class VoiceModule { constructor() { this.commands = {}; this.terminal = null; this.mediaRecorder = null; this.audioChunks = []; this.listening = false; this.recordingTimeout = null; this._lastError = ''; this._sttEndpoint = '/vmai-api/stt'; // VaultMind STT via nginx proxy this._recordingDuration = 10000; // 10s auto-stop this._voices = []; this._autoSpeak = false; } init(terminal) { this.terminal = terminal; // Pre-load TTS voices (Chrome loads async) if (typeof speechSynthesis !== 'undefined') { this._voices = speechSynthesis.getVoices() || []; if (this._voices.length === 0) { var self = this; speechSynthesis.onvoiceschanged = function() { self._voices = speechSynthesis.getVoices() || []; }; } } } _getBestVoice() { var voices = this._voices.length > 0 ? this._voices : (typeof speechSynthesis !== 'undefined' ? speechSynthesis.getVoices() : []); return voices.find(function(v) { return v.lang && v.lang.startsWith('en') && v.name.includes('Google'); }) || voices.find(function(v) { return v.lang && v.lang.startsWith('en'); }) || (voices.length > 0 ? voices[0] : null); } requirements() { const hasMic = typeof navigator !== 'undefined' && !!navigator.mediaDevices?.getUserMedia; const hasRecorder = typeof MediaRecorder !== 'undefined'; if (hasMic && hasRecorder) { return { met: true, missing: [] }; } const missing = []; if (!hasMic) missing.push('Microphone access not available'); if (!hasRecorder) missing.push('MediaRecorder not available'); return { met: false, missing }; } getHelpInfo() { return { 'voice': 'voice [stop] — Start voice recording (Whisper STT), or type "voice stop" to cancel', 'speak': 'speak [text] — Text-to-speech. No args = toggle auto-speak for AI responses' }; } registerCommands() { this.commands['voice'] = (args) => this.handle(args); this.commands['speak'] = (args) => this.speak(args); } speak(args) { const text = args.join(' ').trim(); if (!text) { if (typeof speechSynthesis === 'undefined') { this.terminal.print('🔊 TTS not supported in this browser', 'error'); return; } this._autoSpeak = !this._autoSpeak; this.terminal.print(this._autoSpeak ? '🔊 Auto-speak ON — AI responses will be spoken' : '🔊 Auto-speak OFF', 'info'); return; } if (typeof speechSynthesis === 'undefined') { this.terminal.print('🔊 TTS not supported in this browser', 'error'); return; } this._doSpeak(text); this.terminal.print('🔊 Speaking: "' + text.substring(0, 80) + (text.length > 80 ? '...' : '') + '"', 'info'); } _doSpeak(text) { var self = this; speechSynthesis.cancel(); var utter = new SpeechSynthesisUtterance(text); utter.rate = parseFloat(localStorage.getItem('webx11_tts_rate') || '1.0'); utter.pitch = 1.0; utter.volume = 1.0; var voice = this._getBestVoice(); if (voice) utter.voice = voice; utter.onstart = function() { self._updateButton(true); }; utter.onend = function() { self._updateButton(false); }; utter.onerror = function() { self._updateButton(false); }; // If voices not loaded yet, wait for voiceschanged with 2s timeout if (this._voices.length === 0 && typeof speechSynthesis !== 'undefined') { var spoken = false; var onReady = function() { if (spoken) return; spoken = true; var v = self._getBestVoice(); if (v) utter.voice = v; speechSynthesis.cancel(); speechSynthesis.speak(utter); speechSynthesis.onvoiceschanged = null; }; speechSynthesis.onvoiceschanged = onReady; setTimeout(function() { if (!spoken) { spoken = true; speechSynthesis.onvoiceschanged = null; speechSynthesis.cancel(); speechSynthesis.speak(utter); } }, 2000); } else { speechSynthesis.speak(utter); } } speakText(text) { if (!this._autoSpeak || typeof speechSynthesis === 'undefined') return; speechSynthesis.cancel(); const clean = text.replace(/```[\s\S]*?```/g, ' code block ').replace(/[*_#|`>]/g, '').replace(/\n+/g, '. ').trim(); if (clean.length > 500) { const chunks = clean.match(/.{1,500}/g); for (var i = 0; i < chunks.length; i++) { this._doSpeak(chunks[i]); } } else { this._doSpeak(clean); } } handle(args) { const sub = args[0] && args[0].toLowerCase(); if (sub === 'stop') return this.stopListening(); if (this.listening) return this.terminal.print('Already recording. Say something or type "voice stop"', 'warning'); this.startListening(); } _updateButton(listening, error) { const btn = document.getElementById('voice-toggle-btn'); if (!btn) return; if (listening) { btn.style.borderColor = 'var(--success-color)'; btn.style.color = 'var(--success-color)'; btn.style.animation = 'pulse 1.5s ease-in-out infinite'; btn.textContent = '🔴 Recording'; } else { btn.style.borderColor = ''; btn.style.color = ''; btn.style.animation = ''; if (error) { btn.style.borderColor = 'var(--error-color)'; btn.style.color = 'var(--error-color)'; } else { btn.style.borderColor = ''; btn.style.color = ''; } btn.textContent = '🎤 Mic'; } } startListening() { if (!navigator.mediaDevices?.getUserMedia) { this._updateButton(false, true); this.terminal.print('🎤 Audio recording not supported in this browser', 'error'); return; } this.listening = true; this.audioChunks = []; this._updateButton(true); this.terminal.print('🎤 Recording... (tap mic again or type "voice stop" to finish)', 'info'); navigator.mediaDevices.getUserMedia({ audio: true }) .then(stream => { // Prefer webm/opus (Chrome, Firefox) or fall back to whatever works const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') ? 'audio/webm;codecs=opus' : MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm' : MediaRecorder.isTypeSupported('audio/ogg;codecs=opus') ? 'audio/ogg;codecs=opus' : ''; this.mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {}); this.mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) this.audioChunks.push(e.data); }; this.mediaRecorder.onstop = () => { stream.getTracks().forEach(t => t.stop()); this._transcribeAudio(); }; this.mediaRecorder.start(); // Auto-stop after recording duration this.recordingTimeout = setTimeout(() => { if (this.mediaRecorder?.state === 'recording') { this.mediaRecorder.stop(); } }, this._recordingDuration); }) .catch(err => { this.terminal.print('🎤 Mic access denied: ' + err.message, 'error'); this.listening = false; this._updateButton(false, true); }); } stopListening() { if (this.mediaRecorder?.state === 'recording') { this.mediaRecorder.stop(); clearTimeout(this.recordingTimeout); } this.listening = false; this._updateButton(false); this.terminal.print('🎤 Recording stopped', 'info'); } _transcribeAudio() { const blob = new Blob(this.audioChunks, { type: this.mediaRecorder?.mimeType || 'audio/webm' }); if (blob.size < 100) { this.terminal.print('🎤 No audio detected', 'warning'); this.listening = false; this._updateButton(false); return; } this.terminal.print('🎤 Transcribing...', 'info'); // Convert blob to WAV via AudioContext (MediaRecorder captures compressed) // We send raw audio bytes — faster-whisper accepts WAV const reader = new FileReader(); reader.onload = async () => { try { // Decode compressed audio via Web Audio API → raw PCM const ctx = new (window.AudioContext || window.webkitAudioContext)(); const audioBuffer = await ctx.decodeAudioData(reader.result); // Convert to 16-bit PCM WAV at 16kHz const pcmData = this._audioBufferToWav(audioBuffer); const b64 = btoa(String.fromCharCode(...new Uint8Array(pcmData))); const resp = await fetch(this._sttEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ audio: b64, sample_rate: 16000 }), }); const data = await resp.json(); const text = (data.text || '').trim(); if (text) { this.terminal.print('🎤 Recognized: "' + text + '"', 'success'); const input = document.getElementById('command-input'); if (input) { input.value = text; input.style.height = 'auto'; input.focus(); } } else { this.terminal.print('🎤 No speech detected', 'warning'); } } catch (err) { this.terminal.print('🎤 Transcription error: ' + err.message, 'error'); } this.listening = false; this._updateButton(false); }; reader.readAsArrayBuffer(blob); } _audioBufferToWav(audioBuffer) { // Resample to 16kHz mono 16-bit PCM const targetSampleRate = 16000; const numChannels = 1; const src = audioBuffer.getChannelData(0); // take first channel const srcRate = audioBuffer.sampleRate; // Resample const ratio = srcRate / targetSampleRate; const newLen = Math.round(src.length / ratio); const pcm = new Int16Array(newLen); for (let i = 0; i < newLen; i++) { const srcIdx = Math.round(i * ratio); const sample = Math.max(-1, Math.min(1, src[Math.min(srcIdx, src.length - 1)])); pcm[i] = sample < 0 ? sample * 0x8000 : sample * 0x7FFF; } // Build WAV header + data const buffer = new ArrayBuffer(44 + pcm.byteLength); const view = new DataView(buffer); // RIFF header this._writeString(view, 0, 'RIFF'); view.setUint32(4, 36 + pcm.byteLength, true); this._writeString(view, 8, 'WAVE'); // fmt chunk this._writeString(view, 12, 'fmt '); view.setUint32(16, 16, true); // chunk size view.setUint16(20, 1, true); // PCM view.setUint16(22, numChannels, true); view.setUint32(24, targetSampleRate, true); view.setUint32(28, targetSampleRate * numChannels * 2, true); // byte rate view.setUint16(32, numChannels * 2, true); // block align view.setUint16(34, 16, true); // bits per sample // data chunk this._writeString(view, 36, 'data'); view.setUint32(40, pcm.byteLength, true); new Uint8Array(buffer, 44).set(new Uint8Array(pcm.buffer)); return buffer; } _writeString(view, offset, str) { for (let i = 0; i < str.length; i++) { view.setUint8(offset + i, str.charCodeAt(i)); } } } if (typeof window.webx11 !== 'undefined' && window.webx11) { webx11.moduleManager.registerModule(new VoiceModule(), 'https://webx11.duckdns.org/spaces/newos4/modules/ext/voice.js'); }