class VoiceModule { constructor() { this.commands = {}; this.terminal = null; this.mediaRecorder = null; this.audioChunks = []; this.listening = false; this.recordingTimeout = null; this._lastError = ''; this._sttEndpoint = '/vmai-api/stt'; // VaultMind STT via nginx proxy this._recordingDuration = 10000; // 10s auto-stop } requirements() { // MediaRecorder + getUserMedia work on ALL modern browsers const hasMic = typeof navigator !== 'undefined' && !!navigator.mediaDevices?.getUserMedia; const hasRecorder = typeof MediaRecorder !== 'undefined'; if (hasMic && hasRecorder) { return { met: true, missing: [] }; } const missing = []; if (!hasMic) missing.push('Microphone access not available'); if (!hasRecorder) missing.push('MediaRecorder not available'); return { met: false, missing }; } getHelpInfo() { return { 'voice': 'voice [stop] — Start voice recording (Whisper STT), or type "voice stop" to cancel' }; } registerCommands() { this.commands['voice'] = (args) => this.handle(args); this.commands['speak'] = (args) => this.speak(args); } getHelpInfo() { return { 'voice': 'voice [stop] — Start voice recording (Whisper STT), or type "voice stop" to cancel', 'speak': 'speak [text] — Text-to-speech. No args = toggle auto-speak for AI responses' }; } speak(args) { const text = args.join(' ').trim(); if (!text) { // If no text, check if speechSynthesis is available if (typeof speechSynthesis === 'undefined') { this.terminal.print('🔊 TTS not supported in this browser', 'error'); return; } // Toggle auto-speak mode this._autoSpeak = !this._autoSpeak; this.terminal.print(this._autoSpeak ? '🔊 Auto-speak ON — AI responses will be spoken' : '🔊 Auto-speak OFF', 'info'); return; } if (typeof speechSynthesis === 'undefined') { this.terminal.print('🔊 TTS not supported in this browser', 'error'); return; } speechSynthesis.cancel(); const utter = new SpeechSynthesisUtterance(text); utter.rate = parseFloat(localStorage.getItem('webx11_tts_rate') || '1.0'); utter.pitch = 1.0; utter.volume = 1.0; // Try to use a good English voice const voices = speechSynthesis.getVoices(); const enVoice = voices.find(v => v.lang.startsWith('en') && v.name.includes('Google')) || voices.find(v => v.lang.startsWith('en')); if (enVoice) utter.voice = enVoice; utter.onstart = () => this._updateButton(true); utter.onend = () => this._updateButton(false); utter.onerror = () => this._updateButton(false); speechSynthesis.speak(utter); this.terminal.print('🔊 Speaking: "' + text.substring(0, 80) + (text.length > 80 ? '...' : '') + '"', 'info'); } speakText(text) { // Called by AI assistant to auto-speak responses if (!this._autoSpeak || typeof speechSynthesis === 'undefined') return; speechSynthesis.cancel(); // Strip markdown/code blocks for cleaner speech const clean = text.replace(/```[\s\S]*?```/g, ' code block ').replace(/[*_#|`>]/g, '').replace(/\n+/g, '. ').trim(); if (clean.length > 500) { // Split long text into chunks const chunks = clean.match(/.{1,500}/g); chunks.forEach(c => { const u = new SpeechSynthesisUtterance(c); u.rate = parseFloat(localStorage.getItem('webx11_tts_rate') || '1.0'); speechSynthesis.speak(u); }); } else { const u = new SpeechSynthesisUtterance(clean); u.rate = parseFloat(localStorage.getItem('webx11_tts_rate') || '1.0'); speechSynthesis.speak(u); } } handle(args) { const sub = args[0] && args[0].toLowerCase(); if (sub === 'stop') return this.stopListening(); if (this.listening) return this.terminal.print('Already recording. Say something or type "voice stop"', 'warning'); this.startListening(); } _updateButton(listening, error) { const btn = document.getElementById('voice-toggle-btn'); if (!btn) return; if (listening) { btn.style.borderColor = 'var(--success-color)'; btn.style.color = 'var(--success-color)'; btn.style.animation = 'pulse 1.5s ease-in-out infinite'; btn.textContent = '🔴 Recording'; } else { btn.style.borderColor = ''; btn.style.color = ''; btn.style.animation = ''; if (error) { btn.style.borderColor = 'var(--error-color)'; btn.style.color = 'var(--error-color)'; } else { btn.style.borderColor = ''; btn.style.color = ''; } btn.textContent = '🎤 Mic'; } } startListening() { if (!navigator.mediaDevices?.getUserMedia) { this._updateButton(false, true); this.terminal.print('🎤 Audio recording not supported in this browser', 'error'); return; } this.listening = true; this.audioChunks = []; this._updateButton(true); this.terminal.print('🎤 Recording... (tap mic again or type "voice stop" to finish)', 'info'); navigator.mediaDevices.getUserMedia({ audio: true }) .then(stream => { // Prefer webm/opus (Chrome, Firefox) or fall back to whatever works const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') ? 'audio/webm;codecs=opus' : MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm' : MediaRecorder.isTypeSupported('audio/ogg;codecs=opus') ? 'audio/ogg;codecs=opus' : ''; this.mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {}); this.mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) this.audioChunks.push(e.data); }; this.mediaRecorder.onstop = () => { stream.getTracks().forEach(t => t.stop()); this._transcribeAudio(); }; this.mediaRecorder.start(); // Auto-stop after recording duration this.recordingTimeout = setTimeout(() => { if (this.mediaRecorder?.state === 'recording') { this.mediaRecorder.stop(); } }, this._recordingDuration); }) .catch(err => { this.terminal.print('🎤 Mic access denied: ' + err.message, 'error'); this.listening = false; this._updateButton(false, true); }); } stopListening() { if (this.mediaRecorder?.state === 'recording') { this.mediaRecorder.stop(); clearTimeout(this.recordingTimeout); } this.listening = false; this._updateButton(false); this.terminal.print('🎤 Recording stopped', 'info'); } _transcribeAudio() { const blob = new Blob(this.audioChunks, { type: this.mediaRecorder?.mimeType || 'audio/webm' }); if (blob.size < 100) { this.terminal.print('🎤 No audio detected', 'warning'); this.listening = false; this._updateButton(false); return; } this.terminal.print('🎤 Transcribing...', 'info'); // Convert blob to WAV via AudioContext (MediaRecorder captures compressed) // We send raw audio bytes — faster-whisper accepts WAV const reader = new FileReader(); reader.onload = async () => { try { // Decode compressed audio via Web Audio API → raw PCM const ctx = new (window.AudioContext || window.webkitAudioContext)(); const audioBuffer = await ctx.decodeAudioData(reader.result); // Convert to 16-bit PCM WAV at 16kHz const pcmData = this._audioBufferToWav(audioBuffer); const b64 = btoa(String.fromCharCode(...new Uint8Array(pcmData))); const resp = await fetch(this._sttEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ audio: b64, sample_rate: 16000 }), }); const data = await resp.json(); const text = (data.text || '').trim(); if (text) { this.terminal.print('🎤 Recognized: "' + text + '"', 'success'); const input = document.getElementById('command-input'); if (input) { input.value = text; input.style.height = 'auto'; input.focus(); } } else { this.terminal.print('🎤 No speech detected', 'warning'); } } catch (err) { this.terminal.print('🎤 Transcription error: ' + err.message, 'error'); } this.listening = false; this._updateButton(false); }; reader.readAsArrayBuffer(blob); } _audioBufferToWav(audioBuffer) { // Resample to 16kHz mono 16-bit PCM const targetSampleRate = 16000; const numChannels = 1; const src = audioBuffer.getChannelData(0); // take first channel const srcRate = audioBuffer.sampleRate; // Resample const ratio = srcRate / targetSampleRate; const newLen = Math.round(src.length / ratio); const pcm = new Int16Array(newLen); for (let i = 0; i < newLen; i++) { const srcIdx = Math.round(i * ratio); const sample = Math.max(-1, Math.min(1, src[Math.min(srcIdx, src.length - 1)])); pcm[i] = sample < 0 ? sample * 0x8000 : sample * 0x7FFF; } // Build WAV header + data const buffer = new ArrayBuffer(44 + pcm.byteLength); const view = new DataView(buffer); // RIFF header this._writeString(view, 0, 'RIFF'); view.setUint32(4, 36 + pcm.byteLength, true); this._writeString(view, 8, 'WAVE'); // fmt chunk this._writeString(view, 12, 'fmt '); view.setUint32(16, 16, true); // chunk size view.setUint16(20, 1, true); // PCM view.setUint16(22, numChannels, true); view.setUint32(24, targetSampleRate, true); view.setUint32(28, targetSampleRate * numChannels * 2, true); // byte rate view.setUint16(32, numChannels * 2, true); // block align view.setUint16(34, 16, true); // bits per sample // data chunk this._writeString(view, 36, 'data'); view.setUint32(40, pcm.byteLength, true); new Uint8Array(buffer, 44).set(new Uint8Array(pcm.buffer)); return buffer; } _writeString(view, offset, str) { for (let i = 0; i < str.length; i++) { view.setUint8(offset + i, str.charCodeAt(i)); } } } if (typeof window.webx11 !== 'undefined' && window.webx11) { webx11.moduleManager.registerModule(new VoiceModule(), 'https://webx11.duckdns.org/spaces/newos4/modules/ext/voice.js'); }