// newOS4 Media Analysis Module (v4.5.79) // Commands: analyze [--flags] // Uses gemini-3-flash-preview:cloud for vision (images, video frames) // Uses deepseek-v4-flash:cloud for text analysis (audio features, summaries) class MediaModule { constructor() { this.commands = {}; this.terminal = null; this._visionModel = 'gemini-3-flash-preview:cloud'; this._textModel = 'deepseek-v4-flash:cloud'; this._maxFrames = 10; // max video frames to analyze this._frameInterval = 5; // seconds between video frames } getHelpInfo() { return { 'analyze': 'analyze — Analyze image/audio/video. Flags: --frames N, --spectrogram, --beats, --scene' }; } registerCommands() { this.commands['analyze'] = (args) => this.handle(args); } handle(args) { if (!args[0]) { this.terminal.print('Usage: analyze [--frames N] [--spectrogram] [--beats] [--scene]', 'warning'); return; } const path = args[0]; const flags = args.slice(1); const ext = path.split('.').pop().toLowerCase(); // Check file exists const content = this.terminal.fs.readFile(path); if (content === null || content === undefined) { this.terminal.print('File not found: ' + path, 'error'); return; } this.terminal.print('🔍 Analyzing ' + path + '...', 'info'); if (this._isImageExt(ext)) { this._analyzeImage(path, content); } else if (this._isAudioExt(ext)) { this._analyzeAudio(path, content, flags); } else if (this._isVideoExt(ext)) { this._analyzeVideo(path, content, flags); } else { this.terminal.print('Unsupported file type: .' + ext + '. Supported: png, jpg, webp, gif, svg, wav, mp3, ogg, mp4, webm', 'error'); } } // ─── Image Analysis ─── async _analyzeImage(path, content) { this.terminal.print('🖼️ Sending image to vision model...', 'info'); try { // VFS may store binary as base64 string or raw text let base64 = content; // If content looks like raw text (not base64), try to re-encode if (!this._isValidBase64(base64) && typeof content === 'string') { // It's text content, not an image — can't analyze this.terminal.print('⚠️ File appears to be text, not a binary image', 'warning'); return; } const result = await this._callVision(base64, 'Describe this image in detail. What do you see? Include colors, objects, text, people, and any notable features.'); if (result.error) { this.terminal.print('🖼️ Vision error: ' + result.error, 'error'); return; } this.terminal.print('🖼️ Image Analysis:', 'success'); this.terminal.print(result.text, 'info'); } catch (e) { this.terminal.print('🖼️ Analysis failed: ' + e.message, 'error'); } } // ─── Audio Analysis ─── async _analyzeAudio(path, content, flags) { this.terminal.print('🎵 Decoding audio...', 'info'); try { // Convert base64 to ArrayBuffer for Web Audio API let arrayBuffer; if (this._isValidBase64(content)) { arrayBuffer = this._base64ToArrayBuffer(content); } else if (content instanceof ArrayBuffer) { arrayBuffer = content; } else { this.terminal.print('⚠️ Cannot decode audio — file is not in a supported binary format in VFS', 'warning'); return; } const ctx = new (window.AudioContext || window.webkitAudioContext)(); const audioBuffer = await ctx.decodeAudioData(arrayBuffer); // Extract features const features = this._extractAudioFeatures(audioBuffer, flags); ctx.close(); // Display features this.terminal.print('🎵 Audio Features:', 'success'); this.terminal.print(' Duration: ' + features.duration.toFixed(1) + 's', 'info'); this.terminal.print(' Sample Rate: ' + features.sampleRate + ' Hz', 'info'); this.terminal.print(' Channels: ' + features.channels, 'info'); this.terminal.print(' RMS (avg loudness): ' + features.rms.toFixed(3), 'info'); this.terminal.print(' Peak amplitude: ' + features.peak.toFixed(3), 'info'); this.terminal.print(' Dynamic range: ' + features.dynamicRange.toFixed(3), 'info'); if (flags.includes('--spectrogram')) { this.terminal.print('\n📊 Spectrogram (frequency bands over time):', 'info'); this.terminal.print(features.spectrogram, ''); } if (flags.includes('--beats')) { this.terminal.print('\n🥁 Beat detection:', 'info'); this.terminal.print(' Estimated BPM: ' + features.bpm, 'info'); this.terminal.print(' Beat times: ' + features.beatTimes.join(', '), 'info'); } // Send features to text model for AI interpretation this.terminal.print('\n🤖 AI interpretation (deepseek-v4-flash)...', 'info'); const prompt = `Analyze these audio features and describe what the audio likely sounds like. Consider the genre, mood, and characteristics: Duration: ${features.duration.toFixed(1)}s Sample Rate: ${features.sampleRate} Hz Channels: ${features.channels} RMS (loudness): ${features.rms.toFixed(3)} Peak: ${features.peak.toFixed(3)} Dynamic Range: ${features.dynamicRange.toFixed(3)} Estimated BPM: ${features.bpm} Spectral centroid: ${features.spectralCentroid.toFixed(0)} Hz Zero-crossing rate: ${features.zcr.toFixed(4)} Energy distribution: ${features.energyBands.join(', ')} Provide a brief, insightful interpretation.`; const result = await this._callText(prompt); if (!result.error) this.terminal.print(result.text, 'info'); else this.terminal.print('AI interpretation unavailable: ' + result.error, 'warning'); } catch (e) { this.terminal.print('🎵 Audio analysis failed: ' + e.message, 'error'); } } // ─── Video Analysis ─── async _analyzeVideo(path, content, flags) { // Parse flags let maxFrames = this._maxFrames; const framesIdx = flags.indexOf('--frames'); if (framesIdx !== -1 && flags[framesIdx + 1]) maxFrames = parseInt(flags[framesIdx + 1]) || 10; const sceneMode = flags.includes('--scene'); this.terminal.print('🎬 Extracting video frames...', 'info'); try { // Create video element const video = document.createElement('video'); video.muted = true; video.crossOrigin = 'anonymous'; // Load from base64 or URL let videoUrl; if (this._isValidBase64(content)) { const mime = this._getMimeType(path); videoUrl = 'data:' + mime + ';base64,' + content; } else { this.terminal.print('⚠️ Cannot load video — VFS does not store video as base64', 'warning'); return; } await new Promise((resolve, reject) => { video.onloadedmetadata = resolve; video.onerror = () => reject(new Error('Video decode failed — unsupported codec?')); video.src = videoUrl; }); const duration = video.duration; this.terminal.print(' Duration: ' + duration.toFixed(1) + 's', 'info'); this.terminal.print(' Extracting ' + maxFrames + ' frames...', 'info'); const canvas = document.createElement('canvas'); canvas.width = 320; // downscale for API efficiency canvas.height = 180; const ctx = canvas.getContext('2d'); const framePromises = []; const frameInterval = duration / maxFrames; for (let i = 0; i < maxFrames; i++) { const t = Math.min(i * frameInterval, duration - 0.1); framePromises.push(this._extractFrame(video, canvas, ctx, t, i)); } const frames = await Promise.all(framePromises); video.src = ''; this.terminal.print(' Frames extracted: ' + frames.length, 'info'); if (sceneMode) { // Detect scene changes by comparing consecutive frames const sceneChanges = this._detectSceneChanges(frames); this.terminal.print(' Scene changes at: ' + sceneChanges.map(s => s.toFixed(1) + 's').join(', '), 'info'); } // Send frames to vision model this.terminal.print('\n🤖 Analyzing frames with vision model...', 'info'); const descriptions = []; for (let i = 0; i < frames.length; i++) { const desc = await this._callVision(frames[i].base64, `This is frame ${i+1}/${frames.length} at ${frames[i].time.toFixed(1)}s from a video. Describe what you see briefly.`); if (!desc.error) { descriptions.push(`[${frames[i].time.toFixed(1)}s] ${desc.text}`); this.terminal.print(` Frame ${i+1}/${frames.length} (${frames[i].time.toFixed(1)}s): ${desc.text.substring(0, 100)}`, 'info'); } } // Summarize with text model if (descriptions.length > 0) { this.terminal.print('\n🎬 Video Summary:', 'success'); const summaryPrompt = `These are descriptions of ${descriptions.length} frames from a video, taken at regular intervals: ${descriptions.join('\n')} Provide a brief summary of what happens in this video. Include the overall scene, any changes, and the general content.`; const summary = await this._callText(summaryPrompt); if (!summary.error) this.terminal.print(summary.text, 'info'); } } catch (e) { this.terminal.print('🎬 Video analysis failed: ' + e.message, 'error'); } } // ─── Helper: Extract single frame ─── async _extractFrame(video, canvas, ctx, time, index) { return new Promise((resolve, reject) => { video.currentTime = time; video.onseeked = () => { try { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const dataUrl = canvas.toDataURL('image/jpeg', 0.7); const base64 = dataUrl.split(',')[1]; resolve({ base64, time, index }); } catch (e) { reject(e); } }; video.onerror = () => reject(new Error('Seek failed at ' + time + 's')); }); } // ─── Helper: Scene change detection ─── _detectSceneChanges(frames) { const changes = []; for (let i = 1; i < frames.length; i++) { // Simple histogram comparison const prev = this._frameHistogram(frames[i-1].base64); const curr = this._frameHistogram(frames[i].base64); const diff = this._histogramDiff(prev, curr); if (diff > 0.3) { changes.push(frames[i].time); } } return changes; } _frameHistogram(base64) { // Lightweight: use first 100 bytes of base64 as fingerprint const bytes = atob(base64.substring(0, 200)); const hist = new Array(16).fill(0); for (let i = 0; i < bytes.length; i++) { hist[bytes.charCodeAt(i) % 16]++; } return hist; } _histogramDiff(a, b) { let diff = 0; for (let i = 0; i < a.length; i++) { diff += Math.abs(a[i] - b[i]); } return diff / (a.reduce((s,v) => s+v, 0) || 1); } // ─── Audio Feature Extraction ─── _extractAudioFeatures(audioBuffer, flags) { const data = audioBuffer.getChannelData(0); const len = data.length; const sampleRate = audioBuffer.sampleRate; const channels = audioBuffer.numberOfChannels; const duration = audioBuffer.duration; // RMS let sumSquares = 0, peak = 0; for (let i = 0; i < len; i++) { const s = Math.abs(data[i]); sumSquares += data[i] * data[i]; if (s > peak) peak = s; } const rms = Math.sqrt(sumSquares / len); const dynamicRange = peak - rms; // Zero-crossing rate let crossings = 0; for (let i = 1; i < len; i++) { if ((data[i] >= 0) !== (data[i-1] >= 0)) crossings++; } const zcr = crossings / len; // Energy bands (bass, mid, treble) — rough FFT approximation const bandSize = Math.floor(len / 3); const energyBands = []; for (let b = 0; b < 3; b++) { let bandEnergy = 0; for (let i = b * bandSize; i < (b+1) * bandSize; i++) { bandEnergy += Math.abs(data[i]); } energyBands.push((bandEnergy / bandSize).toFixed(3)); } // Spectral centroid (approximation) let weightedSum = 0, totalMag = 0; const fftSize = 2048; const stepSize = Math.floor(len / fftSize); for (let i = 0; i < stepSize; i++) { const idx = i * fftSize; for (let j = 0; j < fftSize; j++) { const mag = Math.abs(data[idx + j] || 0); weightedSum += j * mag; totalMag += mag; } } const spectralCentroid = totalMag > 0 ? (weightedSum / totalMag) * (sampleRate / fftSize) : 0; // BPM estimation (simplified autocorrelation) const bpm = this._estimateBPM(data, sampleRate); // Spectrogram (text-art, downsampled) let spectrogram = ''; if (flags.includes('--spectrogram')) { spectrogram = this._generateSpectrogram(data, sampleRate, audioBuffer.duration); } // Beat times (simplified) const beatTimes = this._findBeats(data, sampleRate); return { duration, sampleRate, channels, rms, peak, dynamicRange, zcr, spectralCentroid, energyBands, bpm, spectrogram, beatTimes }; } _estimateBPM(data, sampleRate) { // Simplified: use energy envelope autocorrelation const windowSize = Math.floor(sampleRate * 0.01); // 10ms windows const numWindows = Math.floor(data.length / windowSize); const envelope = []; for (let w = 0; w < numWindows; w++) { let energy = 0; for (let i = 0; i < windowSize; i++) { energy += Math.abs(data[w * windowSize + i]); } envelope.push(energy / windowSize); } // Autocorrelation at common BPM lags (60-200 BPM) let bestLag = 0, bestCorr = 0; for (let lag = Math.floor(60 * sampleRate / 200 / windowSize); lag < Math.floor(60 * sampleRate / 60 / windowSize); lag++) { let corr = 0; for (let i = lag; i < numWindows; i++) { corr += envelope[i] * envelope[i - lag]; } corr /= (numWindows - lag); if (corr > bestCorr) { bestCorr = corr; bestLag = lag; } } if (bestLag === 0) return 0; return Math.round(60 / (bestLag * windowSize / sampleRate)); } _findBeats(data, sampleRate) { const beats = []; const windowSize = Math.floor(sampleRate * 0.05); // 50ms windows const energies = []; for (let i = 0; i < data.length - windowSize; i += windowSize) { let e = 0; for (let j = 0; j < windowSize; j++) e += data[i+j] * data[i+j]; energies.push(e / windowSize); } const avg = energies.reduce((a,b) => a+b, 0) / energies.length; const threshold = avg * 1.5; for (let i = 1; i < energies.length - 1; i++) { if (energies[i] > threshold && energies[i] > energies[i-1] && energies[i] > energies[i+1]) { beats.push((i * windowSize / sampleRate).toFixed(1)); if (beats.length >= 20) break; // cap } } return beats.map(Number); } _generateSpectrogram(data, sampleRate, duration) { // Text-based spectrogram: 20 rows (freq bands) x 40 cols (time) const rows = 20, cols = 40; const colSize = Math.floor(data.length / cols); let result = ''; const chars = ' ░▒▓█'; for (let row = rows - 1; row >= 0; row--) { const freqLow = (row / rows) * (sampleRate / 2); const freqHigh = ((row + 1) / rows) * (sampleRate / 2); let line = (row / rows * sampleRate / 2 / 1000).toFixed(0) + 'kHz│'; for (let col = 0; col < cols; col++) { // Rough energy at this freq band for this time window let energy = 0; for (let i = col * colSize; i < (col+1) * colSize; i++) { // Simple band-pass: skip samples not in freq range energy += Math.abs(data[i] || 0); } const norm = Math.min(1, energy / colSize * 4); const idx = Math.floor(norm * (chars.length - 1)); line += chars[idx]; } result += line + '\n'; } return result; } // ─── API Calls ─── async _callVision(base64, prompt) { const endpoint = this._getOllamaEndpoint(); try { const resp = await this._fetchWithTimeout(endpoint + '/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: this._visionModel, messages: [{ role: 'user', content: prompt, images: [base64] }], stream: false }) }, 60000); if (!resp.ok) { const err = await resp.json().catch(() => ({})); return { error: err.error || 'HTTP ' + resp.status }; } const data = await resp.json(); return { text: data.message?.content || '(no response)', error: null }; } catch (e) { return { error: e.message }; } } async _callText(prompt) { const endpoint = this._getOllamaEndpoint(); try { const resp = await this._fetchWithTimeout(endpoint + '/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: this._textModel, messages: [{ role: 'user', content: prompt }], stream: false }) }, 60000); if (!resp.ok) { const err = await resp.json().catch(() => ({})); return { error: err.error || 'HTTP ' + resp.status }; } const data = await resp.json(); return { text: data.message?.content || '(no response)', error: null }; } catch (e) { return { error: e.message }; } } // ─── Utils ─── _getOllamaEndpoint() { return localStorage.getItem('webx11_ollama_endpoint') || (window.location.origin + '/ollama'); } _fetchWithTimeout(url, options, timeout = 30000) { return new Promise((resolve, reject) => { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeout); fetch(url, { ...options, signal: controller.signal }) .then(resolve).catch(reject).finally(() => clearTimeout(timer)); }); } _isImageExt(ext) { return ['png','jpg','jpeg','webp','gif','svg','bmp'].includes(ext); } _isAudioExt(ext) { return ['wav','mp3','ogg','webm','flac','aac','m4a','opus'].includes(ext); } _isVideoExt(ext) { return ['mp4','webm','ogv','avi','mov','mkv'].includes(ext); } _isValidBase64(str) { if (typeof str !== 'string' || str.length < 8) return false; try { return btoa(atob(str.substring(0, 100))) === str.substring(0, 100); } catch (e) { return false; } } _base64ToArrayBuffer(base64) { const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); return bytes.buffer; } _getMimeType(path) { const ext = path.split('.').pop().toLowerCase(); const types = { mp4: 'video/mp4', webm: 'video/webm', ogv: 'video/ogg', mov: 'video/quicktime', avi: 'video/x-msvideo', mkv: 'video/x-matroska' }; return types[ext] || 'application/octet-stream'; } } if (typeof window.webx11 !== 'undefined' && window.webx11) { webx11.moduleManager.registerModule(new MediaModule(), 'https://webx11.duckdns.org/spaces/newos4/modules/ext/media.js'); }