diff options
Diffstat (limited to 'modules/ext/media.js')
| -rw-r--r-- | modules/ext/media.js | 508 |
1 files changed, 508 insertions, 0 deletions
diff --git a/modules/ext/media.js b/modules/ext/media.js new file mode 100644 index 0000000..5723e27 --- /dev/null +++ b/modules/ext/media.js @@ -0,0 +1,508 @@ +// newOS4 Media Analysis Module (v4.5.79) +// Commands: analyze <path> [--flags] +// Uses gemini-3-flash-preview:cloud for vision (images, video frames) +// Uses deepseek-v4-flash:cloud for text analysis (audio features, summaries) +class MediaModule { + constructor() { + this.commands = {}; + this.terminal = null; + this._visionModel = 'gemini-3-flash-preview:cloud'; + this._textModel = 'deepseek-v4-flash:cloud'; + this._maxFrames = 10; // max video frames to analyze + this._frameInterval = 5; // seconds between video frames + } + + getHelpInfo() { + return { + 'analyze': 'analyze <file> — Analyze image/audio/video. Flags: --frames N, --spectrogram, --beats, --scene' + }; + } + + registerCommands() { + this.commands['analyze'] = (args) => this.handle(args); + } + + handle(args) { + if (!args[0]) { + this.terminal.print('Usage: analyze <file> [--frames N] [--spectrogram] [--beats] [--scene]', 'warning'); + return; + } + const path = args[0]; + const flags = args.slice(1); + const ext = path.split('.').pop().toLowerCase(); + + // Check file exists + const content = this.terminal.fs.readFile(path); + if (content === null || content === undefined) { + this.terminal.print('File not found: ' + path, 'error'); + return; + } + + this.terminal.print('🔍 Analyzing ' + path + '...', 'info'); + + if (this._isImageExt(ext)) { + this._analyzeImage(path, content); + } else if (this._isAudioExt(ext)) { + this._analyzeAudio(path, content, flags); + } else if (this._isVideoExt(ext)) { + this._analyzeVideo(path, content, flags); + } else { + this.terminal.print('Unsupported file type: .' + ext + '. Supported: png, jpg, webp, gif, svg, wav, mp3, ogg, mp4, webm', 'error'); + } + } + + // ─── Image Analysis ─── + async _analyzeImage(path, content) { + this.terminal.print('🖼️ Sending image to vision model...', 'info'); + try { + // VFS may store binary as base64 string or raw text + let base64 = content; + // If content looks like raw text (not base64), try to re-encode + if (!this._isValidBase64(base64) && typeof content === 'string') { + // It's text content, not an image — can't analyze + this.terminal.print('⚠️ File appears to be text, not a binary image', 'warning'); + return; + } + const result = await this._callVision(base64, 'Describe this image in detail. What do you see? Include colors, objects, text, people, and any notable features.'); + if (result.error) { + this.terminal.print('🖼️ Vision error: ' + result.error, 'error'); + return; + } + this.terminal.print('🖼️ Image Analysis:', 'success'); + this.terminal.print(result.text, 'info'); + } catch (e) { + this.terminal.print('🖼️ Analysis failed: ' + e.message, 'error'); + } + } + + // ─── Audio Analysis ─── + async _analyzeAudio(path, content, flags) { + this.terminal.print('🎵 Decoding audio...', 'info'); + try { + // Convert base64 to ArrayBuffer for Web Audio API + let arrayBuffer; + if (this._isValidBase64(content)) { + arrayBuffer = this._base64ToArrayBuffer(content); + } else if (content instanceof ArrayBuffer) { + arrayBuffer = content; + } else { + this.terminal.print('⚠️ Cannot decode audio — file is not in a supported binary format in VFS', 'warning'); + return; + } + + const ctx = new (window.AudioContext || window.webkitAudioContext)(); + const audioBuffer = await ctx.decodeAudioData(arrayBuffer); + + // Extract features + const features = this._extractAudioFeatures(audioBuffer, flags); + ctx.close(); + + // Display features + this.terminal.print('🎵 Audio Features:', 'success'); + this.terminal.print(' Duration: ' + features.duration.toFixed(1) + 's', 'info'); + this.terminal.print(' Sample Rate: ' + features.sampleRate + ' Hz', 'info'); + this.terminal.print(' Channels: ' + features.channels, 'info'); + this.terminal.print(' RMS (avg loudness): ' + features.rms.toFixed(3), 'info'); + this.terminal.print(' Peak amplitude: ' + features.peak.toFixed(3), 'info'); + this.terminal.print(' Dynamic range: ' + features.dynamicRange.toFixed(3), 'info'); + + if (flags.includes('--spectrogram')) { + this.terminal.print('\n📊 Spectrogram (frequency bands over time):', 'info'); + this.terminal.print(features.spectrogram, ''); + } + + if (flags.includes('--beats')) { + this.terminal.print('\n🥁 Beat detection:', 'info'); + this.terminal.print(' Estimated BPM: ' + features.bpm, 'info'); + this.terminal.print(' Beat times: ' + features.beatTimes.join(', '), 'info'); + } + + // Send features to text model for AI interpretation + this.terminal.print('\n🤖 AI interpretation (deepseek-v4-flash)...', 'info'); + const prompt = `Analyze these audio features and describe what the audio likely sounds like. Consider the genre, mood, and characteristics: + +Duration: ${features.duration.toFixed(1)}s +Sample Rate: ${features.sampleRate} Hz +Channels: ${features.channels} +RMS (loudness): ${features.rms.toFixed(3)} +Peak: ${features.peak.toFixed(3)} +Dynamic Range: ${features.dynamicRange.toFixed(3)} +Estimated BPM: ${features.bpm} +Spectral centroid: ${features.spectralCentroid.toFixed(0)} Hz +Zero-crossing rate: ${features.zcr.toFixed(4)} +Energy distribution: ${features.energyBands.join(', ')} + +Provide a brief, insightful interpretation.`; + + const result = await this._callText(prompt); + if (!result.error) this.terminal.print(result.text, 'info'); + else this.terminal.print('AI interpretation unavailable: ' + result.error, 'warning'); + + } catch (e) { + this.terminal.print('🎵 Audio analysis failed: ' + e.message, 'error'); + } + } + + // ─── Video Analysis ─── + async _analyzeVideo(path, content, flags) { + // Parse flags + let maxFrames = this._maxFrames; + const framesIdx = flags.indexOf('--frames'); + if (framesIdx !== -1 && flags[framesIdx + 1]) maxFrames = parseInt(flags[framesIdx + 1]) || 10; + const sceneMode = flags.includes('--scene'); + + this.terminal.print('🎬 Extracting video frames...', 'info'); + + try { + // Create video element + const video = document.createElement('video'); + video.muted = true; + video.crossOrigin = 'anonymous'; + + // Load from base64 or URL + let videoUrl; + if (this._isValidBase64(content)) { + const mime = this._getMimeType(path); + videoUrl = 'data:' + mime + ';base64,' + content; + } else { + this.terminal.print('⚠️ Cannot load video — VFS does not store video as base64', 'warning'); + return; + } + + await new Promise((resolve, reject) => { + video.onloadedmetadata = resolve; + video.onerror = () => reject(new Error('Video decode failed — unsupported codec?')); + video.src = videoUrl; + }); + + const duration = video.duration; + this.terminal.print(' Duration: ' + duration.toFixed(1) + 's', 'info'); + this.terminal.print(' Extracting ' + maxFrames + ' frames...', 'info'); + + const canvas = document.createElement('canvas'); + canvas.width = 320; // downscale for API efficiency + canvas.height = 180; + const ctx = canvas.getContext('2d'); + + const framePromises = []; + const frameInterval = duration / maxFrames; + + for (let i = 0; i < maxFrames; i++) { + const t = Math.min(i * frameInterval, duration - 0.1); + framePromises.push(this._extractFrame(video, canvas, ctx, t, i)); + } + + const frames = await Promise.all(framePromises); + video.src = ''; + + this.terminal.print(' Frames extracted: ' + frames.length, 'info'); + + if (sceneMode) { + // Detect scene changes by comparing consecutive frames + const sceneChanges = this._detectSceneChanges(frames); + this.terminal.print(' Scene changes at: ' + sceneChanges.map(s => s.toFixed(1) + 's').join(', '), 'info'); + } + + // Send frames to vision model + this.terminal.print('\n🤖 Analyzing frames with vision model...', 'info'); + const descriptions = []; + for (let i = 0; i < frames.length; i++) { + const desc = await this._callVision(frames[i].base64, `This is frame ${i+1}/${frames.length} at ${frames[i].time.toFixed(1)}s from a video. Describe what you see briefly.`); + if (!desc.error) { + descriptions.push(`[${frames[i].time.toFixed(1)}s] ${desc.text}`); + this.terminal.print(` Frame ${i+1}/${frames.length} (${frames[i].time.toFixed(1)}s): ${desc.text.substring(0, 100)}`, 'info'); + } + } + + // Summarize with text model + if (descriptions.length > 0) { + this.terminal.print('\n🎬 Video Summary:', 'success'); + const summaryPrompt = `These are descriptions of ${descriptions.length} frames from a video, taken at regular intervals: +${descriptions.join('\n')} + +Provide a brief summary of what happens in this video. Include the overall scene, any changes, and the general content.`; + const summary = await this._callText(summaryPrompt); + if (!summary.error) this.terminal.print(summary.text, 'info'); + } + + } catch (e) { + this.terminal.print('🎬 Video analysis failed: ' + e.message, 'error'); + } + } + + // ─── Helper: Extract single frame ─── + async _extractFrame(video, canvas, ctx, time, index) { + return new Promise((resolve, reject) => { + video.currentTime = time; + video.onseeked = () => { + try { + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + const dataUrl = canvas.toDataURL('image/jpeg', 0.7); + const base64 = dataUrl.split(',')[1]; + resolve({ base64, time, index }); + } catch (e) { reject(e); } + }; + video.onerror = () => reject(new Error('Seek failed at ' + time + 's')); + }); + } + + // ─── Helper: Scene change detection ─── + _detectSceneChanges(frames) { + const changes = []; + for (let i = 1; i < frames.length; i++) { + // Simple histogram comparison + const prev = this._frameHistogram(frames[i-1].base64); + const curr = this._frameHistogram(frames[i].base64); + const diff = this._histogramDiff(prev, curr); + if (diff > 0.3) { + changes.push(frames[i].time); + } + } + return changes; + } + + _frameHistogram(base64) { + // Lightweight: use first 100 bytes of base64 as fingerprint + const bytes = atob(base64.substring(0, 200)); + const hist = new Array(16).fill(0); + for (let i = 0; i < bytes.length; i++) { + hist[bytes.charCodeAt(i) % 16]++; + } + return hist; + } + + _histogramDiff(a, b) { + let diff = 0; + for (let i = 0; i < a.length; i++) { + diff += Math.abs(a[i] - b[i]); + } + return diff / (a.reduce((s,v) => s+v, 0) || 1); + } + + // ─── Audio Feature Extraction ─── + _extractAudioFeatures(audioBuffer, flags) { + const data = audioBuffer.getChannelData(0); + const len = data.length; + const sampleRate = audioBuffer.sampleRate; + const channels = audioBuffer.numberOfChannels; + const duration = audioBuffer.duration; + + // RMS + let sumSquares = 0, peak = 0; + for (let i = 0; i < len; i++) { + const s = Math.abs(data[i]); + sumSquares += data[i] * data[i]; + if (s > peak) peak = s; + } + const rms = Math.sqrt(sumSquares / len); + const dynamicRange = peak - rms; + + // Zero-crossing rate + let crossings = 0; + for (let i = 1; i < len; i++) { + if ((data[i] >= 0) !== (data[i-1] >= 0)) crossings++; + } + const zcr = crossings / len; + + // Energy bands (bass, mid, treble) — rough FFT approximation + const bandSize = Math.floor(len / 3); + const energyBands = []; + for (let b = 0; b < 3; b++) { + let bandEnergy = 0; + for (let i = b * bandSize; i < (b+1) * bandSize; i++) { + bandEnergy += Math.abs(data[i]); + } + energyBands.push((bandEnergy / bandSize).toFixed(3)); + } + + // Spectral centroid (approximation) + let weightedSum = 0, totalMag = 0; + const fftSize = 2048; + const stepSize = Math.floor(len / fftSize); + for (let i = 0; i < stepSize; i++) { + const idx = i * fftSize; + for (let j = 0; j < fftSize; j++) { + const mag = Math.abs(data[idx + j] || 0); + weightedSum += j * mag; + totalMag += mag; + } + } + const spectralCentroid = totalMag > 0 ? (weightedSum / totalMag) * (sampleRate / fftSize) : 0; + + // BPM estimation (simplified autocorrelation) + const bpm = this._estimateBPM(data, sampleRate); + + // Spectrogram (text-art, downsampled) + let spectrogram = ''; + if (flags.includes('--spectrogram')) { + spectrogram = this._generateSpectrogram(data, sampleRate, audioBuffer.duration); + } + + // Beat times (simplified) + const beatTimes = this._findBeats(data, sampleRate); + + return { + duration, sampleRate, channels, rms, peak, dynamicRange, + zcr, spectralCentroid, energyBands, bpm, spectrogram, beatTimes + }; + } + + _estimateBPM(data, sampleRate) { + // Simplified: use energy envelope autocorrelation + const windowSize = Math.floor(sampleRate * 0.01); // 10ms windows + const numWindows = Math.floor(data.length / windowSize); + const envelope = []; + for (let w = 0; w < numWindows; w++) { + let energy = 0; + for (let i = 0; i < windowSize; i++) { + energy += Math.abs(data[w * windowSize + i]); + } + envelope.push(energy / windowSize); + } + // Autocorrelation at common BPM lags (60-200 BPM) + let bestLag = 0, bestCorr = 0; + for (let lag = Math.floor(60 * sampleRate / 200 / windowSize); lag < Math.floor(60 * sampleRate / 60 / windowSize); lag++) { + let corr = 0; + for (let i = lag; i < numWindows; i++) { + corr += envelope[i] * envelope[i - lag]; + } + corr /= (numWindows - lag); + if (corr > bestCorr) { bestCorr = corr; bestLag = lag; } + } + if (bestLag === 0) return 0; + return Math.round(60 / (bestLag * windowSize / sampleRate)); + } + + _findBeats(data, sampleRate) { + const beats = []; + const windowSize = Math.floor(sampleRate * 0.05); // 50ms windows + const energies = []; + for (let i = 0; i < data.length - windowSize; i += windowSize) { + let e = 0; + for (let j = 0; j < windowSize; j++) e += data[i+j] * data[i+j]; + energies.push(e / windowSize); + } + const avg = energies.reduce((a,b) => a+b, 0) / energies.length; + const threshold = avg * 1.5; + for (let i = 1; i < energies.length - 1; i++) { + if (energies[i] > threshold && energies[i] > energies[i-1] && energies[i] > energies[i+1]) { + beats.push((i * windowSize / sampleRate).toFixed(1)); + if (beats.length >= 20) break; // cap + } + } + return beats.map(Number); + } + + _generateSpectrogram(data, sampleRate, duration) { + // Text-based spectrogram: 20 rows (freq bands) x 40 cols (time) + const rows = 20, cols = 40; + const colSize = Math.floor(data.length / cols); + let result = ''; + const chars = ' ░▒▓█'; + for (let row = rows - 1; row >= 0; row--) { + const freqLow = (row / rows) * (sampleRate / 2); + const freqHigh = ((row + 1) / rows) * (sampleRate / 2); + let line = (row / rows * sampleRate / 2 / 1000).toFixed(0) + 'kHz│'; + for (let col = 0; col < cols; col++) { + // Rough energy at this freq band for this time window + let energy = 0; + for (let i = col * colSize; i < (col+1) * colSize; i++) { + // Simple band-pass: skip samples not in freq range + energy += Math.abs(data[i] || 0); + } + const norm = Math.min(1, energy / colSize * 4); + const idx = Math.floor(norm * (chars.length - 1)); + line += chars[idx]; + } + result += line + '\n'; + } + return result; + } + + // ─── API Calls ─── + async _callVision(base64, prompt) { + const endpoint = this._getOllamaEndpoint(); + try { + const resp = await this._fetchWithTimeout(endpoint + '/api/chat', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: this._visionModel, + messages: [{ role: 'user', content: prompt, images: [base64] }], + stream: false + }) + }, 60000); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + return { error: err.error || 'HTTP ' + resp.status }; + } + const data = await resp.json(); + return { text: data.message?.content || '(no response)', error: null }; + } catch (e) { + return { error: e.message }; + } + } + + async _callText(prompt) { + const endpoint = this._getOllamaEndpoint(); + try { + const resp = await this._fetchWithTimeout(endpoint + '/api/chat', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: this._textModel, + messages: [{ role: 'user', content: prompt }], + stream: false + }) + }, 60000); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + return { error: err.error || 'HTTP ' + resp.status }; + } + const data = await resp.json(); + return { text: data.message?.content || '(no response)', error: null }; + } catch (e) { + return { error: e.message }; + } + } + + // ─── Utils ─── + _getOllamaEndpoint() { + return localStorage.getItem('webx11_ollama_endpoint') || (window.location.origin + '/ollama'); + } + + _fetchWithTimeout(url, options, timeout = 30000) { + return new Promise((resolve, reject) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + fetch(url, { ...options, signal: controller.signal }) + .then(resolve).catch(reject).finally(() => clearTimeout(timer)); + }); + } + + _isImageExt(ext) { return ['png','jpg','jpeg','webp','gif','svg','bmp'].includes(ext); } + _isAudioExt(ext) { return ['wav','mp3','ogg','webm','flac','aac','m4a','opus'].includes(ext); } + _isVideoExt(ext) { return ['mp4','webm','ogv','avi','mov','mkv'].includes(ext); } + + _isValidBase64(str) { + if (typeof str !== 'string' || str.length < 8) return false; + try { return btoa(atob(str.substring(0, 100))) === str.substring(0, 100); } catch (e) { return false; } + } + + _base64ToArrayBuffer(base64) { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes.buffer; + } + + _getMimeType(path) { + const ext = path.split('.').pop().toLowerCase(); + const types = { mp4: 'video/mp4', webm: 'video/webm', ogv: 'video/ogg', mov: 'video/quicktime', avi: 'video/x-msvideo', mkv: 'video/x-matroska' }; + return types[ext] || 'application/octet-stream'; + } +} + +if (typeof window.webx11 !== 'undefined' && window.webx11) { + webx11.moduleManager.registerModule(new MediaModule(), 'https://webx11.duckdns.org/spaces/newos4/modules/ext/media.js'); +}
\ No newline at end of file |