path: root/modules/ext/media.js

// newOS4 Media Analysis Module (v4.5.79)
// Commands: analyze <path> [--flags]
// Uses gemini-3-flash-preview:cloud for vision (images, video frames)
// Uses deepseek-v4-flash:cloud for text analysis (audio features, summaries)
class MediaModule {
    constructor() {
        this.commands = {};
        this.terminal = null;
        this._visionModel = 'gemini-3-flash-preview:cloud';
        this._textModel = 'deepseek-v4-flash:cloud';
        this._maxFrames = 10;       // max video frames to analyze
        this._frameInterval = 5;    // seconds between video frames
    }

    getHelpInfo() {
        return {
            'analyze': 'analyze <file> — Analyze image/audio/video. Flags: --frames N, --spectrogram, --beats, --scene'
        };
    }

    registerCommands() {
        this.commands['analyze'] = (args) => this.handle(args);
    }

    handle(args) {
        if (!args[0]) {
            this.terminal.print('Usage: analyze <file> [--frames N] [--spectrogram] [--beats] [--scene]', 'warning');
            return;
        }
        const path = args[0];
        const flags = args.slice(1);
        const ext = path.split('.').pop().toLowerCase();

        // Check file exists
        const content = this.terminal.fs.readFile(path);
        if (content === null || content === undefined) {
            this.terminal.print('File not found: ' + path, 'error');
            return;
        }

        this.terminal.print('🔍 Analyzing ' + path + '...', 'info');

        if (this._isImageExt(ext)) {
            this._analyzeImage(path, content);
        } else if (this._isAudioExt(ext)) {
            this._analyzeAudio(path, content, flags);
        } else if (this._isVideoExt(ext)) {
            this._analyzeVideo(path, content, flags);
        } else {
            this.terminal.print('Unsupported file type: .' + ext + '. Supported: png, jpg, webp, gif, svg, wav, mp3, ogg, mp4, webm', 'error');
        }
    }

    // ─── Image Analysis ───
    async _analyzeImage(path, content) {
        this.terminal.print('🖼️ Sending image to vision model...', 'info');
        try {
            // VFS may store binary as base64 string or raw text
            let base64 = content;
            // If content looks like raw text (not base64), try to re-encode
            if (!this._isValidBase64(base64) && typeof content === 'string') {
                // It's text content, not an image — can't analyze
                this.terminal.print('⚠️ File appears to be text, not a binary image', 'warning');
                return;
            }
            const result = await this._callVision(base64, 'Describe this image in detail. What do you see? Include colors, objects, text, people, and any notable features.');
            if (result.error) {
                this.terminal.print('🖼️ Vision error: ' + result.error, 'error');
                return;
            }
            this.terminal.print('🖼️ Image Analysis:', 'success');
            this.terminal.print(result.text, 'info');
        } catch (e) {
            this.terminal.print('🖼️ Analysis failed: ' + e.message, 'error');
        }
    }

    // ─── Audio Analysis ───
    async _analyzeAudio(path, content, flags) {
        this.terminal.print('🎵 Decoding audio...', 'info');
        try {
            // Convert base64 to ArrayBuffer for Web Audio API
            let arrayBuffer;
            if (this._isValidBase64(content)) {
                arrayBuffer = this._base64ToArrayBuffer(content);
            } else if (content instanceof ArrayBuffer) {
                arrayBuffer = content;
            } else {
                this.terminal.print('⚠️ Cannot decode audio — file is not in a supported binary format in VFS', 'warning');
                return;
            }

            const ctx = new (window.AudioContext || window.webkitAudioContext)();
            const audioBuffer = await ctx.decodeAudioData(arrayBuffer);

            // Extract features
            const features = this._extractAudioFeatures(audioBuffer, flags);
            ctx.close();

            // Display features
            this.terminal.print('🎵 Audio Features:', 'success');
            this.terminal.print('  Duration: ' + features.duration.toFixed(1) + 's', 'info');
            this.terminal.print('  Sample Rate: ' + features.sampleRate + ' Hz', 'info');
            this.terminal.print('  Channels: ' + features.channels, 'info');
            this.terminal.print('  RMS (avg loudness): ' + features.rms.toFixed(3), 'info');
            this.terminal.print('  Peak amplitude: ' + features.peak.toFixed(3), 'info');
            this.terminal.print('  Dynamic range: ' + features.dynamicRange.toFixed(3), 'info');

            if (flags.includes('--spectrogram')) {
                this.terminal.print('\n📊 Spectrogram (frequency bands over time):', 'info');
                this.terminal.print(features.spectrogram, '');
            }

            if (flags.includes('--beats')) {
                this.terminal.print('\n🥁 Beat detection:', 'info');
                this.terminal.print('  Estimated BPM: ' + features.bpm, 'info');
                this.terminal.print('  Beat times: ' + features.beatTimes.join(', '), 'info');
            }

            // Send features to text model for AI interpretation
            this.terminal.print('\n🤖 AI interpretation (deepseek-v4-flash)...', 'info');
            const prompt = `Analyze these audio features and describe what the audio likely sounds like. Consider the genre, mood, and characteristics:

Duration: ${features.duration.toFixed(1)}s
Sample Rate: ${features.sampleRate} Hz
Channels: ${features.channels}
RMS (loudness): ${features.rms.toFixed(3)}
Peak: ${features.peak.toFixed(3)}
Dynamic Range: ${features.dynamicRange.toFixed(3)}
Estimated BPM: ${features.bpm}
Spectral centroid: ${features.spectralCentroid.toFixed(0)} Hz
Zero-crossing rate: ${features.zcr.toFixed(4)}
Energy distribution: ${features.energyBands.join(', ')}

Provide a brief, insightful interpretation.`;

            const result = await this._callText(prompt);
            if (!result.error) this.terminal.print(result.text, 'info');
            else this.terminal.print('AI interpretation unavailable: ' + result.error, 'warning');

        } catch (e) {
            this.terminal.print('🎵 Audio analysis failed: ' + e.message, 'error');
        }
    }

    // ─── Video Analysis ───
    async _analyzeVideo(path, content, flags) {
        // Parse flags
        let maxFrames = this._maxFrames;
        const framesIdx = flags.indexOf('--frames');
        if (framesIdx !== -1 && flags[framesIdx + 1]) maxFrames = parseInt(flags[framesIdx + 1]) || 10;
        const sceneMode = flags.includes('--scene');

        this.terminal.print('🎬 Extracting video frames...', 'info');

        try {
            // Create video element
            const video = document.createElement('video');
            video.muted = true;
            video.crossOrigin = 'anonymous';

            // Load from base64 or URL
            let videoUrl;
            if (this._isValidBase64(content)) {
                const mime = this._getMimeType(path);
                videoUrl = 'data:' + mime + ';base64,' + content;
            } else {
                this.terminal.print('⚠️ Cannot load video — VFS does not store video as base64', 'warning');
                return;
            }

            await new Promise((resolve, reject) => {
                video.onloadedmetadata = resolve;
                video.onerror = () => reject(new Error('Video decode failed — unsupported codec?'));
                video.src = videoUrl;
            });

            const duration = video.duration;
            this.terminal.print('  Duration: ' + duration.toFixed(1) + 's', 'info');
            this.terminal.print('  Extracting ' + maxFrames + ' frames...', 'info');

            const canvas = document.createElement('canvas');
            canvas.width = 320;  // downscale for API efficiency
            canvas.height = 180;
            const ctx = canvas.getContext('2d');

            const framePromises = [];
            const frameInterval = duration / maxFrames;

            for (let i = 0; i < maxFrames; i++) {
                const t = Math.min(i * frameInterval, duration - 0.1);
                framePromises.push(this._extractFrame(video, canvas, ctx, t, i));
            }

            const frames = await Promise.all(framePromises);
            video.src = '';

            this.terminal.print('  Frames extracted: ' + frames.length, 'info');

            if (sceneMode) {
                // Detect scene changes by comparing consecutive frames
                const sceneChanges = this._detectSceneChanges(frames);
                this.terminal.print('  Scene changes at: ' + sceneChanges.map(s => s.toFixed(1) + 's').join(', '), 'info');
            }

            // Send frames to vision model
            this.terminal.print('\n🤖 Analyzing frames with vision model...', 'info');
            const descriptions = [];
            for (let i = 0; i < frames.length; i++) {
                const desc = await this._callVision(frames[i].base64, `This is frame ${i+1}/${frames.length} at ${frames[i].time.toFixed(1)}s from a video. Describe what you see briefly.`);
                if (!desc.error) {
                    descriptions.push(`[${frames[i].time.toFixed(1)}s] ${desc.text}`);
                    this.terminal.print(`  Frame ${i+1}/${frames.length} (${frames[i].time.toFixed(1)}s): ${desc.text.substring(0, 100)}`, 'info');
                }
            }

            // Summarize with text model
            if (descriptions.length > 0) {
                this.terminal.print('\n🎬 Video Summary:', 'success');
                const summaryPrompt = `These are descriptions of ${descriptions.length} frames from a video, taken at regular intervals:
${descriptions.join('\n')}

Provide a brief summary of what happens in this video. Include the overall scene, any changes, and the general content.`;
                const summary = await this._callText(summaryPrompt);
                if (!summary.error) this.terminal.print(summary.text, 'info');
            }

        } catch (e) {
            this.terminal.print('🎬 Video analysis failed: ' + e.message, 'error');
        }
    }

    // ─── Helper: Extract single frame ───
    async _extractFrame(video, canvas, ctx, time, index) {
        return new Promise((resolve, reject) => {
            video.currentTime = time;
            video.onseeked = () => {
                try {
                    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
                    const dataUrl = canvas.toDataURL('image/jpeg', 0.7);
                    const base64 = dataUrl.split(',')[1];
                    resolve({ base64, time, index });
                } catch (e) { reject(e); }
            };
            video.onerror = () => reject(new Error('Seek failed at ' + time + 's'));
        });
    }

    // ─── Helper: Scene change detection ───
    _detectSceneChanges(frames) {
        const changes = [];
        for (let i = 1; i < frames.length; i++) {
            // Simple histogram comparison
            const prev = this._frameHistogram(frames[i-1].base64);
            const curr = this._frameHistogram(frames[i].base64);
            const diff = this._histogramDiff(prev, curr);
            if (diff > 0.3) {
                changes.push(frames[i].time);
            }
        }
        return changes;
    }

    _frameHistogram(base64) {
        // Lightweight: use first 100 bytes of base64 as fingerprint
        const bytes = atob(base64.substring(0, 200));
        const hist = new Array(16).fill(0);
        for (let i = 0; i < bytes.length; i++) {
            hist[bytes.charCodeAt(i) % 16]++;
        }
        return hist;
    }

    _histogramDiff(a, b) {
        let diff = 0;
        for (let i = 0; i < a.length; i++) {
            diff += Math.abs(a[i] - b[i]);
        }
        return diff / (a.reduce((s,v) => s+v, 0) || 1);
    }

    // ─── Audio Feature Extraction ───
    _extractAudioFeatures(audioBuffer, flags) {
        const data = audioBuffer.getChannelData(0);
        const len = data.length;
        const sampleRate = audioBuffer.sampleRate;
        const channels = audioBuffer.numberOfChannels;
        const duration = audioBuffer.duration;

        // RMS
        let sumSquares = 0, peak = 0;
        for (let i = 0; i < len; i++) {
            const s = Math.abs(data[i]);
            sumSquares += data[i] * data[i];
            if (s > peak) peak = s;
        }
        const rms = Math.sqrt(sumSquares / len);
        const dynamicRange = peak - rms;

        // Zero-crossing rate
        let crossings = 0;
        for (let i = 1; i < len; i++) {
            if ((data[i] >= 0) !== (data[i-1] >= 0)) crossings++;
        }
        const zcr = crossings / len;

        // Energy bands (bass, mid, treble) — rough FFT approximation
        const bandSize = Math.floor(len / 3);
        const energyBands = [];
        for (let b = 0; b < 3; b++) {
            let bandEnergy = 0;
            for (let i = b * bandSize; i < (b+1) * bandSize; i++) {
                bandEnergy += Math.abs(data[i]);
            }
            energyBands.push((bandEnergy / bandSize).toFixed(3));
        }

        // Spectral centroid (approximation)
        let weightedSum = 0, totalMag = 0;
        const fftSize = 2048;
        const stepSize = Math.floor(len / fftSize);
        for (let i = 0; i < stepSize; i++) {
            const idx = i * fftSize;
            for (let j = 0; j < fftSize; j++) {
                const mag = Math.abs(data[idx + j] || 0);
                weightedSum += j * mag;
                totalMag += mag;
            }
        }
        const spectralCentroid = totalMag > 0 ? (weightedSum / totalMag) * (sampleRate / fftSize) : 0;

        // BPM estimation (simplified autocorrelation)
        const bpm = this._estimateBPM(data, sampleRate);

        // Spectrogram (text-art, downsampled)
        let spectrogram = '';
        if (flags.includes('--spectrogram')) {
            spectrogram = this._generateSpectrogram(data, sampleRate, audioBuffer.duration);
        }

        // Beat times (simplified)
        const beatTimes = this._findBeats(data, sampleRate);

        return {
            duration, sampleRate, channels, rms, peak, dynamicRange,
            zcr, spectralCentroid, energyBands, bpm, spectrogram, beatTimes
        };
    }

    _estimateBPM(data, sampleRate) {
        // Simplified: use energy envelope autocorrelation
        const windowSize = Math.floor(sampleRate * 0.01); // 10ms windows
        const numWindows = Math.floor(data.length / windowSize);
        const envelope = [];
        for (let w = 0; w < numWindows; w++) {
            let energy = 0;
            for (let i = 0; i < windowSize; i++) {
                energy += Math.abs(data[w * windowSize + i]);
            }
            envelope.push(energy / windowSize);
        }
        // Autocorrelation at common BPM lags (60-200 BPM)
        let bestLag = 0, bestCorr = 0;
        for (let lag = Math.floor(60 * sampleRate / 200 / windowSize); lag < Math.floor(60 * sampleRate / 60 / windowSize); lag++) {
            let corr = 0;
            for (let i = lag; i < numWindows; i++) {
                corr += envelope[i] * envelope[i - lag];
            }
            corr /= (numWindows - lag);
            if (corr > bestCorr) { bestCorr = corr; bestLag = lag; }
        }
        if (bestLag === 0) return 0;
        return Math.round(60 / (bestLag * windowSize / sampleRate));
    }

    _findBeats(data, sampleRate) {
        const beats = [];
        const windowSize = Math.floor(sampleRate * 0.05); // 50ms windows
        const energies = [];
        for (let i = 0; i < data.length - windowSize; i += windowSize) {
            let e = 0;
            for (let j = 0; j < windowSize; j++) e += data[i+j] * data[i+j];
            energies.push(e / windowSize);
        }
        const avg = energies.reduce((a,b) => a+b, 0) / energies.length;
        const threshold = avg * 1.5;
        for (let i = 1; i < energies.length - 1; i++) {
            if (energies[i] > threshold && energies[i] > energies[i-1] && energies[i] > energies[i+1]) {
                beats.push((i * windowSize / sampleRate).toFixed(1));
                if (beats.length >= 20) break; // cap
            }
        }
        return beats.map(Number);
    }

    _generateSpectrogram(data, sampleRate, duration) {
        // Text-based spectrogram: 20 rows (freq bands) x 40 cols (time)
        const rows = 20, cols = 40;
        const colSize = Math.floor(data.length / cols);
        let result = '';
        const chars = ' ░▒▓█';
        for (let row = rows - 1; row >= 0; row--) {
            const freqLow = (row / rows) * (sampleRate / 2);
            const freqHigh = ((row + 1) / rows) * (sampleRate / 2);
            let line = (row / rows * sampleRate / 2 / 1000).toFixed(0) + 'kHz│';
            for (let col = 0; col < cols; col++) {
                // Rough energy at this freq band for this time window
                let energy = 0;
                for (let i = col * colSize; i < (col+1) * colSize; i++) {
                    // Simple band-pass: skip samples not in freq range
                    energy += Math.abs(data[i] || 0);
                }
                const norm = Math.min(1, energy / colSize * 4);
                const idx = Math.floor(norm * (chars.length - 1));
                line += chars[idx];
            }
            result += line + '\n';
        }
        return result;
    }

    // ─── API Calls ───
    async _callVision(base64, prompt) {
        const endpoint = this._getOllamaEndpoint();
        try {
            const resp = await this._fetchWithTimeout(endpoint + '/api/chat', {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({
                    model: this._visionModel,
                    messages: [{ role: 'user', content: prompt, images: [base64] }],
                    stream: false
                })
            }, 60000);
            if (!resp.ok) {
                const err = await resp.json().catch(() => ({}));
                return { error: err.error || 'HTTP ' + resp.status };
            }
            const data = await resp.json();
            return { text: data.message?.content || '(no response)', error: null };
        } catch (e) {
            return { error: e.message };
        }
    }

    async _callText(prompt) {
        const endpoint = this._getOllamaEndpoint();
        try {
            const resp = await this._fetchWithTimeout(endpoint + '/api/chat', {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({
                    model: this._textModel,
                    messages: [{ role: 'user', content: prompt }],
                    stream: false
                })
            }, 60000);
            if (!resp.ok) {
                const err = await resp.json().catch(() => ({}));
                return { error: err.error || 'HTTP ' + resp.status };
            }
            const data = await resp.json();
            return { text: data.message?.content || '(no response)', error: null };
        } catch (e) {
            return { error: e.message };
        }
    }

    // ─── Utils ───
    _getOllamaEndpoint() {
        return localStorage.getItem('webx11_ollama_endpoint') || (window.location.origin + '/ollama');
    }

    _fetchWithTimeout(url, options, timeout = 30000) {
        return new Promise((resolve, reject) => {
            const controller = new AbortController();
            const timer = setTimeout(() => controller.abort(), timeout);
            fetch(url, { ...options, signal: controller.signal })
                .then(resolve).catch(reject).finally(() => clearTimeout(timer));
        });
    }

    _isImageExt(ext) { return ['png','jpg','jpeg','webp','gif','svg','bmp'].includes(ext); }
    _isAudioExt(ext) { return ['wav','mp3','ogg','webm','flac','aac','m4a','opus'].includes(ext); }
    _isVideoExt(ext) { return ['mp4','webm','ogv','avi','mov','mkv'].includes(ext); }

    _isValidBase64(str) {
        if (typeof str !== 'string' || str.length < 8) return false;
        try { return btoa(atob(str.substring(0, 100))) === str.substring(0, 100); } catch (e) { return false; }
    }

    _base64ToArrayBuffer(base64) {
        const binary = atob(base64);
        const bytes = new Uint8Array(binary.length);
        for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
        return bytes.buffer;
    }

    _getMimeType(path) {
        const ext = path.split('.').pop().toLowerCase();
        const types = { mp4: 'video/mp4', webm: 'video/webm', ogv: 'video/ogg', mov: 'video/quicktime', avi: 'video/x-msvideo', mkv: 'video/x-matroska' };
        return types[ext] || 'application/octet-stream';
    }
}

if (typeof window.webx11 !== 'undefined' && window.webx11) {
    webx11.moduleManager.registerModule(new MediaModule(), 'https://webx11.duckdns.org/spaces/newos4/modules/ext/media.js');
}