path: root/modules/core/module-predictor.js

// module-predictor.js — Predictive resource allocation engine
// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
// real outcome. v1 counted every observed run in the denominator while incrementing the
// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
// Commands: predict, predictor-status, predictor-config
class PredictorModule {
    constructor() {
        this.commands = {};
        this.terminal = null;
        this._dataPoints = [];      // observed runs — the training set for _findSimilar
        this._pending = [];         // predictions waiting for an outcome (bounded, TTL'd)
        this._predictions = 0;
        this._accuracy = { correct: 0, total: 0 };  // total = paired comparisons only
        this._errorSum = 0;        // SUM of raw |predicted-observed|/observed (mean = /paired)
        this._turnsErr = 0;        // sum of |predicted turns - observed turns|
        this._CONFIDENCE_THRESHOLD = 0.5;
        this._MIN_DATA_POINTS = 5;
        this._PAIR_TTL_MS = 15 * 60 * 1000;
        this._SCHEMA = 2;
        this._loaded = false;
    }

    getHelpInfo() {
        return {
            'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
            'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
            'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
        };
    }

    registerCommands() {
        this.commands['predict'] = (args) => this.predict(args.join(' '));
        this.commands['predictor-status'] = () => this.showStatus();
        this.commands['predictor-config'] = (args) => this.setConfig(args);
    }

    init(terminal) {
        this.terminal = terminal;
        this._loadFromVFS();
        this._loaded = true;
        this.terminal.print('📊 Predictor loaded — analyzing task patterns', 'info');
    }

    // ─── Prediction Engine ───

    async predict(task) {
        if (!task) return this.terminal.print('Usage: predict <task description>', 'error');

        const features = this._extractFeatures(task);
        const similar = this._findSimilar(features);
        const prediction = this._computePrediction(similar, features);

        // Register it as pending so the next matching run can be scored against it.
        const pending = this.notePrediction(task, 'cli');
        pending.similarCount = similar.length;

        this.terminal.print('📈 Task Prediction', 'header');
        this.terminal.print(`  Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');

        if (prediction.confidence >= this._CONFIDENCE_THRESHOLD) {
            this.terminal.print(`  ⏱  Estimated duration: ${prediction.duration}s`, 'info');
            this.terminal.print(`  🔧 Estimated tool turns: ${prediction.toolTurns}`, 'info');
            this.terminal.print(`  📊 Confidence: ${(prediction.confidence * 100).toFixed(0)}% (based on ${similar.length} similar tasks)`, 'success');
        } else if (similar.length > 0) {
            this.terminal.print(`  ⏱  Estimated duration: ${prediction.duration}s (low confidence)`, 'warning');
            this.terminal.print(`  🔧 Tool turns: ~${prediction.toolTurns}`, 'warning');
            this.terminal.print(`  📊 Confidence: ${(prediction.confidence * 100).toFixed(0)}% — only ${similar.length} similar data point(s)`, 'warning');
            this.terminal.print('  Tip: More task data will improve predictions.', 'info');
        } else {
            this.terminal.print('  ❓ No historical data to base prediction on.', 'warning');
            this.terminal.print('  Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
            this.terminal.print('  Default estimate: ~30s duration, ~5 tool turns', 'info');
        }

        if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
            this.terminal.print(`  💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
        }

        this.terminal.print(`  🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
        return prediction;
    }

    _extractFeatures(task) {
        const lower = String(task || '').toLowerCase();
        return {
            length: String(task || '').length,
            wordCount: String(task || '').split(/\s+/).length,
            hasSearch: /\b(search|find|look)\b/.test(lower),
            hasWrite: /\b(write|create|save|edit)\b/.test(lower),
            hasDelete: /\b(delete|remove|rm)\b/.test(lower),
            hasInstall: /\b(install|wpm|package)\b/.test(lower),
            hasRead: /\b(read|cat|view|list)\b/.test(lower),
            hasWeb: /\b(fetch|url|http|web)\b/.test(lower),
            hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
            hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
            hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
            complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
            keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
        };
    }

    // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
    _similarity(a, b) {
        if (!a || !b) return 0;
        const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
                      'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
        const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
        const kw = a.keywords || [];
        const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
        const kScore = overlap / Math.max(kw.length, 1);
        return matches * 0.6 + kScore * 0.4;
    }

    _findSimilar(features) {
        return this._dataPoints.filter(dp => {
            if (!dp.observedDuration) return false;   // outcomes only, never predictions
            if (dp.usable === false) return false;    // aborted runs are not evidence
            if (dp.features) return this._similarity(features, dp.features) > 0.3;
            return false;
        });
    }

    _computePrediction(similar, features) {
        if (similar.length === 0) {
            let baseDuration = 15;
            let baseTurns = 3;
            if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
            if (features.hasWrite) { baseDuration = 20; baseTurns = 4; }
            if (features.hasOrchestrate) { baseDuration = 40; baseTurns = 8; }
            if (features.hasSync) { baseDuration = 30; baseTurns = 6; }
            if (features.complexity > 5) baseDuration *= 1.5;
            return {
                duration: baseDuration,
                toolTurns: baseTurns,
                confidence: 0.1,
                similarCount: 0,
                floored: false
            };
        }

        const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
        const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
        // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
        const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
        const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;

        const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
        const adjustedDuration = avgDuration * Math.max(complexityMultiplier, 0.5);
        const floored = adjustedDuration < 1;          // no prediction is "0 seconds"
        const duration = Math.max(1, Math.round(adjustedDuration));

        return {
            duration,
            toolTurns: Math.max(1, Math.round(avgTurns)),
            confidence: Math.min(confidence, 0.95),
            similarCount: similar.length,
            floored
        };
    }

    // ─── Prediction ↔ Outcome pairing (v2) ───

    // Task text never leaves the module: only a stable hash is stored/serialised.
    _hashTask(t) {
        const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
        let h = 0x811c9dc5;
        for (let i = 0; i < s.length; i++) {
            h ^= s.charCodeAt(i);
            h = (h * 0x01000193) >>> 0;
        }
        return h.toString(16);
    }

    // Register a prediction to be scored later. Returns the pending record (with .id).
    notePrediction(task, source) {
        const features = this._extractFeatures(task || '');
        const p = this._computePrediction(this._findSimilar(features), features);
        const rec = {
            id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
            taskHash: this._hashTask(task),
            features,
            predictedDuration: p.duration,
            predictedTurns: p.toolTurns,
            confidence: p.confidence,
            floored: p.floored === true,
            source: source || 'loop',
            ts: Date.now()
        };
        this._pending.push(rec);
        if (this._pending.length > 50) this._pending = this._pending.slice(-50);
        this._predictions++;
        this._debounceSave();
        return rec;
    }

    _takePending(features, opts) {
        const now = Date.now();
        this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
        const taskHash = opts.task ? this._hashTask(opts.task) : null;

        let idx = -1;
        if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
        if (idx < 0 && taskHash) {
            for (let i = this._pending.length - 1; i >= 0; i--) {
                if (this._pending[i].taskHash === taskHash) { idx = i; break; }
            }
        }
        if (idx < 0) {
            // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
            // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
            let best = -1, bestScore = 0;
            for (let i = 0; i < this._pending.length; i++) {
                const cand = this._pending[i];
                if (cand.source !== 'cli') continue;
                const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
                if (shared === 0) continue;
                const s = this._similarity(features, cand.features);
                if (s > bestScore) { bestScore = s; best = i; }
            }
            if (bestScore >= 0.5) idx = best;
        }
        return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
    }

    // ─── Learning from Results ───

    recordResult(features, duration, toolTurns, success, opts) {
        const o = opts || {};
        const pred = this._takePending(features, o);
        const observed = Number(duration) || 0;

        const rec = {
            features,
            observedDuration: observed,
            observedTurns: toolTurns,
            success: !!success,
            paired: !!pred,
            timestamp: new Date().toISOString()
        };

        rec.usable = observed >= 1.5;    // aborted/instant runs say nothing about duration

        if (pred) {
            const absErr = Math.abs(pred.predictedDuration - observed);
            rec.predictedDuration = pred.predictedDuration;
            rec.predictedTurns = pred.predictedTurns;
            // RAW error — the renderer clamps, the store does not (a 475x miss must stay 475x)
            rec.errorPct = observed > 0 ? absErr / observed : null;
            rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
            rec.scored = rec.usable && rec.errorPct !== null && pred.floored !== true;
            if (rec.scored) {
                this._accuracy.total++;
                this._errorSum += rec.errorPct;
                this._turnsErr += rec.turnsError;
                // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
                if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
            }
        }

        this._dataPoints.push(rec);
        if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
        if (pred) this._saveNow(); else this._debounceSave();
        return rec;
    }

    // ─── Commands ───

    showStatus() {
        const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
        const paired = this._accuracy.total;
        const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
        const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
        const meanErr = paired > 0 ? this._errorSum / paired : 0;
        const mape = paired > 0
            ? ` · mean error ${(Math.min(meanErr, 5) * 100).toFixed(0)}%${meanErr > 5 ? '+' : ''}` +
              ` · turns ±${(this._turnsErr / paired).toFixed(1)}`
            : '';
        const unscored = this._dataPoints.filter(dp => dp.scored === false).length;

        this.terminal.print('📊 Predictor Status', 'header');
        this.terminal.print(`  Total data points: ${this._dataPoints.length}`, 'info');
        this.terminal.print(`  Actual executions: ${actualExecutions}`, 'info');
        this.terminal.print(`  Predictions made: ${this._predictions}`, 'info');
        this.terminal.print(`  Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
        this.terminal.print(`  Awaiting outcome: ${this._pending.length}`, 'info');
        this.terminal.print(`  Unpaired runs: ${unpaired}`, 'info');
        this.terminal.print(`  Unscored pairs: ${unscored} (aborted run or floored prediction)`, 'info');
        this.terminal.print(`  Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
        this.terminal.print(`  Min data points: ${this._MIN_DATA_POINTS}`, 'info');
        this.terminal.print(`  Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
    }

    setConfig(args) {
        if (args.length < 2) return this.terminal.print('Usage: predictor-config <key> <value>', 'error');
        const key = args[0];
        const val = args[1];

        const configMap = {
            'confidence-threshold': () => {
                const n = parseFloat(val);
                if (isNaN(n) || n < 0 || n > 1) return 'Must be 0.0-1.0';
                this._CONFIDENCE_THRESHOLD = n;
                return `confidence-threshold set to ${n}`;
            },
            'min-data-points': () => {
                const n = parseInt(val);
                if (isNaN(n) || n < 1) return 'Must be >= 1';
                this._MIN_DATA_POINTS = n;
                return `min-data-points set to ${n}`;
            },
            'pair-ttl': () => {
                const n = parseInt(val);
                if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
                this._PAIR_TTL_MS = n * 1000;
                return `pair-ttl set to ${n}s`;
            },
        };

        if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
        const msg = configMap[key]();
        this.terminal.print(`✅ ${msg}`, 'success');
    }

    // ─── Persistence ───

    _loadFromVFS() {
        try {
            const fs = this.terminal.fs;
            if (!fs) return;
            const data = fs.fsReadJSON('/system/metrics/predictor.json');
            if (!data) return;

            // Prediction-only points are noise for similarity; keep outcomes.
            this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
            this._predictions = data.predictions || 0;

            if (data.v === this._SCHEMA) {
                const acc = data.accuracy || {};
                this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
                this._errorSum = (typeof data.errorSum === 'number') ? data.errorSum : (data.mape || 0);
                this._turnsErr = data.turnsErr || 0;
                const cfg = data.config || {};
                if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
                if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
                if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
                const now = Date.now();
                this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
            } else {
                // v1 counted every observation in `total` but only some in `correct` → 0%.
                this._accuracy = { correct: 0, total: 0 };
                this._errorSum = 0;
                this._pending = [];
            }
        } catch (e) {
            console.warn('[Predictor] Load error (fresh start):', e.message);
        }
    }

    _saveNow() {
        try {
            const fs = this.terminal.fs;
            if (!fs) return;
            if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
            fs.fsWriteJSON('/system/metrics/predictor.json', {
                v: this._SCHEMA,
                dataPoints: this._dataPoints.slice(-500),
                predictions: this._predictions,
                accuracy: this._accuracy,
                errorSum: this._errorSum,
                turnsErr: this._turnsErr,
                config: {
                    confidenceThreshold: this._CONFIDENCE_THRESHOLD,
                    minDataPoints: this._MIN_DATA_POINTS,
                    pairTtlMs: this._PAIR_TTL_MS
                },
                pending: this._pending.slice(-50)
            });
        } catch (e) {
            console.error('[Predictor] Save error:', e.message);
        }
    }

    _debounceSave() {
        clearTimeout(this._saveTimer);
        this._saveTimer = setTimeout(() => this._saveNow(), 3000);
    }
}

// Required for module system
if (typeof window !== 'undefined') {
    window.PredictorModule = PredictorModule;
}