diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/core/core-bundle.js | 308 | ||||
| -rw-r--r-- | modules/core/module-aiassist.js | 47 | ||||
| -rw-r--r-- | modules/core/module-loader.js | 2 | ||||
| -rw-r--r-- | modules/core/module-predictor.js | 263 | ||||
| -rw-r--r-- | modules/test/test-predictor-accuracy.js | 209 |
5 files changed, 637 insertions, 192 deletions
diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index 67747bf..9ac94ed 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -9041,6 +9041,7 @@ buildSystemPrompt() { await new Promise(r => setTimeout(r, 50)); } this._chatLock = true; + const taskStart = Date.now(); // predictor durations exclude chat-lock wait this.terminal.markDirty(); // Auto-route context based on intent try { @@ -9131,6 +9132,16 @@ buildSystemPrompt() { } } catch (e) { /* world model not available */ } + // v4.6.56: register a prediction for THIS turn so recordResult can score it by id. + let turnPredictionId = null; + try { + const predictor = this.terminal.moduleManager.modules['PredictorModule']; + if (predictor && typeof predictor.notePrediction === 'function') { + const tp = predictor.notePrediction(userMessage, 'loop'); + turnPredictionId = tp && tp.id ? tp.id : null; + } + } catch (e) { /* predictor is best-effort */ } + const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main'; try { loopResult = await this.agenticLoop(messages, opts); @@ -9158,8 +9169,9 @@ buildSystemPrompt() { const predictor = this.terminal.moduleManager.modules['PredictorModule']; if (predictor && typeof predictor.recordResult === 'function') { const features = predictor._extractFeatures(userMessage); - const duration = (Date.now() - lockStart) / 1000; - predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled); + const duration = (Date.now() - taskStart) / 1000; + predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled, + { task: userMessage, predictionId: turnPredictionId }); } } catch (e) { /* predictor learning is best-effort */ } // Record task in memory graph for direct AI conversations (not just orchestrator) @@ -11241,16 +11253,26 @@ buildSystemPrompt() { if (!taskDesc) return JSON.stringify({ error: 'task parameter required' }); var pFeatures = predictor._extractFeatures(taskDesc); var pSimilar = predictor._findSimilar(pFeatures); - var pResult = predictor._computePrediction(pSimilar, pFeatures); + // v4.6.56: track the prediction so the outcome can score it. + var pTracked; + if (typeof predictor.notePrediction === 'function') { + pTracked = predictor.notePrediction(taskDesc, 'tool'); + } else { + var pFallback = predictor._computePrediction(pSimilar, pFeatures); + pTracked = { id: null, predictedDuration: pFallback.duration, + predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence }; + } return JSON.stringify({ task: taskDesc.substring(0, 80), - estimatedDuration: pResult.duration + 's', - estimatedToolTurns: pResult.toolTurns, - confidence: Math.round(pResult.confidence * 100) + '%', - similarTasks: pResult.similarCount, - recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30 - ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's' - : 'No special action needed' + predictionId: pTracked.id || null, + estimatedDuration: pTracked.predictedDuration + 's', + estimatedToolTurns: pTracked.predictedTurns, + confidence: Math.round(pTracked.confidence * 100) + '%', + similarTasks: pSimilar.length, + recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30 + ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's' + : 'No special action needed', + tracked: !!pTracked.id }); } case 'world_model_query': { @@ -12841,9 +12863,10 @@ buildSystemPrompt() { if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length; } catch (e) {} var predictor = mm ? mm.modules['PredictorModule'] : null; - var predictorAcc = 'N/A'; + var predictorAcc = 'n/a (no paired runs)'; if (predictor && predictor._accuracy && predictor._accuracy.total > 0) { - predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%'; + predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' + + predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)'; } data.health = { modules: data.modules.loaded + '/' + data.modules.total, @@ -17736,25 +17759,32 @@ if (typeof window !== 'undefined') { /* === module-predictor.js === */ // module-predictor.js — Predictive resource allocation engine -// Analyzes historical telemetry + memory graph data to predict task duration/turns +// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a +// real outcome. v1 counted every observed run in the denominator while incrementing the +// numerator only when an old `predict` record happened to match, so the dashboard read 0%. // Commands: predict, predictor-status, predictor-config class PredictorModule { constructor() { this.commands = {}; this.terminal = null; - this._dataPoints = []; + this._dataPoints = []; // observed runs — the training set for _findSimilar + this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; - this._accuracy = { correct: 0, total: 0 }; + this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only + this._mape = 0; // sum of capped |predicted-observed|/observed + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; + this._PAIR_TTL_MS = 15 * 60 * 1000; + this._SCHEMA = 2; this._loaded = false; } getHelpInfo() { return { 'predict': 'predict <task description> — Predict task duration, tool turns, and confidence', - 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count', - 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points', + 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points', + 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl', }; } @@ -17776,33 +17806,14 @@ class PredictorModule { async predict(task) { if (!task) return this.terminal.print('Usage: predict <task description>', 'error'); - const startTime = performance.now(); - - // Extract features const features = this._extractFeatures(task); - - // Find similar historical tasks const similar = this._findSimilar(features); - - // Generate prediction const prediction = this._computePrediction(similar, features); - // Record this prediction for accuracy tracking - const duration = ((performance.now() - startTime) / 1000).toFixed(2); + // Register it as pending so the next matching run can be scored against it. + const pending = this.notePrediction(task, 'cli'); + pending.similarCount = similar.length; - this._predictions++; - this._dataPoints.push({ - features, - predictedDuration: prediction.duration, - predictedTurns: prediction.toolTurns, - confidence: prediction.confidence, - similarCount: similar.length, - timestamp: new Date().toISOString() - }); - - this._debounceSave(); - - // Display result this.terminal.print('📈 Task Prediction', 'header'); this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info'); @@ -17818,23 +17829,22 @@ class PredictorModule { } else { this.terminal.print(' ❓ No historical data to base prediction on.', 'warning'); this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info'); - // Provide hardcoded defaults this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info'); } - // If got memory graph, offer recommendations if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) { this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info'); } + this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info'); return prediction; } _extractFeatures(task) { - const lower = task.toLowerCase(); + const lower = String(task || '').toLowerCase(); return { - length: task.length, - wordCount: task.split(/\s+/).length, + length: String(task || '').length, + wordCount: String(task || '').split(/\s+/).length, hasSearch: /\b(search|find|look)\b/.test(lower), hasWrite: /\b(write|create|save|edit)\b/.test(lower), hasDelete: /\b(delete|remove|rm)\b/.test(lower), @@ -17844,42 +17854,33 @@ class PredictorModule { hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower), hasModule: /\b(module|edit_module|view_module)\b/.test(lower), hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower), - complexity: Math.min(task.length / 50, 10), // 0-10 scale + complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10) }; } + // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2). + _similarity(a, b) { + if (!a || !b) return 0; + const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead', + 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync']; + const matches = keys.filter(k => a[k] === b[k]).length / keys.length; + const kw = a.keywords || []; + const overlap = kw.filter(k => (b.keywords || []).includes(k)).length; + const kScore = overlap / Math.max(kw.length, 1); + return matches * 0.6 + kScore * 0.4; + } + _findSimilar(features) { return this._dataPoints.filter(dp => { - // Skip predictions (only use actual executions with observed results) - if (!dp.observedDuration) return false; - const f = dp.features; - if (!f) return false; - // Feature similarity scoring - let score = 0; - const matches = [ - features.hasSearch === f.hasSearch, - features.hasWrite === f.hasWrite, - features.hasDelete === f.hasDelete, - features.hasInstall === f.hasInstall, - features.hasRead === f.hasRead, - features.hasWeb === f.hasWeb, - features.hasOrchestrate === f.hasOrchestrate, - features.hasModule === f.hasModule, - features.hasSync === f.hasSync, - ]; - score = matches.filter(Boolean).length / matches.length; - // Keyword overlap bonus - const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length; - const kScore = kOverlap / Math.max(features.keywords.length, 1); - score = score * 0.6 + kScore * 0.4; - return score > 0.3; + if (!dp.observedDuration) return false; // outcomes only, never predictions + if (!dp.features) return false; + return this._similarity(features, dp.features) > 0.3; }); } _computePrediction(similar, features) { if (similar.length === 0) { - // Hardcoded defaults let baseDuration = 15; let baseTurns = 3; if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; } @@ -17897,9 +17898,10 @@ class PredictorModule { const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length; const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length; - const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1; + // v2: the documented knob now actually scales confidence (v1 hardcoded 20). + const minPts = Math.max(1, this._MIN_DATA_POINTS || 5); + const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; - // Apply task complexity multiplier const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); @@ -17911,43 +17913,126 @@ class PredictorModule { }; } + // ─── Prediction ↔ Outcome pairing (v2) ─── + + // Task text never leaves the module: only a stable hash is stored/serialised. + _hashTask(t) { + const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = (h * 0x01000193) >>> 0; + } + return h.toString(16); + } + + // Register a prediction to be scored later. Returns the pending record (with .id). + notePrediction(task, source) { + const features = this._extractFeatures(task || ''); + const p = this._computePrediction(this._findSimilar(features), features); + const rec = { + id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), + taskHash: this._hashTask(task), + features, + predictedDuration: p.duration, + predictedTurns: p.toolTurns, + confidence: p.confidence, + source: source || 'loop', + ts: Date.now() + }; + this._pending.push(rec); + if (this._pending.length > 50) this._pending = this._pending.slice(-50); + this._predictions++; + this._debounceSave(); + return rec; + } + + _takePending(features, opts) { + const now = Date.now(); + this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + const taskHash = opts.task ? this._hashTask(opts.task) : null; + + let idx = -1; + if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId); + if (idx < 0 && taskHash) { + for (let i = this._pending.length - 1; i >= 0; i--) { + if (this._pending[i].taskHash === taskHash) { idx = i; break; } + } + } + if (idx < 0) { + // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be + // matched by id or exact task, never by shape alone (nine equal booleans score 0.6). + let best = -1, bestScore = 0; + for (let i = 0; i < this._pending.length; i++) { + const cand = this._pending[i]; + if (cand.source !== 'cli') continue; + const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length; + if (shared === 0) continue; + const s = this._similarity(features, cand.features); + if (s > bestScore) { bestScore = s; best = i; } + } + if (bestScore >= 0.5) idx = best; + } + return idx >= 0 ? this._pending.splice(idx, 1)[0] : null; + } + // ─── Learning from Results ─── - recordResult(features, duration, toolTurns, success) { - this._dataPoints.push({ + recordResult(features, duration, toolTurns, success, opts) { + const o = opts || {}; + const pred = this._takePending(features, o); + const observed = Number(duration) || 0; + + const rec = { features, - observedDuration: duration, + observedDuration: observed, observedTurns: toolTurns, - success, + success: !!success, + paired: !!pred, timestamp: new Date().toISOString() - }); - this._accuracy.total++; - // Compare with last prediction for this feature set - const lastPrediction = [...this._dataPoints] - .reverse() - .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch); - if (lastPrediction) { - const error = Math.abs(lastPrediction.predictedDuration - duration) / duration; - if (error < 0.3) this._accuracy.correct++; + }; + + if (pred) { + const absErr = Math.abs(pred.predictedDuration - observed); + rec.predictedDuration = pred.predictedDuration; + rec.predictedTurns = pred.predictedTurns; + // capped: one 15s-vs-1s miss must not poison the mean-error readout forever + rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); + this._accuracy.total++; + this._mape += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; } - this._debounceSave(); + + this._dataPoints.push(rec); + if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500); + if (pred) this._saveNow(); else this._debounceSave(); + return rec; } // ─── Commands ─── showStatus() { const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length; - const accPct = this._accuracy.total > 0 - ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0) - : 'N/A'; + const paired = this._accuracy.total; + const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; + const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const mape = paired > 0 + ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + : ''; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); this.terminal.print(` Actual executions: ${actualExecutions}`, 'info'); this.terminal.print(` Predictions made: ${this._predictions}`, 'info'); - this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info'); + this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); + this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); + this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); + this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); } setConfig(args) { @@ -17968,6 +18053,12 @@ class PredictorModule { this._MIN_DATA_POINTS = n; return `min-data-points set to ${n}`; }, + 'pair-ttl': () => { + const n = parseInt(val); + if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)'; + this._PAIR_TTL_MS = n * 1000; + return `pair-ttl set to ${n}s`; + }, }; if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error'); @@ -17982,10 +18073,28 @@ class PredictorModule { const fs = this.terminal.fs; if (!fs) return; const data = fs.fsReadJSON('/system/metrics/predictor.json'); - if (data) { - this._dataPoints = data.dataPoints || []; - this._predictions = data.predictions || 0; - this._accuracy = data.accuracy || { correct: 0, total: 0 }; + if (!data) return; + + // Prediction-only points are noise for similarity; keep outcomes. + this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration); + this._predictions = data.predictions || 0; + + if (data.v === this._SCHEMA) { + const acc = data.accuracy || {}; + this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; + this._mape = data.mape || 0; + this._turnsErr = data.turnsErr || 0; + const cfg = data.config || {}; + if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; + if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints; + if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs; + const now = Date.now(); + this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + } else { + // v1 counted every observation in `total` but only some in `correct` → 0%. + this._accuracy = { correct: 0, total: 0 }; + this._mape = 0; + this._pending = []; } } catch (e) { console.warn('[Predictor] Load error (fresh start):', e.message); @@ -17998,9 +18107,18 @@ class PredictorModule { if (!fs) return; if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics'); fs.fsWriteJSON('/system/metrics/predictor.json', { - dataPoints: this._dataPoints.slice(-500), // keep last 500 + v: this._SCHEMA, + dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, - accuracy: this._accuracy + accuracy: this._accuracy, + mape: this._mape, + turnsErr: this._turnsErr, + config: { + confidenceThreshold: this._CONFIDENCE_THRESHOLD, + minDataPoints: this._MIN_DATA_POINTS, + pairTtlMs: this._PAIR_TTL_MS + }, + pending: this._pending.slice(-50) }); } catch (e) { console.error('[Predictor] Save error:', e.message); diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index abbe9ce..83d073a 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -820,6 +820,7 @@ buildSystemPrompt() { await new Promise(r => setTimeout(r, 50)); } this._chatLock = true; + const taskStart = Date.now(); // predictor durations exclude chat-lock wait this.terminal.markDirty(); // Auto-route context based on intent try { @@ -910,6 +911,16 @@ buildSystemPrompt() { } } catch (e) { /* world model not available */ } + // v4.6.56: register a prediction for THIS turn so recordResult can score it by id. + let turnPredictionId = null; + try { + const predictor = this.terminal.moduleManager.modules['PredictorModule']; + if (predictor && typeof predictor.notePrediction === 'function') { + const tp = predictor.notePrediction(userMessage, 'loop'); + turnPredictionId = tp && tp.id ? tp.id : null; + } + } catch (e) { /* predictor is best-effort */ } + const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main'; try { loopResult = await this.agenticLoop(messages, opts); @@ -937,8 +948,9 @@ buildSystemPrompt() { const predictor = this.terminal.moduleManager.modules['PredictorModule']; if (predictor && typeof predictor.recordResult === 'function') { const features = predictor._extractFeatures(userMessage); - const duration = (Date.now() - lockStart) / 1000; - predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled); + const duration = (Date.now() - taskStart) / 1000; + predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled, + { task: userMessage, predictionId: turnPredictionId }); } } catch (e) { /* predictor learning is best-effort */ } // Record task in memory graph for direct AI conversations (not just orchestrator) @@ -3020,16 +3032,26 @@ buildSystemPrompt() { if (!taskDesc) return JSON.stringify({ error: 'task parameter required' }); var pFeatures = predictor._extractFeatures(taskDesc); var pSimilar = predictor._findSimilar(pFeatures); - var pResult = predictor._computePrediction(pSimilar, pFeatures); + // v4.6.56: track the prediction so the outcome can score it. + var pTracked; + if (typeof predictor.notePrediction === 'function') { + pTracked = predictor.notePrediction(taskDesc, 'tool'); + } else { + var pFallback = predictor._computePrediction(pSimilar, pFeatures); + pTracked = { id: null, predictedDuration: pFallback.duration, + predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence }; + } return JSON.stringify({ task: taskDesc.substring(0, 80), - estimatedDuration: pResult.duration + 's', - estimatedToolTurns: pResult.toolTurns, - confidence: Math.round(pResult.confidence * 100) + '%', - similarTasks: pResult.similarCount, - recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30 - ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's' - : 'No special action needed' + predictionId: pTracked.id || null, + estimatedDuration: pTracked.predictedDuration + 's', + estimatedToolTurns: pTracked.predictedTurns, + confidence: Math.round(pTracked.confidence * 100) + '%', + similarTasks: pSimilar.length, + recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30 + ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's' + : 'No special action needed', + tracked: !!pTracked.id }); } case 'world_model_query': { @@ -4620,9 +4642,10 @@ buildSystemPrompt() { if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length; } catch (e) {} var predictor = mm ? mm.modules['PredictorModule'] : null; - var predictorAcc = 'N/A'; + var predictorAcc = 'n/a (no paired runs)'; if (predictor && predictor._accuracy && predictor._accuracy.total > 0) { - predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%'; + predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' + + predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)'; } data.health = { modules: data.modules.loaded + '/' + data.modules.total, diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index ff908cd..8b4f606 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -4,7 +4,7 @@ // Core module bundle loaded synchronously in main.html. // This loader only handles version cache-busting, splash, terminal init, and optional ext modules. - const LATEST_VER = 'v4.6.55'; + const LATEST_VER = 'v4.6.56'; const CACHE = '?v=' + LATEST_VER; const stored = localStorage.getItem('webx11_cache_ver'); if (stored !== LATEST_VER) { diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js index 3d868ae..cb1209e 100644 --- a/modules/core/module-predictor.js +++ b/modules/core/module-predictor.js @@ -1,23 +1,30 @@ // module-predictor.js — Predictive resource allocation engine -// Analyzes historical telemetry + memory graph data to predict task duration/turns +// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a +// real outcome. v1 counted every observed run in the denominator while incrementing the +// numerator only when an old `predict` record happened to match, so the dashboard read 0%. // Commands: predict, predictor-status, predictor-config class PredictorModule { constructor() { this.commands = {}; this.terminal = null; - this._dataPoints = []; + this._dataPoints = []; // observed runs — the training set for _findSimilar + this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; - this._accuracy = { correct: 0, total: 0 }; + this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only + this._mape = 0; // sum of capped |predicted-observed|/observed + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; + this._PAIR_TTL_MS = 15 * 60 * 1000; + this._SCHEMA = 2; this._loaded = false; } getHelpInfo() { return { 'predict': 'predict <task description> — Predict task duration, tool turns, and confidence', - 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count', - 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points', + 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points', + 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl', }; } @@ -39,33 +46,14 @@ class PredictorModule { async predict(task) { if (!task) return this.terminal.print('Usage: predict <task description>', 'error'); - const startTime = performance.now(); - - // Extract features const features = this._extractFeatures(task); - - // Find similar historical tasks const similar = this._findSimilar(features); - - // Generate prediction const prediction = this._computePrediction(similar, features); - // Record this prediction for accuracy tracking - const duration = ((performance.now() - startTime) / 1000).toFixed(2); - - this._predictions++; - this._dataPoints.push({ - features, - predictedDuration: prediction.duration, - predictedTurns: prediction.toolTurns, - confidence: prediction.confidence, - similarCount: similar.length, - timestamp: new Date().toISOString() - }); - - this._debounceSave(); + // Register it as pending so the next matching run can be scored against it. + const pending = this.notePrediction(task, 'cli'); + pending.similarCount = similar.length; - // Display result this.terminal.print('📈 Task Prediction', 'header'); this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info'); @@ -81,23 +69,22 @@ class PredictorModule { } else { this.terminal.print(' ❓ No historical data to base prediction on.', 'warning'); this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info'); - // Provide hardcoded defaults this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info'); } - // If got memory graph, offer recommendations if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) { this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info'); } + this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info'); return prediction; } _extractFeatures(task) { - const lower = task.toLowerCase(); + const lower = String(task || '').toLowerCase(); return { - length: task.length, - wordCount: task.split(/\s+/).length, + length: String(task || '').length, + wordCount: String(task || '').split(/\s+/).length, hasSearch: /\b(search|find|look)\b/.test(lower), hasWrite: /\b(write|create|save|edit)\b/.test(lower), hasDelete: /\b(delete|remove|rm)\b/.test(lower), @@ -107,42 +94,33 @@ class PredictorModule { hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower), hasModule: /\b(module|edit_module|view_module)\b/.test(lower), hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower), - complexity: Math.min(task.length / 50, 10), // 0-10 scale + complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10) }; } + // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2). + _similarity(a, b) { + if (!a || !b) return 0; + const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead', + 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync']; + const matches = keys.filter(k => a[k] === b[k]).length / keys.length; + const kw = a.keywords || []; + const overlap = kw.filter(k => (b.keywords || []).includes(k)).length; + const kScore = overlap / Math.max(kw.length, 1); + return matches * 0.6 + kScore * 0.4; + } + _findSimilar(features) { return this._dataPoints.filter(dp => { - // Skip predictions (only use actual executions with observed results) - if (!dp.observedDuration) return false; - const f = dp.features; - if (!f) return false; - // Feature similarity scoring - let score = 0; - const matches = [ - features.hasSearch === f.hasSearch, - features.hasWrite === f.hasWrite, - features.hasDelete === f.hasDelete, - features.hasInstall === f.hasInstall, - features.hasRead === f.hasRead, - features.hasWeb === f.hasWeb, - features.hasOrchestrate === f.hasOrchestrate, - features.hasModule === f.hasModule, - features.hasSync === f.hasSync, - ]; - score = matches.filter(Boolean).length / matches.length; - // Keyword overlap bonus - const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length; - const kScore = kOverlap / Math.max(features.keywords.length, 1); - score = score * 0.6 + kScore * 0.4; - return score > 0.3; + if (!dp.observedDuration) return false; // outcomes only, never predictions + if (!dp.features) return false; + return this._similarity(features, dp.features) > 0.3; }); } _computePrediction(similar, features) { if (similar.length === 0) { - // Hardcoded defaults let baseDuration = 15; let baseTurns = 3; if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; } @@ -160,9 +138,10 @@ class PredictorModule { const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length; const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length; - const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1; + // v2: the documented knob now actually scales confidence (v1 hardcoded 20). + const minPts = Math.max(1, this._MIN_DATA_POINTS || 5); + const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; - // Apply task complexity multiplier const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); @@ -174,43 +153,126 @@ class PredictorModule { }; } + // ─── Prediction ↔ Outcome pairing (v2) ─── + + // Task text never leaves the module: only a stable hash is stored/serialised. + _hashTask(t) { + const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = (h * 0x01000193) >>> 0; + } + return h.toString(16); + } + + // Register a prediction to be scored later. Returns the pending record (with .id). + notePrediction(task, source) { + const features = this._extractFeatures(task || ''); + const p = this._computePrediction(this._findSimilar(features), features); + const rec = { + id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), + taskHash: this._hashTask(task), + features, + predictedDuration: p.duration, + predictedTurns: p.toolTurns, + confidence: p.confidence, + source: source || 'loop', + ts: Date.now() + }; + this._pending.push(rec); + if (this._pending.length > 50) this._pending = this._pending.slice(-50); + this._predictions++; + this._debounceSave(); + return rec; + } + + _takePending(features, opts) { + const now = Date.now(); + this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + const taskHash = opts.task ? this._hashTask(opts.task) : null; + + let idx = -1; + if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId); + if (idx < 0 && taskHash) { + for (let i = this._pending.length - 1; i >= 0; i--) { + if (this._pending[i].taskHash === taskHash) { idx = i; break; } + } + } + if (idx < 0) { + // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be + // matched by id or exact task, never by shape alone (nine equal booleans score 0.6). + let best = -1, bestScore = 0; + for (let i = 0; i < this._pending.length; i++) { + const cand = this._pending[i]; + if (cand.source !== 'cli') continue; + const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length; + if (shared === 0) continue; + const s = this._similarity(features, cand.features); + if (s > bestScore) { bestScore = s; best = i; } + } + if (bestScore >= 0.5) idx = best; + } + return idx >= 0 ? this._pending.splice(idx, 1)[0] : null; + } + // ─── Learning from Results ─── - recordResult(features, duration, toolTurns, success) { - this._dataPoints.push({ + recordResult(features, duration, toolTurns, success, opts) { + const o = opts || {}; + const pred = this._takePending(features, o); + const observed = Number(duration) || 0; + + const rec = { features, - observedDuration: duration, + observedDuration: observed, observedTurns: toolTurns, - success, + success: !!success, + paired: !!pred, timestamp: new Date().toISOString() - }); - this._accuracy.total++; - // Compare with last prediction for this feature set - const lastPrediction = [...this._dataPoints] - .reverse() - .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch); - if (lastPrediction) { - const error = Math.abs(lastPrediction.predictedDuration - duration) / duration; - if (error < 0.3) this._accuracy.correct++; + }; + + if (pred) { + const absErr = Math.abs(pred.predictedDuration - observed); + rec.predictedDuration = pred.predictedDuration; + rec.predictedTurns = pred.predictedTurns; + // capped: one 15s-vs-1s miss must not poison the mean-error readout forever + rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); + this._accuracy.total++; + this._mape += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; } - this._debounceSave(); + + this._dataPoints.push(rec); + if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500); + if (pred) this._saveNow(); else this._debounceSave(); + return rec; } // ─── Commands ─── showStatus() { const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length; - const accPct = this._accuracy.total > 0 - ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0) - : 'N/A'; + const paired = this._accuracy.total; + const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; + const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const mape = paired > 0 + ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + : ''; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); this.terminal.print(` Actual executions: ${actualExecutions}`, 'info'); this.terminal.print(` Predictions made: ${this._predictions}`, 'info'); - this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info'); + this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); + this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); + this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); + this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); } setConfig(args) { @@ -231,6 +293,12 @@ class PredictorModule { this._MIN_DATA_POINTS = n; return `min-data-points set to ${n}`; }, + 'pair-ttl': () => { + const n = parseInt(val); + if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)'; + this._PAIR_TTL_MS = n * 1000; + return `pair-ttl set to ${n}s`; + }, }; if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error'); @@ -245,10 +313,28 @@ class PredictorModule { const fs = this.terminal.fs; if (!fs) return; const data = fs.fsReadJSON('/system/metrics/predictor.json'); - if (data) { - this._dataPoints = data.dataPoints || []; - this._predictions = data.predictions || 0; - this._accuracy = data.accuracy || { correct: 0, total: 0 }; + if (!data) return; + + // Prediction-only points are noise for similarity; keep outcomes. + this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration); + this._predictions = data.predictions || 0; + + if (data.v === this._SCHEMA) { + const acc = data.accuracy || {}; + this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; + this._mape = data.mape || 0; + this._turnsErr = data.turnsErr || 0; + const cfg = data.config || {}; + if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; + if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints; + if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs; + const now = Date.now(); + this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + } else { + // v1 counted every observation in `total` but only some in `correct` → 0%. + this._accuracy = { correct: 0, total: 0 }; + this._mape = 0; + this._pending = []; } } catch (e) { console.warn('[Predictor] Load error (fresh start):', e.message); @@ -261,9 +347,18 @@ class PredictorModule { if (!fs) return; if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics'); fs.fsWriteJSON('/system/metrics/predictor.json', { - dataPoints: this._dataPoints.slice(-500), // keep last 500 + v: this._SCHEMA, + dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, - accuracy: this._accuracy + accuracy: this._accuracy, + mape: this._mape, + turnsErr: this._turnsErr, + config: { + confidenceThreshold: this._CONFIDENCE_THRESHOLD, + minDataPoints: this._MIN_DATA_POINTS, + pairTtlMs: this._PAIR_TTL_MS + }, + pending: this._pending.slice(-50) }); } catch (e) { console.error('[Predictor] Save error:', e.message); @@ -279,4 +374,4 @@ class PredictorModule { // Required for module system if (typeof window !== 'undefined') { window.PredictorModule = PredictorModule; -}
\ No newline at end of file +} diff --git a/modules/test/test-predictor-accuracy.js b/modules/test/test-predictor-accuracy.js new file mode 100644 index 0000000..1a79167 --- /dev/null +++ b/modules/test/test-predictor-accuracy.js @@ -0,0 +1,209 @@ +// modules/test/test-predictor-accuracy.js — run: node modules/test/test-predictor-accuracy.js +// Proves the predictor's accuracy metric measures PAIRED predictions only. +// RED on v1 code: unpaired runs inflate the denominator, so accuracy is always 0%. +const fs = require('fs'), path = require('path'); +const src = fs.readFileSync(path.join(__dirname, '../core/module-predictor.js'), 'utf8'); +const PredictorModule = new Function(src + '\nreturn PredictorModule;')(); + +function memFs() { + const files = {}; + const dirs = new Set(['/']); + return { + readFile: p => (p in files ? files[p] : null), + writeFile: (p, c) => { files[p] = c; return true; }, + fsReadJSON: p => { try { return JSON.parse(files[p]); } catch (e) { return null; } }, + fsWriteJSON: (p, o) => { files[p] = JSON.stringify(o); return true; }, + readDir: p => (dirs.has(p) ? [] : null), + mkdirRecursive: p => { dirs.add(p); return true; }, + mkdir: p => { dirs.add(p); return true; }, + }; +} + +function mk(persist) { + const out = []; + const term = { print: m => out.push(String(m)), fs: memFs() }; + const p = new PredictorModule(); + p.terminal = term; + if (!persist) { p._saveNow = () => {}; p._debounceSave = () => {}; } // isolate logic from timers + return { p, out, term }; +} + +let fails = 0, passes = 0; +function ok(name, cond, detail) { + if (cond) { passes++; return; } + fails++; + console.log('FAIL ' + name + (detail ? ' — ' + detail : '')); +} + +// A. runs with no prediction must not enter the denominator +{ + const { p } = mk(); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); + p.recordResult(p._extractFeatures('write a report'), 40, 3, true); + ok('A1 total stays 0 without predictions', p._accuracy.total === 0, 'total=' + p._accuracy.total); + ok('A2 observed runs are still recorded', p._dataPoints.filter(d => d.observedDuration).length === 2); +} + +// B. a prediction paired with an on-time outcome counts as correct +{ + const { p } = mk(); + const pred = p.notePrediction('write a report file', 'test'); + ok('B0 notePrediction returns a pending prediction', !!pred && typeof pred.id === 'string'); + p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 1.1, 4, true, { task: 'write a report file' }); + ok('B1 paired once', p._accuracy.total === 1, 'total=' + p._accuracy.total); + ok('B2 on-time prediction scored correct', p._accuracy.correct === 1, 'correct=' + p._accuracy.correct); + ok('B3 pending consumed', p._pending.length === 0, 'pending=' + p._pending.length); +} + +// C. an outcome >30% off is paired but wrong +{ + const { p } = mk(); + const pred = p.notePrediction('write a report file', 'test'); + p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 3 + 5, 4, true, { task: 'write a report file' }); + ok('C1 paired', p._accuracy.total === 1); + ok('C2 not correct', p._accuracy.correct === 0); +} + +// D. an unrelated run must not consume the pending prediction +{ + const { p } = mk(); + p.notePrediction('search the web for ollama docs', 'test'); + p.recordResult(p._extractFeatures('delete the tmp folder'), 9, 1, true, { task: 'delete the tmp folder' }); + ok('D1 unrelated run unpaired', p._accuracy.total === 0, 'total=' + p._accuracy.total); + ok('D2 prediction still pending', p._pending.length === 1, 'pending=' + p._pending.length); +} + +// E. expired predictions are dropped, not paired +{ + const { p } = mk(); + const pred = p.notePrediction('read the logs', 'test'); + pred.ts = Date.now() - (p._PAIR_TTL_MS + 1000); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true, { task: 'read the logs' }); + ok('E1 expired prediction not paired', p._accuracy.total === 0); +} + +// F. the status line never renders "N/A%" +{ + const { p, out } = mk(); + p.showStatus(); + const line = out.find(l => l.includes('Accuracy')) || ''; + ok('F1 empty state says n/a', /\bn\/a\b/i.test(line) && !line.includes('N/A%'), line.trim()); + ok('F2 paired count is shown', /paired/i.test(line), line.trim()); +} + +// G. _MIN_DATA_POINTS actually shapes confidence (it is a dead knob in v1) +{ + const { p } = mk(); + for (let i = 0; i < 3; i++) p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); + const f = p._extractFeatures('read the logs'); + p._MIN_DATA_POINTS = 10; + const a = p._computePrediction(p._findSimilar(f), f).confidence; + p._MIN_DATA_POINTS = 3; + const b = p._computePrediction(p._findSimilar(f), f).confidence; + ok('G1 lowering min-data-points raises confidence', b > a, a.toFixed(3) + ' -> ' + b.toFixed(3)); +} + +// H. v1 payload migration: junk accuracy discarded, observed points kept +{ + const { p, term } = mk(); + term.fs.fsWriteJSON('/system/metrics/predictor.json', { + dataPoints: [{ features: { hasRead: true }, observedDuration: 30, observedTurns: 3 }, + { features: { hasRead: true }, predictedDuration: 30 }], + predictions: 7, + accuracy: { correct: 0, total: 6 } + }); + p._loadFromVFS(); + ok('H1 v1 accuracy discarded', p._accuracy.total === 0 && p._accuracy.correct === 0, JSON.stringify(p._accuracy)); + ok('H2 observed points kept, prediction-only points dropped', p._dataPoints.length === 1, 'n=' + p._dataPoints.length); + ok('H3 predictions counter kept', p._predictions === 7, 'n=' + p._predictions); +} + +// I. save → load round-trip keeps paired accuracy; stale pending is dropped +{ + const { p, term } = mk(true); + const pred = p.notePrediction('read the logs', 'test'); + p.recordResult(p._extractFeatures('read the logs'), pred.predictedDuration, 2, true, { task: 'read the logs' }); + p._saveNow(); + const q = new PredictorModule(); q.terminal = term; q._loadFromVFS(); + ok('I1 accuracy survives reload', q._accuracy.total === 1 && q._accuracy.correct === 1, JSON.stringify(q._accuracy)); + const stale = p.notePrediction('read the logs again', 'test'); + stale.ts = Date.now() - (p._PAIR_TTL_MS + 60000); // expired while pending + p._saveNow(); + const s = new PredictorModule(); s.terminal = term; s._loadFromVFS(); + ok('I2 expired pending dropped on load', s._pending.length === 0, 'pending=' + s._pending.length); +} + + +// J. 3 s absolute floor: a small miss on a short task still counts +{ + const { p } = mk(); + p.notePrediction('read a tiny file', 'test'); + p._pending[0].predictedDuration = 5; // integer seconds vs fractional observed + p.recordResult(p._extractFeatures('read a tiny file'), 3.8, 1, true, { task: 'read a tiny file' }); + ok('J1 1.2 s miss on a 3.8 s task is inside the 3 s floor', p._accuracy.correct === 1, JSON.stringify(p._accuracy)); +} + +// K. a wild miss is capped, so mean error stays readable +{ + const { p } = mk(); + p.notePrediction('read the logs', 'test'); + p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' }); + const rec = p._dataPoints[p._dataPoints.length - 1]; + ok('K1 errorPct capped at 500%', rec.errorPct <= 5, 'errorPct=' + rec.errorPct); +} + +// L. a tool/loop prediction is never consumed by the similarity fallback +{ + const { p } = mk(); + p.notePrediction('write a file report', 'tool'); + p.recordResult(p._extractFeatures('write a file report somewhere else'), 40, 3, true); // no task, no id + ok('L1 shape-alike run does not consume a tool prediction', p._accuracy.total === 0, 'total=' + p._accuracy.total); +} + +// M. the cli fallback needs at least one shared keyword +{ + const { p } = mk(); + p.notePrediction('alpha bravo charlie', 'cli'); + p.recordResult(p._extractFeatures('delta echo foxtrot'), 30, 3, true); + ok('M1 zero keyword overlap is not a match', p._accuracy.total === 0, 'total=' + p._accuracy.total); +} + +// N. tuning survives a reload +{ + const { p, term } = mk(true); + p._MIN_DATA_POINTS = 9; + p._PAIR_TTL_MS = 600000; + p._saveNow(); + const q = new PredictorModule(); q.terminal = term; q._loadFromVFS(); + ok('N1 config persisted', q._MIN_DATA_POINTS === 9 && q._PAIR_TTL_MS === 600000, + q._MIN_DATA_POINTS + '/' + q._PAIR_TTL_MS); +} + +// O. "unpaired" counts scored-and-missed runs, not legacy observations +{ + const { p, out, term } = mk(); + term.fs.fsWriteJSON('/system/metrics/predictor.json', { + dataPoints: [{ features: { hasRead: true }, observedDuration: 30 }], // v1 point, no `paired` flag + accuracy: { correct: 0, total: 6 } + }); + p._loadFromVFS(); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); // unpaired v2 run + out.length = 0; p.showStatus(); + const line = out.find(l => l.includes('Unpaired runs')) || ''; + ok('O1 only paired:false records count as unpaired', /Unpaired runs: 1\b/.test(line), line.trim()); +} + + +// P. the id threaded by the AI loop wins over text/similarity guessing +{ + const { p } = mk(); + p.notePrediction('something else entirely', 'tool'); // decoy, pushed first + const loopPred = p.notePrediction('read the logs', 'loop'); + p.recordResult(p._extractFeatures('read the logs'), loopPred.predictedDuration, 2, true, + { task: 'read the logs', predictionId: loopPred.id }); + ok('P1 the threaded id is the one consumed', p._accuracy.total === 1 && p._pending.length === 1, + 'total=' + p._accuracy.total + ' pending=' + p._pending.length); +} + +console.log(passes + '/' + (passes + fails) + ' PASS'); +process.exit(fails ? 1 : 0); |