From 7f4732e5089b1430e736bc6a8b8578a674f31f36 Mon Sep 17 00:00:00 2001 From: Agando Date: Fri, 23 Oct 2026 17:38:10 +0200 Subject: fix(newos4): measure predictor accuracy over paired predictions (v4.6.56) The accuracy readout was structurally 0%: module-predictor.js:187 incremented the denominator on every observed run while the numerator only moved when a stored prediction existed, and predictions were stored only by the manual `predict` command (the AI tool predict_task computed one and discarded it). Replaying the AI loop's own call path with 6 tasks printed "Predictions made: 0, Accuracy: 0/6". - pending predictions with a 15m TTL, consumed by threaded id -> task hash -> cli-only similarity >= 0.5; unpaired runs never enter the denominator - duration scored within 30% or 3s (integer predictions vs fractional observations) - errorPct capped at 500% so one miss cannot poison the mean error - schema v2: v1 correct/total counters dropped on load, observed runs and the prediction counter kept; task text stored as an FNV-1a hash only - tuning (confidence-threshold, min-data-points, pair-ttl) persists - min-data-points now scales confidence (was declared, printed, settable, unread) - AI loop registers a prediction per turn and threads its id; predict_task returns predictionId; durations exclude chat-lock queue wait - status/dashboard say "n/a" instead of "N/A%", and carry paired count, mean error, turns error, pending and unpaired counts - test: modules/test/test-predictor-accuracy.js (26 assertions) Live on main.html v4.6.56: fresh session -> "Accuracy: n/a (0/0 paired)"; one paired run -> "100% (1/1 paired) - mean error 5% - turns +-0.0". --- modules/core/core-bundle.js | 308 ++++++++++++++++++++++++++++++-------------- 1 file changed, 213 insertions(+), 95 deletions(-) (limited to 'modules/core/core-bundle.js') diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index 67747bf..9ac94ed 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -9041,6 +9041,7 @@ buildSystemPrompt() { await new Promise(r => setTimeout(r, 50)); } this._chatLock = true; + const taskStart = Date.now(); // predictor durations exclude chat-lock wait this.terminal.markDirty(); // Auto-route context based on intent try { @@ -9131,6 +9132,16 @@ buildSystemPrompt() { } } catch (e) { /* world model not available */ } + // v4.6.56: register a prediction for THIS turn so recordResult can score it by id. + let turnPredictionId = null; + try { + const predictor = this.terminal.moduleManager.modules['PredictorModule']; + if (predictor && typeof predictor.notePrediction === 'function') { + const tp = predictor.notePrediction(userMessage, 'loop'); + turnPredictionId = tp && tp.id ? tp.id : null; + } + } catch (e) { /* predictor is best-effort */ } + const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main'; try { loopResult = await this.agenticLoop(messages, opts); @@ -9158,8 +9169,9 @@ buildSystemPrompt() { const predictor = this.terminal.moduleManager.modules['PredictorModule']; if (predictor && typeof predictor.recordResult === 'function') { const features = predictor._extractFeatures(userMessage); - const duration = (Date.now() - lockStart) / 1000; - predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled); + const duration = (Date.now() - taskStart) / 1000; + predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled, + { task: userMessage, predictionId: turnPredictionId }); } } catch (e) { /* predictor learning is best-effort */ } // Record task in memory graph for direct AI conversations (not just orchestrator) @@ -11241,16 +11253,26 @@ buildSystemPrompt() { if (!taskDesc) return JSON.stringify({ error: 'task parameter required' }); var pFeatures = predictor._extractFeatures(taskDesc); var pSimilar = predictor._findSimilar(pFeatures); - var pResult = predictor._computePrediction(pSimilar, pFeatures); + // v4.6.56: track the prediction so the outcome can score it. + var pTracked; + if (typeof predictor.notePrediction === 'function') { + pTracked = predictor.notePrediction(taskDesc, 'tool'); + } else { + var pFallback = predictor._computePrediction(pSimilar, pFeatures); + pTracked = { id: null, predictedDuration: pFallback.duration, + predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence }; + } return JSON.stringify({ task: taskDesc.substring(0, 80), - estimatedDuration: pResult.duration + 's', - estimatedToolTurns: pResult.toolTurns, - confidence: Math.round(pResult.confidence * 100) + '%', - similarTasks: pResult.similarCount, - recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30 - ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's' - : 'No special action needed' + predictionId: pTracked.id || null, + estimatedDuration: pTracked.predictedDuration + 's', + estimatedToolTurns: pTracked.predictedTurns, + confidence: Math.round(pTracked.confidence * 100) + '%', + similarTasks: pSimilar.length, + recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30 + ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's' + : 'No special action needed', + tracked: !!pTracked.id }); } case 'world_model_query': { @@ -12841,9 +12863,10 @@ buildSystemPrompt() { if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length; } catch (e) {} var predictor = mm ? mm.modules['PredictorModule'] : null; - var predictorAcc = 'N/A'; + var predictorAcc = 'n/a (no paired runs)'; if (predictor && predictor._accuracy && predictor._accuracy.total > 0) { - predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%'; + predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' + + predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)'; } data.health = { modules: data.modules.loaded + '/' + data.modules.total, @@ -17736,25 +17759,32 @@ if (typeof window !== 'undefined') { /* === module-predictor.js === */ // module-predictor.js — Predictive resource allocation engine -// Analyzes historical telemetry + memory graph data to predict task duration/turns +// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a +// real outcome. v1 counted every observed run in the denominator while incrementing the +// numerator only when an old `predict` record happened to match, so the dashboard read 0%. // Commands: predict, predictor-status, predictor-config class PredictorModule { constructor() { this.commands = {}; this.terminal = null; - this._dataPoints = []; + this._dataPoints = []; // observed runs — the training set for _findSimilar + this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; - this._accuracy = { correct: 0, total: 0 }; + this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only + this._mape = 0; // sum of capped |predicted-observed|/observed + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; + this._PAIR_TTL_MS = 15 * 60 * 1000; + this._SCHEMA = 2; this._loaded = false; } getHelpInfo() { return { 'predict': 'predict — Predict task duration, tool turns, and confidence', - 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count', - 'predictor-config': 'predictor-config — Tune: confidence-threshold, min-data-points', + 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points', + 'predictor-config': 'predictor-config — Tune: confidence-threshold, min-data-points, pair-ttl', }; } @@ -17776,33 +17806,14 @@ class PredictorModule { async predict(task) { if (!task) return this.terminal.print('Usage: predict ', 'error'); - const startTime = performance.now(); - - // Extract features const features = this._extractFeatures(task); - - // Find similar historical tasks const similar = this._findSimilar(features); - - // Generate prediction const prediction = this._computePrediction(similar, features); - // Record this prediction for accuracy tracking - const duration = ((performance.now() - startTime) / 1000).toFixed(2); + // Register it as pending so the next matching run can be scored against it. + const pending = this.notePrediction(task, 'cli'); + pending.similarCount = similar.length; - this._predictions++; - this._dataPoints.push({ - features, - predictedDuration: prediction.duration, - predictedTurns: prediction.toolTurns, - confidence: prediction.confidence, - similarCount: similar.length, - timestamp: new Date().toISOString() - }); - - this._debounceSave(); - - // Display result this.terminal.print('📈 Task Prediction', 'header'); this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info'); @@ -17818,23 +17829,22 @@ class PredictorModule { } else { this.terminal.print(' ❓ No historical data to base prediction on.', 'warning'); this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info'); - // Provide hardcoded defaults this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info'); } - // If got memory graph, offer recommendations if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) { this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info'); } + this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info'); return prediction; } _extractFeatures(task) { - const lower = task.toLowerCase(); + const lower = String(task || '').toLowerCase(); return { - length: task.length, - wordCount: task.split(/\s+/).length, + length: String(task || '').length, + wordCount: String(task || '').split(/\s+/).length, hasSearch: /\b(search|find|look)\b/.test(lower), hasWrite: /\b(write|create|save|edit)\b/.test(lower), hasDelete: /\b(delete|remove|rm)\b/.test(lower), @@ -17844,42 +17854,33 @@ class PredictorModule { hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower), hasModule: /\b(module|edit_module|view_module)\b/.test(lower), hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower), - complexity: Math.min(task.length / 50, 10), // 0-10 scale + complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10) }; } + // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2). + _similarity(a, b) { + if (!a || !b) return 0; + const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead', + 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync']; + const matches = keys.filter(k => a[k] === b[k]).length / keys.length; + const kw = a.keywords || []; + const overlap = kw.filter(k => (b.keywords || []).includes(k)).length; + const kScore = overlap / Math.max(kw.length, 1); + return matches * 0.6 + kScore * 0.4; + } + _findSimilar(features) { return this._dataPoints.filter(dp => { - // Skip predictions (only use actual executions with observed results) - if (!dp.observedDuration) return false; - const f = dp.features; - if (!f) return false; - // Feature similarity scoring - let score = 0; - const matches = [ - features.hasSearch === f.hasSearch, - features.hasWrite === f.hasWrite, - features.hasDelete === f.hasDelete, - features.hasInstall === f.hasInstall, - features.hasRead === f.hasRead, - features.hasWeb === f.hasWeb, - features.hasOrchestrate === f.hasOrchestrate, - features.hasModule === f.hasModule, - features.hasSync === f.hasSync, - ]; - score = matches.filter(Boolean).length / matches.length; - // Keyword overlap bonus - const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length; - const kScore = kOverlap / Math.max(features.keywords.length, 1); - score = score * 0.6 + kScore * 0.4; - return score > 0.3; + if (!dp.observedDuration) return false; // outcomes only, never predictions + if (!dp.features) return false; + return this._similarity(features, dp.features) > 0.3; }); } _computePrediction(similar, features) { if (similar.length === 0) { - // Hardcoded defaults let baseDuration = 15; let baseTurns = 3; if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; } @@ -17897,9 +17898,10 @@ class PredictorModule { const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length; const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length; - const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1; + // v2: the documented knob now actually scales confidence (v1 hardcoded 20). + const minPts = Math.max(1, this._MIN_DATA_POINTS || 5); + const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; - // Apply task complexity multiplier const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); @@ -17911,43 +17913,126 @@ class PredictorModule { }; } + // ─── Prediction ↔ Outcome pairing (v2) ─── + + // Task text never leaves the module: only a stable hash is stored/serialised. + _hashTask(t) { + const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = (h * 0x01000193) >>> 0; + } + return h.toString(16); + } + + // Register a prediction to be scored later. Returns the pending record (with .id). + notePrediction(task, source) { + const features = this._extractFeatures(task || ''); + const p = this._computePrediction(this._findSimilar(features), features); + const rec = { + id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), + taskHash: this._hashTask(task), + features, + predictedDuration: p.duration, + predictedTurns: p.toolTurns, + confidence: p.confidence, + source: source || 'loop', + ts: Date.now() + }; + this._pending.push(rec); + if (this._pending.length > 50) this._pending = this._pending.slice(-50); + this._predictions++; + this._debounceSave(); + return rec; + } + + _takePending(features, opts) { + const now = Date.now(); + this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + const taskHash = opts.task ? this._hashTask(opts.task) : null; + + let idx = -1; + if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId); + if (idx < 0 && taskHash) { + for (let i = this._pending.length - 1; i >= 0; i--) { + if (this._pending[i].taskHash === taskHash) { idx = i; break; } + } + } + if (idx < 0) { + // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be + // matched by id or exact task, never by shape alone (nine equal booleans score 0.6). + let best = -1, bestScore = 0; + for (let i = 0; i < this._pending.length; i++) { + const cand = this._pending[i]; + if (cand.source !== 'cli') continue; + const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length; + if (shared === 0) continue; + const s = this._similarity(features, cand.features); + if (s > bestScore) { bestScore = s; best = i; } + } + if (bestScore >= 0.5) idx = best; + } + return idx >= 0 ? this._pending.splice(idx, 1)[0] : null; + } + // ─── Learning from Results ─── - recordResult(features, duration, toolTurns, success) { - this._dataPoints.push({ + recordResult(features, duration, toolTurns, success, opts) { + const o = opts || {}; + const pred = this._takePending(features, o); + const observed = Number(duration) || 0; + + const rec = { features, - observedDuration: duration, + observedDuration: observed, observedTurns: toolTurns, - success, + success: !!success, + paired: !!pred, timestamp: new Date().toISOString() - }); - this._accuracy.total++; - // Compare with last prediction for this feature set - const lastPrediction = [...this._dataPoints] - .reverse() - .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch); - if (lastPrediction) { - const error = Math.abs(lastPrediction.predictedDuration - duration) / duration; - if (error < 0.3) this._accuracy.correct++; + }; + + if (pred) { + const absErr = Math.abs(pred.predictedDuration - observed); + rec.predictedDuration = pred.predictedDuration; + rec.predictedTurns = pred.predictedTurns; + // capped: one 15s-vs-1s miss must not poison the mean-error readout forever + rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); + this._accuracy.total++; + this._mape += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; } - this._debounceSave(); + + this._dataPoints.push(rec); + if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500); + if (pred) this._saveNow(); else this._debounceSave(); + return rec; } // ─── Commands ─── showStatus() { const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length; - const accPct = this._accuracy.total > 0 - ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0) - : 'N/A'; + const paired = this._accuracy.total; + const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; + const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const mape = paired > 0 + ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + : ''; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); this.terminal.print(` Actual executions: ${actualExecutions}`, 'info'); this.terminal.print(` Predictions made: ${this._predictions}`, 'info'); - this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info'); + this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); + this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); + this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); + this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); } setConfig(args) { @@ -17968,6 +18053,12 @@ class PredictorModule { this._MIN_DATA_POINTS = n; return `min-data-points set to ${n}`; }, + 'pair-ttl': () => { + const n = parseInt(val); + if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)'; + this._PAIR_TTL_MS = n * 1000; + return `pair-ttl set to ${n}s`; + }, }; if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error'); @@ -17982,10 +18073,28 @@ class PredictorModule { const fs = this.terminal.fs; if (!fs) return; const data = fs.fsReadJSON('/system/metrics/predictor.json'); - if (data) { - this._dataPoints = data.dataPoints || []; - this._predictions = data.predictions || 0; - this._accuracy = data.accuracy || { correct: 0, total: 0 }; + if (!data) return; + + // Prediction-only points are noise for similarity; keep outcomes. + this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration); + this._predictions = data.predictions || 0; + + if (data.v === this._SCHEMA) { + const acc = data.accuracy || {}; + this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; + this._mape = data.mape || 0; + this._turnsErr = data.turnsErr || 0; + const cfg = data.config || {}; + if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; + if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints; + if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs; + const now = Date.now(); + this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + } else { + // v1 counted every observation in `total` but only some in `correct` → 0%. + this._accuracy = { correct: 0, total: 0 }; + this._mape = 0; + this._pending = []; } } catch (e) { console.warn('[Predictor] Load error (fresh start):', e.message); @@ -17998,9 +18107,18 @@ class PredictorModule { if (!fs) return; if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics'); fs.fsWriteJSON('/system/metrics/predictor.json', { - dataPoints: this._dataPoints.slice(-500), // keep last 500 + v: this._SCHEMA, + dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, - accuracy: this._accuracy + accuracy: this._accuracy, + mape: this._mape, + turnsErr: this._turnsErr, + config: { + confidenceThreshold: this._CONFIDENCE_THRESHOLD, + minDataPoints: this._MIN_DATA_POINTS, + pairTtlMs: this._PAIR_TTL_MS + }, + pending: this._pending.slice(-50) }); } catch (e) { console.error('[Predictor] Save error:', e.message); -- cgit v1.2.3