aboutsummaryrefslogtreecommitdiff
path: root/modules/core/module-aiassist.js
diff options
context:
space:
mode:
authorAgando <roduccomm@gmail.com>2026-10-23 17:38:10 +0200
committerAgando <roduccomm@gmail.com>2026-10-23 17:38:10 +0200
commit7f4732e5089b1430e736bc6a8b8578a674f31f36 (patch)
treec51e8da70e8468d234a60510d678fcfb4a797eab /modules/core/module-aiassist.js
parent0c01cf48e83eecca953d0c6cc2dc949aed552f2f (diff)
fix(newos4): measure predictor accuracy over paired predictions (v4.6.56)
The accuracy readout was structurally 0%: module-predictor.js:187 incremented the denominator on every observed run while the numerator only moved when a stored prediction existed, and predictions were stored only by the manual `predict` command (the AI tool predict_task computed one and discarded it). Replaying the AI loop's own call path with 6 tasks printed "Predictions made: 0, Accuracy: 0/6". - pending predictions with a 15m TTL, consumed by threaded id -> task hash -> cli-only similarity >= 0.5; unpaired runs never enter the denominator - duration scored within 30% or 3s (integer predictions vs fractional observations) - errorPct capped at 500% so one miss cannot poison the mean error - schema v2: v1 correct/total counters dropped on load, observed runs and the prediction counter kept; task text stored as an FNV-1a hash only - tuning (confidence-threshold, min-data-points, pair-ttl) persists - min-data-points now scales confidence (was declared, printed, settable, unread) - AI loop registers a prediction per turn and threads its id; predict_task returns predictionId; durations exclude chat-lock queue wait - status/dashboard say "n/a" instead of "N/A%", and carry paired count, mean error, turns error, pending and unpaired counts - test: modules/test/test-predictor-accuracy.js (26 assertions) Live on main.html v4.6.56: fresh session -> "Accuracy: n/a (0/0 paired)"; one paired run -> "100% (1/1 paired) - mean error 5% - turns +-0.0".
Diffstat (limited to 'modules/core/module-aiassist.js')
-rw-r--r--modules/core/module-aiassist.js47
1 files changed, 35 insertions, 12 deletions
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index abbe9ce..83d073a 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -820,6 +820,7 @@ buildSystemPrompt() {
await new Promise(r => setTimeout(r, 50));
}
this._chatLock = true;
+ const taskStart = Date.now(); // predictor durations exclude chat-lock wait
this.terminal.markDirty();
// Auto-route context based on intent
try {
@@ -910,6 +911,16 @@ buildSystemPrompt() {
}
} catch (e) { /* world model not available */ }
+ // v4.6.56: register a prediction for THIS turn so recordResult can score it by id.
+ let turnPredictionId = null;
+ try {
+ const predictor = this.terminal.moduleManager.modules['PredictorModule'];
+ if (predictor && typeof predictor.notePrediction === 'function') {
+ const tp = predictor.notePrediction(userMessage, 'loop');
+ turnPredictionId = tp && tp.id ? tp.id : null;
+ }
+ } catch (e) { /* predictor is best-effort */ }
+
const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main';
try {
loopResult = await this.agenticLoop(messages, opts);
@@ -937,8 +948,9 @@ buildSystemPrompt() {
const predictor = this.terminal.moduleManager.modules['PredictorModule'];
if (predictor && typeof predictor.recordResult === 'function') {
const features = predictor._extractFeatures(userMessage);
- const duration = (Date.now() - lockStart) / 1000;
- predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled);
+ const duration = (Date.now() - taskStart) / 1000;
+ predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled,
+ { task: userMessage, predictionId: turnPredictionId });
}
} catch (e) { /* predictor learning is best-effort */ }
// Record task in memory graph for direct AI conversations (not just orchestrator)
@@ -3020,16 +3032,26 @@ buildSystemPrompt() {
if (!taskDesc) return JSON.stringify({ error: 'task parameter required' });
var pFeatures = predictor._extractFeatures(taskDesc);
var pSimilar = predictor._findSimilar(pFeatures);
- var pResult = predictor._computePrediction(pSimilar, pFeatures);
+ // v4.6.56: track the prediction so the outcome can score it.
+ var pTracked;
+ if (typeof predictor.notePrediction === 'function') {
+ pTracked = predictor.notePrediction(taskDesc, 'tool');
+ } else {
+ var pFallback = predictor._computePrediction(pSimilar, pFeatures);
+ pTracked = { id: null, predictedDuration: pFallback.duration,
+ predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence };
+ }
return JSON.stringify({
task: taskDesc.substring(0, 80),
- estimatedDuration: pResult.duration + 's',
- estimatedToolTurns: pResult.toolTurns,
- confidence: Math.round(pResult.confidence * 100) + '%',
- similarTasks: pResult.similarCount,
- recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30
- ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's'
- : 'No special action needed'
+ predictionId: pTracked.id || null,
+ estimatedDuration: pTracked.predictedDuration + 's',
+ estimatedToolTurns: pTracked.predictedTurns,
+ confidence: Math.round(pTracked.confidence * 100) + '%',
+ similarTasks: pSimilar.length,
+ recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30
+ ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's'
+ : 'No special action needed',
+ tracked: !!pTracked.id
});
}
case 'world_model_query': {
@@ -4620,9 +4642,10 @@ buildSystemPrompt() {
if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length;
} catch (e) {}
var predictor = mm ? mm.modules['PredictorModule'] : null;
- var predictorAcc = 'N/A';
+ var predictorAcc = 'n/a (no paired runs)';
if (predictor && predictor._accuracy && predictor._accuracy.total > 0) {
- predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%';
+ predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' +
+ predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)';
}
data.health = {
modules: data.modules.loaded + '/' + data.modules.total,