1 files changed, 209 insertions, 0 deletions
diff --git a/modules/test/test-predictor-accuracy.js b/modules/test/test-predictor-accuracy.js
new file mode 100644
index 0000000..1a79167
--- /dev/null
+++ b/modules/test/test-predictor-accuracy.js
@@ -0,0 +1,209 @@
+// modules/test/test-predictor-accuracy.js — run: node modules/test/test-predictor-accuracy.js
+// Proves the predictor's accuracy metric measures PAIRED predictions only.
+// RED on v1 code: unpaired runs inflate the denominator, so accuracy is always 0%.
+const fs = require('fs'), path = require('path');
+const src = fs.readFileSync(path.join(__dirname, '../core/module-predictor.js'), 'utf8');
+const PredictorModule = new Function(src + '\nreturn PredictorModule;')();
+
+function memFs() {
+    const files = {};
+    const dirs = new Set(['/']);
+    return {
+        readFile: p => (p in files ? files[p] : null),
+        writeFile: (p, c) => { files[p] = c; return true; },
+        fsReadJSON: p => { try { return JSON.parse(files[p]); } catch (e) { return null; } },
+        fsWriteJSON: (p, o) => { files[p] = JSON.stringify(o); return true; },
+        readDir: p => (dirs.has(p) ? [] : null),
+        mkdirRecursive: p => { dirs.add(p); return true; },
+        mkdir: p => { dirs.add(p); return true; },
+    };
+}
+
+function mk(persist) {
+    const out = [];
+    const term = { print: m => out.push(String(m)), fs: memFs() };
+    const p = new PredictorModule();
+    p.terminal = term;
+    if (!persist) { p._saveNow = () => {}; p._debounceSave = () => {}; }   // isolate logic from timers
+    return { p, out, term };
+}
+
+let fails = 0, passes = 0;
+function ok(name, cond, detail) {
+    if (cond) { passes++; return; }
+    fails++;
+    console.log('FAIL ' + name + (detail ? ' — ' + detail : ''));
+}
+
+// A. runs with no prediction must not enter the denominator
+{
+    const { p } = mk();
+    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
+    p.recordResult(p._extractFeatures('write a report'), 40, 3, true);
+    ok('A1 total stays 0 without predictions', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+    ok('A2 observed runs are still recorded', p._dataPoints.filter(d => d.observedDuration).length === 2);
+}
+
+// B. a prediction paired with an on-time outcome counts as correct
+{
+    const { p } = mk();
+    const pred = p.notePrediction('write a report file', 'test');
+    ok('B0 notePrediction returns a pending prediction', !!pred && typeof pred.id === 'string');
+    p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 1.1, 4, true, { task: 'write a report file' });
+    ok('B1 paired once', p._accuracy.total === 1, 'total=' + p._accuracy.total);
+    ok('B2 on-time prediction scored correct', p._accuracy.correct === 1, 'correct=' + p._accuracy.correct);
+    ok('B3 pending consumed', p._pending.length === 0, 'pending=' + p._pending.length);
+}
+
+// C. an outcome >30% off is paired but wrong
+{
+    const { p } = mk();
+    const pred = p.notePrediction('write a report file', 'test');
+    p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 3 + 5, 4, true, { task: 'write a report file' });
+    ok('C1 paired', p._accuracy.total === 1);
+    ok('C2 not correct', p._accuracy.correct === 0);
+}
+
+// D. an unrelated run must not consume the pending prediction
+{
+    const { p } = mk();
+    p.notePrediction('search the web for ollama docs', 'test');
+    p.recordResult(p._extractFeatures('delete the tmp folder'), 9, 1, true, { task: 'delete the tmp folder' });
+    ok('D1 unrelated run unpaired', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+    ok('D2 prediction still pending', p._pending.length === 1, 'pending=' + p._pending.length);
+}
+
+// E. expired predictions are dropped, not paired
+{
+    const { p } = mk();
+    const pred = p.notePrediction('read the logs', 'test');
+    pred.ts = Date.now() - (p._PAIR_TTL_MS + 1000);
+    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true, { task: 'read the logs' });
+    ok('E1 expired prediction not paired', p._accuracy.total === 0);
+}
+
+// F. the status line never renders "N/A%"
+{
+    const { p, out } = mk();
+    p.showStatus();
+    const line = out.find(l => l.includes('Accuracy')) || '';
+    ok('F1 empty state says n/a', /\bn\/a\b/i.test(line) && !line.includes('N/A%'), line.trim());
+    ok('F2 paired count is shown', /paired/i.test(line), line.trim());
+}
+
+// G. _MIN_DATA_POINTS actually shapes confidence (it is a dead knob in v1)
+{
+    const { p } = mk();
+    for (let i = 0; i < 3; i++) p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
+    const f = p._extractFeatures('read the logs');
+    p._MIN_DATA_POINTS = 10;
+    const a = p._computePrediction(p._findSimilar(f), f).confidence;
+    p._MIN_DATA_POINTS = 3;
+    const b = p._computePrediction(p._findSimilar(f), f).confidence;
+    ok('G1 lowering min-data-points raises confidence', b > a, a.toFixed(3) + ' -> ' + b.toFixed(3));
+}
+
+// H. v1 payload migration: junk accuracy discarded, observed points kept
+{
+    const { p, term } = mk();
+    term.fs.fsWriteJSON('/system/metrics/predictor.json', {
+        dataPoints: [{ features: { hasRead: true }, observedDuration: 30, observedTurns: 3 },
+                     { features: { hasRead: true }, predictedDuration: 30 }],
+        predictions: 7,
+        accuracy: { correct: 0, total: 6 }
+    });
+    p._loadFromVFS();
+    ok('H1 v1 accuracy discarded', p._accuracy.total === 0 && p._accuracy.correct === 0, JSON.stringify(p._accuracy));
+    ok('H2 observed points kept, prediction-only points dropped', p._dataPoints.length === 1, 'n=' + p._dataPoints.length);
+    ok('H3 predictions counter kept', p._predictions === 7, 'n=' + p._predictions);
+}
+
+// I. save → load round-trip keeps paired accuracy; stale pending is dropped
+{
+    const { p, term } = mk(true);
+    const pred = p.notePrediction('read the logs', 'test');
+    p.recordResult(p._extractFeatures('read the logs'), pred.predictedDuration, 2, true, { task: 'read the logs' });
+    p._saveNow();
+    const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
+    ok('I1 accuracy survives reload', q._accuracy.total === 1 && q._accuracy.correct === 1, JSON.stringify(q._accuracy));
+    const stale = p.notePrediction('read the logs again', 'test');
+    stale.ts = Date.now() - (p._PAIR_TTL_MS + 60000);   // expired while pending
+    p._saveNow();
+    const s = new PredictorModule(); s.terminal = term; s._loadFromVFS();
+    ok('I2 expired pending dropped on load', s._pending.length === 0, 'pending=' + s._pending.length);
+}
+
+
+// J. 3 s absolute floor: a small miss on a short task still counts
+{
+    const { p } = mk();
+    p.notePrediction('read a tiny file', 'test');
+    p._pending[0].predictedDuration = 5;                 // integer seconds vs fractional observed
+    p.recordResult(p._extractFeatures('read a tiny file'), 3.8, 1, true, { task: 'read a tiny file' });
+    ok('J1 1.2 s miss on a 3.8 s task is inside the 3 s floor', p._accuracy.correct === 1, JSON.stringify(p._accuracy));
+}
+
+// K. a wild miss is capped, so mean error stays readable
+{
+    const { p } = mk();
+    p.notePrediction('read the logs', 'test');
+    p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' });
+    const rec = p._dataPoints[p._dataPoints.length - 1];
+    ok('K1 errorPct capped at 500%', rec.errorPct <= 5, 'errorPct=' + rec.errorPct);
+}
+
+// L. a tool/loop prediction is never consumed by the similarity fallback
+{
+    const { p } = mk();
+    p.notePrediction('write a file report', 'tool');
+    p.recordResult(p._extractFeatures('write a file report somewhere else'), 40, 3, true);   // no task, no id
+    ok('L1 shape-alike run does not consume a tool prediction', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+}
+
+// M. the cli fallback needs at least one shared keyword
+{
+    const { p } = mk();
+    p.notePrediction('alpha bravo charlie', 'cli');
+    p.recordResult(p._extractFeatures('delta echo foxtrot'), 30, 3, true);
+    ok('M1 zero keyword overlap is not a match', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+}
+
+// N. tuning survives a reload
+{
+    const { p, term } = mk(true);
+    p._MIN_DATA_POINTS = 9;
+    p._PAIR_TTL_MS = 600000;
+    p._saveNow();
+    const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
+    ok('N1 config persisted', q._MIN_DATA_POINTS === 9 && q._PAIR_TTL_MS === 600000,
+        q._MIN_DATA_POINTS + '/' + q._PAIR_TTL_MS);
+}
+
+// O. "unpaired" counts scored-and-missed runs, not legacy observations
+{
+    const { p, out, term } = mk();
+    term.fs.fsWriteJSON('/system/metrics/predictor.json', {
+        dataPoints: [{ features: { hasRead: true }, observedDuration: 30 }],   // v1 point, no `paired` flag
+        accuracy: { correct: 0, total: 6 }
+    });
+    p._loadFromVFS();
+    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);           // unpaired v2 run
+    out.length = 0; p.showStatus();
+    const line = out.find(l => l.includes('Unpaired runs')) || '';
+    ok('O1 only paired:false records count as unpaired', /Unpaired runs: 1\b/.test(line), line.trim());
+}
+
+
+// P. the id threaded by the AI loop wins over text/similarity guessing
+{
+    const { p } = mk();
+    p.notePrediction('something else entirely', 'tool');          // decoy, pushed first
+    const loopPred = p.notePrediction('read the logs', 'loop');
+    p.recordResult(p._extractFeatures('read the logs'), loopPred.predictedDuration, 2, true,
+                   { task: 'read the logs', predictionId: loopPred.id });
+    ok('P1 the threaded id is the one consumed', p._accuracy.total === 1 && p._pending.length === 1,
+        'total=' + p._accuracy.total + ' pending=' + p._pending.length);
+}
+
+console.log(passes + '/' + (passes + fails) + ' PASS');
+process.exit(fails ? 1 : 0);