path: root/modules/test/test-predictor-accuracy.js

// modules/test/test-predictor-accuracy.js — run: node modules/test/test-predictor-accuracy.js
// Proves the predictor's accuracy metric measures PAIRED predictions only.
// RED on v1 code: unpaired runs inflate the denominator, so accuracy is always 0%.
const fs = require('fs'), path = require('path');
const src = fs.readFileSync(path.join(__dirname, '../core/module-predictor.js'), 'utf8');
const PredictorModule = new Function(src + '\nreturn PredictorModule;')();

function memFs() {
    const files = {};
    const dirs = new Set(['/']);
    return {
        readFile: p => (p in files ? files[p] : null),
        writeFile: (p, c) => { files[p] = c; return true; },
        fsReadJSON: p => { try { return JSON.parse(files[p]); } catch (e) { return null; } },
        fsWriteJSON: (p, o) => { files[p] = JSON.stringify(o); return true; },
        readDir: p => (dirs.has(p) ? [] : null),
        mkdirRecursive: p => { dirs.add(p); return true; },
        mkdir: p => { dirs.add(p); return true; },
    };
}

function mk(persist) {
    const out = [];
    const term = { print: m => out.push(String(m)), fs: memFs() };
    const p = new PredictorModule();
    p.terminal = term;
    if (!persist) { p._saveNow = () => {}; p._debounceSave = () => {}; }   // isolate logic from timers
    return { p, out, term };
}

let fails = 0, passes = 0;
function ok(name, cond, detail) {
    if (cond) { passes++; return; }
    fails++;
    console.log('FAIL ' + name + (detail ? ' — ' + detail : ''));
}

// A. runs with no prediction must not enter the denominator
{
    const { p } = mk();
    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
    p.recordResult(p._extractFeatures('write a report'), 40, 3, true);
    ok('A1 total stays 0 without predictions', p._accuracy.total === 0, 'total=' + p._accuracy.total);
    ok('A2 observed runs are still recorded', p._dataPoints.filter(d => d.observedDuration).length === 2);
}

// B. a prediction paired with an on-time outcome counts as correct
{
    const { p } = mk();
    const pred = p.notePrediction('write a report file', 'test');
    ok('B0 notePrediction returns a pending prediction', !!pred && typeof pred.id === 'string');
    p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 1.1, 4, true, { task: 'write a report file' });
    ok('B1 paired once', p._accuracy.total === 1, 'total=' + p._accuracy.total);
    ok('B2 on-time prediction scored correct', p._accuracy.correct === 1, 'correct=' + p._accuracy.correct);
    ok('B3 pending consumed', p._pending.length === 0, 'pending=' + p._pending.length);
}

// C. an outcome >30% off is paired but wrong
{
    const { p } = mk();
    const pred = p.notePrediction('write a report file', 'test');
    p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 3 + 5, 4, true, { task: 'write a report file' });
    ok('C1 paired', p._accuracy.total === 1);
    ok('C2 not correct', p._accuracy.correct === 0);
}

// D. an unrelated run must not consume the pending prediction
{
    const { p } = mk();
    p.notePrediction('search the web for ollama docs', 'test');
    p.recordResult(p._extractFeatures('delete the tmp folder'), 9, 1, true, { task: 'delete the tmp folder' });
    ok('D1 unrelated run unpaired', p._accuracy.total === 0, 'total=' + p._accuracy.total);
    ok('D2 prediction still pending', p._pending.length === 1, 'pending=' + p._pending.length);
}

// E. expired predictions are dropped, not paired
{
    const { p } = mk();
    const pred = p.notePrediction('read the logs', 'test');
    pred.ts = Date.now() - (p._PAIR_TTL_MS + 1000);
    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true, { task: 'read the logs' });
    ok('E1 expired prediction not paired', p._accuracy.total === 0);
}

// F. the status line never renders "N/A%"
{
    const { p, out } = mk();
    p.showStatus();
    const line = out.find(l => l.includes('Accuracy')) || '';
    ok('F1 empty state says n/a', /\bn\/a\b/i.test(line) && !line.includes('N/A%'), line.trim());
    ok('F2 paired count is shown', /paired/i.test(line), line.trim());
}

// G. _MIN_DATA_POINTS actually shapes confidence (it is a dead knob in v1)
{
    const { p } = mk();
    for (let i = 0; i < 3; i++) p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
    const f = p._extractFeatures('read the logs');
    p._MIN_DATA_POINTS = 10;
    const a = p._computePrediction(p._findSimilar(f), f).confidence;
    p._MIN_DATA_POINTS = 3;
    const b = p._computePrediction(p._findSimilar(f), f).confidence;
    ok('G1 lowering min-data-points raises confidence', b > a, a.toFixed(3) + ' -> ' + b.toFixed(3));
}

// H. v1 payload migration: junk accuracy discarded, observed points kept
{
    const { p, term } = mk();
    term.fs.fsWriteJSON('/system/metrics/predictor.json', {
        dataPoints: [{ features: { hasRead: true }, observedDuration: 30, observedTurns: 3 },
                     { features: { hasRead: true }, predictedDuration: 30 }],
        predictions: 7,
        accuracy: { correct: 0, total: 6 }
    });
    p._loadFromVFS();
    ok('H1 v1 accuracy discarded', p._accuracy.total === 0 && p._accuracy.correct === 0, JSON.stringify(p._accuracy));
    ok('H2 observed points kept, prediction-only points dropped', p._dataPoints.length === 1, 'n=' + p._dataPoints.length);
    ok('H3 predictions counter kept', p._predictions === 7, 'n=' + p._predictions);
}

// I. save → load round-trip keeps paired accuracy; stale pending is dropped
{
    const { p, term } = mk(true);
    const pred = p.notePrediction('read the logs', 'test');
    p.recordResult(p._extractFeatures('read the logs'), pred.predictedDuration, 2, true, { task: 'read the logs' });
    p._saveNow();
    const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
    ok('I1 accuracy survives reload', q._accuracy.total === 1 && q._accuracy.correct === 1, JSON.stringify(q._accuracy));
    const stale = p.notePrediction('read the logs again', 'test');
    stale.ts = Date.now() - (p._PAIR_TTL_MS + 60000);   // expired while pending
    p._saveNow();
    const s = new PredictorModule(); s.terminal = term; s._loadFromVFS();
    ok('I2 expired pending dropped on load', s._pending.length === 0, 'pending=' + s._pending.length);
}


// J. 3 s absolute floor: a small miss on a short task still counts
{
    const { p } = mk();
    p.notePrediction('read a tiny file', 'test');
    p._pending[0].predictedDuration = 5;                 // integer seconds vs fractional observed
    p.recordResult(p._extractFeatures('read a tiny file'), 3.8, 1, true, { task: 'read a tiny file' });
    ok('J1 1.2 s miss on a 3.8 s task is inside the 3 s floor', p._accuracy.correct === 1, JSON.stringify(p._accuracy));
}

// K. a wild miss is capped, so mean error stays readable
{
    const { p } = mk();
    p.notePrediction('read the logs', 'test');
    p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' });
    const rec = p._dataPoints[p._dataPoints.length - 1];
    ok('K1 errorPct is stored raw', rec.errorPct > 5, 'errorPct=' + rec.errorPct);
}

// L. a tool/loop prediction is never consumed by the similarity fallback
{
    const { p } = mk();
    p.notePrediction('write a file report', 'tool');
    p.recordResult(p._extractFeatures('write a file report somewhere else'), 40, 3, true);   // no task, no id
    ok('L1 shape-alike run does not consume a tool prediction', p._accuracy.total === 0, 'total=' + p._accuracy.total);
}

// M. the cli fallback needs at least one shared keyword
{
    const { p } = mk();
    p.notePrediction('alpha bravo charlie', 'cli');
    p.recordResult(p._extractFeatures('delta echo foxtrot'), 30, 3, true);
    ok('M1 zero keyword overlap is not a match', p._accuracy.total === 0, 'total=' + p._accuracy.total);
}

// N. tuning survives a reload
{
    const { p, term } = mk(true);
    p._MIN_DATA_POINTS = 9;
    p._PAIR_TTL_MS = 600000;
    p._saveNow();
    const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
    ok('N1 config persisted', q._MIN_DATA_POINTS === 9 && q._PAIR_TTL_MS === 600000,
        q._MIN_DATA_POINTS + '/' + q._PAIR_TTL_MS);
}

// O. "unpaired" counts scored-and-missed runs, not legacy observations
{
    const { p, out, term } = mk();
    term.fs.fsWriteJSON('/system/metrics/predictor.json', {
        dataPoints: [{ features: { hasRead: true }, observedDuration: 30 }],   // v1 point, no `paired` flag
        accuracy: { correct: 0, total: 6 }
    });
    p._loadFromVFS();
    p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);           // unpaired v2 run
    out.length = 0; p.showStatus();
    const line = out.find(l => l.includes('Unpaired runs')) || '';
    ok('O1 only paired:false records count as unpaired', /Unpaired runs: 1\b/.test(line), line.trim());
}


// P. the id threaded by the AI loop wins over text/similarity guessing
{
    const { p } = mk();
    p.notePrediction('something else entirely', 'tool');          // decoy, pushed first
    const loopPred = p.notePrediction('read the logs', 'loop');
    p.recordResult(p._extractFeatures('read the logs'), loopPred.predictedDuration, 2, true,
                   { task: 'read the logs', predictionId: loopPred.id });
    ok('P1 the threaded id is the one consumed', p._accuracy.total === 1 && p._pending.length === 1,
        'total=' + p._accuracy.total + ' pending=' + p._pending.length);
}


// Q. an aborted 0.079 s run must not drive the next prediction to zero (§5 point 1→2)
{
    const { p } = mk();
    const task = 'paste a 2158 char status report';
    p.notePrediction(task, 'loop');
    p.recordResult(p._extractFeatures(task), 0.079, 1, true, { task });
    const next = p.notePrediction(task, 'loop');
    ok('Q1 prediction is never 0 s', next.predictedDuration >= 1, 'pred=' + next.predictedDuration);
    ok('Q2 the aborted run is marked unusable', p._dataPoints[0].usable === false);
    ok('Q3 it never trains _findSimilar', p._findSimilar(p._extractFeatures(task)).length === 0);
}

// R. the persisted field is a SUM and the display divides by the paired count
{
    const { p, out, term } = mk(true);
    for (let i = 0; i < 2; i++) {
        const task = 'read the logs';
        p.notePrediction(task, 'loop');
        p.recordResult(p._extractFeatures(task), 20, 2, true, { task });   // 15 s predicted vs 20 s observed
    }
    p._saveNow();
    const blob = JSON.parse(term.fs.readFile('/system/metrics/predictor.json'));
    ok('R1 stored accumulator is a sum and is named errorSum', typeof blob.errorSum === 'number' && !('mape' in blob),
        JSON.stringify(Object.keys(blob)));
    out.length = 0; p.showStatus();
    const line = out.find(l => l.includes('Accuracy')) || '';
    ok('R2 display shows the mean with its sample count', /Accuracy: \d+% \(2\/2 paired\) · mean error \d+%/.test(line), line.trim());
}

// S. a floored prediction is consumed but not scored as error (§5 defect C)
{
    const { p } = mk();
    const pred = p.notePrediction('read the logs', 'test');
    p._pending[0].floored = true;
    p._pending[0].predictedDuration = 1;
    p.recordResult(p._extractFeatures('read the logs'), 54.7, 2, true, { task: 'read the logs' });
    ok('S1 floored prediction does not enter accuracy', p._accuracy.total === 0, 'total=' + p._accuracy.total);
    ok('S2 it is still recorded and marked unscored', p._dataPoints[0].scored === false);
    ok('S3 pending consumed', p._pending.length === 0);
}

console.log(passes + '/' + (passes + fails) + ' PASS');
process.exit(fails ? 1 : 0);