fix(newos4): measure predictor accuracy over paired predictions (v4.6.56)

The accuracy readout was structurally 0%: module-predictor.js:187 incremented the
denominator on every observed run while the numerator only moved when a stored
prediction existed, and predictions were stored only by the manual `predict`
command (the AI tool predict_task computed one and discarded it). Replaying the
AI loop's own call path with 6 tasks printed "Predictions made: 0, Accuracy: 0/6".

- pending predictions with a 15m TTL, consumed by threaded id -> task hash ->
  cli-only similarity >= 0.5; unpaired runs never enter the denominator
- duration scored within 30% or 3s (integer predictions vs fractional observations)
- errorPct capped at 500% so one miss cannot poison the mean error
- schema v2: v1 correct/total counters dropped on load, observed runs and the
  prediction counter kept; task text stored as an FNV-1a hash only
- tuning (confidence-threshold, min-data-points, pair-ttl) persists
- min-data-points now scales confidence (was declared, printed, settable, unread)
- AI loop registers a prediction per turn and threads its id; predict_task
  returns predictionId; durations exclude chat-lock queue wait
- status/dashboard say "n/a" instead of "N/A%", and carry paired count, mean
  error, turns error, pending and unpaired counts
- test: modules/test/test-predictor-accuracy.js (26 assertions)

Live on main.html v4.6.56: fresh session -> "Accuracy: n/a (0/0 paired)";
one paired run -> "100% (1/1 paired) - mean error 5% - turns +-0.0".

1 files changed, 179 insertions, 84 deletions

diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js
index 3d868ae..cb1209e 100644
--- a/modules/core/module-predictor.js
+++ b/modules/core/module-predictor.js
@@ -1,23 +1,30 @@
 // module-predictor.js — Predictive resource allocation engine
-// Analyzes historical telemetry + memory graph data to predict task duration/turns
+// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
+// real outcome. v1 counted every observed run in the denominator while incrementing the
+// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
 // Commands: predict, predictor-status, predictor-config
 class PredictorModule {
     constructor() {
         this.commands = {};
         this.terminal = null;
-        this._dataPoints = [];
+        this._dataPoints = [];      // observed runs — the training set for _findSimilar
+        this._pending = [];         // predictions waiting for an outcome (bounded, TTL'd)
         this._predictions = 0;
-        this._accuracy = { correct: 0, total: 0 };
+        this._accuracy = { correct: 0, total: 0 };  // total = paired comparisons only
+        this._mape = 0;             // sum of capped |predicted-observed|/observed
+        this._turnsErr = 0;         // sum of |predicted turns - observed turns|
         this._CONFIDENCE_THRESHOLD = 0.5;
         this._MIN_DATA_POINTS = 5;
+        this._PAIR_TTL_MS = 15 * 60 * 1000;
+        this._SCHEMA = 2;
         this._loaded = false;
     }
 
     getHelpInfo() {
         return {
             'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
-            'predictor-status': 'predictor-status — Model accuracy, data points, prediction count',
-            'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points',
+            'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
+            'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
         };
     }
 
@@ -39,33 +46,14 @@ class PredictorModule {
     async predict(task) {
         if (!task) return this.terminal.print('Usage: predict <task description>', 'error');
 
-        const startTime = performance.now();
-
-        // Extract features
         const features = this._extractFeatures(task);
-
-        // Find similar historical tasks
         const similar = this._findSimilar(features);
-
-        // Generate prediction
         const prediction = this._computePrediction(similar, features);
 
-        // Record this prediction for accuracy tracking
-        const duration = ((performance.now() - startTime) / 1000).toFixed(2);
-
-        this._predictions++;
-        this._dataPoints.push({
-            features,
-            predictedDuration: prediction.duration,
-            predictedTurns: prediction.toolTurns,
-            confidence: prediction.confidence,
-            similarCount: similar.length,
-            timestamp: new Date().toISOString()
-        });
-
-        this._debounceSave();
+        // Register it as pending so the next matching run can be scored against it.
+        const pending = this.notePrediction(task, 'cli');
+        pending.similarCount = similar.length;
 
-        // Display result
         this.terminal.print('📈 Task Prediction', 'header');
         this.terminal.print(`  Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');
 
@@ -81,23 +69,22 @@ class PredictorModule {
         } else {
             this.terminal.print('  ❓ No historical data to base prediction on.', 'warning');
             this.terminal.print('  Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
-            // Provide hardcoded defaults
             this.terminal.print('  Default estimate: ~30s duration, ~5 tool turns', 'info');
         }
 
-        // If got memory graph, offer recommendations
         if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
             this.terminal.print(`  💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
         }
 
+        this.terminal.print(`  🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
         return prediction;
     }
 
     _extractFeatures(task) {
-        const lower = task.toLowerCase();
+        const lower = String(task || '').toLowerCase();
         return {
-            length: task.length,
-            wordCount: task.split(/\s+/).length,
+            length: String(task || '').length,
+            wordCount: String(task || '').split(/\s+/).length,
             hasSearch: /\b(search|find|look)\b/.test(lower),
             hasWrite: /\b(write|create|save|edit)\b/.test(lower),
             hasDelete: /\b(delete|remove|rm)\b/.test(lower),
@@ -107,42 +94,33 @@ class PredictorModule {
             hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
             hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
             hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
-            complexity: Math.min(task.length / 50, 10), // 0-10 scale
+            complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
             keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
         };
     }
 
+    // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
+    _similarity(a, b) {
+        if (!a || !b) return 0;
+        const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
+                      'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
+        const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
+        const kw = a.keywords || [];
+        const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
+        const kScore = overlap / Math.max(kw.length, 1);
+        return matches * 0.6 + kScore * 0.4;
+    }
+
     _findSimilar(features) {
         return this._dataPoints.filter(dp => {
-            // Skip predictions (only use actual executions with observed results)
-            if (!dp.observedDuration) return false;
-            const f = dp.features;
-            if (!f) return false;
-            // Feature similarity scoring
-            let score = 0;
-            const matches = [
-                features.hasSearch === f.hasSearch,
-                features.hasWrite === f.hasWrite,
-                features.hasDelete === f.hasDelete,
-                features.hasInstall === f.hasInstall,
-                features.hasRead === f.hasRead,
-                features.hasWeb === f.hasWeb,
-                features.hasOrchestrate === f.hasOrchestrate,
-                features.hasModule === f.hasModule,
-                features.hasSync === f.hasSync,
-            ];
-            score = matches.filter(Boolean).length / matches.length;
-            // Keyword overlap bonus
-            const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length;
-            const kScore = kOverlap / Math.max(features.keywords.length, 1);
-            score = score * 0.6 + kScore * 0.4;
-            return score > 0.3;
+            if (!dp.observedDuration) return false;   // outcomes only, never predictions
+            if (!dp.features) return false;
+            return this._similarity(features, dp.features) > 0.3;
         });
     }
 
     _computePrediction(similar, features) {
         if (similar.length === 0) {
-            // Hardcoded defaults
             let baseDuration = 15;
             let baseTurns = 3;
             if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
@@ -160,9 +138,10 @@ class PredictorModule {
 
         const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
         const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
-        const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1;
+        // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
+        const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
+        const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;
 
-        // Apply task complexity multiplier
         const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
         const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5));
 
@@ -174,43 +153,126 @@ class PredictorModule {
         };
     }
 
+    // ─── Prediction ↔ Outcome pairing (v2) ───
+
+    // Task text never leaves the module: only a stable hash is stored/serialised.
+    _hashTask(t) {
+        const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
+        let h = 0x811c9dc5;
+        for (let i = 0; i < s.length; i++) {
+            h ^= s.charCodeAt(i);
+            h = (h * 0x01000193) >>> 0;
+        }
+        return h.toString(16);
+    }
+
+    // Register a prediction to be scored later. Returns the pending record (with .id).
+    notePrediction(task, source) {
+        const features = this._extractFeatures(task || '');
+        const p = this._computePrediction(this._findSimilar(features), features);
+        const rec = {
+            id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+            taskHash: this._hashTask(task),
+            features,
+            predictedDuration: p.duration,
+            predictedTurns: p.toolTurns,
+            confidence: p.confidence,
+            source: source || 'loop',
+            ts: Date.now()
+        };
+        this._pending.push(rec);
+        if (this._pending.length > 50) this._pending = this._pending.slice(-50);
+        this._predictions++;
+        this._debounceSave();
+        return rec;
+    }
+
+    _takePending(features, opts) {
+        const now = Date.now();
+        this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+        const taskHash = opts.task ? this._hashTask(opts.task) : null;
+
+        let idx = -1;
+        if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
+        if (idx < 0 && taskHash) {
+            for (let i = this._pending.length - 1; i >= 0; i--) {
+                if (this._pending[i].taskHash === taskHash) { idx = i; break; }
+            }
+        }
+        if (idx < 0) {
+            // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
+            // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
+            let best = -1, bestScore = 0;
+            for (let i = 0; i < this._pending.length; i++) {
+                const cand = this._pending[i];
+                if (cand.source !== 'cli') continue;
+                const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
+                if (shared === 0) continue;
+                const s = this._similarity(features, cand.features);
+                if (s > bestScore) { bestScore = s; best = i; }
+            }
+            if (bestScore >= 0.5) idx = best;
+        }
+        return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
+    }
+
     // ─── Learning from Results ───
 
-    recordResult(features, duration, toolTurns, success) {
-        this._dataPoints.push({
+    recordResult(features, duration, toolTurns, success, opts) {
+        const o = opts || {};
+        const pred = this._takePending(features, o);
+        const observed = Number(duration) || 0;
+
+        const rec = {
             features,
-            observedDuration: duration,
+            observedDuration: observed,
             observedTurns: toolTurns,
-            success,
+            success: !!success,
+            paired: !!pred,
             timestamp: new Date().toISOString()
-        });
-        this._accuracy.total++;
-        // Compare with last prediction for this feature set
-        const lastPrediction = [...this._dataPoints]
-            .reverse()
-            .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch);
-        if (lastPrediction) {
-            const error = Math.abs(lastPrediction.predictedDuration - duration) / duration;
-            if (error < 0.3) this._accuracy.correct++;
+        };
+
+        if (pred) {
+            const absErr = Math.abs(pred.predictedDuration - observed);
+            rec.predictedDuration = pred.predictedDuration;
+            rec.predictedTurns = pred.predictedTurns;
+            // capped: one 15s-vs-1s miss must not poison the mean-error readout forever
+            rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5);
+            rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
+            this._accuracy.total++;
+            this._mape += rec.errorPct;
+            this._turnsErr += rec.turnsError;
+            // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
+            if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
         }
-        this._debounceSave();
+
+        this._dataPoints.push(rec);
+        if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
+        if (pred) this._saveNow(); else this._debounceSave();
+        return rec;
     }
 
     // ─── Commands ───
 
     showStatus() {
         const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
-        const accPct = this._accuracy.total > 0
-            ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0)
-            : 'N/A';
+        const paired = this._accuracy.total;
+        const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
+        const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
+        const mape = paired > 0
+            ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}`
+            : '';
 
         this.terminal.print('📊 Predictor Status', 'header');
         this.terminal.print(`  Total data points: ${this._dataPoints.length}`, 'info');
         this.terminal.print(`  Actual executions: ${actualExecutions}`, 'info');
         this.terminal.print(`  Predictions made: ${this._predictions}`, 'info');
-        this.terminal.print(`  Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info');
+        this.terminal.print(`  Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
+        this.terminal.print(`  Awaiting outcome: ${this._pending.length}`, 'info');
+        this.terminal.print(`  Unpaired runs: ${unpaired}`, 'info');
         this.terminal.print(`  Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
         this.terminal.print(`  Min data points: ${this._MIN_DATA_POINTS}`, 'info');
+        this.terminal.print(`  Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
     }
 
     setConfig(args) {
@@ -231,6 +293,12 @@ class PredictorModule {
                 this._MIN_DATA_POINTS = n;
                 return `min-data-points set to ${n}`;
             },
+            'pair-ttl': () => {
+                const n = parseInt(val);
+                if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
+                this._PAIR_TTL_MS = n * 1000;
+                return `pair-ttl set to ${n}s`;
+            },
         };
 
         if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
@@ -245,10 +313,28 @@ class PredictorModule {
             const fs = this.terminal.fs;
             if (!fs) return;
             const data = fs.fsReadJSON('/system/metrics/predictor.json');
-            if (data) {
-                this._dataPoints = data.dataPoints || [];
-                this._predictions = data.predictions || 0;
-                this._accuracy = data.accuracy || { correct: 0, total: 0 };
+            if (!data) return;
+
+            // Prediction-only points are noise for similarity; keep outcomes.
+            this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
+            this._predictions = data.predictions || 0;
+
+            if (data.v === this._SCHEMA) {
+                const acc = data.accuracy || {};
+                this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
+                this._mape = data.mape || 0;
+                this._turnsErr = data.turnsErr || 0;
+                const cfg = data.config || {};
+                if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
+                if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
+                if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
+                const now = Date.now();
+                this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+            } else {
+                // v1 counted every observation in `total` but only some in `correct` → 0%.
+                this._accuracy = { correct: 0, total: 0 };
+                this._mape = 0;
+                this._pending = [];
             }
         } catch (e) {
             console.warn('[Predictor] Load error (fresh start):', e.message);
@@ -261,9 +347,18 @@ class PredictorModule {
             if (!fs) return;
             if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
             fs.fsWriteJSON('/system/metrics/predictor.json', {
-                dataPoints: this._dataPoints.slice(-500),  // keep last 500
+                v: this._SCHEMA,
+                dataPoints: this._dataPoints.slice(-500),
                 predictions: this._predictions,
-                accuracy: this._accuracy
+                accuracy: this._accuracy,
+                mape: this._mape,
+                turnsErr: this._turnsErr,
+                config: {
+                    confidenceThreshold: this._CONFIDENCE_THRESHOLD,
+                    minDataPoints: this._MIN_DATA_POINTS,
+                    pairTtlMs: this._PAIR_TTL_MS
+                },
+                pending: this._pending.slice(-50)
             });
         } catch (e) {
             console.error('[Predictor] Save error:', e.message);
@@ -279,4 +374,4 @@ class PredictorModule {
 // Required for module system
 if (typeof window !== 'undefined') {
     window.PredictorModule = PredictorModule;
-}
\ No newline at end of file
+}