aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--downloads/system-info.html262
-rw-r--r--help.html10
-rw-r--r--main.html6
-rw-r--r--modules/core/core-bundle.js308
-rw-r--r--modules/core/module-aiassist.js47
-rw-r--r--modules/core/module-loader.js2
-rw-r--r--modules/core/module-predictor.js263
-rw-r--r--modules/test/test-predictor-accuracy.js209
8 files changed, 912 insertions, 195 deletions
diff --git a/downloads/system-info.html b/downloads/system-info.html
new file mode 100644
index 0000000..4991292
--- /dev/null
+++ b/downloads/system-info.html
@@ -0,0 +1,262 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
+ <meta name="theme-color" content="#0f1420">
+ <meta name="color-scheme" content="dark">
+ <meta name="description" content="newOS4 v4.6.27 system snapshot — modules, skills, agents, token usage, and health metrics.">
+ <meta property="og:title" content="newOS4 System Info">
+ <meta property="og:description" content="System snapshot — 46 modules, 39 skills, token usage, agents, and health metrics.">
+ <meta property="og:type" content="website">
+ <title>newOS4 System Info</title>
+ <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Ctext y='13' font-size='13'%3E%E2%9A%99%EF%B8%8F%3C/text%3E%3C/svg%3E">
+ <style>
+ * { box-sizing: border-box; margin: 0; padding: 0; }
+ html { -webkit-text-size-adjust: 100%; text-size-adjust: 100%; }
+ body {
+ font-family: system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif;
+ font-size: 16px;
+ background: #0f1420;
+ color: #e4e4e4;
+ min-height: 100vh;
+ padding: clamp(0.75rem, 3vw, 2rem);
+ padding-left: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-left));
+ padding-right: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-right));
+ padding-bottom: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-bottom));
+ overflow-wrap: break-word;
+ -webkit-tap-highlight-color: transparent;
+ }
+ .container { max-width: 1200px; margin: 0 auto; }
+ h1 { text-align: center; color: #00d9ff; font-size: clamp(1.5rem, 6vw, 2.5rem); margin-bottom: 0.35rem; text-shadow: 0 0 20px rgba(0,217,255,0.5); }
+ .subtitle { text-align: center; color: #8b93a7; font-size: clamp(0.8rem, 3vw, 0.95rem); margin-bottom: clamp(1rem, 4vw, 2rem); }
+ .grid { display: grid; grid-template-columns: 1fr; gap: clamp(0.75rem, 3vw, 1.5rem); }
+ @media (min-width: 640px) { .grid { grid-template-columns: repeat(2, 1fr); } }
+ @media (min-width: 1024px) { .grid { grid-template-columns: repeat(3, 1fr); } }
+ .card {
+ background: #151b2e; border-radius: 12px;
+ padding: clamp(0.9rem, 4vw, 1.5rem);
+ border: 1px solid rgba(255,255,255,0.08);
+ min-width: 0;
+ }
+ .card h2 { color: #00d9ff; font-size: clamp(0.95rem, 4vw, 1.2rem); margin-bottom: 0.85rem; border-bottom: 1px solid rgba(0,217,255,0.3); padding-bottom: 0.5rem; }
+ dl.stat-list { display: block; }
+ dl.stat-list > div { display: flex; justify-content: space-between; align-items: center; gap: 0.75rem; padding: 0.55rem 0; min-height: 2.4rem; border-bottom: 1px solid rgba(255,255,255,0.05); }
+ dl.stat-list > div:last-child { border-bottom: none; }
+ dt.stat-label { color: #a8b0c4; min-width: 0; }
+ dd.stat-value { font-weight: 600; color: #fff; text-align: right; min-width: 0; }
+ .stat-value.ok { color: #00ff88; }
+ .stat-value.warn { color: #ffaa00; }
+ .stat-value.error { color: #ff4466; }
+ .note { margin-top: 0.85rem; color: #8b93a7; font-size: 0.8rem; line-height: 1.45; }
+ .module-list { display: flex; flex-wrap: wrap; gap: 0.4rem; margin-top: 0.5rem; }
+ .tag { background: rgba(0,217,255,0.16); padding: 0.3rem 0.6rem; border-radius: 20px; font-size: 0.72rem; line-height: 1.4; color: #00d9ff; }
+ .tag.core { background: rgba(0,255,136,0.14); color: #00ff88; }
+ .tag.ext { background: rgba(255,170,0,0.14); color: #ffaa00; }
+ .tag.modified { background: rgba(255,68,102,0.16); color: #ff4466; border: 1px solid #ff4466; }
+ .model-item { margin: 0.65rem 0; min-width: 0; }
+ .model-top { display: flex; justify-content: space-between; gap: 0.6rem; font-size: 0.78rem; margin-bottom: 0.3rem; }
+ .model-name { color: #b8c0d4; min-width: 0; overflow-wrap: break-word; }
+ .model-count { color: #8b93a7; flex-shrink: 0; }
+ .model-track { background: rgba(255,255,255,0.08); border-radius: 8px; overflow: hidden; }
+ .model-fill { height: 26px; border-radius: 8px; display: flex; align-items: center; justify-content: flex-end; padding: 0 0.6rem; font-size: 0.75rem; font-weight: 600; color: #fff; min-width: 2.6rem; }
+ .model-fill.c1 { background: #0088cc; }
+ .model-fill.c2 { background: #7a3db8; }
+ .model-fill.c3 { background: #009977; }
+ .model-fill.c4 { background: #cc5500; }
+ .model-fill.c5 { background: #8a6d1f; }
+ .model-fill.c6 { background: #a83d6e; }
+ .agent-row { display: flex; align-items: center; gap: 0.55rem; padding: 0.55rem 0; min-height: 2.4rem; font-size: 0.85rem; border-bottom: 1px solid rgba(255,255,255,0.05); }
+ .agent-row:last-child { border-bottom: none; }
+ .agent-icon { width: 8px; height: 8px; border-radius: 50%; background: #6c7a95; flex-shrink: 0; }
+ .agent-name { min-width: 0; overflow-wrap: break-word; }
+ .badge { display: inline-block; padding: 0.2rem 0.5rem; border-radius: 4px; font-size: 0.68rem; font-weight: 600; flex-shrink: 0; }
+ .badge-auto { background: rgba(0,217,255,0.16); color: #00d9ff; }
+ .badge-arch { background: rgba(255,170,0,0.16); color: #ffaa00; }
+ .badge-search { background: rgba(0,255,136,0.16); color: #00ff88; }
+ .badge-sleep { background: rgba(120,120,140,0.2); color: #a8b0c4; margin-left: auto; }
+ .timestamp { text-align: center; color: #8b93a7; font-size: 0.78rem; margin-top: clamp(1rem, 4vw, 2rem); }
+ @media (prefers-reduced-motion: reduce) { h1 { text-shadow: none; } }
+ @media (min-width: 1024px) { .card { transition: border-color 0.3s ease; } .card:hover { border-color: rgba(0,217,255,0.35); } }
+ @media print {
+ body { background: #fff; color: #111; }
+ h1 { color: #005f80; text-shadow: none; }
+ .card { background: #f6f6f6; border: 1px solid #ccc; break-inside: avoid; }
+ .card h2 { color: #005f80; }
+ .stat-label, .model-name, .model-count, .note, .timestamp, .subtitle { color: #555; }
+ .stat-value { color: #000; }
+ .stat-value.ok { color: #0a7a3d; }
+ .stat-value.warn { color: #9c6200; }
+ .stat-value.error { color: #b3001e; }
+ .model-track { background: #ddd; }
+ .model-fill { color: #fff; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+ }
+ </style>
+</head>
+<body>
+ <script type="application/json" id="system-data">
+{
+ "version": "v4.6.27",
+ "vfs": { "files": 467, "dirs": 181, "size": "1431.7KB" },
+ "memory": { "heap": "16.3MB", "graphNodes": 41, "graphEdges": 49 },
+ "modules": { "loaded": 46, "core": 23, "ext": 24, "available": 47, "modified": ["OrchestratorModule"] },
+ "skills": 39,
+ "agents": [
+ { "id": "agent-mvdsl9qg-5sq9-auto-learner", "role": "auto-learner", "status": "sleeping", "caps": 4, "badge": "auto" },
+ { "id": "agent-mvdsl9r2-bkm1-kernel-architect", "role": "kernel-architect", "status": "sleeping", "caps": 4, "badge": "arch" },
+ { "id": "agent-mvdsl9rl-bq7d-searcher", "role": "searcher", "status": "sleeping", "caps": 3, "badge": "search" }
+ ],
+ "ollama": { "connected": true, "model": "glm-5.3:cloud" },
+ "usage": {
+ "total": 500, "prompt": 10006288, "completion": 956373,
+ "avgLatency": 18610, "timeSpanHours": 22.1, "provider": "Ollama",
+ "byModel": {
+ "glm-5.3:cloud": { "requests": 203, "tokens": 7696218 },
+ "minimax-m2.7:cloud": { "requests": 218, "tokens": 2114960 },
+ "kimi-k3:cloud": { "requests": 47, "tokens": 844472 },
+ "gemma4:cloud": { "requests": 18, "tokens": 176391 },
+ "gpt-oss:120b-cloud": { "requests": 7, "tokens": 72567 },
+ "kimi-k2.7-code:cloud": { "requests": 7, "tokens": 58053 }
+ }
+ },
+ "latency": [
+ { "name": "web_fetch", "calls": 4, "errors": 2 },
+ { "name": "create_skill", "calls": 2, "errors": 0 },
+ { "name": "list_skills", "calls": 2, "errors": 0 },
+ { "name": "write_file", "calls": 3, "errors": 0 },
+ { "name": "read_file", "calls": 2, "errors": 0 },
+ { "name": "world_model_query", "calls": 1, "errors": 0 },
+ { "name": "get_status", "calls": 1, "errors": 0 },
+ { "name": "get_usage_stats", "calls": 1, "errors": 0 },
+ { "name": "get_module_status", "calls": 1, "errors": 0 },
+ { "name": "list_agents", "calls": 1, "errors": 0 }
+ ],
+ "health": { "online": true, "errors": 7, "predictorAccuracy": "n/a (no paired runs)" },
+ "timestamp": 1792327737761
+}
+ </script>
+
+ <div class="container">
+ <h1>⚙️ newOS4 System Info</h1>
+ <p class="subtitle">v4.6.27 • System Snapshot • Mobile-Optimized</p>
+
+ <div class="grid">
+ <!-- System Overview -->
+ <div class="card">
+ <h2>🖥️ System Overview</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">Version</dt><dd class="stat-value">v4.6.27</dd></div>
+ <div><dt class="stat-label">VFS Files</dt><dd class="stat-value ok">467</dd></div>
+ <div><dt class="stat-label">VFS Dirs / Size</dt><dd class="stat-value">181 · 1.4 MB</dd></div>
+ <div><dt class="stat-label">Memory Heap</dt><dd class="stat-value ok">16.3 MB</dd></div>
+ <div><dt class="stat-label">Graph Nodes / Edges</dt><dd class="stat-value">41 / 49</dd></div>
+ <div><dt class="stat-label">Ollama</dt><dd class="stat-value ok">Connected</dd></div>
+ <div><dt class="stat-label">Active Model</dt><dd class="stat-value">glm-5.3:cloud</dd></div>
+ <div><dt class="stat-label">System Errors</dt><dd class="stat-value error">7 ⚠️</dd></div>
+ </dl>
+ </div>
+
+ <!-- Modules -->
+ <div class="card">
+ <h2>📦 Modules</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">Loaded</dt><dd class="stat-value ok">46</dd></div>
+ <div><dt class="stat-label">Core</dt><dd class="stat-value">23</dd></div>
+ <div><dt class="stat-label">Extensions</dt><dd class="stat-value">24 available</dd></div>
+ <div><dt class="stat-label">Modified</dt><dd class="stat-value warn">1 (Orchestrator)</dd></div>
+ </dl>
+ <p class="note">Core: filesystem, AI, agents, memory graph, neural VFS, orchestration. Extensions: editors, shell, WebDAV, cron, mindmaps, Three, Course, Media, JSConsole, and more.</p>
+ </div>
+
+ <!-- Token Usage -->
+ <div class="card">
+ <h2>📊 Token Usage</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">Total Requests</dt><dd class="stat-value">500</dd></div>
+ <div><dt class="stat-label">Total Tokens</dt><dd class="stat-value ok">10.96M</dd></div>
+ <div><dt class="stat-label">Prompt / Completion</dt><dd class="stat-value">10.01M / 956K</dd></div>
+ <div><dt class="stat-label">Time Span</dt><dd class="stat-value">~22 hours</dd></div>
+ <div><dt class="stat-label">Provider</dt><dd class="stat-value">Ollama (100%)</dd></div>
+ <div><dt class="stat-label">Avg Latency</dt><dd class="stat-value">18.6s</dd></div>
+ </dl>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">glm-5.3:cloud · 203 req</span><span class="model-count">7.70M</span></div>
+ <div class="model-track" role="img" aria-label="glm-5.3:cloud usage, 70.2% share"><div class="model-fill c1" style="width:100%">70.2%</div></div>
+ </div>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">minimax-m2.7:cloud · 218 req</span><span class="model-count">2.11M</span></div>
+ <div class="model-track" role="img" aria-label="minimax-m2.7:cloud usage, 19.3% share"><div class="model-fill c2" style="width:27.5%">19.3%</div></div>
+ </div>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">kimi-k3:cloud · 47 req</span><span class="model-count">844K</span></div>
+ <div class="model-track" role="img" aria-label="kimi-k3:cloud usage, 7.7% share"><div class="model-fill c3" style="width:10.9%">7.7%</div></div>
+ </div>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">gemma4:cloud · 18 req</span><span class="model-count">176K</span></div>
+ <div class="model-track" role="img" aria-label="gemma4:cloud usage, 1.6% share"><div class="model-fill c4" style="width:2.3%">1.6%</div></div>
+ </div>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">gpt-oss:120b-cloud · 7 req</span><span class="model-count">72.6K</span></div>
+ <div class="model-track" role="img" aria-label="gpt-oss:120b-cloud usage, 0.7% share"><div class="model-fill c5" style="width:0.94%">0.7%</div></div>
+ </div>
+ <div class="model-item">
+ <div class="model-top"><span class="model-name">kimi-k2.7-code:cloud · 7 req</span><span class="model-count">58.1K</span></div>
+ <div class="model-track" role="img" aria-label="kimi-k2.7-code:cloud usage, 0.5% share"><div class="model-fill c6" style="width:0.75%">0.5%</div></div>
+ </div>
+ </div>
+
+ <!-- Skills -->
+ <div class="card">
+ <h2>🎓 Skills</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">Total Skills</dt><dd class="stat-value ok">39</dd></div>
+ <div><dt class="stat-label">Recently Added</dt><dd class="stat-value">generate-info-page</dd></div>
+ </dl>
+ <p class="note">Categories: system, learning, dev, process, business. Includes newOS4 interactive course, memgraph visualization, mobile HTML optimization, and more.</p>
+ </div>
+
+ <!-- Agents -->
+ <div class="card">
+ <h2>🤖 Agents</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">Total Agents</dt><dd class="stat-value">3</dd></div>
+ <div><dt class="stat-label">Status</dt><dd class="stat-value">All sleeping 💤</dd></div>
+ </dl>
+ <div class="agent-row">
+ <span class="agent-icon" aria-hidden="true"></span>
+ <span class="agent-name">auto-learner</span>
+ <span class="badge badge-auto">AUTO</span>
+ <span class="badge badge-sleep">sleeping</span>
+ </div>
+ <div class="agent-row">
+ <span class="agent-icon" aria-hidden="true"></span>
+ <span class="agent-name">kernel-architect</span>
+ <span class="badge badge-arch">ARCH</span>
+ <span class="badge badge-sleep">sleeping</span>
+ </div>
+ <div class="agent-row">
+ <span class="agent-icon" aria-hidden="true"></span>
+ <span class="agent-name">searcher</span>
+ <span class="badge badge-search">SEARCH</span>
+ <span class="badge badge-sleep">sleeping</span>
+ </div>
+ </div>
+
+ <!-- Tool Latency & Health -->
+ <div class="card">
+ <h2>⚡ Tools & Health</h2>
+ <dl class="stat-list">
+ <div><dt class="stat-label">System Online</dt><dd class="stat-value ok">Yes</dd></div>
+ <div><dt class="stat-label">System Errors</dt><dd class="stat-value error">7 ⚠️</dd></div>
+ <div><dt class="stat-label">Predictor Accuracy</dt><dd class="stat-value warn">0%</dd></div>
+ <div><dt class="stat-label">web_fetch Errors</dt><dd class="stat-value warn">2 of 4 calls</dd></div>
+ <div><dt class="stat-label">Auto-Lessons</dt><dd class="stat-value">3 saved</dd></div>
+ </dl>
+ <p class="note">web_fetch shows 2 errors — monitor if it persists. Auto-lessons flag anomalies (high token usage, near turn limits) for self-improvement.</p>
+ </div>
+ </div>
+
+ <p class="timestamp">Snapshot: <time datetime="2026-10-18T12:48:57Z">Sat Oct 18 2026, 12:48 UTC</time> · Regenerate via <code>generate-info-page</code> skill</p>
+ </div>
+</body>
+</html> \ No newline at end of file
diff --git a/help.html b/help.html
index 18a03ef..9a9a363 100644
--- a/help.html
+++ b/help.html
@@ -1660,6 +1660,16 @@ listmods</pre>
<h2>📋 Changelog</h2>
<div class="version-item">
+ <div class="v-header"><span class="v-num">v4.6.56 — October 2026</span><span class="v-date">2026-10-23</span></div>
+ <ul>
+ <li><strong>Predictor accuracy was structurally 0%</strong> — the denominator counted every observed run while the numerator only moved when a <em>stored prediction</em> matched, and predictions were stored only by the manual <code>predict</code> command. Unpaired runs no longer enter the accuracy denominator; the AI loop now registers a prediction per turn (<code>notePrediction</code>) and <code>predict_task</code> returns a tracked <code>predictionId</code> instead of discarding its estimate.</li>
+ <li><strong>Paired scoring</strong> — a prediction is consumed by threaded id → task hash → cli-only feature similarity ≥ 0.5, within a 15-minute TTL (<code>predictor-config pair-ttl &lt;seconds&gt;</code>). A run counts as correct within 30% <em>or</em> 3 seconds of the predicted duration; <code>predictor-status</code> shows paired accuracy, mean error, turns error, pending predictions and unpaired runs.</li>
+ <li><strong>Schema v2</strong> — stored payloads from v1 are migrated on load: observed runs and the prediction counter are kept, the meaningless <code>correct/total</code> counters are dropped. Task text is stored as an FNV-1a hash, never in plaintext. Tuning (<code>confidence-threshold</code>, <code>min-data-points</code>, <code>pair-ttl</code>) now persists.</li>
+ <li><strong>Knobs that do something</strong> — <code>min-data-points</code> now scales prediction confidence (it was declared, displayed and settable but never read), durations exclude chat-lock queue wait, and the confidence percentages on <code>predict</code>/<code>predict_task</code> will move for the same history (fewer points ⇒ lower confidence).</li>
+ </ul>
+ </div>
+
+ <div class="version-item">
<div class="v-header"><span class="v-num">v4.6.52 — October 2026</span><span class="v-date">2026-10-22</span></div>
<ul>
<li><strong>Documentation backlog cleared</strong> — 74 commands that the code registered but this page never documented are now in their own sections: <code>dashboard</code> (with <code>--watch</code>/<code>--stop</code>), <code>usage</code>, <code>ps</code>/<code>top</code>/<code>df</code>/<code>free</code>, the six <code>a2a-*</code> mailbox commands, <code>webxai</code>/<code>webx</code>/<code>wx</code>, <code>context</code>/<code>contexts</code>/<code>context-new</code>/<code>context-delete</code>/<code>context-auto</code>/<code>smart-ask</code>, the <code>commit-*</code> approval trio, <code>module-load|unload|enable|reset</code>, <code>memgraph-agent|session|decay-config</code>, <code>orchestrator-schedule|schedules|unschedule</code>, <code>skill-validate|audit|restore|pack-core</code>, <code>learn-skill</code>, <code>encrypt</code>/<code>decrypt</code>, <code>gzip</code>/<code>wget</code>, <code>kanban</code>, <code>three</code>, <code>course</code>, <code>videos</code>, <code>speak</code>/<code>voice</code>/<code>analyze</code>, <code>gui-run|text|clear</code>, the <code>vaultmind-*</code> set, <code>debug-sync</code>/<code>debug-sync-url</code>, <code>setfont</code>, <code>bug</code>, <code>session-new</code>, <code>files</code>, <code>node</code>, <code>sed</code>, <code>nvfs-index-graph</code>, <code>wallet-lookup</code>, <code>ai-model</code>, <code>ai-ignore</code>, <code>webdav-login</code>, <code>view_module</code>, <code>run</code>, <code>sysmsg</code>, <code>history</code>.</li>
diff --git a/main.html b/main.html
index e89534c..c84421b 100644
--- a/main.html
+++ b/main.html
@@ -1349,13 +1349,13 @@
</script>
<!-- v4.5.63: A2A scratchpad fixes + agent-register arg fix -->
-<script src="modules/core/core-bundle.js?v=v4.6.55"></script>
+<script src="modules/core/core-bundle.js?v=v4.6.56"></script>
<!-- Async dependency-ordered module loader now only initializes terminal and loads optional ext modules -->
-<script src="modules/core/module-loader.js?v=v4.6.55"></script>
+<script src="modules/core/module-loader.js?v=v4.6.56"></script>
<!-- Critical ext bundle loads AFTER webx11 exists so registerModule calls work -->
-<script src="modules/ext/critical-ext-bundle.js?v=v4.6.55"></script>
+<script src="modules/ext/critical-ext-bundle.js?v=v4.6.56"></script>
<script>
// ═══ ONE TRUE VERSION SOURCE ═══
diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js
index 67747bf..9ac94ed 100644
--- a/modules/core/core-bundle.js
+++ b/modules/core/core-bundle.js
@@ -9041,6 +9041,7 @@ buildSystemPrompt() {
await new Promise(r => setTimeout(r, 50));
}
this._chatLock = true;
+ const taskStart = Date.now(); // predictor durations exclude chat-lock wait
this.terminal.markDirty();
// Auto-route context based on intent
try {
@@ -9131,6 +9132,16 @@ buildSystemPrompt() {
}
} catch (e) { /* world model not available */ }
+ // v4.6.56: register a prediction for THIS turn so recordResult can score it by id.
+ let turnPredictionId = null;
+ try {
+ const predictor = this.terminal.moduleManager.modules['PredictorModule'];
+ if (predictor && typeof predictor.notePrediction === 'function') {
+ const tp = predictor.notePrediction(userMessage, 'loop');
+ turnPredictionId = tp && tp.id ? tp.id : null;
+ }
+ } catch (e) { /* predictor is best-effort */ }
+
const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main';
try {
loopResult = await this.agenticLoop(messages, opts);
@@ -9158,8 +9169,9 @@ buildSystemPrompt() {
const predictor = this.terminal.moduleManager.modules['PredictorModule'];
if (predictor && typeof predictor.recordResult === 'function') {
const features = predictor._extractFeatures(userMessage);
- const duration = (Date.now() - lockStart) / 1000;
- predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled);
+ const duration = (Date.now() - taskStart) / 1000;
+ predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled,
+ { task: userMessage, predictionId: turnPredictionId });
}
} catch (e) { /* predictor learning is best-effort */ }
// Record task in memory graph for direct AI conversations (not just orchestrator)
@@ -11241,16 +11253,26 @@ buildSystemPrompt() {
if (!taskDesc) return JSON.stringify({ error: 'task parameter required' });
var pFeatures = predictor._extractFeatures(taskDesc);
var pSimilar = predictor._findSimilar(pFeatures);
- var pResult = predictor._computePrediction(pSimilar, pFeatures);
+ // v4.6.56: track the prediction so the outcome can score it.
+ var pTracked;
+ if (typeof predictor.notePrediction === 'function') {
+ pTracked = predictor.notePrediction(taskDesc, 'tool');
+ } else {
+ var pFallback = predictor._computePrediction(pSimilar, pFeatures);
+ pTracked = { id: null, predictedDuration: pFallback.duration,
+ predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence };
+ }
return JSON.stringify({
task: taskDesc.substring(0, 80),
- estimatedDuration: pResult.duration + 's',
- estimatedToolTurns: pResult.toolTurns,
- confidence: Math.round(pResult.confidence * 100) + '%',
- similarTasks: pResult.similarCount,
- recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30
- ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's'
- : 'No special action needed'
+ predictionId: pTracked.id || null,
+ estimatedDuration: pTracked.predictedDuration + 's',
+ estimatedToolTurns: pTracked.predictedTurns,
+ confidence: Math.round(pTracked.confidence * 100) + '%',
+ similarTasks: pSimilar.length,
+ recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30
+ ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's'
+ : 'No special action needed',
+ tracked: !!pTracked.id
});
}
case 'world_model_query': {
@@ -12841,9 +12863,10 @@ buildSystemPrompt() {
if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length;
} catch (e) {}
var predictor = mm ? mm.modules['PredictorModule'] : null;
- var predictorAcc = 'N/A';
+ var predictorAcc = 'n/a (no paired runs)';
if (predictor && predictor._accuracy && predictor._accuracy.total > 0) {
- predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%';
+ predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' +
+ predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)';
}
data.health = {
modules: data.modules.loaded + '/' + data.modules.total,
@@ -17736,25 +17759,32 @@ if (typeof window !== 'undefined') {
/* === module-predictor.js === */
// module-predictor.js — Predictive resource allocation engine
-// Analyzes historical telemetry + memory graph data to predict task duration/turns
+// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
+// real outcome. v1 counted every observed run in the denominator while incrementing the
+// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
// Commands: predict, predictor-status, predictor-config
class PredictorModule {
constructor() {
this.commands = {};
this.terminal = null;
- this._dataPoints = [];
+ this._dataPoints = []; // observed runs — the training set for _findSimilar
+ this._pending = []; // predictions waiting for an outcome (bounded, TTL'd)
this._predictions = 0;
- this._accuracy = { correct: 0, total: 0 };
+ this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only
+ this._mape = 0; // sum of capped |predicted-observed|/observed
+ this._turnsErr = 0; // sum of |predicted turns - observed turns|
this._CONFIDENCE_THRESHOLD = 0.5;
this._MIN_DATA_POINTS = 5;
+ this._PAIR_TTL_MS = 15 * 60 * 1000;
+ this._SCHEMA = 2;
this._loaded = false;
}
getHelpInfo() {
return {
'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
- 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count',
- 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points',
+ 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
+ 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
};
}
@@ -17776,33 +17806,14 @@ class PredictorModule {
async predict(task) {
if (!task) return this.terminal.print('Usage: predict <task description>', 'error');
- const startTime = performance.now();
-
- // Extract features
const features = this._extractFeatures(task);
-
- // Find similar historical tasks
const similar = this._findSimilar(features);
-
- // Generate prediction
const prediction = this._computePrediction(similar, features);
- // Record this prediction for accuracy tracking
- const duration = ((performance.now() - startTime) / 1000).toFixed(2);
+ // Register it as pending so the next matching run can be scored against it.
+ const pending = this.notePrediction(task, 'cli');
+ pending.similarCount = similar.length;
- this._predictions++;
- this._dataPoints.push({
- features,
- predictedDuration: prediction.duration,
- predictedTurns: prediction.toolTurns,
- confidence: prediction.confidence,
- similarCount: similar.length,
- timestamp: new Date().toISOString()
- });
-
- this._debounceSave();
-
- // Display result
this.terminal.print('📈 Task Prediction', 'header');
this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');
@@ -17818,23 +17829,22 @@ class PredictorModule {
} else {
this.terminal.print(' ❓ No historical data to base prediction on.', 'warning');
this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
- // Provide hardcoded defaults
this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info');
}
- // If got memory graph, offer recommendations
if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
}
+ this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
return prediction;
}
_extractFeatures(task) {
- const lower = task.toLowerCase();
+ const lower = String(task || '').toLowerCase();
return {
- length: task.length,
- wordCount: task.split(/\s+/).length,
+ length: String(task || '').length,
+ wordCount: String(task || '').split(/\s+/).length,
hasSearch: /\b(search|find|look)\b/.test(lower),
hasWrite: /\b(write|create|save|edit)\b/.test(lower),
hasDelete: /\b(delete|remove|rm)\b/.test(lower),
@@ -17844,42 +17854,33 @@ class PredictorModule {
hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
- complexity: Math.min(task.length / 50, 10), // 0-10 scale
+ complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
};
}
+ // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
+ _similarity(a, b) {
+ if (!a || !b) return 0;
+ const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
+ 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
+ const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
+ const kw = a.keywords || [];
+ const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
+ const kScore = overlap / Math.max(kw.length, 1);
+ return matches * 0.6 + kScore * 0.4;
+ }
+
_findSimilar(features) {
return this._dataPoints.filter(dp => {
- // Skip predictions (only use actual executions with observed results)
- if (!dp.observedDuration) return false;
- const f = dp.features;
- if (!f) return false;
- // Feature similarity scoring
- let score = 0;
- const matches = [
- features.hasSearch === f.hasSearch,
- features.hasWrite === f.hasWrite,
- features.hasDelete === f.hasDelete,
- features.hasInstall === f.hasInstall,
- features.hasRead === f.hasRead,
- features.hasWeb === f.hasWeb,
- features.hasOrchestrate === f.hasOrchestrate,
- features.hasModule === f.hasModule,
- features.hasSync === f.hasSync,
- ];
- score = matches.filter(Boolean).length / matches.length;
- // Keyword overlap bonus
- const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length;
- const kScore = kOverlap / Math.max(features.keywords.length, 1);
- score = score * 0.6 + kScore * 0.4;
- return score > 0.3;
+ if (!dp.observedDuration) return false; // outcomes only, never predictions
+ if (!dp.features) return false;
+ return this._similarity(features, dp.features) > 0.3;
});
}
_computePrediction(similar, features) {
if (similar.length === 0) {
- // Hardcoded defaults
let baseDuration = 15;
let baseTurns = 3;
if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
@@ -17897,9 +17898,10 @@ class PredictorModule {
const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
- const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1;
+ // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
+ const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
+ const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;
- // Apply task complexity multiplier
const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5));
@@ -17911,43 +17913,126 @@ class PredictorModule {
};
}
+ // ─── Prediction ↔ Outcome pairing (v2) ───
+
+ // Task text never leaves the module: only a stable hash is stored/serialised.
+ _hashTask(t) {
+ const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
+ let h = 0x811c9dc5;
+ for (let i = 0; i < s.length; i++) {
+ h ^= s.charCodeAt(i);
+ h = (h * 0x01000193) >>> 0;
+ }
+ return h.toString(16);
+ }
+
+ // Register a prediction to be scored later. Returns the pending record (with .id).
+ notePrediction(task, source) {
+ const features = this._extractFeatures(task || '');
+ const p = this._computePrediction(this._findSimilar(features), features);
+ const rec = {
+ id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+ taskHash: this._hashTask(task),
+ features,
+ predictedDuration: p.duration,
+ predictedTurns: p.toolTurns,
+ confidence: p.confidence,
+ source: source || 'loop',
+ ts: Date.now()
+ };
+ this._pending.push(rec);
+ if (this._pending.length > 50) this._pending = this._pending.slice(-50);
+ this._predictions++;
+ this._debounceSave();
+ return rec;
+ }
+
+ _takePending(features, opts) {
+ const now = Date.now();
+ this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ const taskHash = opts.task ? this._hashTask(opts.task) : null;
+
+ let idx = -1;
+ if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
+ if (idx < 0 && taskHash) {
+ for (let i = this._pending.length - 1; i >= 0; i--) {
+ if (this._pending[i].taskHash === taskHash) { idx = i; break; }
+ }
+ }
+ if (idx < 0) {
+ // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
+ // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
+ let best = -1, bestScore = 0;
+ for (let i = 0; i < this._pending.length; i++) {
+ const cand = this._pending[i];
+ if (cand.source !== 'cli') continue;
+ const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
+ if (shared === 0) continue;
+ const s = this._similarity(features, cand.features);
+ if (s > bestScore) { bestScore = s; best = i; }
+ }
+ if (bestScore >= 0.5) idx = best;
+ }
+ return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
+ }
+
// ─── Learning from Results ───
- recordResult(features, duration, toolTurns, success) {
- this._dataPoints.push({
+ recordResult(features, duration, toolTurns, success, opts) {
+ const o = opts || {};
+ const pred = this._takePending(features, o);
+ const observed = Number(duration) || 0;
+
+ const rec = {
features,
- observedDuration: duration,
+ observedDuration: observed,
observedTurns: toolTurns,
- success,
+ success: !!success,
+ paired: !!pred,
timestamp: new Date().toISOString()
- });
- this._accuracy.total++;
- // Compare with last prediction for this feature set
- const lastPrediction = [...this._dataPoints]
- .reverse()
- .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch);
- if (lastPrediction) {
- const error = Math.abs(lastPrediction.predictedDuration - duration) / duration;
- if (error < 0.3) this._accuracy.correct++;
+ };
+
+ if (pred) {
+ const absErr = Math.abs(pred.predictedDuration - observed);
+ rec.predictedDuration = pred.predictedDuration;
+ rec.predictedTurns = pred.predictedTurns;
+ // capped: one 15s-vs-1s miss must not poison the mean-error readout forever
+ rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5);
+ rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
+ this._accuracy.total++;
+ this._mape += rec.errorPct;
+ this._turnsErr += rec.turnsError;
+ // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
+ if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
}
- this._debounceSave();
+
+ this._dataPoints.push(rec);
+ if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
+ if (pred) this._saveNow(); else this._debounceSave();
+ return rec;
}
// ─── Commands ───
showStatus() {
const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
- const accPct = this._accuracy.total > 0
- ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0)
- : 'N/A';
+ const paired = this._accuracy.total;
+ const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
+ const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
+ const mape = paired > 0
+ ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}`
+ : '';
this.terminal.print('📊 Predictor Status', 'header');
this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info');
this.terminal.print(` Actual executions: ${actualExecutions}`, 'info');
this.terminal.print(` Predictions made: ${this._predictions}`, 'info');
- this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info');
+ this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
+ this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info');
+ this.terminal.print(` Unpaired runs: ${unpaired}`, 'info');
this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info');
+ this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
}
setConfig(args) {
@@ -17968,6 +18053,12 @@ class PredictorModule {
this._MIN_DATA_POINTS = n;
return `min-data-points set to ${n}`;
},
+ 'pair-ttl': () => {
+ const n = parseInt(val);
+ if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
+ this._PAIR_TTL_MS = n * 1000;
+ return `pair-ttl set to ${n}s`;
+ },
};
if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
@@ -17982,10 +18073,28 @@ class PredictorModule {
const fs = this.terminal.fs;
if (!fs) return;
const data = fs.fsReadJSON('/system/metrics/predictor.json');
- if (data) {
- this._dataPoints = data.dataPoints || [];
- this._predictions = data.predictions || 0;
- this._accuracy = data.accuracy || { correct: 0, total: 0 };
+ if (!data) return;
+
+ // Prediction-only points are noise for similarity; keep outcomes.
+ this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
+ this._predictions = data.predictions || 0;
+
+ if (data.v === this._SCHEMA) {
+ const acc = data.accuracy || {};
+ this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
+ this._mape = data.mape || 0;
+ this._turnsErr = data.turnsErr || 0;
+ const cfg = data.config || {};
+ if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
+ if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
+ if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
+ const now = Date.now();
+ this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ } else {
+ // v1 counted every observation in `total` but only some in `correct` → 0%.
+ this._accuracy = { correct: 0, total: 0 };
+ this._mape = 0;
+ this._pending = [];
}
} catch (e) {
console.warn('[Predictor] Load error (fresh start):', e.message);
@@ -17998,9 +18107,18 @@ class PredictorModule {
if (!fs) return;
if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
fs.fsWriteJSON('/system/metrics/predictor.json', {
- dataPoints: this._dataPoints.slice(-500), // keep last 500
+ v: this._SCHEMA,
+ dataPoints: this._dataPoints.slice(-500),
predictions: this._predictions,
- accuracy: this._accuracy
+ accuracy: this._accuracy,
+ mape: this._mape,
+ turnsErr: this._turnsErr,
+ config: {
+ confidenceThreshold: this._CONFIDENCE_THRESHOLD,
+ minDataPoints: this._MIN_DATA_POINTS,
+ pairTtlMs: this._PAIR_TTL_MS
+ },
+ pending: this._pending.slice(-50)
});
} catch (e) {
console.error('[Predictor] Save error:', e.message);
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index abbe9ce..83d073a 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -820,6 +820,7 @@ buildSystemPrompt() {
await new Promise(r => setTimeout(r, 50));
}
this._chatLock = true;
+ const taskStart = Date.now(); // predictor durations exclude chat-lock wait
this.terminal.markDirty();
// Auto-route context based on intent
try {
@@ -910,6 +911,16 @@ buildSystemPrompt() {
}
} catch (e) { /* world model not available */ }
+ // v4.6.56: register a prediction for THIS turn so recordResult can score it by id.
+ let turnPredictionId = null;
+ try {
+ const predictor = this.terminal.moduleManager.modules['PredictorModule'];
+ if (predictor && typeof predictor.notePrediction === 'function') {
+ const tp = predictor.notePrediction(userMessage, 'loop');
+ turnPredictionId = tp && tp.id ? tp.id : null;
+ }
+ } catch (e) { /* predictor is best-effort */ }
+
const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main';
try {
loopResult = await this.agenticLoop(messages, opts);
@@ -937,8 +948,9 @@ buildSystemPrompt() {
const predictor = this.terminal.moduleManager.modules['PredictorModule'];
if (predictor && typeof predictor.recordResult === 'function') {
const features = predictor._extractFeatures(userMessage);
- const duration = (Date.now() - lockStart) / 1000;
- predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled);
+ const duration = (Date.now() - taskStart) / 1000;
+ predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled,
+ { task: userMessage, predictionId: turnPredictionId });
}
} catch (e) { /* predictor learning is best-effort */ }
// Record task in memory graph for direct AI conversations (not just orchestrator)
@@ -3020,16 +3032,26 @@ buildSystemPrompt() {
if (!taskDesc) return JSON.stringify({ error: 'task parameter required' });
var pFeatures = predictor._extractFeatures(taskDesc);
var pSimilar = predictor._findSimilar(pFeatures);
- var pResult = predictor._computePrediction(pSimilar, pFeatures);
+ // v4.6.56: track the prediction so the outcome can score it.
+ var pTracked;
+ if (typeof predictor.notePrediction === 'function') {
+ pTracked = predictor.notePrediction(taskDesc, 'tool');
+ } else {
+ var pFallback = predictor._computePrediction(pSimilar, pFeatures);
+ pTracked = { id: null, predictedDuration: pFallback.duration,
+ predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence };
+ }
return JSON.stringify({
task: taskDesc.substring(0, 80),
- estimatedDuration: pResult.duration + 's',
- estimatedToolTurns: pResult.toolTurns,
- confidence: Math.round(pResult.confidence * 100) + '%',
- similarTasks: pResult.similarCount,
- recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30
- ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's'
- : 'No special action needed'
+ predictionId: pTracked.id || null,
+ estimatedDuration: pTracked.predictedDuration + 's',
+ estimatedToolTurns: pTracked.predictedTurns,
+ confidence: Math.round(pTracked.confidence * 100) + '%',
+ similarTasks: pSimilar.length,
+ recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30
+ ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's'
+ : 'No special action needed',
+ tracked: !!pTracked.id
});
}
case 'world_model_query': {
@@ -4620,9 +4642,10 @@ buildSystemPrompt() {
if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length;
} catch (e) {}
var predictor = mm ? mm.modules['PredictorModule'] : null;
- var predictorAcc = 'N/A';
+ var predictorAcc = 'n/a (no paired runs)';
if (predictor && predictor._accuracy && predictor._accuracy.total > 0) {
- predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%';
+ predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' +
+ predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)';
}
data.health = {
modules: data.modules.loaded + '/' + data.modules.total,
diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js
index ff908cd..8b4f606 100644
--- a/modules/core/module-loader.js
+++ b/modules/core/module-loader.js
@@ -4,7 +4,7 @@
// Core module bundle loaded synchronously in main.html.
// This loader only handles version cache-busting, splash, terminal init, and optional ext modules.
- const LATEST_VER = 'v4.6.55';
+ const LATEST_VER = 'v4.6.56';
const CACHE = '?v=' + LATEST_VER;
const stored = localStorage.getItem('webx11_cache_ver');
if (stored !== LATEST_VER) {
diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js
index 3d868ae..cb1209e 100644
--- a/modules/core/module-predictor.js
+++ b/modules/core/module-predictor.js
@@ -1,23 +1,30 @@
// module-predictor.js — Predictive resource allocation engine
-// Analyzes historical telemetry + memory graph data to predict task duration/turns
+// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
+// real outcome. v1 counted every observed run in the denominator while incrementing the
+// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
// Commands: predict, predictor-status, predictor-config
class PredictorModule {
constructor() {
this.commands = {};
this.terminal = null;
- this._dataPoints = [];
+ this._dataPoints = []; // observed runs — the training set for _findSimilar
+ this._pending = []; // predictions waiting for an outcome (bounded, TTL'd)
this._predictions = 0;
- this._accuracy = { correct: 0, total: 0 };
+ this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only
+ this._mape = 0; // sum of capped |predicted-observed|/observed
+ this._turnsErr = 0; // sum of |predicted turns - observed turns|
this._CONFIDENCE_THRESHOLD = 0.5;
this._MIN_DATA_POINTS = 5;
+ this._PAIR_TTL_MS = 15 * 60 * 1000;
+ this._SCHEMA = 2;
this._loaded = false;
}
getHelpInfo() {
return {
'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
- 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count',
- 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points',
+ 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
+ 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
};
}
@@ -39,33 +46,14 @@ class PredictorModule {
async predict(task) {
if (!task) return this.terminal.print('Usage: predict <task description>', 'error');
- const startTime = performance.now();
-
- // Extract features
const features = this._extractFeatures(task);
-
- // Find similar historical tasks
const similar = this._findSimilar(features);
-
- // Generate prediction
const prediction = this._computePrediction(similar, features);
- // Record this prediction for accuracy tracking
- const duration = ((performance.now() - startTime) / 1000).toFixed(2);
-
- this._predictions++;
- this._dataPoints.push({
- features,
- predictedDuration: prediction.duration,
- predictedTurns: prediction.toolTurns,
- confidence: prediction.confidence,
- similarCount: similar.length,
- timestamp: new Date().toISOString()
- });
-
- this._debounceSave();
+ // Register it as pending so the next matching run can be scored against it.
+ const pending = this.notePrediction(task, 'cli');
+ pending.similarCount = similar.length;
- // Display result
this.terminal.print('📈 Task Prediction', 'header');
this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');
@@ -81,23 +69,22 @@ class PredictorModule {
} else {
this.terminal.print(' ❓ No historical data to base prediction on.', 'warning');
this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
- // Provide hardcoded defaults
this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info');
}
- // If got memory graph, offer recommendations
if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
}
+ this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
return prediction;
}
_extractFeatures(task) {
- const lower = task.toLowerCase();
+ const lower = String(task || '').toLowerCase();
return {
- length: task.length,
- wordCount: task.split(/\s+/).length,
+ length: String(task || '').length,
+ wordCount: String(task || '').split(/\s+/).length,
hasSearch: /\b(search|find|look)\b/.test(lower),
hasWrite: /\b(write|create|save|edit)\b/.test(lower),
hasDelete: /\b(delete|remove|rm)\b/.test(lower),
@@ -107,42 +94,33 @@ class PredictorModule {
hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
- complexity: Math.min(task.length / 50, 10), // 0-10 scale
+ complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
};
}
+ // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
+ _similarity(a, b) {
+ if (!a || !b) return 0;
+ const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
+ 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
+ const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
+ const kw = a.keywords || [];
+ const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
+ const kScore = overlap / Math.max(kw.length, 1);
+ return matches * 0.6 + kScore * 0.4;
+ }
+
_findSimilar(features) {
return this._dataPoints.filter(dp => {
- // Skip predictions (only use actual executions with observed results)
- if (!dp.observedDuration) return false;
- const f = dp.features;
- if (!f) return false;
- // Feature similarity scoring
- let score = 0;
- const matches = [
- features.hasSearch === f.hasSearch,
- features.hasWrite === f.hasWrite,
- features.hasDelete === f.hasDelete,
- features.hasInstall === f.hasInstall,
- features.hasRead === f.hasRead,
- features.hasWeb === f.hasWeb,
- features.hasOrchestrate === f.hasOrchestrate,
- features.hasModule === f.hasModule,
- features.hasSync === f.hasSync,
- ];
- score = matches.filter(Boolean).length / matches.length;
- // Keyword overlap bonus
- const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length;
- const kScore = kOverlap / Math.max(features.keywords.length, 1);
- score = score * 0.6 + kScore * 0.4;
- return score > 0.3;
+ if (!dp.observedDuration) return false; // outcomes only, never predictions
+ if (!dp.features) return false;
+ return this._similarity(features, dp.features) > 0.3;
});
}
_computePrediction(similar, features) {
if (similar.length === 0) {
- // Hardcoded defaults
let baseDuration = 15;
let baseTurns = 3;
if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
@@ -160,9 +138,10 @@ class PredictorModule {
const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
- const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1;
+ // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
+ const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
+ const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;
- // Apply task complexity multiplier
const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5));
@@ -174,43 +153,126 @@ class PredictorModule {
};
}
+ // ─── Prediction ↔ Outcome pairing (v2) ───
+
+ // Task text never leaves the module: only a stable hash is stored/serialised.
+ _hashTask(t) {
+ const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
+ let h = 0x811c9dc5;
+ for (let i = 0; i < s.length; i++) {
+ h ^= s.charCodeAt(i);
+ h = (h * 0x01000193) >>> 0;
+ }
+ return h.toString(16);
+ }
+
+ // Register a prediction to be scored later. Returns the pending record (with .id).
+ notePrediction(task, source) {
+ const features = this._extractFeatures(task || '');
+ const p = this._computePrediction(this._findSimilar(features), features);
+ const rec = {
+ id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+ taskHash: this._hashTask(task),
+ features,
+ predictedDuration: p.duration,
+ predictedTurns: p.toolTurns,
+ confidence: p.confidence,
+ source: source || 'loop',
+ ts: Date.now()
+ };
+ this._pending.push(rec);
+ if (this._pending.length > 50) this._pending = this._pending.slice(-50);
+ this._predictions++;
+ this._debounceSave();
+ return rec;
+ }
+
+ _takePending(features, opts) {
+ const now = Date.now();
+ this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ const taskHash = opts.task ? this._hashTask(opts.task) : null;
+
+ let idx = -1;
+ if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
+ if (idx < 0 && taskHash) {
+ for (let i = this._pending.length - 1; i >= 0; i--) {
+ if (this._pending[i].taskHash === taskHash) { idx = i; break; }
+ }
+ }
+ if (idx < 0) {
+ // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
+ // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
+ let best = -1, bestScore = 0;
+ for (let i = 0; i < this._pending.length; i++) {
+ const cand = this._pending[i];
+ if (cand.source !== 'cli') continue;
+ const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
+ if (shared === 0) continue;
+ const s = this._similarity(features, cand.features);
+ if (s > bestScore) { bestScore = s; best = i; }
+ }
+ if (bestScore >= 0.5) idx = best;
+ }
+ return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
+ }
+
// ─── Learning from Results ───
- recordResult(features, duration, toolTurns, success) {
- this._dataPoints.push({
+ recordResult(features, duration, toolTurns, success, opts) {
+ const o = opts || {};
+ const pred = this._takePending(features, o);
+ const observed = Number(duration) || 0;
+
+ const rec = {
features,
- observedDuration: duration,
+ observedDuration: observed,
observedTurns: toolTurns,
- success,
+ success: !!success,
+ paired: !!pred,
timestamp: new Date().toISOString()
- });
- this._accuracy.total++;
- // Compare with last prediction for this feature set
- const lastPrediction = [...this._dataPoints]
- .reverse()
- .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch);
- if (lastPrediction) {
- const error = Math.abs(lastPrediction.predictedDuration - duration) / duration;
- if (error < 0.3) this._accuracy.correct++;
+ };
+
+ if (pred) {
+ const absErr = Math.abs(pred.predictedDuration - observed);
+ rec.predictedDuration = pred.predictedDuration;
+ rec.predictedTurns = pred.predictedTurns;
+ // capped: one 15s-vs-1s miss must not poison the mean-error readout forever
+ rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5);
+ rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
+ this._accuracy.total++;
+ this._mape += rec.errorPct;
+ this._turnsErr += rec.turnsError;
+ // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
+ if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
}
- this._debounceSave();
+
+ this._dataPoints.push(rec);
+ if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
+ if (pred) this._saveNow(); else this._debounceSave();
+ return rec;
}
// ─── Commands ───
showStatus() {
const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
- const accPct = this._accuracy.total > 0
- ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0)
- : 'N/A';
+ const paired = this._accuracy.total;
+ const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
+ const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
+ const mape = paired > 0
+ ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}`
+ : '';
this.terminal.print('📊 Predictor Status', 'header');
this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info');
this.terminal.print(` Actual executions: ${actualExecutions}`, 'info');
this.terminal.print(` Predictions made: ${this._predictions}`, 'info');
- this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info');
+ this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
+ this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info');
+ this.terminal.print(` Unpaired runs: ${unpaired}`, 'info');
this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info');
+ this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
}
setConfig(args) {
@@ -231,6 +293,12 @@ class PredictorModule {
this._MIN_DATA_POINTS = n;
return `min-data-points set to ${n}`;
},
+ 'pair-ttl': () => {
+ const n = parseInt(val);
+ if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
+ this._PAIR_TTL_MS = n * 1000;
+ return `pair-ttl set to ${n}s`;
+ },
};
if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
@@ -245,10 +313,28 @@ class PredictorModule {
const fs = this.terminal.fs;
if (!fs) return;
const data = fs.fsReadJSON('/system/metrics/predictor.json');
- if (data) {
- this._dataPoints = data.dataPoints || [];
- this._predictions = data.predictions || 0;
- this._accuracy = data.accuracy || { correct: 0, total: 0 };
+ if (!data) return;
+
+ // Prediction-only points are noise for similarity; keep outcomes.
+ this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
+ this._predictions = data.predictions || 0;
+
+ if (data.v === this._SCHEMA) {
+ const acc = data.accuracy || {};
+ this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
+ this._mape = data.mape || 0;
+ this._turnsErr = data.turnsErr || 0;
+ const cfg = data.config || {};
+ if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
+ if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
+ if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
+ const now = Date.now();
+ this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ } else {
+ // v1 counted every observation in `total` but only some in `correct` → 0%.
+ this._accuracy = { correct: 0, total: 0 };
+ this._mape = 0;
+ this._pending = [];
}
} catch (e) {
console.warn('[Predictor] Load error (fresh start):', e.message);
@@ -261,9 +347,18 @@ class PredictorModule {
if (!fs) return;
if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
fs.fsWriteJSON('/system/metrics/predictor.json', {
- dataPoints: this._dataPoints.slice(-500), // keep last 500
+ v: this._SCHEMA,
+ dataPoints: this._dataPoints.slice(-500),
predictions: this._predictions,
- accuracy: this._accuracy
+ accuracy: this._accuracy,
+ mape: this._mape,
+ turnsErr: this._turnsErr,
+ config: {
+ confidenceThreshold: this._CONFIDENCE_THRESHOLD,
+ minDataPoints: this._MIN_DATA_POINTS,
+ pairTtlMs: this._PAIR_TTL_MS
+ },
+ pending: this._pending.slice(-50)
});
} catch (e) {
console.error('[Predictor] Save error:', e.message);
@@ -279,4 +374,4 @@ class PredictorModule {
// Required for module system
if (typeof window !== 'undefined') {
window.PredictorModule = PredictorModule;
-} \ No newline at end of file
+}
diff --git a/modules/test/test-predictor-accuracy.js b/modules/test/test-predictor-accuracy.js
new file mode 100644
index 0000000..1a79167
--- /dev/null
+++ b/modules/test/test-predictor-accuracy.js
@@ -0,0 +1,209 @@
+// modules/test/test-predictor-accuracy.js — run: node modules/test/test-predictor-accuracy.js
+// Proves the predictor's accuracy metric measures PAIRED predictions only.
+// RED on v1 code: unpaired runs inflate the denominator, so accuracy is always 0%.
+const fs = require('fs'), path = require('path');
+const src = fs.readFileSync(path.join(__dirname, '../core/module-predictor.js'), 'utf8');
+const PredictorModule = new Function(src + '\nreturn PredictorModule;')();
+
+function memFs() {
+ const files = {};
+ const dirs = new Set(['/']);
+ return {
+ readFile: p => (p in files ? files[p] : null),
+ writeFile: (p, c) => { files[p] = c; return true; },
+ fsReadJSON: p => { try { return JSON.parse(files[p]); } catch (e) { return null; } },
+ fsWriteJSON: (p, o) => { files[p] = JSON.stringify(o); return true; },
+ readDir: p => (dirs.has(p) ? [] : null),
+ mkdirRecursive: p => { dirs.add(p); return true; },
+ mkdir: p => { dirs.add(p); return true; },
+ };
+}
+
+function mk(persist) {
+ const out = [];
+ const term = { print: m => out.push(String(m)), fs: memFs() };
+ const p = new PredictorModule();
+ p.terminal = term;
+ if (!persist) { p._saveNow = () => {}; p._debounceSave = () => {}; } // isolate logic from timers
+ return { p, out, term };
+}
+
+let fails = 0, passes = 0;
+function ok(name, cond, detail) {
+ if (cond) { passes++; return; }
+ fails++;
+ console.log('FAIL ' + name + (detail ? ' — ' + detail : ''));
+}
+
+// A. runs with no prediction must not enter the denominator
+{
+ const { p } = mk();
+ p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
+ p.recordResult(p._extractFeatures('write a report'), 40, 3, true);
+ ok('A1 total stays 0 without predictions', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+ ok('A2 observed runs are still recorded', p._dataPoints.filter(d => d.observedDuration).length === 2);
+}
+
+// B. a prediction paired with an on-time outcome counts as correct
+{
+ const { p } = mk();
+ const pred = p.notePrediction('write a report file', 'test');
+ ok('B0 notePrediction returns a pending prediction', !!pred && typeof pred.id === 'string');
+ p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 1.1, 4, true, { task: 'write a report file' });
+ ok('B1 paired once', p._accuracy.total === 1, 'total=' + p._accuracy.total);
+ ok('B2 on-time prediction scored correct', p._accuracy.correct === 1, 'correct=' + p._accuracy.correct);
+ ok('B3 pending consumed', p._pending.length === 0, 'pending=' + p._pending.length);
+}
+
+// C. an outcome >30% off is paired but wrong
+{
+ const { p } = mk();
+ const pred = p.notePrediction('write a report file', 'test');
+ p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 3 + 5, 4, true, { task: 'write a report file' });
+ ok('C1 paired', p._accuracy.total === 1);
+ ok('C2 not correct', p._accuracy.correct === 0);
+}
+
+// D. an unrelated run must not consume the pending prediction
+{
+ const { p } = mk();
+ p.notePrediction('search the web for ollama docs', 'test');
+ p.recordResult(p._extractFeatures('delete the tmp folder'), 9, 1, true, { task: 'delete the tmp folder' });
+ ok('D1 unrelated run unpaired', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+ ok('D2 prediction still pending', p._pending.length === 1, 'pending=' + p._pending.length);
+}
+
+// E. expired predictions are dropped, not paired
+{
+ const { p } = mk();
+ const pred = p.notePrediction('read the logs', 'test');
+ pred.ts = Date.now() - (p._PAIR_TTL_MS + 1000);
+ p.recordResult(p._extractFeatures('read the logs'), 20, 2, true, { task: 'read the logs' });
+ ok('E1 expired prediction not paired', p._accuracy.total === 0);
+}
+
+// F. the status line never renders "N/A%"
+{
+ const { p, out } = mk();
+ p.showStatus();
+ const line = out.find(l => l.includes('Accuracy')) || '';
+ ok('F1 empty state says n/a', /\bn\/a\b/i.test(line) && !line.includes('N/A%'), line.trim());
+ ok('F2 paired count is shown', /paired/i.test(line), line.trim());
+}
+
+// G. _MIN_DATA_POINTS actually shapes confidence (it is a dead knob in v1)
+{
+ const { p } = mk();
+ for (let i = 0; i < 3; i++) p.recordResult(p._extractFeatures('read the logs'), 20, 2, true);
+ const f = p._extractFeatures('read the logs');
+ p._MIN_DATA_POINTS = 10;
+ const a = p._computePrediction(p._findSimilar(f), f).confidence;
+ p._MIN_DATA_POINTS = 3;
+ const b = p._computePrediction(p._findSimilar(f), f).confidence;
+ ok('G1 lowering min-data-points raises confidence', b > a, a.toFixed(3) + ' -> ' + b.toFixed(3));
+}
+
+// H. v1 payload migration: junk accuracy discarded, observed points kept
+{
+ const { p, term } = mk();
+ term.fs.fsWriteJSON('/system/metrics/predictor.json', {
+ dataPoints: [{ features: { hasRead: true }, observedDuration: 30, observedTurns: 3 },
+ { features: { hasRead: true }, predictedDuration: 30 }],
+ predictions: 7,
+ accuracy: { correct: 0, total: 6 }
+ });
+ p._loadFromVFS();
+ ok('H1 v1 accuracy discarded', p._accuracy.total === 0 && p._accuracy.correct === 0, JSON.stringify(p._accuracy));
+ ok('H2 observed points kept, prediction-only points dropped', p._dataPoints.length === 1, 'n=' + p._dataPoints.length);
+ ok('H3 predictions counter kept', p._predictions === 7, 'n=' + p._predictions);
+}
+
+// I. save → load round-trip keeps paired accuracy; stale pending is dropped
+{
+ const { p, term } = mk(true);
+ const pred = p.notePrediction('read the logs', 'test');
+ p.recordResult(p._extractFeatures('read the logs'), pred.predictedDuration, 2, true, { task: 'read the logs' });
+ p._saveNow();
+ const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
+ ok('I1 accuracy survives reload', q._accuracy.total === 1 && q._accuracy.correct === 1, JSON.stringify(q._accuracy));
+ const stale = p.notePrediction('read the logs again', 'test');
+ stale.ts = Date.now() - (p._PAIR_TTL_MS + 60000); // expired while pending
+ p._saveNow();
+ const s = new PredictorModule(); s.terminal = term; s._loadFromVFS();
+ ok('I2 expired pending dropped on load', s._pending.length === 0, 'pending=' + s._pending.length);
+}
+
+
+// J. 3 s absolute floor: a small miss on a short task still counts
+{
+ const { p } = mk();
+ p.notePrediction('read a tiny file', 'test');
+ p._pending[0].predictedDuration = 5; // integer seconds vs fractional observed
+ p.recordResult(p._extractFeatures('read a tiny file'), 3.8, 1, true, { task: 'read a tiny file' });
+ ok('J1 1.2 s miss on a 3.8 s task is inside the 3 s floor', p._accuracy.correct === 1, JSON.stringify(p._accuracy));
+}
+
+// K. a wild miss is capped, so mean error stays readable
+{
+ const { p } = mk();
+ p.notePrediction('read the logs', 'test');
+ p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' });
+ const rec = p._dataPoints[p._dataPoints.length - 1];
+ ok('K1 errorPct capped at 500%', rec.errorPct <= 5, 'errorPct=' + rec.errorPct);
+}
+
+// L. a tool/loop prediction is never consumed by the similarity fallback
+{
+ const { p } = mk();
+ p.notePrediction('write a file report', 'tool');
+ p.recordResult(p._extractFeatures('write a file report somewhere else'), 40, 3, true); // no task, no id
+ ok('L1 shape-alike run does not consume a tool prediction', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+}
+
+// M. the cli fallback needs at least one shared keyword
+{
+ const { p } = mk();
+ p.notePrediction('alpha bravo charlie', 'cli');
+ p.recordResult(p._extractFeatures('delta echo foxtrot'), 30, 3, true);
+ ok('M1 zero keyword overlap is not a match', p._accuracy.total === 0, 'total=' + p._accuracy.total);
+}
+
+// N. tuning survives a reload
+{
+ const { p, term } = mk(true);
+ p._MIN_DATA_POINTS = 9;
+ p._PAIR_TTL_MS = 600000;
+ p._saveNow();
+ const q = new PredictorModule(); q.terminal = term; q._loadFromVFS();
+ ok('N1 config persisted', q._MIN_DATA_POINTS === 9 && q._PAIR_TTL_MS === 600000,
+ q._MIN_DATA_POINTS + '/' + q._PAIR_TTL_MS);
+}
+
+// O. "unpaired" counts scored-and-missed runs, not legacy observations
+{
+ const { p, out, term } = mk();
+ term.fs.fsWriteJSON('/system/metrics/predictor.json', {
+ dataPoints: [{ features: { hasRead: true }, observedDuration: 30 }], // v1 point, no `paired` flag
+ accuracy: { correct: 0, total: 6 }
+ });
+ p._loadFromVFS();
+ p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); // unpaired v2 run
+ out.length = 0; p.showStatus();
+ const line = out.find(l => l.includes('Unpaired runs')) || '';
+ ok('O1 only paired:false records count as unpaired', /Unpaired runs: 1\b/.test(line), line.trim());
+}
+
+
+// P. the id threaded by the AI loop wins over text/similarity guessing
+{
+ const { p } = mk();
+ p.notePrediction('something else entirely', 'tool'); // decoy, pushed first
+ const loopPred = p.notePrediction('read the logs', 'loop');
+ p.recordResult(p._extractFeatures('read the logs'), loopPred.predictedDuration, 2, true,
+ { task: 'read the logs', predictionId: loopPred.id });
+ ok('P1 the threaded id is the one consumed', p._accuracy.total === 1 && p._pending.length === 1,
+ 'total=' + p._accuracy.total + ' pending=' + p._pending.length);
+}
+
+console.log(passes + '/' + (passes + fails) + ' PASS');
+process.exit(fails ? 1 : 0);