diff options
| -rw-r--r-- | downloads/system-info.html | 262 | ||||
| -rw-r--r-- | help.html | 10 | ||||
| -rw-r--r-- | main.html | 6 | ||||
| -rw-r--r-- | modules/core/core-bundle.js | 308 | ||||
| -rw-r--r-- | modules/core/module-aiassist.js | 47 | ||||
| -rw-r--r-- | modules/core/module-loader.js | 2 | ||||
| -rw-r--r-- | modules/core/module-predictor.js | 263 | ||||
| -rw-r--r-- | modules/test/test-predictor-accuracy.js | 209 |
8 files changed, 912 insertions, 195 deletions
diff --git a/downloads/system-info.html b/downloads/system-info.html new file mode 100644 index 0000000..4991292 --- /dev/null +++ b/downloads/system-info.html @@ -0,0 +1,262 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover"> + <meta name="theme-color" content="#0f1420"> + <meta name="color-scheme" content="dark"> + <meta name="description" content="newOS4 v4.6.27 system snapshot — modules, skills, agents, token usage, and health metrics."> + <meta property="og:title" content="newOS4 System Info"> + <meta property="og:description" content="System snapshot — 46 modules, 39 skills, token usage, agents, and health metrics."> + <meta property="og:type" content="website"> + <title>newOS4 System Info</title> + <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Ctext y='13' font-size='13'%3E%E2%9A%99%EF%B8%8F%3C/text%3E%3C/svg%3E"> + <style> + * { box-sizing: border-box; margin: 0; padding: 0; } + html { -webkit-text-size-adjust: 100%; text-size-adjust: 100%; } + body { + font-family: system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif; + font-size: 16px; + background: #0f1420; + color: #e4e4e4; + min-height: 100vh; + padding: clamp(0.75rem, 3vw, 2rem); + padding-left: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-left)); + padding-right: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-right)); + padding-bottom: max(clamp(0.75rem, 3vw, 2rem), env(safe-area-inset-bottom)); + overflow-wrap: break-word; + -webkit-tap-highlight-color: transparent; + } + .container { max-width: 1200px; margin: 0 auto; } + h1 { text-align: center; color: #00d9ff; font-size: clamp(1.5rem, 6vw, 2.5rem); margin-bottom: 0.35rem; text-shadow: 0 0 20px rgba(0,217,255,0.5); } + .subtitle { text-align: center; color: #8b93a7; font-size: clamp(0.8rem, 3vw, 0.95rem); margin-bottom: clamp(1rem, 4vw, 2rem); } + .grid { display: grid; grid-template-columns: 1fr; gap: clamp(0.75rem, 3vw, 1.5rem); } + @media (min-width: 640px) { .grid { grid-template-columns: repeat(2, 1fr); } } + @media (min-width: 1024px) { .grid { grid-template-columns: repeat(3, 1fr); } } + .card { + background: #151b2e; border-radius: 12px; + padding: clamp(0.9rem, 4vw, 1.5rem); + border: 1px solid rgba(255,255,255,0.08); + min-width: 0; + } + .card h2 { color: #00d9ff; font-size: clamp(0.95rem, 4vw, 1.2rem); margin-bottom: 0.85rem; border-bottom: 1px solid rgba(0,217,255,0.3); padding-bottom: 0.5rem; } + dl.stat-list { display: block; } + dl.stat-list > div { display: flex; justify-content: space-between; align-items: center; gap: 0.75rem; padding: 0.55rem 0; min-height: 2.4rem; border-bottom: 1px solid rgba(255,255,255,0.05); } + dl.stat-list > div:last-child { border-bottom: none; } + dt.stat-label { color: #a8b0c4; min-width: 0; } + dd.stat-value { font-weight: 600; color: #fff; text-align: right; min-width: 0; } + .stat-value.ok { color: #00ff88; } + .stat-value.warn { color: #ffaa00; } + .stat-value.error { color: #ff4466; } + .note { margin-top: 0.85rem; color: #8b93a7; font-size: 0.8rem; line-height: 1.45; } + .module-list { display: flex; flex-wrap: wrap; gap: 0.4rem; margin-top: 0.5rem; } + .tag { background: rgba(0,217,255,0.16); padding: 0.3rem 0.6rem; border-radius: 20px; font-size: 0.72rem; line-height: 1.4; color: #00d9ff; } + .tag.core { background: rgba(0,255,136,0.14); color: #00ff88; } + .tag.ext { background: rgba(255,170,0,0.14); color: #ffaa00; } + .tag.modified { background: rgba(255,68,102,0.16); color: #ff4466; border: 1px solid #ff4466; } + .model-item { margin: 0.65rem 0; min-width: 0; } + .model-top { display: flex; justify-content: space-between; gap: 0.6rem; font-size: 0.78rem; margin-bottom: 0.3rem; } + .model-name { color: #b8c0d4; min-width: 0; overflow-wrap: break-word; } + .model-count { color: #8b93a7; flex-shrink: 0; } + .model-track { background: rgba(255,255,255,0.08); border-radius: 8px; overflow: hidden; } + .model-fill { height: 26px; border-radius: 8px; display: flex; align-items: center; justify-content: flex-end; padding: 0 0.6rem; font-size: 0.75rem; font-weight: 600; color: #fff; min-width: 2.6rem; } + .model-fill.c1 { background: #0088cc; } + .model-fill.c2 { background: #7a3db8; } + .model-fill.c3 { background: #009977; } + .model-fill.c4 { background: #cc5500; } + .model-fill.c5 { background: #8a6d1f; } + .model-fill.c6 { background: #a83d6e; } + .agent-row { display: flex; align-items: center; gap: 0.55rem; padding: 0.55rem 0; min-height: 2.4rem; font-size: 0.85rem; border-bottom: 1px solid rgba(255,255,255,0.05); } + .agent-row:last-child { border-bottom: none; } + .agent-icon { width: 8px; height: 8px; border-radius: 50%; background: #6c7a95; flex-shrink: 0; } + .agent-name { min-width: 0; overflow-wrap: break-word; } + .badge { display: inline-block; padding: 0.2rem 0.5rem; border-radius: 4px; font-size: 0.68rem; font-weight: 600; flex-shrink: 0; } + .badge-auto { background: rgba(0,217,255,0.16); color: #00d9ff; } + .badge-arch { background: rgba(255,170,0,0.16); color: #ffaa00; } + .badge-search { background: rgba(0,255,136,0.16); color: #00ff88; } + .badge-sleep { background: rgba(120,120,140,0.2); color: #a8b0c4; margin-left: auto; } + .timestamp { text-align: center; color: #8b93a7; font-size: 0.78rem; margin-top: clamp(1rem, 4vw, 2rem); } + @media (prefers-reduced-motion: reduce) { h1 { text-shadow: none; } } + @media (min-width: 1024px) { .card { transition: border-color 0.3s ease; } .card:hover { border-color: rgba(0,217,255,0.35); } } + @media print { + body { background: #fff; color: #111; } + h1 { color: #005f80; text-shadow: none; } + .card { background: #f6f6f6; border: 1px solid #ccc; break-inside: avoid; } + .card h2 { color: #005f80; } + .stat-label, .model-name, .model-count, .note, .timestamp, .subtitle { color: #555; } + .stat-value { color: #000; } + .stat-value.ok { color: #0a7a3d; } + .stat-value.warn { color: #9c6200; } + .stat-value.error { color: #b3001e; } + .model-track { background: #ddd; } + .model-fill { color: #fff; -webkit-print-color-adjust: exact; print-color-adjust: exact; } + } + </style> +</head> +<body> + <script type="application/json" id="system-data"> +{ + "version": "v4.6.27", + "vfs": { "files": 467, "dirs": 181, "size": "1431.7KB" }, + "memory": { "heap": "16.3MB", "graphNodes": 41, "graphEdges": 49 }, + "modules": { "loaded": 46, "core": 23, "ext": 24, "available": 47, "modified": ["OrchestratorModule"] }, + "skills": 39, + "agents": [ + { "id": "agent-mvdsl9qg-5sq9-auto-learner", "role": "auto-learner", "status": "sleeping", "caps": 4, "badge": "auto" }, + { "id": "agent-mvdsl9r2-bkm1-kernel-architect", "role": "kernel-architect", "status": "sleeping", "caps": 4, "badge": "arch" }, + { "id": "agent-mvdsl9rl-bq7d-searcher", "role": "searcher", "status": "sleeping", "caps": 3, "badge": "search" } + ], + "ollama": { "connected": true, "model": "glm-5.3:cloud" }, + "usage": { + "total": 500, "prompt": 10006288, "completion": 956373, + "avgLatency": 18610, "timeSpanHours": 22.1, "provider": "Ollama", + "byModel": { + "glm-5.3:cloud": { "requests": 203, "tokens": 7696218 }, + "minimax-m2.7:cloud": { "requests": 218, "tokens": 2114960 }, + "kimi-k3:cloud": { "requests": 47, "tokens": 844472 }, + "gemma4:cloud": { "requests": 18, "tokens": 176391 }, + "gpt-oss:120b-cloud": { "requests": 7, "tokens": 72567 }, + "kimi-k2.7-code:cloud": { "requests": 7, "tokens": 58053 } + } + }, + "latency": [ + { "name": "web_fetch", "calls": 4, "errors": 2 }, + { "name": "create_skill", "calls": 2, "errors": 0 }, + { "name": "list_skills", "calls": 2, "errors": 0 }, + { "name": "write_file", "calls": 3, "errors": 0 }, + { "name": "read_file", "calls": 2, "errors": 0 }, + { "name": "world_model_query", "calls": 1, "errors": 0 }, + { "name": "get_status", "calls": 1, "errors": 0 }, + { "name": "get_usage_stats", "calls": 1, "errors": 0 }, + { "name": "get_module_status", "calls": 1, "errors": 0 }, + { "name": "list_agents", "calls": 1, "errors": 0 } + ], + "health": { "online": true, "errors": 7, "predictorAccuracy": "n/a (no paired runs)" }, + "timestamp": 1792327737761 +} + </script> + + <div class="container"> + <h1>⚙️ newOS4 System Info</h1> + <p class="subtitle">v4.6.27 • System Snapshot • Mobile-Optimized</p> + + <div class="grid"> + <!-- System Overview --> + <div class="card"> + <h2>🖥️ System Overview</h2> + <dl class="stat-list"> + <div><dt class="stat-label">Version</dt><dd class="stat-value">v4.6.27</dd></div> + <div><dt class="stat-label">VFS Files</dt><dd class="stat-value ok">467</dd></div> + <div><dt class="stat-label">VFS Dirs / Size</dt><dd class="stat-value">181 · 1.4 MB</dd></div> + <div><dt class="stat-label">Memory Heap</dt><dd class="stat-value ok">16.3 MB</dd></div> + <div><dt class="stat-label">Graph Nodes / Edges</dt><dd class="stat-value">41 / 49</dd></div> + <div><dt class="stat-label">Ollama</dt><dd class="stat-value ok">Connected</dd></div> + <div><dt class="stat-label">Active Model</dt><dd class="stat-value">glm-5.3:cloud</dd></div> + <div><dt class="stat-label">System Errors</dt><dd class="stat-value error">7 ⚠️</dd></div> + </dl> + </div> + + <!-- Modules --> + <div class="card"> + <h2>📦 Modules</h2> + <dl class="stat-list"> + <div><dt class="stat-label">Loaded</dt><dd class="stat-value ok">46</dd></div> + <div><dt class="stat-label">Core</dt><dd class="stat-value">23</dd></div> + <div><dt class="stat-label">Extensions</dt><dd class="stat-value">24 available</dd></div> + <div><dt class="stat-label">Modified</dt><dd class="stat-value warn">1 (Orchestrator)</dd></div> + </dl> + <p class="note">Core: filesystem, AI, agents, memory graph, neural VFS, orchestration. Extensions: editors, shell, WebDAV, cron, mindmaps, Three, Course, Media, JSConsole, and more.</p> + </div> + + <!-- Token Usage --> + <div class="card"> + <h2>📊 Token Usage</h2> + <dl class="stat-list"> + <div><dt class="stat-label">Total Requests</dt><dd class="stat-value">500</dd></div> + <div><dt class="stat-label">Total Tokens</dt><dd class="stat-value ok">10.96M</dd></div> + <div><dt class="stat-label">Prompt / Completion</dt><dd class="stat-value">10.01M / 956K</dd></div> + <div><dt class="stat-label">Time Span</dt><dd class="stat-value">~22 hours</dd></div> + <div><dt class="stat-label">Provider</dt><dd class="stat-value">Ollama (100%)</dd></div> + <div><dt class="stat-label">Avg Latency</dt><dd class="stat-value">18.6s</dd></div> + </dl> + <div class="model-item"> + <div class="model-top"><span class="model-name">glm-5.3:cloud · 203 req</span><span class="model-count">7.70M</span></div> + <div class="model-track" role="img" aria-label="glm-5.3:cloud usage, 70.2% share"><div class="model-fill c1" style="width:100%">70.2%</div></div> + </div> + <div class="model-item"> + <div class="model-top"><span class="model-name">minimax-m2.7:cloud · 218 req</span><span class="model-count">2.11M</span></div> + <div class="model-track" role="img" aria-label="minimax-m2.7:cloud usage, 19.3% share"><div class="model-fill c2" style="width:27.5%">19.3%</div></div> + </div> + <div class="model-item"> + <div class="model-top"><span class="model-name">kimi-k3:cloud · 47 req</span><span class="model-count">844K</span></div> + <div class="model-track" role="img" aria-label="kimi-k3:cloud usage, 7.7% share"><div class="model-fill c3" style="width:10.9%">7.7%</div></div> + </div> + <div class="model-item"> + <div class="model-top"><span class="model-name">gemma4:cloud · 18 req</span><span class="model-count">176K</span></div> + <div class="model-track" role="img" aria-label="gemma4:cloud usage, 1.6% share"><div class="model-fill c4" style="width:2.3%">1.6%</div></div> + </div> + <div class="model-item"> + <div class="model-top"><span class="model-name">gpt-oss:120b-cloud · 7 req</span><span class="model-count">72.6K</span></div> + <div class="model-track" role="img" aria-label="gpt-oss:120b-cloud usage, 0.7% share"><div class="model-fill c5" style="width:0.94%">0.7%</div></div> + </div> + <div class="model-item"> + <div class="model-top"><span class="model-name">kimi-k2.7-code:cloud · 7 req</span><span class="model-count">58.1K</span></div> + <div class="model-track" role="img" aria-label="kimi-k2.7-code:cloud usage, 0.5% share"><div class="model-fill c6" style="width:0.75%">0.5%</div></div> + </div> + </div> + + <!-- Skills --> + <div class="card"> + <h2>🎓 Skills</h2> + <dl class="stat-list"> + <div><dt class="stat-label">Total Skills</dt><dd class="stat-value ok">39</dd></div> + <div><dt class="stat-label">Recently Added</dt><dd class="stat-value">generate-info-page</dd></div> + </dl> + <p class="note">Categories: system, learning, dev, process, business. Includes newOS4 interactive course, memgraph visualization, mobile HTML optimization, and more.</p> + </div> + + <!-- Agents --> + <div class="card"> + <h2>🤖 Agents</h2> + <dl class="stat-list"> + <div><dt class="stat-label">Total Agents</dt><dd class="stat-value">3</dd></div> + <div><dt class="stat-label">Status</dt><dd class="stat-value">All sleeping 💤</dd></div> + </dl> + <div class="agent-row"> + <span class="agent-icon" aria-hidden="true"></span> + <span class="agent-name">auto-learner</span> + <span class="badge badge-auto">AUTO</span> + <span class="badge badge-sleep">sleeping</span> + </div> + <div class="agent-row"> + <span class="agent-icon" aria-hidden="true"></span> + <span class="agent-name">kernel-architect</span> + <span class="badge badge-arch">ARCH</span> + <span class="badge badge-sleep">sleeping</span> + </div> + <div class="agent-row"> + <span class="agent-icon" aria-hidden="true"></span> + <span class="agent-name">searcher</span> + <span class="badge badge-search">SEARCH</span> + <span class="badge badge-sleep">sleeping</span> + </div> + </div> + + <!-- Tool Latency & Health --> + <div class="card"> + <h2>⚡ Tools & Health</h2> + <dl class="stat-list"> + <div><dt class="stat-label">System Online</dt><dd class="stat-value ok">Yes</dd></div> + <div><dt class="stat-label">System Errors</dt><dd class="stat-value error">7 ⚠️</dd></div> + <div><dt class="stat-label">Predictor Accuracy</dt><dd class="stat-value warn">0%</dd></div> + <div><dt class="stat-label">web_fetch Errors</dt><dd class="stat-value warn">2 of 4 calls</dd></div> + <div><dt class="stat-label">Auto-Lessons</dt><dd class="stat-value">3 saved</dd></div> + </dl> + <p class="note">web_fetch shows 2 errors — monitor if it persists. Auto-lessons flag anomalies (high token usage, near turn limits) for self-improvement.</p> + </div> + </div> + + <p class="timestamp">Snapshot: <time datetime="2026-10-18T12:48:57Z">Sat Oct 18 2026, 12:48 UTC</time> · Regenerate via <code>generate-info-page</code> skill</p> + </div> +</body> +</html>
\ No newline at end of file @@ -1660,6 +1660,16 @@ listmods</pre> <h2>📋 Changelog</h2> <div class="version-item"> + <div class="v-header"><span class="v-num">v4.6.56 — October 2026</span><span class="v-date">2026-10-23</span></div> + <ul> + <li><strong>Predictor accuracy was structurally 0%</strong> — the denominator counted every observed run while the numerator only moved when a <em>stored prediction</em> matched, and predictions were stored only by the manual <code>predict</code> command. Unpaired runs no longer enter the accuracy denominator; the AI loop now registers a prediction per turn (<code>notePrediction</code>) and <code>predict_task</code> returns a tracked <code>predictionId</code> instead of discarding its estimate.</li> + <li><strong>Paired scoring</strong> — a prediction is consumed by threaded id → task hash → cli-only feature similarity ≥ 0.5, within a 15-minute TTL (<code>predictor-config pair-ttl <seconds></code>). A run counts as correct within 30% <em>or</em> 3 seconds of the predicted duration; <code>predictor-status</code> shows paired accuracy, mean error, turns error, pending predictions and unpaired runs.</li> + <li><strong>Schema v2</strong> — stored payloads from v1 are migrated on load: observed runs and the prediction counter are kept, the meaningless <code>correct/total</code> counters are dropped. Task text is stored as an FNV-1a hash, never in plaintext. Tuning (<code>confidence-threshold</code>, <code>min-data-points</code>, <code>pair-ttl</code>) now persists.</li> + <li><strong>Knobs that do something</strong> — <code>min-data-points</code> now scales prediction confidence (it was declared, displayed and settable but never read), durations exclude chat-lock queue wait, and the confidence percentages on <code>predict</code>/<code>predict_task</code> will move for the same history (fewer points ⇒ lower confidence).</li> + </ul> + </div> + + <div class="version-item"> <div class="v-header"><span class="v-num">v4.6.52 — October 2026</span><span class="v-date">2026-10-22</span></div> <ul> <li><strong>Documentation backlog cleared</strong> — 74 commands that the code registered but this page never documented are now in their own sections: <code>dashboard</code> (with <code>--watch</code>/<code>--stop</code>), <code>usage</code>, <code>ps</code>/<code>top</code>/<code>df</code>/<code>free</code>, the six <code>a2a-*</code> mailbox commands, <code>webxai</code>/<code>webx</code>/<code>wx</code>, <code>context</code>/<code>contexts</code>/<code>context-new</code>/<code>context-delete</code>/<code>context-auto</code>/<code>smart-ask</code>, the <code>commit-*</code> approval trio, <code>module-load|unload|enable|reset</code>, <code>memgraph-agent|session|decay-config</code>, <code>orchestrator-schedule|schedules|unschedule</code>, <code>skill-validate|audit|restore|pack-core</code>, <code>learn-skill</code>, <code>encrypt</code>/<code>decrypt</code>, <code>gzip</code>/<code>wget</code>, <code>kanban</code>, <code>three</code>, <code>course</code>, <code>videos</code>, <code>speak</code>/<code>voice</code>/<code>analyze</code>, <code>gui-run|text|clear</code>, the <code>vaultmind-*</code> set, <code>debug-sync</code>/<code>debug-sync-url</code>, <code>setfont</code>, <code>bug</code>, <code>session-new</code>, <code>files</code>, <code>node</code>, <code>sed</code>, <code>nvfs-index-graph</code>, <code>wallet-lookup</code>, <code>ai-model</code>, <code>ai-ignore</code>, <code>webdav-login</code>, <code>view_module</code>, <code>run</code>, <code>sysmsg</code>, <code>history</code>.</li> @@ -1349,13 +1349,13 @@ </script> <!-- v4.5.63: A2A scratchpad fixes + agent-register arg fix --> -<script src="modules/core/core-bundle.js?v=v4.6.55"></script> +<script src="modules/core/core-bundle.js?v=v4.6.56"></script> <!-- Async dependency-ordered module loader now only initializes terminal and loads optional ext modules --> -<script src="modules/core/module-loader.js?v=v4.6.55"></script> +<script src="modules/core/module-loader.js?v=v4.6.56"></script> <!-- Critical ext bundle loads AFTER webx11 exists so registerModule calls work --> -<script src="modules/ext/critical-ext-bundle.js?v=v4.6.55"></script> +<script src="modules/ext/critical-ext-bundle.js?v=v4.6.56"></script> <script> // ═══ ONE TRUE VERSION SOURCE ═══ diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index 67747bf..9ac94ed 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -9041,6 +9041,7 @@ buildSystemPrompt() { await new Promise(r => setTimeout(r, 50)); } this._chatLock = true; + const taskStart = Date.now(); // predictor durations exclude chat-lock wait this.terminal.markDirty(); // Auto-route context based on intent try { @@ -9131,6 +9132,16 @@ buildSystemPrompt() { } } catch (e) { /* world model not available */ } + // v4.6.56: register a prediction for THIS turn so recordResult can score it by id. + let turnPredictionId = null; + try { + const predictor = this.terminal.moduleManager.modules['PredictorModule']; + if (predictor && typeof predictor.notePrediction === 'function') { + const tp = predictor.notePrediction(userMessage, 'loop'); + turnPredictionId = tp && tp.id ? tp.id : null; + } + } catch (e) { /* predictor is best-effort */ } + const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main'; try { loopResult = await this.agenticLoop(messages, opts); @@ -9158,8 +9169,9 @@ buildSystemPrompt() { const predictor = this.terminal.moduleManager.modules['PredictorModule']; if (predictor && typeof predictor.recordResult === 'function') { const features = predictor._extractFeatures(userMessage); - const duration = (Date.now() - lockStart) / 1000; - predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled); + const duration = (Date.now() - taskStart) / 1000; + predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled, + { task: userMessage, predictionId: turnPredictionId }); } } catch (e) { /* predictor learning is best-effort */ } // Record task in memory graph for direct AI conversations (not just orchestrator) @@ -11241,16 +11253,26 @@ buildSystemPrompt() { if (!taskDesc) return JSON.stringify({ error: 'task parameter required' }); var pFeatures = predictor._extractFeatures(taskDesc); var pSimilar = predictor._findSimilar(pFeatures); - var pResult = predictor._computePrediction(pSimilar, pFeatures); + // v4.6.56: track the prediction so the outcome can score it. + var pTracked; + if (typeof predictor.notePrediction === 'function') { + pTracked = predictor.notePrediction(taskDesc, 'tool'); + } else { + var pFallback = predictor._computePrediction(pSimilar, pFeatures); + pTracked = { id: null, predictedDuration: pFallback.duration, + predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence }; + } return JSON.stringify({ task: taskDesc.substring(0, 80), - estimatedDuration: pResult.duration + 's', - estimatedToolTurns: pResult.toolTurns, - confidence: Math.round(pResult.confidence * 100) + '%', - similarTasks: pResult.similarCount, - recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30 - ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's' - : 'No special action needed' + predictionId: pTracked.id || null, + estimatedDuration: pTracked.predictedDuration + 's', + estimatedToolTurns: pTracked.predictedTurns, + confidence: Math.round(pTracked.confidence * 100) + '%', + similarTasks: pSimilar.length, + recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30 + ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's' + : 'No special action needed', + tracked: !!pTracked.id }); } case 'world_model_query': { @@ -12841,9 +12863,10 @@ buildSystemPrompt() { if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length; } catch (e) {} var predictor = mm ? mm.modules['PredictorModule'] : null; - var predictorAcc = 'N/A'; + var predictorAcc = 'n/a (no paired runs)'; if (predictor && predictor._accuracy && predictor._accuracy.total > 0) { - predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%'; + predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' + + predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)'; } data.health = { modules: data.modules.loaded + '/' + data.modules.total, @@ -17736,25 +17759,32 @@ if (typeof window !== 'undefined') { /* === module-predictor.js === */ // module-predictor.js — Predictive resource allocation engine -// Analyzes historical telemetry + memory graph data to predict task duration/turns +// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a +// real outcome. v1 counted every observed run in the denominator while incrementing the +// numerator only when an old `predict` record happened to match, so the dashboard read 0%. // Commands: predict, predictor-status, predictor-config class PredictorModule { constructor() { this.commands = {}; this.terminal = null; - this._dataPoints = []; + this._dataPoints = []; // observed runs — the training set for _findSimilar + this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; - this._accuracy = { correct: 0, total: 0 }; + this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only + this._mape = 0; // sum of capped |predicted-observed|/observed + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; + this._PAIR_TTL_MS = 15 * 60 * 1000; + this._SCHEMA = 2; this._loaded = false; } getHelpInfo() { return { 'predict': 'predict <task description> — Predict task duration, tool turns, and confidence', - 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count', - 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points', + 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points', + 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl', }; } @@ -17776,33 +17806,14 @@ class PredictorModule { async predict(task) { if (!task) return this.terminal.print('Usage: predict <task description>', 'error'); - const startTime = performance.now(); - - // Extract features const features = this._extractFeatures(task); - - // Find similar historical tasks const similar = this._findSimilar(features); - - // Generate prediction const prediction = this._computePrediction(similar, features); - // Record this prediction for accuracy tracking - const duration = ((performance.now() - startTime) / 1000).toFixed(2); + // Register it as pending so the next matching run can be scored against it. + const pending = this.notePrediction(task, 'cli'); + pending.similarCount = similar.length; - this._predictions++; - this._dataPoints.push({ - features, - predictedDuration: prediction.duration, - predictedTurns: prediction.toolTurns, - confidence: prediction.confidence, - similarCount: similar.length, - timestamp: new Date().toISOString() - }); - - this._debounceSave(); - - // Display result this.terminal.print('📈 Task Prediction', 'header'); this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info'); @@ -17818,23 +17829,22 @@ class PredictorModule { } else { this.terminal.print(' ❓ No historical data to base prediction on.', 'warning'); this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info'); - // Provide hardcoded defaults this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info'); } - // If got memory graph, offer recommendations if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) { this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info'); } + this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info'); return prediction; } _extractFeatures(task) { - const lower = task.toLowerCase(); + const lower = String(task || '').toLowerCase(); return { - length: task.length, - wordCount: task.split(/\s+/).length, + length: String(task || '').length, + wordCount: String(task || '').split(/\s+/).length, hasSearch: /\b(search|find|look)\b/.test(lower), hasWrite: /\b(write|create|save|edit)\b/.test(lower), hasDelete: /\b(delete|remove|rm)\b/.test(lower), @@ -17844,42 +17854,33 @@ class PredictorModule { hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower), hasModule: /\b(module|edit_module|view_module)\b/.test(lower), hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower), - complexity: Math.min(task.length / 50, 10), // 0-10 scale + complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10) }; } + // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2). + _similarity(a, b) { + if (!a || !b) return 0; + const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead', + 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync']; + const matches = keys.filter(k => a[k] === b[k]).length / keys.length; + const kw = a.keywords || []; + const overlap = kw.filter(k => (b.keywords || []).includes(k)).length; + const kScore = overlap / Math.max(kw.length, 1); + return matches * 0.6 + kScore * 0.4; + } + _findSimilar(features) { return this._dataPoints.filter(dp => { - // Skip predictions (only use actual executions with observed results) - if (!dp.observedDuration) return false; - const f = dp.features; - if (!f) return false; - // Feature similarity scoring - let score = 0; - const matches = [ - features.hasSearch === f.hasSearch, - features.hasWrite === f.hasWrite, - features.hasDelete === f.hasDelete, - features.hasInstall === f.hasInstall, - features.hasRead === f.hasRead, - features.hasWeb === f.hasWeb, - features.hasOrchestrate === f.hasOrchestrate, - features.hasModule === f.hasModule, - features.hasSync === f.hasSync, - ]; - score = matches.filter(Boolean).length / matches.length; - // Keyword overlap bonus - const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length; - const kScore = kOverlap / Math.max(features.keywords.length, 1); - score = score * 0.6 + kScore * 0.4; - return score > 0.3; + if (!dp.observedDuration) return false; // outcomes only, never predictions + if (!dp.features) return false; + return this._similarity(features, dp.features) > 0.3; }); } _computePrediction(similar, features) { if (similar.length === 0) { - // Hardcoded defaults let baseDuration = 15; let baseTurns = 3; if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; } @@ -17897,9 +17898,10 @@ class PredictorModule { const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length; const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length; - const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1; + // v2: the documented knob now actually scales confidence (v1 hardcoded 20). + const minPts = Math.max(1, this._MIN_DATA_POINTS || 5); + const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; - // Apply task complexity multiplier const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); @@ -17911,43 +17913,126 @@ class PredictorModule { }; } + // ─── Prediction ↔ Outcome pairing (v2) ─── + + // Task text never leaves the module: only a stable hash is stored/serialised. + _hashTask(t) { + const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = (h * 0x01000193) >>> 0; + } + return h.toString(16); + } + + // Register a prediction to be scored later. Returns the pending record (with .id). + notePrediction(task, source) { + const features = this._extractFeatures(task || ''); + const p = this._computePrediction(this._findSimilar(features), features); + const rec = { + id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), + taskHash: this._hashTask(task), + features, + predictedDuration: p.duration, + predictedTurns: p.toolTurns, + confidence: p.confidence, + source: source || 'loop', + ts: Date.now() + }; + this._pending.push(rec); + if (this._pending.length > 50) this._pending = this._pending.slice(-50); + this._predictions++; + this._debounceSave(); + return rec; + } + + _takePending(features, opts) { + const now = Date.now(); + this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + const taskHash = opts.task ? this._hashTask(opts.task) : null; + + let idx = -1; + if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId); + if (idx < 0 && taskHash) { + for (let i = this._pending.length - 1; i >= 0; i--) { + if (this._pending[i].taskHash === taskHash) { idx = i; break; } + } + } + if (idx < 0) { + // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be + // matched by id or exact task, never by shape alone (nine equal booleans score 0.6). + let best = -1, bestScore = 0; + for (let i = 0; i < this._pending.length; i++) { + const cand = this._pending[i]; + if (cand.source !== 'cli') continue; + const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length; + if (shared === 0) continue; + const s = this._similarity(features, cand.features); + if (s > bestScore) { bestScore = s; best = i; } + } + if (bestScore >= 0.5) idx = best; + } + return idx >= 0 ? this._pending.splice(idx, 1)[0] : null; + } + // ─── Learning from Results ─── - recordResult(features, duration, toolTurns, success) { - this._dataPoints.push({ + recordResult(features, duration, toolTurns, success, opts) { + const o = opts || {}; + const pred = this._takePending(features, o); + const observed = Number(duration) || 0; + + const rec = { features, - observedDuration: duration, + observedDuration: observed, observedTurns: toolTurns, - success, + success: !!success, + paired: !!pred, timestamp: new Date().toISOString() - }); - this._accuracy.total++; - // Compare with last prediction for this feature set - const lastPrediction = [...this._dataPoints] - .reverse() - .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch); - if (lastPrediction) { - const error = Math.abs(lastPrediction.predictedDuration - duration) / duration; - if (error < 0.3) this._accuracy.correct++; + }; + + if (pred) { + const absErr = Math.abs(pred.predictedDuration - observed); + rec.predictedDuration = pred.predictedDuration; + rec.predictedTurns = pred.predictedTurns; + // capped: one 15s-vs-1s miss must not poison the mean-error readout forever + rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); + this._accuracy.total++; + this._mape += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; } - this._debounceSave(); + + this._dataPoints.push(rec); + if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500); + if (pred) this._saveNow(); else this._debounceSave(); + return rec; } // ─── Commands ─── showStatus() { const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length; - const accPct = this._accuracy.total > 0 - ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0) - : 'N/A'; + const paired = this._accuracy.total; + const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; + const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const mape = paired > 0 + ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + : ''; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); this.terminal.print(` Actual executions: ${actualExecutions}`, 'info'); this.terminal.print(` Predictions made: ${this._predictions}`, 'info'); - this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info'); + this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); + this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); + this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); + this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); } setConfig(args) { @@ -17968,6 +18053,12 @@ class PredictorModule { this._MIN_DATA_POINTS = n; return `min-data-points set to ${n}`; }, + 'pair-ttl': () => { + const n = parseInt(val); + if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)'; + this._PAIR_TTL_MS = n * 1000; + return `pair-ttl set to ${n}s`; + }, }; if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error'); @@ -17982,10 +18073,28 @@ class PredictorModule { const fs = this.terminal.fs; if (!fs) return; const data = fs.fsReadJSON('/system/metrics/predictor.json'); - if (data) { - this._dataPoints = data.dataPoints || []; - this._predictions = data.predictions || 0; - this._accuracy = data.accuracy || { correct: 0, total: 0 }; + if (!data) return; + + // Prediction-only points are noise for similarity; keep outcomes. + this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration); + this._predictions = data.predictions || 0; + + if (data.v === this._SCHEMA) { + const acc = data.accuracy || {}; + this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; + this._mape = data.mape || 0; + this._turnsErr = data.turnsErr || 0; + const cfg = data.config || {}; + if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; + if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints; + if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs; + const now = Date.now(); + this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + } else { + // v1 counted every observation in `total` but only some in `correct` → 0%. + this._accuracy = { correct: 0, total: 0 }; + this._mape = 0; + this._pending = []; } } catch (e) { console.warn('[Predictor] Load error (fresh start):', e.message); @@ -17998,9 +18107,18 @@ class PredictorModule { if (!fs) return; if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics'); fs.fsWriteJSON('/system/metrics/predictor.json', { - dataPoints: this._dataPoints.slice(-500), // keep last 500 + v: this._SCHEMA, + dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, - accuracy: this._accuracy + accuracy: this._accuracy, + mape: this._mape, + turnsErr: this._turnsErr, + config: { + confidenceThreshold: this._CONFIDENCE_THRESHOLD, + minDataPoints: this._MIN_DATA_POINTS, + pairTtlMs: this._PAIR_TTL_MS + }, + pending: this._pending.slice(-50) }); } catch (e) { console.error('[Predictor] Save error:', e.message); diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index abbe9ce..83d073a 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -820,6 +820,7 @@ buildSystemPrompt() { await new Promise(r => setTimeout(r, 50)); } this._chatLock = true; + const taskStart = Date.now(); // predictor durations exclude chat-lock wait this.terminal.markDirty(); // Auto-route context based on intent try { @@ -910,6 +911,16 @@ buildSystemPrompt() { } } catch (e) { /* world model not available */ } + // v4.6.56: register a prediction for THIS turn so recordResult can score it by id. + let turnPredictionId = null; + try { + const predictor = this.terminal.moduleManager.modules['PredictorModule']; + if (predictor && typeof predictor.notePrediction === 'function') { + const tp = predictor.notePrediction(userMessage, 'loop'); + turnPredictionId = tp && tp.id ? tp.id : null; + } + } catch (e) { /* predictor is best-effort */ } + const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main'; try { loopResult = await this.agenticLoop(messages, opts); @@ -937,8 +948,9 @@ buildSystemPrompt() { const predictor = this.terminal.moduleManager.modules['PredictorModule']; if (predictor && typeof predictor.recordResult === 'function') { const features = predictor._extractFeatures(userMessage); - const duration = (Date.now() - lockStart) / 1000; - predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled); + const duration = (Date.now() - taskStart) / 1000; + predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled, + { task: userMessage, predictionId: turnPredictionId }); } } catch (e) { /* predictor learning is best-effort */ } // Record task in memory graph for direct AI conversations (not just orchestrator) @@ -3020,16 +3032,26 @@ buildSystemPrompt() { if (!taskDesc) return JSON.stringify({ error: 'task parameter required' }); var pFeatures = predictor._extractFeatures(taskDesc); var pSimilar = predictor._findSimilar(pFeatures); - var pResult = predictor._computePrediction(pSimilar, pFeatures); + // v4.6.56: track the prediction so the outcome can score it. + var pTracked; + if (typeof predictor.notePrediction === 'function') { + pTracked = predictor.notePrediction(taskDesc, 'tool'); + } else { + var pFallback = predictor._computePrediction(pSimilar, pFeatures); + pTracked = { id: null, predictedDuration: pFallback.duration, + predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence }; + } return JSON.stringify({ task: taskDesc.substring(0, 80), - estimatedDuration: pResult.duration + 's', - estimatedToolTurns: pResult.toolTurns, - confidence: Math.round(pResult.confidence * 100) + '%', - similarTasks: pResult.similarCount, - recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30 - ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's' - : 'No special action needed' + predictionId: pTracked.id || null, + estimatedDuration: pTracked.predictedDuration + 's', + estimatedToolTurns: pTracked.predictedTurns, + confidence: Math.round(pTracked.confidence * 100) + '%', + similarTasks: pSimilar.length, + recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30 + ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's' + : 'No special action needed', + tracked: !!pTracked.id }); } case 'world_model_query': { @@ -4620,9 +4642,10 @@ buildSystemPrompt() { if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length; } catch (e) {} var predictor = mm ? mm.modules['PredictorModule'] : null; - var predictorAcc = 'N/A'; + var predictorAcc = 'n/a (no paired runs)'; if (predictor && predictor._accuracy && predictor._accuracy.total > 0) { - predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%'; + predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' + + predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)'; } data.health = { modules: data.modules.loaded + '/' + data.modules.total, diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index ff908cd..8b4f606 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -4,7 +4,7 @@ // Core module bundle loaded synchronously in main.html. // This loader only handles version cache-busting, splash, terminal init, and optional ext modules. - const LATEST_VER = 'v4.6.55'; + const LATEST_VER = 'v4.6.56'; const CACHE = '?v=' + LATEST_VER; const stored = localStorage.getItem('webx11_cache_ver'); if (stored !== LATEST_VER) { diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js index 3d868ae..cb1209e 100644 --- a/modules/core/module-predictor.js +++ b/modules/core/module-predictor.js @@ -1,23 +1,30 @@ // module-predictor.js — Predictive resource allocation engine -// Analyzes historical telemetry + memory graph data to predict task duration/turns +// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a +// real outcome. v1 counted every observed run in the denominator while incrementing the +// numerator only when an old `predict` record happened to match, so the dashboard read 0%. // Commands: predict, predictor-status, predictor-config class PredictorModule { constructor() { this.commands = {}; this.terminal = null; - this._dataPoints = []; + this._dataPoints = []; // observed runs — the training set for _findSimilar + this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; - this._accuracy = { correct: 0, total: 0 }; + this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only + this._mape = 0; // sum of capped |predicted-observed|/observed + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; + this._PAIR_TTL_MS = 15 * 60 * 1000; + this._SCHEMA = 2; this._loaded = false; } getHelpInfo() { return { 'predict': 'predict <task description> — Predict task duration, tool turns, and confidence', - 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count', - 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points', + 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points', + 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl', }; } @@ -39,33 +46,14 @@ class PredictorModule { async predict(task) { if (!task) return this.terminal.print('Usage: predict <task description>', 'error'); - const startTime = performance.now(); - - // Extract features const features = this._extractFeatures(task); - - // Find similar historical tasks const similar = this._findSimilar(features); - - // Generate prediction const prediction = this._computePrediction(similar, features); - // Record this prediction for accuracy tracking - const duration = ((performance.now() - startTime) / 1000).toFixed(2); - - this._predictions++; - this._dataPoints.push({ - features, - predictedDuration: prediction.duration, - predictedTurns: prediction.toolTurns, - confidence: prediction.confidence, - similarCount: similar.length, - timestamp: new Date().toISOString() - }); - - this._debounceSave(); + // Register it as pending so the next matching run can be scored against it. + const pending = this.notePrediction(task, 'cli'); + pending.similarCount = similar.length; - // Display result this.terminal.print('📈 Task Prediction', 'header'); this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info'); @@ -81,23 +69,22 @@ class PredictorModule { } else { this.terminal.print(' ❓ No historical data to base prediction on.', 'warning'); this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info'); - // Provide hardcoded defaults this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info'); } - // If got memory graph, offer recommendations if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) { this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info'); } + this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info'); return prediction; } _extractFeatures(task) { - const lower = task.toLowerCase(); + const lower = String(task || '').toLowerCase(); return { - length: task.length, - wordCount: task.split(/\s+/).length, + length: String(task || '').length, + wordCount: String(task || '').split(/\s+/).length, hasSearch: /\b(search|find|look)\b/.test(lower), hasWrite: /\b(write|create|save|edit)\b/.test(lower), hasDelete: /\b(delete|remove|rm)\b/.test(lower), @@ -107,42 +94,33 @@ class PredictorModule { hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower), hasModule: /\b(module|edit_module|view_module)\b/.test(lower), hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower), - complexity: Math.min(task.length / 50, 10), // 0-10 scale + complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10) }; } + // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2). + _similarity(a, b) { + if (!a || !b) return 0; + const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead', + 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync']; + const matches = keys.filter(k => a[k] === b[k]).length / keys.length; + const kw = a.keywords || []; + const overlap = kw.filter(k => (b.keywords || []).includes(k)).length; + const kScore = overlap / Math.max(kw.length, 1); + return matches * 0.6 + kScore * 0.4; + } + _findSimilar(features) { return this._dataPoints.filter(dp => { - // Skip predictions (only use actual executions with observed results) - if (!dp.observedDuration) return false; - const f = dp.features; - if (!f) return false; - // Feature similarity scoring - let score = 0; - const matches = [ - features.hasSearch === f.hasSearch, - features.hasWrite === f.hasWrite, - features.hasDelete === f.hasDelete, - features.hasInstall === f.hasInstall, - features.hasRead === f.hasRead, - features.hasWeb === f.hasWeb, - features.hasOrchestrate === f.hasOrchestrate, - features.hasModule === f.hasModule, - features.hasSync === f.hasSync, - ]; - score = matches.filter(Boolean).length / matches.length; - // Keyword overlap bonus - const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length; - const kScore = kOverlap / Math.max(features.keywords.length, 1); - score = score * 0.6 + kScore * 0.4; - return score > 0.3; + if (!dp.observedDuration) return false; // outcomes only, never predictions + if (!dp.features) return false; + return this._similarity(features, dp.features) > 0.3; }); } _computePrediction(similar, features) { if (similar.length === 0) { - // Hardcoded defaults let baseDuration = 15; let baseTurns = 3; if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; } @@ -160,9 +138,10 @@ class PredictorModule { const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length; const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length; - const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1; + // v2: the documented knob now actually scales confidence (v1 hardcoded 20). + const minPts = Math.max(1, this._MIN_DATA_POINTS || 5); + const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; - // Apply task complexity multiplier const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); @@ -174,43 +153,126 @@ class PredictorModule { }; } + // ─── Prediction ↔ Outcome pairing (v2) ─── + + // Task text never leaves the module: only a stable hash is stored/serialised. + _hashTask(t) { + const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200); + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = (h * 0x01000193) >>> 0; + } + return h.toString(16); + } + + // Register a prediction to be scored later. Returns the pending record (with .id). + notePrediction(task, source) { + const features = this._extractFeatures(task || ''); + const p = this._computePrediction(this._findSimilar(features), features); + const rec = { + id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), + taskHash: this._hashTask(task), + features, + predictedDuration: p.duration, + predictedTurns: p.toolTurns, + confidence: p.confidence, + source: source || 'loop', + ts: Date.now() + }; + this._pending.push(rec); + if (this._pending.length > 50) this._pending = this._pending.slice(-50); + this._predictions++; + this._debounceSave(); + return rec; + } + + _takePending(features, opts) { + const now = Date.now(); + this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + const taskHash = opts.task ? this._hashTask(opts.task) : null; + + let idx = -1; + if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId); + if (idx < 0 && taskHash) { + for (let i = this._pending.length - 1; i >= 0; i--) { + if (this._pending[i].taskHash === taskHash) { idx = i; break; } + } + } + if (idx < 0) { + // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be + // matched by id or exact task, never by shape alone (nine equal booleans score 0.6). + let best = -1, bestScore = 0; + for (let i = 0; i < this._pending.length; i++) { + const cand = this._pending[i]; + if (cand.source !== 'cli') continue; + const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length; + if (shared === 0) continue; + const s = this._similarity(features, cand.features); + if (s > bestScore) { bestScore = s; best = i; } + } + if (bestScore >= 0.5) idx = best; + } + return idx >= 0 ? this._pending.splice(idx, 1)[0] : null; + } + // ─── Learning from Results ─── - recordResult(features, duration, toolTurns, success) { - this._dataPoints.push({ + recordResult(features, duration, toolTurns, success, opts) { + const o = opts || {}; + const pred = this._takePending(features, o); + const observed = Number(duration) || 0; + + const rec = { features, - observedDuration: duration, + observedDuration: observed, observedTurns: toolTurns, - success, + success: !!success, + paired: !!pred, timestamp: new Date().toISOString() - }); - this._accuracy.total++; - // Compare with last prediction for this feature set - const lastPrediction = [...this._dataPoints] - .reverse() - .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch); - if (lastPrediction) { - const error = Math.abs(lastPrediction.predictedDuration - duration) / duration; - if (error < 0.3) this._accuracy.correct++; + }; + + if (pred) { + const absErr = Math.abs(pred.predictedDuration - observed); + rec.predictedDuration = pred.predictedDuration; + rec.predictedTurns = pred.predictedTurns; + // capped: one 15s-vs-1s miss must not poison the mean-error readout forever + rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); + this._accuracy.total++; + this._mape += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; } - this._debounceSave(); + + this._dataPoints.push(rec); + if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500); + if (pred) this._saveNow(); else this._debounceSave(); + return rec; } // ─── Commands ─── showStatus() { const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length; - const accPct = this._accuracy.total > 0 - ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0) - : 'N/A'; + const paired = this._accuracy.total; + const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; + const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const mape = paired > 0 + ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + : ''; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); this.terminal.print(` Actual executions: ${actualExecutions}`, 'info'); this.terminal.print(` Predictions made: ${this._predictions}`, 'info'); - this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info'); + this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); + this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); + this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); + this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); } setConfig(args) { @@ -231,6 +293,12 @@ class PredictorModule { this._MIN_DATA_POINTS = n; return `min-data-points set to ${n}`; }, + 'pair-ttl': () => { + const n = parseInt(val); + if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)'; + this._PAIR_TTL_MS = n * 1000; + return `pair-ttl set to ${n}s`; + }, }; if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error'); @@ -245,10 +313,28 @@ class PredictorModule { const fs = this.terminal.fs; if (!fs) return; const data = fs.fsReadJSON('/system/metrics/predictor.json'); - if (data) { - this._dataPoints = data.dataPoints || []; - this._predictions = data.predictions || 0; - this._accuracy = data.accuracy || { correct: 0, total: 0 }; + if (!data) return; + + // Prediction-only points are noise for similarity; keep outcomes. + this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration); + this._predictions = data.predictions || 0; + + if (data.v === this._SCHEMA) { + const acc = data.accuracy || {}; + this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; + this._mape = data.mape || 0; + this._turnsErr = data.turnsErr || 0; + const cfg = data.config || {}; + if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; + if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints; + if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs; + const now = Date.now(); + this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS); + } else { + // v1 counted every observation in `total` but only some in `correct` → 0%. + this._accuracy = { correct: 0, total: 0 }; + this._mape = 0; + this._pending = []; } } catch (e) { console.warn('[Predictor] Load error (fresh start):', e.message); @@ -261,9 +347,18 @@ class PredictorModule { if (!fs) return; if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics'); fs.fsWriteJSON('/system/metrics/predictor.json', { - dataPoints: this._dataPoints.slice(-500), // keep last 500 + v: this._SCHEMA, + dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, - accuracy: this._accuracy + accuracy: this._accuracy, + mape: this._mape, + turnsErr: this._turnsErr, + config: { + confidenceThreshold: this._CONFIDENCE_THRESHOLD, + minDataPoints: this._MIN_DATA_POINTS, + pairTtlMs: this._PAIR_TTL_MS + }, + pending: this._pending.slice(-50) }); } catch (e) { console.error('[Predictor] Save error:', e.message); @@ -279,4 +374,4 @@ class PredictorModule { // Required for module system if (typeof window !== 'undefined') { window.PredictorModule = PredictorModule; -}
\ No newline at end of file +} diff --git a/modules/test/test-predictor-accuracy.js b/modules/test/test-predictor-accuracy.js new file mode 100644 index 0000000..1a79167 --- /dev/null +++ b/modules/test/test-predictor-accuracy.js @@ -0,0 +1,209 @@ +// modules/test/test-predictor-accuracy.js — run: node modules/test/test-predictor-accuracy.js +// Proves the predictor's accuracy metric measures PAIRED predictions only. +// RED on v1 code: unpaired runs inflate the denominator, so accuracy is always 0%. +const fs = require('fs'), path = require('path'); +const src = fs.readFileSync(path.join(__dirname, '../core/module-predictor.js'), 'utf8'); +const PredictorModule = new Function(src + '\nreturn PredictorModule;')(); + +function memFs() { + const files = {}; + const dirs = new Set(['/']); + return { + readFile: p => (p in files ? files[p] : null), + writeFile: (p, c) => { files[p] = c; return true; }, + fsReadJSON: p => { try { return JSON.parse(files[p]); } catch (e) { return null; } }, + fsWriteJSON: (p, o) => { files[p] = JSON.stringify(o); return true; }, + readDir: p => (dirs.has(p) ? [] : null), + mkdirRecursive: p => { dirs.add(p); return true; }, + mkdir: p => { dirs.add(p); return true; }, + }; +} + +function mk(persist) { + const out = []; + const term = { print: m => out.push(String(m)), fs: memFs() }; + const p = new PredictorModule(); + p.terminal = term; + if (!persist) { p._saveNow = () => {}; p._debounceSave = () => {}; } // isolate logic from timers + return { p, out, term }; +} + +let fails = 0, passes = 0; +function ok(name, cond, detail) { + if (cond) { passes++; return; } + fails++; + console.log('FAIL ' + name + (detail ? ' — ' + detail : '')); +} + +// A. runs with no prediction must not enter the denominator +{ + const { p } = mk(); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); + p.recordResult(p._extractFeatures('write a report'), 40, 3, true); + ok('A1 total stays 0 without predictions', p._accuracy.total === 0, 'total=' + p._accuracy.total); + ok('A2 observed runs are still recorded', p._dataPoints.filter(d => d.observedDuration).length === 2); +} + +// B. a prediction paired with an on-time outcome counts as correct +{ + const { p } = mk(); + const pred = p.notePrediction('write a report file', 'test'); + ok('B0 notePrediction returns a pending prediction', !!pred && typeof pred.id === 'string'); + p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 1.1, 4, true, { task: 'write a report file' }); + ok('B1 paired once', p._accuracy.total === 1, 'total=' + p._accuracy.total); + ok('B2 on-time prediction scored correct', p._accuracy.correct === 1, 'correct=' + p._accuracy.correct); + ok('B3 pending consumed', p._pending.length === 0, 'pending=' + p._pending.length); +} + +// C. an outcome >30% off is paired but wrong +{ + const { p } = mk(); + const pred = p.notePrediction('write a report file', 'test'); + p.recordResult(p._extractFeatures('write a report file'), pred.predictedDuration * 3 + 5, 4, true, { task: 'write a report file' }); + ok('C1 paired', p._accuracy.total === 1); + ok('C2 not correct', p._accuracy.correct === 0); +} + +// D. an unrelated run must not consume the pending prediction +{ + const { p } = mk(); + p.notePrediction('search the web for ollama docs', 'test'); + p.recordResult(p._extractFeatures('delete the tmp folder'), 9, 1, true, { task: 'delete the tmp folder' }); + ok('D1 unrelated run unpaired', p._accuracy.total === 0, 'total=' + p._accuracy.total); + ok('D2 prediction still pending', p._pending.length === 1, 'pending=' + p._pending.length); +} + +// E. expired predictions are dropped, not paired +{ + const { p } = mk(); + const pred = p.notePrediction('read the logs', 'test'); + pred.ts = Date.now() - (p._PAIR_TTL_MS + 1000); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true, { task: 'read the logs' }); + ok('E1 expired prediction not paired', p._accuracy.total === 0); +} + +// F. the status line never renders "N/A%" +{ + const { p, out } = mk(); + p.showStatus(); + const line = out.find(l => l.includes('Accuracy')) || ''; + ok('F1 empty state says n/a', /\bn\/a\b/i.test(line) && !line.includes('N/A%'), line.trim()); + ok('F2 paired count is shown', /paired/i.test(line), line.trim()); +} + +// G. _MIN_DATA_POINTS actually shapes confidence (it is a dead knob in v1) +{ + const { p } = mk(); + for (let i = 0; i < 3; i++) p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); + const f = p._extractFeatures('read the logs'); + p._MIN_DATA_POINTS = 10; + const a = p._computePrediction(p._findSimilar(f), f).confidence; + p._MIN_DATA_POINTS = 3; + const b = p._computePrediction(p._findSimilar(f), f).confidence; + ok('G1 lowering min-data-points raises confidence', b > a, a.toFixed(3) + ' -> ' + b.toFixed(3)); +} + +// H. v1 payload migration: junk accuracy discarded, observed points kept +{ + const { p, term } = mk(); + term.fs.fsWriteJSON('/system/metrics/predictor.json', { + dataPoints: [{ features: { hasRead: true }, observedDuration: 30, observedTurns: 3 }, + { features: { hasRead: true }, predictedDuration: 30 }], + predictions: 7, + accuracy: { correct: 0, total: 6 } + }); + p._loadFromVFS(); + ok('H1 v1 accuracy discarded', p._accuracy.total === 0 && p._accuracy.correct === 0, JSON.stringify(p._accuracy)); + ok('H2 observed points kept, prediction-only points dropped', p._dataPoints.length === 1, 'n=' + p._dataPoints.length); + ok('H3 predictions counter kept', p._predictions === 7, 'n=' + p._predictions); +} + +// I. save → load round-trip keeps paired accuracy; stale pending is dropped +{ + const { p, term } = mk(true); + const pred = p.notePrediction('read the logs', 'test'); + p.recordResult(p._extractFeatures('read the logs'), pred.predictedDuration, 2, true, { task: 'read the logs' }); + p._saveNow(); + const q = new PredictorModule(); q.terminal = term; q._loadFromVFS(); + ok('I1 accuracy survives reload', q._accuracy.total === 1 && q._accuracy.correct === 1, JSON.stringify(q._accuracy)); + const stale = p.notePrediction('read the logs again', 'test'); + stale.ts = Date.now() - (p._PAIR_TTL_MS + 60000); // expired while pending + p._saveNow(); + const s = new PredictorModule(); s.terminal = term; s._loadFromVFS(); + ok('I2 expired pending dropped on load', s._pending.length === 0, 'pending=' + s._pending.length); +} + + +// J. 3 s absolute floor: a small miss on a short task still counts +{ + const { p } = mk(); + p.notePrediction('read a tiny file', 'test'); + p._pending[0].predictedDuration = 5; // integer seconds vs fractional observed + p.recordResult(p._extractFeatures('read a tiny file'), 3.8, 1, true, { task: 'read a tiny file' }); + ok('J1 1.2 s miss on a 3.8 s task is inside the 3 s floor', p._accuracy.correct === 1, JSON.stringify(p._accuracy)); +} + +// K. a wild miss is capped, so mean error stays readable +{ + const { p } = mk(); + p.notePrediction('read the logs', 'test'); + p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' }); + const rec = p._dataPoints[p._dataPoints.length - 1]; + ok('K1 errorPct capped at 500%', rec.errorPct <= 5, 'errorPct=' + rec.errorPct); +} + +// L. a tool/loop prediction is never consumed by the similarity fallback +{ + const { p } = mk(); + p.notePrediction('write a file report', 'tool'); + p.recordResult(p._extractFeatures('write a file report somewhere else'), 40, 3, true); // no task, no id + ok('L1 shape-alike run does not consume a tool prediction', p._accuracy.total === 0, 'total=' + p._accuracy.total); +} + +// M. the cli fallback needs at least one shared keyword +{ + const { p } = mk(); + p.notePrediction('alpha bravo charlie', 'cli'); + p.recordResult(p._extractFeatures('delta echo foxtrot'), 30, 3, true); + ok('M1 zero keyword overlap is not a match', p._accuracy.total === 0, 'total=' + p._accuracy.total); +} + +// N. tuning survives a reload +{ + const { p, term } = mk(true); + p._MIN_DATA_POINTS = 9; + p._PAIR_TTL_MS = 600000; + p._saveNow(); + const q = new PredictorModule(); q.terminal = term; q._loadFromVFS(); + ok('N1 config persisted', q._MIN_DATA_POINTS === 9 && q._PAIR_TTL_MS === 600000, + q._MIN_DATA_POINTS + '/' + q._PAIR_TTL_MS); +} + +// O. "unpaired" counts scored-and-missed runs, not legacy observations +{ + const { p, out, term } = mk(); + term.fs.fsWriteJSON('/system/metrics/predictor.json', { + dataPoints: [{ features: { hasRead: true }, observedDuration: 30 }], // v1 point, no `paired` flag + accuracy: { correct: 0, total: 6 } + }); + p._loadFromVFS(); + p.recordResult(p._extractFeatures('read the logs'), 20, 2, true); // unpaired v2 run + out.length = 0; p.showStatus(); + const line = out.find(l => l.includes('Unpaired runs')) || ''; + ok('O1 only paired:false records count as unpaired', /Unpaired runs: 1\b/.test(line), line.trim()); +} + + +// P. the id threaded by the AI loop wins over text/similarity guessing +{ + const { p } = mk(); + p.notePrediction('something else entirely', 'tool'); // decoy, pushed first + const loopPred = p.notePrediction('read the logs', 'loop'); + p.recordResult(p._extractFeatures('read the logs'), loopPred.predictedDuration, 2, true, + { task: 'read the logs', predictionId: loopPred.id }); + ok('P1 the threaded id is the one consumed', p._accuracy.total === 1 && p._pending.length === 1, + 'total=' + p._accuracy.total + ' pending=' + p._pending.length); +} + +console.log(passes + '/' + (passes + fails) + ' PASS'); +process.exit(fails ? 1 : 0); |