diff options
| -rw-r--r-- | main.html | 6 | ||||
| -rw-r--r-- | modules/core/core-bundle.js | 77 | ||||
| -rw-r--r-- | modules/core/module-aiassist.js | 14 | ||||
| -rw-r--r-- | modules/core/module-fs.js | 8 | ||||
| -rw-r--r-- | modules/core/module-loader.js | 2 | ||||
| -rw-r--r-- | modules/core/module-predictor.js | 55 | ||||
| -rw-r--r-- | modules/test/test-predictor-accuracy.js | 44 |
7 files changed, 151 insertions, 55 deletions
@@ -1349,13 +1349,13 @@ </script> <!-- v4.5.63: A2A scratchpad fixes + agent-register arg fix --> -<script src="modules/core/core-bundle.js?v=v4.6.56"></script> +<script src="modules/core/core-bundle.js?v=v4.6.57"></script> <!-- Async dependency-ordered module loader now only initializes terminal and loads optional ext modules --> -<script src="modules/core/module-loader.js?v=v4.6.56"></script> +<script src="modules/core/module-loader.js?v=v4.6.57"></script> <!-- Critical ext bundle loads AFTER webx11 exists so registerModule calls work --> -<script src="modules/ext/critical-ext-bundle.js?v=v4.6.56"></script> +<script src="modules/ext/critical-ext-bundle.js?v=v4.6.57"></script> <script> // ═══ ONE TRUE VERSION SOURCE ═══ diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index 9ac94ed..c654021 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -6391,7 +6391,13 @@ class FileSystemModule { const total = filtered.reduce((s, f) => s + (f.size || 0), 0); this.terminal.print('total ' + filtered.length + ' (' + total + ' bytes)', 'text-muted'); } else { - this.terminal.printClickable(filtered); + // v4.6.57: printClickable() writes straight to the DOM, so anything that + // captures output (the AI run_command tool, the diagnostics sweep) saw nothing + // and reported "directory empty" (packet §1b). Always emit a text line. + const names = filtered.map(f => f.name + (f.type === 'dir' ? '/' : '')).join(' '); + this.terminal.print(names); + const capturing = !!(this.terminal._printCaptureStack && this.terminal._printCaptureStack.length); + if (!capturing && typeof this.terminal.printClickable === 'function') this.terminal.printClickable(filtered); } }; this.commands['cat'] = (args) => { @@ -11315,9 +11321,11 @@ buildSystemPrompt() { case 'view_module': { const modName = a.name || ''; const entry = this.terminal.moduleManager.moduleRegistry.find(m => m.name === modName); - const code = this.terminal.fs.readFile(`_mods/${modName}.js`); - if (code) return code; const url = entry ? entry.url : null; + // v4.6.57: deployed source FIRST. The unversioned `_mods/<Name>.js` edit cache + // used to shadow it forever, so view_module returned pre-fix code while the + // running bundle was new (packet §10). The cache is only a labelled fallback. + const deployedVer = (typeof localStorage !== 'undefined' && localStorage.getItem('webx11_cache_ver')) || 'dev'; // Try registry URL first if (url) { try { @@ -11325,7 +11333,7 @@ buildSystemPrompt() { if (resp.ok) { const src = await resp.text(); this.terminal.fs.writeFile(`_mods/${modName}.orig.js`, src); - return src; + return '// deployed: ' + url + ' @ ' + deployedVer + '\n' + src; } } catch (e) { /* fall through */ } } @@ -11347,11 +11355,15 @@ buildSystemPrompt() { if (resp.ok) { const src = await resp.text(); this.terminal.fs.writeFile(`_mods/${modName}.orig.js`, src); - return src; + return '// deployed: ' + fetchUrl + ' @ ' + deployedVer + '\n' + src; } } } } catch (e) { /* fall through */ } + const cachedSrc = this.terminal.fs.readFile(`_mods/${modName}.js`); + if (cachedSrc) { + return '// VFS COPY of ' + modName + '.js — may be stale (not the deployed file)\n' + cachedSrc; + } return JSON.stringify({ error: `Cannot find source for ${modName} — not in registry, not in modules-index.json` }); } case 'edit_module': { @@ -17771,8 +17783,8 @@ class PredictorModule { this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only - this._mape = 0; // sum of capped |predicted-observed|/observed - this._turnsErr = 0; // sum of |predicted turns - observed turns| + this._errorSum = 0; // SUM of raw |predicted-observed|/observed (mean = /paired) + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; this._PAIR_TTL_MS = 15 * 60 * 1000; @@ -17874,8 +17886,9 @@ class PredictorModule { _findSimilar(features) { return this._dataPoints.filter(dp => { if (!dp.observedDuration) return false; // outcomes only, never predictions - if (!dp.features) return false; - return this._similarity(features, dp.features) > 0.3; + if (dp.usable === false) return false; // aborted runs are not evidence + if (dp.features) return this._similarity(features, dp.features) > 0.3; + return false; }); } @@ -17892,7 +17905,8 @@ class PredictorModule { duration: baseDuration, toolTurns: baseTurns, confidence: 0.1, - similarCount: 0 + similarCount: 0, + floored: false }; } @@ -17903,13 +17917,16 @@ class PredictorModule { const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; - const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); + const adjustedDuration = avgDuration * Math.max(complexityMultiplier, 0.5); + const floored = adjustedDuration < 1; // no prediction is "0 seconds" + const duration = Math.max(1, Math.round(adjustedDuration)); return { - duration: adjustedDuration, - toolTurns: Math.round(avgTurns), + duration, + toolTurns: Math.max(1, Math.round(avgTurns)), confidence: Math.min(confidence, 0.95), - similarCount: similar.length + similarCount: similar.length, + floored }; } @@ -17937,6 +17954,7 @@ class PredictorModule { predictedDuration: p.duration, predictedTurns: p.toolTurns, confidence: p.confidence, + floored: p.floored === true, source: source || 'loop', ts: Date.now() }; @@ -17992,18 +18010,23 @@ class PredictorModule { timestamp: new Date().toISOString() }; + rec.usable = observed >= 1.5; // aborted/instant runs say nothing about duration + if (pred) { const absErr = Math.abs(pred.predictedDuration - observed); rec.predictedDuration = pred.predictedDuration; rec.predictedTurns = pred.predictedTurns; - // capped: one 15s-vs-1s miss must not poison the mean-error readout forever - rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + // RAW error — the renderer clamps, the store does not (a 475x miss must stay 475x) + rec.errorPct = observed > 0 ? absErr / observed : null; rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); - this._accuracy.total++; - this._mape += rec.errorPct; - this._turnsErr += rec.turnsError; - // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict - if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + rec.scored = rec.usable && rec.errorPct !== null && pred.floored !== true; + if (rec.scored) { + this._accuracy.total++; + this._errorSum += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + } } this._dataPoints.push(rec); @@ -18019,9 +18042,12 @@ class PredictorModule { const paired = this._accuracy.total; const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const meanErr = paired > 0 ? this._errorSum / paired : 0; const mape = paired > 0 - ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + ? ` · mean error ${(Math.min(meanErr, 5) * 100).toFixed(0)}%${meanErr > 5 ? '+' : ''}` + + ` · turns ±${(this._turnsErr / paired).toFixed(1)}` : ''; + const unscored = this._dataPoints.filter(dp => dp.scored === false).length; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); @@ -18030,6 +18056,7 @@ class PredictorModule { this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); + this.terminal.print(` Unscored pairs: ${unscored} (aborted run or floored prediction)`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); @@ -18082,7 +18109,7 @@ class PredictorModule { if (data.v === this._SCHEMA) { const acc = data.accuracy || {}; this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; - this._mape = data.mape || 0; + this._errorSum = (typeof data.errorSum === 'number') ? data.errorSum : (data.mape || 0); this._turnsErr = data.turnsErr || 0; const cfg = data.config || {}; if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; @@ -18093,7 +18120,7 @@ class PredictorModule { } else { // v1 counted every observation in `total` but only some in `correct` → 0%. this._accuracy = { correct: 0, total: 0 }; - this._mape = 0; + this._errorSum = 0; this._pending = []; } } catch (e) { @@ -18111,7 +18138,7 @@ class PredictorModule { dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, accuracy: this._accuracy, - mape: this._mape, + errorSum: this._errorSum, turnsErr: this._turnsErr, config: { confidenceThreshold: this._CONFIDENCE_THRESHOLD, diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index 83d073a..628e68a 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -3094,9 +3094,11 @@ buildSystemPrompt() { case 'view_module': { const modName = a.name || ''; const entry = this.terminal.moduleManager.moduleRegistry.find(m => m.name === modName); - const code = this.terminal.fs.readFile(`_mods/${modName}.js`); - if (code) return code; const url = entry ? entry.url : null; + // v4.6.57: deployed source FIRST. The unversioned `_mods/<Name>.js` edit cache + // used to shadow it forever, so view_module returned pre-fix code while the + // running bundle was new (packet §10). The cache is only a labelled fallback. + const deployedVer = (typeof localStorage !== 'undefined' && localStorage.getItem('webx11_cache_ver')) || 'dev'; // Try registry URL first if (url) { try { @@ -3104,7 +3106,7 @@ buildSystemPrompt() { if (resp.ok) { const src = await resp.text(); this.terminal.fs.writeFile(`_mods/${modName}.orig.js`, src); - return src; + return '// deployed: ' + url + ' @ ' + deployedVer + '\n' + src; } } catch (e) { /* fall through */ } } @@ -3126,11 +3128,15 @@ buildSystemPrompt() { if (resp.ok) { const src = await resp.text(); this.terminal.fs.writeFile(`_mods/${modName}.orig.js`, src); - return src; + return '// deployed: ' + fetchUrl + ' @ ' + deployedVer + '\n' + src; } } } } catch (e) { /* fall through */ } + const cachedSrc = this.terminal.fs.readFile(`_mods/${modName}.js`); + if (cachedSrc) { + return '// VFS COPY of ' + modName + '.js — may be stale (not the deployed file)\n' + cachedSrc; + } return JSON.stringify({ error: `Cannot find source for ${modName} — not in registry, not in modules-index.json` }); } case 'edit_module': { diff --git a/modules/core/module-fs.js b/modules/core/module-fs.js index 686a1a6..03c3733 100644 --- a/modules/core/module-fs.js +++ b/modules/core/module-fs.js @@ -63,7 +63,13 @@ class FileSystemModule { const total = filtered.reduce((s, f) => s + (f.size || 0), 0); this.terminal.print('total ' + filtered.length + ' (' + total + ' bytes)', 'text-muted'); } else { - this.terminal.printClickable(filtered); + // v4.6.57: printClickable() writes straight to the DOM, so anything that + // captures output (the AI run_command tool, the diagnostics sweep) saw nothing + // and reported "directory empty" (packet §1b). Always emit a text line. + const names = filtered.map(f => f.name + (f.type === 'dir' ? '/' : '')).join(' '); + this.terminal.print(names); + const capturing = !!(this.terminal._printCaptureStack && this.terminal._printCaptureStack.length); + if (!capturing && typeof this.terminal.printClickable === 'function') this.terminal.printClickable(filtered); } }; this.commands['cat'] = (args) => { diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index 8b4f606..b31fd83 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -4,7 +4,7 @@ // Core module bundle loaded synchronously in main.html. // This loader only handles version cache-busting, splash, terminal init, and optional ext modules. - const LATEST_VER = 'v4.6.56'; + const LATEST_VER = 'v4.6.57'; const CACHE = '?v=' + LATEST_VER; const stored = localStorage.getItem('webx11_cache_ver'); if (stored !== LATEST_VER) { diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js index cb1209e..c8d526d 100644 --- a/modules/core/module-predictor.js +++ b/modules/core/module-predictor.js @@ -11,8 +11,8 @@ class PredictorModule { this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only - this._mape = 0; // sum of capped |predicted-observed|/observed - this._turnsErr = 0; // sum of |predicted turns - observed turns| + this._errorSum = 0; // SUM of raw |predicted-observed|/observed (mean = /paired) + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; this._PAIR_TTL_MS = 15 * 60 * 1000; @@ -114,8 +114,9 @@ class PredictorModule { _findSimilar(features) { return this._dataPoints.filter(dp => { if (!dp.observedDuration) return false; // outcomes only, never predictions - if (!dp.features) return false; - return this._similarity(features, dp.features) > 0.3; + if (dp.usable === false) return false; // aborted runs are not evidence + if (dp.features) return this._similarity(features, dp.features) > 0.3; + return false; }); } @@ -132,7 +133,8 @@ class PredictorModule { duration: baseDuration, toolTurns: baseTurns, confidence: 0.1, - similarCount: 0 + similarCount: 0, + floored: false }; } @@ -143,13 +145,16 @@ class PredictorModule { const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; - const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); + const adjustedDuration = avgDuration * Math.max(complexityMultiplier, 0.5); + const floored = adjustedDuration < 1; // no prediction is "0 seconds" + const duration = Math.max(1, Math.round(adjustedDuration)); return { - duration: adjustedDuration, - toolTurns: Math.round(avgTurns), + duration, + toolTurns: Math.max(1, Math.round(avgTurns)), confidence: Math.min(confidence, 0.95), - similarCount: similar.length + similarCount: similar.length, + floored }; } @@ -177,6 +182,7 @@ class PredictorModule { predictedDuration: p.duration, predictedTurns: p.toolTurns, confidence: p.confidence, + floored: p.floored === true, source: source || 'loop', ts: Date.now() }; @@ -232,18 +238,23 @@ class PredictorModule { timestamp: new Date().toISOString() }; + rec.usable = observed >= 1.5; // aborted/instant runs say nothing about duration + if (pred) { const absErr = Math.abs(pred.predictedDuration - observed); rec.predictedDuration = pred.predictedDuration; rec.predictedTurns = pred.predictedTurns; - // capped: one 15s-vs-1s miss must not poison the mean-error readout forever - rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + // RAW error — the renderer clamps, the store does not (a 475x miss must stay 475x) + rec.errorPct = observed > 0 ? absErr / observed : null; rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); - this._accuracy.total++; - this._mape += rec.errorPct; - this._turnsErr += rec.turnsError; - // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict - if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + rec.scored = rec.usable && rec.errorPct !== null && pred.floored !== true; + if (rec.scored) { + this._accuracy.total++; + this._errorSum += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + } } this._dataPoints.push(rec); @@ -259,9 +270,12 @@ class PredictorModule { const paired = this._accuracy.total; const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const meanErr = paired > 0 ? this._errorSum / paired : 0; const mape = paired > 0 - ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + ? ` · mean error ${(Math.min(meanErr, 5) * 100).toFixed(0)}%${meanErr > 5 ? '+' : ''}` + + ` · turns ±${(this._turnsErr / paired).toFixed(1)}` : ''; + const unscored = this._dataPoints.filter(dp => dp.scored === false).length; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); @@ -270,6 +284,7 @@ class PredictorModule { this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); + this.terminal.print(` Unscored pairs: ${unscored} (aborted run or floored prediction)`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); @@ -322,7 +337,7 @@ class PredictorModule { if (data.v === this._SCHEMA) { const acc = data.accuracy || {}; this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; - this._mape = data.mape || 0; + this._errorSum = (typeof data.errorSum === 'number') ? data.errorSum : (data.mape || 0); this._turnsErr = data.turnsErr || 0; const cfg = data.config || {}; if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; @@ -333,7 +348,7 @@ class PredictorModule { } else { // v1 counted every observation in `total` but only some in `correct` → 0%. this._accuracy = { correct: 0, total: 0 }; - this._mape = 0; + this._errorSum = 0; this._pending = []; } } catch (e) { @@ -351,7 +366,7 @@ class PredictorModule { dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, accuracy: this._accuracy, - mape: this._mape, + errorSum: this._errorSum, turnsErr: this._turnsErr, config: { confidenceThreshold: this._CONFIDENCE_THRESHOLD, diff --git a/modules/test/test-predictor-accuracy.js b/modules/test/test-predictor-accuracy.js index 1a79167..3688b03 100644 --- a/modules/test/test-predictor-accuracy.js +++ b/modules/test/test-predictor-accuracy.js @@ -149,7 +149,7 @@ function ok(name, cond, detail) { p.notePrediction('read the logs', 'test'); p.recordResult(p._extractFeatures('read the logs'), 0.5, 1, true, { task: 'read the logs' }); const rec = p._dataPoints[p._dataPoints.length - 1]; - ok('K1 errorPct capped at 500%', rec.errorPct <= 5, 'errorPct=' + rec.errorPct); + ok('K1 errorPct is stored raw', rec.errorPct > 5, 'errorPct=' + rec.errorPct); } // L. a tool/loop prediction is never consumed by the similarity fallback @@ -205,5 +205,47 @@ function ok(name, cond, detail) { 'total=' + p._accuracy.total + ' pending=' + p._pending.length); } + +// Q. an aborted 0.079 s run must not drive the next prediction to zero (§5 point 1→2) +{ + const { p } = mk(); + const task = 'paste a 2158 char status report'; + p.notePrediction(task, 'loop'); + p.recordResult(p._extractFeatures(task), 0.079, 1, true, { task }); + const next = p.notePrediction(task, 'loop'); + ok('Q1 prediction is never 0 s', next.predictedDuration >= 1, 'pred=' + next.predictedDuration); + ok('Q2 the aborted run is marked unusable', p._dataPoints[0].usable === false); + ok('Q3 it never trains _findSimilar', p._findSimilar(p._extractFeatures(task)).length === 0); +} + +// R. the persisted field is a SUM and the display divides by the paired count +{ + const { p, out, term } = mk(true); + for (let i = 0; i < 2; i++) { + const task = 'read the logs'; + p.notePrediction(task, 'loop'); + p.recordResult(p._extractFeatures(task), 20, 2, true, { task }); // 15 s predicted vs 20 s observed + } + p._saveNow(); + const blob = JSON.parse(term.fs.readFile('/system/metrics/predictor.json')); + ok('R1 stored accumulator is a sum and is named errorSum', typeof blob.errorSum === 'number' && !('mape' in blob), + JSON.stringify(Object.keys(blob))); + out.length = 0; p.showStatus(); + const line = out.find(l => l.includes('Accuracy')) || ''; + ok('R2 display shows the mean with its sample count', /Accuracy: \d+% \(2\/2 paired\) · mean error \d+%/.test(line), line.trim()); +} + +// S. a floored prediction is consumed but not scored as error (§5 defect C) +{ + const { p } = mk(); + const pred = p.notePrediction('read the logs', 'test'); + p._pending[0].floored = true; + p._pending[0].predictedDuration = 1; + p.recordResult(p._extractFeatures('read the logs'), 54.7, 2, true, { task: 'read the logs' }); + ok('S1 floored prediction does not enter accuracy', p._accuracy.total === 0, 'total=' + p._accuracy.total); + ok('S2 it is still recorded and marked unscored', p._dataPoints[0].scored === false); + ok('S3 pending consumed', p._pending.length === 0); +} + console.log(passes + '/' + (passes + fails) + ' PASS'); process.exit(fails ? 1 : 0); |