aboutsummaryrefslogtreecommitdiff
path: root/modules/core
diff options
context:
space:
mode:
Diffstat (limited to 'modules/core')
-rw-r--r--modules/core/core-bundle.js308
-rw-r--r--modules/core/module-aiassist.js47
-rw-r--r--modules/core/module-loader.js2
-rw-r--r--modules/core/module-predictor.js263
4 files changed, 428 insertions, 192 deletions
diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js
index 67747bf..9ac94ed 100644
--- a/modules/core/core-bundle.js
+++ b/modules/core/core-bundle.js
@@ -9041,6 +9041,7 @@ buildSystemPrompt() {
await new Promise(r => setTimeout(r, 50));
}
this._chatLock = true;
+ const taskStart = Date.now(); // predictor durations exclude chat-lock wait
this.terminal.markDirty();
// Auto-route context based on intent
try {
@@ -9131,6 +9132,16 @@ buildSystemPrompt() {
}
} catch (e) { /* world model not available */ }
+ // v4.6.56: register a prediction for THIS turn so recordResult can score it by id.
+ let turnPredictionId = null;
+ try {
+ const predictor = this.terminal.moduleManager.modules['PredictorModule'];
+ if (predictor && typeof predictor.notePrediction === 'function') {
+ const tp = predictor.notePrediction(userMessage, 'loop');
+ turnPredictionId = tp && tp.id ? tp.id : null;
+ }
+ } catch (e) { /* predictor is best-effort */ }
+
const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main';
try {
loopResult = await this.agenticLoop(messages, opts);
@@ -9158,8 +9169,9 @@ buildSystemPrompt() {
const predictor = this.terminal.moduleManager.modules['PredictorModule'];
if (predictor && typeof predictor.recordResult === 'function') {
const features = predictor._extractFeatures(userMessage);
- const duration = (Date.now() - lockStart) / 1000;
- predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled);
+ const duration = (Date.now() - taskStart) / 1000;
+ predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled,
+ { task: userMessage, predictionId: turnPredictionId });
}
} catch (e) { /* predictor learning is best-effort */ }
// Record task in memory graph for direct AI conversations (not just orchestrator)
@@ -11241,16 +11253,26 @@ buildSystemPrompt() {
if (!taskDesc) return JSON.stringify({ error: 'task parameter required' });
var pFeatures = predictor._extractFeatures(taskDesc);
var pSimilar = predictor._findSimilar(pFeatures);
- var pResult = predictor._computePrediction(pSimilar, pFeatures);
+ // v4.6.56: track the prediction so the outcome can score it.
+ var pTracked;
+ if (typeof predictor.notePrediction === 'function') {
+ pTracked = predictor.notePrediction(taskDesc, 'tool');
+ } else {
+ var pFallback = predictor._computePrediction(pSimilar, pFeatures);
+ pTracked = { id: null, predictedDuration: pFallback.duration,
+ predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence };
+ }
return JSON.stringify({
task: taskDesc.substring(0, 80),
- estimatedDuration: pResult.duration + 's',
- estimatedToolTurns: pResult.toolTurns,
- confidence: Math.round(pResult.confidence * 100) + '%',
- similarTasks: pResult.similarCount,
- recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30
- ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's'
- : 'No special action needed'
+ predictionId: pTracked.id || null,
+ estimatedDuration: pTracked.predictedDuration + 's',
+ estimatedToolTurns: pTracked.predictedTurns,
+ confidence: Math.round(pTracked.confidence * 100) + '%',
+ similarTasks: pSimilar.length,
+ recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30
+ ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's'
+ : 'No special action needed',
+ tracked: !!pTracked.id
});
}
case 'world_model_query': {
@@ -12841,9 +12863,10 @@ buildSystemPrompt() {
if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length;
} catch (e) {}
var predictor = mm ? mm.modules['PredictorModule'] : null;
- var predictorAcc = 'N/A';
+ var predictorAcc = 'n/a (no paired runs)';
if (predictor && predictor._accuracy && predictor._accuracy.total > 0) {
- predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%';
+ predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' +
+ predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)';
}
data.health = {
modules: data.modules.loaded + '/' + data.modules.total,
@@ -17736,25 +17759,32 @@ if (typeof window !== 'undefined') {
/* === module-predictor.js === */
// module-predictor.js — Predictive resource allocation engine
-// Analyzes historical telemetry + memory graph data to predict task duration/turns
+// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
+// real outcome. v1 counted every observed run in the denominator while incrementing the
+// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
// Commands: predict, predictor-status, predictor-config
class PredictorModule {
constructor() {
this.commands = {};
this.terminal = null;
- this._dataPoints = [];
+ this._dataPoints = []; // observed runs — the training set for _findSimilar
+ this._pending = []; // predictions waiting for an outcome (bounded, TTL'd)
this._predictions = 0;
- this._accuracy = { correct: 0, total: 0 };
+ this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only
+ this._mape = 0; // sum of capped |predicted-observed|/observed
+ this._turnsErr = 0; // sum of |predicted turns - observed turns|
this._CONFIDENCE_THRESHOLD = 0.5;
this._MIN_DATA_POINTS = 5;
+ this._PAIR_TTL_MS = 15 * 60 * 1000;
+ this._SCHEMA = 2;
this._loaded = false;
}
getHelpInfo() {
return {
'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
- 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count',
- 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points',
+ 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
+ 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
};
}
@@ -17776,33 +17806,14 @@ class PredictorModule {
async predict(task) {
if (!task) return this.terminal.print('Usage: predict <task description>', 'error');
- const startTime = performance.now();
-
- // Extract features
const features = this._extractFeatures(task);
-
- // Find similar historical tasks
const similar = this._findSimilar(features);
-
- // Generate prediction
const prediction = this._computePrediction(similar, features);
- // Record this prediction for accuracy tracking
- const duration = ((performance.now() - startTime) / 1000).toFixed(2);
+ // Register it as pending so the next matching run can be scored against it.
+ const pending = this.notePrediction(task, 'cli');
+ pending.similarCount = similar.length;
- this._predictions++;
- this._dataPoints.push({
- features,
- predictedDuration: prediction.duration,
- predictedTurns: prediction.toolTurns,
- confidence: prediction.confidence,
- similarCount: similar.length,
- timestamp: new Date().toISOString()
- });
-
- this._debounceSave();
-
- // Display result
this.terminal.print('📈 Task Prediction', 'header');
this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');
@@ -17818,23 +17829,22 @@ class PredictorModule {
} else {
this.terminal.print(' ❓ No historical data to base prediction on.', 'warning');
this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
- // Provide hardcoded defaults
this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info');
}
- // If got memory graph, offer recommendations
if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
}
+ this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
return prediction;
}
_extractFeatures(task) {
- const lower = task.toLowerCase();
+ const lower = String(task || '').toLowerCase();
return {
- length: task.length,
- wordCount: task.split(/\s+/).length,
+ length: String(task || '').length,
+ wordCount: String(task || '').split(/\s+/).length,
hasSearch: /\b(search|find|look)\b/.test(lower),
hasWrite: /\b(write|create|save|edit)\b/.test(lower),
hasDelete: /\b(delete|remove|rm)\b/.test(lower),
@@ -17844,42 +17854,33 @@ class PredictorModule {
hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
- complexity: Math.min(task.length / 50, 10), // 0-10 scale
+ complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
};
}
+ // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
+ _similarity(a, b) {
+ if (!a || !b) return 0;
+ const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
+ 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
+ const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
+ const kw = a.keywords || [];
+ const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
+ const kScore = overlap / Math.max(kw.length, 1);
+ return matches * 0.6 + kScore * 0.4;
+ }
+
_findSimilar(features) {
return this._dataPoints.filter(dp => {
- // Skip predictions (only use actual executions with observed results)
- if (!dp.observedDuration) return false;
- const f = dp.features;
- if (!f) return false;
- // Feature similarity scoring
- let score = 0;
- const matches = [
- features.hasSearch === f.hasSearch,
- features.hasWrite === f.hasWrite,
- features.hasDelete === f.hasDelete,
- features.hasInstall === f.hasInstall,
- features.hasRead === f.hasRead,
- features.hasWeb === f.hasWeb,
- features.hasOrchestrate === f.hasOrchestrate,
- features.hasModule === f.hasModule,
- features.hasSync === f.hasSync,
- ];
- score = matches.filter(Boolean).length / matches.length;
- // Keyword overlap bonus
- const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length;
- const kScore = kOverlap / Math.max(features.keywords.length, 1);
- score = score * 0.6 + kScore * 0.4;
- return score > 0.3;
+ if (!dp.observedDuration) return false; // outcomes only, never predictions
+ if (!dp.features) return false;
+ return this._similarity(features, dp.features) > 0.3;
});
}
_computePrediction(similar, features) {
if (similar.length === 0) {
- // Hardcoded defaults
let baseDuration = 15;
let baseTurns = 3;
if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
@@ -17897,9 +17898,10 @@ class PredictorModule {
const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
- const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1;
+ // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
+ const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
+ const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;
- // Apply task complexity multiplier
const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5));
@@ -17911,43 +17913,126 @@ class PredictorModule {
};
}
+ // ─── Prediction ↔ Outcome pairing (v2) ───
+
+ // Task text never leaves the module: only a stable hash is stored/serialised.
+ _hashTask(t) {
+ const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
+ let h = 0x811c9dc5;
+ for (let i = 0; i < s.length; i++) {
+ h ^= s.charCodeAt(i);
+ h = (h * 0x01000193) >>> 0;
+ }
+ return h.toString(16);
+ }
+
+ // Register a prediction to be scored later. Returns the pending record (with .id).
+ notePrediction(task, source) {
+ const features = this._extractFeatures(task || '');
+ const p = this._computePrediction(this._findSimilar(features), features);
+ const rec = {
+ id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+ taskHash: this._hashTask(task),
+ features,
+ predictedDuration: p.duration,
+ predictedTurns: p.toolTurns,
+ confidence: p.confidence,
+ source: source || 'loop',
+ ts: Date.now()
+ };
+ this._pending.push(rec);
+ if (this._pending.length > 50) this._pending = this._pending.slice(-50);
+ this._predictions++;
+ this._debounceSave();
+ return rec;
+ }
+
+ _takePending(features, opts) {
+ const now = Date.now();
+ this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ const taskHash = opts.task ? this._hashTask(opts.task) : null;
+
+ let idx = -1;
+ if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
+ if (idx < 0 && taskHash) {
+ for (let i = this._pending.length - 1; i >= 0; i--) {
+ if (this._pending[i].taskHash === taskHash) { idx = i; break; }
+ }
+ }
+ if (idx < 0) {
+ // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
+ // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
+ let best = -1, bestScore = 0;
+ for (let i = 0; i < this._pending.length; i++) {
+ const cand = this._pending[i];
+ if (cand.source !== 'cli') continue;
+ const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
+ if (shared === 0) continue;
+ const s = this._similarity(features, cand.features);
+ if (s > bestScore) { bestScore = s; best = i; }
+ }
+ if (bestScore >= 0.5) idx = best;
+ }
+ return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
+ }
+
// ─── Learning from Results ───
- recordResult(features, duration, toolTurns, success) {
- this._dataPoints.push({
+ recordResult(features, duration, toolTurns, success, opts) {
+ const o = opts || {};
+ const pred = this._takePending(features, o);
+ const observed = Number(duration) || 0;
+
+ const rec = {
features,
- observedDuration: duration,
+ observedDuration: observed,
observedTurns: toolTurns,
- success,
+ success: !!success,
+ paired: !!pred,
timestamp: new Date().toISOString()
- });
- this._accuracy.total++;
- // Compare with last prediction for this feature set
- const lastPrediction = [...this._dataPoints]
- .reverse()
- .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch);
- if (lastPrediction) {
- const error = Math.abs(lastPrediction.predictedDuration - duration) / duration;
- if (error < 0.3) this._accuracy.correct++;
+ };
+
+ if (pred) {
+ const absErr = Math.abs(pred.predictedDuration - observed);
+ rec.predictedDuration = pred.predictedDuration;
+ rec.predictedTurns = pred.predictedTurns;
+ // capped: one 15s-vs-1s miss must not poison the mean-error readout forever
+ rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5);
+ rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
+ this._accuracy.total++;
+ this._mape += rec.errorPct;
+ this._turnsErr += rec.turnsError;
+ // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
+ if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
}
- this._debounceSave();
+
+ this._dataPoints.push(rec);
+ if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
+ if (pred) this._saveNow(); else this._debounceSave();
+ return rec;
}
// ─── Commands ───
showStatus() {
const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
- const accPct = this._accuracy.total > 0
- ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0)
- : 'N/A';
+ const paired = this._accuracy.total;
+ const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
+ const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
+ const mape = paired > 0
+ ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}`
+ : '';
this.terminal.print('📊 Predictor Status', 'header');
this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info');
this.terminal.print(` Actual executions: ${actualExecutions}`, 'info');
this.terminal.print(` Predictions made: ${this._predictions}`, 'info');
- this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info');
+ this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
+ this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info');
+ this.terminal.print(` Unpaired runs: ${unpaired}`, 'info');
this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info');
+ this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
}
setConfig(args) {
@@ -17968,6 +18053,12 @@ class PredictorModule {
this._MIN_DATA_POINTS = n;
return `min-data-points set to ${n}`;
},
+ 'pair-ttl': () => {
+ const n = parseInt(val);
+ if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
+ this._PAIR_TTL_MS = n * 1000;
+ return `pair-ttl set to ${n}s`;
+ },
};
if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
@@ -17982,10 +18073,28 @@ class PredictorModule {
const fs = this.terminal.fs;
if (!fs) return;
const data = fs.fsReadJSON('/system/metrics/predictor.json');
- if (data) {
- this._dataPoints = data.dataPoints || [];
- this._predictions = data.predictions || 0;
- this._accuracy = data.accuracy || { correct: 0, total: 0 };
+ if (!data) return;
+
+ // Prediction-only points are noise for similarity; keep outcomes.
+ this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
+ this._predictions = data.predictions || 0;
+
+ if (data.v === this._SCHEMA) {
+ const acc = data.accuracy || {};
+ this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
+ this._mape = data.mape || 0;
+ this._turnsErr = data.turnsErr || 0;
+ const cfg = data.config || {};
+ if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
+ if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
+ if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
+ const now = Date.now();
+ this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ } else {
+ // v1 counted every observation in `total` but only some in `correct` → 0%.
+ this._accuracy = { correct: 0, total: 0 };
+ this._mape = 0;
+ this._pending = [];
}
} catch (e) {
console.warn('[Predictor] Load error (fresh start):', e.message);
@@ -17998,9 +18107,18 @@ class PredictorModule {
if (!fs) return;
if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
fs.fsWriteJSON('/system/metrics/predictor.json', {
- dataPoints: this._dataPoints.slice(-500), // keep last 500
+ v: this._SCHEMA,
+ dataPoints: this._dataPoints.slice(-500),
predictions: this._predictions,
- accuracy: this._accuracy
+ accuracy: this._accuracy,
+ mape: this._mape,
+ turnsErr: this._turnsErr,
+ config: {
+ confidenceThreshold: this._CONFIDENCE_THRESHOLD,
+ minDataPoints: this._MIN_DATA_POINTS,
+ pairTtlMs: this._PAIR_TTL_MS
+ },
+ pending: this._pending.slice(-50)
});
} catch (e) {
console.error('[Predictor] Save error:', e.message);
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index abbe9ce..83d073a 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -820,6 +820,7 @@ buildSystemPrompt() {
await new Promise(r => setTimeout(r, 50));
}
this._chatLock = true;
+ const taskStart = Date.now(); // predictor durations exclude chat-lock wait
this.terminal.markDirty();
// Auto-route context based on intent
try {
@@ -910,6 +911,16 @@ buildSystemPrompt() {
}
} catch (e) { /* world model not available */ }
+ // v4.6.56: register a prediction for THIS turn so recordResult can score it by id.
+ let turnPredictionId = null;
+ try {
+ const predictor = this.terminal.moduleManager.modules['PredictorModule'];
+ if (predictor && typeof predictor.notePrediction === 'function') {
+ const tp = predictor.notePrediction(userMessage, 'loop');
+ turnPredictionId = tp && tp.id ? tp.id : null;
+ }
+ } catch (e) { /* predictor is best-effort */ }
+
const _traceTag = opts && opts.agentTag ? opts.agentTag : 'main';
try {
loopResult = await this.agenticLoop(messages, opts);
@@ -937,8 +948,9 @@ buildSystemPrompt() {
const predictor = this.terminal.moduleManager.modules['PredictorModule'];
if (predictor && typeof predictor.recordResult === 'function') {
const features = predictor._extractFeatures(userMessage);
- const duration = (Date.now() - lockStart) / 1000;
- predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled);
+ const duration = (Date.now() - taskStart) / 1000;
+ predictor.recordResult(features, duration, loopResult.toolTurns, !this._cancelled,
+ { task: userMessage, predictionId: turnPredictionId });
}
} catch (e) { /* predictor learning is best-effort */ }
// Record task in memory graph for direct AI conversations (not just orchestrator)
@@ -3020,16 +3032,26 @@ buildSystemPrompt() {
if (!taskDesc) return JSON.stringify({ error: 'task parameter required' });
var pFeatures = predictor._extractFeatures(taskDesc);
var pSimilar = predictor._findSimilar(pFeatures);
- var pResult = predictor._computePrediction(pSimilar, pFeatures);
+ // v4.6.56: track the prediction so the outcome can score it.
+ var pTracked;
+ if (typeof predictor.notePrediction === 'function') {
+ pTracked = predictor.notePrediction(taskDesc, 'tool');
+ } else {
+ var pFallback = predictor._computePrediction(pSimilar, pFeatures);
+ pTracked = { id: null, predictedDuration: pFallback.duration,
+ predictedTurns: pFallback.toolTurns, confidence: pFallback.confidence };
+ }
return JSON.stringify({
task: taskDesc.substring(0, 80),
- estimatedDuration: pResult.duration + 's',
- estimatedToolTurns: pResult.toolTurns,
- confidence: Math.round(pResult.confidence * 100) + '%',
- similarTasks: pResult.similarCount,
- recommendation: pResult.confidence >= predictor._CONFIDENCE_THRESHOLD && pResult.duration > 30
- ? 'Set TTL to ' + Math.ceil(pResult.duration * 1.5) + 's'
- : 'No special action needed'
+ predictionId: pTracked.id || null,
+ estimatedDuration: pTracked.predictedDuration + 's',
+ estimatedToolTurns: pTracked.predictedTurns,
+ confidence: Math.round(pTracked.confidence * 100) + '%',
+ similarTasks: pSimilar.length,
+ recommendation: pTracked.confidence >= predictor._CONFIDENCE_THRESHOLD && pTracked.predictedDuration > 30
+ ? 'Set TTL to ' + Math.ceil(pTracked.predictedDuration * 1.5) + 's'
+ : 'No special action needed',
+ tracked: !!pTracked.id
});
}
case 'world_model_query': {
@@ -4620,9 +4642,10 @@ buildSystemPrompt() {
if (sm && sm._latencyLog) totalErrors += sm._latencyLog.filter(function(e) { return e.error; }).length;
} catch (e) {}
var predictor = mm ? mm.modules['PredictorModule'] : null;
- var predictorAcc = 'N/A';
+ var predictorAcc = 'n/a (no paired runs)';
if (predictor && predictor._accuracy && predictor._accuracy.total > 0) {
- predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '%';
+ predictorAcc = Math.round(predictor._accuracy.correct / predictor._accuracy.total * 100) + '% (' +
+ predictor._accuracy.correct + '/' + predictor._accuracy.total + ' paired)';
}
data.health = {
modules: data.modules.loaded + '/' + data.modules.total,
diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js
index ff908cd..8b4f606 100644
--- a/modules/core/module-loader.js
+++ b/modules/core/module-loader.js
@@ -4,7 +4,7 @@
// Core module bundle loaded synchronously in main.html.
// This loader only handles version cache-busting, splash, terminal init, and optional ext modules.
- const LATEST_VER = 'v4.6.55';
+ const LATEST_VER = 'v4.6.56';
const CACHE = '?v=' + LATEST_VER;
const stored = localStorage.getItem('webx11_cache_ver');
if (stored !== LATEST_VER) {
diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js
index 3d868ae..cb1209e 100644
--- a/modules/core/module-predictor.js
+++ b/modules/core/module-predictor.js
@@ -1,23 +1,30 @@
// module-predictor.js — Predictive resource allocation engine
-// Analyzes historical telemetry + memory graph data to predict task duration/turns
+// v2 (2026-10-23): accuracy is measured ONLY over predictions that were paired with a
+// real outcome. v1 counted every observed run in the denominator while incrementing the
+// numerator only when an old `predict` record happened to match, so the dashboard read 0%.
// Commands: predict, predictor-status, predictor-config
class PredictorModule {
constructor() {
this.commands = {};
this.terminal = null;
- this._dataPoints = [];
+ this._dataPoints = []; // observed runs — the training set for _findSimilar
+ this._pending = []; // predictions waiting for an outcome (bounded, TTL'd)
this._predictions = 0;
- this._accuracy = { correct: 0, total: 0 };
+ this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only
+ this._mape = 0; // sum of capped |predicted-observed|/observed
+ this._turnsErr = 0; // sum of |predicted turns - observed turns|
this._CONFIDENCE_THRESHOLD = 0.5;
this._MIN_DATA_POINTS = 5;
+ this._PAIR_TTL_MS = 15 * 60 * 1000;
+ this._SCHEMA = 2;
this._loaded = false;
}
getHelpInfo() {
return {
'predict': 'predict <task description> — Predict task duration, tool turns, and confidence',
- 'predictor-status': 'predictor-status — Model accuracy, data points, prediction count',
- 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points',
+ 'predictor-status': 'predictor-status — Paired accuracy, pending predictions, data points',
+ 'predictor-config': 'predictor-config <key> <value> — Tune: confidence-threshold, min-data-points, pair-ttl',
};
}
@@ -39,33 +46,14 @@ class PredictorModule {
async predict(task) {
if (!task) return this.terminal.print('Usage: predict <task description>', 'error');
- const startTime = performance.now();
-
- // Extract features
const features = this._extractFeatures(task);
-
- // Find similar historical tasks
const similar = this._findSimilar(features);
-
- // Generate prediction
const prediction = this._computePrediction(similar, features);
- // Record this prediction for accuracy tracking
- const duration = ((performance.now() - startTime) / 1000).toFixed(2);
-
- this._predictions++;
- this._dataPoints.push({
- features,
- predictedDuration: prediction.duration,
- predictedTurns: prediction.toolTurns,
- confidence: prediction.confidence,
- similarCount: similar.length,
- timestamp: new Date().toISOString()
- });
-
- this._debounceSave();
+ // Register it as pending so the next matching run can be scored against it.
+ const pending = this.notePrediction(task, 'cli');
+ pending.similarCount = similar.length;
- // Display result
this.terminal.print('📈 Task Prediction', 'header');
this.terminal.print(` Task: ${task.substring(0, 50)}${task.length > 50 ? '...' : ''}`, 'info');
@@ -81,23 +69,22 @@ class PredictorModule {
} else {
this.terminal.print(' ❓ No historical data to base prediction on.', 'warning');
this.terminal.print(' Tip: Run `orchestrate` with various tasks to build a baseline.', 'info');
- // Provide hardcoded defaults
this.terminal.print(' Default estimate: ~30s duration, ~5 tool turns', 'info');
}
- // If got memory graph, offer recommendations
if (prediction.confidence >= this._CONFIDENCE_THRESHOLD && prediction.duration > 30) {
this.terminal.print(` 💡 Recommendation: Set TTL to ${Math.ceil(prediction.duration * 1.5)}s for predictable completion`, 'info');
}
+ this.terminal.print(` 🎯 Tracked as ${pending.id} — the next matching run scores it`, 'info');
return prediction;
}
_extractFeatures(task) {
- const lower = task.toLowerCase();
+ const lower = String(task || '').toLowerCase();
return {
- length: task.length,
- wordCount: task.split(/\s+/).length,
+ length: String(task || '').length,
+ wordCount: String(task || '').split(/\s+/).length,
hasSearch: /\b(search|find|look)\b/.test(lower),
hasWrite: /\b(write|create|save|edit)\b/.test(lower),
hasDelete: /\b(delete|remove|rm)\b/.test(lower),
@@ -107,42 +94,33 @@ class PredictorModule {
hasOrchestrate: /\b(orchestrat|delegate|agent)\b/.test(lower),
hasModule: /\b(module|edit_module|view_module)\b/.test(lower),
hasSync: /\b(sync|recall|remember|previous session|state sync)\b/.test(lower),
- complexity: Math.min(task.length / 50, 10), // 0-10 scale
+ complexity: Math.min(String(task || '').length / 50, 10), // 0-10 scale
keywords: [...new Set(lower.split(/\s+/).filter(w => w.length > 3))].slice(0, 10)
};
}
+ // Same scoring _findSimilar uses, exposed so pairing can reuse it (v2).
+ _similarity(a, b) {
+ if (!a || !b) return 0;
+ const keys = ['hasSearch', 'hasWrite', 'hasDelete', 'hasInstall', 'hasRead',
+ 'hasWeb', 'hasOrchestrate', 'hasModule', 'hasSync'];
+ const matches = keys.filter(k => a[k] === b[k]).length / keys.length;
+ const kw = a.keywords || [];
+ const overlap = kw.filter(k => (b.keywords || []).includes(k)).length;
+ const kScore = overlap / Math.max(kw.length, 1);
+ return matches * 0.6 + kScore * 0.4;
+ }
+
_findSimilar(features) {
return this._dataPoints.filter(dp => {
- // Skip predictions (only use actual executions with observed results)
- if (!dp.observedDuration) return false;
- const f = dp.features;
- if (!f) return false;
- // Feature similarity scoring
- let score = 0;
- const matches = [
- features.hasSearch === f.hasSearch,
- features.hasWrite === f.hasWrite,
- features.hasDelete === f.hasDelete,
- features.hasInstall === f.hasInstall,
- features.hasRead === f.hasRead,
- features.hasWeb === f.hasWeb,
- features.hasOrchestrate === f.hasOrchestrate,
- features.hasModule === f.hasModule,
- features.hasSync === f.hasSync,
- ];
- score = matches.filter(Boolean).length / matches.length;
- // Keyword overlap bonus
- const kOverlap = features.keywords.filter(k => f.keywords?.includes(k)).length;
- const kScore = kOverlap / Math.max(features.keywords.length, 1);
- score = score * 0.6 + kScore * 0.4;
- return score > 0.3;
+ if (!dp.observedDuration) return false; // outcomes only, never predictions
+ if (!dp.features) return false;
+ return this._similarity(features, dp.features) > 0.3;
});
}
_computePrediction(similar, features) {
if (similar.length === 0) {
- // Hardcoded defaults
let baseDuration = 15;
let baseTurns = 3;
if (features.hasSearch || features.hasWeb) { baseDuration = 25; baseTurns = 5; }
@@ -160,9 +138,10 @@ class PredictorModule {
const avgDuration = similar.reduce((s, dp) => s + (dp.observedDuration || 30), 0) / similar.length;
const avgTurns = similar.reduce((s, dp) => s + (dp.observedTurns || 3), 0) / similar.length;
- const confidence = Math.min(similar.length / 20, 1) * 0.8 + 0.1;
+ // v2: the documented knob now actually scales confidence (v1 hardcoded 20).
+ const minPts = Math.max(1, this._MIN_DATA_POINTS || 5);
+ const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1;
- // Apply task complexity multiplier
const complexityMultiplier = 1 + (features.complexity - 3) * 0.1;
const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5));
@@ -174,43 +153,126 @@ class PredictorModule {
};
}
+ // ─── Prediction ↔ Outcome pairing (v2) ───
+
+ // Task text never leaves the module: only a stable hash is stored/serialised.
+ _hashTask(t) {
+ const s = String(t || '').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 200);
+ let h = 0x811c9dc5;
+ for (let i = 0; i < s.length; i++) {
+ h ^= s.charCodeAt(i);
+ h = (h * 0x01000193) >>> 0;
+ }
+ return h.toString(16);
+ }
+
+ // Register a prediction to be scored later. Returns the pending record (with .id).
+ notePrediction(task, source) {
+ const features = this._extractFeatures(task || '');
+ const p = this._computePrediction(this._findSimilar(features), features);
+ const rec = {
+ id: 'p' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+ taskHash: this._hashTask(task),
+ features,
+ predictedDuration: p.duration,
+ predictedTurns: p.toolTurns,
+ confidence: p.confidence,
+ source: source || 'loop',
+ ts: Date.now()
+ };
+ this._pending.push(rec);
+ if (this._pending.length > 50) this._pending = this._pending.slice(-50);
+ this._predictions++;
+ this._debounceSave();
+ return rec;
+ }
+
+ _takePending(features, opts) {
+ const now = Date.now();
+ this._pending = this._pending.filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ const taskHash = opts.task ? this._hashTask(opts.task) : null;
+
+ let idx = -1;
+ if (opts.predictionId) idx = this._pending.findIndex(p => p.id === opts.predictionId);
+ if (idx < 0 && taskHash) {
+ for (let i = this._pending.length - 1; i >= 0; i--) {
+ if (this._pending[i].taskHash === taskHash) { idx = i; break; }
+ }
+ }
+ if (idx < 0) {
+ // Fallback only for a manual `predict` (source 'cli'): a tool/loop prediction must be
+ // matched by id or exact task, never by shape alone (nine equal booleans score 0.6).
+ let best = -1, bestScore = 0;
+ for (let i = 0; i < this._pending.length; i++) {
+ const cand = this._pending[i];
+ if (cand.source !== 'cli') continue;
+ const shared = (features.keywords || []).filter(k => (cand.features?.keywords || []).includes(k)).length;
+ if (shared === 0) continue;
+ const s = this._similarity(features, cand.features);
+ if (s > bestScore) { bestScore = s; best = i; }
+ }
+ if (bestScore >= 0.5) idx = best;
+ }
+ return idx >= 0 ? this._pending.splice(idx, 1)[0] : null;
+ }
+
// ─── Learning from Results ───
- recordResult(features, duration, toolTurns, success) {
- this._dataPoints.push({
+ recordResult(features, duration, toolTurns, success, opts) {
+ const o = opts || {};
+ const pred = this._takePending(features, o);
+ const observed = Number(duration) || 0;
+
+ const rec = {
features,
- observedDuration: duration,
+ observedDuration: observed,
observedTurns: toolTurns,
- success,
+ success: !!success,
+ paired: !!pred,
timestamp: new Date().toISOString()
- });
- this._accuracy.total++;
- // Compare with last prediction for this feature set
- const lastPrediction = [...this._dataPoints]
- .reverse()
- .find(dp => dp.predictedDuration && dp.features?.hasSearch === features.hasSearch);
- if (lastPrediction) {
- const error = Math.abs(lastPrediction.predictedDuration - duration) / duration;
- if (error < 0.3) this._accuracy.correct++;
+ };
+
+ if (pred) {
+ const absErr = Math.abs(pred.predictedDuration - observed);
+ rec.predictedDuration = pred.predictedDuration;
+ rec.predictedTurns = pred.predictedTurns;
+ // capped: one 15s-vs-1s miss must not poison the mean-error readout forever
+ rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5);
+ rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0));
+ this._accuracy.total++;
+ this._mape += rec.errorPct;
+ this._turnsErr += rec.turnsError;
+ // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict
+ if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++;
}
- this._debounceSave();
+
+ this._dataPoints.push(rec);
+ if (this._dataPoints.length > 500) this._dataPoints = this._dataPoints.slice(-500);
+ if (pred) this._saveNow(); else this._debounceSave();
+ return rec;
}
// ─── Commands ───
showStatus() {
const actualExecutions = this._dataPoints.filter(dp => dp.observedDuration).length;
- const accPct = this._accuracy.total > 0
- ? ((this._accuracy.correct / this._accuracy.total) * 100).toFixed(0)
- : 'N/A';
+ const paired = this._accuracy.total;
+ const unpaired = this._dataPoints.filter(dp => dp.paired === false).length;
+ const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a';
+ const mape = paired > 0
+ ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}`
+ : '';
this.terminal.print('📊 Predictor Status', 'header');
this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info');
this.terminal.print(` Actual executions: ${actualExecutions}`, 'info');
this.terminal.print(` Predictions made: ${this._predictions}`, 'info');
- this.terminal.print(` Accuracy: ${accPct}% (${this._accuracy.correct}/${this._accuracy.total})`, 'info');
+ this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info');
+ this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info');
+ this.terminal.print(` Unpaired runs: ${unpaired}`, 'info');
this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info');
this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info');
+ this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info');
}
setConfig(args) {
@@ -231,6 +293,12 @@ class PredictorModule {
this._MIN_DATA_POINTS = n;
return `min-data-points set to ${n}`;
},
+ 'pair-ttl': () => {
+ const n = parseInt(val);
+ if (isNaN(n) || n < 60) return 'Must be >= 60 (seconds)';
+ this._PAIR_TTL_MS = n * 1000;
+ return `pair-ttl set to ${n}s`;
+ },
};
if (!configMap[key]) return this.terminal.print(`Unknown: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
@@ -245,10 +313,28 @@ class PredictorModule {
const fs = this.terminal.fs;
if (!fs) return;
const data = fs.fsReadJSON('/system/metrics/predictor.json');
- if (data) {
- this._dataPoints = data.dataPoints || [];
- this._predictions = data.predictions || 0;
- this._accuracy = data.accuracy || { correct: 0, total: 0 };
+ if (!data) return;
+
+ // Prediction-only points are noise for similarity; keep outcomes.
+ this._dataPoints = (data.dataPoints || []).filter(dp => dp && dp.observedDuration);
+ this._predictions = data.predictions || 0;
+
+ if (data.v === this._SCHEMA) {
+ const acc = data.accuracy || {};
+ this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 };
+ this._mape = data.mape || 0;
+ this._turnsErr = data.turnsErr || 0;
+ const cfg = data.config || {};
+ if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold;
+ if (typeof cfg.minDataPoints === 'number') this._MIN_DATA_POINTS = cfg.minDataPoints;
+ if (typeof cfg.pairTtlMs === 'number') this._PAIR_TTL_MS = cfg.pairTtlMs;
+ const now = Date.now();
+ this._pending = (data.pending || []).filter(p => p && p.ts && (now - p.ts) <= this._PAIR_TTL_MS);
+ } else {
+ // v1 counted every observation in `total` but only some in `correct` → 0%.
+ this._accuracy = { correct: 0, total: 0 };
+ this._mape = 0;
+ this._pending = [];
}
} catch (e) {
console.warn('[Predictor] Load error (fresh start):', e.message);
@@ -261,9 +347,18 @@ class PredictorModule {
if (!fs) return;
if (!fs.readDir('/system/metrics')) fs.mkdirRecursive('/system/metrics');
fs.fsWriteJSON('/system/metrics/predictor.json', {
- dataPoints: this._dataPoints.slice(-500), // keep last 500
+ v: this._SCHEMA,
+ dataPoints: this._dataPoints.slice(-500),
predictions: this._predictions,
- accuracy: this._accuracy
+ accuracy: this._accuracy,
+ mape: this._mape,
+ turnsErr: this._turnsErr,
+ config: {
+ confidenceThreshold: this._CONFIDENCE_THRESHOLD,
+ minDataPoints: this._MIN_DATA_POINTS,
+ pairTtlMs: this._PAIR_TTL_MS
+ },
+ pending: this._pending.slice(-50)
});
} catch (e) {
console.error('[Predictor] Save error:', e.message);
@@ -279,4 +374,4 @@ class PredictorModule {
// Required for module system
if (typeof window !== 'undefined') {
window.PredictorModule = PredictorModule;
-} \ No newline at end of file
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-06 18:48:59 +0000