diff options
Diffstat (limited to 'modules/core/module-predictor.js')
| -rw-r--r-- | modules/core/module-predictor.js | 55 |
1 files changed, 35 insertions, 20 deletions
diff --git a/modules/core/module-predictor.js b/modules/core/module-predictor.js index cb1209e..c8d526d 100644 --- a/modules/core/module-predictor.js +++ b/modules/core/module-predictor.js @@ -11,8 +11,8 @@ class PredictorModule { this._pending = []; // predictions waiting for an outcome (bounded, TTL'd) this._predictions = 0; this._accuracy = { correct: 0, total: 0 }; // total = paired comparisons only - this._mape = 0; // sum of capped |predicted-observed|/observed - this._turnsErr = 0; // sum of |predicted turns - observed turns| + this._errorSum = 0; // SUM of raw |predicted-observed|/observed (mean = /paired) + this._turnsErr = 0; // sum of |predicted turns - observed turns| this._CONFIDENCE_THRESHOLD = 0.5; this._MIN_DATA_POINTS = 5; this._PAIR_TTL_MS = 15 * 60 * 1000; @@ -114,8 +114,9 @@ class PredictorModule { _findSimilar(features) { return this._dataPoints.filter(dp => { if (!dp.observedDuration) return false; // outcomes only, never predictions - if (!dp.features) return false; - return this._similarity(features, dp.features) > 0.3; + if (dp.usable === false) return false; // aborted runs are not evidence + if (dp.features) return this._similarity(features, dp.features) > 0.3; + return false; }); } @@ -132,7 +133,8 @@ class PredictorModule { duration: baseDuration, toolTurns: baseTurns, confidence: 0.1, - similarCount: 0 + similarCount: 0, + floored: false }; } @@ -143,13 +145,16 @@ class PredictorModule { const confidence = Math.min(similar.length / minPts, 1) * 0.8 + 0.1; const complexityMultiplier = 1 + (features.complexity - 3) * 0.1; - const adjustedDuration = Math.round(avgDuration * Math.max(complexityMultiplier, 0.5)); + const adjustedDuration = avgDuration * Math.max(complexityMultiplier, 0.5); + const floored = adjustedDuration < 1; // no prediction is "0 seconds" + const duration = Math.max(1, Math.round(adjustedDuration)); return { - duration: adjustedDuration, - toolTurns: Math.round(avgTurns), + duration, + toolTurns: Math.max(1, Math.round(avgTurns)), confidence: Math.min(confidence, 0.95), - similarCount: similar.length + similarCount: similar.length, + floored }; } @@ -177,6 +182,7 @@ class PredictorModule { predictedDuration: p.duration, predictedTurns: p.toolTurns, confidence: p.confidence, + floored: p.floored === true, source: source || 'loop', ts: Date.now() }; @@ -232,18 +238,23 @@ class PredictorModule { timestamp: new Date().toISOString() }; + rec.usable = observed >= 1.5; // aborted/instant runs say nothing about duration + if (pred) { const absErr = Math.abs(pred.predictedDuration - observed); rec.predictedDuration = pred.predictedDuration; rec.predictedTurns = pred.predictedTurns; - // capped: one 15s-vs-1s miss must not poison the mean-error readout forever - rec.errorPct = Math.min(observed > 0 ? absErr / observed : 5, 5); + // RAW error — the renderer clamps, the store does not (a 475x miss must stay 475x) + rec.errorPct = observed > 0 ? absErr / observed : null; rec.turnsError = Math.abs((pred.predictedTurns || 0) - (toolTurns || 0)); - this._accuracy.total++; - this._mape += rec.errorPct; - this._turnsErr += rec.turnsError; - // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict - if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + rec.scored = rec.usable && rec.errorPct !== null && pred.floored !== true; + if (rec.scored) { + this._accuracy.total++; + this._errorSum += rec.errorPct; + this._turnsErr += rec.turnsError; + // 30% relative, with a 3 s floor so rounding on short tasks does not flip the verdict + if (absErr <= Math.max(0.30 * observed, 3)) this._accuracy.correct++; + } } this._dataPoints.push(rec); @@ -259,9 +270,12 @@ class PredictorModule { const paired = this._accuracy.total; const unpaired = this._dataPoints.filter(dp => dp.paired === false).length; const accPct = paired > 0 ? ((this._accuracy.correct / paired) * 100).toFixed(0) + '%' : 'n/a'; + const meanErr = paired > 0 ? this._errorSum / paired : 0; const mape = paired > 0 - ? ` · mean error ${((this._mape / paired) * 100).toFixed(0)}% · turns ±${(this._turnsErr / paired).toFixed(1)}` + ? ` · mean error ${(Math.min(meanErr, 5) * 100).toFixed(0)}%${meanErr > 5 ? '+' : ''}` + + ` · turns ±${(this._turnsErr / paired).toFixed(1)}` : ''; + const unscored = this._dataPoints.filter(dp => dp.scored === false).length; this.terminal.print('📊 Predictor Status', 'header'); this.terminal.print(` Total data points: ${this._dataPoints.length}`, 'info'); @@ -270,6 +284,7 @@ class PredictorModule { this.terminal.print(` Accuracy: ${accPct} (${this._accuracy.correct}/${paired} paired)${mape}`, 'info'); this.terminal.print(` Awaiting outcome: ${this._pending.length}`, 'info'); this.terminal.print(` Unpaired runs: ${unpaired}`, 'info'); + this.terminal.print(` Unscored pairs: ${unscored} (aborted run or floored prediction)`, 'info'); this.terminal.print(` Confidence threshold: ${this._CONFIDENCE_THRESHOLD}`, 'info'); this.terminal.print(` Min data points: ${this._MIN_DATA_POINTS}`, 'info'); this.terminal.print(` Pair TTL: ${Math.round(this._PAIR_TTL_MS / 60000)}m`, 'info'); @@ -322,7 +337,7 @@ class PredictorModule { if (data.v === this._SCHEMA) { const acc = data.accuracy || {}; this._accuracy = { correct: acc.correct || 0, total: acc.total || 0 }; - this._mape = data.mape || 0; + this._errorSum = (typeof data.errorSum === 'number') ? data.errorSum : (data.mape || 0); this._turnsErr = data.turnsErr || 0; const cfg = data.config || {}; if (typeof cfg.confidenceThreshold === 'number') this._CONFIDENCE_THRESHOLD = cfg.confidenceThreshold; @@ -333,7 +348,7 @@ class PredictorModule { } else { // v1 counted every observation in `total` but only some in `correct` → 0%. this._accuracy = { correct: 0, total: 0 }; - this._mape = 0; + this._errorSum = 0; this._pending = []; } } catch (e) { @@ -351,7 +366,7 @@ class PredictorModule { dataPoints: this._dataPoints.slice(-500), predictions: this._predictions, accuracy: this._accuracy, - mape: this._mape, + errorSum: this._errorSum, turnsErr: this._turnsErr, config: { confidenceThreshold: this._CONFIDENCE_THRESHOLD, |