diff options
| author | Agando <roduccomm@gmail.com> | 2026-10-27 16:32:21 +0100 |
|---|---|---|
| committer | Agando <roduccomm@gmail.com> | 2026-10-27 16:32:21 +0100 |
| commit | 675397058b3562a67319e92bd455761168908090 (patch) | |
| tree | 257277206b29bc61b1a640174d73ce9cdb192766 | |
| parent | 40c4f5ccd9a54b60cad3a3f29c6c9d32d3fbdac5 (diff) | |
test(newos4): prove the mixed-batch loop ceiling stops an agent at 9 identical calls
MAINT-006 was the last open board item and it needed a live run, not another code argument.
The fix landed in 5058d3b (the hard stop was only reachable when EVERY call in a batch was blocked,
so one repeated call + one novel call per turn ran to maxTurns).
- Live probe on v4.6.116 with the deployed class and a deterministic model stub (repeat read_file
+ a fresh list_files each turn): the loop stopped at turn 9 with
"⛔ main: no progress after 9 identical calls — ending loop with partial result" and returned
{finalAnswer, toolTurns} with a 400-char partial answer.
- test-agent-loop-ceiling.js (9 asserts, real module via its constructor): A mixed batch stops before
maxTurns / B all-blocked batch still stops / C a productive loop that never repeats is NOT stopped.
- RED on 52d188a (pre-fix): 8 passed, 1 FAILED — the mixed batch ran to "AI reached the 12/12 tool
limit" with no ceiling line. Suite in that checkout: 19 files, this one failing.
- Suite now 19/19 files; maintain OK.
No bundle change: the shipped code already contains the ceiling.
| -rw-r--r-- | modules/test/test-agent-loop-ceiling.js | 78 |
1 files changed, 78 insertions, 0 deletions
diff --git a/modules/test/test-agent-loop-ceiling.js b/modules/test/test-agent-loop-ceiling.js new file mode 100644 index 0000000..9a00e2b --- /dev/null +++ b/modules/test/test-agent-loop-ceiling.js @@ -0,0 +1,78 @@ +// modules/test/test-agent-loop-ceiling.js — run: node +// MAINT-006: the hard stop used to be reachable only when EVERY call in a batch was blocked, so a +// turn that mixed one repeated call with one novel call ran to maxTurns. This drives the real +// agenticLoop with a deterministic model stub and asserts the mixed batch ends at the ceiling. +const fsMod = require('fs'), path = require('path'); +const ROOT = path.join(__dirname, '..', 'core'); +const store = {}; +global.localStorage = { getItem: k => (k in store ? store[k] : null), setItem: (k, v) => { store[k] = String(v); }, removeItem: k => { delete store[k]; } }; +global.indexedDB = { open() { throw new Error('no idb'); }, deleteDatabase() { throw new Error('no idb'); } }; +global.window = global; + +const load = (file, cls) => new Function(fsMod.readFileSync(path.join(ROOT, file), 'utf8') + '\nreturn ' + cls + ';')(); +const AI = load('module-aiassist.js', 'WebX11AIAssistantModule'); +const VFS = load('virtual-fs.js', 'VirtualFileSystem'); + +let fails = 0, passes = 0; +const ok = (name, cond, detail) => { if (cond) { passes++; return; } fails++; console.log('FAIL ' + name + (detail ? ' — ' + detail : '')); }; + +async function runLoop(shape, maxTurns) { + const out = []; + const vfs = new VFS(); + vfs.saveToStorage = () => true; + vfs.writeFile('/probe.txt', 'alpha\nbeta\ngamma'); + const ai = new AI(); // real constructor: the loop reads _commandFreq/_toolCallLog + let turn = 0; + ai.terminal = { + fs: vfs, + print: (m) => out.push(String(m)), + log: () => {}, + moduleManager: { modules: {}, getAllCommands: () => ({ read_file: 'AI', list_files: 'AI' }) }, + ollamaManager: { + chatWithTools: async () => { + turn++; + const calls = shape(turn); + return { message: { role: 'assistant', content: '', tool_calls: calls }, tool_calls: calls }; + }, + }, + }; + ai._debugPanelEnabled = false; + let res = null, err = null; + try { res = await ai.agenticLoop([{ role: 'user', content: 'loop ceiling probe' }], { maxTurns, agent: 'probe' }); } + catch (e) { err = e.message; } + return { out, res, err, turn }; +} + +// A. mixed batch: one identical call every turn + one novel call every turn +(async () => { + const { out, res, err, turn } = await runLoop((t) => ([ + { id: 'rep-' + t, type: 'function', function: { name: 'read_file', arguments: JSON.stringify({ path: '/probe.txt' }) } }, + { id: 'nov-' + t, type: 'function', function: { name: 'list_files', arguments: JSON.stringify({ path: '/novel-' + t }) } }, + ]), 20); + const joined = out.join('\n'); + ok('A1 no exception', err === null, String(err)); + ok('A2 the ceiling fires', /no progress after \d+ identical calls/.test(joined), joined.slice(-200)); + ok('A3 it stops well before maxTurns', turn < 20, 'ran ' + turn + ' turns'); + ok('A4 it returns a partial answer', !!res && typeof res.finalAnswer === 'string', JSON.stringify(res && Object.keys(res))); +})(); + +// B. all-identical batch still stops (the original path must not regress) +setTimeout(async () => { + const { out, err, turn } = await runLoop((t) => ([ + { id: 'r' + t, type: 'function', function: { name: 'read_file', arguments: JSON.stringify({ path: '/probe.txt' }) } }, + ]), 20); + const joined = out.join('\n'); + ok('B1 all-blocked batch has no exception', err === null, String(err)); + ok('B2 all-blocked batch stops early', turn < 20, 'ran ' + turn + ' turns'); + ok('B3 all-blocked batch reports the ceiling', /no progress after \d+ identical calls/.test(joined), joined.slice(-200)); + + // C. a genuinely productive loop is NOT stopped early: every call differs + const third = await runLoop((t) => ([ + { id: 'u' + t, type: 'function', function: { name: 'list_files', arguments: JSON.stringify({ path: '/dir-' + t }) } }, + ]), 3); + ok('C1 a productive loop runs to maxTurns', third.turn >= 3, 'ran ' + third.turn + ' turns'); + ok('C2 no ceiling message on a productive loop', !/no progress after/.test(third.out.join('\n')), third.out.join('|').slice(-160)); + + if (fails === 0) console.log('test-agent-loop-ceiling: ' + passes + ' passed, 0 failed'); + else { console.log('test-agent-loop-ceiling: ' + passes + ' passed, ' + fails + ' FAILED'); process.exit(1); } +}, 50); |