diff options
Diffstat (limited to 'modules/test/test-tool-arg-coercion.js')
| -rw-r--r-- | modules/test/test-tool-arg-coercion.js | 280 |
1 files changed, 280 insertions, 0 deletions
diff --git a/modules/test/test-tool-arg-coercion.js b/modules/test/test-tool-arg-coercion.js new file mode 100644 index 0000000..42b4d80 --- /dev/null +++ b/modules/test/test-tool-arg-coercion.js @@ -0,0 +1,280 @@ +// node modules/test/test-tool-arg-coercion.js +// Regression for the whole "tool argument is not a string" crash class in +// module-aiassist.js. The tool schemas declare string/integer types, but a model +// can send a number, array or object anyway; handlers do `const x = a.field || ''` +// (which only catches FALSY values) and then call .trim()/.split()/.replace()/ +// .match()/.startsWith() — every such call became a TypeError and a filed +// auto-report (user_set.key, create_skill.name/content, delegate_task.task, +// create_mindmap.children, web_fetch.method, patch_skill.content, ...). +// +// The fix is ONE schema-driven coercion at the dispatch boundary: +// a = this._coerceToolArgs(name, a); +// These tests check the coercer itself against every tool definition, then +// drive the formerly-unsafe tools through the real runTool with hostile args. +const fsMod = require('fs'), path = require('path'); +const ROOT = path.join(__dirname, '..', 'core'); +const SRC = process.env.AIASSIST_SRC || path.join(ROOT, 'module-aiassist.js'); + +const store = {}; +global.localStorage = { + getItem: k => (k in store ? store[k] : null), + setItem: (k, v) => { store[k] = String(v); }, + removeItem: k => { delete store[k]; }, + clear: () => { for (const k of Object.keys(store)) delete store[k]; } +}; +global.window = global; +global.indexedDB = { open() { throw new Error('no idb'); }, deleteDatabase() { throw new Error('no idb'); } }; +global.document = { getElementById: () => null, createElement: () => ({ style: {} }), body: { appendChild() {}, removeChild() {} } }; +global.navigator = global.navigator || {}; +// Offline fetch: every request fails cleanly so tools that would hit the network +// answer with their own error JSON instead of a TypeError. +let fetchCalls = []; +global.fetch = async (u) => { + fetchCalls.push(String(u)); + return { ok: false, status: 503, json: async () => ({}), text: async () => '' }; +}; + +const load = (file, cls) => new Function(fsMod.readFileSync(path.join(ROOT, file), 'utf8') + '\nreturn ' + cls + ';')(); +const VFS = load('virtual-fs.js', 'VirtualFileSystem'); +const AI = new Function(fsMod.readFileSync(SRC, 'utf8') + '\nreturn WebX11AIAssistantModule;')(); + +let fails = 0, passes = 0; +function ok(label, cond, detail) { + if (cond) { passes++; return; } + fails++; + console.log('FAIL ' + label + (detail ? ' — ' + detail : '')); +} +const show = v => { try { return JSON.stringify(v); } catch (e) { return String(v); } }; + +function mk() { + const out = []; + const vfs = new VFS(); + vfs.saveToStorage = () => true; + const ai = Object.create(AI.prototype); // skip the DOM-touching constructor + ai.terminal = { + print: (m, t) => out.push({ text: String(m), type: t || '' }), + log: () => {}, printHTML: () => {}, + fs: vfs, + moduleManager: { + modules: {}, moduleRegistry: [], + loadModuleFromCode: () => true, unloadModule: () => {}, unregisterModule: () => {} + }, + security: { _aiToolDepth: 0, checkAndRun: async () => {} }, + _pushPrintCapture: () => {}, _popPrintCapture: () => {} + }; + ai.checkGuardrail = () => null; + ai._errorLog = []; + ai._toolCallLog = []; + ai._toolCache = new Map(); + ai._toolEpoch = 0; + ai._currentTag = 'main'; + ai._userProfile = {}; + ai._loopProfiles = {}; + ai._toolLimit = () => 8000; + fetchCalls = []; + return { ai, vfs, out }; +} + +// ── Layer A: unit — the coercer against EVERY declared property ─────────────── +{ + const { ai } = mk(); + const defs = ai.getToolDefinitions(); + ok('defs: getToolDefinitions() populated the cache', Array.isArray(ai._toolDefs) && ai._toolDefs.length === defs.length, + 'cache=' + (ai._toolDefs ? ai._toolDefs.length : 'none') + ' defs=' + defs.length); + ok('defs: 50+ tools defined', defs.length >= 50, 'got ' + defs.length); + + const hostile = [42, { a: 1 }, ['x']]; + const coveredTools = new Set(); + let stringProps = 0, numericProps = 0; + + for (const def of defs) { + const fn = def.function, props = (fn.parameters && fn.parameters.properties) || {}; + for (const prop of Object.keys(props)) { + const type = props[prop].type; + const label = fn.name + '.' + prop; + if (type === 'string') { + stringProps++; + coveredTools.add(fn.name); + for (const v of hostile) { + const input = {}; input[prop] = v; + let out; + try { out = ai._coerceToolArgs(fn.name, input); } + catch (e) { ok(label + ' ' + show(v) + ': did not throw', false, e.message); continue; } + ok(label + ' ' + show(v) + ': coerced to a string', typeof out[prop] === 'string', + 'got ' + typeof out[prop] + ' ' + show(out[prop])); + } + // structured values must survive as readable JSON, not "[object Object]" + const objIn = {}; objIn[prop] = { a: 1 }; + const objOut = ai._coerceToolArgs(fn.name, objIn); + ok(label + ' object: kept as JSON text', + objOut[prop] === '{"a":1}' && objOut[prop].indexOf('[object Object]') === -1, show(objOut[prop])); + // a real string is untouched + const sIn = {}; sIn[prop] = ' keep me '; + ok(label + ' string: untouched', ai._coerceToolArgs(fn.name, sIn)[prop] === ' keep me '); + // falsy-but-string stays a string; optional args stay absent/null + const zeroIn = {}; zeroIn[prop] = 0; + ok(label + ' 0: becomes "0"', ai._coerceToolArgs(fn.name, zeroIn)[prop] === '0'); + const nullIn = {}; nullIn[prop] = null; + const nullOut = ai._coerceToolArgs(fn.name, nullIn); + ok(label + ' null: left for _pathArg/handler defaults', nullOut[prop] === null, show(nullOut[prop])); + const undefIn = {}; undefIn[prop] = undefined; + const undefOut = ai._coerceToolArgs(fn.name, undefIn); + ok(label + ' undefined: left absent', prop in undefOut && undefOut[prop] === undefined, show(undefOut)); + } else if (type === 'integer' || type === 'number') { + numericProps++; + const numIn = {}; numIn[prop] = '42'; + const numOut = ai._coerceToolArgs(fn.name, numIn)[prop]; + ok(label + " '42': becomes the number 42", numOut === 42 && typeof numOut === 'number', typeof numOut + ' ' + show(numOut)); + const badIn = {}; badIn[prop] = 'abc'; + ok(label + " 'abc': left for the handler", ai._coerceToolArgs(fn.name, badIn)[prop] === 'abc'); + const objIn2 = {}; objIn2[prop] = { a: 1 }; + const objOut2 = ai._coerceToolArgs(fn.name, objIn2)[prop]; + ok(label + ' object: unchanged', typeof objOut2 === 'object' && objOut2.a === 1, show(objOut2)); + const realIn = {}; realIn[prop] = 7; + ok(label + ' 7: unchanged', ai._coerceToolArgs(fn.name, realIn)[prop] === 7); + } + } + } + ok('unit: every string prop of every tool was exercised', stringProps >= 30 && coveredTools.size >= 25, + 'stringProps=' + stringProps + ' tools=' + coveredTools.size); + ok('unit: numeric props exercised', numericProps >= 4, 'got ' + numericProps); + + // unknown tool / non-object args / broken defs: pass through, never throw + const passthrough = { x: 1, y: { a: 1 }, z: 'ok' }; + ok('unknown tool: returns the same object', ai._coerceToolArgs('no_such_tool_xyz', passthrough) === passthrough); + ok('unknown tool: values untouched', passthrough.y.a === 1); + for (const weird of [null, undefined, 'str', 42, []]) { + let out, threw = null; + try { out = ai._coerceToolArgs('user_get', weird); } catch (e) { threw = e; } + ok('non-object args ' + show(weird) + ': no throw', threw === null, threw && threw.message); + } + const broken = Object.create(AI.prototype); + broken._buildToolDefs = () => { throw new Error('boom'); }; + const target = { key: { a: 1 } }; + let brokenOut, brokenThrew = null; + try { brokenOut = broken._coerceToolArgs('user_get', target); } catch (e) { brokenThrew = e; } + ok('broken defs: no throw and original returned', brokenThrew === null && brokenOut === target, brokenThrew && brokenThrew.message); + + console.log('unit: ' + defs.length + ' tool defs, ' + stringProps + ' string props in ' + + coveredTools.size + ' tools, ' + numericProps + ' numeric props'); + console.log('unit: string-prop tools — ' + Array.from(coveredTools).sort().join(', ')); +} + +// ── Layer B: end-to-end through the real runTool ────────────────────────────── +async function call(ai, name, args, label) { + let res = null, threw = null; + try { res = await ai.runTool(name, args); } catch (e) { threw = e; } + ok((label || name) + ': runTool did not throw', threw === null, threw && threw.message); + ok((label || name) + ': returned a string', typeof res === 'string', typeof res); + ok((label || name) + ': no string-method TypeError leaked', + typeof res === 'string' && !/is not a function/.test(res), String(res).slice(0, 240)); + let parsed = null; + try { parsed = JSON.parse(res); } catch (e) {} + return { res, parsed }; +} + +(async () => { + // 1. key/value tools — user_get/user_set (:2911-2934 area). a.key was .split() + { + const { ai } = mk(); + const g = await call(ai, 'user_get', { key: { a: 1 } }); + ok('user_get: object key reached the handler as JSON', !!g.parsed && g.parsed.key === '{"a":1}', g.res); + const s = await call(ai, 'user_set', { key: 42, value: { theme: 'dark' } }); + ok('user_set: numeric key coerced', !!s.parsed && s.parsed.success === true && s.parsed.key === '42', s.res); + ok('user_set: object value stored as JSON', ai._userProfile['42'] === '{"theme":"dark"}', show(ai._userProfile)); + } + + // 2. skill creator — create_skill (:3164-3168): name.trim()/toLowerCase(), + // content.startsWith(), tags.split() + { + const { ai, vfs } = mk(); + const r = await call(ai, 'create_skill', { name: { a: 1 }, description: 7, content: ['line1', 'line2'], tags: 42 }); + ok('create_skill: success with hostile args', !!r.parsed && r.parsed.success === true, r.res); + ok('create_skill: name is the JSON text, slugified', !!r.parsed && r.parsed.name === '{"a":1}', r.res); + const written = vfs.readFile('/skills/{"a":1}.md') || ''; + ok('create_skill: array content written as readable JSON', + written.indexOf('line1') !== -1 && written.indexOf('[object Object]') === -1, written.slice(0, 200)); + ok('create_skill: numeric tags coerced to a string list', /tags: \[42\]/.test(written), written.slice(0, 200)); + } + + // 3. patch_skill (:3185-3192): pcontent.match() / .split() + { + const { ai, vfs } = mk(); + vfs.mkdirRecursive('/skills'); + vfs.fsWriteJSON('/skills/index.json', { demo: { name: 'demo', description: '', tags: [], category: 'general' } }); + const r = await call(ai, 'patch_skill', { name: 'demo', content: { steps: ['a', 'b'] } }); + ok('patch_skill: success with object content', !!r.parsed && r.parsed.success === true, r.res); + ok('patch_skill: object content written as JSON', + (vfs.readFile('/skills/demo.md') || '') === '{"steps":["a","b"]}', vfs.readFile('/skills/demo.md')); + } + + // 4. delegation — delegate_task (:3064): task.substring() + { + const { ai } = mk(); + ai.terminal.moduleManager.modules['OrchestratorModule'] = { + delegateTask: async t => 'worker saw: ' + t + }; + const r = await call(ai, 'delegate_task', { task: { goal: 'ship it' } }); + ok('delegate_task: object task reached the worker as JSON', + !!r.parsed && r.parsed.success === true && /worker saw: \{"goal":"ship it"\}/.test(r.parsed.result || ''), r.res); + } + + // 5. mindmap — create_mindmap (:3305-3312): topic.split(), children.split() + { + const { ai } = mk(); + const added = []; + ai.terminal.moduleManager.modules['MindmapModule'] = { + _new: () => {}, _ensureMindmap: () => ({ root: { id: 'root-1' } }), + _add: a => added.push(a), _svg: () => {}, _countNodes: () => 2 + }; + const r = await call(ai, 'create_mindmap', { topic: 42, children: { a: 1 } }); + ok('create_mindmap: success with hostile args', !!r.parsed && r.parsed.success === true, r.res); + ok('create_mindmap: topic coerced to "42"', !!r.parsed && r.parsed.topic === '42', r.res); + ok('create_mindmap: one child parsed from the JSON text', added.length === 1 && added[0][1] === '{"a":1}', show(added)); + } + + // 6. web_fetch (:2785-2788): (a.method || 'GET').toUpperCase() + body + { + const { ai } = mk(); + const r = await call(ai, 'web_fetch', { url: 'https://example.invalid/x', method: { verb: 'POST' }, body: ['x'], max_chars: '500' }); + ok('web_fetch: reached its own error path (offline stub), not a TypeError', + !!r.parsed && typeof r.parsed.error === 'string', r.res); + ok('web_fetch: coercion let the request be made (method was a string by then)', + fetchCalls.length === 1, show(fetchCalls)); + } + + // 7. filter tool — list_skills (:3208): (a.category || '').toLowerCase() + { + const { ai, vfs } = mk(); + vfs.mkdirRecursive('/skills'); + vfs.fsWriteJSON('/skills/index.json', { demo: { name: 'demo', description: '', tags: [], category: 'general' } }); + const r = await call(ai, 'list_skills', { category: { cat: 'general' } }); + ok('list_skills: no matching category, still a clean list', !!r.parsed && r.parsed.count === 0, r.res); + } + + // 8. module tools live behind a SECOND parse+switch (runModuleTool) that + // runTool delegates to — and ship_module calls newCode.split() at step 5. + // Stub only the pipeline's inner stages so the object `code` reaches it. + { + const { ai, vfs } = mk(); + vfs.mkdirRecursive('_mods'); + vfs.writeFile('_mods/Demo.js', '// old\nconst a = 1;\n'); + const orig = ai.runModuleTool.bind(ai); + ai.runModuleTool = async (n, argsStr) => { + if (n === 'ship_module') return orig(n, argsStr); + if (n === 'test_module') return JSON.stringify({ success: true, module: 'Demo', commands: 1 }); + if (n === 'health_check') return JSON.stringify({ healthy: true }); + return JSON.stringify({ success: true }); + }; + ai._runRegressionSuite = async () => ({ allPassed: true }); + ai._calculateConfidence = () => 80; + const r = await call(ai, 'ship_module', { name: 'Demo', code: { src: 'x' }, tests: [] }, 'ship_module'); + ok('ship_module: object code survived to step 5 (.split)', + !!r.parsed && r.parsed.success === true && /dry-run/.test(r.parsed.pipeline || ''), r.res); + ok('ship_module: object code written as JSON, not [object Object]', + (vfs.readFile('_mods/Demo.js') || '') === '{"src":"x"}', vfs.readFile('_mods/Demo.js')); + } + + console.log('test-tool-arg-coercion: ' + passes + ' passed, ' + fails + ' failed'); + process.exit(fails ? 1 : 0); +})(); |