feat(newos4): configurable tool budgets, per-model defaults, real tool telemetry

Closes STEER-20261102-201234-7200 (read_file capped at 6000 while the schema
promised 32000).

Budgets (module-aiassist.js + new modules/core/model-profiles.js):
- one resolver, precedence: call arg -> loop profile -> model profile ->
  ai-config -> code default. Keys: read-page-chars, read-result-chars,
  tool-context-budget, fetch-chars.
- model classes tiny/small/mid/large keyed by id then by size token; `mid`
  reproduces the old numbers so nothing moves for a mid model.
- _clipToolResult now clips the `content` field, never the serialised JSON
  envelope: clipping the envelope deleted truncated/next_offset, so a 16k page
  lost its continuation pointer (the reported symptom).
- read_file's schema no longer promises a number the model class may not give.

Telemetry:
- _recordToolCall keeps sizes, duration, ok and a 120-char arg preview instead
  of the whole payload; ai-stats prints calls / ok% / avg ms / KB / truncation
  from localStorage webx11_tool_stats.

Server (api/proxy-api.py):
- /api/extract honours max_chars (default 12000, cap 100000) and returns
  content_chars + truncated; the bs4 branch is no longer uncapped while the
  regex fallback was silently cut at 10 000.

BEHAVIOUR CHANGE: num_ctx for a model with no explicit setting now follows the
class default (16384 -> 65536 for cloud/large, 8192 for tiny). Cloud models bill
on tokens used, so cost is unchanged; local models get a bigger KV cache.

Two build traps, both now documented in AGENTS.md:
- model-profiles.js must be the FIRST entry in CORE_FILES and must export the
  MODEL_PROFILES global (a TDZ typeof throws; without the global every budget
  silently falls back to the code default).
- a budget key needs the setter branch AND the constructor loader at
  module-aiassist.js:49 - persisting without restoring looks fine until reload.

Verified: test-tool-context-bound 23/23 (RED before: E1-E4 + TypeError on
_toolLimit); live bundle read_page 32000 / result 12000 / budget 48000 /
fetch 12000 and _ctxTokens 65536 for gemma4:31b-cloud; /api/extract 28970 ->
500 chars with truncated true; ai-config write+restore probe 6/6.

1 files changed, 132 insertions, 19 deletions

diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index e4b7b7e..365bf94 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -46,7 +46,7 @@
                 this._loopProfiles = {};
                 this._initLoopProfiles();
                 // Restore AI config from localStorage (maxContext, MAX_TOOL_TURNS, etc.)
-                try { const cfg = localStorage.getItem('webx11_ai_config'); if (cfg) { const p = JSON.parse(cfg); if (p.maxTurns) this.MAX_TOOL_TURNS = p.maxTurns; if (p.maxContext) this.maxContext = p.maxContext; if (p.maxContextTokens) this.maxContextTokens = p.maxContextTokens; if (p.confirmTimeoutMs) this._confirmTimeoutMs = p.confirmTimeoutMs; if (p.skillDebounceMs) this._skillDebounceMs = p.skillDebounceMs; if (p.chatLockTimeoutMs) this._chatLockTimeoutMs = p.chatLockTimeoutMs; } } catch (e) {}
+                try { const cfg = localStorage.getItem('webx11_ai_config'); if (cfg) { const p = JSON.parse(cfg); if (p.maxTurns) this.MAX_TOOL_TURNS = p.maxTurns; if (p.maxContext) this.maxContext = p.maxContext; if (p.maxContextTokens) this.maxContextTokens = p.maxContextTokens; if (p.confirmTimeoutMs) this._confirmTimeoutMs = p.confirmTimeoutMs; if (p.skillDebounceMs) this._skillDebounceMs = p.skillDebounceMs; if (p.chatLockTimeoutMs) this._chatLockTimeoutMs = p.chatLockTimeoutMs; if (p.maxReadPageChars) this.maxReadPageChars = p.maxReadPageChars; if (p.maxToolResultChars) this.maxToolResultChars = p.maxToolResultChars; if (p.toolContextBudgetChars) this.toolContextBudgetChars = p.toolContextBudgetChars; if (p.maxFetchChars) this.maxFetchChars = p.maxFetchChars; } } catch (e) {}
                 try { const s = localStorage.getItem('webx11_ai_patterns'); if (s) this._commandFreq = JSON.parse(s); } catch (e) {}
                 try { const p = localStorage.getItem('webx11_user_profile'); if (p) this._userProfile = JSON.parse(p); } catch (e) {}
                 // Restore AI conversation context from localStorage
@@ -162,7 +162,7 @@ buildSystemPrompt() {
                         parameters: { type: 'object', properties: {
                             path: { type: 'string', description: 'File path' },
                             offset: { type: 'integer', description: 'Start character offset (default 0)' },
-                            limit: { type: 'integer', description: 'Max characters to return (default 8000, max 32000)' }
+                            limit: { type: 'integer', description: 'Max characters per page (default 8000). The effective page size follows the current model\'s budget and is capped again before the result enters context — the response carries returned_chars/total_chars/truncated/next_offset, page on those' }
                         }, required: ['path'] }
                     }
                 }, {
@@ -225,7 +225,8 @@ buildSystemPrompt() {
                                 url: { type: 'string', description: 'Full URL' },
                                 method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' },
                                 body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' },
-                                content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' }
+                                content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' },
+max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' },
                             },
                             required: ['url']
                         }
@@ -755,6 +756,7 @@ buildSystemPrompt() {
                 this.commands['ai-model'] = (args) => this.showModel(args);
                 this.commands['ai-compress'] = () => this._manualCompress();
                 this.commands['ai-config'] = (args) => this.aiConfig(args);
+                this.commands['ai-stats'] = () => this.aiStats();
                 this.commands['commit-approve'] = (args) => this._executeCommit(args.join(' '));
                 this.commands['commit-deny'] = () => {
                     if (!this._pendingCommit) return this.terminal.print('No pending commit.', 'warning');
@@ -1015,7 +1017,10 @@ buildSystemPrompt() {
                     this.terminal.log('INFO', 'AI', 'Query: ' + userMessage.substring(0, 100));
                     if (loopResult && loopResult.toolTurns) {
                         const toolNames = this._toolCallLog.slice(-(loopResult.toolTurns || 0)).map(function(e) { return e.name; }).join(', ');
-                        this.terminal.log('INFO', 'AI', 'Used ' + loopResult.toolTurns + ' tool(s): ' + toolNames);
+                        this.terminal.log('INFO', 'AI', 'Used ' + loopResult.toolTurns + ' tool(s): ' + toolNames
+                            + ' · returned ' + Math.round(recent.reduce((a, e) => a + e.returnedBytes, 0) / 1024) + 'KB'
+                            + ' · clipped ' + Math.round(recent.reduce((a, e) => a + e.clippedBytes, 0) / 1024) + 'KB'
+                            + ' · ' + recent.filter(e => e.truncated).length + ' truncated');
                     }
                 }
                 // Auto-archive every 10 messages
@@ -1344,10 +1349,87 @@ buildSystemPrompt() {
             // went 6 → 112 705 tokens across 8 model calls (maxContextTokens is 12 000). Two
             // limits keep a single turn inside the window: clip each observation as it arrives,
             // and elide the oldest ones once the running total passes the budget.
+            _recordToolCall(entry) {
+                const r = String(entry.result || '');
+                let truncated = false, contentChars = null;
+                try {
+                    if (r.charAt(0) === '{') {
+                        const o = JSON.parse(r);
+                        truncated = !!(o.truncated || o.clipped);
+                        contentChars = (typeof o.content_chars === 'number') ? o.content_chars : null;
+                    }
+                } catch (e) {}
+                const rec = {
+                    turn: entry.turn, name: entry.name,
+                    argsPreview: String(entry.args || '').slice(0, 120),
+                    argsBytes: String(entry.args || '').length,
+                    resultBytes: r.length, contentChars: contentChars,
+                    returnedBytes: Number(entry.returnedBytes || r.length),
+                    clippedBytes: Math.max(0, r.length - Number(entry.returnedBytes || r.length)),
+                    truncated: truncated, durationMs: entry.durationMs || 0,
+                    ok: !entry.error, errorKind: entry.errorKind || null,
+                };
+                this._toolCallLog.push(rec);
+                if (this._toolCallLog.length > 50) this._toolCallLog.shift();
+                this._toolStatsAdd(rec);
+                return rec;
+            }
+            _toolStatsAdd(rec) {
+                try {
+                    const K = 'webx11_tool_stats';
+                    const s = JSON.parse(localStorage.getItem(K) || '{"v":1,"tools":{}}');
+                    const t = s.tools[rec.name] || (s.tools[rec.name] = { calls: 0, ok: 0, ms: 0, bytes: 0, truncated: 0 });
+                    t.calls++; t.ok += rec.ok ? 1 : 0; t.ms += rec.durationMs;
+                    t.bytes += rec.returnedBytes; t.truncated += rec.truncated ? 1 : 0;
+                    localStorage.setItem(K, JSON.stringify(s));
+                } catch (e) {}
+            }
+            // ── Tool budgets: call arg > loop profile > model profile > ai-config > code default ──
+            // ── Tool budgets: call arg > loop profile > model profile > ai-config > code default ──
+            _modelProfile() {
+                try {
+                    // `this.modelProfiles` is set by the node harness; in the browser the bundle
+                    // supplies the top-level MODEL_PROFILES constant. Never require() a sibling here —
+                    // the path resolves relative to whoever called the module (found the hard way).
+                    const mp = this.modelProfiles
+                        || (typeof MODEL_PROFILES !== 'undefined' ? MODEL_PROFILES : null);
+                    const id = this.terminal && this.terminal.ollamaManager
+                        ? this.terminal.ollamaManager.currentModel : '';
+                    return mp && mp.profileFor ? mp.profileFor(id) : null;
+                } catch (e) { return null; }
+            }
+            _toolLimit(key, callArg) {
+                const explicit = { readPageChars: this.maxReadPageChars, toolResultChars: this.maxToolResultChars,
+                                   toolContextBudgetChars: this.toolContextBudgetChars, fetchChars: this.maxFetchChars }[key];
+                const prof = this._modelProfile() || {};
+                const codeDefault = { readPageChars: 16000, toolResultChars: 6000,
+                                      toolContextBudgetChars: 24000, fetchChars: 4000 }[key];
+                if (Number(callArg) > 0) return Number(callArg);
+                if (Number(explicit) > 0) return Number(explicit);
+                if (Number(prof[key]) > 0) return Number(prof[key]);
+                return codeDefault;
+            }
             _clipToolResult(result) {
+                const max = this._toolLimit('toolResultChars');
                 const text = typeof result === 'string' ? result : JSON.stringify(result);
-                const max = this.maxToolResultChars || 6000;
                 if (!text || text.length <= max) return text;
+                // A tool envelope is JSON with a `content` field. Clipping the envelope blind deletes
+                // `truncated` / `next_offset`, so paging loses its continuation pointer (measured
+                // 2026-11-02: a 16 000-char page serialises to ~16 080 chars, cut at 6 000).
+                if (text.charAt(0) === '{') {
+                    try {
+                        const obj = JSON.parse(text);
+                        if (obj && typeof obj.content === 'string' && obj.content.length > 0) {
+                            const keep = Math.max(500, max - 200);   // headroom for the other keys
+                            obj.returned_chars = keep;
+                            obj.content = obj.content.slice(0, keep) +
+                                '\n…[clipped: ' + (obj.content.length - keep) + ' of ' + obj.content.length +
+                                ' chars withheld — re-read with offset/limit if you need the rest]';
+                            obj.clipped = true;
+                            return JSON.stringify(obj);
+                        }
+                    } catch (e) { /* not JSON → fall through to the string clip */ }
+                }
                 return text.slice(0, max)
                     + '\n…[clipped: ' + (text.length - max) + ' of ' + text.length
                     + ' chars withheld — re-read with offset/limit if you need the rest]';
@@ -1736,15 +1818,9 @@ buildSystemPrompt() {
                                 toolResult = await this.runTool(fn.name, fn.arguments);
                                 this._cacheSet(fn.name, _cg.key, toolResult);
                             }
-                            const t1 = Math.round((Date.now() - t0) / 1000);
-                            this._toolCallLog.push({
-                                turn: toolTurns,
-                                name: fn.name,
-                                args: fn.arguments,
-                                result: toolResult,
-                                duration: t1,
-                                error: this._isToolError(toolResult) ? true : false
-                            });
+                            const t1 = Date.now() - t0;
+                            this._recordToolCall({ turn: toolTurns, name: fn.name, args: fn.arguments,
+                                result: toolResult, durationMs: t1, error: this._isToolError(toolResult) });
                             _loopLog.push(this._toolCallLog[this._toolCallLog.length - 1]);
                             // Auto-trim to 50 — prevents unbounded memory growth
                             if (this._toolCallLog.length > 50) {
@@ -2475,7 +2551,8 @@ buildSystemPrompt() {
                             // resume. Now the caller pages with offset/limit and gets next_offset.
                             const total = content.length;
                             let offset = Math.max(0, Math.min(parseInt(a.offset, 10) || 0, total));
-                            const limit = Math.max(500, Math.min(parseInt(a.limit, 10) || 8000, 16000));
+                            const pageCap = Math.max(500, Math.min(this._toolLimit('readPageChars'), 200000));
+                            const limit = Math.max(500, Math.min(parseInt(a.limit, 10) || 8000, pageCap));
                             // Never split a surrogate pair (emoji/CJK-ext) across pages:
                             // a lone surrogate makes JSON.stringify emit an invalid body.
                             if (offset > 0) {
@@ -2578,8 +2655,9 @@ buildSystemPrompt() {
                             const contentType = a.content_type || (method === 'POST' ? 'application/json' : '');
                             try {
                                 // Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content)
+                                const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000));
                                 const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract';
-                                const params = new URLSearchParams({ url });
+                                const params = new URLSearchParams({ url, max_chars: String(maxChars) });
                                 if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); }
                                 const resp = await fetch(endpoint + '?' + params.toString());
                                 if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' });
@@ -2590,10 +2668,15 @@ buildSystemPrompt() {
                                 // For POST/route, the response is raw text; for GET/extract it's JSON
                                 if (method === 'POST') {
                                     const text = await resp.text();
-                                    return JSON.stringify({ method, content: text.substring(0, 4000), content_length: text.length });
+                                    return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length,
+                                                            returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars });
                                 }
                                 const data = await resp.json();
-                                return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, 4000), word_count: data.word_count || 0, method_proxy: data.method || 'extract' });
+                                return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars),
+                                                        content_chars: data.content_chars || (data.content || '').length,
+                                                        returned_chars: Math.min((data.content || '').length, maxChars),
+                                                        truncated: !!(data.truncated || (data.content_chars > maxChars)),
+                                                        word_count: data.word_count || 0, method_proxy: data.method || 'extract' });
                             } catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); }
                         }
                         case 'wpm_install': {
@@ -3867,12 +3950,30 @@ buildSystemPrompt() {
             _getAiConfigKeys() {
                 return ['max-turns', 'max-context', 'max-context-tokens', 'confirm-timeout', 'chat-lock-timeout', 'skill-debounce', 'trivial-optimization', 'temperature', 'top-p', 'max-tokens', 'thinking'];
             }
+            aiStats() {
+                let s = { tools: {} };
+                try { s = JSON.parse(localStorage.getItem('webx11_tool_stats') || '{"tools":{}}'); } catch (e) {}
+                const rows = Object.keys(s.tools || {}).sort((a, b) => s.tools[b].calls - s.tools[a].calls);
+                if (!rows.length) return this.terminal.print('ai-stats: no tool calls recorded yet', 'info');
+                this.terminal.print('=== Tool telemetry (this browser) ===', 'info');
+                for (const n of rows) {
+                    const t = s.tools[n];
+                    this.terminal.print(`${n}: ${t.calls} calls · ok ${Math.round(100 * t.ok / t.calls)}% · ` +
+                        `avg ${Math.round(t.ms / t.calls)}ms · returned ${(t.bytes / 1024).toFixed(1)}KB · ` +
+                        `truncated ${t.truncated} (${Math.round(100 * t.truncated / t.calls)}%)`, '');
+                }
+                this.terminal.print('ai-stats reset — clear localStorage webx11_tool_stats', 'info');
+            }
             aiConfig(args) {
                 if (args.length === 0) {
                     this.terminal.print('=== AI Config ===', 'info');
                     this.terminal.print(`max-turns: ${this.MAX_TOOL_TURNS} (default: 8)`, '');
                     this.terminal.print(`max-context: ${this.maxContext} (default: 10)`, '');
                     this.terminal.print(`max-context-tokens: ${this.maxContextTokens} (code default: 12000)`, '');
+this.terminal.print(`read-page-chars: ${this._toolLimit('readPageChars')} (model default applied; code default 16000)`, '');
+this.terminal.print(`read-result-chars: ${this._toolLimit('toolResultChars')} (model default applied; code default 6000)`, '');
+this.terminal.print(`tool-context-budget: ${this._toolLimit('toolContextBudgetChars')} (code default 24000)`, '');
+this.terminal.print(`fetch-chars: ${this._toolLimit('fetchChars')} (code default 4000)`, '');
                     this.terminal.print(`trivial-optimization: ${(this.terminal?.ollamaManager?.optimizationEnabled) ? 'on' : 'off (default)'}`, '');
                     this.terminal.print(`confirm-timeout: ${this._confirmTimeoutMs > 0 ? this._confirmTimeoutMs / 1000 + 's' : 'disabled (no auto-deny)'} (default: 120s)`, '');
                     this.terminal.print(`chat-lock-timeout: ${this._chatLockTimeoutMs / 1000}s (default: 10s)`, '');
@@ -3938,6 +4039,14 @@ buildSystemPrompt() {
                     if (!validModes.includes(val)) return this.terminal.print(`thinking must be one of: ${validModes.join(', ')}`, 'warning');
                     const om = this.terminal?.ollamaManager;
                     if (om && om.modelParams) { om.modelParams.thinking = val; om._saveModelParams(); }
+} else if (key === 'read-page-chars' || key === 'read-result-chars'
+        || key === 'tool-context-budget' || key === 'fetch-chars') {
+    const n = parseInt(val, 10);
+    if (isNaN(n) || n < 500 || n > 200000) return this.terminal.print(key + ' must be 500-200000', 'warning');
+    if (key === 'read-page-chars') this.maxReadPageChars = n;
+    if (key === 'read-result-chars') this.maxToolResultChars = n;
+    if (key === 'tool-context-budget') this.toolContextBudgetChars = n;
+    if (key === 'fetch-chars') this.maxFetchChars = n;
                 } else {
                     return this.terminal.print(`Unknown key: ${key}. Valid: max-turns, max-context, max-context-tokens, confirm-timeout, chat-lock-timeout, skill-debounce, trivial-optimization, temperature, top-p, max-tokens, thinking`, 'warning');
                 }
@@ -3948,7 +4057,11 @@ buildSystemPrompt() {
                     maxContextTokens: this.maxContextTokens,
                     confirmTimeoutMs: this._confirmTimeoutMs,
                     chatLockTimeoutMs: this._chatLockTimeoutMs,
-                    skillDebounceMs: this._skillDebounceMs
+                    skillDebounceMs: this._skillDebounceMs,
+maxReadPageChars: this.maxReadPageChars,
+maxToolResultChars: this.maxToolResultChars,
+toolContextBudgetChars: this.toolContextBudgetChars,
+maxFetchChars: this.maxFetchChars,
                 }));
                 this.terminal.print(`ai-config ${key} set to ${val}`, 'success');
             }