diff options
Diffstat (limited to 'modules/core/module-aiassist.js')
| -rw-r--r-- | modules/core/module-aiassist.js | 62 |
1 files changed, 51 insertions, 11 deletions
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index 0a0ea71..a3ffb6f 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -226,7 +226,8 @@ buildSystemPrompt() { method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' }, body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' }, content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' }, -max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' }, +max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; a page is capped by the tool-result budget, so page with offset/next_offset for longer pages)' }, + offset: { type: 'integer', description: 'Start character offset into the extracted text (default 0). Use the next_offset from the previous reply to continue.' }, }, required: ['url'] } @@ -1455,6 +1456,11 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re if (keep < obj.content.length) { const c = obj.content.charCodeAt(keep - 1); if (c >= 0xD800 && c <= 0xDBFF) keep -= 1; + } else if (obj.content.length <= keep) { + // The page fits; only the envelope is fat. Do not claim a loss that + // did not happen (measured 2026-11-06: "clipped: 0 of 11800 withheld" + // appended while every character was returned). + return JSON.stringify(obj); } const hadMore = (obj.next_offset !== null && obj.next_offset !== undefined); const full = obj.content.length; @@ -2706,9 +2712,14 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const contentType = a.content_type || (method === 'POST' ? 'application/json' : ''); try { // Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content) - const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000)); + // A page larger than toolResultChars-200 is a page the model never sees + // (measured 2026-11-06: asked 50 000, received 11 800, no next_offset). + const wanted = parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'); + const clipMax = this._toolLimit('toolResultChars'); + const limit = Math.max(500, Math.min(wanted, clipMax - 200)); // request window + const offset = Math.max(0, parseInt(a.offset, 10) || 0); const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract'; - const params = new URLSearchParams({ url, max_chars: String(maxChars) }); + const params = new URLSearchParams({ url, max_chars: String(Math.min(100000, offset + limit)) }); if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); } const resp = await fetch(endpoint + '?' + params.toString()); if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' }); @@ -2716,18 +2727,47 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const err = await resp.json().catch(() => ({})); return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' }); } - // For POST/route, the response is raw text; for GET/extract it's JSON + // POST: /api/fetch returns raw text, so page it by characters. if (method === 'POST') { const text = await resp.text(); - return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length, - returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars }); + const start = Math.min(offset, text.length); + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = text.slice(start, start + lim); + const nextOffset = start + slice.length < text.length ? start + slice.length : null; + env = { method, offset: start, content: slice, returned_chars: slice.length, + content_chars: text.length, total_chars: text.length, source_bytes: text.length, + truncated: nextOffset !== null, next_offset: nextOffset }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } + // GET: /api/extract carries content_chars = the true length of the extracted + // text, so offset/next_offset are exact rather than hopeful. const data = await resp.json(); - return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars), - content_chars: data.content_chars || (data.content || '').length, - returned_chars: Math.min((data.content || '').length, maxChars), - truncated: !!(data.truncated || (data.content_chars > maxChars)), - word_count: data.word_count || 0, method_proxy: data.method || 'extract' }); + const full = data.content || ''; + // UTF-16 length when the proxy supplies it: Python len() counts code points, + // JS counts UTF-16 units, and 7 astral chars were enough to leave a 7-char + // gap at the end of a 90 KB page (measured 2026-11-06). + const totalChars = Number(data.content_chars_utf16) || Number(data.content_chars) || full.length; + let start = Math.min(offset, totalChars); + if (start > 0) { const prev = full.charCodeAt(start - 1); if (prev >= 0xD800 && prev <= 0xDBFF) start += 1; } + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = full.slice(start, start + lim); + const nextOffset = start + slice.length < totalChars ? start + slice.length : null; + env = { method, title: data.title || '', offset: start, content: slice, returned_chars: slice.length, + content_chars: totalChars, total_chars: totalChars, + source_bytes: (data.source_bytes === undefined ? null : data.source_bytes), + truncated: nextOffset !== null, next_offset: nextOffset, + word_count: data.word_count || 0, method_proxy: data.method || 'extract' }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); } } case 'wpm_install': { |