From 0cb252dfbeb02d23cd92bad3ca8df76be22d9f92 Mon Sep 17 00:00:00 2001 From: Agando Date: Fri, 6 Nov 2026 16:19:59 +0100 Subject: release: v4.6.151 — wget that actually saves, web_fetch that actually pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S1 (reported by agent-os4-maint-01, wrongly refuted by the chat auto-reply): wget existed but was unusable and dishonest. - Cause 1: _cmdWget used the page's own fetch(url), so every cross-origin URL died on CORS ("Failed to fetch"). Measured in the served v4.6.150 page: fetch(cgit README) threw, /api/fetch returned 200/1954 bytes. - Cause 2: _vfsWriteBytes ignored writeFile()'s boolean, so the command printed a green tick when the target directory did not exist. Measured: fs.writeFile('nosuchdir/x.txt') === false. - Cause 3: bytes were round-tripped through String.fromCharCode, mangling UTF-8. - Fix: fetch through /api/fetch (raw source), mkdir -p, writeFile + read-back verification, error instead of a tick on failure. Register curl with -s/-o parsing delegating to the same writer so the two cannot drift. S2 (same reporter): web_fetch could not page and said it could. - Cause: the handler honoured max_chars up to 50 000, then _clipToolResult kept toolResultChars-200 = 11 800, so 38 200 chars vanished silently; offset was not in the schema and was never read, so next_offset pointed at a page the caller could not request again. - Fix: offset/next_offset/total_chars/source_bytes in the contract; the page is built, measured against the clip budget and shrunk until the envelope fits, so next_offset - returned_chars is always 0. - Also fixed: _clipToolResult appended "0 of N chars withheld" when the content already fitted, claiming a loss that did not happen. C1: fetch --save=FILE — the only terminal path from a remote URL to the VFS, so a 90 KB source page no longer has to be read through 3 000-char scrollback. Proxy: /api/extract now returns source_bytes (raw page size) and content_chars_utf16. Python len() counts code points while JS String.length counts UTF-16 units; 7 astral characters in the changelog left a 7-char gap at the end of a 90 KB page (measured). _html_ok declares charset=utf-8. Verified: node --check on all four modules; py_compile; bundles rebuilt; served bytes identical to disk for all five assets at ?v=v4.6.151; live in the browser: curl -s -o review/main.md saved 1954 chars with the correct head, a 404 wrote no file, fetch --save wrote 91 518 chars, web_fetch p1 returned 11 754 with next_offset 11 754 (gap 0), page 2 started at a different offset with next 23 496. --- modules/core/core-bundle.js | 79 ++++++++++++++++++++++++++++++++++------- modules/core/module-aiassist.js | 62 ++++++++++++++++++++++++++------ modules/core/module-fetcher.js | 17 ++++++++- modules/core/module-loader.js | 2 +- modules/ext/cryptogzip.js | 67 +++++++++++++++++++++++++++------- 5 files changed, 189 insertions(+), 38 deletions(-) (limited to 'modules') diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index c37d13d..4483709 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -9144,7 +9144,8 @@ buildSystemPrompt() { method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' }, body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' }, content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' }, -max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' }, +max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; a page is capped by the tool-result budget, so page with offset/next_offset for longer pages)' }, + offset: { type: 'integer', description: 'Start character offset into the extracted text (default 0). Use the next_offset from the previous reply to continue.' }, }, required: ['url'] } @@ -10373,6 +10374,11 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re if (keep < obj.content.length) { const c = obj.content.charCodeAt(keep - 1); if (c >= 0xD800 && c <= 0xDBFF) keep -= 1; + } else if (obj.content.length <= keep) { + // The page fits; only the envelope is fat. Do not claim a loss that + // did not happen (measured 2026-11-06: "clipped: 0 of 11800 withheld" + // appended while every character was returned). + return JSON.stringify(obj); } const hadMore = (obj.next_offset !== null && obj.next_offset !== undefined); const full = obj.content.length; @@ -11624,9 +11630,14 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const contentType = a.content_type || (method === 'POST' ? 'application/json' : ''); try { // Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content) - const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000)); + // A page larger than toolResultChars-200 is a page the model never sees + // (measured 2026-11-06: asked 50 000, received 11 800, no next_offset). + const wanted = parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'); + const clipMax = this._toolLimit('toolResultChars'); + const limit = Math.max(500, Math.min(wanted, clipMax - 200)); // request window + const offset = Math.max(0, parseInt(a.offset, 10) || 0); const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract'; - const params = new URLSearchParams({ url, max_chars: String(maxChars) }); + const params = new URLSearchParams({ url, max_chars: String(Math.min(100000, offset + limit)) }); if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); } const resp = await fetch(endpoint + '?' + params.toString()); if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' }); @@ -11634,18 +11645,47 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const err = await resp.json().catch(() => ({})); return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' }); } - // For POST/route, the response is raw text; for GET/extract it's JSON + // POST: /api/fetch returns raw text, so page it by characters. if (method === 'POST') { const text = await resp.text(); - return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length, - returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars }); + const start = Math.min(offset, text.length); + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = text.slice(start, start + lim); + const nextOffset = start + slice.length < text.length ? start + slice.length : null; + env = { method, offset: start, content: slice, returned_chars: slice.length, + content_chars: text.length, total_chars: text.length, source_bytes: text.length, + truncated: nextOffset !== null, next_offset: nextOffset }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } + // GET: /api/extract carries content_chars = the true length of the extracted + // text, so offset/next_offset are exact rather than hopeful. const data = await resp.json(); - return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars), - content_chars: data.content_chars || (data.content || '').length, - returned_chars: Math.min((data.content || '').length, maxChars), - truncated: !!(data.truncated || (data.content_chars > maxChars)), - word_count: data.word_count || 0, method_proxy: data.method || 'extract' }); + const full = data.content || ''; + // UTF-16 length when the proxy supplies it: Python len() counts code points, + // JS counts UTF-16 units, and 7 astral chars were enough to leave a 7-char + // gap at the end of a 90 KB page (measured 2026-11-06). + const totalChars = Number(data.content_chars_utf16) || Number(data.content_chars) || full.length; + let start = Math.min(offset, totalChars); + if (start > 0) { const prev = full.charCodeAt(start - 1); if (prev >= 0xD800 && prev <= 0xDBFF) start += 1; } + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = full.slice(start, start + lim); + const nextOffset = start + slice.length < totalChars ? start + slice.length : null; + env = { method, title: data.title || '', offset: start, content: slice, returned_chars: slice.length, + content_chars: totalChars, total_chars: totalChars, + source_bytes: (data.source_bytes === undefined ? null : data.source_bytes), + truncated: nextOffset !== null, next_offset: nextOffset, + word_count: data.word_count || 0, method_proxy: data.method || 'extract' }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); } } case 'wpm_install': { @@ -14062,7 +14102,7 @@ class WebFetcherModule { } getHelpInfo() { return { - 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content', + 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] [--save=FILE] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content, --save writes it to the VFS', 'extract': 'extract — Extract readable article content (title + body text) from a URL', }; } @@ -14126,6 +14166,21 @@ class WebFetcherModule { return; } const text = await resp.text(); + // --save: the only terminal path from a remote URL to the VFS. Without it a + // 100 KB source page can only be read through 3 000-char scrollback, which is + // what sent the in-OS agent looking for a shell redirect (reported 2026-11-06). + if (flags.save) { + const path = String(flags.save === true ? (url.split('/').pop().split('?')[0] || 'fetch.txt') : flags.save); + const dir = path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : ''; + if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} } + const wrote = this.terminal.fs.writeFile(path, text); + if (!wrote || this.terminal.fs.readFile(path) !== text) { + this.terminal.print(`Save failed: could not write ${path}`, 'error'); + return; + } + this.terminal.print(`💾 Saved ${text.length} chars to ${path}`, 'success'); + return; + } const limit = showRaw ? text.length : 3000; this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success'); this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : '')); diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index 0a0ea71..a3ffb6f 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -226,7 +226,8 @@ buildSystemPrompt() { method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' }, body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' }, content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' }, -max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' }, +max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; a page is capped by the tool-result budget, so page with offset/next_offset for longer pages)' }, + offset: { type: 'integer', description: 'Start character offset into the extracted text (default 0). Use the next_offset from the previous reply to continue.' }, }, required: ['url'] } @@ -1455,6 +1456,11 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re if (keep < obj.content.length) { const c = obj.content.charCodeAt(keep - 1); if (c >= 0xD800 && c <= 0xDBFF) keep -= 1; + } else if (obj.content.length <= keep) { + // The page fits; only the envelope is fat. Do not claim a loss that + // did not happen (measured 2026-11-06: "clipped: 0 of 11800 withheld" + // appended while every character was returned). + return JSON.stringify(obj); } const hadMore = (obj.next_offset !== null && obj.next_offset !== undefined); const full = obj.content.length; @@ -2706,9 +2712,14 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const contentType = a.content_type || (method === 'POST' ? 'application/json' : ''); try { // Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content) - const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000)); + // A page larger than toolResultChars-200 is a page the model never sees + // (measured 2026-11-06: asked 50 000, received 11 800, no next_offset). + const wanted = parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'); + const clipMax = this._toolLimit('toolResultChars'); + const limit = Math.max(500, Math.min(wanted, clipMax - 200)); // request window + const offset = Math.max(0, parseInt(a.offset, 10) || 0); const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract'; - const params = new URLSearchParams({ url, max_chars: String(maxChars) }); + const params = new URLSearchParams({ url, max_chars: String(Math.min(100000, offset + limit)) }); if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); } const resp = await fetch(endpoint + '?' + params.toString()); if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' }); @@ -2716,18 +2727,47 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re const err = await resp.json().catch(() => ({})); return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' }); } - // For POST/route, the response is raw text; for GET/extract it's JSON + // POST: /api/fetch returns raw text, so page it by characters. if (method === 'POST') { const text = await resp.text(); - return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length, - returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars }); + const start = Math.min(offset, text.length); + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = text.slice(start, start + lim); + const nextOffset = start + slice.length < text.length ? start + slice.length : null; + env = { method, offset: start, content: slice, returned_chars: slice.length, + content_chars: text.length, total_chars: text.length, source_bytes: text.length, + truncated: nextOffset !== null, next_offset: nextOffset }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } + // GET: /api/extract carries content_chars = the true length of the extracted + // text, so offset/next_offset are exact rather than hopeful. const data = await resp.json(); - return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars), - content_chars: data.content_chars || (data.content || '').length, - returned_chars: Math.min((data.content || '').length, maxChars), - truncated: !!(data.truncated || (data.content_chars > maxChars)), - word_count: data.word_count || 0, method_proxy: data.method || 'extract' }); + const full = data.content || ''; + // UTF-16 length when the proxy supplies it: Python len() counts code points, + // JS counts UTF-16 units, and 7 astral chars were enough to leave a 7-char + // gap at the end of a 90 KB page (measured 2026-11-06). + const totalChars = Number(data.content_chars_utf16) || Number(data.content_chars) || full.length; + let start = Math.min(offset, totalChars); + if (start > 0) { const prev = full.charCodeAt(start - 1); if (prev >= 0xD800 && prev <= 0xDBFF) start += 1; } + let lim = Math.max(500, Math.min(limit, clipMax - 200)); + let env; + do { + const slice = full.slice(start, start + lim); + const nextOffset = start + slice.length < totalChars ? start + slice.length : null; + env = { method, title: data.title || '', offset: start, content: slice, returned_chars: slice.length, + content_chars: totalChars, total_chars: totalChars, + source_bytes: (data.source_bytes === undefined ? null : data.source_bytes), + truncated: nextOffset !== null, next_offset: nextOffset, + word_count: data.word_count || 0, method_proxy: data.method || 'extract' }; + if (JSON.stringify(env).length <= clipMax || lim <= 500) break; + lim -= Math.max(1, JSON.stringify(env).length - clipMax); + } while (true); + return JSON.stringify(env); } catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); } } case 'wpm_install': { diff --git a/modules/core/module-fetcher.js b/modules/core/module-fetcher.js index 45505a7..d1b7bcc 100644 --- a/modules/core/module-fetcher.js +++ b/modules/core/module-fetcher.js @@ -5,7 +5,7 @@ class WebFetcherModule { } getHelpInfo() { return { - 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content', + 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] [--save=FILE] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content, --save writes it to the VFS', 'extract': 'extract — Extract readable article content (title + body text) from a URL', }; } @@ -69,6 +69,21 @@ class WebFetcherModule { return; } const text = await resp.text(); + // --save: the only terminal path from a remote URL to the VFS. Without it a + // 100 KB source page can only be read through 3 000-char scrollback, which is + // what sent the in-OS agent looking for a shell redirect (reported 2026-11-06). + if (flags.save) { + const path = String(flags.save === true ? (url.split('/').pop().split('?')[0] || 'fetch.txt') : flags.save); + const dir = path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : ''; + if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} } + const wrote = this.terminal.fs.writeFile(path, text); + if (!wrote || this.terminal.fs.readFile(path) !== text) { + this.terminal.print(`Save failed: could not write ${path}`, 'error'); + return; + } + this.terminal.print(`💾 Saved ${text.length} chars to ${path}`, 'success'); + return; + } const limit = showRaw ? text.length : 3000; this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success'); this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : '')); diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index 1005463..1f931fd 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -4,7 +4,7 @@ // Core module bundle loaded synchronously in main.html. // This loader only handles version cache-busting, splash, terminal init, and optional ext modules. - const LATEST_VER = 'v4.6.150'; + const LATEST_VER = 'v4.6.151'; const CACHE = '?v=' + LATEST_VER; // Exposed so ext modules can register their own URL with the same cache-busting version // instead of an unversioned path the browser may already have cached (shell.js did). diff --git a/modules/ext/cryptogzip.js b/modules/ext/cryptogzip.js index 200abdd..cc275df 100644 --- a/modules/ext/cryptogzip.js +++ b/modules/ext/cryptogzip.js @@ -15,7 +15,8 @@ class CryptoGzipModule { 'decrypt': 'decrypt — Decrypt a .enc file', 'tar': 'tar — Bundle a VFS directory into a tar archive (gzip if .gz)', 'untar': 'untar [dest-dir] — Extract a tar archive into VFS (auto-detects gzip)', - 'wget': 'wget [filename] — Download a URL and save to VFS' + 'wget': 'wget [filename] — Download a URL (via /api/fetch) and save it to VFS', + 'curl': 'curl [-s] [-o FILE] — Same as wget: proxy fetch + verified VFS write' }; } @@ -342,7 +343,10 @@ class CryptoGzipModule { } // ============================================================ - // WGET — fetch URL and save to VFS (binary-safe) + // WGET — fetch a URL through the /api/fetch proxy and save to VFS + // A direct browser fetch() only works same-origin: cross-origin URLs + // throw "Failed to fetch" (CORS), measured 2026-11-06 on the served + // v4.6.150 page. /api/fetch returns the raw source as UTF-8 text. // ============================================================ async _cmdWget(args) { @@ -350,20 +354,40 @@ class CryptoGzipModule { this.terminal.print('Usage: wget [filename] — Download a URL and save to VFS', 'warning'); return; } - const url = args[0]; - const filename = args[1] || url.split('/').pop() || 'download.bin'; + const url = /^https?:\/\//i.test(args[0]) ? args[0] : 'https://' + args[0]; + const filename = args[1] || url.split('/').pop().split('?')[0] || 'download.bin'; this.terminal.print(`Downloading ${url}...`, 'info'); + let text; try { - const resp = await fetch(url); - if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`); - const buf = await resp.arrayBuffer(); - const bytes = new Uint8Array(buf); - this._vfsWriteBytes(filename, bytes); - const sizeKb = (bytes.length / 1024).toFixed(1); - this.terminal.print(`✅ wget: ${filename} (${sizeKb} KB, ${bytes.length} bytes)`, 'success'); + const resp = await fetch('/api/fetch?url=' + encodeURIComponent(url)); + if (resp.status === 403) { + this.terminal.print('wget error: URL blocked — internal network', 'error'); + return; + } + if (!resp.ok) { + let detail = 'HTTP ' + resp.status; + try { const j = await resp.json(); detail = j.error || detail; } catch (e) {} + this.terminal.print('wget error: ' + detail, 'error'); + return; + } + text = await resp.text(); } catch (e) { - this.terminal.print(`wget error: ${e.message}`, 'error'); + this.terminal.print('wget error: ' + e.message, 'error'); + return; + } + // writeFile() returns false rather than throwing when the parent directory is + // missing — the old code printed ✅ on that path (measured: writeFile('nosuchdir/x.txt') === false). + const dir = filename.includes('/') ? filename.slice(0, filename.lastIndexOf('/')) : ''; + if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} } + if (!this.terminal.fs.writeFile(filename, text)) { + this.terminal.print(`wget error: could not write ${filename} (missing directory, or path is a directory)`, 'error'); + return; } + if (this.terminal.fs.readFile(filename) !== text) { + this.terminal.print(`wget error: write verification failed for ${filename}`, 'error'); + return; + } + this.terminal.print(`✅ wget: ${filename} (${(text.length / 1024).toFixed(1)} KB, ${text.length} chars)`, 'success'); } // ============================================================ @@ -582,6 +606,22 @@ class CryptoGzipModule { this.commands['tar'] = async (args) => self._cmdTar(args); this.commands['untar'] = async (args) => self._cmdUntar(args); this.commands['wget'] = async (args) => self._cmdWget(args); + // curl — the flags an agent actually types. No shell semantics: -s/-L/-f are + // accepted and ignored, -o/--output selects the VFS target. Delegates to the + // single verified writer above so the two cannot drift. + this.commands['curl'] = async (args) => { + const positional = []; + let out = null; + for (let i = 0; i < args.length; i++) { + const a = String(args[i]); + if (a === '-o' || a === '-O' || a === '--output') { out = args[++i]; } + else if (a.startsWith('-')) { /* accepted, ignored */ } + else positional.push(a); + } + if (!positional[0]) { this.terminal.print('Usage: curl [-s] [-o FILE] ', 'warning'); return; } + const target = out || (positional[0].split('/').pop().split('?')[0] || 'download.bin'); + return self._cmdWget([positional[0], target]); + }; this.commandsMeta = { gzip: { argType: 'file', description: 'Compress a VFS file to .gz' }, gunzip: { argType: 'file', description: 'Decompress a .gz file (binary-safe)' }, @@ -589,7 +629,8 @@ class CryptoGzipModule { decrypt: { argType: 'file', description: 'Decrypt a .enc file' }, tar: { argType: 'file', description: 'Bundle a VFS directory into a tar archive' }, untar: { argType: 'file', description: 'Extract a tar archive into VFS' }, - wget: { argType: 'url', description: 'Download a URL and save to VFS' } + wget: { argType: 'url', description: 'Download a URL and save to VFS' }, + curl: { argType: 'url', description: 'curl [-s] [-o FILE] — same as wget' } }; } } -- cgit v1.2.3