aboutsummaryrefslogtreecommitdiff
path: root/modules
diff options
context:
space:
mode:
Diffstat (limited to 'modules')
-rw-r--r--modules/core/core-bundle.js79
-rw-r--r--modules/core/module-aiassist.js62
-rw-r--r--modules/core/module-fetcher.js17
-rw-r--r--modules/core/module-loader.js2
-rw-r--r--modules/ext/cryptogzip.js67
5 files changed, 189 insertions, 38 deletions
diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js
index c37d13d..4483709 100644
--- a/modules/core/core-bundle.js
+++ b/modules/core/core-bundle.js
@@ -9144,7 +9144,8 @@ buildSystemPrompt() {
method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' },
body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' },
content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' },
-max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' },
+max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; a page is capped by the tool-result budget, so page with offset/next_offset for longer pages)' },
+ offset: { type: 'integer', description: 'Start character offset into the extracted text (default 0). Use the next_offset from the previous reply to continue.' },
},
required: ['url']
}
@@ -10373,6 +10374,11 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
if (keep < obj.content.length) {
const c = obj.content.charCodeAt(keep - 1);
if (c >= 0xD800 && c <= 0xDBFF) keep -= 1;
+ } else if (obj.content.length <= keep) {
+ // The page fits; only the envelope is fat. Do not claim a loss that
+ // did not happen (measured 2026-11-06: "clipped: 0 of 11800 withheld"
+ // appended while every character was returned).
+ return JSON.stringify(obj);
}
const hadMore = (obj.next_offset !== null && obj.next_offset !== undefined);
const full = obj.content.length;
@@ -11624,9 +11630,14 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
const contentType = a.content_type || (method === 'POST' ? 'application/json' : '');
try {
// Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content)
- const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000));
+ // A page larger than toolResultChars-200 is a page the model never sees
+ // (measured 2026-11-06: asked 50 000, received 11 800, no next_offset).
+ const wanted = parseInt(a.max_chars, 10) || this._toolLimit('fetchChars');
+ const clipMax = this._toolLimit('toolResultChars');
+ const limit = Math.max(500, Math.min(wanted, clipMax - 200)); // request window
+ const offset = Math.max(0, parseInt(a.offset, 10) || 0);
const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract';
- const params = new URLSearchParams({ url, max_chars: String(maxChars) });
+ const params = new URLSearchParams({ url, max_chars: String(Math.min(100000, offset + limit)) });
if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); }
const resp = await fetch(endpoint + '?' + params.toString());
if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' });
@@ -11634,18 +11645,47 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
const err = await resp.json().catch(() => ({}));
return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' });
}
- // For POST/route, the response is raw text; for GET/extract it's JSON
+ // POST: /api/fetch returns raw text, so page it by characters.
if (method === 'POST') {
const text = await resp.text();
- return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length,
- returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars });
+ const start = Math.min(offset, text.length);
+ let lim = Math.max(500, Math.min(limit, clipMax - 200));
+ let env;
+ do {
+ const slice = text.slice(start, start + lim);
+ const nextOffset = start + slice.length < text.length ? start + slice.length : null;
+ env = { method, offset: start, content: slice, returned_chars: slice.length,
+ content_chars: text.length, total_chars: text.length, source_bytes: text.length,
+ truncated: nextOffset !== null, next_offset: nextOffset };
+ if (JSON.stringify(env).length <= clipMax || lim <= 500) break;
+ lim -= Math.max(1, JSON.stringify(env).length - clipMax);
+ } while (true);
+ return JSON.stringify(env);
}
+ // GET: /api/extract carries content_chars = the true length of the extracted
+ // text, so offset/next_offset are exact rather than hopeful.
const data = await resp.json();
- return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars),
- content_chars: data.content_chars || (data.content || '').length,
- returned_chars: Math.min((data.content || '').length, maxChars),
- truncated: !!(data.truncated || (data.content_chars > maxChars)),
- word_count: data.word_count || 0, method_proxy: data.method || 'extract' });
+ const full = data.content || '';
+ // UTF-16 length when the proxy supplies it: Python len() counts code points,
+ // JS counts UTF-16 units, and 7 astral chars were enough to leave a 7-char
+ // gap at the end of a 90 KB page (measured 2026-11-06).
+ const totalChars = Number(data.content_chars_utf16) || Number(data.content_chars) || full.length;
+ let start = Math.min(offset, totalChars);
+ if (start > 0) { const prev = full.charCodeAt(start - 1); if (prev >= 0xD800 && prev <= 0xDBFF) start += 1; }
+ let lim = Math.max(500, Math.min(limit, clipMax - 200));
+ let env;
+ do {
+ const slice = full.slice(start, start + lim);
+ const nextOffset = start + slice.length < totalChars ? start + slice.length : null;
+ env = { method, title: data.title || '', offset: start, content: slice, returned_chars: slice.length,
+ content_chars: totalChars, total_chars: totalChars,
+ source_bytes: (data.source_bytes === undefined ? null : data.source_bytes),
+ truncated: nextOffset !== null, next_offset: nextOffset,
+ word_count: data.word_count || 0, method_proxy: data.method || 'extract' };
+ if (JSON.stringify(env).length <= clipMax || lim <= 500) break;
+ lim -= Math.max(1, JSON.stringify(env).length - clipMax);
+ } while (true);
+ return JSON.stringify(env);
} catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); }
}
case 'wpm_install': {
@@ -14062,7 +14102,7 @@ class WebFetcherModule {
}
getHelpInfo() {
return {
- 'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content',
+ 'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] [--save=FILE] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content, --save writes it to the VFS',
'extract': 'extract <url> — Extract readable article content (title + body text) from a URL',
};
}
@@ -14126,6 +14166,21 @@ class WebFetcherModule {
return;
}
const text = await resp.text();
+ // --save: the only terminal path from a remote URL to the VFS. Without it a
+ // 100 KB source page can only be read through 3 000-char scrollback, which is
+ // what sent the in-OS agent looking for a shell redirect (reported 2026-11-06).
+ if (flags.save) {
+ const path = String(flags.save === true ? (url.split('/').pop().split('?')[0] || 'fetch.txt') : flags.save);
+ const dir = path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '';
+ if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} }
+ const wrote = this.terminal.fs.writeFile(path, text);
+ if (!wrote || this.terminal.fs.readFile(path) !== text) {
+ this.terminal.print(`Save failed: could not write ${path}`, 'error');
+ return;
+ }
+ this.terminal.print(`💾 Saved ${text.length} chars to ${path}`, 'success');
+ return;
+ }
const limit = showRaw ? text.length : 3000;
this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success');
this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : ''));
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index 0a0ea71..a3ffb6f 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -226,7 +226,8 @@ buildSystemPrompt() {
method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' },
body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' },
content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' },
-max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; max 50000)' },
+max_chars: { type: 'integer', description: 'Max characters of page content to return (default from the model profile; a page is capped by the tool-result budget, so page with offset/next_offset for longer pages)' },
+ offset: { type: 'integer', description: 'Start character offset into the extracted text (default 0). Use the next_offset from the previous reply to continue.' },
},
required: ['url']
}
@@ -1455,6 +1456,11 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
if (keep < obj.content.length) {
const c = obj.content.charCodeAt(keep - 1);
if (c >= 0xD800 && c <= 0xDBFF) keep -= 1;
+ } else if (obj.content.length <= keep) {
+ // The page fits; only the envelope is fat. Do not claim a loss that
+ // did not happen (measured 2026-11-06: "clipped: 0 of 11800 withheld"
+ // appended while every character was returned).
+ return JSON.stringify(obj);
}
const hadMore = (obj.next_offset !== null && obj.next_offset !== undefined);
const full = obj.content.length;
@@ -2706,9 +2712,14 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
const contentType = a.content_type || (method === 'POST' ? 'application/json' : '');
try {
// Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content)
- const maxChars = Math.max(500, Math.min(parseInt(a.max_chars, 10) || this._toolLimit('fetchChars'), 50000));
+ // A page larger than toolResultChars-200 is a page the model never sees
+ // (measured 2026-11-06: asked 50 000, received 11 800, no next_offset).
+ const wanted = parseInt(a.max_chars, 10) || this._toolLimit('fetchChars');
+ const clipMax = this._toolLimit('toolResultChars');
+ const limit = Math.max(500, Math.min(wanted, clipMax - 200)); // request window
+ const offset = Math.max(0, parseInt(a.offset, 10) || 0);
const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract';
- const params = new URLSearchParams({ url, max_chars: String(maxChars) });
+ const params = new URLSearchParams({ url, max_chars: String(Math.min(100000, offset + limit)) });
if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); }
const resp = await fetch(endpoint + '?' + params.toString());
if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' });
@@ -2716,18 +2727,47 @@ max_chars: { type: 'integer', description: 'Max characters of page content to re
const err = await resp.json().catch(() => ({}));
return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' });
}
- // For POST/route, the response is raw text; for GET/extract it's JSON
+ // POST: /api/fetch returns raw text, so page it by characters.
if (method === 'POST') {
const text = await resp.text();
- return JSON.stringify({ method, content: text.substring(0, maxChars), content_chars: text.length,
- returned_chars: Math.min(text.length, maxChars), truncated: text.length > maxChars });
+ const start = Math.min(offset, text.length);
+ let lim = Math.max(500, Math.min(limit, clipMax - 200));
+ let env;
+ do {
+ const slice = text.slice(start, start + lim);
+ const nextOffset = start + slice.length < text.length ? start + slice.length : null;
+ env = { method, offset: start, content: slice, returned_chars: slice.length,
+ content_chars: text.length, total_chars: text.length, source_bytes: text.length,
+ truncated: nextOffset !== null, next_offset: nextOffset };
+ if (JSON.stringify(env).length <= clipMax || lim <= 500) break;
+ lim -= Math.max(1, JSON.stringify(env).length - clipMax);
+ } while (true);
+ return JSON.stringify(env);
}
+ // GET: /api/extract carries content_chars = the true length of the extracted
+ // text, so offset/next_offset are exact rather than hopeful.
const data = await resp.json();
- return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, maxChars),
- content_chars: data.content_chars || (data.content || '').length,
- returned_chars: Math.min((data.content || '').length, maxChars),
- truncated: !!(data.truncated || (data.content_chars > maxChars)),
- word_count: data.word_count || 0, method_proxy: data.method || 'extract' });
+ const full = data.content || '';
+ // UTF-16 length when the proxy supplies it: Python len() counts code points,
+ // JS counts UTF-16 units, and 7 astral chars were enough to leave a 7-char
+ // gap at the end of a 90 KB page (measured 2026-11-06).
+ const totalChars = Number(data.content_chars_utf16) || Number(data.content_chars) || full.length;
+ let start = Math.min(offset, totalChars);
+ if (start > 0) { const prev = full.charCodeAt(start - 1); if (prev >= 0xD800 && prev <= 0xDBFF) start += 1; }
+ let lim = Math.max(500, Math.min(limit, clipMax - 200));
+ let env;
+ do {
+ const slice = full.slice(start, start + lim);
+ const nextOffset = start + slice.length < totalChars ? start + slice.length : null;
+ env = { method, title: data.title || '', offset: start, content: slice, returned_chars: slice.length,
+ content_chars: totalChars, total_chars: totalChars,
+ source_bytes: (data.source_bytes === undefined ? null : data.source_bytes),
+ truncated: nextOffset !== null, next_offset: nextOffset,
+ word_count: data.word_count || 0, method_proxy: data.method || 'extract' };
+ if (JSON.stringify(env).length <= clipMax || lim <= 500) break;
+ lim -= Math.max(1, JSON.stringify(env).length - clipMax);
+ } while (true);
+ return JSON.stringify(env);
} catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); }
}
case 'wpm_install': {
diff --git a/modules/core/module-fetcher.js b/modules/core/module-fetcher.js
index 45505a7..d1b7bcc 100644
--- a/modules/core/module-fetcher.js
+++ b/modules/core/module-fetcher.js
@@ -5,7 +5,7 @@ class WebFetcherModule {
}
getHelpInfo() {
return {
- 'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content',
+ 'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] [--save=FILE] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content, --save writes it to the VFS',
'extract': 'extract <url> — Extract readable article content (title + body text) from a URL',
};
}
@@ -69,6 +69,21 @@ class WebFetcherModule {
return;
}
const text = await resp.text();
+ // --save: the only terminal path from a remote URL to the VFS. Without it a
+ // 100 KB source page can only be read through 3 000-char scrollback, which is
+ // what sent the in-OS agent looking for a shell redirect (reported 2026-11-06).
+ if (flags.save) {
+ const path = String(flags.save === true ? (url.split('/').pop().split('?')[0] || 'fetch.txt') : flags.save);
+ const dir = path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '';
+ if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} }
+ const wrote = this.terminal.fs.writeFile(path, text);
+ if (!wrote || this.terminal.fs.readFile(path) !== text) {
+ this.terminal.print(`Save failed: could not write ${path}`, 'error');
+ return;
+ }
+ this.terminal.print(`💾 Saved ${text.length} chars to ${path}`, 'success');
+ return;
+ }
const limit = showRaw ? text.length : 3000;
this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success');
this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : ''));
diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js
index 1005463..1f931fd 100644
--- a/modules/core/module-loader.js
+++ b/modules/core/module-loader.js
@@ -4,7 +4,7 @@
// Core module bundle loaded synchronously in main.html.
// This loader only handles version cache-busting, splash, terminal init, and optional ext modules.
- const LATEST_VER = 'v4.6.150';
+ const LATEST_VER = 'v4.6.151';
const CACHE = '?v=' + LATEST_VER;
// Exposed so ext modules can register their own URL with the same cache-busting version
// instead of an unversioned path the browser may already have cached (shell.js did).
diff --git a/modules/ext/cryptogzip.js b/modules/ext/cryptogzip.js
index 200abdd..cc275df 100644
--- a/modules/ext/cryptogzip.js
+++ b/modules/ext/cryptogzip.js
@@ -15,7 +15,8 @@ class CryptoGzipModule {
'decrypt': 'decrypt <path.enc> <passphrase> — Decrypt a .enc file',
'tar': 'tar <src-dir> <out.tar[.gz]> — Bundle a VFS directory into a tar archive (gzip if .gz)',
'untar': 'untar <path.tar[.gz]> [dest-dir] — Extract a tar archive into VFS (auto-detects gzip)',
- 'wget': 'wget <url> [filename] — Download a URL and save to VFS'
+ 'wget': 'wget <url> [filename] — Download a URL (via /api/fetch) and save it to VFS',
+ 'curl': 'curl [-s] [-o FILE] <url> — Same as wget: proxy fetch + verified VFS write'
};
}
@@ -342,7 +343,10 @@ class CryptoGzipModule {
}
// ============================================================
- // WGET — fetch URL and save to VFS (binary-safe)
+ // WGET — fetch a URL through the /api/fetch proxy and save to VFS
+ // A direct browser fetch() only works same-origin: cross-origin URLs
+ // throw "Failed to fetch" (CORS), measured 2026-11-06 on the served
+ // v4.6.150 page. /api/fetch returns the raw source as UTF-8 text.
// ============================================================
async _cmdWget(args) {
@@ -350,20 +354,40 @@ class CryptoGzipModule {
this.terminal.print('Usage: wget <url> [filename] — Download a URL and save to VFS', 'warning');
return;
}
- const url = args[0];
- const filename = args[1] || url.split('/').pop() || 'download.bin';
+ const url = /^https?:\/\//i.test(args[0]) ? args[0] : 'https://' + args[0];
+ const filename = args[1] || url.split('/').pop().split('?')[0] || 'download.bin';
this.terminal.print(`Downloading ${url}...`, 'info');
+ let text;
try {
- const resp = await fetch(url);
- if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
- const buf = await resp.arrayBuffer();
- const bytes = new Uint8Array(buf);
- this._vfsWriteBytes(filename, bytes);
- const sizeKb = (bytes.length / 1024).toFixed(1);
- this.terminal.print(`✅ wget: ${filename} (${sizeKb} KB, ${bytes.length} bytes)`, 'success');
+ const resp = await fetch('/api/fetch?url=' + encodeURIComponent(url));
+ if (resp.status === 403) {
+ this.terminal.print('wget error: URL blocked — internal network', 'error');
+ return;
+ }
+ if (!resp.ok) {
+ let detail = 'HTTP ' + resp.status;
+ try { const j = await resp.json(); detail = j.error || detail; } catch (e) {}
+ this.terminal.print('wget error: ' + detail, 'error');
+ return;
+ }
+ text = await resp.text();
} catch (e) {
- this.terminal.print(`wget error: ${e.message}`, 'error');
+ this.terminal.print('wget error: ' + e.message, 'error');
+ return;
+ }
+ // writeFile() returns false rather than throwing when the parent directory is
+ // missing — the old code printed ✅ on that path (measured: writeFile('nosuchdir/x.txt') === false).
+ const dir = filename.includes('/') ? filename.slice(0, filename.lastIndexOf('/')) : '';
+ if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} }
+ if (!this.terminal.fs.writeFile(filename, text)) {
+ this.terminal.print(`wget error: could not write ${filename} (missing directory, or path is a directory)`, 'error');
+ return;
}
+ if (this.terminal.fs.readFile(filename) !== text) {
+ this.terminal.print(`wget error: write verification failed for ${filename}`, 'error');
+ return;
+ }
+ this.terminal.print(`✅ wget: ${filename} (${(text.length / 1024).toFixed(1)} KB, ${text.length} chars)`, 'success');
}
// ============================================================
@@ -582,6 +606,22 @@ class CryptoGzipModule {
this.commands['tar'] = async (args) => self._cmdTar(args);
this.commands['untar'] = async (args) => self._cmdUntar(args);
this.commands['wget'] = async (args) => self._cmdWget(args);
+ // curl — the flags an agent actually types. No shell semantics: -s/-L/-f are
+ // accepted and ignored, -o/--output selects the VFS target. Delegates to the
+ // single verified writer above so the two cannot drift.
+ this.commands['curl'] = async (args) => {
+ const positional = [];
+ let out = null;
+ for (let i = 0; i < args.length; i++) {
+ const a = String(args[i]);
+ if (a === '-o' || a === '-O' || a === '--output') { out = args[++i]; }
+ else if (a.startsWith('-')) { /* accepted, ignored */ }
+ else positional.push(a);
+ }
+ if (!positional[0]) { this.terminal.print('Usage: curl [-s] [-o FILE] <url>', 'warning'); return; }
+ const target = out || (positional[0].split('/').pop().split('?')[0] || 'download.bin');
+ return self._cmdWget([positional[0], target]);
+ };
this.commandsMeta = {
gzip: { argType: 'file', description: 'Compress a VFS file to .gz' },
gunzip: { argType: 'file', description: 'Decompress a .gz file (binary-safe)' },
@@ -589,7 +629,8 @@ class CryptoGzipModule {
decrypt: { argType: 'file', description: 'Decrypt a .enc file' },
tar: { argType: 'file', description: 'Bundle a VFS directory into a tar archive' },
untar: { argType: 'file', description: 'Extract a tar archive into VFS' },
- wget: { argType: 'url', description: 'Download a URL and save to VFS' }
+ wget: { argType: 'url', description: 'Download a URL and save to VFS' },
+ curl: { argType: 'url', description: 'curl [-s] [-o FILE] <url> — same as wget' }
};
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-06 21:29:44 +0000