release: v4.6.151 — wget that actually saves, web_fetch that actually pagesHEADv4.6.151main

S1 (reported by agent-os4-maint-01, wrongly refuted by the chat auto-reply):
wget existed but was unusable and dishonest.
- Cause 1: _cmdWget used the page's own fetch(url), so every cross-origin URL
  died on CORS ("Failed to fetch"). Measured in the served v4.6.150 page:
  fetch(cgit README) threw, /api/fetch returned 200/1954 bytes.
- Cause 2: _vfsWriteBytes ignored writeFile()'s boolean, so the command printed
  a green tick when the target directory did not exist. Measured:
  fs.writeFile('nosuchdir/x.txt') === false.
- Cause 3: bytes were round-tripped through String.fromCharCode, mangling UTF-8.
- Fix: fetch through /api/fetch (raw source), mkdir -p, writeFile + read-back
  verification, error instead of a tick on failure. Register curl with
  -s/-o parsing delegating to the same writer so the two cannot drift.

S2 (same reporter): web_fetch could not page and said it could.
- Cause: the handler honoured max_chars up to 50 000, then _clipToolResult kept
  toolResultChars-200 = 11 800, so 38 200 chars vanished silently; offset was
  not in the schema and was never read, so next_offset pointed at a page the
  caller could not request again.
- Fix: offset/next_offset/total_chars/source_bytes in the contract; the page is
  built, measured against the clip budget and shrunk until the envelope fits,
  so next_offset - returned_chars is always 0.
- Also fixed: _clipToolResult appended "0 of N chars withheld" when the content
  already fitted, claiming a loss that did not happen.

C1: fetch <url> --save=FILE — the only terminal path from a remote URL to the
VFS, so a 90 KB source page no longer has to be read through 3 000-char
scrollback.

Proxy: /api/extract now returns source_bytes (raw page size) and
content_chars_utf16. Python len() counts code points while JS String.length
counts UTF-16 units; 7 astral characters in the changelog left a 7-char gap at
the end of a 90 KB page (measured). _html_ok declares charset=utf-8.

Verified: node --check on all four modules; py_compile; bundles rebuilt;
served bytes identical to disk for all five assets at ?v=v4.6.151; live in the
browser: curl -s -o review/main.md <cross-origin> saved 1954 chars with the
correct head, a 404 wrote no file, fetch --save wrote 91 518 chars,
web_fetch p1 returned 11 754 with next_offset 11 754 (gap 0), page 2 started at
a different offset with next 23 496.

1 files changed, 54 insertions, 13 deletions

diff --git a/modules/ext/cryptogzip.js b/modules/ext/cryptogzip.js
index 200abdd..cc275df 100644
--- a/modules/ext/cryptogzip.js
+++ b/modules/ext/cryptogzip.js
@@ -15,7 +15,8 @@ class CryptoGzipModule {
             'decrypt': 'decrypt <path.enc> <passphrase> — Decrypt a .enc file',
             'tar': 'tar <src-dir> <out.tar[.gz]> — Bundle a VFS directory into a tar archive (gzip if .gz)',
             'untar': 'untar <path.tar[.gz]> [dest-dir] — Extract a tar archive into VFS (auto-detects gzip)',
-            'wget': 'wget <url> [filename] — Download a URL and save to VFS'
+            'wget': 'wget <url> [filename] — Download a URL (via /api/fetch) and save it to VFS',
+            'curl': 'curl [-s] [-o FILE] <url> — Same as wget: proxy fetch + verified VFS write'
         };
     }
 
@@ -342,7 +343,10 @@ class CryptoGzipModule {
     }
 
     // ============================================================
-    // WGET — fetch URL and save to VFS (binary-safe)
+    // WGET — fetch a URL through the /api/fetch proxy and save to VFS
+    // A direct browser fetch() only works same-origin: cross-origin URLs
+    // throw "Failed to fetch" (CORS), measured 2026-11-06 on the served
+    // v4.6.150 page. /api/fetch returns the raw source as UTF-8 text.
     // ============================================================
 
     async _cmdWget(args) {
@@ -350,20 +354,40 @@ class CryptoGzipModule {
             this.terminal.print('Usage: wget <url> [filename] — Download a URL and save to VFS', 'warning');
             return;
         }
-        const url = args[0];
-        const filename = args[1] || url.split('/').pop() || 'download.bin';
+        const url = /^https?:\/\//i.test(args[0]) ? args[0] : 'https://' + args[0];
+        const filename = args[1] || url.split('/').pop().split('?')[0] || 'download.bin';
         this.terminal.print(`Downloading ${url}...`, 'info');
+        let text;
         try {
-            const resp = await fetch(url);
-            if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
-            const buf = await resp.arrayBuffer();
-            const bytes = new Uint8Array(buf);
-            this._vfsWriteBytes(filename, bytes);
-            const sizeKb = (bytes.length / 1024).toFixed(1);
-            this.terminal.print(`✅ wget: ${filename} (${sizeKb} KB, ${bytes.length} bytes)`, 'success');
+            const resp = await fetch('/api/fetch?url=' + encodeURIComponent(url));
+            if (resp.status === 403) {
+                this.terminal.print('wget error: URL blocked — internal network', 'error');
+                return;
+            }
+            if (!resp.ok) {
+                let detail = 'HTTP ' + resp.status;
+                try { const j = await resp.json(); detail = j.error || detail; } catch (e) {}
+                this.terminal.print('wget error: ' + detail, 'error');
+                return;
+            }
+            text = await resp.text();
         } catch (e) {
-            this.terminal.print(`wget error: ${e.message}`, 'error');
+            this.terminal.print('wget error: ' + e.message, 'error');
+            return;
+        }
+        // writeFile() returns false rather than throwing when the parent directory is
+        // missing — the old code printed ✅ on that path (measured: writeFile('nosuchdir/x.txt') === false).
+        const dir = filename.includes('/') ? filename.slice(0, filename.lastIndexOf('/')) : '';
+        if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} }
+        if (!this.terminal.fs.writeFile(filename, text)) {
+            this.terminal.print(`wget error: could not write ${filename} (missing directory, or path is a directory)`, 'error');
+            return;
         }
+        if (this.terminal.fs.readFile(filename) !== text) {
+            this.terminal.print(`wget error: write verification failed for ${filename}`, 'error');
+            return;
+        }
+        this.terminal.print(`✅ wget: ${filename} (${(text.length / 1024).toFixed(1)} KB, ${text.length} chars)`, 'success');
     }
 
     // ============================================================
@@ -582,6 +606,22 @@ class CryptoGzipModule {
         this.commands['tar'] = async (args) => self._cmdTar(args);
         this.commands['untar'] = async (args) => self._cmdUntar(args);
         this.commands['wget'] = async (args) => self._cmdWget(args);
+        // curl — the flags an agent actually types. No shell semantics: -s/-L/-f are
+        // accepted and ignored, -o/--output selects the VFS target. Delegates to the
+        // single verified writer above so the two cannot drift.
+        this.commands['curl'] = async (args) => {
+            const positional = [];
+            let out = null;
+            for (let i = 0; i < args.length; i++) {
+                const a = String(args[i]);
+                if (a === '-o' || a === '-O' || a === '--output') { out = args[++i]; }
+                else if (a.startsWith('-')) { /* accepted, ignored */ }
+                else positional.push(a);
+            }
+            if (!positional[0]) { this.terminal.print('Usage: curl [-s] [-o FILE] <url>', 'warning'); return; }
+            const target = out || (positional[0].split('/').pop().split('?')[0] || 'download.bin');
+            return self._cmdWget([positional[0], target]);
+        };
         this.commandsMeta = {
             gzip: { argType: 'file', description: 'Compress a VFS file to .gz' },
             gunzip: { argType: 'file', description: 'Decompress a .gz file (binary-safe)' },
@@ -589,7 +629,8 @@ class CryptoGzipModule {
             decrypt: { argType: 'file', description: 'Decrypt a .enc file' },
             tar: { argType: 'file', description: 'Bundle a VFS directory into a tar archive' },
             untar: { argType: 'file', description: 'Extract a tar archive into VFS' },
-            wget: { argType: 'url', description: 'Download a URL and save to VFS' }
+            wget: { argType: 'url', description: 'Download a URL and save to VFS' },
+            curl: { argType: 'url', description: 'curl [-s] [-o FILE] <url> — same as wget' }
         };
     }
 }