1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
class WebFetcherModule {
constructor() {
this.commands = {};
this.terminal = null;
}
getHelpInfo() {
return {
'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] [--save=FILE] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content, --save writes it to the VFS',
'extract': 'extract <url> — Extract readable article content (title + body text) from a URL',
};
}
registerCommands() {
this.commands['fetch'] = (args) => this.fetchUrl(args);
this.commands['extract'] = (args) => this.extractUrl(args);
}
async _fetchWithTimeout(url, timeoutMs = 30000) {
// Wrap fetch with AbortController for client-side timeout
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, { signal: controller.signal });
} finally {
clearTimeout(timeout);
}
}
_parseFlags(args) {
// Extract --key=value flags from args; return {flags, positional}
const flags = {};
const positional = [];
for (const a of args) {
const m = String(a).match(/^--([a-zA-Z][\w-]*)(?:=(.*))?$/);
if (m) {
flags[m[1]] = m[2] !== undefined ? m[2] : true;
} else {
positional.push(a);
}
}
return { flags, positional };
}
async fetchUrl(args) {
const { flags, positional } = this._parseFlags(args || []);
if (!positional[0]) {
this.terminal.print("Usage: fetch <url> [--post=BODY] [--type=CT] [--raw]", "warning");
this.terminal.print(" Example: fetch https://example.com", "info");
this.terminal.print(" Example: fetch https://api.hyperliquid.xyz/info --post='{\"type\":\"l2Book\",\"coin\":\"HYPE\"}' --type=application/json", "info");
return;
}
let url = positional[0];
if (!url.match(/^https?:\/\//i)) url = 'https://' + url;
const method = flags.post ? 'POST' : 'GET';
const showRaw = !!flags.raw;
this.terminal.print(`🌐 Fetching (${method}): ${url}`, "info");
if (flags.post) this.terminal.print(`📤 Body: ${String(flags.post).substring(0, 200)}${String(flags.post).length > 200 ? '...' : ''}`, "info");
try {
const params = new URLSearchParams({ url });
if (flags.post) { params.set('method', 'POST'); params.set('body', flags.post); }
if (flags.type) params.set('content_type', flags.type);
const resp = await this._fetchWithTimeout('/api/fetch?' + params.toString(), 30000);
if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; }
if (!resp.ok) {
// Try to parse JSON error from proxy
const ct = resp.headers.get('Content-Type') || '';
if (ct.includes('application/json')) {
const err = await resp.json().catch(() => ({}));
this.terminal.print('Fetch failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error');
} else {
this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error');
}
return;
}
const text = await resp.text();
// --save: the only terminal path from a remote URL to the VFS. Without it a
// 100 KB source page can only be read through 3 000-char scrollback, which is
// what sent the in-OS agent looking for a shell redirect (reported 2026-11-06).
if (flags.save) {
const path = String(flags.save === true ? (url.split('/').pop().split('?')[0] || 'fetch.txt') : flags.save);
const dir = path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '';
if (dir) { try { this.terminal.fs.mkdir(dir); } catch (e) {} }
const wrote = this.terminal.fs.writeFile(path, text);
if (!wrote || this.terminal.fs.readFile(path) !== text) {
this.terminal.print(`Save failed: could not write ${path}`, 'error');
return;
}
this.terminal.print(`💾 Saved ${text.length} chars to ${path}`, 'success');
return;
}
const limit = showRaw ? text.length : 3000;
this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success');
this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : ''));
this.terminal.print(`\n📏 Content Length: ${text.length} chars${showRaw ? ' (raw, untruncated)' : ''}`, 'info');
} catch (e) {
if (e.name === 'AbortError') {
this.terminal.print(`⏱ Request timed out (30s). Try \`extract\` instead of \`fetch\` for slow sites.`, 'error');
} else {
this.terminal.print('Fetch failed: ' + e.message, 'error');
}
}
}
async extractUrl(args) {
if (!args[0]) {
this.terminal.print("Usage: extract <url>", "warning");
this.terminal.print("Example: extract https://example.com", "info");
return;
}
let url = args[0];
if (!url.match(/^https?:\/\//i)) url = 'https://' + url;
this.terminal.print(`📄 Extracting: ${url}`, "info");
try {
const resp = await this._fetchWithTimeout('/api/extract?url=' + encodeURIComponent(url), 30000);
if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; }
if (!resp.ok) {
const err = await resp.json().catch(() => ({}));
this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error');
return;
}
const data = await resp.json();
if (data.title) this.terminal.print(`\n📰 ${data.title}\n`, 'success');
this.terminal.print(data.content || '(no content extracted)');
this.terminal.print(`\n📏 ${data.word_count || 0} words (via ${data.method || 'unknown'})`, 'info');
// F5: Follow-up hint
this.terminal.print(`💡 Try \`summarize ${url}\` for AI summary, or \`nvfs-resolve <concept>\` to find related files`, 'info');
} catch (e) {
if (e.name === 'AbortError') {
this.terminal.print(`⏱ Extract timed out (30s). Site may be slow or blocking automated access.`, 'error');
} else {
this.terminal.print('Extract failed: ' + e.message, 'error');
}
}
}
}
|
