1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
class WebFetcherModule {
constructor() {
this.commands = {};
this.terminal = null;
}
getHelpInfo() {
return {
'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content',
'extract': 'extract <url> — Extract readable article content (title + body text) from a URL',
};
}
registerCommands() {
this.commands['fetch'] = (args) => this.fetchUrl(args);
this.commands['extract'] = (args) => this.extractUrl(args);
}
async _fetchWithTimeout(url, timeoutMs = 30000) {
// Wrap fetch with AbortController for client-side timeout
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, { signal: controller.signal });
} finally {
clearTimeout(timeout);
}
}
_parseFlags(args) {
// Extract --key=value flags from args; return {flags, positional}
const flags = {};
const positional = [];
for (const a of args) {
const m = String(a).match(/^--([a-zA-Z][\w-]*)(?:=(.*))?$/);
if (m) {
flags[m[1]] = m[2] !== undefined ? m[2] : true;
} else {
positional.push(a);
}
}
return { flags, positional };
}
async fetchUrl(args) {
const { flags, positional } = this._parseFlags(args || []);
if (!positional[0]) {
this.terminal.print("Usage: fetch <url> [--post=BODY] [--type=CT] [--raw]", "warning");
this.terminal.print(" Example: fetch https://example.com", "info");
this.terminal.print(" Example: fetch https://api.hyperliquid.xyz/info --post='{\"type\":\"l2Book\",\"coin\":\"HYPE\"}' --type=application/json", "info");
return;
}
let url = positional[0];
if (!url.match(/^https?:\/\//i)) url = 'https://' + url;
const method = flags.post ? 'POST' : 'GET';
const showRaw = !!flags.raw;
this.terminal.print(`🌐 Fetching (${method}): ${url}`, "info");
if (flags.post) this.terminal.print(`📤 Body: ${String(flags.post).substring(0, 200)}${String(flags.post).length > 200 ? '...' : ''}`, "info");
try {
const params = new URLSearchParams({ url });
if (flags.post) { params.set('method', 'POST'); params.set('body', flags.post); }
if (flags.type) params.set('content_type', flags.type);
const resp = await this._fetchWithTimeout('/api/fetch?' + params.toString(), 30000);
if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; }
if (!resp.ok) {
// Try to parse JSON error from proxy
const ct = resp.headers.get('Content-Type') || '';
if (ct.includes('application/json')) {
const err = await resp.json().catch(() => ({}));
this.terminal.print('Fetch failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error');
} else {
this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error');
}
return;
}
const text = await resp.text();
const limit = showRaw ? text.length : 3000;
this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success');
this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : ''));
this.terminal.print(`\n📏 Content Length: ${text.length} chars${showRaw ? ' (raw, untruncated)' : ''}`, 'info');
} catch (e) {
if (e.name === 'AbortError') {
this.terminal.print(`⏱ Request timed out (30s). Try \`extract\` instead of \`fetch\` for slow sites.`, 'error');
} else {
this.terminal.print('Fetch failed: ' + e.message, 'error');
}
}
}
async extractUrl(args) {
if (!args[0]) {
this.terminal.print("Usage: extract <url>", "warning");
this.terminal.print("Example: extract https://example.com", "info");
return;
}
let url = args[0];
if (!url.match(/^https?:\/\//i)) url = 'https://' + url;
this.terminal.print(`📄 Extracting: ${url}`, "info");
try {
const resp = await this._fetchWithTimeout('/api/extract?url=' + encodeURIComponent(url), 30000);
if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; }
if (!resp.ok) {
const err = await resp.json().catch(() => ({}));
this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error');
return;
}
const data = await resp.json();
if (data.title) this.terminal.print(`\n📰 ${data.title}\n`, 'success');
this.terminal.print(data.content || '(no content extracted)');
this.terminal.print(`\n📏 ${data.word_count || 0} words (via ${data.method || 'unknown'})`, 'info');
// F5: Follow-up hint
this.terminal.print(`💡 Try \`summarize ${url}\` for AI summary, or \`nvfs-resolve <concept>\` to find related files`, 'info');
} catch (e) {
if (e.name === 'AbortError') {
this.terminal.print(`⏱ Extract timed out (30s). Site may be slow or blocking automated access.`, 'error');
} else {
this.terminal.print('Extract failed: ' + e.message, 'error');
}
}
}
}
|
