diff options
Diffstat (limited to 'modules/core/module-fetcher.js')
| -rw-r--r-- | modules/core/module-fetcher.js | 86 |
1 files changed, 68 insertions, 18 deletions
diff --git a/modules/core/module-fetcher.js b/modules/core/module-fetcher.js index fea84d2..45505a7 100644 --- a/modules/core/module-fetcher.js +++ b/modules/core/module-fetcher.js @@ -5,36 +5,80 @@ class WebFetcherModule { } getHelpInfo() { return { - 'fetch': 'fetch <url> — Fetch and display raw HTML/text content from a URL', - 'web': 'web <url> — Alias for fetch', + 'fetch': 'fetch <url> [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content', 'extract': 'extract <url> — Extract readable article content (title + body text) from a URL', }; } registerCommands() { this.commands['fetch'] = (args) => this.fetchUrl(args); - this.commands['web'] = (args) => this.fetchUrl(args); this.commands['extract'] = (args) => this.extractUrl(args); } + async _fetchWithTimeout(url, timeoutMs = 30000) { + // Wrap fetch with AbortController for client-side timeout + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { signal: controller.signal }); + } finally { + clearTimeout(timeout); + } + } + _parseFlags(args) { + // Extract --key=value flags from args; return {flags, positional} + const flags = {}; + const positional = []; + for (const a of args) { + const m = String(a).match(/^--([a-zA-Z][\w-]*)(?:=(.*))?$/); + if (m) { + flags[m[1]] = m[2] !== undefined ? m[2] : true; + } else { + positional.push(a); + } + } + return { flags, positional }; + } async fetchUrl(args) { - if (!args[0]) { - this.terminal.print("Usage: fetch <url>", "warning"); - this.terminal.print("Example: fetch https://example.com", "info"); + const { flags, positional } = this._parseFlags(args || []); + if (!positional[0]) { + this.terminal.print("Usage: fetch <url> [--post=BODY] [--type=CT] [--raw]", "warning"); + this.terminal.print(" Example: fetch https://example.com", "info"); + this.terminal.print(" Example: fetch https://api.hyperliquid.xyz/info --post='{\"type\":\"l2Book\",\"coin\":\"HYPE\"}' --type=application/json", "info"); return; } - let url = args[0]; + let url = positional[0]; if (!url.match(/^https?:\/\//i)) url = 'https://' + url; - this.terminal.print(`🌐 Fetching: ${url}`, "info"); + const method = flags.post ? 'POST' : 'GET'; + const showRaw = !!flags.raw; + this.terminal.print(`🌐 Fetching (${method}): ${url}`, "info"); + if (flags.post) this.terminal.print(`📤 Body: ${String(flags.post).substring(0, 200)}${String(flags.post).length > 200 ? '...' : ''}`, "info"); try { - const resp = await fetch('/api/fetch?url=' + encodeURIComponent(url)); + const params = new URLSearchParams({ url }); + if (flags.post) { params.set('method', 'POST'); params.set('body', flags.post); } + if (flags.type) params.set('content_type', flags.type); + const resp = await this._fetchWithTimeout('/api/fetch?' + params.toString(), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } - if (!resp.ok) { this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error'); return; } + if (!resp.ok) { + // Try to parse JSON error from proxy + const ct = resp.headers.get('Content-Type') || ''; + if (ct.includes('application/json')) { + const err = await resp.json().catch(() => ({})); + this.terminal.print('Fetch failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); + } else { + this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error'); + } + return; + } const text = await resp.text(); - const cleanText = text.replace(/\s+/g, ' ').trim(); - this.terminal.print(`\n=== Content from ${url} ===\n`, 'success'); - this.terminal.print(cleanText.substring(0, 3000) + (cleanText.length > 3000 ? '...' : '')); - this.terminal.print(`\n📏 Content Length: ${cleanText.length} chars`, 'info'); + const limit = showRaw ? text.length : 3000; + this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success'); + this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : '')); + this.terminal.print(`\n📏 Content Length: ${text.length} chars${showRaw ? ' (raw, untruncated)' : ''}`, 'info'); } catch (e) { - this.terminal.print('Fetch failed: ' + e.message, 'error'); + if (e.name === 'AbortError') { + this.terminal.print(`⏱ Request timed out (30s). Try \`extract\` instead of \`fetch\` for slow sites.`, 'error'); + } else { + this.terminal.print('Fetch failed: ' + e.message, 'error'); + } } } async extractUrl(args) { @@ -47,19 +91,25 @@ class WebFetcherModule { if (!url.match(/^https?:\/\//i)) url = 'https://' + url; this.terminal.print(`📄 Extracting: ${url}`, "info"); try { - const resp = await fetch('/api/extract?url=' + encodeURIComponent(url)); + const resp = await this._fetchWithTimeout('/api/extract?url=' + encodeURIComponent(url), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } if (!resp.ok) { const err = await resp.json().catch(() => ({})); - this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status), 'error'); + this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); return; } const data = await resp.json(); if (data.title) this.terminal.print(`\n📰 ${data.title}\n`, 'success'); this.terminal.print(data.content || '(no content extracted)'); this.terminal.print(`\n📏 ${data.word_count || 0} words (via ${data.method || 'unknown'})`, 'info'); + // F5: Follow-up hint + this.terminal.print(`💡 Try \`summarize ${url}\` for AI summary, or \`nvfs-resolve <concept>\` to find related files`, 'info'); } catch (e) { - this.terminal.print('Extract failed: ' + e.message, 'error'); + if (e.name === 'AbortError') { + this.terminal.print(`⏱ Extract timed out (30s). Site may be slow or blocking automated access.`, 'error'); + } else { + this.terminal.print('Extract failed: ' + e.message, 'error'); + } } } } |