class WebFetcherModule { constructor() { this.commands = {}; this.terminal = null; } getHelpInfo() { return { 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content', 'extract': 'extract — Extract readable article content (title + body text) from a URL', }; } registerCommands() { this.commands['fetch'] = (args) => this.fetchUrl(args); this.commands['extract'] = (args) => this.extractUrl(args); } async _fetchWithTimeout(url, timeoutMs = 30000) { // Wrap fetch with AbortController for client-side timeout const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { signal: controller.signal }); } finally { clearTimeout(timeout); } } _parseFlags(args) { // Extract --key=value flags from args; return {flags, positional} const flags = {}; const positional = []; for (const a of args) { const m = String(a).match(/^--([a-zA-Z][\w-]*)(?:=(.*))?$/); if (m) { flags[m[1]] = m[2] !== undefined ? m[2] : true; } else { positional.push(a); } } return { flags, positional }; } async fetchUrl(args) { const { flags, positional } = this._parseFlags(args || []); if (!positional[0]) { this.terminal.print("Usage: fetch [--post=BODY] [--type=CT] [--raw]", "warning"); this.terminal.print(" Example: fetch https://example.com", "info"); this.terminal.print(" Example: fetch https://api.hyperliquid.xyz/info --post='{\"type\":\"l2Book\",\"coin\":\"HYPE\"}' --type=application/json", "info"); return; } let url = positional[0]; if (!url.match(/^https?:\/\//i)) url = 'https://' + url; const method = flags.post ? 'POST' : 'GET'; const showRaw = !!flags.raw; this.terminal.print(`🌐 Fetching (${method}): ${url}`, "info"); if (flags.post) this.terminal.print(`šŸ“¤ Body: ${String(flags.post).substring(0, 200)}${String(flags.post).length > 200 ? '...' : ''}`, "info"); try { const params = new URLSearchParams({ url }); if (flags.post) { params.set('method', 'POST'); params.set('body', flags.post); } if (flags.type) params.set('content_type', flags.type); const resp = await this._fetchWithTimeout('/api/fetch?' + params.toString(), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } if (!resp.ok) { // Try to parse JSON error from proxy const ct = resp.headers.get('Content-Type') || ''; if (ct.includes('application/json')) { const err = await resp.json().catch(() => ({})); this.terminal.print('Fetch failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); } else { this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error'); } return; } const text = await resp.text(); const limit = showRaw ? text.length : 3000; this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success'); this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : '')); this.terminal.print(`\nšŸ“ Content Length: ${text.length} chars${showRaw ? ' (raw, untruncated)' : ''}`, 'info'); } catch (e) { if (e.name === 'AbortError') { this.terminal.print(`ā± Request timed out (30s). Try \`extract\` instead of \`fetch\` for slow sites.`, 'error'); } else { this.terminal.print('Fetch failed: ' + e.message, 'error'); } } } async extractUrl(args) { if (!args[0]) { this.terminal.print("Usage: extract ", "warning"); this.terminal.print("Example: extract https://example.com", "info"); return; } let url = args[0]; if (!url.match(/^https?:\/\//i)) url = 'https://' + url; this.terminal.print(`šŸ“„ Extracting: ${url}`, "info"); try { const resp = await this._fetchWithTimeout('/api/extract?url=' + encodeURIComponent(url), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } if (!resp.ok) { const err = await resp.json().catch(() => ({})); this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); return; } const data = await resp.json(); if (data.title) this.terminal.print(`\nšŸ“° ${data.title}\n`, 'success'); this.terminal.print(data.content || '(no content extracted)'); this.terminal.print(`\nšŸ“ ${data.word_count || 0} words (via ${data.method || 'unknown'})`, 'info'); // F5: Follow-up hint this.terminal.print(`šŸ’” Try \`summarize ${url}\` for AI summary, or \`nvfs-resolve \` to find related files`, 'info'); } catch (e) { if (e.name === 'AbortError') { this.terminal.print(`ā± Extract timed out (30s). Site may be slow or blocking automated access.`, 'error'); } else { this.terminal.print('Extract failed: ' + e.message, 'error'); } } } }