From c3d5fe64c268d7ca85d1e303b98201391f28c9e9 Mon Sep 17 00:00:00 2001 From: Agando Date: Wed, 10 Jun 2026 18:00:19 +0200 Subject: auto: 2026-06-10_18:00 changes --- modules/core/module-aiassist.js | 46 +++++++-- modules/core/module-fetcher.js | 86 ++++++++++++---- modules/core/module-loader.js | 2 +- modules/core/module-neural-vfs.js | 210 +++++++++++++++++++++++++++++++++----- modules/core/module-searxng.js | 13 ++- modules/core/terminal.js | 70 ++++++++++--- 6 files changed, 356 insertions(+), 71 deletions(-) (limited to 'modules/core') diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js index 3af7f9b..5a0c389 100644 --- a/modules/core/module-aiassist.js +++ b/modules/core/module-aiassist.js @@ -66,9 +66,10 @@ buildSystemPrompt() { } } catch (e) {} return 'Current date: ' + dateStr + nvfsNote + '\n' - + 'VFS\u2192Modules\u2192AI. nvfs-index for commands. Tool\u2192Observe\u2192Answer. nvfs-resolve. Destructive cmds need confirm.\n' - + 'FS: list/read/write/delete | NVFS: nvfs-resolve | Web: search/fetch | Mod: view/edit/test/reload | Config | Skills\n' - + 'Other: floating, webdav, cron, vault, sandbox, agents, wpm, mindmap, memgraph, guardrail, wget, tar, secret-* \u2014 discoverable via help'; + + 'VFS→Modules→AI. nvfs-index for commands. Tool→Observe→Answer. nvfs-resolve. Destructive cmds need confirm.\n' + + 'FS: list/read/write/delete | NVFS: nvfs-resolve | Web: search/fetch [POST=method+body] | Mod: view/edit/test/reload | Config | Skills\n' + + 'web_fetch tool: GET for normal pages (returns title+content+word_count), POST for JSON APIs like Hyperliquid /info (pass method="POST" and body as JSON string).\n' + + 'Other: floating, webdav, cron, vault, sandbox, agents, wpm, mindmap, memgraph, guardrail, wget, tar, secret-* — discoverable via help'; } getToolDefinitions() { if (!this._toolDefs) this._toolDefs = this._buildToolDefs(); @@ -135,8 +136,17 @@ buildSystemPrompt() { type: 'function', function: { name: 'web_fetch', - description: 'Fetch & extract URL content', - parameters: { type: 'object', properties: { url: { type: 'string', description: 'Full URL' } }, required: ['url'] } + description: 'Fetch & extract URL content. Supports GET (default) and POST (for APIs like Hyperliquid /info).', + parameters: { + type: 'object', + properties: { + url: { type: 'string', description: 'Full URL' }, + method: { type: 'string', enum: ['GET', 'POST'], description: 'HTTP method (default GET)' }, + body: { type: 'string', description: 'Request body for POST (JSON string, max 10KB)' }, + content_type: { type: 'string', description: 'Content-Type header (default application/json for POST)' } + }, + required: ['url'] + } } }, { type: 'function', @@ -1466,7 +1476,10 @@ buildSystemPrompt() { if (!query) return JSON.stringify({ error: 'No query provided' }); try { const resp = await fetch('/api/search?q=' + encodeURIComponent(query)); - if (!resp.ok) { return JSON.stringify({ error: 'Search API returned HTTP ' + resp.status }); } + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + return JSON.stringify({ error: 'Search failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' }); + } const data = await resp.json(); const results = data.results || []; if (results.length === 0) return JSON.stringify({ message: 'No results found' }); @@ -1476,12 +1489,27 @@ buildSystemPrompt() { case 'web_fetch': { const url = a.url || ''; if (!url) return JSON.stringify({ error: 'No URL provided' }); + const method = (a.method || 'GET').toUpperCase(); + const body = a.body || ''; + const contentType = a.content_type || (method === 'POST' ? 'application/json' : ''); try { - const resp = await fetch('/api/extract?url=' + encodeURIComponent(url)); + // Use /api/fetch for POST, /api/extract for GET (extract is cleaner for readable content) + const endpoint = method === 'POST' ? '/api/fetch' : '/api/extract'; + const params = new URLSearchParams({ url }); + if (method === 'POST') { params.set('method', 'POST'); params.set('body', body); if (contentType) params.set('content_type', contentType); } + const resp = await fetch(endpoint + '?' + params.toString()); if (resp.status === 403) return JSON.stringify({ error: 'URL blocked — internal network' }); - if (!resp.ok) { return JSON.stringify({ error: 'Fetch API returned HTTP ' + resp.status }); } + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + return JSON.stringify({ error: 'Fetch failed: ' + (err.error || `HTTP ${resp.status}`), status: resp.status, details: err.details || '' }); + } + // For POST/route, the response is raw text; for GET/extract it's JSON + if (method === 'POST') { + const text = await resp.text(); + return JSON.stringify({ method, content: text.substring(0, 4000), content_length: text.length }); + } const data = await resp.json(); - return JSON.stringify({ title: data.title || '', content: (data.content || '').substring(0, 4000), word_count: data.word_count || 0, method: data.method || 'extract' }); + return JSON.stringify({ method, title: data.title || '', content: (data.content || '').substring(0, 4000), word_count: data.word_count || 0, method_proxy: data.method || 'extract' }); } catch (e) { return JSON.stringify({ error: 'Fetch failed: ' + e.message }); } } case 'wpm_install': { diff --git a/modules/core/module-fetcher.js b/modules/core/module-fetcher.js index fea84d2..45505a7 100644 --- a/modules/core/module-fetcher.js +++ b/modules/core/module-fetcher.js @@ -5,36 +5,80 @@ class WebFetcherModule { } getHelpInfo() { return { - 'fetch': 'fetch — Fetch and display raw HTML/text content from a URL', - 'web': 'web — Alias for fetch', + 'fetch': 'fetch [--post=BODY] [--type=CT] [--raw] — Fetch URL (raw HTML/text); --post enables POST with body, --type sets Content-Type, --raw shows untruncated content', 'extract': 'extract — Extract readable article content (title + body text) from a URL', }; } registerCommands() { this.commands['fetch'] = (args) => this.fetchUrl(args); - this.commands['web'] = (args) => this.fetchUrl(args); this.commands['extract'] = (args) => this.extractUrl(args); } + async _fetchWithTimeout(url, timeoutMs = 30000) { + // Wrap fetch with AbortController for client-side timeout + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { signal: controller.signal }); + } finally { + clearTimeout(timeout); + } + } + _parseFlags(args) { + // Extract --key=value flags from args; return {flags, positional} + const flags = {}; + const positional = []; + for (const a of args) { + const m = String(a).match(/^--([a-zA-Z][\w-]*)(?:=(.*))?$/); + if (m) { + flags[m[1]] = m[2] !== undefined ? m[2] : true; + } else { + positional.push(a); + } + } + return { flags, positional }; + } async fetchUrl(args) { - if (!args[0]) { - this.terminal.print("Usage: fetch ", "warning"); - this.terminal.print("Example: fetch https://example.com", "info"); + const { flags, positional } = this._parseFlags(args || []); + if (!positional[0]) { + this.terminal.print("Usage: fetch [--post=BODY] [--type=CT] [--raw]", "warning"); + this.terminal.print(" Example: fetch https://example.com", "info"); + this.terminal.print(" Example: fetch https://api.hyperliquid.xyz/info --post='{\"type\":\"l2Book\",\"coin\":\"HYPE\"}' --type=application/json", "info"); return; } - let url = args[0]; + let url = positional[0]; if (!url.match(/^https?:\/\//i)) url = 'https://' + url; - this.terminal.print(`🌐 Fetching: ${url}`, "info"); + const method = flags.post ? 'POST' : 'GET'; + const showRaw = !!flags.raw; + this.terminal.print(`🌐 Fetching (${method}): ${url}`, "info"); + if (flags.post) this.terminal.print(`📤 Body: ${String(flags.post).substring(0, 200)}${String(flags.post).length > 200 ? '...' : ''}`, "info"); try { - const resp = await fetch('/api/fetch?url=' + encodeURIComponent(url)); + const params = new URLSearchParams({ url }); + if (flags.post) { params.set('method', 'POST'); params.set('body', flags.post); } + if (flags.type) params.set('content_type', flags.type); + const resp = await this._fetchWithTimeout('/api/fetch?' + params.toString(), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } - if (!resp.ok) { this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error'); return; } + if (!resp.ok) { + // Try to parse JSON error from proxy + const ct = resp.headers.get('Content-Type') || ''; + if (ct.includes('application/json')) { + const err = await resp.json().catch(() => ({})); + this.terminal.print('Fetch failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); + } else { + this.terminal.print('Fetch failed: HTTP ' + resp.status, 'error'); + } + return; + } const text = await resp.text(); - const cleanText = text.replace(/\s+/g, ' ').trim(); - this.terminal.print(`\n=== Content from ${url} ===\n`, 'success'); - this.terminal.print(cleanText.substring(0, 3000) + (cleanText.length > 3000 ? '...' : '')); - this.terminal.print(`\n📏 Content Length: ${cleanText.length} chars`, 'info'); + const limit = showRaw ? text.length : 3000; + this.terminal.print(`\n=== Content from ${url} (${method}) ===\n`, 'success'); + this.terminal.print(text.substring(0, limit) + (text.length > limit && !showRaw ? '...' : '')); + this.terminal.print(`\n📏 Content Length: ${text.length} chars${showRaw ? ' (raw, untruncated)' : ''}`, 'info'); } catch (e) { - this.terminal.print('Fetch failed: ' + e.message, 'error'); + if (e.name === 'AbortError') { + this.terminal.print(`⏱ Request timed out (30s). Try \`extract\` instead of \`fetch\` for slow sites.`, 'error'); + } else { + this.terminal.print('Fetch failed: ' + e.message, 'error'); + } } } async extractUrl(args) { @@ -47,19 +91,25 @@ class WebFetcherModule { if (!url.match(/^https?:\/\//i)) url = 'https://' + url; this.terminal.print(`📄 Extracting: ${url}`, "info"); try { - const resp = await fetch('/api/extract?url=' + encodeURIComponent(url)); + const resp = await this._fetchWithTimeout('/api/extract?url=' + encodeURIComponent(url), 30000); if (resp.status === 403) { this.terminal.print('Blocked: internal network addresses not allowed', 'error'); return; } if (!resp.ok) { const err = await resp.json().catch(() => ({})); - this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status), 'error'); + this.terminal.print('Extract failed: ' + (err.error || 'HTTP ' + resp.status) + (err.details ? ' — ' + err.details.split('\n')[0] : ''), 'error'); return; } const data = await resp.json(); if (data.title) this.terminal.print(`\n📰 ${data.title}\n`, 'success'); this.terminal.print(data.content || '(no content extracted)'); this.terminal.print(`\n📏 ${data.word_count || 0} words (via ${data.method || 'unknown'})`, 'info'); + // F5: Follow-up hint + this.terminal.print(`💡 Try \`summarize ${url}\` for AI summary, or \`nvfs-resolve \` to find related files`, 'info'); } catch (e) { - this.terminal.print('Extract failed: ' + e.message, 'error'); + if (e.name === 'AbortError') { + this.terminal.print(`⏱ Extract timed out (30s). Site may be slow or blocking automated access.`, 'error'); + } else { + this.terminal.print('Extract failed: ' + e.message, 'error'); + } } } } diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index ab73202..50962dd 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -105,7 +105,7 @@ window.__moduleLoaderReload = reloadWithSelection; - const LATEST_VER = '2026-06-10-v4.5.34'; + const LATEST_VER = '2026-06-10-v4.5.40'; const CACHE = '?v=' + LATEST_VER; const stored = localStorage.getItem('webx11_cache_ver'); if (stored !== LATEST_VER) { diff --git a/modules/core/module-neural-vfs.js b/modules/core/module-neural-vfs.js index 613d53a..2e5b9ac 100644 --- a/modules/core/module-neural-vfs.js +++ b/modules/core/module-neural-vfs.js @@ -4,10 +4,12 @@ class NeuralVFSModule { this.commandsMeta = { 'nvfs-index': { argType: 'file' }, 'nvfs-status': { argType: 'none' }, 'nvfs-resolve': { argType: 'file' }, - 'nvfs-tree': { argType: 'file' }, 'nvfs-config': { argType: 'file' }, - 'nvfs-alias': { argType: 'file' }, - 'nvfs-sync': { argType: 'file' }, + 'nvfs-tree': { argType: 'file' }, + 'nvfs-config': { argType: 'enum', enumFn: '_getNvfsConfigKeys' }, + 'nvfs-alias': { argType: 'enum', enumFn: '_getNvfsAliasActions' }, + 'nvfs-sync': { argType: 'none' }, 'nvfs-save': { argType: 'none' }, + 'nvfs-cancel': { argType: 'none' }, }; this.terminal = null; this.index = {}; @@ -32,6 +34,8 @@ class NeuralVFSModule { this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; + // Cancel flag + this._cancelRequested = false; // Status bar polling — updates #status-text with indexing indicator if (typeof window !== 'undefined') { this._statusInterval = setInterval(() => this._updateStatusBar(), 2000); @@ -57,7 +61,8 @@ class NeuralVFSModule { 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS config (fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index, deep-model)', 'nvfs-alias': 'nvfs-alias list|add|remove|clear — Manage concept aliases', 'nvfs-sync': 'nvfs-sync [--deep] — Incremental background sync (auto-sync dirty files)', - 'nvfs-save': 'nvfs-save — Force-save current in-memory index to VFS (survives refresh)' + 'nvfs-save': 'nvfs-save — Force-save current in-memory index to VFS (survives refresh)', + 'nvfs-cancel': 'nvfs-cancel — Cancel running index operation' }; } @@ -69,7 +74,8 @@ class NeuralVFSModule { this.commands['nvfs-config'] = (args) => this.nvfsConfig(args); this.commands['nvfs-alias'] = (args) => this.nvfsAlias(args); this.commands['nvfs-sync'] = async (args) => this.nvfsSync(args); - this.commands['nvfs-save'] = (args) => this.nvfsSave(); + this.commands['nvfs-save'] = async (args) => this.nvfsSave(); + this.commands['nvfs-cancel'] = (args) => this.cancelIndex(); } // ── Core Index Builder (async, chunked) ── @@ -80,12 +86,14 @@ class NeuralVFSModule { if (!fs) return this.terminal.print('VFS not available', 'error'); // Safety guard: warn if deep mode is off but the saved index has LLM concepts - if (!useDeep) { + // (show once per session — doesn't repeat on every nvfs-index call) + if (!useDeep && !this._warnedAboutLLMStrip) { const prevIndex = fs.fsReadJSON(this.indexPath) || {}; const hasLLM = Object.values(prevIndex).some(entries => Array.isArray(entries) && entries.some(e => e.source === 'llm') ); if (hasLLM) { + this._warnedAboutLLMStrip = true; this.terminal.print('⚠️ Deep indexing is OFF but the saved index contains LLM-generated concepts.', 'warning'); this.terminal.print(' Running nvfs-index without --deep will STRIP all LLM concepts from re-indexed files.', 'warning'); this.terminal.print(' To keep them: nvfs-config deep-index on or nvfs-index --deep', 'warning'); @@ -134,6 +142,13 @@ class NeuralVFSModule { const chunkSize = 10; let i = 0; const processChunk = () => { + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepQueue = []; + this._deepIndexing = false; + this.terminal.print('⛔ Index cancelled', 'warning'); + return; + } const end = Math.min(i + chunkSize, fileList.length); for (let j = i; j < end; j++) { const { path, node } = fileList[j]; @@ -308,6 +323,13 @@ class NeuralVFSModule { this.terminal.print(` Files indexed: ${totalFiles.size}`, ''); this.terminal.print(` Last scan: ${this.lastScan || 'never'}`, ''); this.terminal.print(` Mode: ${this._deepIndexEnabled ? '🌊 Deep (LLM-assisted)' : '⚡ Fast (metadata-only)'}`, ''); + // LLM/fast concept breakdown — proves persistence is working without DevTools + const allEntries = Object.values(this.index).flat(); + const llmCount = allEntries.filter(e => e.source === 'llm').length; + const fastCount = allEntries.length - llmCount; + const llmFiles = new Set(allEntries.filter(e => e.source === 'llm').map(e => e.path)).size; + this.terminal.print(` Fast concepts: ${fastCount}`, ''); + this.terminal.print(` LLM concepts: ${llmCount} (across ${llmFiles} file${llmFiles === 1 ? '' : 's'})`, ''); // Staleness check if (this.lastScan && this.lastScan !== '(loaded from disk)') { @@ -469,8 +491,10 @@ class NeuralVFSModule { _isSystemFile(path) { const normalized = path.startsWith('/') ? path : '/' + path; - const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/_mods', '/backups']; - return sysPrefixes.some(p => normalized.startsWith(p) || path.includes('/node_modules')); + const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/sys', '/_mods', '/backups']; + if (sysPrefixes.some(p => normalized.startsWith(p))) return true; + if (normalized.includes('/node_modules/')) return true; + return false; } _contentHash(str) { @@ -525,14 +549,18 @@ class NeuralVFSModule { } } - // Extract #tags from content (markdown frontmatter or inline) + // Extract #tags from content (YAML frontmatter or inline) if (content) { - const tagMatches = content.match(/tags:\s*\[([^\]]+)\]/i) || content.match(/#[a-zA-Z]\w+/g); - if (tagMatches) { - const tags = tagMatches[0].replace(/tags:\s*\[/i, '').replace(/\]/, '').split(',').map(t => t.trim().replace(/^#/, '').replace(/"/g, '')); - for (const tag of tags) { - if (tag.length > 2) concepts.add(tag.toLowerCase()); - } + let tags = []; + const yamlMatch = content.match(/tags:\s*\[([^\]]+)\]/i); + if (yamlMatch) { + tags = yamlMatch[1].split(',').map(t => t.trim().replace(/^["']|["']$/g, '').replace(/^#/, '')); + } else { + const hashTags = content.match(/#[a-zA-Z]\w+/g) || []; + tags = hashTags.map(t => t.slice(1)); + } + for (const tag of tags) { + if (tag.length > 2) concepts.add(tag.toLowerCase()); } // Extract first H1/H2 as concept @@ -642,6 +670,7 @@ class NeuralVFSModule { const start = Date.now(); let reindexed = 0; + const reindexedPaths = []; this._loadHashCache(fs); for (const path of [...this._dirtyPaths]) { @@ -653,6 +682,7 @@ class NeuralVFSModule { this._reindexFile(path, content, this.index, this._deepIndexEnabled); this._hashCache[path] = { hash }; reindexed++; + reindexedPaths.push(path); } } else { // File was deleted — remove from index @@ -670,9 +700,26 @@ class NeuralVFSModule { const elapsed = ((Date.now() - start) / 1000).toFixed(1); if (reindexed > 0) { this.terminal?.print(`🔄 Auto-sync: re-indexed ${reindexed} changed file(s) in ${elapsed}s`, 'success'); + const cap = 20; + for (const p of reindexedPaths.slice(0, cap)) { + this.terminal?.print(` • ${p}`, ''); + } + if (reindexedPaths.length > cap) { + this.terminal?.print(` ... and ${reindexedPaths.length - cap} more`, 'info'); + } } } + // ── Enum helpers for autocomplete ── + + _getNvfsConfigKeys() { + return ['fuzzy-limit', 'tree-files', 'tree-concepts', 'resolve-files', 'deep-index', 'deep-model']; + } + + _getNvfsAliasActions() { + return ['list', 'add', 'remove', 'clear']; + } + // ── Aliases ── _loadAliases() { @@ -749,10 +796,48 @@ Concepts:`; const results = []; while (this._deepQueue.length > 0) { + // Check cancel between batches + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + this.terminal?.print('⛔ Deep index cancelled', 'warning'); + return; + } const batch = this._deepQueue.splice(0, batchSize); for (const item of batch) { + // Check cancel before each LLM call so user can abort mid-batch + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + // Persist any concepts collected so far before exiting + const vfs = this.terminal?.fs; + if (vfs) this._saveIndex(vfs); + this.terminal?.print('⛔ Deep index cancelled mid-batch', 'warning'); + return; + } const context = { path: item.path, content: item.content }; const llmResponse = await this._callLLM('', context); + // Check cancel after the (possibly long) LLM call before processing + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + const vfs = this.terminal?.fs; + if (vfs) this._saveIndex(vfs); + this.terminal?.print('⛔ Deep index cancelled after LLM call', 'warning'); + return; + } if (llmResponse) { const concepts = this._parseLLMConcepts(llmResponse); results.push({ path: item.path, concepts }); @@ -782,29 +867,57 @@ Concepts:`; if (results.length > 0) { const llmConcepts = results.reduce((s, r) => s + r.concepts.length, 0); - this.terminal?.print(`🧠 Deep pass complete: ${llmConcepts} LLM-generated concepts across ${results.length} files`, 'success'); + const totalLLM = Object.values(index).flat().filter(e => e.source === 'llm').length; + this.terminal?.print(`🧠 Deep pass: added ${llmConcepts} new LLM concepts (${results.length} file${results.length === 1 ? '' : 's'}). Total in index: ${totalLLM}`, 'success'); + // List processed files so user can see if one file keeps re-processing (loop detection) + const cap = 20; + if (results.length <= cap) { + this.terminal?.print(` Files processed this pass:`, 'info'); + for (const r of results) { + this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); + } + } else { + this.terminal?.print(` First ${cap} of ${results.length} files:`, 'info'); + for (const r of results.slice(0, cap)) { + this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); + } + } } - // Persist LLM-generated concepts to disk so they survive page refresh - // Must force-save because _saveIndex() calls saveToStorage() which has a - // debounce guard (_storageSavePending) that may still be set from the - // initial buildIndex() save — without clearing it, the LLM concepts are - // never written to localStorage and are lost on page refresh. + // Persist LLM-generated concepts to disk so they survive page refresh. + // Do NOT bypass _storageSavePending (race condition: bypassing re-triggers + // a debounced saveToStorage 2s later that can clobber LLM concepts if the + // user edits a file in the meantime). Just trigger a normal save — the + // 2s debounce is fine because the user isn't waiting on the keyboard. const vfs = this.terminal?.fs; if (vfs) { - vfs._storageSavePending = false; + // await any in-flight save so this write lands on a clean slate + if (vfs._storageSavePending) { + await new Promise(resolve => { + const check = setInterval(() => { + if (!vfs._storageSavePending) { clearInterval(check); resolve(); } + }, 50); + }); + } this._saveIndex(vfs); } } // ── Nvfs Save (force-persist in-memory index) ── - nvfsSave() { + async nvfsSave() { const fs = this.terminal?.fs; if (!fs) return this.terminal?.print('VFS not available', 'error'); const count = Object.keys(this.index).length; if (count === 0) return this.terminal?.print('No index in memory to save', 'warning'); - // Bypass debounce guard to ensure write - fs._storageSavePending = false; + // Await any in-flight VFS save first — otherwise our write lands mid-flush + // and the debounce timer fires 2s later, clobbering our concepts. + if (fs._storageSavePending) { + await new Promise(resolve => { + const check = setInterval(() => { + if (!fs._storageSavePending) { clearInterval(check); resolve(); } + }, 50); + }); + } this._saveIndex(fs); this.terminal?.print(`✅ Saved ${count} concepts to ${this.indexPath}`, 'success'); this.terminal?.print(' Index will survive page refresh', 'info'); @@ -828,6 +941,53 @@ Concepts:`; await this._processDeepQueue(this.index); } } + + cancelIndex() { + this._cancelRequested = true; + this.terminal?.print('⏳ Cancelling index...', 'warning'); + } + + async init(terminal) { + // Auto-load saved index from VFS on startup so last index survives refresh + const fs = terminal?.fs || this.terminal?.fs; + if (!fs) return; + // Await VFS async loadFromStorage() — it uses CompressionStream which is async. + // module-manager.js awaits this function (line 33), so the rest of init runs + // before any other module can call nvfs-status. + await Promise.resolve(fs._ready); + const saved = fs.fsReadJSON(this.indexPath); + if (saved && typeof saved === 'object' && Object.keys(saved).length > 0) { + this.index = saved; + this._loadHashCache(fs); + this._loadAliases(); + this.lastScan = '(loaded from disk)'; + // One-time migration: purge any stale /sys/... entries left over from + // pre-v4.5.34 builds where the /sys prefix wasn't in _isSystemFile(). + // These entries cause a permanent re-index loop because /sys files use + // _factory dynamic content (timestamps, heap stats) so their hash + // never matches and they get re-queued every nvfs-index --deep run. + let purgedConcepts = 0; + let purgedFiles = 0; + for (const [concept, entries] of Object.entries(this.index)) { + if (!Array.isArray(entries)) continue; + const before = entries.length; + const filtered = entries.filter(e => !(e.path && e.path.startsWith('/sys'))); + if (filtered.length !== before) purgedConcepts += (before - filtered.length); + if (filtered.length > 0) this.index[concept] = filtered; + else delete this.index[concept]; + } + for (const path of Object.keys(this._hashCache)) { + if (path.startsWith('/sys')) { + delete this._hashCache[path]; + purgedFiles++; + } + } + if (purgedConcepts > 0 || purgedFiles > 0) { + this._saveIndex(fs); + this.terminal?.print(`🧹 Migrated stale index: purged ${purgedConcepts} /sys concepts + ${purgedFiles} hash entries from pre-v4.5.34 build`, 'warning'); + } + } + } } // Required for module system diff --git a/modules/core/module-searxng.js b/modules/core/module-searxng.js index 24db0f5..2feb982 100644 --- a/modules/core/module-searxng.js +++ b/modules/core/module-searxng.js @@ -38,13 +38,18 @@ class SearXNGModule { const categoryLabel = category === 'images' ? '🖼️' : (category === 'videos' ? '🎬' : '🔍'); this.terminal.print(`${categoryLabel} Searching ${category}: ${query}`, 'info'); + if (summarize) this.terminal.print('⏳ AI overview will be generated (1-3s)...', 'info'); let params = 'q=' + encodeURIComponent(query) + '&categories=' + category; if (summarize) params += '&summarize=true'; if (fetchContent) params += '&fetch_content=true'; try { - const resp = await fetch('/api/search?' + params); + // 30s client-side timeout (matches module-fetcher) + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 30000); + const resp = await fetch('/api/search?' + params, { signal: controller.signal }); + clearTimeout(timer); if (!resp.ok) { const err = await resp.json().catch(() => ({})); this.terminal.print('Search failed: ' + (err.error || 'HTTP ' + resp.status), 'error'); @@ -100,7 +105,11 @@ class SearXNGModule { } }); } catch (e) { - this.terminal.print('Search failed: ' + e.message, 'error'); + if (e.name === 'AbortError') { + this.terminal.print('⏱ Search timed out (30s). Try removing --summarize or --fetch flags.', 'error'); + } else { + this.terminal.print('Search failed: ' + e.message, 'error'); + } } } } diff --git a/modules/core/terminal.js b/modules/core/terminal.js index fb9f332..647060a 100644 --- a/modules/core/terminal.js +++ b/modules/core/terminal.js @@ -238,6 +238,11 @@ this._historySavePending = false; this.MAX_OUTPUT_LINES = 200; this._historyCap = 200; + // Apply history cap from localStorage on startup (was applied "on next save" before — broken) + try { + const cfgCap = parseInt(localStorage.getItem('webx11_terminal_cmd_history')); + if (cfgCap && cfgCap > 0) this._historyCap = cfgCap; + } catch (e) {} this._aiBusy = false; this._dirty = false; this.history = JSON.parse(localStorage.getItem('webx11_command_history') || '[]'); @@ -317,12 +322,42 @@ } else if (e.key === 'Tab') { e.preventDefault(); if (this.suggestions.length > 0) { - this.selectedSuggestion = (this.selectedSuggestion + 1) % this.suggestions.length; + if (e.shiftKey) { + this.selectedSuggestion = this.selectedSuggestion <= 0 + ? this.suggestions.length - 1 + : this.selectedSuggestion - 1; + } else { + this.selectedSuggestion = (this.selectedSuggestion + 1) % this.suggestions.length; + } this.updateAutocompleteDisplay(); } } else if (e.ctrlKey && (e.key === 'k' || e.key === 'K')) { e.preventDefault(); this.toggleCmdCards(); + } else if (e.ctrlKey && (e.key === 'l' || e.key === 'L')) { + // Ctrl+L — clear output (standard terminal) + e.preventDefault(); + // Preserve welcome/loading indicators — only clear actual output lines + if (this.outputContainer) { + for (const child of [...this.outputContainer.children]) { + if (!child.id || (!child.id.includes('welcome') && !child.id.includes('loading'))) { + child.remove(); + } + } + } + } else if (e.ctrlKey && (e.key === 'c' || e.key === 'C') && this.inputField.value) { + // Ctrl+C — cancel current input line (standard terminal) + e.preventDefault(); + this.print('^C', 'warning'); + this.inputField.value = ''; + this.inputField.style.height = 'auto'; + this.hideAutocomplete(); + return; + } else if (e.ctrlKey && (e.key === 'u' || e.key === 'U')) { + // Ctrl+U — clear line from cursor to start (standard terminal) + e.preventDefault(); + this.inputField.value = ''; + this.inputField.style.height = 'auto'; } else if (e.key === 'Escape') { if (this._aiBusy) { this.executeCommand('ai-stop'); @@ -918,17 +953,18 @@ showAutocomplete() { if (!this.moduleManager || !this.security) return; const input = this.inputField.value; + // Always clear dropdown innerHTML at the start — prevents stale state from prior input + this.dropdown.innerHTML = ''; + this.suggestions = []; + this.selectedSuggestion = -1; + this._isCdAutocomplete = false; const trimmed = input.trim(); const commands = this.moduleManager.getAllCommands() || {}; // Merge built-in internals into autocomplete (they're handled via if-chain, not module commands) for (const c of this.security.commandMeta.keys()) { if (!(c in commands)) commands[c] = 'builtin'; } - this.suggestions = []; - this.selectedSuggestion = -1; - this._isCdAutocomplete = false; - - if (trimmed.length === 0) { + if (input.trim().length === 0) { this.hideAutocomplete(); return; } @@ -937,9 +973,11 @@ const cmdPart = parts[0].toLowerCase(); const lowerInput = input.toLowerCase(); - // Expanded command lists for autocomplete - const fileArgCommands = ['cat', 'preview', 'rm', 'rmdir', 'edit', 'view', 'cp', 'mv', 'touch', 'put', 'run', 'chmod', 'gzip', 'gunzip', 'encrypt', 'decrypt', 'download', 'webdav-backup', 'webdav-restore', 'webdav-upload', 'webdav-download', 'webdav-list', 'trash', 'ls-trash', 'upload', 'vfs-export', 'vfs-import', 'files', 'preview', 'skill-list', 'skill-view', 'skill-create', 'skill-patch', 'skill-delete', 'skill-sync', 'webdav-ls', 'webdav-push', 'webdav-pull', 'webdav-rm', 'webdav-mkdir', 'webdav-rename', 'webdav-move', 'webdav-config', 'nvfs-index', 'nvfs-resolve', 'nvfs-tree', 'nvfs-alias', 'nvfs-sync']; - const dirArgCommands = ['ls', 'cd', 'mkdir', 'rmdir', 'upload', 'webdav-ls', 'webdav-mkdir', 'vfs-export', 'vfs-import', 'skill-list', 'skill-create']; + // Expanded command lists for autocomplete (fallback when commandsMeta lacks argType) + // NOTE: Commands with argType:'enum' in their commandsMeta are handled by _tryEnumAutocomplete + // and should NOT be in these hardcoded lists (the enum check is called BEFORE these). + const fileArgCommands = ['cat', 'preview', 'rm', 'rmdir', 'edit', 'view', 'cp', 'mv', 'touch', 'put', 'run', 'chmod', 'gzip', 'gunzip', 'encrypt', 'decrypt', 'download', 'webdav-backup', 'webdav-restore', 'webdav-upload', 'webdav-download', 'webdav-list', 'trash', 'ls-trash', 'upload', 'vfs-export', 'vfs-import', 'files', 'preview', 'skill-view', 'skill-create', 'skill-patch', 'skill-delete', 'skill-sync', 'webdav-ls', 'webdav-push', 'webdav-pull', 'webdav-rm', 'webdav-mkdir', 'webdav-rename', 'webdav-move', 'webdav-config', 'nvfs-index', 'nvfs-resolve', 'nvfs-tree']; + const dirArgCommands = ['ls', 'cd', 'mkdir', 'rmdir', 'upload', 'webdav-ls', 'webdav-mkdir', 'vfs-export', 'vfs-import', 'skill-create']; const sessionArgCommands = ['session-load', 'session-delete', 'session-export', 'session-save', 'session-list', 'session-import']; const pathArgCommands = ['cat', 'preview', 'rm', 'edit', 'view', 'cp', 'mv', 'touch', 'put', 'run', 'chmod', 'gzip', 'gunzip', 'encrypt', 'decrypt', 'download', 'webdav-backup', 'webdav-restore', 'webdav-upload', 'webdav-download', 'webdav-list', 'ls', 'cd', 'mkdir', 'rmdir', 'cat', 'preview']; @@ -966,15 +1004,15 @@ return; } - if (sessionArgCommands.includes(cmdPart)) { + if (this._tryEnumAutocomplete(cmdPart)) { + this.suggestions = this.suggestions.filter(s => s.startsWith(partialTrimmed)); + } else if (sessionArgCommands.includes(cmdPart)) { this.suggestions = this._getPathSuggestions(partialTrimmed, 'session'); } else if (dirArgCommands.includes(cmdPart) || extDirCmds.has(cmdPart)) { this.suggestions = this._getPathSuggestions(partialTrimmed, 'dir'); this._isCdAutocomplete = true; } else if (fileArgCommands.includes(cmdPart) || extFileCmds.has(cmdPart)) { this.suggestions = this._getPathSuggestions(partialTrimmed, 'file'); - } else if (this._tryEnumAutocomplete(cmdPart)) { - this.suggestions = this.suggestions.filter(s => s.startsWith(partialTrimmed)); } else { // Fallback: show files anyway this.suggestions = this._getPathSuggestions(partialTrimmed, 'file'); @@ -991,15 +1029,15 @@ // Just typed a space after command — show all matching entries if (parts.length > 1) { - if (sessionArgCommands.includes(cmdPart)) { + if (this._tryEnumAutocomplete(cmdPart)) { + // suggestions already populated + } else if (sessionArgCommands.includes(cmdPart)) { this.suggestions = this._getPathSuggestions('', 'session'); } else if (dirArgCommands.includes(cmdPart) || extDirCmds.has(cmdPart)) { this.suggestions = this._getPathSuggestions('', 'dir'); this._isCdAutocomplete = true; } else if (fileArgCommands.includes(cmdPart) || extFileCmds.has(cmdPart)) { this.suggestions = this._getPathSuggestions('', 'file'); - } else if (this._tryEnumAutocomplete(cmdPart)) { - // suggestions already populated } else { // Fallback: show files this.suggestions = this._getPathSuggestions('', 'file'); @@ -2078,7 +2116,7 @@ 'module-fetcher.js','module-skills.js','module-chatstudio.js' ]; const extFiles = [ - 'calc.js','voice.js','cron.js','theme.js','wpm.js','mindmap.js','webdav.js','skill-pack-core.js','editor.js','diff.js','alias.js','finder.js','filemgr.js','cryptogzip.js','tutorial.js','shell.js','busybox.js','floating.js' + 'calc.js','voice.js','cron.js','theme.js','wpm.js','mindmap.js','webdav.js','skill-pack-core.js','editor.js','diff.js','alias.js','finder.js','filemgr.js','cryptogzip.js','tutorial.js','shell.js','busybox.js','floating.js','console.js' ]; const addSection = (title) => { -- cgit v1.2.3