diff options
Diffstat (limited to 'modules/core/module-neural-vfs.js')
| -rw-r--r-- | modules/core/module-neural-vfs.js | 370 |
1 files changed, 359 insertions, 11 deletions
diff --git a/modules/core/module-neural-vfs.js b/modules/core/module-neural-vfs.js index 2644e85..5c30718 100644 --- a/modules/core/module-neural-vfs.js +++ b/modules/core/module-neural-vfs.js @@ -12,6 +12,26 @@ class NeuralVFSModule { this._treeConcepts = 15; this._resolveFiles = 20; this._loadConfig(); + // Auto-sync state + this._dirtyPaths = new Set(); + this._autoSyncTimer = null; + this._aliasMap = {}; + this._loadAliases(); + // Deep Indexing toggle + this._deepIndexEnabled = localStorage.getItem('webx11_nvfs_deep') === 'true'; + this._deepModel = localStorage.getItem('webx11_nvfs_deep_model') || 'ministral-3:3b-cloud'; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + // Status bar polling — updates #status-text with indexing indicator + if (typeof window !== 'undefined') { + this._statusInterval = setInterval(() => this._updateStatusBar(), 2000); + } + // Register as window-level event listener for mutation events from VirtualFS + if (typeof window !== 'undefined') { + window.__nvfsMarkDirty = (path) => this.markDirty(path); + } } _loadConfig() { try { const c = localStorage.getItem('webx11_nvfs_config'); if (c) { const p = JSON.parse(c); if (p.fuzzyLimit) this._fuzzyLimit = p.fuzzyLimit; if (p.treeFiles) this._treeFiles = p.treeFiles; if (p.treeConcepts) this._treeConcepts = p.treeConcepts; if (p.resolveFiles) this._resolveFiles = p.resolveFiles; } } catch (e) {} @@ -22,29 +42,34 @@ class NeuralVFSModule { getHelpInfo() { return { - 'nvfs-index': 'nvfs-index — Scan VFS and build semantic concept index', + 'nvfs-index': 'nvfs-index [--deep] — Scan VFS and build semantic concept index', 'nvfs-status': 'nvfs-status — Show semantic index stats and last scan time', 'nvfs-resolve': 'nvfs-resolve <concept> — Find files related to a concept', 'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree', - 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS display limits (fuzzy-limit, tree-files, tree-concepts, resolve-files)' + 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS config (fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index, deep-model)', + 'nvfs-alias': 'nvfs-alias list|add|remove|clear — Manage concept aliases', + 'nvfs-sync': 'nvfs-sync [--deep] — Incremental background sync (auto-sync dirty files)' }; } registerCommands() { - this.commands['nvfs-index'] = (args) => this.buildIndex(); + this.commands['nvfs-index'] = (args) => this.buildIndex(args); this.commands['nvfs-status'] = (args) => this.showStatus(); this.commands['nvfs-resolve'] = (args) => this.resolve(args); this.commands['nvfs-tree'] = (args) => this.tree(args); this.commands['nvfs-config'] = (args) => this.nvfsConfig(args); + this.commands['nvfs-alias'] = (args) => this.nvfsAlias(args); + this.commands['nvfs-sync'] = async (args) => this.nvfsSync(args); } // ── Core Index Builder (async, chunked) ── - async buildIndex() { + async buildIndex(args) { + const useDeep = this._deepIndexEnabled || (args && args.includes('--deep')); const fs = this.terminal.fs; if (!fs) return this.terminal.print('VFS not available', 'error'); - this.terminal.print('🔍 Scanning VFS for semantic concepts...', 'info'); + this.terminal.print(`🔍 Scanning VFS for semantic concepts${useDeep ? ' (deep mode)' : ''}...`, 'info'); const start = Date.now(); let newIndex = {}; @@ -99,13 +124,13 @@ class NeuralVFSModule { cachedCount++; if (!livePaths.has(path)) { // File was in cache but somehow missing from prevIndex - this._reindexFile(path, content, newIndex); + this._reindexFile(path, content, newIndex, useDeep); } continue; } // Changed or new file — extract concepts fresh - this._reindexFile(path, content, newIndex); + this._reindexFile(path, content, newIndex, useDeep); this._hashCache[path] = { hash }; } i = end; @@ -133,7 +158,22 @@ class NeuralVFSModule { processChunk(); } - _reindexFile(path, content, index) { + _reindexFile(path, content, index, useDeep) { + if (useDeep) { + // Deep mode: queue for async LLM concept extraction — fast-path filename concepts first + const fastConcepts = this._extractConcepts(path, ''); + for (const concept of fastConcepts) { + if (!index[concept]) index[concept] = []; + const exists = index[concept].some(e => e.path === path); + if (!exists) index[concept].push({ path, summary: this._summarize(content), source: 'filename' }); + } + this._deepQueue.push({ path, content }); + if (!this._deepIndexing) { + this._deepIndexing = true; + this._processDeepQueue(index).catch(e => console.error('Deep index error:', e)); + } + return; + } const concepts = this._extractConcepts(path, content); // Remove old entries for this path for (const entries of Object.values(index)) { @@ -170,6 +210,15 @@ class NeuralVFSModule { if (!args[0]) return this.terminal.print('Usage: nvfs-resolve <concept>', 'warning'); const concept = args.join(' ').toLowerCase(); + + // Check aliases first + if (this._aliasMap[concept]) { + const targets = this._aliasMap[concept]; + this.terminal.print(`🔗 Alias "${concept}" → ${targets.length} target(s):`, 'info'); + for (const t of targets) this.terminal.print(` ${t}`, 'success'); + return; + } + const matches = this.index[concept] || []; // Fuzzy fallback — find partial matches @@ -178,6 +227,18 @@ class NeuralVFSModule { ).filter(k => k !== concept); if (matches.length === 0 && fuzzy.length === 0) { + // Hybrid discovery: fall back to live VFS scan + const shellResult = this._hybridShellCheck(concept); + if (shellResult.length > 0) { + this.terminal.print(`🔍 Index empty for "${concept}" but found ${shellResult.length} file(s) via live scan. Run nvfs-index to refresh.`, 'warning'); + for (const f of shellResult.slice(0, this._resolveFiles)) { + this.terminal.print(` ${f.path}${f.summary ? ' — ' + f.summary : ''}`, 'success'); + } + if (shellResult.length > this._resolveFiles) { + this.terminal.print(` ... and ${shellResult.length - this._resolveFiles} more`, 'info'); + } + return; + } return this.terminal.print(`No files found for concept "${concept}"`, 'warning'); } @@ -223,6 +284,31 @@ class NeuralVFSModule { this.terminal.print(` Concepts: ${Object.keys(this.index).length}`, ''); this.terminal.print(` Files indexed: ${totalFiles.size}`, ''); this.terminal.print(` Last scan: ${this.lastScan || 'never'}`, ''); + this.terminal.print(` Mode: ${this._deepIndexEnabled ? '🌊 Deep (LLM-assisted)' : '⚡ Fast (metadata-only)'}`, ''); + + // Staleness check + if (this.lastScan && this.lastScan !== '(loaded from disk)') { + const ageMs = Date.now() - new Date(this.lastScan).getTime(); + const ageHours = Math.floor(ageMs / 3600000); + const ageMinutes = Math.floor((ageMs % 3600000) / 60000); + if (ageHours >= 1) { + this.terminal.print(` Age: ${ageHours}h${ageMinutes > 0 ? ` ${ageMinutes}m` : ''}`, ''); + } + if (ageHours >= 4) { + this.terminal.print(` ⚠️ Index is ${ageHours} hours old — structural changes may be missing. Run nvfs-index to refresh.`, 'warning'); + } + } + + // Dirty file warning + if (this._dirtyPaths && this._dirtyPaths.size > 0) { + this.terminal.print(` ⚠️ ${this._dirtyPaths.size} file(s) changed since last scan (auto-sync pending)`, 'warning'); + } + + // Alias count + const aliasCount = Object.keys(this._aliasMap).length; + if (aliasCount > 0) { + this.terminal.print(` Aliases: ${aliasCount}`, ''); + } } // ── Tree View ── @@ -260,22 +346,89 @@ class NeuralVFSModule { this.terminal.print(`tree-files: ${this._treeFiles} (default: 5)`, ''); this.terminal.print(`tree-concepts: ${this._treeConcepts} (default: 15)`, ''); this.terminal.print(`resolve-files: ${this._resolveFiles} (default: 20)`, ''); + this.terminal.print(`deep-index: ${this._deepIndexEnabled ? 'on' : 'off'} (LLM-assisted concept extraction)`, ''); + this.terminal.print(`deep-model: ${this._deepModel} (model for deep indexing)`, ''); this.terminal.print('Usage: nvfs-config <key> <value>', 'info'); - this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files', 'info'); + this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index (on|off), deep-model <name>', 'info'); return; } const key = args[0]; + if (key === 'deep-index') { + const val = args[1]; + if (val !== 'on' && val !== 'off') return this.terminal.print('deep-index must be "on" or "off"', 'warning'); + this._deepIndexEnabled = val === 'on'; + localStorage.setItem('webx11_nvfs_deep', this._deepIndexEnabled ? 'true' : 'false'); + // If turning off mid-queue, abort pending deep work + if (!this._deepIndexEnabled && this._deepIndexing) { + this._deepQueue = []; + this._deepIndexing = false; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + this.terminal.print(' Deep index queue aborted', 'warning'); + } + this.terminal.print(`nvfs-config deep-index set to ${val}`, 'success'); + if (this._deepIndexEnabled) this.terminal.print(` Deep mode uses ${this._deepModel} for semantic concept extraction`, 'info'); + return; + } + if (key === 'deep-model') { + const val = args.slice(1).join(' '); + if (!val) return this.terminal.print('Usage: nvfs-config deep-model <model-name>', 'warning'); + this._deepModel = val; + localStorage.setItem('webx11_nvfs_deep_model', val); + this.terminal.print(`nvfs-config deep-model set to ${val}`, 'success'); + return; + } const val = parseInt(args[1]); if (isNaN(val) || val < 1 || val > 200) return this.terminal.print('Value must be 1-200', 'warning'); if (key === 'fuzzy-limit') { this._fuzzyLimit = val; } else if (key === 'tree-files') { this._treeFiles = val; } else if (key === 'tree-concepts') { this._treeConcepts = val; } else if (key === 'resolve-files') { this._resolveFiles = val; } - else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files`, 'warning'); } + else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index (on|off), deep-model <name>`, 'warning'); } this._saveConfig(); this.terminal.print(`nvfs-config ${key} set to ${val}`, 'success'); } + // ── Nvfs Alias ── + + nvfsAlias(args) { + const action = args[0]; + if (!action || action === 'list') { + const keys = Object.keys(this._aliasMap); + if (keys.length === 0) return this.terminal.print('No aliases defined. Use nvfs-alias add <name> <target,paths>', 'info'); + this.terminal.print('🔗 Concept Aliases:', 'info'); + for (const [name, targets] of Object.entries(this._aliasMap)) { + this.terminal.print(` ${name} → ${targets.join(', ')}`, 'success'); + } + return; + } + if (action === 'add') { + const name = args[1]; + if (!name || !args[2]) return this.terminal.print('Usage: nvfs-alias add <name> <target1,target2,...>', 'warning'); + const targets = args.slice(2).join(' ').split(',').map(t => t.trim()).filter(Boolean); + this._aliasMap[name] = targets; + this._saveAliases(); + this.terminal.print(`Alias "${name}" added (${targets.length} target(s))`, 'success'); + return; + } + if (action === 'remove') { + const name = args[1]; + if (!name || !this._aliasMap[name]) return this.terminal.print('Alias not found', 'warning'); + delete this._aliasMap[name]; + this._saveAliases(); + this.terminal.print(`Alias "${name}" removed`, 'success'); + return; + } + if (action === 'clear') { + this._aliasMap = {}; + this._saveAliases(); + this.terminal.print('All aliases cleared', 'success'); + return; + } + this.terminal.print('Usage: nvfs-alias list|add <name> <targets>|remove <name>|clear', 'warning'); + } + // ── Private Helpers ── _walk(path, fs, index, cb) { @@ -292,7 +445,7 @@ class NeuralVFSModule { } _isSystemFile(path) { - const sysPrefixes = ['/.trash', '/.semantic_index', '/_snapshots', '/_sessions']; + const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/_mods', '/backups']; return sysPrefixes.some(p => path.startsWith(p) || path.includes('/node_modules')); } @@ -371,6 +524,22 @@ class NeuralVFSModule { return [...concepts]; } + _hybridShellCheck(concept) { + const fs = this.terminal?.fs; + if (!fs) return []; + const words = concept.split(/\s+/).filter(w => w.length > 2); + if (words.length === 0) return []; + const results = []; + this._walk('/', fs, {}, (path, node) => { + if (this._isSystemFile(path)) return; + const lower = path.toLowerCase(); + if (words.some(w => lower.includes(w))) { + results.push({ path, summary: this._summarize(node.content || '') }); + } + }); + return results; + } + _summarize(content) { if (!content || content.length < 10) return ''; // Strip YAML frontmatter @@ -432,6 +601,185 @@ class NeuralVFSModule { console.error('NeuralVFS save error:', e); } } + + // ── Auto-Sync ── + + markDirty(path) { + this._dirtyPaths.add(path); + // Debounced auto-sync: wait 10s after last mutation + if (this._autoSyncTimer) clearTimeout(this._autoSyncTimer); + this._autoSyncTimer = setTimeout(() => this._reindexDirtyPaths(), 10000); + } + + async _reindexDirtyPaths() { + if (this._dirtyPaths.size === 0) return; + const fs = this.terminal?.fs; + if (!fs) { this._dirtyPaths.clear(); return; } + + const start = Date.now(); + let reindexed = 0; + this._loadHashCache(fs); + + for (const path of [...this._dirtyPaths]) { + const content = fs.readFile(path); + if (content !== null) { + const hash = this._contentHash(content); + const cached = this._getCached(path, hash); + if (!cached) { + this._reindexFile(path, content, this.index); + this._hashCache[path] = { hash }; + reindexed++; + } + } else { + // File was deleted — remove from index + for (const entries of Object.values(this.index)) { + const idx = entries.findIndex(e => e.path === path); + if (idx !== -1) entries.splice(idx, 1); + } + delete this._hashCache[path]; + } + } + this._dirtyPaths.clear(); + this._autoSyncTimer = null; + this.lastScan = new Date().toISOString(); + this._saveIndex(fs); + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + if (reindexed > 0) { + this.terminal?.print(`🔄 Auto-sync: re-indexed ${reindexed} changed file(s) in ${elapsed}s`, 'success'); + } + } + + // ── Aliases ── + + _loadAliases() { + try { const r = localStorage.getItem('webx11_nvfs_aliases'); if (r) this._aliasMap = JSON.parse(r); } catch (e) {} + } + _saveAliases() { + localStorage.setItem('webx11_nvfs_aliases', JSON.stringify(this._aliasMap)); + } + + // ── Status Bar Polling ── + + _updateStatusBar() { + if (!this.terminal || !this.terminal.statusBar) return; + const el = this.terminal.statusBar; + if (this._deepIndexing) { + const pct = this._deepTotal > 0 ? Math.round(this._deepProcessed / this._deepTotal * 100) : 0; + el.textContent = `🧠 Deep Indexing... ${this._deepProcessed}/${this._deepTotal} (${pct}%)`; + el.style.color = 'var(--accent, #00ff88)'; + } else if (this._autoSyncTimer !== null || this._dirtyPaths.size > 0) { + el.textContent = `🔄 Auto-sync pending (${this._dirtyPaths.size} dirty)`; + el.style.color = 'var(--warning, #ffaa00)'; + } else { + const count = Object.keys(this.index).length; + el.textContent = `🧠 ${count} concepts${this._deepIndexEnabled ? ' 🌊' : ''}`; + el.style.color = ''; + } + } + + // ── Deep Indexing (LLM-assisted) ── + + _parseLLMConcepts(llmResponse) { + // LLM returns markdown bullet list: "- concept: description" + const concepts = []; + if (!llmResponse) return concepts; + const lines = llmResponse.split('\n'); + for (const line of lines) { + const trimmed = line.replace(/^[-*\s]+/, '').trim(); + if (!trimmed) continue; + // Split on first colon or comma or dash + const colonIdx = trimmed.indexOf(':'); + const commaIdx = trimmed.indexOf(','); + const delimIdx = colonIdx > 0 ? colonIdx : (commaIdx > 0 ? commaIdx : -1); + const concept = delimIdx > 0 ? trimmed.slice(0, delimIdx).trim().toLowerCase() : trimmed.toLowerCase(); + if (concept && concept.length > 2 && concept.length < 60) { + concepts.push(concept); + } + } + return [...new Set(concepts)]; + } + + async _callLLM(prompt, context) { + const om = this.terminal?.ollamaManager; + if (!om || !om.connected) return null; + const fullPrompt = `Extract 2-5 key searchable concepts from this file. Return one per line as lowercase keywords (1-3 words). + +File: ${context.path} +Content: +${context.content.slice(0, 2000)} + +Concepts:`; + try { + const result = await om._chatWithProvider(om.currentProvider, fullPrompt, this._deepModel); + return result; + } catch (e) { + console.warn('LLM concept extraction failed:', e); + return null; + } + } + + async _processDeepQueue(index) { + this._deepProcessed = 0; + this._deepTotal = this._deepQueue.length; + const batchSize = 3; + const results = []; + + while (this._deepQueue.length > 0) { + const batch = this._deepQueue.splice(0, batchSize); + for (const item of batch) { + const context = { path: item.path, content: item.content }; + const llmResponse = await this._callLLM('', context); + if (llmResponse) { + const concepts = this._parseLLMConcepts(llmResponse); + results.push({ path: item.path, concepts }); + for (const concept of concepts) { + if (!index[concept]) index[concept] = []; + const exists = index[concept].some(e => e.path === item.path); + if (!exists) index[concept].push({ + path: item.path, + summary: this._summarize(item.content), + source: 'llm' + }); + } + } + this._deepProcessed++; + // Update status bar live + this._updateStatusBar(); + } + // Yield to UI between batches + await new Promise(r => setTimeout(r, 50)); + } + + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + + if (results.length > 0) { + const llmConcepts = results.reduce((s, r) => s + r.concepts.length, 0); + this.terminal?.print(`🧠 Deep pass complete: ${llmConcepts} LLM-generated concepts across ${results.length} files`, 'success'); + } + } + + // ── Nvfs Sync (incremental background) ── + + async nvfsSync(args) { + const useDeep = args && args.includes('--deep'); + if (this._dirtyPaths.size === 0) { + this.terminal.print('No dirty files to sync', 'info'); + return; + } + this.terminal.print(`🔄 Syncing ${this._dirtyPaths.size} dirty file(s)${useDeep ? ' (deep mode)' : ''}...`, 'info'); + const fs = this.terminal?.fs; + if (!fs) return; + await this._reindexDirtyPaths(); + // After reindex, if deep mode, queue remaining queued items for LLM processing + if (useDeep && this._deepQueue.length > 0) { + this._deepIndexing = true; + await this._processDeepQueue(this.index); + } + } } // Required for module system |