diff options
Diffstat (limited to 'modules/core/module-neural-vfs.js')
| -rw-r--r-- | modules/core/module-neural-vfs.js | 210 |
1 files changed, 185 insertions, 25 deletions
diff --git a/modules/core/module-neural-vfs.js b/modules/core/module-neural-vfs.js index 613d53a..2e5b9ac 100644 --- a/modules/core/module-neural-vfs.js +++ b/modules/core/module-neural-vfs.js @@ -4,10 +4,12 @@ class NeuralVFSModule { this.commandsMeta = { 'nvfs-index': { argType: 'file' }, 'nvfs-status': { argType: 'none' }, 'nvfs-resolve': { argType: 'file' }, - 'nvfs-tree': { argType: 'file' }, 'nvfs-config': { argType: 'file' }, - 'nvfs-alias': { argType: 'file' }, - 'nvfs-sync': { argType: 'file' }, + 'nvfs-tree': { argType: 'file' }, + 'nvfs-config': { argType: 'enum', enumFn: '_getNvfsConfigKeys' }, + 'nvfs-alias': { argType: 'enum', enumFn: '_getNvfsAliasActions' }, + 'nvfs-sync': { argType: 'none' }, 'nvfs-save': { argType: 'none' }, + 'nvfs-cancel': { argType: 'none' }, }; this.terminal = null; this.index = {}; @@ -32,6 +34,8 @@ class NeuralVFSModule { this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; + // Cancel flag + this._cancelRequested = false; // Status bar polling — updates #status-text with indexing indicator if (typeof window !== 'undefined') { this._statusInterval = setInterval(() => this._updateStatusBar(), 2000); @@ -57,7 +61,8 @@ class NeuralVFSModule { 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS config (fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index, deep-model)', 'nvfs-alias': 'nvfs-alias list|add|remove|clear — Manage concept aliases', 'nvfs-sync': 'nvfs-sync [--deep] — Incremental background sync (auto-sync dirty files)', - 'nvfs-save': 'nvfs-save — Force-save current in-memory index to VFS (survives refresh)' + 'nvfs-save': 'nvfs-save — Force-save current in-memory index to VFS (survives refresh)', + 'nvfs-cancel': 'nvfs-cancel — Cancel running index operation' }; } @@ -69,7 +74,8 @@ class NeuralVFSModule { this.commands['nvfs-config'] = (args) => this.nvfsConfig(args); this.commands['nvfs-alias'] = (args) => this.nvfsAlias(args); this.commands['nvfs-sync'] = async (args) => this.nvfsSync(args); - this.commands['nvfs-save'] = (args) => this.nvfsSave(); + this.commands['nvfs-save'] = async (args) => this.nvfsSave(); + this.commands['nvfs-cancel'] = (args) => this.cancelIndex(); } // ── Core Index Builder (async, chunked) ── @@ -80,12 +86,14 @@ class NeuralVFSModule { if (!fs) return this.terminal.print('VFS not available', 'error'); // Safety guard: warn if deep mode is off but the saved index has LLM concepts - if (!useDeep) { + // (show once per session — doesn't repeat on every nvfs-index call) + if (!useDeep && !this._warnedAboutLLMStrip) { const prevIndex = fs.fsReadJSON(this.indexPath) || {}; const hasLLM = Object.values(prevIndex).some(entries => Array.isArray(entries) && entries.some(e => e.source === 'llm') ); if (hasLLM) { + this._warnedAboutLLMStrip = true; this.terminal.print('⚠️ Deep indexing is OFF but the saved index contains LLM-generated concepts.', 'warning'); this.terminal.print(' Running nvfs-index without --deep will STRIP all LLM concepts from re-indexed files.', 'warning'); this.terminal.print(' To keep them: nvfs-config deep-index on or nvfs-index --deep', 'warning'); @@ -134,6 +142,13 @@ class NeuralVFSModule { const chunkSize = 10; let i = 0; const processChunk = () => { + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepQueue = []; + this._deepIndexing = false; + this.terminal.print('⛔ Index cancelled', 'warning'); + return; + } const end = Math.min(i + chunkSize, fileList.length); for (let j = i; j < end; j++) { const { path, node } = fileList[j]; @@ -308,6 +323,13 @@ class NeuralVFSModule { this.terminal.print(` Files indexed: ${totalFiles.size}`, ''); this.terminal.print(` Last scan: ${this.lastScan || 'never'}`, ''); this.terminal.print(` Mode: ${this._deepIndexEnabled ? '🌊 Deep (LLM-assisted)' : '⚡ Fast (metadata-only)'}`, ''); + // LLM/fast concept breakdown — proves persistence is working without DevTools + const allEntries = Object.values(this.index).flat(); + const llmCount = allEntries.filter(e => e.source === 'llm').length; + const fastCount = allEntries.length - llmCount; + const llmFiles = new Set(allEntries.filter(e => e.source === 'llm').map(e => e.path)).size; + this.terminal.print(` Fast concepts: ${fastCount}`, ''); + this.terminal.print(` LLM concepts: ${llmCount} (across ${llmFiles} file${llmFiles === 1 ? '' : 's'})`, ''); // Staleness check if (this.lastScan && this.lastScan !== '(loaded from disk)') { @@ -469,8 +491,10 @@ class NeuralVFSModule { _isSystemFile(path) { const normalized = path.startsWith('/') ? path : '/' + path; - const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/_mods', '/backups']; - return sysPrefixes.some(p => normalized.startsWith(p) || path.includes('/node_modules')); + const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/sys', '/_mods', '/backups']; + if (sysPrefixes.some(p => normalized.startsWith(p))) return true; + if (normalized.includes('/node_modules/')) return true; + return false; } _contentHash(str) { @@ -525,14 +549,18 @@ class NeuralVFSModule { } } - // Extract #tags from content (markdown frontmatter or inline) + // Extract #tags from content (YAML frontmatter or inline) if (content) { - const tagMatches = content.match(/tags:\s*\[([^\]]+)\]/i) || content.match(/#[a-zA-Z]\w+/g); - if (tagMatches) { - const tags = tagMatches[0].replace(/tags:\s*\[/i, '').replace(/\]/, '').split(',').map(t => t.trim().replace(/^#/, '').replace(/"/g, '')); - for (const tag of tags) { - if (tag.length > 2) concepts.add(tag.toLowerCase()); - } + let tags = []; + const yamlMatch = content.match(/tags:\s*\[([^\]]+)\]/i); + if (yamlMatch) { + tags = yamlMatch[1].split(',').map(t => t.trim().replace(/^["']|["']$/g, '').replace(/^#/, '')); + } else { + const hashTags = content.match(/#[a-zA-Z]\w+/g) || []; + tags = hashTags.map(t => t.slice(1)); + } + for (const tag of tags) { + if (tag.length > 2) concepts.add(tag.toLowerCase()); } // Extract first H1/H2 as concept @@ -642,6 +670,7 @@ class NeuralVFSModule { const start = Date.now(); let reindexed = 0; + const reindexedPaths = []; this._loadHashCache(fs); for (const path of [...this._dirtyPaths]) { @@ -653,6 +682,7 @@ class NeuralVFSModule { this._reindexFile(path, content, this.index, this._deepIndexEnabled); this._hashCache[path] = { hash }; reindexed++; + reindexedPaths.push(path); } } else { // File was deleted — remove from index @@ -670,9 +700,26 @@ class NeuralVFSModule { const elapsed = ((Date.now() - start) / 1000).toFixed(1); if (reindexed > 0) { this.terminal?.print(`🔄 Auto-sync: re-indexed ${reindexed} changed file(s) in ${elapsed}s`, 'success'); + const cap = 20; + for (const p of reindexedPaths.slice(0, cap)) { + this.terminal?.print(` • ${p}`, ''); + } + if (reindexedPaths.length > cap) { + this.terminal?.print(` ... and ${reindexedPaths.length - cap} more`, 'info'); + } } } + // ── Enum helpers for autocomplete ── + + _getNvfsConfigKeys() { + return ['fuzzy-limit', 'tree-files', 'tree-concepts', 'resolve-files', 'deep-index', 'deep-model']; + } + + _getNvfsAliasActions() { + return ['list', 'add', 'remove', 'clear']; + } + // ── Aliases ── _loadAliases() { @@ -749,10 +796,48 @@ Concepts:`; const results = []; while (this._deepQueue.length > 0) { + // Check cancel between batches + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + this.terminal?.print('⛔ Deep index cancelled', 'warning'); + return; + } const batch = this._deepQueue.splice(0, batchSize); for (const item of batch) { + // Check cancel before each LLM call so user can abort mid-batch + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + // Persist any concepts collected so far before exiting + const vfs = this.terminal?.fs; + if (vfs) this._saveIndex(vfs); + this.terminal?.print('⛔ Deep index cancelled mid-batch', 'warning'); + return; + } const context = { path: item.path, content: item.content }; const llmResponse = await this._callLLM('', context); + // Check cancel after the (possibly long) LLM call before processing + if (this._cancelRequested) { + this._cancelRequested = false; + this._deepIndexing = false; + this._deepQueue = []; + this._deepProcessed = 0; + this._deepTotal = 0; + this._updateStatusBar(); + const vfs = this.terminal?.fs; + if (vfs) this._saveIndex(vfs); + this.terminal?.print('⛔ Deep index cancelled after LLM call', 'warning'); + return; + } if (llmResponse) { const concepts = this._parseLLMConcepts(llmResponse); results.push({ path: item.path, concepts }); @@ -782,29 +867,57 @@ Concepts:`; if (results.length > 0) { const llmConcepts = results.reduce((s, r) => s + r.concepts.length, 0); - this.terminal?.print(`🧠 Deep pass complete: ${llmConcepts} LLM-generated concepts across ${results.length} files`, 'success'); + const totalLLM = Object.values(index).flat().filter(e => e.source === 'llm').length; + this.terminal?.print(`🧠 Deep pass: added ${llmConcepts} new LLM concepts (${results.length} file${results.length === 1 ? '' : 's'}). Total in index: ${totalLLM}`, 'success'); + // List processed files so user can see if one file keeps re-processing (loop detection) + const cap = 20; + if (results.length <= cap) { + this.terminal?.print(` Files processed this pass:`, 'info'); + for (const r of results) { + this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); + } + } else { + this.terminal?.print(` First ${cap} of ${results.length} files:`, 'info'); + for (const r of results.slice(0, cap)) { + this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); + } + } } - // Persist LLM-generated concepts to disk so they survive page refresh - // Must force-save because _saveIndex() calls saveToStorage() which has a - // debounce guard (_storageSavePending) that may still be set from the - // initial buildIndex() save — without clearing it, the LLM concepts are - // never written to localStorage and are lost on page refresh. + // Persist LLM-generated concepts to disk so they survive page refresh. + // Do NOT bypass _storageSavePending (race condition: bypassing re-triggers + // a debounced saveToStorage 2s later that can clobber LLM concepts if the + // user edits a file in the meantime). Just trigger a normal save — the + // 2s debounce is fine because the user isn't waiting on the keyboard. const vfs = this.terminal?.fs; if (vfs) { - vfs._storageSavePending = false; + // await any in-flight save so this write lands on a clean slate + if (vfs._storageSavePending) { + await new Promise(resolve => { + const check = setInterval(() => { + if (!vfs._storageSavePending) { clearInterval(check); resolve(); } + }, 50); + }); + } this._saveIndex(vfs); } } // ── Nvfs Save (force-persist in-memory index) ── - nvfsSave() { + async nvfsSave() { const fs = this.terminal?.fs; if (!fs) return this.terminal?.print('VFS not available', 'error'); const count = Object.keys(this.index).length; if (count === 0) return this.terminal?.print('No index in memory to save', 'warning'); - // Bypass debounce guard to ensure write - fs._storageSavePending = false; + // Await any in-flight VFS save first — otherwise our write lands mid-flush + // and the debounce timer fires 2s later, clobbering our concepts. + if (fs._storageSavePending) { + await new Promise(resolve => { + const check = setInterval(() => { + if (!fs._storageSavePending) { clearInterval(check); resolve(); } + }, 50); + }); + } this._saveIndex(fs); this.terminal?.print(`✅ Saved ${count} concepts to ${this.indexPath}`, 'success'); this.terminal?.print(' Index will survive page refresh', 'info'); @@ -828,6 +941,53 @@ Concepts:`; await this._processDeepQueue(this.index); } } + + cancelIndex() { + this._cancelRequested = true; + this.terminal?.print('⏳ Cancelling index...', 'warning'); + } + + async init(terminal) { + // Auto-load saved index from VFS on startup so last index survives refresh + const fs = terminal?.fs || this.terminal?.fs; + if (!fs) return; + // Await VFS async loadFromStorage() — it uses CompressionStream which is async. + // module-manager.js awaits this function (line 33), so the rest of init runs + // before any other module can call nvfs-status. + await Promise.resolve(fs._ready); + const saved = fs.fsReadJSON(this.indexPath); + if (saved && typeof saved === 'object' && Object.keys(saved).length > 0) { + this.index = saved; + this._loadHashCache(fs); + this._loadAliases(); + this.lastScan = '(loaded from disk)'; + // One-time migration: purge any stale /sys/... entries left over from + // pre-v4.5.34 builds where the /sys prefix wasn't in _isSystemFile(). + // These entries cause a permanent re-index loop because /sys files use + // _factory dynamic content (timestamps, heap stats) so their hash + // never matches and they get re-queued every nvfs-index --deep run. + let purgedConcepts = 0; + let purgedFiles = 0; + for (const [concept, entries] of Object.entries(this.index)) { + if (!Array.isArray(entries)) continue; + const before = entries.length; + const filtered = entries.filter(e => !(e.path && e.path.startsWith('/sys'))); + if (filtered.length !== before) purgedConcepts += (before - filtered.length); + if (filtered.length > 0) this.index[concept] = filtered; + else delete this.index[concept]; + } + for (const path of Object.keys(this._hashCache)) { + if (path.startsWith('/sys')) { + delete this._hashCache[path]; + purgedFiles++; + } + } + if (purgedConcepts > 0 || purgedFiles > 0) { + this._saveIndex(fs); + this.terminal?.print(`🧹 Migrated stale index: purged ${purgedConcepts} /sys concepts + ${purgedFiles} hash entries from pre-v4.5.34 build`, 'warning'); + } + } + } } // Required for module system |