class NeuralVFSModule { constructor() { this.commands = {}; this.commandsMeta = { 'nvfs-index': { argType: 'file' }, 'nvfs-status': { argType: 'none' }, 'nvfs-resolve': { argType: 'file' }, 'nvfs-tree': { argType: 'file' }, 'nvfs-config': { argType: 'enum', enumFn: '_getNvfsConfigKeys' }, 'nvfs-alias': { argType: 'enum', enumFn: '_getNvfsAliasActions' }, 'nvfs-sync': { argType: 'none' }, 'nvfs-save': { argType: 'none' }, 'nvfs-cancel': { argType: 'none' }, }; this.terminal = null; this.index = {}; this.indexPath = '/.semantic_index/map.json'; this.hashCachePath = '/.semantic_index/hash_cache.json'; this._hashCache = {}; // Configurable display limits this._fuzzyLimit = 10; this._treeFiles = 5; this._treeConcepts = 15; this._resolveFiles = 20; this._excludePatterns = []; this._excludeDirs = []; this._excludeFiles = []; this._includeOnly = []; this._loadConfig(); // Auto-sync state this._dirtyPaths = new Set(); this._autoSyncTimer = null; this._aliasMap = {}; this._loadAliases(); // Deep Indexing toggle this._deepIndexEnabled = localStorage.getItem('webx11_nvfs_deep') === 'true'; this._deepModel = localStorage.getItem('webx11_nvfs_deep_model') || 'gemma4:31b-cloud'; this._deepIndexing = false; this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; // Cancel flag this._cancelRequested = false; // Status bar polling — updates #status-text with indexing indicator if (typeof window !== 'undefined') { this._statusInterval = setInterval(() => this._updateStatusBar(), 2000); } // Register as window-level event listener for mutation events from VirtualFS if (typeof window !== 'undefined') { window.__nvfsMarkDirty = (path) => this.markDirty(path); } } _loadConfig() { try { const c = localStorage.getItem('webx11_nvfs_config'); if (c) { const p = JSON.parse(c); if (p.fuzzyLimit) this._fuzzyLimit = p.fuzzyLimit; if (p.treeFiles) this._treeFiles = p.treeFiles; if (p.treeConcepts) this._treeConcepts = p.treeConcepts; if (p.resolveFiles) this._resolveFiles = p.resolveFiles; if (Array.isArray(p.excludePatterns)) this._excludePatterns = p.excludePatterns; if (Array.isArray(p.excludeDirs)) this._excludeDirs = p.excludeDirs; if (Array.isArray(p.excludeFiles)) this._excludeFiles = p.excludeFiles; if (Array.isArray(p.includeOnly)) this._includeOnly = p.includeOnly; } } catch (e) {} } _saveConfig() { localStorage.setItem('webx11_nvfs_config', JSON.stringify({ fuzzyLimit: this._fuzzyLimit, treeFiles: this._treeFiles, treeConcepts: this._treeConcepts, resolveFiles: this._resolveFiles, excludePatterns: this._excludePatterns, excludeDirs: this._excludeDirs, excludeFiles: this._excludeFiles, includeOnly: this._includeOnly })); } getHelpInfo() { return { 'nvfs-index': 'nvfs-index [--deep] — Scan VFS and build semantic concept index', 'nvfs-status': 'nvfs-status — Show semantic index stats and last scan time', 'nvfs-resolve': 'nvfs-resolve — Find files related to a concept', 'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree', 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS config (fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index, deep-model, exclude-patterns, exclude-dirs, exclude-files, include-only)', 'nvfs-alias': 'nvfs-alias list|add|remove|clear — Manage concept aliases', 'nvfs-sync': 'nvfs-sync [--deep] — Incremental background sync (auto-sync dirty files)', 'nvfs-save': 'nvfs-save — Force-save current in-memory index to VFS (survives refresh)', 'nvfs-cancel': 'nvfs-cancel — Cancel running index operation' }; } registerCommands() { this.commands['nvfs-index'] = (args) => this.buildIndex(args); this.commands['nvfs-status'] = (args) => this.showStatus(); this.commands['nvfs-resolve'] = (args) => this.resolve(args); this.commands['nvfs-tree'] = (args) => this.tree(args); this.commands['nvfs-config'] = (args) => this.nvfsConfig(args); this.commands['nvfs-alias'] = (args) => this.nvfsAlias(args); this.commands['nvfs-sync'] = async (args) => this.nvfsSync(args); this.commands['nvfs-save'] = async (args) => this.nvfsSave(); this.commands['nvfs-cancel'] = (args) => this.cancelIndex(); this.commands['nvfs-index-graph'] = () => this.indexGraphFacts(); } // ── Core Index Builder (async, chunked) ── async buildIndex(args) { const useDeep = this._deepIndexEnabled || (args && args.includes('--deep')); const fs = this.terminal.fs; if (!fs) return this.terminal.print('VFS not available', 'error'); // Safety guard: warn if deep mode is off but the saved index has LLM concepts // (show once per session — doesn't repeat on every nvfs-index call) if (!useDeep && !this._warnedAboutLLMStrip) { const prevIndex = fs.fsReadJSON(this.indexPath) || {}; const hasLLM = Object.values(prevIndex).some(entries => Array.isArray(entries) && entries.some(e => e.source === 'llm') ); if (hasLLM) { this._warnedAboutLLMStrip = true; this.terminal.print('⚠️ Deep indexing is OFF but the saved index contains LLM-generated concepts.', 'warning'); this.terminal.print(' Running nvfs-index without --deep will STRIP all LLM concepts from re-indexed files.', 'warning'); this.terminal.print(' To keep them: nvfs-config deep-index on or nvfs-index --deep', 'warning'); } } this.terminal.print(`🔍 Scanning VFS for semantic concepts${useDeep ? ' (deep mode)' : ''}...`, 'info'); const start = Date.now(); let newIndex = {}; let fileCount = 0; let conceptCount = 0; let cachedCount = 0; // Load previous index and hash cache for incremental diffing const prevIndex = fs.fsReadJSON(this.indexPath) || {}; this._loadHashCache(fs); // Collect current file list const fileList = []; this._collectFiles('/', fs, fileList); if (fileList.length === 0) { return this.terminal.print('No files found to index', 'warning'); } // Detect deleted files — strip orphaned entries from prevIndex const currentPaths = new Set(fileList.map(f => f.path)); let deletedCount = 0; for (const [concept, entries] of Object.entries(prevIndex)) { const remaining = entries.filter(e => currentPaths.has(e.path)); if (remaining.length !== entries.length) deletedCount++; if (remaining.length > 0) prevIndex[concept] = remaining; else delete prevIndex[concept]; } // Warm start: clone prevIndex and count live paths for stats newIndex = JSON.parse(JSON.stringify(prevIndex)); const livePaths = new Set(); for (const entries of Object.values(newIndex)) { for (const e of entries) livePaths.add(e.path); } conceptCount = Object.keys(newIndex).length; // Chunked async processing — yields to UI between chunks const chunkSize = 10; let i = 0; const processChunk = () => { if (this._cancelRequested) { this._cancelRequested = false; this._deepQueue = []; this._deepIndexing = false; this.terminal.print('⛔ Index cancelled', 'warning'); return; } const end = Math.min(i + chunkSize, fileList.length); for (let j = i; j < end; j++) { const { path, node } = fileList[j]; fileCount++; const content = node.content || ''; const hash = this._contentHash(content); // Skip if hash unchanged — content is same as last scan const cached = this._getCached(path, hash); if (cached) { cachedCount++; if (!livePaths.has(path)) { // File was in cache but somehow missing from prevIndex this._reindexFile(path, content, newIndex, useDeep); } continue; } // Changed or new file — extract concepts fresh this._reindexFile(path, content, newIndex, useDeep); this._hashCache[path] = { hash }; } i = end; if (i < fileList.length) { const pct = Math.round(i / fileList.length * 100); const cMsg = cachedCount > 0 ? ` (${cachedCount} cached)` : ''; this.terminal.print(`⏳ Indexing... ${pct}% (${i}/${fileList.length} files${cMsg})`, 'info'); setTimeout(processChunk, 0); } else { // Also index skills this._indexSkills(fs, newIndex); this.index = newIndex; this.lastScan = new Date().toISOString(); this.fileCount = fileCount; this.conceptCount = Object.keys(newIndex).length; this._saveIndex(fs); if (fs._clearConceptCache) fs._clearConceptCache(); const elapsed = ((Date.now() - start) / 1000).toFixed(1); const processedMsg = cachedCount > 0 ? ` (${cachedCount} cached, ${fileCount - cachedCount} reindexed)` : ''; this.terminal.print(`✅ Indexed ${fileCount} files → ${Object.keys(newIndex).length} concepts in ${elapsed}s${processedMsg}`, 'success'); if (deletedCount > 0) this.terminal.print(` Purged ${deletedCount} orphaned concepts from deleted files`, 'info'); this.terminal.print(` Index saved to ${this.indexPath}`, 'info'); } }; processChunk(); } _reindexFile(path, content, index, useDeep) { if (useDeep) { // Deep mode: queue for async LLM concept extraction — fast-path filename concepts first const fastConcepts = this._extractConcepts(path, ''); for (const concept of fastConcepts) { if (!index[concept]) index[concept] = []; const exists = index[concept].some(e => e.path === path); if (!exists) index[concept].push({ path, summary: this._summarize(content), source: 'filename' }); } this._deepQueue.push({ path, content }); if (!this._deepIndexing) { this._deepIndexing = true; this._processDeepQueue(index).catch(e => console.error('Deep index error:', e)); } return; } const concepts = this._extractConcepts(path, content); // Remove old entries for this path for (const entries of Object.values(index)) { const idx = entries.findIndex(e => e.path === path); if (idx !== -1) entries.splice(idx, 1); } // Add new entries for (const concept of concepts) { if (!index[concept]) { index[concept] = []; this.conceptCount++; } index[concept].push({ path, summary: this._summarize(content) }); } } _collectFiles(path, fs, list) { const entries = fs.readDir(path); if (!entries) return; for (const entry of entries) { const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name; if (this._isExcluded(fullPath)) continue; if (entry.type === 'file') { if (this._isSystemFile(fullPath)) continue; list.push({ path: fullPath, node: { type: 'file', content: fs.readFile(fullPath) || '' } }); } else if (entry.type === 'dir') { this._collectFiles(fullPath, fs, list); } } } // ── Resolve ── resolve(args) { if (!args[0]) return this.terminal.print('Usage: nvfs-resolve ', 'warning'); const concept = args.join(' ').toLowerCase(); // Check aliases first if (this._aliasMap[concept]) { const targets = this._aliasMap[concept]; this.terminal.print(`🔗 Alias "${concept}" → ${targets.length} target(s):`, 'info'); for (const t of targets) this.terminal.print(` ${t}`, 'success'); return; } const matches = this.index[concept] || []; // Fuzzy fallback — find partial matches const fuzzy = Object.keys(this.index).filter(k => k.includes(concept) || concept.includes(k) ).filter(k => k !== concept); if (matches.length === 0 && fuzzy.length === 0) { // Hybrid discovery: fall back to live VFS scan const shellResult = this._hybridShellCheck(concept); if (shellResult.length > 0) { this.terminal.print(`🔍 Index empty for "${concept}" but found ${shellResult.length} file(s) via live scan. Run nvfs-index to refresh.`, 'warning'); for (const f of shellResult.slice(0, this._resolveFiles)) { this.terminal.print(` ${f.path}${f.summary ? ' — ' + f.summary : ''}`, 'success'); } if (shellResult.length > this._resolveFiles) { this.terminal.print(` ... and ${shellResult.length - this._resolveFiles} more`, 'info'); } return; } return this.terminal.print(`No files found for concept "${concept}"`, 'warning'); } if (matches.length > 0) { this.terminal.print(`📂 "${concept}" → ${matches.length} file(s):`, 'info'); for (const m of matches.slice(0, this._resolveFiles)) { this.terminal.print(` ${m.path}`, 'success'); if (m.summary) this.terminal.print(` ${m.summary}`, '') } if (matches.length > this._resolveFiles) { this.terminal.print(` ... and ${matches.length - this._resolveFiles} more`, 'info'); } } if (fuzzy.length > 0) { this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, this._fuzzyLimit).join(', ')}`, 'info'); } } // ── Status ── showStatus() { const count = Object.keys(this.index).length; if (count === 0) { // Try loading from VFS const fs = this.terminal.fs; const saved = fs.fsReadJSON(this.indexPath); if (saved) { this.index = saved; this.lastScan = '(loaded from disk)'; this.terminal.print(`📖 Loaded index from ${this.indexPath}`, 'info'); } else { return this.terminal.print('No semantic index found. Run nvfs-index first.', 'warning'); } } const totalFiles = new Set(); for (const entries of Object.values(this.index)) { for (const e of entries) totalFiles.add(e.path); } this.terminal.print(`🧠 Neural VFS Index`, 'info'); this.terminal.print(` Concepts: ${Object.keys(this.index).length}`, ''); this.terminal.print(` Files indexed: ${totalFiles.size}`, ''); this.terminal.print(` Last scan: ${this.lastScan || 'never'}`, ''); this.terminal.print(` Mode: ${this._deepIndexEnabled ? '🌊 Deep (LLM-assisted)' : '⚡ Fast (metadata-only)'}`, ''); // LLM/fast concept breakdown — proves persistence is working without DevTools const allEntries = Object.values(this.index).flat(); const llmCount = allEntries.filter(e => e.source === 'llm').length; const fastCount = allEntries.length - llmCount; const llmFiles = new Set(allEntries.filter(e => e.source === 'llm').map(e => e.path)).size; this.terminal.print(` Fast concepts: ${fastCount}`, ''); this.terminal.print(` LLM concepts: ${llmCount} (across ${llmFiles} file${llmFiles === 1 ? '' : 's'})`, ''); // Staleness check if (this.lastScan && this.lastScan !== '(loaded from disk)') { const ageMs = Date.now() - new Date(this.lastScan).getTime(); const ageHours = Math.floor(ageMs / 3600000); const ageMinutes = Math.floor((ageMs % 3600000) / 60000); if (ageHours >= 1) { this.terminal.print(` Age: ${ageHours}h${ageMinutes > 0 ? ` ${ageMinutes}m` : ''}`, ''); } if (ageHours >= 4) { this.terminal.print(` ⚠️ Index is ${ageHours} hours old — structural changes may be missing. Run nvfs-index to refresh.`, 'warning'); } } // Dirty file warning if (this._dirtyPaths && this._dirtyPaths.size > 0) { this.terminal.print(` ⚠️ ${this._dirtyPaths.size} file(s) changed since last scan (auto-sync pending)`, 'warning'); } // Alias count const aliasCount = Object.keys(this._aliasMap).length; if (aliasCount > 0) { this.terminal.print(` Aliases: ${aliasCount}`, ''); } } // ── Tree View ── tree(args) { const concepts = Object.keys(this.index); if (concepts.length === 0) { return this.terminal.print('No index. Run nvfs-index first.', 'warning'); } const filter = args[0] ? args[0].toLowerCase() : null; const filtered = filter ? concepts.filter(c => c.includes(filter)) : concepts; this.terminal.print(`🌳 Concept Tree${filter ? ` (filter: "${filter}")` : ''}:`, 'info'); for (const concept of filtered.slice(0, this._treeConcepts)) { this.terminal.print(` 📁 ${concept}`, 'success'); for (const entry of this.index[concept].slice(0, this._treeFiles)) { this.terminal.print(` └─ ${entry.path}`, ''); } if (this.index[concept].length > this._treeFiles) { this.terminal.print(` └─ ... (+${this.index[concept].length - this._treeFiles} more)`, ''); } } if (filtered.length > this._treeConcepts) { this.terminal.print(` ... and ${filtered.length - this._treeConcepts} more concepts`, 'info'); } } // ── Nvfs Config ── nvfsConfig(args) { const listConfig = () => { this.terminal.print('=== Neural VFS Config ===', 'info'); this.terminal.print(`fuzzy-limit: ${this._fuzzyLimit} (default: 10)`, ''); this.terminal.print(`tree-files: ${this._treeFiles} (default: 5)`, ''); this.terminal.print(`tree-concepts: ${this._treeConcepts} (default: 15)`, ''); this.terminal.print(`resolve-files: ${this._resolveFiles} (default: 20)`, ''); this.terminal.print(`deep-index: ${this._deepIndexEnabled ? 'on' : 'off'} (LLM-assisted concept extraction)`, ''); this.terminal.print(`deep-model: ${this._deepModel} (model for deep indexing)`, ''); this.terminal.print(`exclude-patterns: ${this._excludePatterns.length ? this._excludePatterns.join(', ') : '(none)'}`, ''); this.terminal.print(`exclude-dirs: ${this._excludeDirs.length ? this._excludeDirs.join(', ') : '(none)'}`, ''); this.terminal.print(`exclude-files: ${this._excludeFiles.length ? this._excludeFiles.join(', ') : '(none)'}`, ''); this.terminal.print(`include-only: ${this._includeOnly.length ? this._includeOnly.join(', ') : '(all)'}`, ''); this.terminal.print('Usage: nvfs-config ', 'info'); this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index (on|off), deep-model, exclude-patterns, exclude-dirs, exclude-files, include-only', 'info'); this.terminal.print('Lists are comma-separated. Use "+item" to append, "-item" to remove, or a plain comma list to replace.', 'info'); }; if (args.length === 0) { listConfig(); return; } const key = args[0]; if (key === 'deep-index') { const val = args[1]; if (val !== 'on' && val !== 'off') return this.terminal.print('deep-index must be "on" or "off"', 'warning'); this._deepIndexEnabled = val === 'on'; localStorage.setItem('webx11_nvfs_deep', this._deepIndexEnabled ? 'true' : 'false'); if (!this._deepIndexEnabled && this._deepIndexing) { this._deepQueue = []; this._deepIndexing = false; this._deepProcessed = 0; this._deepTotal = 0; this._updateStatusBar(); this.terminal.print(' Deep index queue aborted', 'warning'); } this.terminal.print(`nvfs-config deep-index set to ${val}`, 'success'); if (this._deepIndexEnabled) this.terminal.print(` Deep mode uses ${this._deepModel} for semantic concept extraction`, 'info'); return; } if (key === 'deep-model') { const val = args.slice(1).join(' '); if (!val) return this.terminal.print('Usage: nvfs-config deep-model ', 'warning'); this._deepModel = val; localStorage.setItem('webx11_nvfs_deep_model', val); this.terminal.print(`nvfs-config deep-model set to ${val}`, 'success'); return; } // List-type config keys const listKeys = { 'exclude-patterns': '_excludePatterns', 'exclude-dirs': '_excludeDirs', 'exclude-files': '_excludeFiles', 'include-only': '_includeOnly' }; if (listKeys[key]) { const raw = args.slice(1).join(' ').trim(); if (!raw) { this.terminal.print(`${key}: ${this[listKeys[key]].join(', ') || '(empty)'}`, 'info'); return; } let current = this[listKeys[key]]; if (raw.startsWith('+')) { const add = raw.slice(1).split(',').map(s => s.trim()).filter(Boolean); current = [...new Set([...current, ...add])]; } else if (raw.startsWith('-')) { const rem = raw.slice(1).split(',').map(s => s.trim()).filter(Boolean); current = current.filter(c => !rem.includes(c)); } else { current = raw.split(',').map(s => s.trim()).filter(Boolean); } this[listKeys[key]] = current; this._saveConfig(); this.terminal.print(`${key}: ${current.join(', ')}`, 'success'); return; } const val = parseInt(args[1]); if (isNaN(val) || val < 1 || val > 200) return this.terminal.print('Value must be 1-200', 'warning'); if (key === 'fuzzy-limit') { this._fuzzyLimit = val; } else if (key === 'tree-files') { this._treeFiles = val; } else if (key === 'tree-concepts') { this._treeConcepts = val; } else if (key === 'resolve-files') { this._resolveFiles = val; } else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files, deep-index (on|off), deep-model, exclude-patterns, exclude-dirs, exclude-files, include-only`, 'warning'); } this._saveConfig(); this.terminal.print(`nvfs-config ${key} set to ${val}`, 'success'); } // ── Nvfs Alias ── nvfsAlias(args) { const action = args[0]; if (!action || action === 'list') { const keys = Object.keys(this._aliasMap); if (keys.length === 0) return this.terminal.print('No aliases defined. Use nvfs-alias add ', 'info'); this.terminal.print('🔗 Concept Aliases:', 'info'); for (const [name, targets] of Object.entries(this._aliasMap)) { this.terminal.print(` ${name} → ${targets.join(', ')}`, 'success'); } return; } if (action === 'add') { const name = args[1]; if (!name || !args[2]) return this.terminal.print('Usage: nvfs-alias add ', 'warning'); const targets = args.slice(2).join(' ').split(',').map(t => t.trim()).filter(Boolean); this._aliasMap[name] = targets; this._saveAliases(); this.terminal.print(`Alias "${name}" added (${targets.length} target(s))`, 'success'); return; } if (action === 'remove') { const name = args[1]; if (!name || !this._aliasMap[name]) return this.terminal.print('Alias not found', 'warning'); delete this._aliasMap[name]; this._saveAliases(); this.terminal.print(`Alias "${name}" removed`, 'success'); return; } if (action === 'clear') { this._aliasMap = {}; this._saveAliases(); this.terminal.print('All aliases cleared', 'success'); return; } this.terminal.print('Usage: nvfs-alias list|add |remove |clear', 'warning'); } // ── Private Helpers ── _walk(path, fs, index, cb) { const entries = fs.readDir(path); if (!entries) return; for (const entry of entries) { const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name; if (entry.type === 'file') { cb(fullPath, { type: 'file', content: fs.readFile(fullPath) || '' }); } else if (entry.type === 'dir') { this._walk(fullPath, fs, index, cb); } } } _isSystemFile(path) { const normalized = path.startsWith('/') ? path : '/' + path; const sysPrefixes = ['/.trash', '/.semantic_index', '/.memory_graph', '/_snapshots', '/_sessions', '/logs', '/memory', '/system', '/sys', '/_mods', '/backups', '/agents', '/_agents', '/leases']; if (sysPrefixes.some(p => normalized.startsWith(p))) return true; if (normalized.includes('/node_modules/')) return true; return false; } _isExcluded(path) { const normalized = path.startsWith('/') ? path : '/' + path; // Explicit file or directory names const excludeNames = ['agents', 'leases', 'agent-registry', 'lease-system']; const parts = normalized.split('/').filter(Boolean); for (const part of parts) { if (excludeNames.includes(part.toLowerCase())) return true; if (this._excludeDirs.includes(part)) return true; if (this._excludeFiles.includes(part)) return true; } if (this._excludePatterns.some(p => { try { return new RegExp(p).test(normalized); } catch (e) { return normalized.includes(p); } })) return true; if (this._includeOnly.length > 0) { const isIncluded = this._includeOnly.some(p => { try { return new RegExp(p).test(normalized); } catch (e) { return normalized.includes(p); } }); if (!isIncluded) return true; } return false; } _contentHash(str) { if (!str) return ''; let hash = 5381; for (let i = 0; i < str.length; i++) { hash = ((hash << 5) + hash) + str.charCodeAt(i); hash = hash & hash; } return Math.abs(hash).toString(36).slice(0, 12); } _loadHashCache(fs) { const saved = fs.fsReadJSON(this.hashCachePath); if (saved && typeof saved === 'object') this._hashCache = saved; } _getCached(path, hash) { const c = this._hashCache[path]; if (c && c.hash === hash) return c; return null; } _extractConcepts(path, content) { const concepts = new Set(); const name = path.split('/').pop() || ''; const nameLower = name.toLowerCase(); // Extract from filename (split on camelCase, kebab, dots, underscores) const words = nameLower .replace(/([a-z])([A-Z])/g, '$1 $2') .replace(/[-_.]/g, ' ') .replace(/\.(md|js|html|json|css|txt|yaml|yml|toml|env|sh)$/i, '') .split(/\s+/) .filter(w => w.length > 2 && !/^\d+$/.test(w)); for (const word of words) { concepts.add(word); } // Extract file extension as concept const ext = path.split('.').pop(); if (ext && ['md', 'js', 'html', 'json', 'css'].includes(ext)) { concepts.add(ext + '-file'); } // Extract path-level concepts const parts = path.split('/').filter(p => p.length > 0); for (const part of parts) { if (['modules', 'skills', 'core', 'ext', 'backups', 'chat', 'downloads'].includes(part)) { concepts.add(part); } } // Extract #tags from content (YAML frontmatter or inline) if (content) { let tags = []; const yamlMatch = content.match(/tags:\s*\[([^\]]+)\]/i); if (yamlMatch) { tags = yamlMatch[1].split(',').map(t => t.trim().replace(/^["']|["']$/g, '').replace(/^#/, '')); } else { const hashTags = content.match(/#[a-zA-Z]\w+/g) || []; tags = hashTags.map(t => t.slice(1)); } for (const tag of tags) { if (tag.length > 2) concepts.add(tag.toLowerCase()); } // Extract first H1/H2 as concept const headerMatch = content.match(/^#\s+(.+)/m) || content.match(/^##\s+(.+)/m); if (headerMatch) { const header = headerMatch[1].toLowerCase().replace(/[^a-z0-9\s]/g, '').trim(); if (header.length > 3 && header.length < 40) { concepts.add(header); } } } return [...concepts]; } _hybridShellCheck(concept) { const fs = this.terminal?.fs; if (!fs) return []; const words = concept.split(/\s+/).filter(w => w.length > 2); if (words.length === 0) return []; const results = []; this._walk('/', fs, {}, (path, node) => { if (this._isSystemFile(path)) return; const lower = path.toLowerCase(); if (words.some(w => lower.includes(w))) { results.push({ path, summary: this._summarize(node.content || '') }); } }); return results; } _summarize(content) { if (!content || content.length < 10) return ''; // Strip YAML frontmatter const cleaned = content.replace(/---[\s\S]*?---/, '').trim(); // 1. Try first ## section heading const sectionMatch = cleaned.match(/^##\s+(.+)/m); if (sectionMatch && sectionMatch[1].trim().length > 5) { const s = sectionMatch[1].trim(); return s.slice(0, 100) + (s.length > 100 ? '...' : ''); } // 2. Try first paragraph > 40 chars after frontmatter const lines = cleaned.split('\n').filter(l => l.trim().length > 0); for (const line of lines) { const trimmed = line.trim(); if (trimmed.length > 40 && !trimmed.startsWith('#') && !trimmed.startsWith('//')) { return trimmed.slice(0, 100) + (trimmed.length > 100 ? '...' : ''); } } // 3. First meaningful sentence (> 30 chars) const sentence = cleaned.match(/[^.!?\n]+[.!?]/); if (sentence && sentence[0].trim().length > 30) { const s = sentence[0].trim(); return s.slice(0, 100) + (s.length > 100 ? '...' : ''); } // 4. Fallback: first line > 20 chars (original behavior) const firstLine = lines.find(l => l.trim().length > 20); if (firstLine) { const tl = firstLine.trim(); return tl.slice(0, 100) + (tl.length > 100 ? '...' : ''); } return ''; } _indexSkills(fs, index) { const skills = fs.readDir('/skills'); if (!skills) return; for (const skill of skills) { if (skill.type !== 'file' || !skill.name.endsWith('.md')) continue; // Already indexed by _walk — but we add a "skill" tag const base = skill.name.replace('.md', ''); if (!index['skill']) index['skill'] = []; index['skill'].push({ path: '/skills/' + skill.name, summary: 'Skill: ' + base }); } } _saveIndex(fs) { try { if (!fs.readDir('/.semantic_index')) { fs.mkdirRecursive('/.semantic_index'); } fs.fsWriteJSON(this.indexPath, this.index); // Also persist content hash cache for incremental diffing fs.fsWriteJSON(this.hashCachePath, this._hashCache); } catch (e) { console.error('NeuralVFS save error:', e); } } // ── Auto-Sync ── markDirty(path) { this._dirtyPaths.add(path); // Debounced auto-sync: wait 10s after last mutation if (this._autoSyncTimer) clearTimeout(this._autoSyncTimer); this._autoSyncTimer = setTimeout(() => this._reindexDirtyPaths(), 10000); } async _reindexDirtyPaths() { if (this._dirtyPaths.size === 0) return; const fs = this.terminal?.fs; if (!fs) { this._dirtyPaths.clear(); return; } const start = Date.now(); let reindexed = 0; const reindexedPaths = []; this._loadHashCache(fs); for (const path of [...this._dirtyPaths]) { const content = fs.readFile(path); if (content !== null) { const hash = this._contentHash(content); const cached = this._getCached(path, hash); if (!cached) { this._reindexFile(path, content, this.index, this._deepIndexEnabled); this._hashCache[path] = { hash }; reindexed++; reindexedPaths.push(path); } } else { // File was deleted — remove from index for (const entries of Object.values(this.index)) { const idx = entries.findIndex(e => e.path === path); if (idx !== -1) entries.splice(idx, 1); } delete this._hashCache[path]; } } this._dirtyPaths.clear(); this._autoSyncTimer = null; this.lastScan = new Date().toISOString(); this._saveIndex(fs); const elapsed = ((Date.now() - start) / 1000).toFixed(1); if (reindexed > 0) { this.terminal?.print(`🔄 Auto-sync: re-indexed ${reindexed} changed file(s) in ${elapsed}s`, 'success'); const cap = 20; for (const p of reindexedPaths.slice(0, cap)) { this.terminal?.print(` • ${p}`, ''); } if (reindexedPaths.length > cap) { this.terminal?.print(` ... and ${reindexedPaths.length - cap} more`, 'info'); } } } // ── Enum helpers for autocomplete ── _getNvfsConfigKeys() { return ['fuzzy-limit', 'tree-files', 'tree-concepts', 'resolve-files', 'deep-index', 'deep-model', 'exclude-patterns', 'exclude-dirs', 'exclude-files', 'include-only']; } _getNvfsAliasActions() { return ['list', 'add', 'remove', 'clear']; } // ── Aliases ── _loadAliases() { try { const r = localStorage.getItem('webx11_nvfs_aliases'); if (r) this._aliasMap = JSON.parse(r); } catch (e) {} } _saveAliases() { localStorage.setItem('webx11_nvfs_aliases', JSON.stringify(this._aliasMap)); } // ── Status Bar Polling ── _updateStatusBar() { if (!this.terminal || !this.terminal.statusBar) return; const el = this.terminal.statusBar; if (this._deepIndexing) { const pct = this._deepTotal > 0 ? Math.round(this._deepProcessed / this._deepTotal * 100) : 0; el.textContent = `🧠 Deep Indexing... ${this._deepProcessed}/${this._deepTotal} (${pct}%)`; el.style.color = 'var(--accent, #00ff88)'; } else if (this._autoSyncTimer !== null || this._dirtyPaths.size > 0) { el.textContent = `🔄 Auto-sync pending (${this._dirtyPaths.size} dirty)`; el.style.color = 'var(--warning, #ffaa00)'; } else { const count = Object.keys(this.index).length; el.textContent = `🧠 ${count} concepts${this._deepIndexEnabled ? ' 🌊' : ''}`; el.style.color = ''; } } // ── Deep Indexing (LLM-assisted) ── _parseLLMConcepts(llmResponse) { // LLM returns markdown bullet list: "- concept: description" const concepts = []; if (!llmResponse) return concepts; const lines = llmResponse.split('\n'); for (const line of lines) { const trimmed = line.replace(/^[-*\s]+/, '').trim(); if (!trimmed) continue; // Split on first colon or comma or dash const colonIdx = trimmed.indexOf(':'); const commaIdx = trimmed.indexOf(','); const delimIdx = colonIdx > 0 ? colonIdx : (commaIdx > 0 ? commaIdx : -1); const concept = delimIdx > 0 ? trimmed.slice(0, delimIdx).trim().toLowerCase() : trimmed.toLowerCase(); if (concept && concept.length > 2 && concept.length < 60) { concepts.push(concept); } } return [...new Set(concepts)]; } async _callLLM(prompt, context) { const om = this.terminal?.ollamaManager; if (!om || !om.connected) return null; const fullPrompt = `Extract 2-5 key searchable concepts from this file. Return one per line as lowercase keywords (1-3 words). File: ${context.path} Content: ${context.content.slice(0, 2000)} Concepts:`; try { const result = await om._chatWithProvider(om.currentProvider, fullPrompt, this._deepModel); return result; } catch (e) { console.warn('LLM concept extraction failed:', e); return null; } } async _processDeepQueue(index) { this._deepProcessed = 0; this._deepTotal = this._deepQueue.length; const batchSize = 3; const results = []; while (this._deepQueue.length > 0) { // Check cancel between batches if (this._cancelRequested) { this._cancelRequested = false; this._deepIndexing = false; this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; this._updateStatusBar(); this.terminal?.print('⛔ Deep index cancelled', 'warning'); return; } const batch = this._deepQueue.splice(0, batchSize); for (const item of batch) { // Check cancel before each LLM call so user can abort mid-batch if (this._cancelRequested) { this._cancelRequested = false; this._deepIndexing = false; this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; this._updateStatusBar(); // Persist any concepts collected so far before exiting const vfs = this.terminal?.fs; if (vfs) this._saveIndex(vfs); this.terminal?.print('⛔ Deep index cancelled mid-batch', 'warning'); return; } const context = { path: item.path, content: item.content }; const llmResponse = await this._callLLM('', context); // Check cancel after the (possibly long) LLM call before processing if (this._cancelRequested) { this._cancelRequested = false; this._deepIndexing = false; this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; this._updateStatusBar(); const vfs = this.terminal?.fs; if (vfs) this._saveIndex(vfs); this.terminal?.print('⛔ Deep index cancelled after LLM call', 'warning'); return; } if (llmResponse) { const concepts = this._parseLLMConcepts(llmResponse); results.push({ path: item.path, concepts }); for (const concept of concepts) { if (!index[concept]) index[concept] = []; const exists = index[concept].some(e => e.path === item.path); if (!exists) index[concept].push({ path: item.path, summary: this._summarize(item.content), source: 'llm' }); } } this._deepProcessed++; // Update status bar live this._updateStatusBar(); } // Yield to UI between batches await new Promise(r => setTimeout(r, 50)); } this._deepIndexing = false; this._deepQueue = []; this._deepProcessed = 0; this._deepTotal = 0; this._updateStatusBar(); if (results.length > 0) { const llmConcepts = results.reduce((s, r) => s + r.concepts.length, 0); const totalLLM = Object.values(index).flat().filter(e => e.source === 'llm').length; this.terminal?.print(`🧠 Deep pass: added ${llmConcepts} new LLM concepts (${results.length} file${results.length === 1 ? '' : 's'}). Total in index: ${totalLLM}`, 'success'); // List processed files so user can see if one file keeps re-processing (loop detection) const cap = 20; if (results.length <= cap) { this.terminal?.print(` Files processed this pass:`, 'info'); for (const r of results) { this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); } } else { this.terminal?.print(` First ${cap} of ${results.length} files:`, 'info'); for (const r of results.slice(0, cap)) { this.terminal?.print(` • ${r.path} (${r.concepts.length} concepts)`, ''); } } } // Persist LLM-generated concepts to disk so they survive page refresh. // Do NOT bypass _storageSavePending (race condition: bypassing re-triggers // a debounced saveToStorage 2s later that can clobber LLM concepts if the // user edits a file in the meantime). Just trigger a normal save — the // 2s debounce is fine because the user isn't waiting on the keyboard. const vfs = this.terminal?.fs; if (vfs) { // await any in-flight save so this write lands on a clean slate if (vfs._storageSavePending) { await new Promise(resolve => { const check = setInterval(() => { if (!vfs._storageSavePending) { clearInterval(check); resolve(); } }, 50); }); } this._saveIndex(vfs); } } // ── Nvfs Save (force-persist in-memory index) ── async nvfsSave() { const fs = this.terminal?.fs; if (!fs) return this.terminal?.print('VFS not available', 'error'); const count = Object.keys(this.index).length; if (count === 0) return this.terminal?.print('No index in memory to save', 'warning'); // Await any in-flight VFS save first — otherwise our write lands mid-flush // and the debounce timer fires 2s later, clobbering our concepts. if (fs._storageSavePending) { await new Promise(resolve => { const check = setInterval(() => { if (!fs._storageSavePending) { clearInterval(check); resolve(); } }, 50); }); } this._saveIndex(fs); this.terminal?.print(`✅ Saved ${count} concepts to ${this.indexPath}`, 'success'); this.terminal?.print(' Index will survive page refresh', 'info'); } // ── Nvfs Sync (incremental background) ── async nvfsSync(args) { const useDeep = args && args.includes('--deep'); if (this._dirtyPaths.size === 0) { this.terminal.print('No dirty files to sync', 'info'); return; } this.terminal.print(`🔄 Syncing ${this._dirtyPaths.size} dirty file(s)${useDeep ? ' (deep mode)' : ''}...`, 'info'); const fs = this.terminal?.fs; if (!fs) return; await this._reindexDirtyPaths(); // After reindex, if deep mode, queue remaining queued items for LLM processing if (useDeep && this._deepQueue.length > 0) { this._deepIndexing = true; await this._processDeepQueue(this.index); } } cancelIndex() { this._cancelRequested = true; this.terminal?.print('⏳ Cancelling index...', 'warning'); } async init(terminal) { // Auto-load saved index from VFS on startup so last index survives refresh const fs = terminal?.fs || this.terminal?.fs; if (!fs) return; // Await VFS async loadFromStorage() — it uses CompressionStream which is async. // module-manager.js awaits this function (line 33), so the rest of init runs // before any other module can call nvfs-status. await Promise.resolve(fs._ready); const saved = fs.fsReadJSON(this.indexPath); if (saved && typeof saved === 'object' && Object.keys(saved).length > 0) { this.index = saved; this._loadHashCache(fs); this._loadAliases(); this.lastScan = '(loaded from disk)'; // One-time migration: purge any stale /sys/... entries left over from // pre-v4.5.34 builds where the /sys prefix wasn't in _isSystemFile(). // These entries cause a permanent re-index loop because /sys files use // _factory dynamic content (timestamps, heap stats) so their hash // never matches and they get re-queued every nvfs-index --deep run. let purgedConcepts = 0; let purgedFiles = 0; for (const [concept, entries] of Object.entries(this.index)) { if (!Array.isArray(entries)) continue; const before = entries.length; const filtered = entries.filter(e => !(e.path && e.path.startsWith('/sys'))); if (filtered.length !== before) purgedConcepts += (before - filtered.length); if (filtered.length > 0) this.index[concept] = filtered; else delete this.index[concept]; } for (const path of Object.keys(this._hashCache)) { if (path.startsWith('/sys')) { delete this._hashCache[path]; purgedFiles++; } } if (purgedConcepts > 0 || purgedFiles > 0) { this._saveIndex(fs); this.terminal?.print(`🧹 Migrated stale index: purged ${purgedConcepts} /sys concepts + ${purgedFiles} hash entries from pre-v4.5.34 build`, 'warning'); } } } // v4.5.89: Semantic bridge — mirror graph facts into VFS for unified search indexGraphFacts() { const mg = this.terminal.moduleManager.modules['MemoryGraphModule']; if (!mg) return this.terminal.print('❌ Memory Graph module not loaded', 'error'); const fs = this.terminal.fs; if (!fs) return this.terminal.print('❌ VFS not available', 'error'); const factNodes = Object.values(mg._nodes).filter(function(n) { return n.type === 'concept' || n.type === 'fact' || n.type === 'entity' || n.type === 'task'; }); if (factNodes.length === 0) { return this.terminal.print('No graph fact nodes to index. Use memgraph-learn first.', 'info'); } if (!fs.readDir('/_graph_facts')) fs.mkdir('_graph_facts'); // v4.5.89b fix: Clean stale files for deleted nodes (P2 from review) const validKeys = new Set(factNodes.map(function(n) { return n.id.replace(/[^a-zA-Z0-9_-]/g, '_') + '.txt'; })); try { const existing = fs.readDir('/_graph_facts') || []; for (const f of existing) { if (!validKeys.has(f)) fs.deleteFile('/_graph_facts/' + f); } } catch (_) {} let indexed = 0; for (const node of factNodes) { let content = node.label + '\n'; if (node.metadata) { if (node.metadata.full) content += node.metadata.full + '\n'; if (node.metadata.result) content += 'Result: ' + node.metadata.result + '\n'; } content += 'Type: ' + node.type + '\n'; content += 'Namespace: ' + (node.namespace || 'global') + '\n'; if (node.source_session) content += 'Session: ' + node.source_session + '\n'; const safeName = node.id.replace(/[^a-zA-Z0-9_-]/g, '_'); fs.writeFile('/_graph_facts/' + safeName + '.txt', content); indexed++; } this.terminal.print('🔗 Indexed ' + indexed + ' graph facts into /_graph_facts/', 'success'); this.terminal.print(' Run nvfs-index to include them in semantic search', 'info'); } // v4.5.89b: Index a single fact — O(1) instead of O(n) re-index (P2 from review) indexSingleFact(factKey, entity, fact) { const fs = this.terminal.fs; if (!fs) return; if (!fs.readDir('/_graph_facts')) fs.mkdir('_graph_facts'); const safeName = factKey.replace(/[^a-zA-Z0-9_-]/g, '_'); const content = entity + '\n' + fact + '\nType: fact\nNamespace: global\n'; fs.writeFile('/_graph_facts/' + safeName + '.txt', content); } } // Required for module system if (typeof window !== 'undefined') { window.NeuralVFSModule = NeuralVFSModule; }