From ccc7d9956559d2195e20345e7a794662b52cd403 Mon Sep 17 00:00:00 2001 From: Agando Date: Sat, 6 Jun 2026 06:00:27 +0200 Subject: auto: 2026-06-06_06:00 changes --- modules/core/module-neural-vfs.js | 265 +++++++++++++++++++++++++++++++------- 1 file changed, 217 insertions(+), 48 deletions(-) (limited to 'modules/core/module-neural-vfs.js') diff --git a/modules/core/module-neural-vfs.js b/modules/core/module-neural-vfs.js index e9ffafb..2644e85 100644 --- a/modules/core/module-neural-vfs.js +++ b/modules/core/module-neural-vfs.js @@ -4,6 +4,20 @@ class NeuralVFSModule { this.terminal = null; this.index = {}; this.indexPath = '/.semantic_index/map.json'; + this.hashCachePath = '/.semantic_index/hash_cache.json'; + this._hashCache = {}; + // Configurable display limits + this._fuzzyLimit = 10; + this._treeFiles = 5; + this._treeConcepts = 15; + this._resolveFiles = 20; + this._loadConfig(); + } + _loadConfig() { + try { const c = localStorage.getItem('webx11_nvfs_config'); if (c) { const p = JSON.parse(c); if (p.fuzzyLimit) this._fuzzyLimit = p.fuzzyLimit; if (p.treeFiles) this._treeFiles = p.treeFiles; if (p.treeConcepts) this._treeConcepts = p.treeConcepts; if (p.resolveFiles) this._resolveFiles = p.resolveFiles; } } catch (e) {} + } + _saveConfig() { + localStorage.setItem('webx11_nvfs_config', JSON.stringify({ fuzzyLimit: this._fuzzyLimit, treeFiles: this._treeFiles, treeConcepts: this._treeConcepts, resolveFiles: this._resolveFiles })); } getHelpInfo() { @@ -11,7 +25,8 @@ class NeuralVFSModule { 'nvfs-index': 'nvfs-index — Scan VFS and build semantic concept index', 'nvfs-status': 'nvfs-status — Show semantic index stats and last scan time', 'nvfs-resolve': 'nvfs-resolve — Find files related to a concept', - 'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree' + 'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree', + 'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS display limits (fuzzy-limit, tree-files, tree-concepts, resolve-files)' }; } @@ -20,56 +35,133 @@ class NeuralVFSModule { this.commands['nvfs-status'] = (args) => this.showStatus(); this.commands['nvfs-resolve'] = (args) => this.resolve(args); this.commands['nvfs-tree'] = (args) => this.tree(args); + this.commands['nvfs-config'] = (args) => this.nvfsConfig(args); } - // ── Core Index Builder ── + // ── Core Index Builder (async, chunked) ── - buildIndex() { + async buildIndex() { const fs = this.terminal.fs; if (!fs) return this.terminal.print('VFS not available', 'error'); this.terminal.print('🔍 Scanning VFS for semantic concepts...', 'info'); const start = Date.now(); - const newIndex = {}; + let newIndex = {}; let fileCount = 0; let conceptCount = 0; + let cachedCount = 0; - // Walk VFS recursively — skip system dirs - this._walk('/', fs, newIndex, (path, node) => { - if (node.type !== 'file') return; - if (this._isSystemFile(path)) return; - - fileCount++; - const concepts = this._extractConcepts(path, node.content || ''); - for (const concept of concepts) { - if (!newIndex[concept]) { - newIndex[concept] = []; - conceptCount++; - } - // Avoid dupes - if (!newIndex[concept].find(e => e.path === path)) { - newIndex[concept].push({ path, summary: this._summarize(node.content) }); - } - } - }); + // Load previous index and hash cache for incremental diffing + const prevIndex = fs.fsReadJSON(this.indexPath) || {}; + this._loadHashCache(fs); + + // Collect current file list + const fileList = []; + this._collectFiles('/', fs, fileList); - // Also index skills - this._indexSkills(fs, newIndex); + if (fileList.length === 0) { + return this.terminal.print('No files found to index', 'warning'); + } + + // Detect deleted files — strip orphaned entries from prevIndex + const currentPaths = new Set(fileList.map(f => f.path)); + let deletedCount = 0; + for (const [concept, entries] of Object.entries(prevIndex)) { + const remaining = entries.filter(e => currentPaths.has(e.path)); + if (remaining.length !== entries.length) deletedCount++; + if (remaining.length > 0) prevIndex[concept] = remaining; + else delete prevIndex[concept]; + } + + // Warm start: clone prevIndex and count live paths for stats + newIndex = JSON.parse(JSON.stringify(prevIndex)); + const livePaths = new Set(); + for (const entries of Object.values(newIndex)) { + for (const e of entries) livePaths.add(e.path); + } + conceptCount = Object.keys(newIndex).length; + + // Chunked async processing — yields to UI between chunks + const chunkSize = 10; + let i = 0; + const processChunk = () => { + const end = Math.min(i + chunkSize, fileList.length); + for (let j = i; j < end; j++) { + const { path, node } = fileList[j]; + fileCount++; + const content = node.content || ''; + const hash = this._contentHash(content); + + // Skip if hash unchanged — content is same as last scan + const cached = this._getCached(path, hash); + if (cached) { + cachedCount++; + if (!livePaths.has(path)) { + // File was in cache but somehow missing from prevIndex + this._reindexFile(path, content, newIndex); + } + continue; + } - this.index = newIndex; - this.lastScan = new Date().toISOString(); - this.fileCount = fileCount; - this.conceptCount = conceptCount; + // Changed or new file — extract concepts fresh + this._reindexFile(path, content, newIndex); + this._hashCache[path] = { hash }; + } + i = end; + if (i < fileList.length) { + const pct = Math.round(i / fileList.length * 100); + const cMsg = cachedCount > 0 ? ` (${cachedCount} cached)` : ''; + this.terminal.print(`⏳ Indexing... ${pct}% (${i}/${fileList.length} files${cMsg})`, 'info'); + setTimeout(processChunk, 0); + } else { + // Also index skills + this._indexSkills(fs, newIndex); + this.index = newIndex; + this.lastScan = new Date().toISOString(); + this.fileCount = fileCount; + this.conceptCount = Object.keys(newIndex).length; + this._saveIndex(fs); + if (fs._clearConceptCache) fs._clearConceptCache(); + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + const processedMsg = cachedCount > 0 ? ` (${cachedCount} cached, ${fileCount - cachedCount} reindexed)` : ''; + this.terminal.print(`✅ Indexed ${fileCount} files → ${Object.keys(newIndex).length} concepts in ${elapsed}s${processedMsg}`, 'success'); + if (deletedCount > 0) this.terminal.print(` Purged ${deletedCount} orphaned concepts from deleted files`, 'info'); + this.terminal.print(` Index saved to ${this.indexPath}`, 'info'); + } + }; + processChunk(); + } - // Save to VFS - this._saveIndex(fs); - // Clear VFS concept cache so concept:// URIs reflect the new index - if (fs._clearConceptCache) fs._clearConceptCache(); + _reindexFile(path, content, index) { + const concepts = this._extractConcepts(path, content); + // Remove old entries for this path + for (const entries of Object.values(index)) { + const idx = entries.findIndex(e => e.path === path); + if (idx !== -1) entries.splice(idx, 1); + } + // Add new entries + for (const concept of concepts) { + if (!index[concept]) { + index[concept] = []; + this.conceptCount++; + } + index[concept].push({ path, summary: this._summarize(content) }); + } + } - const elapsed = ((Date.now() - start) / 1000).toFixed(1); - this.terminal.print(`✅ Indexed ${fileCount} files → ${conceptCount} concepts in ${elapsed}s`, 'success'); - this.terminal.print(` Index saved to ${this.indexPath}`, 'info'); + _collectFiles(path, fs, list) { + const entries = fs.readDir(path); + if (!entries) return; + for (const entry of entries) { + const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name; + if (entry.type === 'file') { + if (this._isSystemFile(fullPath)) continue; + list.push({ path: fullPath, node: { type: 'file', content: fs.readFile(fullPath) || '' } }); + } else if (entry.type === 'dir') { + this._collectFiles(fullPath, fs, list); + } + } } // ── Resolve ── @@ -91,17 +183,17 @@ class NeuralVFSModule { if (matches.length > 0) { this.terminal.print(`📂 "${concept}" → ${matches.length} file(s):`, 'info'); - for (const m of matches.slice(0, 20)) { + for (const m of matches.slice(0, this._resolveFiles)) { this.terminal.print(` ${m.path}`, 'success'); if (m.summary) this.terminal.print(` ${m.summary}`, '') } - if (matches.length > 20) { - this.terminal.print(` ... and ${matches.length - 20} more`, 'info'); + if (matches.length > this._resolveFiles) { + this.terminal.print(` ... and ${matches.length - this._resolveFiles} more`, 'info'); } } if (fuzzy.length > 0) { - this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, 10).join(', ')}`, 'info'); + this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, this._fuzzyLimit).join(', ')}`, 'info'); } } @@ -145,20 +237,45 @@ class NeuralVFSModule { const filtered = filter ? concepts.filter(c => c.includes(filter)) : concepts; this.terminal.print(`🌳 Concept Tree${filter ? ` (filter: "${filter}")` : ''}:`, 'info'); - for (const concept of filtered.slice(0, 15)) { + for (const concept of filtered.slice(0, this._treeConcepts)) { this.terminal.print(` 📁 ${concept}`, 'success'); - for (const entry of this.index[concept].slice(0, 5)) { + for (const entry of this.index[concept].slice(0, this._treeFiles)) { this.terminal.print(` └─ ${entry.path}`, ''); } - if (this.index[concept].length > 5) { - this.terminal.print(` └─ ... (+${this.index[concept].length - 5} more)`, ''); + if (this.index[concept].length > this._treeFiles) { + this.terminal.print(` └─ ... (+${this.index[concept].length - this._treeFiles} more)`, ''); } } - if (filtered.length > 15) { - this.terminal.print(` ... and ${filtered.length - 15} more concepts`, 'info'); + if (filtered.length > this._treeConcepts) { + this.terminal.print(` ... and ${filtered.length - this._treeConcepts} more concepts`, 'info'); } } + // ── Nvfs Config ── + + nvfsConfig(args) { + if (args.length === 0) { + this.terminal.print('=== Neural VFS Config ===', 'info'); + this.terminal.print(`fuzzy-limit: ${this._fuzzyLimit} (default: 10)`, ''); + this.terminal.print(`tree-files: ${this._treeFiles} (default: 5)`, ''); + this.terminal.print(`tree-concepts: ${this._treeConcepts} (default: 15)`, ''); + this.terminal.print(`resolve-files: ${this._resolveFiles} (default: 20)`, ''); + this.terminal.print('Usage: nvfs-config ', 'info'); + this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files', 'info'); + return; + } + const key = args[0]; + const val = parseInt(args[1]); + if (isNaN(val) || val < 1 || val > 200) return this.terminal.print('Value must be 1-200', 'warning'); + if (key === 'fuzzy-limit') { this._fuzzyLimit = val; } + else if (key === 'tree-files') { this._treeFiles = val; } + else if (key === 'tree-concepts') { this._treeConcepts = val; } + else if (key === 'resolve-files') { this._resolveFiles = val; } + else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files`, 'warning'); } + this._saveConfig(); + this.terminal.print(`nvfs-config ${key} set to ${val}`, 'success'); + } + // ── Private Helpers ── _walk(path, fs, index, cb) { @@ -179,6 +296,27 @@ class NeuralVFSModule { return sysPrefixes.some(p => path.startsWith(p) || path.includes('/node_modules')); } + _contentHash(str) { + if (!str) return ''; + let hash = 5381; + for (let i = 0; i < str.length; i++) { + hash = ((hash << 5) + hash) + str.charCodeAt(i); + hash = hash & hash; + } + return Math.abs(hash).toString(36).slice(0, 12); + } + + _loadHashCache(fs) { + const saved = fs.fsReadJSON(this.hashCachePath); + if (saved && typeof saved === 'object') this._hashCache = saved; + } + + _getCached(path, hash) { + const c = this._hashCache[path]; + if (c && c.hash === hash) return c; + return null; + } + _extractConcepts(path, content) { const concepts = new Set(); const name = path.split('/').pop() || ''; @@ -235,10 +373,39 @@ class NeuralVFSModule { _summarize(content) { if (!content || content.length < 10) return ''; + // Strip YAML frontmatter const cleaned = content.replace(/---[\s\S]*?---/, '').trim(); - const firstLine = cleaned.split('\n').find(l => l.trim().length > 20); - if (!firstLine) return ''; - return firstLine.trim().slice(0, 100) + (firstLine.length > 100 ? '...' : ''); + + // 1. Try first ## section heading + const sectionMatch = cleaned.match(/^##\s+(.+)/m); + if (sectionMatch && sectionMatch[1].trim().length > 5) { + const s = sectionMatch[1].trim(); + return s.slice(0, 100) + (s.length > 100 ? '...' : ''); + } + + // 2. Try first paragraph > 40 chars after frontmatter + const lines = cleaned.split('\n').filter(l => l.trim().length > 0); + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.length > 40 && !trimmed.startsWith('#') && !trimmed.startsWith('//')) { + return trimmed.slice(0, 100) + (trimmed.length > 100 ? '...' : ''); + } + } + + // 3. First meaningful sentence (> 30 chars) + const sentence = cleaned.match(/[^.!?\n]+[.!?]/); + if (sentence && sentence[0].trim().length > 30) { + const s = sentence[0].trim(); + return s.slice(0, 100) + (s.length > 100 ? '...' : ''); + } + + // 4. Fallback: first line > 20 chars (original behavior) + const firstLine = lines.find(l => l.trim().length > 20); + if (firstLine) { + const tl = firstLine.trim(); + return tl.slice(0, 100) + (tl.length > 100 ? '...' : ''); + } + return ''; } _indexSkills(fs, index) { @@ -259,6 +426,8 @@ class NeuralVFSModule { fs.mkdirRecursive('/.semantic_index'); } fs.fsWriteJSON(this.indexPath, this.index); + // Also persist content hash cache for incremental diffing + fs.fsWriteJSON(this.hashCachePath, this._hashCache); } catch (e) { console.error('NeuralVFS save error:', e); } -- cgit v1.2.3