auto: 2026-06-06_06:00 changes

1 files changed, 217 insertions, 48 deletions

diff --git a/modules/core/module-neural-vfs.js b/modules/core/module-neural-vfs.js
index e9ffafb..2644e85 100644
--- a/modules/core/module-neural-vfs.js
+++ b/modules/core/module-neural-vfs.js
@@ -4,6 +4,20 @@ class NeuralVFSModule {
         this.terminal = null;
         this.index = {};
         this.indexPath = '/.semantic_index/map.json';
+        this.hashCachePath = '/.semantic_index/hash_cache.json';
+        this._hashCache = {};
+        // Configurable display limits
+        this._fuzzyLimit = 10;
+        this._treeFiles = 5;
+        this._treeConcepts = 15;
+        this._resolveFiles = 20;
+        this._loadConfig();
+    }
+    _loadConfig() {
+        try { const c = localStorage.getItem('webx11_nvfs_config'); if (c) { const p = JSON.parse(c); if (p.fuzzyLimit) this._fuzzyLimit = p.fuzzyLimit; if (p.treeFiles) this._treeFiles = p.treeFiles; if (p.treeConcepts) this._treeConcepts = p.treeConcepts; if (p.resolveFiles) this._resolveFiles = p.resolveFiles; } } catch (e) {}
+    }
+    _saveConfig() {
+        localStorage.setItem('webx11_nvfs_config', JSON.stringify({ fuzzyLimit: this._fuzzyLimit, treeFiles: this._treeFiles, treeConcepts: this._treeConcepts, resolveFiles: this._resolveFiles }));
     }
 
     getHelpInfo() {
@@ -11,7 +25,8 @@ class NeuralVFSModule {
             'nvfs-index': 'nvfs-index — Scan VFS and build semantic concept index',
             'nvfs-status': 'nvfs-status — Show semantic index stats and last scan time',
             'nvfs-resolve': 'nvfs-resolve <concept> — Find files related to a concept',
-            'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree'
+            'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree',
+            'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS display limits (fuzzy-limit, tree-files, tree-concepts, resolve-files)'
         };
     }
 
@@ -20,56 +35,133 @@ class NeuralVFSModule {
         this.commands['nvfs-status'] = (args) => this.showStatus();
         this.commands['nvfs-resolve'] = (args) => this.resolve(args);
         this.commands['nvfs-tree'] = (args) => this.tree(args);
+        this.commands['nvfs-config'] = (args) => this.nvfsConfig(args);
     }
 
-    // ── Core Index Builder ──
+    // ── Core Index Builder (async, chunked) ──
 
-    buildIndex() {
+    async buildIndex() {
         const fs = this.terminal.fs;
         if (!fs) return this.terminal.print('VFS not available', 'error');
 
         this.terminal.print('🔍 Scanning VFS for semantic concepts...', 'info');
 
         const start = Date.now();
-        const newIndex = {};
+        let newIndex = {};
         let fileCount = 0;
         let conceptCount = 0;
+        let cachedCount = 0;
 
-        // Walk VFS recursively — skip system dirs
-        this._walk('/', fs, newIndex, (path, node) => {
-            if (node.type !== 'file') return;
-            if (this._isSystemFile(path)) return;
-
-            fileCount++;
-            const concepts = this._extractConcepts(path, node.content || '');
-            for (const concept of concepts) {
-                if (!newIndex[concept]) {
-                    newIndex[concept] = [];
-                    conceptCount++;
-                }
-                // Avoid dupes
-                if (!newIndex[concept].find(e => e.path === path)) {
-                    newIndex[concept].push({ path, summary: this._summarize(node.content) });
-                }
-            }
-        });
+        // Load previous index and hash cache for incremental diffing
+        const prevIndex = fs.fsReadJSON(this.indexPath) || {};
+        this._loadHashCache(fs);
+
+        // Collect current file list
+        const fileList = [];
+        this._collectFiles('/', fs, fileList);
 
-        // Also index skills
-        this._indexSkills(fs, newIndex);
+        if (fileList.length === 0) {
+            return this.terminal.print('No files found to index', 'warning');
+        }
+
+        // Detect deleted files — strip orphaned entries from prevIndex
+        const currentPaths = new Set(fileList.map(f => f.path));
+        let deletedCount = 0;
+        for (const [concept, entries] of Object.entries(prevIndex)) {
+            const remaining = entries.filter(e => currentPaths.has(e.path));
+            if (remaining.length !== entries.length) deletedCount++;
+            if (remaining.length > 0) prevIndex[concept] = remaining;
+            else delete prevIndex[concept];
+        }
+
+        // Warm start: clone prevIndex and count live paths for stats
+        newIndex = JSON.parse(JSON.stringify(prevIndex));
+        const livePaths = new Set();
+        for (const entries of Object.values(newIndex)) {
+            for (const e of entries) livePaths.add(e.path);
+        }
+        conceptCount = Object.keys(newIndex).length;
+
+        // Chunked async processing — yields to UI between chunks
+        const chunkSize = 10;
+        let i = 0;
+        const processChunk = () => {
+            const end = Math.min(i + chunkSize, fileList.length);
+            for (let j = i; j < end; j++) {
+                const { path, node } = fileList[j];
+                fileCount++;
+                const content = node.content || '';
+                const hash = this._contentHash(content);
+
+                // Skip if hash unchanged — content is same as last scan
+                const cached = this._getCached(path, hash);
+                if (cached) {
+                    cachedCount++;
+                    if (!livePaths.has(path)) {
+                        // File was in cache but somehow missing from prevIndex
+                        this._reindexFile(path, content, newIndex);
+                    }
+                    continue;
+                }
 
-        this.index = newIndex;
-        this.lastScan = new Date().toISOString();
-        this.fileCount = fileCount;
-        this.conceptCount = conceptCount;
+                // Changed or new file — extract concepts fresh
+                this._reindexFile(path, content, newIndex);
+                this._hashCache[path] = { hash };
+            }
+            i = end;
+            if (i < fileList.length) {
+                const pct = Math.round(i / fileList.length * 100);
+                const cMsg = cachedCount > 0 ? ` (${cachedCount} cached)` : '';
+                this.terminal.print(`⏳ Indexing... ${pct}% (${i}/${fileList.length} files${cMsg})`, 'info');
+                setTimeout(processChunk, 0);
+            } else {
+                // Also index skills
+                this._indexSkills(fs, newIndex);
+                this.index = newIndex;
+                this.lastScan = new Date().toISOString();
+                this.fileCount = fileCount;
+                this.conceptCount = Object.keys(newIndex).length;
+                this._saveIndex(fs);
+                if (fs._clearConceptCache) fs._clearConceptCache();
+                const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+                const processedMsg = cachedCount > 0 ? ` (${cachedCount} cached, ${fileCount - cachedCount} reindexed)` : '';
+                this.terminal.print(`✅ Indexed ${fileCount} files → ${Object.keys(newIndex).length} concepts in ${elapsed}s${processedMsg}`, 'success');
+                if (deletedCount > 0) this.terminal.print(`   Purged ${deletedCount} orphaned concepts from deleted files`, 'info');
+                this.terminal.print(`   Index saved to ${this.indexPath}`, 'info');
+            }
+        };
+        processChunk();
+    }
 
-        // Save to VFS
-        this._saveIndex(fs);
-        // Clear VFS concept cache so concept:// URIs reflect the new index
-        if (fs._clearConceptCache) fs._clearConceptCache();
+    _reindexFile(path, content, index) {
+        const concepts = this._extractConcepts(path, content);
+        // Remove old entries for this path
+        for (const entries of Object.values(index)) {
+            const idx = entries.findIndex(e => e.path === path);
+            if (idx !== -1) entries.splice(idx, 1);
+        }
+        // Add new entries
+        for (const concept of concepts) {
+            if (!index[concept]) {
+                index[concept] = [];
+                this.conceptCount++;
+            }
+            index[concept].push({ path, summary: this._summarize(content) });
+        }
+    }
 
-        const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-        this.terminal.print(`✅ Indexed ${fileCount} files → ${conceptCount} concepts in ${elapsed}s`, 'success');
-        this.terminal.print(`   Index saved to ${this.indexPath}`, 'info');
+    _collectFiles(path, fs, list) {
+        const entries = fs.readDir(path);
+        if (!entries) return;
+        for (const entry of entries) {
+            const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name;
+            if (entry.type === 'file') {
+                if (this._isSystemFile(fullPath)) continue;
+                list.push({ path: fullPath, node: { type: 'file', content: fs.readFile(fullPath) || '' } });
+            } else if (entry.type === 'dir') {
+                this._collectFiles(fullPath, fs, list);
+            }
+        }
     }
 
     // ── Resolve ──
@@ -91,17 +183,17 @@ class NeuralVFSModule {
 
         if (matches.length > 0) {
             this.terminal.print(`📂 "${concept}" → ${matches.length} file(s):`, 'info');
-            for (const m of matches.slice(0, 20)) {
+            for (const m of matches.slice(0, this._resolveFiles)) {
                 this.terminal.print(`   ${m.path}`, 'success');
                 if (m.summary) this.terminal.print(`     ${m.summary}`, '')
             }
-            if (matches.length > 20) {
-                this.terminal.print(`   ... and ${matches.length - 20} more`, 'info');
+            if (matches.length > this._resolveFiles) {
+                this.terminal.print(`   ... and ${matches.length - this._resolveFiles} more`, 'info');
             }
         }
 
         if (fuzzy.length > 0) {
-            this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, 10).join(', ')}`, 'info');
+            this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, this._fuzzyLimit).join(', ')}`, 'info');
         }
     }
 
@@ -145,20 +237,45 @@ class NeuralVFSModule {
         const filtered = filter ? concepts.filter(c => c.includes(filter)) : concepts;
 
         this.terminal.print(`🌳 Concept Tree${filter ? ` (filter: "${filter}")` : ''}:`, 'info');
-        for (const concept of filtered.slice(0, 15)) {
+        for (const concept of filtered.slice(0, this._treeConcepts)) {
             this.terminal.print(`  📁 ${concept}`, 'success');
-            for (const entry of this.index[concept].slice(0, 5)) {
+            for (const entry of this.index[concept].slice(0, this._treeFiles)) {
                 this.terminal.print(`     └─ ${entry.path}`, '');
             }
-            if (this.index[concept].length > 5) {
-                this.terminal.print(`     └─ ... (+${this.index[concept].length - 5} more)`, '');
+            if (this.index[concept].length > this._treeFiles) {
+                this.terminal.print(`     └─ ... (+${this.index[concept].length - this._treeFiles} more)`, '');
             }
         }
-        if (filtered.length > 15) {
-            this.terminal.print(`  ... and ${filtered.length - 15} more concepts`, 'info');
+        if (filtered.length > this._treeConcepts) {
+            this.terminal.print(`  ... and ${filtered.length - this._treeConcepts} more concepts`, 'info');
         }
     }
 
+    // ── Nvfs Config ──
+
+    nvfsConfig(args) {
+        if (args.length === 0) {
+            this.terminal.print('=== Neural VFS Config ===', 'info');
+            this.terminal.print(`fuzzy-limit: ${this._fuzzyLimit} (default: 10)`, '');
+            this.terminal.print(`tree-files: ${this._treeFiles} (default: 5)`, '');
+            this.terminal.print(`tree-concepts: ${this._treeConcepts} (default: 15)`, '');
+            this.terminal.print(`resolve-files: ${this._resolveFiles} (default: 20)`, '');
+            this.terminal.print('Usage: nvfs-config <key> <value>', 'info');
+            this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files', 'info');
+            return;
+        }
+        const key = args[0];
+        const val = parseInt(args[1]);
+        if (isNaN(val) || val < 1 || val > 200) return this.terminal.print('Value must be 1-200', 'warning');
+        if (key === 'fuzzy-limit') { this._fuzzyLimit = val; }
+        else if (key === 'tree-files') { this._treeFiles = val; }
+        else if (key === 'tree-concepts') { this._treeConcepts = val; }
+        else if (key === 'resolve-files') { this._resolveFiles = val; }
+        else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files`, 'warning'); }
+        this._saveConfig();
+        this.terminal.print(`nvfs-config ${key} set to ${val}`, 'success');
+    }
+
     // ── Private Helpers ──
 
     _walk(path, fs, index, cb) {
@@ -179,6 +296,27 @@ class NeuralVFSModule {
         return sysPrefixes.some(p => path.startsWith(p) || path.includes('/node_modules'));
     }
 
+    _contentHash(str) {
+        if (!str) return '';
+        let hash = 5381;
+        for (let i = 0; i < str.length; i++) {
+            hash = ((hash << 5) + hash) + str.charCodeAt(i);
+            hash = hash & hash;
+        }
+        return Math.abs(hash).toString(36).slice(0, 12);
+    }
+
+    _loadHashCache(fs) {
+        const saved = fs.fsReadJSON(this.hashCachePath);
+        if (saved && typeof saved === 'object') this._hashCache = saved;
+    }
+
+    _getCached(path, hash) {
+        const c = this._hashCache[path];
+        if (c && c.hash === hash) return c;
+        return null;
+    }
+
     _extractConcepts(path, content) {
         const concepts = new Set();
         const name = path.split('/').pop() || '';
@@ -235,10 +373,39 @@ class NeuralVFSModule {
 
     _summarize(content) {
         if (!content || content.length < 10) return '';
+        // Strip YAML frontmatter
         const cleaned = content.replace(/---[\s\S]*?---/, '').trim();
-        const firstLine = cleaned.split('\n').find(l => l.trim().length > 20);
-        if (!firstLine) return '';
-        return firstLine.trim().slice(0, 100) + (firstLine.length > 100 ? '...' : '');
+        
+        // 1. Try first ## section heading
+        const sectionMatch = cleaned.match(/^##\s+(.+)/m);
+        if (sectionMatch && sectionMatch[1].trim().length > 5) {
+            const s = sectionMatch[1].trim();
+            return s.slice(0, 100) + (s.length > 100 ? '...' : '');
+        }
+        
+        // 2. Try first paragraph > 40 chars after frontmatter
+        const lines = cleaned.split('\n').filter(l => l.trim().length > 0);
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (trimmed.length > 40 && !trimmed.startsWith('#') && !trimmed.startsWith('//')) {
+                return trimmed.slice(0, 100) + (trimmed.length > 100 ? '...' : '');
+            }
+        }
+        
+        // 3. First meaningful sentence (> 30 chars)
+        const sentence = cleaned.match(/[^.!?\n]+[.!?]/);
+        if (sentence && sentence[0].trim().length > 30) {
+            const s = sentence[0].trim();
+            return s.slice(0, 100) + (s.length > 100 ? '...' : '');
+        }
+        
+        // 4. Fallback: first line > 20 chars (original behavior)
+        const firstLine = lines.find(l => l.trim().length > 20);
+        if (firstLine) {
+            const tl = firstLine.trim();
+            return tl.slice(0, 100) + (tl.length > 100 ? '...' : '');
+        }
+        return '';
     }
 
     _indexSkills(fs, index) {
@@ -259,6 +426,8 @@ class NeuralVFSModule {
                 fs.mkdirRecursive('/.semantic_index');
             }
             fs.fsWriteJSON(this.indexPath, this.index);
+            // Also persist content hash cache for incremental diffing
+            fs.fsWriteJSON(this.hashCachePath, this._hashCache);
         } catch (e) {
             console.error('NeuralVFS save error:', e);
         }