path: root/modules/core/module-neural-vfs.js

class NeuralVFSModule {
    constructor() {
        this.commands = {};
        this.terminal = null;
        this.index = {};
        this.indexPath = '/.semantic_index/map.json';
        this.hashCachePath = '/.semantic_index/hash_cache.json';
        this._hashCache = {};
        // Configurable display limits
        this._fuzzyLimit = 10;
        this._treeFiles = 5;
        this._treeConcepts = 15;
        this._resolveFiles = 20;
        this._loadConfig();
    }
    _loadConfig() {
        try { const c = localStorage.getItem('webx11_nvfs_config'); if (c) { const p = JSON.parse(c); if (p.fuzzyLimit) this._fuzzyLimit = p.fuzzyLimit; if (p.treeFiles) this._treeFiles = p.treeFiles; if (p.treeConcepts) this._treeConcepts = p.treeConcepts; if (p.resolveFiles) this._resolveFiles = p.resolveFiles; } } catch (e) {}
    }
    _saveConfig() {
        localStorage.setItem('webx11_nvfs_config', JSON.stringify({ fuzzyLimit: this._fuzzyLimit, treeFiles: this._treeFiles, treeConcepts: this._treeConcepts, resolveFiles: this._resolveFiles }));
    }

    getHelpInfo() {
        return {
            'nvfs-index': 'nvfs-index — Scan VFS and build semantic concept index',
            'nvfs-status': 'nvfs-status — Show semantic index stats and last scan time',
            'nvfs-resolve': 'nvfs-resolve <concept> — Find files related to a concept',
            'nvfs-tree': 'nvfs-tree [concept] — Show concept-to-file mapping as a tree',
            'nvfs-config': 'nvfs-config [key] [value] — View or set Neural VFS display limits (fuzzy-limit, tree-files, tree-concepts, resolve-files)'
        };
    }

    registerCommands() {
        this.commands['nvfs-index'] = (args) => this.buildIndex();
        this.commands['nvfs-status'] = (args) => this.showStatus();
        this.commands['nvfs-resolve'] = (args) => this.resolve(args);
        this.commands['nvfs-tree'] = (args) => this.tree(args);
        this.commands['nvfs-config'] = (args) => this.nvfsConfig(args);
    }

    // ── Core Index Builder (async, chunked) ──

    async buildIndex() {
        const fs = this.terminal.fs;
        if (!fs) return this.terminal.print('VFS not available', 'error');

        this.terminal.print('🔍 Scanning VFS for semantic concepts...', 'info');

        const start = Date.now();
        let newIndex = {};
        let fileCount = 0;
        let conceptCount = 0;
        let cachedCount = 0;

        // Load previous index and hash cache for incremental diffing
        const prevIndex = fs.fsReadJSON(this.indexPath) || {};
        this._loadHashCache(fs);

        // Collect current file list
        const fileList = [];
        this._collectFiles('/', fs, fileList);

        if (fileList.length === 0) {
            return this.terminal.print('No files found to index', 'warning');
        }

        // Detect deleted files — strip orphaned entries from prevIndex
        const currentPaths = new Set(fileList.map(f => f.path));
        let deletedCount = 0;
        for (const [concept, entries] of Object.entries(prevIndex)) {
            const remaining = entries.filter(e => currentPaths.has(e.path));
            if (remaining.length !== entries.length) deletedCount++;
            if (remaining.length > 0) prevIndex[concept] = remaining;
            else delete prevIndex[concept];
        }

        // Warm start: clone prevIndex and count live paths for stats
        newIndex = JSON.parse(JSON.stringify(prevIndex));
        const livePaths = new Set();
        for (const entries of Object.values(newIndex)) {
            for (const e of entries) livePaths.add(e.path);
        }
        conceptCount = Object.keys(newIndex).length;

        // Chunked async processing — yields to UI between chunks
        const chunkSize = 10;
        let i = 0;
        const processChunk = () => {
            const end = Math.min(i + chunkSize, fileList.length);
            for (let j = i; j < end; j++) {
                const { path, node } = fileList[j];
                fileCount++;
                const content = node.content || '';
                const hash = this._contentHash(content);

                // Skip if hash unchanged — content is same as last scan
                const cached = this._getCached(path, hash);
                if (cached) {
                    cachedCount++;
                    if (!livePaths.has(path)) {
                        // File was in cache but somehow missing from prevIndex
                        this._reindexFile(path, content, newIndex);
                    }
                    continue;
                }

                // Changed or new file — extract concepts fresh
                this._reindexFile(path, content, newIndex);
                this._hashCache[path] = { hash };
            }
            i = end;
            if (i < fileList.length) {
                const pct = Math.round(i / fileList.length * 100);
                const cMsg = cachedCount > 0 ? ` (${cachedCount} cached)` : '';
                this.terminal.print(`⏳ Indexing... ${pct}% (${i}/${fileList.length} files${cMsg})`, 'info');
                setTimeout(processChunk, 0);
            } else {
                // Also index skills
                this._indexSkills(fs, newIndex);
                this.index = newIndex;
                this.lastScan = new Date().toISOString();
                this.fileCount = fileCount;
                this.conceptCount = Object.keys(newIndex).length;
                this._saveIndex(fs);
                if (fs._clearConceptCache) fs._clearConceptCache();
                const elapsed = ((Date.now() - start) / 1000).toFixed(1);
                const processedMsg = cachedCount > 0 ? ` (${cachedCount} cached, ${fileCount - cachedCount} reindexed)` : '';
                this.terminal.print(`✅ Indexed ${fileCount} files → ${Object.keys(newIndex).length} concepts in ${elapsed}s${processedMsg}`, 'success');
                if (deletedCount > 0) this.terminal.print(`   Purged ${deletedCount} orphaned concepts from deleted files`, 'info');
                this.terminal.print(`   Index saved to ${this.indexPath}`, 'info');
            }
        };
        processChunk();
    }

    _reindexFile(path, content, index) {
        const concepts = this._extractConcepts(path, content);
        // Remove old entries for this path
        for (const entries of Object.values(index)) {
            const idx = entries.findIndex(e => e.path === path);
            if (idx !== -1) entries.splice(idx, 1);
        }
        // Add new entries
        for (const concept of concepts) {
            if (!index[concept]) {
                index[concept] = [];
                this.conceptCount++;
            }
            index[concept].push({ path, summary: this._summarize(content) });
        }
    }

    _collectFiles(path, fs, list) {
        const entries = fs.readDir(path);
        if (!entries) return;
        for (const entry of entries) {
            const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name;
            if (entry.type === 'file') {
                if (this._isSystemFile(fullPath)) continue;
                list.push({ path: fullPath, node: { type: 'file', content: fs.readFile(fullPath) || '' } });
            } else if (entry.type === 'dir') {
                this._collectFiles(fullPath, fs, list);
            }
        }
    }

    // ── Resolve ──

    resolve(args) {
        if (!args[0]) return this.terminal.print('Usage: nvfs-resolve <concept>', 'warning');

        const concept = args.join(' ').toLowerCase();
        const matches = this.index[concept] || [];

        // Fuzzy fallback — find partial matches
        const fuzzy = Object.keys(this.index).filter(k =>
            k.includes(concept) || concept.includes(k)
        ).filter(k => k !== concept);

        if (matches.length === 0 && fuzzy.length === 0) {
            return this.terminal.print(`No files found for concept "${concept}"`, 'warning');
        }

        if (matches.length > 0) {
            this.terminal.print(`📂 "${concept}" → ${matches.length} file(s):`, 'info');
            for (const m of matches.slice(0, this._resolveFiles)) {
                this.terminal.print(`   ${m.path}`, 'success');
                if (m.summary) this.terminal.print(`     ${m.summary}`, '')
            }
            if (matches.length > this._resolveFiles) {
                this.terminal.print(`   ... and ${matches.length - this._resolveFiles} more`, 'info');
            }
        }

        if (fuzzy.length > 0) {
            this.terminal.print(`\n🔀 Similar concepts: ${fuzzy.slice(0, this._fuzzyLimit).join(', ')}`, 'info');
        }
    }

    // ── Status ──

    showStatus() {
        const count = Object.keys(this.index).length;
        if (count === 0) {
            // Try loading from VFS
            const fs = this.terminal.fs;
            const saved = fs.fsReadJSON(this.indexPath);
            if (saved) {
                this.index = saved;
                this.lastScan = '(loaded from disk)';
                this.terminal.print(`📖 Loaded index from ${this.indexPath}`, 'info');
            } else {
                return this.terminal.print('No semantic index found. Run nvfs-index first.', 'warning');
            }
        }

        const totalFiles = new Set();
        for (const entries of Object.values(this.index)) {
            for (const e of entries) totalFiles.add(e.path);
        }

        this.terminal.print(`🧠 Neural VFS Index`, 'info');
        this.terminal.print(`   Concepts: ${Object.keys(this.index).length}`, '');
        this.terminal.print(`   Files indexed: ${totalFiles.size}`, '');
        this.terminal.print(`   Last scan: ${this.lastScan || 'never'}`, '');
    }

    // ── Tree View ──

    tree(args) {
        const concepts = Object.keys(this.index);
        if (concepts.length === 0) {
            return this.terminal.print('No index. Run nvfs-index first.', 'warning');
        }

        const filter = args[0] ? args[0].toLowerCase() : null;
        const filtered = filter ? concepts.filter(c => c.includes(filter)) : concepts;

        this.terminal.print(`🌳 Concept Tree${filter ? ` (filter: "${filter}")` : ''}:`, 'info');
        for (const concept of filtered.slice(0, this._treeConcepts)) {
            this.terminal.print(`  📁 ${concept}`, 'success');
            for (const entry of this.index[concept].slice(0, this._treeFiles)) {
                this.terminal.print(`     └─ ${entry.path}`, '');
            }
            if (this.index[concept].length > this._treeFiles) {
                this.terminal.print(`     └─ ... (+${this.index[concept].length - this._treeFiles} more)`, '');
            }
        }
        if (filtered.length > this._treeConcepts) {
            this.terminal.print(`  ... and ${filtered.length - this._treeConcepts} more concepts`, 'info');
        }
    }

    // ── Nvfs Config ──

    nvfsConfig(args) {
        if (args.length === 0) {
            this.terminal.print('=== Neural VFS Config ===', 'info');
            this.terminal.print(`fuzzy-limit: ${this._fuzzyLimit} (default: 10)`, '');
            this.terminal.print(`tree-files: ${this._treeFiles} (default: 5)`, '');
            this.terminal.print(`tree-concepts: ${this._treeConcepts} (default: 15)`, '');
            this.terminal.print(`resolve-files: ${this._resolveFiles} (default: 20)`, '');
            this.terminal.print('Usage: nvfs-config <key> <value>', 'info');
            this.terminal.print('Keys: fuzzy-limit, tree-files, tree-concepts, resolve-files', 'info');
            return;
        }
        const key = args[0];
        const val = parseInt(args[1]);
        if (isNaN(val) || val < 1 || val > 200) return this.terminal.print('Value must be 1-200', 'warning');
        if (key === 'fuzzy-limit') { this._fuzzyLimit = val; }
        else if (key === 'tree-files') { this._treeFiles = val; }
        else if (key === 'tree-concepts') { this._treeConcepts = val; }
        else if (key === 'resolve-files') { this._resolveFiles = val; }
        else { return this.terminal.print(`Unknown key: ${key}. Valid: fuzzy-limit, tree-files, tree-concepts, resolve-files`, 'warning'); }
        this._saveConfig();
        this.terminal.print(`nvfs-config ${key} set to ${val}`, 'success');
    }

    // ── Private Helpers ──

    _walk(path, fs, index, cb) {
        const entries = fs.readDir(path);
        if (!entries) return;
        for (const entry of entries) {
            const fullPath = path === '/' ? '/' + entry.name : path + '/' + entry.name;
            if (entry.type === 'file') {
                cb(fullPath, { type: 'file', content: fs.readFile(fullPath) || '' });
            } else if (entry.type === 'dir') {
                this._walk(fullPath, fs, index, cb);
            }
        }
    }

    _isSystemFile(path) {
        const sysPrefixes = ['/.trash', '/.semantic_index', '/_snapshots', '/_sessions'];
        return sysPrefixes.some(p => path.startsWith(p) || path.includes('/node_modules'));
    }

    _contentHash(str) {
        if (!str) return '';
        let hash = 5381;
        for (let i = 0; i < str.length; i++) {
            hash = ((hash << 5) + hash) + str.charCodeAt(i);
            hash = hash & hash;
        }
        return Math.abs(hash).toString(36).slice(0, 12);
    }

    _loadHashCache(fs) {
        const saved = fs.fsReadJSON(this.hashCachePath);
        if (saved && typeof saved === 'object') this._hashCache = saved;
    }

    _getCached(path, hash) {
        const c = this._hashCache[path];
        if (c && c.hash === hash) return c;
        return null;
    }

    _extractConcepts(path, content) {
        const concepts = new Set();
        const name = path.split('/').pop() || '';
        const nameLower = name.toLowerCase();

        // Extract from filename (split on camelCase, kebab, dots, underscores)
        const words = nameLower
            .replace(/([a-z])([A-Z])/g, '$1 $2')
            .replace(/[-_.]/g, ' ')
            .replace(/\.(md|js|html|json|css|txt|yaml|yml|toml|env|sh)$/i, '')
            .split(/\s+/)
            .filter(w => w.length > 2 && !/^\d+$/.test(w));

        for (const word of words) {
            concepts.add(word);
        }

        // Extract file extension as concept
        const ext = path.split('.').pop();
        if (ext && ['md', 'js', 'html', 'json', 'css'].includes(ext)) {
            concepts.add(ext + '-file');
        }

        // Extract path-level concepts
        const parts = path.split('/').filter(p => p.length > 0);
        for (const part of parts) {
            if (['modules', 'skills', 'core', 'ext', 'backups', 'chat', 'downloads'].includes(part)) {
                concepts.add(part);
            }
        }

        // Extract #tags from content (markdown frontmatter or inline)
        if (content) {
            const tagMatches = content.match(/tags:\s*\[([^\]]+)\]/i) || content.match(/#[a-zA-Z]\w+/g);
            if (tagMatches) {
                const tags = tagMatches[0].replace(/tags:\s*\[/i, '').replace(/\]/, '').split(',').map(t => t.trim().replace(/^#/, '').replace(/"/g, ''));
                for (const tag of tags) {
                    if (tag.length > 2) concepts.add(tag.toLowerCase());
                }
            }

            // Extract first H1/H2 as concept
            const headerMatch = content.match(/^#\s+(.+)/m) || content.match(/^##\s+(.+)/m);
            if (headerMatch) {
                const header = headerMatch[1].toLowerCase().replace(/[^a-z0-9\s]/g, '').trim();
                if (header.length > 3 && header.length < 40) {
                    concepts.add(header);
                }
            }
        }

        return [...concepts];
    }

    _summarize(content) {
        if (!content || content.length < 10) return '';
        // Strip YAML frontmatter
        const cleaned = content.replace(/---[\s\S]*?---/, '').trim();
        
        // 1. Try first ## section heading
        const sectionMatch = cleaned.match(/^##\s+(.+)/m);
        if (sectionMatch && sectionMatch[1].trim().length > 5) {
            const s = sectionMatch[1].trim();
            return s.slice(0, 100) + (s.length > 100 ? '...' : '');
        }
        
        // 2. Try first paragraph > 40 chars after frontmatter
        const lines = cleaned.split('\n').filter(l => l.trim().length > 0);
        for (const line of lines) {
            const trimmed = line.trim();
            if (trimmed.length > 40 && !trimmed.startsWith('#') && !trimmed.startsWith('//')) {
                return trimmed.slice(0, 100) + (trimmed.length > 100 ? '...' : '');
            }
        }
        
        // 3. First meaningful sentence (> 30 chars)
        const sentence = cleaned.match(/[^.!?\n]+[.!?]/);
        if (sentence && sentence[0].trim().length > 30) {
            const s = sentence[0].trim();
            return s.slice(0, 100) + (s.length > 100 ? '...' : '');
        }
        
        // 4. Fallback: first line > 20 chars (original behavior)
        const firstLine = lines.find(l => l.trim().length > 20);
        if (firstLine) {
            const tl = firstLine.trim();
            return tl.slice(0, 100) + (tl.length > 100 ? '...' : '');
        }
        return '';
    }

    _indexSkills(fs, index) {
        const skills = fs.readDir('/skills');
        if (!skills) return;
        for (const skill of skills) {
            if (skill.type !== 'file' || !skill.name.endsWith('.md')) continue;
            // Already indexed by _walk — but we add a "skill" tag
            const base = skill.name.replace('.md', '');
            if (!index['skill']) index['skill'] = [];
            index['skill'].push({ path: '/skills/' + skill.name, summary: 'Skill: ' + base });
        }
    }

    _saveIndex(fs) {
        try {
            if (!fs.readDir('/.semantic_index')) {
                fs.mkdirRecursive('/.semantic_index');
            }
            fs.fsWriteJSON(this.indexPath, this.index);
            // Also persist content hash cache for incremental diffing
            fs.fsWriteJSON(this.hashCachePath, this._hashCache);
        } catch (e) {
            console.error('NeuralVFS save error:', e);
        }
    }
}

// Required for module system
if (typeof window !== 'undefined') {
    window.NeuralVFSModule = NeuralVFSModule;
}