diff options
| -rw-r--r-- | main.html | 6 | ||||
| -rw-r--r-- | modules/core/core-bundle.js | 336 | ||||
| -rw-r--r-- | modules/core/module-fs.js | 100 | ||||
| -rw-r--r-- | modules/core/module-loader.js | 2 | ||||
| -rw-r--r-- | modules/core/module-servermon.js | 4 | ||||
| -rw-r--r-- | modules/core/module-system.js | 202 | ||||
| -rw-r--r-- | modules/core/terminal.js | 9 | ||||
| -rw-r--r-- | modules/core/virtual-fs.js | 23 | ||||
| -rw-r--r-- | modules/test/test-doctor-thresholds.js | 341 | ||||
| -rw-r--r-- | modules/test/test-vfs-size-bytes.js | 216 | ||||
| -rw-r--r-- | modules/test/test-virtual-fs-require-leaf.js | 160 |
11 files changed, 1290 insertions, 109 deletions
@@ -1350,13 +1350,13 @@ </script> <!-- v4.5.63: A2A scratchpad fixes + agent-register arg fix --> -<script src="modules/core/core-bundle.js?v=v4.6.159"></script> +<script src="modules/core/core-bundle.js?v=v4.6.160"></script> <!-- Async dependency-ordered module loader now only initializes terminal and loads optional ext modules --> -<script src="modules/core/module-loader.js?v=v4.6.159"></script> +<script src="modules/core/module-loader.js?v=v4.6.160"></script> <!-- Critical ext bundle loads AFTER webx11 exists so registerModule calls work --> -<script src="modules/ext/critical-ext-bundle.js?v=v4.6.159"></script> +<script src="modules/ext/critical-ext-bundle.js?v=v4.6.160"></script> <script> // ═══ ONE TRUE VERSION SOURCE ═══ diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js index b365fa0..ddcac45 100644 --- a/modules/core/core-bundle.js +++ b/modules/core/core-bundle.js @@ -557,6 +557,19 @@ class VirtualFileSystem { } return { dir, name: segs[segs.length - 1] }; } + // newOS4 patch: _resolvePath() validates intermediate segments only, so it returns a + // result for a leaf that does not exist. Mutating callers then acted on a phantom + // path and reported success (deleteCheckpoint printed 'Deleted checkpoint' and + // deleted nothing). _requireLeaf() refuses any resolved path whose leaf is absent. + _requireLeaf(resolved) { + if (!resolved) return null; + if (resolved.name) { + const node = (resolved.dir && resolved.dir.content) ? resolved.dir.content[resolved.name] : null; + if (!node) return null; + } + return resolved; + } + _resolveDir(path) { if (this._isVirtualPath(path)) { const r = this._resolveVirtualPath(path); @@ -626,7 +639,7 @@ class VirtualFileSystem { } deleteFile(path) { if (this._isVirtualPath(path)) return false; - const resolved = this._resolvePath(path); + const resolved = this._requireLeaf(this._resolvePath(path)); if (!resolved || !resolved.name) return false; const node = resolved.dir.content[resolved.name]; if (node && node.type === 'file') { @@ -1013,6 +1026,10 @@ class VirtualFileSystem { delete rootClone.content['_sessions']; delete rootClone.content['_snapshots']; delete rootClone.content['.trash']; + // newOS4 patch: module originals and the semantic index dominated every + // snapshot (one checkpoint reached 4.13MB) and grew on each new checkpoint. + delete rootClone.content['_mods']; + delete rootClone.content['.semantic_index']; } const data = { version: 1, @@ -1047,7 +1064,7 @@ class VirtualFileSystem { // clone verbatim deletes every checkpoint, every agent session and the trash, so keep the // live copies and re-attach them after the root swap. const keep = {}; - for (const dir of ['_snapshots', '_sessions', '.trash']) { + for (const dir of ['_snapshots', '_sessions', '.trash', '_mods', '.semantic_index']) { const node = this.root && this.root.content ? this.root.content[dir] : null; if (node) keep[dir] = node; } @@ -1067,7 +1084,7 @@ class VirtualFileSystem { } deleteCheckpoint(name) { const path = '/_snapshots/checkpoint-' + name + '.json'; - const resolved = this._resolvePath(path); + const resolved = this._requireLeaf(this._resolvePath(path)); if (resolved && resolved.name) { delete resolved.dir.content[resolved.name]; this.saveToStorage(); @@ -3253,9 +3270,14 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm } } updateNetworkStatus() { + // navigator.onLine is a hint, not a fact: an Android WebView reports false while + // fully connected (the same lie `status` used to print). Never turn that hint into + // "❌ Offline" — claim Online only when the browser says so, otherwise say unknown + // and point at `doctor`, which probes for real. const isOnline = navigator.onLine; - this.networkStatus.textContent = isOnline ? '📶 Online' : '❌ Offline'; - this.networkStatus.style.color = isOnline ? 'var(--text-color)' : 'var(--error-color)'; + this.networkStatus.textContent = isOnline ? '📶 Online' : '📶 Network?'; + this.networkStatus.title = isOnline ? '' : 'Browser reports offline — run doctor for a live probe'; + this.networkStatus.style.color = isOnline ? 'var(--text-color)' : 'var(--text-muted)'; } updatePrompt() { const path = '/' + this.fs.currentPath.join('/'); @@ -5886,6 +5908,84 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm 'checkpoint-del': 'checkpoint-del <name> — Delete a VFS checkpoint', }; } + + // ── doctor measurement helpers ────────────────────────────────────── + // UTF-8 byte length of a string. Blob is the reference implementation + // (Node 17+/every target browser), TextEncoder the browser fallback, and a + // code-point walk the last resort so a stripped-down WebView still gets a + // number instead of a thrown command. + _utf8Bytes(str) { + if (typeof str !== 'string' || str.length === 0) return 0; + try { + if (typeof Blob === 'function') return new Blob([str]).size; + } catch (e) {} + try { + if (typeof TextEncoder === 'function') return new TextEncoder().encode(str).length; + } catch (e) {} + let n = 0; + for (let i = 0; i < str.length; i++) { + const cp = str.codePointAt(i); + if (cp > 0xffff) i++; // surrogate pair consumed + n += cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4; + } + return n; + } + + // Sum of UTF-8 bytes of every file's content under `root`. This is the unit + // `vfs-size` reports, without JSON.stringify's braces/quoting overhead and + // without the old `chars * 2` fudge. Depth- and node-capped so a malformed + // or cyclic tree cannot hang the command. + _vfsContentBytes(root) { + let bytes = 0, nodes = 0; + const walk = (node, depth) => { + if (!node || typeof node !== 'object' || depth > 64) return; + if (node.type === 'file') { + if (typeof node.content === 'string') bytes += this._utf8Bytes(node.content); + return; + } + const kids = node.content; + if (!kids || typeof kids !== 'object') return; + for (const name of Object.keys(kids)) { + if (++nodes > 20000) return; + walk(kids[name], depth + 1); + } + }; + walk(root, 0); + return bytes; + } + + // Live reachability probe. The legacy `onLine` boolean is not consulted anywhere: + // in Android WebView it reports false while the page is fully connected, which is + // exactly how `status` used to tell working users they were offline. + async _probeNetwork() { + try { + if (typeof fetch !== 'function') return { state: 'unprobed', detail: 'fetch unavailable' }; + } catch (e) { + return { state: 'unprobed', detail: 'fetch unavailable' }; + } + let ctl = null, timer = null; + try { if (typeof AbortController === 'function') ctl = new AbortController(); } catch (e) {} + try { + if (ctl) timer = setTimeout(() => { try { ctl.abort(); } catch (e) {} }, 4000); + const resp = await fetch('./', { + method: 'HEAD', + cache: 'no-store', + signal: ctl ? ctl.signal : undefined, + }); + if (!resp) return { state: 'unreachable', detail: 'empty response' }; + if (resp.ok) return { state: 'reachable', detail: 'HTTP ' + resp.status }; + return { state: 'http-' + resp.status, detail: 'HTTP ' + resp.status }; + } catch (e) { + const aborted = e && (e.name === 'AbortError' || e.name === 'TimeoutError'); + return { + state: 'unreachable', + detail: aborted ? 'timed out after 4s' : ((e && e.message) || 'request failed'), + }; + } finally { + if (timer) clearTimeout(timer); + } + } + registerCommands() { this.commands['help'] = (args) => this.handleHelp(args); this.commands['echo'] = (args) => this.terminal.print(args.join(' ')); @@ -5934,7 +6034,10 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm }, 500); this.commands['status'] = () => { this.terminal.print(`System: WebX11 ${window.NEWOS4_VERSION || 'v4.0'}`); - this.terminal.print(`Network: ${navigator.onLine ? 'Online' : 'Offline'}`); + // The legacy `onLine` boolean is NOT evidence of reachability: Android + // WebView reports false on a fully connected page, and true on a captive + // portal. Say so, and point at the command that actually probes. + this.terminal.print('Network: unknown — run doctor for a live probe'); this.terminal.print(`Files: ${this.terminal.fs.ls().length} items`); this.terminal.print(`Modules: ${Object.keys(this.terminal.moduleManager.modules).length} loaded`); this.terminal.print(`Ollama: ${this.terminal.ollamaManager.connected ? 'Connected' : 'Disconnected'}`); @@ -5961,7 +6064,7 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm this.terminal.print(` Build: ${buildDate}`); this.terminal.print(` Cache key: ${cacheKey}`); this.terminal.print(` Browser: ${ua}`); - this.terminal.print(` Online: ${navigator.onLine ? 'yes' : 'no'}`); + this.terminal.print(` Network: unknown — run doctor for a live probe`); }; // v4.5.43: uptime — time since page load + last save time @@ -6017,40 +6120,102 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm }; // v4.5.43: doctor — run all health checks, rank issues, show top 3 - this.commands['doctor'] = () => { + // v4.6.x: the legacy `onLine` boolean is GONE from this file. It reports false + // on a fully connected Android WebView, which is how `status`/`doctor` used to + // announce an outage on a working page. Reachability now comes from a real HEAD + // probe, and every size is measured in the unit the store actually charges for. + this.commands['doctor'] = async () => { const issues = []; const fs = this.terminal.fs; + const MB = 1024 * 1024; - // Check 1: Network - if (!navigator.onLine) issues.push({ severity: 'high', msg: 'Network: offline' }); + // Check 1: Network — live probe, 4s budget + const net = await this._probeNetwork(); + if (net.state === 'unreachable') { + issues.push({ severity: 'high', msg: `Network: unreachable (HEAD ./ — ${net.detail})` }); + } else if (net.state.indexOf('http-') === 0) { + // A non-ok HEAD is not proof the page is broken (some servers answer + // 405 to HEAD); report the status and let the user judge. + issues.push({ severity: 'medium', msg: `Network: HTTP probe to ./ returned ${net.state} (page may still work)` }); + } // Check 2: Ollama if (!this.terminal.ollamaManager.connected) { issues.push({ severity: 'medium', msg: 'Ollama: not connected (AI features limited)' }); } - // Check 3: VFS size + // Origin quota — context only. It is the whole-origin quota (Cache Storage + // included, typically hundreds of MB to GBs). localStorage has its own + // ~5 MB per-origin cap, so this number must NEVER set a severity: a 4.5 MB + // localStorage is 96% of its cap and 0.2% of a 2 GB origin quota. + let originUsage = 0, originQuota = 0; try { - const vfsBytes = JSON.stringify(fs.root).length; - if (vfsBytes > 524288) issues.push({ severity: 'high', msg: `VFS: ${(vfsBytes/1024).toFixed(0)}KB > 512KB limit` }); - else if (vfsBytes > 262144) issues.push({ severity: 'medium', msg: `VFS: ${(vfsBytes/1024).toFixed(0)}KB > 256KB (consider reset-vfs)` }); + if (typeof navigator !== 'undefined' && navigator.storage && + typeof navigator.storage.estimate === 'function') { + const est = await navigator.storage.estimate(); + if (est && typeof est.quota === 'number' && est.quota > 0) { + originQuota = est.quota; + originUsage = typeof est.usage === 'number' ? est.usage : 0; + } + } + } catch (e) {} + + // Check 3: VFS — UTF-8 bytes of file content (the unit vfs-size reports), + // never JSON.stringify(root).length and never "chars × 2". + let vfsBytes = null; + try { vfsBytes = this._vfsContentBytes(fs.root); } catch (e) { vfsBytes = null; } + + // Check 4: localStorage — JS strings are UTF-16, so each key and value is + // charged 2 bytes per code unit. Σ (key.length + value.length) × 2 is the + // real storage cost, not a fudge factor. + let lsBytes = 0, lsKeys = 0; + try { + for (let i = 0; i < localStorage.length; i++) { + const k = localStorage.key(i); + if (!k) continue; + lsKeys++; + lsBytes += (k.length + (localStorage.getItem(k) || '').length) * 2; + } } catch (e) {} - // Check 4: Heap + // VFS thresholds: a VFS only threatens the browser budget when it is BOTH + // large on its own and the bulk of what is actually stored. + if (vfsBytes !== null) { + const vfsMB = vfsBytes / MB; + const share = lsBytes > 0 ? vfsBytes / lsBytes : 0; + const budget = lsBytes > 0 + ? `, ${(share * 100).toFixed(0)}% of the ${(lsBytes / MB).toFixed(1)} MB localStorage budget` + : ', no localStorage copy measured'; + if (vfsBytes > 3 * MB && (lsBytes === 0 || share > 0.8)) { + issues.push({ severity: 'high', msg: `VFS: ${vfsMB.toFixed(1)} MB of file content${budget} — close to the ~5 MB per-origin cap` }); + } else if (vfsBytes > 2 * MB) { + issues.push({ severity: 'medium', msg: `VFS: ${vfsMB.toFixed(1)} MB of file content${budget} (consider deleting stale checkpoints)` }); + } + } + + // localStorage thresholds: the cap is ~5 MB per origin, whatever + // navigator.storage.estimate() says the whole origin may use. + if (lsBytes > 4 * MB) { + issues.push({ severity: 'high', msg: `localStorage: ${(lsBytes / MB).toFixed(1)} MB of UTF-16 data in ${lsKeys} keys — approaching the ~5 MB per-origin cap` }); + } else if (lsBytes > 3 * MB) { + issues.push({ severity: 'medium', msg: `localStorage: ${(lsBytes / MB).toFixed(1)} MB of UTF-16 data in ${lsKeys} keys — past half of the ~5 MB per-origin cap` }); + } + + // Check 5: Heap if (performance.memory) { const heapMB = performance.memory.usedJSHeapSize / 1048576; if (heapMB > 200) issues.push({ severity: 'high', msg: `Heap: ${heapMB.toFixed(0)}MB > 200MB` }); else if (heapMB > 100) issues.push({ severity: 'medium', msg: `Heap: ${heapMB.toFixed(0)}MB > 100MB` }); } - // Check 5: Logs + // Check 6: Logs try { const logContent = fs.readFile('/logs/current.log') || ''; const logLines = logContent.split('\n').filter(l => l).length; if (logLines > 500) issues.push({ severity: 'low', msg: `Logs: ${logLines} lines (consider logrotate)` }); } catch (e) {} - // Check 6: Errors in last 100 log lines + // Check 7: Errors in last 100 log lines try { const logContent = fs.readFile('/logs/current.log') || ''; const recent = logContent.split('\n').filter(l => l).slice(-100); @@ -6059,16 +6224,6 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm else if (errorCount > 0) issues.push({ severity: 'medium', msg: `Errors: ${errorCount} errors in last 100 log lines` }); } catch (e) {} - // Check 7: localStorage quota - let lsBytes = 0; - try { - for (let i = 0; i < localStorage.length; i++) { - const k = localStorage.key(i); - if (k) lsBytes += (k.length + (localStorage.getItem(k) || '').length) * 2; - } - } catch (e) {} - if (lsBytes > 4 * 1024 * 1024) issues.push({ severity: 'high', msg: `localStorage: ${(lsBytes/1024/1024).toFixed(1)}MB approaching 5MB quota` }); - // Check 8: Disabled modules let disabled = []; try { disabled = JSON.parse(localStorage.getItem('webx11_module_disabled') || '[]'); } catch (e) {} @@ -6077,6 +6232,11 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm } this.terminal.print('🩺 Doctor Report', 'header'); + this.terminal.print(` Measured: VFS ${vfsBytes === null ? 'unavailable' : (vfsBytes / MB).toFixed(2) + ' MB UTF-8'} · localStorage ${(lsBytes / MB).toFixed(2)} MB UTF-16 in ${lsKeys} keys`, 'info'); + this.terminal.print(` Network: ${net.state === 'reachable' ? 'reachable (' + net.detail + ')' : net.state + (net.detail ? ' — ' + net.detail : '')}`, 'info'); + if (originQuota > 0) { + this.terminal.print(` Origin usage ${(originUsage / MB).toFixed(1)} MB of ~${(originQuota / MB).toFixed(0)} MB quota (whole origin, mostly Cache Storage — localStorage has its own ~5 MB cap)`, 'info'); + } if (issues.length === 0) { this.terminal.print(' ✅ All systems healthy', 'success'); return; @@ -6085,20 +6245,21 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm const sevOrder = { high: 0, medium: 1, low: 2 }; issues.sort((a, b) => sevOrder[a.severity] - sevOrder[b.severity]); this.terminal.print(` Found ${issues.length} issue(s):`, 'warning'); - for (const i of issues.slice(0, 5)) { + for (const i of issues.slice(0, 3)) { const icon = i.severity === 'high' ? '🔴' : i.severity === 'medium' ? '🟡' : '🟢'; this.terminal.print(` ${icon} [${i.severity.toUpperCase()}] ${i.msg}`, i.severity === 'high' ? 'error' : 'warning'); } - if (issues.length > 5) this.terminal.print(` ... and ${issues.length - 5} more`, 'info'); + if (issues.length > 3) this.terminal.print(` ... and ${issues.length - 3} more`, 'info'); this.terminal.print('', ''); this.terminal.print(' Recommended actions:', 'info'); - if (issues.some(i => i.msg.startsWith('Network'))) this.terminal.print(' → Check internet connection'); + if (issues.some(i => i.msg.startsWith('Network: unreachable'))) this.terminal.print(' → Check connectivity; reload before trusting this probe'); + if (issues.some(i => i.msg.startsWith('Network: HTTP probe'))) this.terminal.print(' → Server answered the probe with an error status — check the server, not the network'); if (issues.some(i => i.msg.startsWith('Ollama'))) this.terminal.print(' → Check Ollama is running (ollama serve)'); - if (issues.some(i => i.msg.startsWith('VFS'))) this.terminal.print(' → Run reset-vfs --confirm to clear or checkpoint/rollback'); + if (issues.some(i => i.msg.startsWith('VFS'))) this.terminal.print(' → Delete stale snapshots (checkpoint-list, checkpoint-del <name>) or rollback; do NOT reset-vfs'); if (issues.some(i => i.msg.startsWith('Heap'))) this.terminal.print(' → Refresh the page to clear memory'); if (issues.some(i => i.msg.startsWith('Logs'))) this.terminal.print(' → Run logrotate'); if (issues.some(i => i.msg.startsWith('Errors'))) this.terminal.print(' → Run errors to see details, or bug-report to save state'); - if (issues.some(i => i.msg.startsWith('localStorage'))) this.terminal.print(' → Run factory-reset --confirm (last resort)'); + if (issues.some(i => i.msg.startsWith('localStorage'))) this.terminal.print(' → Free space by deleting stale VFS entries or checkpoints; do NOT factory-reset'); }; // v4.5.43: bug-report — generate markdown report to /tmp/ @@ -6112,7 +6273,7 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm lines.push('## System'); lines.push(`- Version: ${window.NEWOS4_VERSION || 'unknown'}`); lines.push(`- Cache key: ${localStorage.getItem('webx11_cache_ver') || 'unknown'}`); - lines.push(`- Online: ${navigator.onLine ? 'yes' : 'no'}`); + lines.push(`- Online: unknown (not probed; run doctor)`); lines.push(`- Browser: ${navigator.userAgent}`); lines.push(`- Screen: ${screen.width}x${screen.height}`); lines.push(`- Heap: ${performance.memory ? (performance.memory.usedJSHeapSize / 1048576).toFixed(1) + 'MB' : 'N/A'}`); @@ -6176,7 +6337,10 @@ Only output XML function calls for tool tasks. If no tool is needed, answer norm version: window.NEWOS4_VERSION || 'unknown', url: location.href, ua: navigator.userAgent, - online: navigator.onLine, + // Legacy key, kept so stored reports keep their shape. The old onLine + // boolean lied in Android WebView, so null = "not asserted". + online: null, + network: 'unknown', modules: Object.keys(this.terminal.moduleManager.modules).length, errors: [], }; @@ -7416,17 +7580,52 @@ class FileSystemModule { }; this.commands['vfs-size'] = () => { try { - const json = JSON.stringify(this.terminal.fs.root); - const totalSize = (json.length / 1024 / 1024).toFixed(2); + // Every number this command prints is STORED BYTES: the UTF-8 length of each + // file's content. It used to report JSON.stringify(root).length, which is UTF-16 + // string units plus every structural character of the serialisation — roughly 2x + // the real size of a plain-text tree, and wrong the other way for non-ASCII. + const utf8Len = (s) => { + if (typeof s !== 'string' || s.length === 0) return 0; + if (typeof Blob !== 'undefined') { + try { return new Blob([s]).size; } catch (e) { /* fall through */ } + } + if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(s).length; + return s.length; // last resort only: no Blob and no TextEncoder + }; + const fmtSize = (bytes) => { + if (bytes < 1024) return bytes + ' B'; + if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB'; + return (bytes / 1024 / 1024).toFixed(2) + ' MB'; + }; + // Recursively total one subtree. Each file is visited exactly once, so a nested + // tree is never double counted and no directory node adds bytes of its own. + const measure = (node) => { + const out = { bytes: 0, files: 0 }; + if (!node || !node.content) return out; + for (const child of Object.values(node.content)) { + if (!child) continue; + if (child.type === 'file') { + out.bytes += utf8Len(child.content); + out.files += 1; + } else if (child.type === 'dir') { + const sub = measure(child); + out.bytes += sub.bytes; + out.files += sub.files; + } + } + return out; + }; const root = this.terminal.fs.root; + const total = measure(root); + // Per-file table, same byte metric. const sizeMap = []; function walk(node, path) { if (!node || !node.content) return; for (const [name, child] of Object.entries(node.content)) { + if (!child) continue; const fullPath = path + '/' + name; if (child.type === 'file') { - const size = (child.content ? child.content.length : 0); - sizeMap.push({ path: fullPath, size, sizeKB: (size / 1024).toFixed(1) }); + sizeMap.push({ path: fullPath, size: utf8Len(child.content) }); } else if (child.type === 'dir') { walk(child, fullPath); } @@ -7434,23 +7633,54 @@ class FileSystemModule { } walk(root, ''); sizeMap.sort((a, b) => b.size - a.size); + // Top-level groups, so the table accounts for the whole tree: one row per + // top-level directory plus a row for files stored directly at root. + const groups = []; + const rootFiles = { bytes: 0, files: 0 }; + for (const [name, child] of Object.entries((root && root.content) || {})) { + if (!child) continue; + if (child.type === 'dir') { + const m = measure(child); + groups.push({ name: '/' + name, bytes: m.bytes, files: m.files }); + } else if (child.type === 'file') { + rootFiles.bytes += utf8Len(child.content); + rootFiles.files += 1; + } + } + if (rootFiles.files > 0) groups.push({ name: '/(root files)', bytes: rootFiles.bytes, files: rootFiles.files }); + groups.sort((a, b) => b.bytes - a.bytes); const totalFiles = sizeMap.length; - const top5 = sizeMap.slice(0, 5); this.terminal.print('📊 VFS Size Report', 'text-muted'); - this.terminal.print(' Total: ' + totalSize + ' MB across ' + totalFiles + ' files'); - this.terminal.print(' Largest:'); - top5.forEach((f, i) => { - this.terminal.print(' ' + (i + 1) + '. ' + f.path + ' — ' + f.sizeKB + ' KB'); - }); - // Show system dir sizes - const sysDirs = ['/_snapshots', '/_sessions', '/.trash']; + this.terminal.print(' Total: ' + fmtSize(total.bytes) + ' (' + total.bytes + ' bytes) across ' + totalFiles + (totalFiles === 1 ? ' file' : ' files')); + if (totalFiles > 0) { + this.terminal.print(' Largest:'); + const shownFiles = sizeMap.slice(0, 10); + shownFiles.forEach((f, i) => { + this.terminal.print(' ' + (i + 1) + '. ' + f.path + ' — ' + fmtSize(f.size)); + }); + // The rows must add up to the total above, so the tail is folded into one + // row instead of dropped: this table used to be top-5 only, and the rest of + // the tree was missing from the report entirely. + if (totalFiles > shownFiles.length) { + let restBytes = 0; + for (const f of sizeMap.slice(shownFiles.length)) restBytes += f.size; + this.terminal.print(' (+' + (totalFiles - shownFiles.length) + ' more files — ' + fmtSize(restBytes) + ')'); + } + } + // Every top-level group is listed (root breadth, not depth), so these rows + // always sum to the total above. + this.terminal.print(' Top directories by stored bytes:', 'info'); + for (const g of groups) { + this.terminal.print(' ' + g.name + ' — ' + fmtSize(g.bytes) + ' (' + g.files + (g.files === 1 ? ' file)' : ' files)')); + } + // Show system dir sizes — real bytes, not JSON.stringify(list of entries).length + const sysDirs = ['/_snapshots', '/_sessions', '/.trash', '/_mods', '/.semantic_index']; for (const dir of sysDirs) { - const dirNode = this.terminal.fs.readDir(dir); + const name = dir.replace(/^\//, ''); + const dirNode = (root && root.content && root.content[name]) || null; if (dirNode) { - const cnt = dirNode.length; - // estimate dir size - const dirSize = JSON.stringify(dirNode).length; - this.terminal.print(' ' + dir + ': ' + cnt + ' files, ~' + (dirSize / 1024).toFixed(1) + ' KB', dirSize > 102400 ? 'warning' : 'text-muted'); + const m = measure(dirNode); + this.terminal.print(' ' + dir + ': ' + m.files + (m.files === 1 ? ' file, ' : ' files, ') + fmtSize(m.bytes), m.bytes > 102400 ? 'warning' : 'text-muted'); } } } catch (e) { @@ -8553,7 +8783,9 @@ class ServerMonitorModule { this.terminal.print('🌐 Network Test', 'header'); // 1. Browser online const online = navigator.onLine; - this.terminal.print(` ${online ? '✅' : '🔴'} Browser: ${online ? 'online' : 'offline'}`, online ? 'success' : 'error'); + // Reported, not measured: WebViews answer false while connected. The real verdict comes + // from the proxy probe below, so this line must not read as a failure. + this.terminal.print(` ${online ? '✅' : '❔'} Browser reports: ${online ? 'online' : 'offline (unreliable)'}`, online ? 'success' : 'warning'); // 2. Proxy API try { const t0 = Date.now(); diff --git a/modules/core/module-fs.js b/modules/core/module-fs.js index 0dea0d5..b6cf272 100644 --- a/modules/core/module-fs.js +++ b/modules/core/module-fs.js @@ -730,17 +730,52 @@ class FileSystemModule { }; this.commands['vfs-size'] = () => { try { - const json = JSON.stringify(this.terminal.fs.root); - const totalSize = (json.length / 1024 / 1024).toFixed(2); + // Every number this command prints is STORED BYTES: the UTF-8 length of each + // file's content. It used to report JSON.stringify(root).length, which is UTF-16 + // string units plus every structural character of the serialisation — roughly 2x + // the real size of a plain-text tree, and wrong the other way for non-ASCII. + const utf8Len = (s) => { + if (typeof s !== 'string' || s.length === 0) return 0; + if (typeof Blob !== 'undefined') { + try { return new Blob([s]).size; } catch (e) { /* fall through */ } + } + if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(s).length; + return s.length; // last resort only: no Blob and no TextEncoder + }; + const fmtSize = (bytes) => { + if (bytes < 1024) return bytes + ' B'; + if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB'; + return (bytes / 1024 / 1024).toFixed(2) + ' MB'; + }; + // Recursively total one subtree. Each file is visited exactly once, so a nested + // tree is never double counted and no directory node adds bytes of its own. + const measure = (node) => { + const out = { bytes: 0, files: 0 }; + if (!node || !node.content) return out; + for (const child of Object.values(node.content)) { + if (!child) continue; + if (child.type === 'file') { + out.bytes += utf8Len(child.content); + out.files += 1; + } else if (child.type === 'dir') { + const sub = measure(child); + out.bytes += sub.bytes; + out.files += sub.files; + } + } + return out; + }; const root = this.terminal.fs.root; + const total = measure(root); + // Per-file table, same byte metric. const sizeMap = []; function walk(node, path) { if (!node || !node.content) return; for (const [name, child] of Object.entries(node.content)) { + if (!child) continue; const fullPath = path + '/' + name; if (child.type === 'file') { - const size = (child.content ? child.content.length : 0); - sizeMap.push({ path: fullPath, size, sizeKB: (size / 1024).toFixed(1) }); + sizeMap.push({ path: fullPath, size: utf8Len(child.content) }); } else if (child.type === 'dir') { walk(child, fullPath); } @@ -748,23 +783,54 @@ class FileSystemModule { } walk(root, ''); sizeMap.sort((a, b) => b.size - a.size); + // Top-level groups, so the table accounts for the whole tree: one row per + // top-level directory plus a row for files stored directly at root. + const groups = []; + const rootFiles = { bytes: 0, files: 0 }; + for (const [name, child] of Object.entries((root && root.content) || {})) { + if (!child) continue; + if (child.type === 'dir') { + const m = measure(child); + groups.push({ name: '/' + name, bytes: m.bytes, files: m.files }); + } else if (child.type === 'file') { + rootFiles.bytes += utf8Len(child.content); + rootFiles.files += 1; + } + } + if (rootFiles.files > 0) groups.push({ name: '/(root files)', bytes: rootFiles.bytes, files: rootFiles.files }); + groups.sort((a, b) => b.bytes - a.bytes); const totalFiles = sizeMap.length; - const top5 = sizeMap.slice(0, 5); this.terminal.print('📊 VFS Size Report', 'text-muted'); - this.terminal.print(' Total: ' + totalSize + ' MB across ' + totalFiles + ' files'); - this.terminal.print(' Largest:'); - top5.forEach((f, i) => { - this.terminal.print(' ' + (i + 1) + '. ' + f.path + ' — ' + f.sizeKB + ' KB'); - }); - // Show system dir sizes - const sysDirs = ['/_snapshots', '/_sessions', '/.trash']; + this.terminal.print(' Total: ' + fmtSize(total.bytes) + ' (' + total.bytes + ' bytes) across ' + totalFiles + (totalFiles === 1 ? ' file' : ' files')); + if (totalFiles > 0) { + this.terminal.print(' Largest:'); + const shownFiles = sizeMap.slice(0, 10); + shownFiles.forEach((f, i) => { + this.terminal.print(' ' + (i + 1) + '. ' + f.path + ' — ' + fmtSize(f.size)); + }); + // The rows must add up to the total above, so the tail is folded into one + // row instead of dropped: this table used to be top-5 only, and the rest of + // the tree was missing from the report entirely. + if (totalFiles > shownFiles.length) { + let restBytes = 0; + for (const f of sizeMap.slice(shownFiles.length)) restBytes += f.size; + this.terminal.print(' (+' + (totalFiles - shownFiles.length) + ' more files — ' + fmtSize(restBytes) + ')'); + } + } + // Every top-level group is listed (root breadth, not depth), so these rows + // always sum to the total above. + this.terminal.print(' Top directories by stored bytes:', 'info'); + for (const g of groups) { + this.terminal.print(' ' + g.name + ' — ' + fmtSize(g.bytes) + ' (' + g.files + (g.files === 1 ? ' file)' : ' files)')); + } + // Show system dir sizes — real bytes, not JSON.stringify(list of entries).length + const sysDirs = ['/_snapshots', '/_sessions', '/.trash', '/_mods', '/.semantic_index']; for (const dir of sysDirs) { - const dirNode = this.terminal.fs.readDir(dir); + const name = dir.replace(/^\//, ''); + const dirNode = (root && root.content && root.content[name]) || null; if (dirNode) { - const cnt = dirNode.length; - // estimate dir size - const dirSize = JSON.stringify(dirNode).length; - this.terminal.print(' ' + dir + ': ' + cnt + ' files, ~' + (dirSize / 1024).toFixed(1) + ' KB', dirSize > 102400 ? 'warning' : 'text-muted'); + const m = measure(dirNode); + this.terminal.print(' ' + dir + ': ' + m.files + (m.files === 1 ? ' file, ' : ' files, ') + fmtSize(m.bytes), m.bytes > 102400 ? 'warning' : 'text-muted'); } } } catch (e) { diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js index b926c4a..801bdb7 100644 --- a/modules/core/module-loader.js +++ b/modules/core/module-loader.js @@ -4,7 +4,7 @@ // Core module bundle loaded synchronously in main.html. // This loader only handles version cache-busting, splash, terminal init, and optional ext modules. - const LATEST_VER = 'v4.6.159'; + const LATEST_VER = 'v4.6.160'; const CACHE = '?v=' + LATEST_VER; // Exposed so ext modules can register their own URL with the same cache-busting version // instead of an unversioned path the browser may already have cached (shell.js did). diff --git a/modules/core/module-servermon.js b/modules/core/module-servermon.js index 600618a..2d320f4 100644 --- a/modules/core/module-servermon.js +++ b/modules/core/module-servermon.js @@ -32,7 +32,9 @@ class ServerMonitorModule { this.terminal.print('🌐 Network Test', 'header'); // 1. Browser online const online = navigator.onLine; - this.terminal.print(` ${online ? '✅' : '🔴'} Browser: ${online ? 'online' : 'offline'}`, online ? 'success' : 'error'); + // Reported, not measured: WebViews answer false while connected. The real verdict comes + // from the proxy probe below, so this line must not read as a failure. + this.terminal.print(` ${online ? '✅' : '❔'} Browser reports: ${online ? 'online' : 'offline (unreliable)'}`, online ? 'success' : 'warning'); // 2. Proxy API try { const t0 = Date.now(); diff --git a/modules/core/module-system.js b/modules/core/module-system.js index 43bb0b1..ef37b11 100644 --- a/modules/core/module-system.js +++ b/modules/core/module-system.js @@ -35,6 +35,84 @@ 'checkpoint-del': 'checkpoint-del <name> — Delete a VFS checkpoint', }; } + + // ── doctor measurement helpers ────────────────────────────────────── + // UTF-8 byte length of a string. Blob is the reference implementation + // (Node 17+/every target browser), TextEncoder the browser fallback, and a + // code-point walk the last resort so a stripped-down WebView still gets a + // number instead of a thrown command. + _utf8Bytes(str) { + if (typeof str !== 'string' || str.length === 0) return 0; + try { + if (typeof Blob === 'function') return new Blob([str]).size; + } catch (e) {} + try { + if (typeof TextEncoder === 'function') return new TextEncoder().encode(str).length; + } catch (e) {} + let n = 0; + for (let i = 0; i < str.length; i++) { + const cp = str.codePointAt(i); + if (cp > 0xffff) i++; // surrogate pair consumed + n += cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4; + } + return n; + } + + // Sum of UTF-8 bytes of every file's content under `root`. This is the unit + // `vfs-size` reports, without JSON.stringify's braces/quoting overhead and + // without the old `chars * 2` fudge. Depth- and node-capped so a malformed + // or cyclic tree cannot hang the command. + _vfsContentBytes(root) { + let bytes = 0, nodes = 0; + const walk = (node, depth) => { + if (!node || typeof node !== 'object' || depth > 64) return; + if (node.type === 'file') { + if (typeof node.content === 'string') bytes += this._utf8Bytes(node.content); + return; + } + const kids = node.content; + if (!kids || typeof kids !== 'object') return; + for (const name of Object.keys(kids)) { + if (++nodes > 20000) return; + walk(kids[name], depth + 1); + } + }; + walk(root, 0); + return bytes; + } + + // Live reachability probe. The legacy `onLine` boolean is not consulted anywhere: + // in Android WebView it reports false while the page is fully connected, which is + // exactly how `status` used to tell working users they were offline. + async _probeNetwork() { + try { + if (typeof fetch !== 'function') return { state: 'unprobed', detail: 'fetch unavailable' }; + } catch (e) { + return { state: 'unprobed', detail: 'fetch unavailable' }; + } + let ctl = null, timer = null; + try { if (typeof AbortController === 'function') ctl = new AbortController(); } catch (e) {} + try { + if (ctl) timer = setTimeout(() => { try { ctl.abort(); } catch (e) {} }, 4000); + const resp = await fetch('./', { + method: 'HEAD', + cache: 'no-store', + signal: ctl ? ctl.signal : undefined, + }); + if (!resp) return { state: 'unreachable', detail: 'empty response' }; + if (resp.ok) return { state: 'reachable', detail: 'HTTP ' + resp.status }; + return { state: 'http-' + resp.status, detail: 'HTTP ' + resp.status }; + } catch (e) { + const aborted = e && (e.name === 'AbortError' || e.name === 'TimeoutError'); + return { + state: 'unreachable', + detail: aborted ? 'timed out after 4s' : ((e && e.message) || 'request failed'), + }; + } finally { + if (timer) clearTimeout(timer); + } + } + registerCommands() { this.commands['help'] = (args) => this.handleHelp(args); this.commands['echo'] = (args) => this.terminal.print(args.join(' ')); @@ -83,7 +161,10 @@ }, 500); this.commands['status'] = () => { this.terminal.print(`System: WebX11 ${window.NEWOS4_VERSION || 'v4.0'}`); - this.terminal.print(`Network: ${navigator.onLine ? 'Online' : 'Offline'}`); + // The legacy `onLine` boolean is NOT evidence of reachability: Android + // WebView reports false on a fully connected page, and true on a captive + // portal. Say so, and point at the command that actually probes. + this.terminal.print('Network: unknown — run doctor for a live probe'); this.terminal.print(`Files: ${this.terminal.fs.ls().length} items`); this.terminal.print(`Modules: ${Object.keys(this.terminal.moduleManager.modules).length} loaded`); this.terminal.print(`Ollama: ${this.terminal.ollamaManager.connected ? 'Connected' : 'Disconnected'}`); @@ -110,7 +191,7 @@ this.terminal.print(` Build: ${buildDate}`); this.terminal.print(` Cache key: ${cacheKey}`); this.terminal.print(` Browser: ${ua}`); - this.terminal.print(` Online: ${navigator.onLine ? 'yes' : 'no'}`); + this.terminal.print(` Network: unknown — run doctor for a live probe`); }; // v4.5.43: uptime — time since page load + last save time @@ -166,40 +247,102 @@ }; // v4.5.43: doctor — run all health checks, rank issues, show top 3 - this.commands['doctor'] = () => { + // v4.6.x: the legacy `onLine` boolean is GONE from this file. It reports false + // on a fully connected Android WebView, which is how `status`/`doctor` used to + // announce an outage on a working page. Reachability now comes from a real HEAD + // probe, and every size is measured in the unit the store actually charges for. + this.commands['doctor'] = async () => { const issues = []; const fs = this.terminal.fs; - - // Check 1: Network - if (!navigator.onLine) issues.push({ severity: 'high', msg: 'Network: offline' }); + const MB = 1024 * 1024; + + // Check 1: Network — live probe, 4s budget + const net = await this._probeNetwork(); + if (net.state === 'unreachable') { + issues.push({ severity: 'high', msg: `Network: unreachable (HEAD ./ — ${net.detail})` }); + } else if (net.state.indexOf('http-') === 0) { + // A non-ok HEAD is not proof the page is broken (some servers answer + // 405 to HEAD); report the status and let the user judge. + issues.push({ severity: 'medium', msg: `Network: HTTP probe to ./ returned ${net.state} (page may still work)` }); + } // Check 2: Ollama if (!this.terminal.ollamaManager.connected) { issues.push({ severity: 'medium', msg: 'Ollama: not connected (AI features limited)' }); } - // Check 3: VFS size + // Origin quota — context only. It is the whole-origin quota (Cache Storage + // included, typically hundreds of MB to GBs). localStorage has its own + // ~5 MB per-origin cap, so this number must NEVER set a severity: a 4.5 MB + // localStorage is 96% of its cap and 0.2% of a 2 GB origin quota. + let originUsage = 0, originQuota = 0; + try { + if (typeof navigator !== 'undefined' && navigator.storage && + typeof navigator.storage.estimate === 'function') { + const est = await navigator.storage.estimate(); + if (est && typeof est.quota === 'number' && est.quota > 0) { + originQuota = est.quota; + originUsage = typeof est.usage === 'number' ? est.usage : 0; + } + } + } catch (e) {} + + // Check 3: VFS — UTF-8 bytes of file content (the unit vfs-size reports), + // never JSON.stringify(root).length and never "chars × 2". + let vfsBytes = null; + try { vfsBytes = this._vfsContentBytes(fs.root); } catch (e) { vfsBytes = null; } + + // Check 4: localStorage — JS strings are UTF-16, so each key and value is + // charged 2 bytes per code unit. Σ (key.length + value.length) × 2 is the + // real storage cost, not a fudge factor. + let lsBytes = 0, lsKeys = 0; try { - const vfsBytes = JSON.stringify(fs.root).length; - if (vfsBytes > 524288) issues.push({ severity: 'high', msg: `VFS: ${(vfsBytes/1024).toFixed(0)}KB > 512KB limit` }); - else if (vfsBytes > 262144) issues.push({ severity: 'medium', msg: `VFS: ${(vfsBytes/1024).toFixed(0)}KB > 256KB (consider reset-vfs)` }); + for (let i = 0; i < localStorage.length; i++) { + const k = localStorage.key(i); + if (!k) continue; + lsKeys++; + lsBytes += (k.length + (localStorage.getItem(k) || '').length) * 2; + } } catch (e) {} - // Check 4: Heap + // VFS thresholds: a VFS only threatens the browser budget when it is BOTH + // large on its own and the bulk of what is actually stored. + if (vfsBytes !== null) { + const vfsMB = vfsBytes / MB; + const share = lsBytes > 0 ? vfsBytes / lsBytes : 0; + const budget = lsBytes > 0 + ? `, ${(share * 100).toFixed(0)}% of the ${(lsBytes / MB).toFixed(1)} MB localStorage budget` + : ', no localStorage copy measured'; + if (vfsBytes > 3 * MB && (lsBytes === 0 || share > 0.8)) { + issues.push({ severity: 'high', msg: `VFS: ${vfsMB.toFixed(1)} MB of file content${budget} — close to the ~5 MB per-origin cap` }); + } else if (vfsBytes > 2 * MB) { + issues.push({ severity: 'medium', msg: `VFS: ${vfsMB.toFixed(1)} MB of file content${budget} (consider deleting stale checkpoints)` }); + } + } + + // localStorage thresholds: the cap is ~5 MB per origin, whatever + // navigator.storage.estimate() says the whole origin may use. + if (lsBytes > 4 * MB) { + issues.push({ severity: 'high', msg: `localStorage: ${(lsBytes / MB).toFixed(1)} MB of UTF-16 data in ${lsKeys} keys — approaching the ~5 MB per-origin cap` }); + } else if (lsBytes > 3 * MB) { + issues.push({ severity: 'medium', msg: `localStorage: ${(lsBytes / MB).toFixed(1)} MB of UTF-16 data in ${lsKeys} keys — past half of the ~5 MB per-origin cap` }); + } + + // Check 5: Heap if (performance.memory) { const heapMB = performance.memory.usedJSHeapSize / 1048576; if (heapMB > 200) issues.push({ severity: 'high', msg: `Heap: ${heapMB.toFixed(0)}MB > 200MB` }); else if (heapMB > 100) issues.push({ severity: 'medium', msg: `Heap: ${heapMB.toFixed(0)}MB > 100MB` }); } - // Check 5: Logs + // Check 6: Logs try { const logContent = fs.readFile('/logs/current.log') || ''; const logLines = logContent.split('\n').filter(l => l).length; if (logLines > 500) issues.push({ severity: 'low', msg: `Logs: ${logLines} lines (consider logrotate)` }); } catch (e) {} - // Check 6: Errors in last 100 log lines + // Check 7: Errors in last 100 log lines try { const logContent = fs.readFile('/logs/current.log') || ''; const recent = logContent.split('\n').filter(l => l).slice(-100); @@ -208,16 +351,6 @@ else if (errorCount > 0) issues.push({ severity: 'medium', msg: `Errors: ${errorCount} errors in last 100 log lines` }); } catch (e) {} - // Check 7: localStorage quota - let lsBytes = 0; - try { - for (let i = 0; i < localStorage.length; i++) { - const k = localStorage.key(i); - if (k) lsBytes += (k.length + (localStorage.getItem(k) || '').length) * 2; - } - } catch (e) {} - if (lsBytes > 4 * 1024 * 1024) issues.push({ severity: 'high', msg: `localStorage: ${(lsBytes/1024/1024).toFixed(1)}MB approaching 5MB quota` }); - // Check 8: Disabled modules let disabled = []; try { disabled = JSON.parse(localStorage.getItem('webx11_module_disabled') || '[]'); } catch (e) {} @@ -226,6 +359,11 @@ } this.terminal.print('🩺 Doctor Report', 'header'); + this.terminal.print(` Measured: VFS ${vfsBytes === null ? 'unavailable' : (vfsBytes / MB).toFixed(2) + ' MB UTF-8'} · localStorage ${(lsBytes / MB).toFixed(2)} MB UTF-16 in ${lsKeys} keys`, 'info'); + this.terminal.print(` Network: ${net.state === 'reachable' ? 'reachable (' + net.detail + ')' : net.state + (net.detail ? ' — ' + net.detail : '')}`, 'info'); + if (originQuota > 0) { + this.terminal.print(` Origin usage ${(originUsage / MB).toFixed(1)} MB of ~${(originQuota / MB).toFixed(0)} MB quota (whole origin, mostly Cache Storage — localStorage has its own ~5 MB cap)`, 'info'); + } if (issues.length === 0) { this.terminal.print(' ✅ All systems healthy', 'success'); return; @@ -234,20 +372,21 @@ const sevOrder = { high: 0, medium: 1, low: 2 }; issues.sort((a, b) => sevOrder[a.severity] - sevOrder[b.severity]); this.terminal.print(` Found ${issues.length} issue(s):`, 'warning'); - for (const i of issues.slice(0, 5)) { + for (const i of issues.slice(0, 3)) { const icon = i.severity === 'high' ? '🔴' : i.severity === 'medium' ? '🟡' : '🟢'; this.terminal.print(` ${icon} [${i.severity.toUpperCase()}] ${i.msg}`, i.severity === 'high' ? 'error' : 'warning'); } - if (issues.length > 5) this.terminal.print(` ... and ${issues.length - 5} more`, 'info'); + if (issues.length > 3) this.terminal.print(` ... and ${issues.length - 3} more`, 'info'); this.terminal.print('', ''); this.terminal.print(' Recommended actions:', 'info'); - if (issues.some(i => i.msg.startsWith('Network'))) this.terminal.print(' → Check internet connection'); + if (issues.some(i => i.msg.startsWith('Network: unreachable'))) this.terminal.print(' → Check connectivity; reload before trusting this probe'); + if (issues.some(i => i.msg.startsWith('Network: HTTP probe'))) this.terminal.print(' → Server answered the probe with an error status — check the server, not the network'); if (issues.some(i => i.msg.startsWith('Ollama'))) this.terminal.print(' → Check Ollama is running (ollama serve)'); - if (issues.some(i => i.msg.startsWith('VFS'))) this.terminal.print(' → Run reset-vfs --confirm to clear or checkpoint/rollback'); + if (issues.some(i => i.msg.startsWith('VFS'))) this.terminal.print(' → Delete stale snapshots (checkpoint-list, checkpoint-del <name>) or rollback; do NOT reset-vfs'); if (issues.some(i => i.msg.startsWith('Heap'))) this.terminal.print(' → Refresh the page to clear memory'); if (issues.some(i => i.msg.startsWith('Logs'))) this.terminal.print(' → Run logrotate'); if (issues.some(i => i.msg.startsWith('Errors'))) this.terminal.print(' → Run errors to see details, or bug-report to save state'); - if (issues.some(i => i.msg.startsWith('localStorage'))) this.terminal.print(' → Run factory-reset --confirm (last resort)'); + if (issues.some(i => i.msg.startsWith('localStorage'))) this.terminal.print(' → Free space by deleting stale VFS entries or checkpoints; do NOT factory-reset'); }; // v4.5.43: bug-report — generate markdown report to /tmp/ @@ -261,7 +400,7 @@ lines.push('## System'); lines.push(`- Version: ${window.NEWOS4_VERSION || 'unknown'}`); lines.push(`- Cache key: ${localStorage.getItem('webx11_cache_ver') || 'unknown'}`); - lines.push(`- Online: ${navigator.onLine ? 'yes' : 'no'}`); + lines.push(`- Online: unknown (not probed; run doctor)`); lines.push(`- Browser: ${navigator.userAgent}`); lines.push(`- Screen: ${screen.width}x${screen.height}`); lines.push(`- Heap: ${performance.memory ? (performance.memory.usedJSHeapSize / 1048576).toFixed(1) + 'MB' : 'N/A'}`); @@ -325,7 +464,10 @@ version: window.NEWOS4_VERSION || 'unknown', url: location.href, ua: navigator.userAgent, - online: navigator.onLine, + // Legacy key, kept so stored reports keep their shape. The old onLine + // boolean lied in Android WebView, so null = "not asserted". + online: null, + network: 'unknown', modules: Object.keys(this.terminal.moduleManager.modules).length, errors: [], }; diff --git a/modules/core/terminal.js b/modules/core/terminal.js index d0f6cff..9750c24 100644 --- a/modules/core/terminal.js +++ b/modules/core/terminal.js @@ -564,9 +564,14 @@ } } updateNetworkStatus() { + // navigator.onLine is a hint, not a fact: an Android WebView reports false while + // fully connected (the same lie `status` used to print). Never turn that hint into + // "❌ Offline" — claim Online only when the browser says so, otherwise say unknown + // and point at `doctor`, which probes for real. const isOnline = navigator.onLine; - this.networkStatus.textContent = isOnline ? '📶 Online' : '❌ Offline'; - this.networkStatus.style.color = isOnline ? 'var(--text-color)' : 'var(--error-color)'; + this.networkStatus.textContent = isOnline ? '📶 Online' : '📶 Network?'; + this.networkStatus.title = isOnline ? '' : 'Browser reports offline — run doctor for a live probe'; + this.networkStatus.style.color = isOnline ? 'var(--text-color)' : 'var(--text-muted)'; } updatePrompt() { const path = '/' + this.fs.currentPath.join('/'); diff --git a/modules/core/virtual-fs.js b/modules/core/virtual-fs.js index 3a3e119..57c6578 100644 --- a/modules/core/virtual-fs.js +++ b/modules/core/virtual-fs.js @@ -510,6 +510,19 @@ class VirtualFileSystem { } return { dir, name: segs[segs.length - 1] }; } + // newOS4 patch: _resolvePath() validates intermediate segments only, so it returns a + // result for a leaf that does not exist. Mutating callers then acted on a phantom + // path and reported success (deleteCheckpoint printed 'Deleted checkpoint' and + // deleted nothing). _requireLeaf() refuses any resolved path whose leaf is absent. + _requireLeaf(resolved) { + if (!resolved) return null; + if (resolved.name) { + const node = (resolved.dir && resolved.dir.content) ? resolved.dir.content[resolved.name] : null; + if (!node) return null; + } + return resolved; + } + _resolveDir(path) { if (this._isVirtualPath(path)) { const r = this._resolveVirtualPath(path); @@ -579,7 +592,7 @@ class VirtualFileSystem { } deleteFile(path) { if (this._isVirtualPath(path)) return false; - const resolved = this._resolvePath(path); + const resolved = this._requireLeaf(this._resolvePath(path)); if (!resolved || !resolved.name) return false; const node = resolved.dir.content[resolved.name]; if (node && node.type === 'file') { @@ -966,6 +979,10 @@ class VirtualFileSystem { delete rootClone.content['_sessions']; delete rootClone.content['_snapshots']; delete rootClone.content['.trash']; + // newOS4 patch: module originals and the semantic index dominated every + // snapshot (one checkpoint reached 4.13MB) and grew on each new checkpoint. + delete rootClone.content['_mods']; + delete rootClone.content['.semantic_index']; } const data = { version: 1, @@ -1000,7 +1017,7 @@ class VirtualFileSystem { // clone verbatim deletes every checkpoint, every agent session and the trash, so keep the // live copies and re-attach them after the root swap. const keep = {}; - for (const dir of ['_snapshots', '_sessions', '.trash']) { + for (const dir of ['_snapshots', '_sessions', '.trash', '_mods', '.semantic_index']) { const node = this.root && this.root.content ? this.root.content[dir] : null; if (node) keep[dir] = node; } @@ -1020,7 +1037,7 @@ class VirtualFileSystem { } deleteCheckpoint(name) { const path = '/_snapshots/checkpoint-' + name + '.json'; - const resolved = this._resolvePath(path); + const resolved = this._requireLeaf(this._resolvePath(path)); if (resolved && resolved.name) { delete resolved.dir.content[resolved.name]; this.saveToStorage(); diff --git a/modules/test/test-doctor-thresholds.js b/modules/test/test-doctor-thresholds.js new file mode 100644 index 0000000..120b941 --- /dev/null +++ b/modules/test/test-doctor-thresholds.js @@ -0,0 +1,341 @@ +// node modules/test/test-doctor-thresholds.js +// Proves `doctor` measures real storage cost and can actually fire: +// · network comes from a live HEAD probe, never from the legacy onLine boolean; +// · localStorage is charged as UTF-16 (Σ (key+value) × 2), against its own ~5 MB cap; +// · VFS is summed as UTF-8 bytes of file content (Blob), the same unit vfs-size uses; +// · navigator.storage.estimate() is CONTEXT ONLY — a 2 GB origin quota must NOT suppress a +// 4.5 MB localStorage finding. That is the exact defect in the proposed patch, whose +// `lsBytes > quota * 0.8` comparison could never be true on a real browser. +// Run through the real class loaded from modules/core/module-system.js. +const fsMod = require('fs'), path = require('path'); +const ROOT = path.join(__dirname, '..', 'core'); +// N4_SRC lets the same assertions run against another build of the file (mutation check). +const SRC_PATH = process.env.N4_SRC || path.join(ROOT, 'module-system.js'); + +global.window = global; + +const src = fsMod.readFileSync(SRC_PATH, 'utf8'); +const SystemModule = new Function(src + '\nreturn SystemModule;')(); + +// ── a localStorage that charges what a browser charges ────────────────────── +// Values are held as given; the module must do the ×2 itself, so this fake reports +// .length (UTF-16 code units) exactly like the real Storage object does. +function makeLS(pairs) { + const map = new Map(Object.entries(pairs || {})); + return { + get length() { return map.size; }, + key(i) { return Array.from(map.keys())[i] ?? null; }, + getItem(k) { return map.has(k) ? map.get(k) : null; }, + setItem(k, v) { map.set(k, String(v)); }, + removeItem(k) { map.delete(k); }, + clear() { map.clear(); }, + _map: map, + }; +} + +// ── terminal stub ────────────────────────────────────────────────────────── +function mkTerminal(opts) { + const out = []; + const files = (opts && opts.files) || {}; + const dirs = (opts && opts.dirs) || { '/': [] }; + return { + out, + text: () => out.map(l => l.msg).join('\n'), + fs: { + root: (opts && opts.root) || { type: 'dir', content: {} }, + ls: () => (dirs['/'] || []), + readDir: (p) => dirs[p] || null, + readFile: (p) => (p in files ? files[p] : null), + }, + ollamaManager: { connected: (opts && opts.ollama !== undefined) ? opts.ollama : true }, + moduleManager: { modules: { SystemModule: {}, VirtualFSModule: {} } }, + print: (msg, cls) => out.push({ msg: String(msg), cls: cls || '' }), + }; +} + +function mkModule(opts) { + const t = mkTerminal(opts); + const m = new SystemModule(); + m.terminal = t; + m.registerCommands(); + return { m, t }; +} + +let fails = 0, passes = 0; +function ok(name, cond, detail) { + if (cond) { passes++; return; } + fails++; + console.log('FAIL ' + name + (detail ? ' — ' + detail : '')); +} + +// ── helpers to build payloads of an exact byte size ──────────────────────── +const kb = 1024; +function asciiFile(mb) { return 'a'.repeat(Math.round(mb * 1024 * 1024)); } + +// A VFS file node tree holding one big ASCII file. +function rootWithFile(content) { + return { type: 'dir', content: { tmp: { type: 'dir', content: { 'big.txt': { type: 'file', content } } } } }; +} + +// Drive `doctor` with a given fake browser surface. +async function doctor(opts) { + const { m, t } = mkModule(opts); + const o = opts || {}; + const savedNavigator = global.navigator; + const savedFetch = global.fetch; + const savedLS = global.localStorage; + global.localStorage = o.ls || makeLS({}); + if ('fetch' in o) { + if (o.fetch === undefined) delete global.fetch; + else global.fetch = o.fetch; + } + if ('storage' in o) { + const nav = Object.assign({}, savedNavigator); + if (o.storage === undefined) delete nav.storage; + else nav.storage = o.storage; + Object.defineProperty(global, 'navigator', { value: nav, configurable: true, writable: true }); + } + try { + const r = m.commands['doctor'](); + if (r && typeof r.then === 'function') await r; + } finally { + global.fetch = savedFetch; + global.localStorage = savedLS; + Object.defineProperty(global, 'navigator', { value: savedNavigator, configurable: true, writable: true }); + } + return t; +} + +// Extract the issue lines the report printed (top 3) plus the full issue count line. +function issueLines(t) { + return t.out.map(l => l.msg).filter(s => /^\s{4}\S+ \[(HIGH|MEDIUM|LOW)\]/.test(s)); +} +function report(t) { return t.text(); } + +function okFetch(status) { + return () => Promise.resolve({ ok: status >= 200 && status < 300, status }); +} + +// ══════════════════════════════════════════════════════════════════════════ +// A. Network — live probe, not the legacy boolean +// ══════════════════════════════════════════════════════════════════════════ +(async () => { + // A1: a healthy fetch must produce NO Network issue at all. + { + const t = await doctor({ fetch: okFetch(200), files: {} }); + ok('A1 fetch ok → no Network issue', !/Network: (unreachable|HTTP probe)/.test(report(t)), report(t)); + ok('A1b report says reachable', /Network:\s+reachable/.test(report(t)), report(t)); + } + + // A2: a rejected fetch must be a HIGH unreachable finding. + { + const t = await doctor({ fetch: () => Promise.reject(new Error('net::ERR_NAME_NOT_RESOLVED')) }); + const txt = report(t); + ok('A2 fetch reject → high Network: unreachable', + /\[HIGH\] Network: unreachable/.test(txt), txt); + } + + // A3: a 503 must be named, not flattened to "offline". + { + const t = await doctor({ fetch: okFetch(503) }); + const txt = report(t); + ok('A3 fetch 503 → Network issue naming http-503', + /Network:.*http-503/.test(txt), txt); + ok('A3b 503 is not reported as unreachable', + !/Network: unreachable/.test(txt), txt); + } + + // A4: legacy boolean must not exist in the source at all. + { + const hits = src.split('\n').filter(l => /navigator\s*\.\s*onLine/.test(l)).length; + ok('A4 no navigator.onLine reference in module-system.js', hits === 0, hits + ' hit(s)'); + } + + // ══════════════════════════════════════════════════════════════════════ + // B. localStorage — UTF-16 cost against its own ~5 MB cap + // ══════════════════════════════════════════════════════════════════════ + // 4.5 MB of UTF-16 = 2.25 M code units of value + key. + { + const t = await doctor({ + fetch: okFetch(200), + ls: makeLS({ webx11_big: 'a'.repeat(Math.round(4.5 * 1024 * 1024 / 2)) }), + }); + const txt = report(t); + ok('B1 localStorage 4.5 MB → HIGH localStorage issue', + /\[HIGH\] localStorage: 4\.5 MB/.test(txt), txt); + ok('B1b message names the ~5 MB per-origin cap', + /~5 MB per-origin cap/.test(txt), txt); + } + { + // 3.2 MB of UTF-16 = 1.6 M code units. + const t = await doctor({ + fetch: okFetch(200), + ls: makeLS({ webx11_mid: 'b'.repeat(Math.round(3.2 * 1024 * 1024 / 2)) }), + }); + const txt = report(t); + ok('B2 localStorage 3.2 MB → MEDIUM', /\[MEDIUM\] localStorage: 3\.2 MB/.test(txt), txt); + ok('B2b no HIGH localStorage at 3.2 MB', !/\[HIGH\] localStorage/.test(txt), txt); + } + { + const t = await doctor({ + fetch: okFetch(200), + ls: makeLS({ webx11_small: 'c'.repeat(Math.round(1 * 1024 * 1024 / 2)) }), + }); + ok('B3 localStorage 1 MB → no localStorage issue', + !/localStorage:/.test(report(t)), report(t)); + } + + // ══════════════════════════════════════════════════════════════════════ + // C. VFS — UTF-8 bytes of content, share of the localStorage budget + // ══════════════════════════════════════════════════════════════════════ + { + // 3.5 MB of ASCII VFS + 4 MB localStorage budget → 87% share → HIGH. + const t = await doctor({ + fetch: okFetch(200), + root: rootWithFile(asciiFile(3.5)), + ls: makeLS({ webx11_vfs: 'd'.repeat(Math.round(4 * 1024 * 1024 / 2)) }), + }); + const txt = report(t); + ok('C1 VFS 3.5 MB of 4 MB budget → HIGH VFS issue', + /\[HIGH\] VFS: 3\.5 MB/.test(txt), txt); + ok('C1b message states the measured share', /87% of the 4\.0 MB localStorage budget/.test(txt), txt); + ok('C1c VFS does not claim a JSON limit', !/512KB|256KB/.test(txt), txt); + } + { + // 1 MB VFS → under both thresholds, nothing. + const t = await doctor({ + fetch: okFetch(200), + root: rootWithFile(asciiFile(1)), + ls: makeLS({ webx11_vfs: 'e'.repeat(Math.round(4 * 1024 * 1024 / 2)) }), + }); + ok('C2 VFS 1 MB → no VFS issue', !/VFS: /.test(report(t)), report(t)); + } + { + // 2.5 MB VFS → MEDIUM (large alone, but not a big share of an 8 MB budget). + const t = await doctor({ + fetch: okFetch(200), + root: rootWithFile(asciiFile(2.5)), + ls: makeLS({ webx11_vfs: 'f'.repeat(Math.round(8 * 1024 * 1024 / 2)) }), + }); + const txt = report(t); + ok('C3 VFS 2.5 MB → MEDIUM', /\[MEDIUM\] VFS: 2\.5 MB/.test(txt), txt); + } + { + // UTF-8, not chars: 1 MiB of 'é' is 2.00 MB of UTF-8 (chars would read 1.00 MB). + const t = await doctor({ + fetch: okFetch(200), + root: rootWithFile('é'.repeat(1024 * 1024)), + ls: makeLS({ webx11_vfs: 'g'.repeat(Math.round(8 * 1024 * 1024 / 2)) }), + }); + ok('C4 multibyte counted as UTF-8 bytes (1M é = 2.00 MB)', + /Measured: VFS 2\.00 MB UTF-8/.test(report(t)), report(t)); + } + + // ══════════════════════════════════════════════════════════════════════ + // D. The patch's bug: origin quota must not suppress a real finding + // ══════════════════════════════════════════════════════════════════════ + { + const est = () => Promise.resolve({ usage: 40 * 1024 * 1024, quota: 2 * 1024 * 1024 * 1024 }); + const t = await doctor({ + fetch: okFetch(200), + storage: { estimate: est }, + ls: makeLS({ webx11_big: 'h'.repeat(Math.round(4.5 * 1024 * 1024 / 2)) }), + }); + const txt = report(t); + ok('D1 2 GB origin quota does NOT suppress a 4.5 MB localStorage finding', + /\[HIGH\] localStorage: 4\.5 MB/.test(txt), txt); + ok('D2 quota is printed as context only', + /Origin usage 40\.0 MB of ~2048 MB quota/.test(txt), txt); + ok('D3 context line states localStorage has its own cap', + /localStorage has its own ~5 MB cap/.test(txt), txt); + ok('D4 quota does not turn a big VFS healthy either', !/All systems healthy/.test(txt), txt); + } + { + // And a big quota must not invent an issue where none exists. + const t = await doctor({ + fetch: okFetch(200), + storage: { estimate: () => Promise.resolve({ usage: 10 * 1024 * 1024, quota: 2 * 1024 * 1024 * 1024 }) }, + ls: makeLS({ webx11_a: 'tiny' }), + }); + ok('D5 healthy storage + huge quota → all healthy', + /All systems healthy/.test(report(t)), report(t)); + } + + // ══════════════════════════════════════════════════════════════════════ + // E. Degraded browsers — every probe is individually guarded + // ══════════════════════════════════════════════════════════════════════ + { + const t = await doctor({ fetch: undefined, files: {} }); + const txt = report(t); + ok('E1 fetch undefined → completes and prints a report', + /Doctor Report/.test(txt) && /Measured: VFS/.test(txt), txt); + ok('E1b no Network issue invented without a probe', !/Network: unreachable/.test(txt), txt); + } + { + // Blob removed: the UTF-8 fallback must still measure 2 bytes per 'é'. + const savedBlob = global.Blob; + delete global.Blob; + try { + const t = await doctor({ + fetch: okFetch(200), + root: rootWithFile('é'.repeat(1024 * 1024)), + ls: makeLS({ webx11_vfs: 'i'.repeat(Math.round(8 * 1024 * 1024 / 2)) }), + }); + ok('E2 Blob undefined → completes and still counts UTF-8 (2.00 MB)', + /Doctor Report/.test(report(t)) && /Measured: VFS 2\.00 MB UTF-8/.test(report(t)), report(t)); + } finally { + global.Blob = savedBlob; + } + } + { + const t = await doctor({ fetch: okFetch(200), storage: undefined, files: {} }); + const txt = report(t); + ok('E3 storage.estimate undefined → completes, no quota line', + /Doctor Report/.test(txt) && !/Origin usage/.test(txt), txt); + } + { + // estimate() that throws must not kill the report. + const t = await doctor({ + fetch: okFetch(200), + storage: { estimate: () => Promise.reject(new Error('SecurityError')) }, + }); + ok('E4 estimate() rejecting → completes', /Doctor Report/.test(report(t)), report(t)); + } + + // ══════════════════════════════════════════════════════════════════════ + // F. status must not claim Online/Offline + // ══════════════════════════════════════════════════════════════════════ + { + const { m, t } = mkModule({ fetch: okFetch(200) }); + m.commands['status'](); + const txt = report(t); + const netLine = t.out.map(l => l.msg).find(s => /^Network:/.test(s)) || ''; + ok('F1 status prints a Network line', netLine.length > 0, txt); + ok('F2 status line says neither Online nor Offline', + !/\b(Online|Offline)\b/.test(netLine), netLine); + ok('F3 status points at doctor', /doctor/.test(netLine), netLine); + ok('F4 status still reports files/modules/ollama', + /^Files:/m.test(txt) && /^Modules:/m.test(txt) && /^Ollama:/m.test(txt), txt); + } + + // ══════════════════════════════════════════════════════════════════════ + // G. Output shape kept: top 3 issues + Recommended actions + // ══════════════════════════════════════════════════════════════════════ + { + const t = await doctor({ + fetch: () => Promise.reject(new Error('down')), + ollama: false, + root: rootWithFile(asciiFile(3.5)), + ls: makeLS({ webx11_big: 'j'.repeat(Math.round(4.5 * 1024 * 1024 / 2)) }), + files: { '/logs/current.log': Array.from({ length: 900 }, () => 'x').join('\n') }, + }); + const txt = report(t); + ok('G1 at most 3 issue lines printed', issueLines(t).length <= 3, issueLines(t).length + ' lines'); + ok('G2 remainder is summarised', /\.\.\. and \d+ more/.test(txt), txt); + ok('G3 Recommended actions section present', /Recommended actions:/.test(txt), txt); + } + + // ── summary ─────────────────────────────────────────────────────────── + console.log('\n' + passes + '/' + (passes + fails) + ' assertions passed'); + if (fails > 0) { console.log(fails + ' FAILED'); process.exit(1); } +})(); diff --git a/modules/test/test-vfs-size-bytes.js b/modules/test/test-vfs-size-bytes.js new file mode 100644 index 0000000..109c58e --- /dev/null +++ b/modules/test/test-vfs-size-bytes.js @@ -0,0 +1,216 @@ +// node modules/test/test-vfs-size-bytes.js +// test-vfs-size-bytes — `vfs-size` must report REAL UTF-8 STORED BYTES, one unit, everywhere. +// +// RED on the pre-patch command (v4.6.159): it printed three different units — +// total = JSON.stringify(fs.root).length / 1MB +// per file = child.content.length (UTF-16 code units) +// system dir = JSON.stringify(readDir(dir)).length (a list of entries, not bytes) +// The total also carried every structural character of the serialisation, so a text +// tree reported ~2x its real stored size, while 'é' and '☃' were under-counted. +// +// This test pins the NUMBERS THE COMMAND PRINTS, never a helper: +// 1. unit facts on one-file trees: 'abcd' 4 B, 'é' 2 B, '☃' 3 B, 'é☃' 5 B, +// 1000 ASCII chars 1000 B, '' 0 B; +// 2. a 12-file nested tree: total == sum of the per-file rows == sum of the +// top-directory rows (each file counted once, no structure overhead); +// 3. every system-dir line == a manual sum of that dir's files; +// 4. a tree of directories with no files at all neither throws nor prints NaN. +// +// Set FS_MODULE_SRC=<path> to point the harness at another copy of module-fs.js +// (used to prove the test is RED against the pre-fix / mutated file). +const fsMod = require('fs'); +const path = require('path'); +const SRC = process.env.FS_MODULE_SRC || path.join(__dirname, '..', 'core', 'module-fs.js'); + +// ── harness ──────────────────────────────────────────────────────────────── +let pass = 0; +const fails = []; +function ok(name, cond, detail) { + if (cond) { pass++; console.log(' ok ' + name); } + else { fails.push(name + (detail ? ' — ' + detail : '')); console.log(' FAIL ' + name + (detail ? ' — ' + detail : '')); } +} +function eq(name, actual, expected) { + ok(name, actual === expected, 'got ' + JSON.stringify(actual) + ', expected ' + JSON.stringify(expected)); +} + +// ── load the real class out of the real file (no re-implementation) ──────── +const src = fsMod.readFileSync(SRC, 'utf8'); +const FileSystemModule = new Function(src + '\nreturn FileSystemModule;')(); + +// ── tree builders ────────────────────────────────────────────────────────── +function dir(children) { return { type: 'dir', content: children }; } +function file(text) { return { type: 'file', content: text }; } +const B = (s) => Buffer.byteLength(s, 'utf8'); // independent of Blob + +// ── drive the shipped command and capture what it prints ─────────────────── +function run(tree) { + const out = []; + const mod = new FileSystemModule(); + mod.terminal = { fs: { root: tree }, print: (text) => out.push(String(text)) }; + mod.registerCommands(); + mod.commands['vfs-size'](); // a throw escapes here and fails the test loudly + return out; +} + +const UNITS = { B: 1, KB: 1024, MB: 1024 * 1024 }; +function toBytes(human) { + const m = /^([\d.]+) (B|KB|MB)$/.exec(human); + if (!m) return null; + return Math.round(parseFloat(m[1]) * UNITS[m[2]]); +} +function parse(out) { + const r = { totalHuman: null, totalExact: null, files: null, rows: [], rest: null, dirs: [], sys: [] }; + for (const line of out) { + let m; + if ((m = /^ {2}Total: ([\d.]+ (?:B|KB|MB)) \((\d+) bytes\) across (\d+) files?$/.exec(line))) { + r.totalHuman = toBytes(m[1]); r.totalExact = Number(m[2]); r.files = Number(m[3]); + } else if ((m = /^ {4}\d+\. (\S+) — ([\d.]+ (?:B|KB|MB))$/.exec(line))) { + r.rows.push({ path: m[1], bytes: toBytes(m[2]) }); + } else if ((m = /^ {4}\(\+(\d+) more files? — ([\d.]+ (?:B|KB|MB))\)$/.exec(line))) { + r.rest = { count: Number(m[1]), bytes: toBytes(m[2]) }; + } else if ((m = /^ {4}(.+?) — ([\d.]+ (?:B|KB|MB)) \((\d+) files?\)$/.exec(line))) { + r.dirs.push({ name: m[1], bytes: toBytes(m[2]), files: Number(m[3]) }); + } else if ((m = /^ {2}(\/[\w.]+): (\d+) files?, ([\d.]+ (?:B|KB|MB))$/.exec(line))) { + r.sys.push({ name: m[1], files: Number(m[2]), bytes: toBytes(m[3]) }); + } + } + return r; +} +const sum = (a) => a.reduce((x, y) => x + y, 0); + +console.log('module: ' + SRC); +console.log('node ' + process.version + ', Blob is a global: ' + (typeof Blob === 'function')); + +// ── 1. unit facts, one file per tree (always row 1, never truncated) ─────── +console.log('\n── 1. one-file trees: bytes actually counted ──'); +const CASES = [ + ['abcd', 'abcd', 4, 'a file containing "abcd" counts 4 bytes'], + ['é', 'é', 2, '"é" counts 2 bytes (1 UTF-16 unit, 2 UTF-8 bytes)'], + ['☃', '☃', 3, '"☃" counts 3 bytes'], + ['é☃', 'é☃', 5, '"é☃" counts 5 bytes'], + ['x'.repeat(1000), '1000 ASCII chars', 1000, 'a 1000-char ASCII file counts 1000 bytes'], + ['', '(empty)', 0, 'an empty file counts 0 bytes'], +]; +for (const [content, label, want, name] of CASES) { + const p = parse(run(dir({ 'f.txt': file(content) }))); + console.log(' ' + String(label).slice(0, 24).padEnd(26) + String(p.totalExact).padStart(6) + ' B row1=' + (p.rows[0] ? p.rows[0].bytes : 'MISSING')); + eq(name, p.totalExact, want); + eq(' row 1 for ' + label + ' is the same number', p.rows[0] && p.rows[0].bytes, want); + eq(' file count for ' + label, p.files, 1); +} + +// ── 2. a 12-file nested tree ────────────────────────────────────────────── +const tree = dir({ + 'a.txt': file('abcd'), // 4 (root-level file -> own group row) + 'dirA': dir({ + 'e.txt': file('é'), // 2 + 'sub': dir({ 'c.txt': file('☃') }), // 3 (three levels deep) + }), + 'dirB': dir({ + 'f.txt': file('x'.repeat(1000)), // 1000 + 'deep': dir({ 'g.txt': file('é☃') }), // 5 + }), + 'dirC': dir({ 'onlySub': dir({ 'deeper': dir({}) }) }), // 0 (no files at any depth) + '_snapshots': dir({ 'cp1.json': file('{"a":1}'), 'cp2.json': file('{"b":"é"}') }), // 7 + 11 + '.trash': dir({ 'old.txt': file('☃☃') }), // 6 + '_mods': dir({ 'calc.js': file('é'), 'nested': dir({ 'x.js': file('ab☃') }) }), // 2 + 5 + '.semantic_index': dir({ 'index.json': file('[]') }), // 2 + 'empty.txt': file(''), // 0 + // '_sessions' deliberately absent -> line skipped, must not throw +}); +const EXPECT = B('abcd') + B('é') + B('☃') + B('x'.repeat(1000)) + B('é☃') + B('') + + B('{"a":1}') + B('{"b":"é"}') + B('☃☃') + B('é') + B('ab☃') + B('[]'); +const EXPECT_FILES = 12; + +let out, p; +try { out = run(tree); p = parse(out); } +catch (e) { console.log(' FAIL vfs-size threw on the nested tree: ' + e.message); process.exit(1); } +const text = out.join('\n'); +const rowSum = sum(p.rows.map(r => r.bytes)) + (p.rest ? p.rest.bytes : 0); +const dirSum = sum(p.dirs.map(d => d.bytes)); + +console.log('\n── 2. nested tree totals ──'); +console.log(' printed total (exact) ' + p.totalExact + ' B'); +console.log(' printed total (human) ' + p.totalHuman + ' B (1-decimal, rounded from the exact field)'); +console.log(' per-file rows (' + p.rows.length + ') + remainder ' + (p.rest ? p.rest.bytes : 0) + ' = ' + rowSum); +console.log(' top-dir rows ' + dirSum); +console.log(' independent expectation ' + EXPECT + ' B over ' + EXPECT_FILES + ' files'); +console.log(' old metric JSON.stringify(root).length = ' + JSON.stringify(tree).length + ' (would have been printed as MB)'); +eq('file count', p.files, EXPECT_FILES); +eq('printed total is the UTF-8 byte sum', p.totalExact, EXPECT); +ok('human total is the same number (1-decimal KB tolerance)', + Math.abs(p.totalHuman - p.totalExact) <= 51, 'human=' + p.totalHuman + ' exact=' + p.totalExact); +eq('per-file rows + remainder sum to the printed total', rowSum, p.totalExact); +eq('top-directory rows sum to the printed total', dirSum, p.totalExact); +ok('printed total is NOT the old JSON.stringify(root).length', + p.totalExact !== JSON.stringify(tree).length, + 'old metric = ' + JSON.stringify(tree).length); + +console.log('\n── 2b. per-file table ──'); +for (const r of p.rows) console.log(' ' + r.path.padEnd(22) + String(r.bytes).padStart(6) + ' B'); +if (p.rest) console.log(' (+' + p.rest.count + ' more files — ' + p.rest.bytes + ' B)'); +const rowFor = (path) => (p.rows.find(r => r.path === path) || {}).bytes; +eq('largest file wins', rowFor('/dirB/f.txt'), 1000); +eq('nested file is listed once at its own path', rowFor('/dirA/sub/c.txt'), 3); +eq('truncated tail is one row, not silently dropped', p.rest && p.rest.count, EXPECT_FILES - p.rows.length); + +console.log('\n── 2c. top directories ──'); +for (const d of p.dirs) console.log(' ' + d.name.padEnd(22) + String(d.bytes).padStart(6) + ' B ' + d.files + ' files'); +const dirFor = (n) => p.dirs.find(d => d.name === n) || {}; +eq('/dirA counts its subdir once (2 + 3)', dirFor('/dirA').bytes, 5); +eq('/dirB counts deep/g.txt once (1000 + 5)', dirFor('/dirB').bytes, 1005); +eq('/dirB file count is 2, not 3', dirFor('/dirB').files, 2); +eq('root-level files get their own row', dirFor('/(root files)').bytes, 4); +eq('directory with only subdirs is 0 bytes', dirFor('/dirC').bytes, 0); +eq('every top-level dir appears (7 dirs + the root-files row)', p.dirs.length, 8); + +// ── 3. system dirs: real bytes, equal to a manual sum ───────────────────── +console.log('\n── 3. system dirs ──'); +for (const s of p.sys) console.log(' ' + s.name.padEnd(22) + String(s.bytes).padStart(6) + ' B ' + s.files + ' files'); +const sysFor = (n) => p.sys.find(s => s.name === n) || {}; +const SYS_EXPECT = { + '/_snapshots': { files: 2, bytes: B('{"a":1}') + B('{"b":"é"}') }, + '/.trash': { files: 1, bytes: B('☃☃') }, + '/_mods': { files: 2, bytes: B('é') + B('ab☃') }, + '/.semantic_index': { files: 1, bytes: B('[]') }, +}; +for (const [name, want] of Object.entries(SYS_EXPECT)) { + eq(name + ' bytes == manual sum', sysFor(name).bytes, want.bytes); + eq(name + ' file count (recursive)', sysFor(name).files, want.files); +} +ok('missing /_sessions prints no line instead of throwing', !sysFor('/_sessions').name); +eq('exactly the four present system dirs printed', p.sys.length, 4); +ok('_mods and .semantic_index are covered (the two dirs that dominate a checkpoint)', + !!sysFor('/_mods').name && !!sysFor('/.semantic_index').name); + +// ── 4. a tree of directories with no files ──────────────────────────────── +console.log('\n── 4. directories only, no files ──'); +let out2; +try { + out2 = run(dir({ 'a': dir({ 'b': dir({ 'c': dir({}) }) }), 'd': dir({}) })); + ok('does not throw', true); +} catch (e) { + ok('does not throw', false, e.message); + out2 = []; +} +const text2 = out2.join('\n'); +const p2 = parse(out2); +eq('total is 0 bytes', p2.totalExact, 0); +eq('file count is 0', p2.files, 0); +ok('no "Largest:" block when there are no files', !/Largest:/.test(text2)); +eq('dir rows still sum to the total', sum(p2.dirs.map(d => d.bytes)), 0); + +// ── 5. no undefined / NaN anywhere ──────────────────────────────────────── +ok('main output has no "undefined"', !/undefined/.test(text)); +ok('main output has no "NaN"', !/NaN/.test(text)); +ok('dirs-only output has no "undefined"', !/undefined/.test(text2)); +ok('dirs-only output has no "NaN"', !/NaN/.test(text2)); + +// ── verdict ─────────────────────────────────────────────────────────────── +console.log('\n' + (pass + fails.length) + ' assertions, ' + pass + ' passed, ' + fails.length + ' failed'); +if (fails.length) { + fails.forEach(f => console.log(' FAILED: ' + f)); + console.log('RESULT: FAIL'); + process.exit(1); +} +console.log('RESULT: PASS — vfs-size reports UTF-8 stored bytes in one unit everywhere'); diff --git a/modules/test/test-virtual-fs-require-leaf.js b/modules/test/test-virtual-fs-require-leaf.js new file mode 100644 index 0000000..7b62d0e --- /dev/null +++ b/modules/test/test-virtual-fs-require-leaf.js @@ -0,0 +1,160 @@ +// node modules/test/test-virtual-fs-require-leaf.js +// Proves the 2026-11-09 virtual-fs patch: phantom deletes are refused (_requireLeaf) and +// checkpoints stop cloning _mods / .semantic_index (while rollback keeps the live copies). +// Offline: no browser, no real localStorage, no IndexedDB, no timers. +const fsMod = require('fs'), path = require('path'); +const ROOT = path.join(__dirname, '..', 'core'); + +// ── persistence stub: in-memory localStorage, IndexedDB that refuses to exist ── +const ls = {}; +global.localStorage = { getItem: k => (k in ls ? ls[k] : null), setItem: (k, v) => { ls[k] = String(v); }, removeItem: k => { delete ls[k]; } }; +global.indexedDB = { open() { throw new Error('no idb in tests'); }, deleteDatabase() { throw new Error('no idb in tests'); } }; +global.window = global; + +// The file ends with `window.VirtualFileSystem = VirtualFileSystem;` — so the class is a +// window global *and* is in scope at the bottom of the script. Return the real class object. +function load(file, cls) { + const src = fsMod.readFileSync(path.join(ROOT, file), 'utf8'); + return new Function(src + '\nreturn ' + cls + ';')(); +} +const VFS = load('virtual-fs.js', 'VirtualFileSystem'); + +let fails = 0, passes = 0; +function ok(name, cond, detail) { + if (cond) { passes++; return; } + fails++; + console.log('FAIL ' + name + (detail ? ' — ' + detail : '')); +} +function mk() { + const vfs = new VFS(); + vfs.saveToStorage = () => true; // keep the 2s setTimeout saver out of the test + vfs.root = { type: 'dir', content: {} }; + vfs.currentPath = []; + if (vfs._ready && typeof vfs._ready.catch === 'function') vfs._ready.catch(() => {}); + return vfs; +} +const keysOf = node => Object.keys(node.content).sort(); + +// ── 0. export shape ──────────────────────────────────────────────────────────── +ok('0 export shape: window.VirtualFileSystem is the class', typeof VFS === 'function' && global.window.VirtualFileSystem === VFS); +ok('0 constructor name', VFS.name === 'VirtualFileSystem', VFS.name); +ok('0 has _requireLeaf guard', typeof VFS.prototype._requireLeaf === 'function'); + +// ── A. deleteFile of a missing leaf must be refused, sibling untouched ───────── +{ + const vfs = mk(); + vfs.mkdirRecursive('/docs'); + vfs.writeFile('/docs/real.txt', 'keep me'); + ok('A1 deleteFile("/docs/ghost.txt") returns false', vfs.deleteFile('/docs/ghost.txt') === false); + ok('A2 sibling survived', vfs.readFile('/docs/real.txt') === 'keep me'); + ok('A3 nothing was trashed', !vfs.root.content['.trash'] || keysOf(vfs.root.content['.trash']).length === 0); + ok('A4 deleteFile("/no/such/dir/x.txt") returns false', vfs.deleteFile('/no/such/dir/x.txt') === false); + + // ── B. deleteFile of a real file removes exactly that file ──────────────── + vfs.writeFile('/docs/other.txt', 'other'); + ok('B1 deleteFile of a real file returns true', vfs.deleteFile('/docs/real.txt') === true); + ok('B2 the file is gone', vfs.readFile('/docs/real.txt') === null); + ok('B3 sibling intact', vfs.readFile('/docs/other.txt') === 'other'); + ok('B4 only that entry was removed', JSON.stringify(keysOf(vfs.root.content.docs)) === '["other.txt"]', JSON.stringify(keysOf(vfs.root.content.docs))); +} + +// ── C. deleteCheckpoint of a nonexistent checkpoint is refused ──────────────── +{ + const vfs = mk(); + vfs.mkdirRecursive('/_snapshots'); + vfs.writeFile('/_snapshots/checkpoint-keep.json', JSON.stringify({ createdAt: '2026-01-01T00:00:00.000Z' })); + ok('C1 deleteCheckpoint("nope") is falsy', !vfs.deleteCheckpoint('nope')); + ok('C2 _snapshots unchanged', JSON.stringify(keysOf(vfs.root.content['_snapshots'])) === '["checkpoint-keep.json"]', JSON.stringify(keysOf(vfs.root.content['_snapshots']))); + ok('C3 existing checkpoint still readable', !!vfs.fsReadJSON('/_snapshots/checkpoint-keep.json')); + ok('C4 real checkpoint delete still returns true', vfs.deleteCheckpoint('keep') === true); + ok('C5 it was actually removed', keysOf(vfs.root.content['_snapshots']).length === 0); +} + +// ── D. a checkpoint must not carry _mods or .semantic_index ─────────────────── +{ + const vfs = mk(); + vfs.mkdirRecursive('/_mods'); vfs.writeFile('/_mods/mod-a.js', 'MOD-ORIGINAL'); + vfs.mkdirRecursive('/.semantic_index'); vfs.writeFile('/.semantic_index/idx.json', '{"nodes":1}'); + vfs.writeFile('/notes.txt', 'plain'); + const name = vfs.checkpoint('excl'); + const snap = vfs.fsReadJSON('/_snapshots/checkpoint-' + name + '.json'); + ok('D1 checkpoint written', !!snap && !!snap.root); + ok('D2 _mods absent from the snapshot root', !('_mods' in snap.root.content), JSON.stringify(Object.keys(snap.root.content))); + ok('D3 .semantic_index absent from the snapshot root', !('.semantic_index' in snap.root.content)); + ok('D4 ordinary files are still snapshotted', snap.root.content['notes.txt'] && snap.root.content['notes.txt'].content === 'plain'); + ok('D5 scaffolding still excluded (_snapshots/_sessions/.trash)', + !('_snapshots' in snap.root.content) && !('_sessions' in snap.root.content) && !('.trash' in snap.root.content)); + + // ── E. rollback keeps the LIVE _mods / .semantic_index ──────────────────── + vfs.writeFile('/notes.txt', 'edited after checkpoint'); + ok('E0 pre-rollback edit in place', vfs.readFile('/notes.txt') === 'edited after checkpoint'); + ok('E1 rollback returns true', vfs.rollback(name) === true); + ok('E2 live _mods survives rollback', vfs.readFile('/_mods/mod-a.js') === 'MOD-ORIGINAL', String(vfs.readFile('/_mods/mod-a.js'))); + ok('E3 live .semantic_index survives rollback', vfs.readFile('/.semantic_index/idx.json') === '{"nodes":1}'); + ok('E4 ordinary file rolled back to the snapshot content', vfs.readFile('/notes.txt') === 'plain', String(vfs.readFile('/notes.txt'))); + ok('E5 snapshot bookkeeping survived rollback', keysOf(vfs.root.content['_snapshots']).indexOf('checkpoint-' + name + '.json') !== -1); +} + +// ── F. a legacy checkpoint that *does* embed _mods must not clobber the live one ── +{ + const vfs = mk(); + vfs.mkdirRecursive('/_mods'); vfs.writeFile('/_mods/live.js', 'LIVE'); + vfs.mkdirRecursive('/.semantic_index'); vfs.writeFile('/.semantic_index/live.json', '{"live":true}'); + vfs.writeFile('/f.txt', 'v1'); + const name = vfs.checkpoint('legacy'); + const p = '/_snapshots/checkpoint-' + name + '.json'; + const snap = vfs.fsReadJSON(p); // simulate a pre-patch 4MB snapshot + snap.root.content['_mods'] = { type: 'dir', content: { 'stale.js': { type: 'file', content: 'STALE' } } }; + snap.root.content['.semantic_index'] = { type: 'dir', content: { 'stale.json': { type: 'file', content: 'STALE' } } }; + vfs.writeFile(p, JSON.stringify(snap)); + ok('F1 rollback of a legacy snapshot succeeds', vfs.rollback(name) === true); + ok('F2 live _mods wins over the stale copy', vfs.readFile('/_mods/live.js') === 'LIVE' && vfs.readFile('/_mods/stale.js') === null); + ok('F3 live .semantic_index wins over the stale copy', + vfs.readFile('/.semantic_index/live.json') === '{"live":true}' && vfs.readFile('/.semantic_index/stale.json') === null); +} + +// ── G. plain checkpoint/restore round trip is unbroken ─────────────────────── +{ + const vfs = mk(); + vfs.mkdirRecursive('/work'); + vfs.writeFile('/work/a.txt', 'version-1'); + vfs.writeFile('/work/b.txt', 'b-1'); + const name = vfs.checkpoint('roundtrip'); + vfs.writeFile('/work/a.txt', 'version-2'); + vfs.deleteFile('/work/b.txt'); + vfs.writeFile('/work/new.txt', 'added-later'); + ok('G1 rollback returns true', vfs.rollback(name) === true); + ok('G2 modified file restored', vfs.readFile('/work/a.txt') === 'version-1', String(vfs.readFile('/work/a.txt'))); + ok('G3 deleted file restored', vfs.readFile('/work/b.txt') === 'b-1'); + ok('G4 post-checkpoint file removed by the rollback', vfs.readFile('/work/new.txt') === null); +} + +// ── H. measured saving on a realistic 2 MB _mods ──────────────────────────── +{ + const vfs = mk(); + vfs.mkdirRecursive('/_mods'); + vfs.writeFile('/_mods/bundle.js', 'x'.repeat(2 * 1024 * 1024)); // 2 MiB module originals + vfs.mkdirRecursive('/.semantic_index'); + vfs.writeFile('/.semantic_index/idx.json', JSON.stringify({ nodes: 'y'.repeat(4096) })); + vfs.writeFile('/doc.txt', 'hello'); + const name = vfs.checkpoint('big'); + const raw = vfs.readFile('/_snapshots/checkpoint-' + name + '.json'); + const after = raw.length; + // what the old code produced: the same snapshot with the two dirs cloned back in + const snap = JSON.parse(raw); + snap.root.content['_mods'] = JSON.parse(JSON.stringify(vfs.root.content['_mods'])); + snap.root.content['.semantic_index'] = JSON.parse(JSON.stringify(vfs.root.content['.semantic_index'])); + const before = JSON.stringify(snap).length; + console.log('MEASURE checkpoint bytes before exclusion: ' + before.toLocaleString('en-US')); + console.log('MEASURE checkpoint bytes after exclusion: ' + after.toLocaleString('en-US')); + console.log('MEASURE saved: ' + (before - after).toLocaleString('en-US') + ' bytes (' + (100 * (before - after) / before).toFixed(1) + '%)'); + ok('H1 excluded snapshot stays small (< 200 KB)', after < 200 * 1024, after + ' bytes'); + ok('H2 the old shape would have carried the 2 MiB _mods', before > 2 * 1024 * 1024, before + ' bytes'); +} + +if (fails === 0) { + console.log('test-virtual-fs-require-leaf: ' + passes + ' passed, 0 failed'); +} else { + console.log('test-virtual-fs-require-leaf: ' + passes + ' passed, ' + fails + ' FAILED'); + process.exit(1); +} |