aboutsummaryrefslogtreecommitdiff
path: root/scripts/newos4-skill-audit.py
diff options
context:
space:
mode:
authorAgando <roduccomm@gmail.com>2026-10-23 14:39:45 +0200
committerAgando <roduccomm@gmail.com>2026-10-23 14:39:45 +0200
commit940bf43ba5cbed2d0a9afa4c70ffbc3df5cb3f90 (patch)
treedba6efc7ea9b7a5e86910104866d494103bfbf9a /scripts/newos4-skill-audit.py
parent25833fb43a0ed3e9c7522dfb6a78f29dfd947720 (diff)
fix(newos4): unmeasured skills are untracked, world-model writes verified, skill audit
External review packet (os4-external-review-v4.6.51.html) triaged against the code: 2 findings confirmed and fixed, 4 could not be reproduced and were recorded as such. - module-memory-graph.js: getStaleSkills() treated a missing lastUsed as 999 days unused, so skill-audit recommended archiving every skill (22 of 22). Skills with no usage telemetry are now collected by getUntrackedSkills() and never counted stale. Live: "Stale: 0" + "Untracked: 22 (no usage telemetry — NOT stale)". - module-world-model.js: _saveToVFS() ignored fsWriteJSON()'s return value and promoted whatever a truthy read returned. Now write -> read-back -> byte compare -> promote, with every return checked. Regression test modules/test/test-world-model-save.js (red on the old code: promoted=true with errs=[]; green now). - scripts/newos4-skill-audit.py: 5 mechanical probes (version x3, read_file paging + context budget, selection symbols, required scripts, extKeys) + skill presence; exits non-zero on drift. 7/7 pass today. Weekly job cb6bc2a16216 runs it Mondays. Not reproduced (recorded in admin/docs/newos4-external-review-verification-2026-10-23.md): world_model_query empty-payload throw (live call returns JSON), pipes-as-command-names, ls skills empty (23 entries via fs.readDir), ~66 agent dirs (4 live).
Diffstat (limited to 'scripts/newos4-skill-audit.py')
-rwxr-xr-xscripts/newos4-skill-audit.py91
1 files changed, 91 insertions, 0 deletions
diff --git a/scripts/newos4-skill-audit.py b/scripts/newos4-skill-audit.py
new file mode 100755
index 0000000..c10274c
--- /dev/null
+++ b/scripts/newos4-skill-audit.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Audit newOS4 skill claims against the live tree. Exit 1 on any drift."""
+import json, os, re, subprocess, sys
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+SKILLS = os.path.expanduser('~/.hermes/profiles/webx11/skills/webx11')
+PAIRS = ['newos4-core-module-editing', 'newos4-module-patch-and-verify']
+
+
+def read(p):
+ try:
+ with open(p, encoding='utf-8') as f:
+ return f.read()
+ except Exception:
+ return ''
+
+
+def probe_version():
+ loader = read(os.path.join(ROOT, 'modules/core/module-loader.js'))
+ m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", loader)
+ if not m:
+ return False, 'LATEST_VER not found in module-loader.js'
+ ver = m.group(1)
+ n = read(os.path.join(ROOT, 'main.html')).count(ver)
+ return n == 3, f'{ver} appears {n}x in main.html (expect 3)'
+
+
+def probe_read_file():
+ src = read(os.path.join(ROOT, 'modules/core/module-aiassist.js'))
+ has_offset = 'offset: { type:' in src
+ has_next = 'next_offset' in src
+ m = re.search(r'maxContextTokens\s*=\s*(\d+)', src)
+ return (has_offset and has_next and bool(m)), \
+ f'offset={has_offset} next_offset={has_next} maxContextTokens={m.group(1) if m else None}'
+
+
+def probe_selection():
+ src = read(os.path.join(ROOT, 'modules/ext/filemgr.js'))
+ return ('_multiSelect' in src and '_getSelectedPaths' in src), \
+ f'_multiSelect={"_multiSelect" in src} _getSelectedPaths={"_getSelectedPaths" in src}'
+
+
+def probe_scripts():
+ need = ['scripts/rebuild-bundles.py', 'scripts/lint-no-undef.py',
+ 'scripts/newos4-maintain.py', 'scripts/newos4-triage.py']
+ missing = [p for p in need if not os.path.exists(os.path.join(ROOT, p))]
+ return not missing, f'missing={missing or "none"}'
+
+
+def probe_bundle_membership():
+ loader = read(os.path.join(ROOT, 'modules/core/module-loader.js'))
+ return "extKeys = [" in loader, 'extKeys literal present'
+
+
+PROBES = {
+ 'version': probe_version,
+ 'read_file_contract': probe_read_file,
+ 'selection_symbols': probe_selection,
+ 'scripts': probe_scripts,
+ 'ext_keys': probe_bundle_membership,
+}
+
+
+def main():
+ results, failed = {}, 0
+ for name, fn in PROBES.items():
+ try:
+ ok, detail = fn()
+ except Exception as e:
+ ok, detail = False, f'probe threw: {e}'
+ results[name] = {'ok': ok, 'detail': detail}
+ print(('PASS ' if ok else 'FAIL ') + name + ': ' + detail)
+ failed += 0 if ok else 1
+
+ for pair in PAIRS:
+ p = os.path.join(SKILLS, pair, 'SKILL.md')
+ results['skill_' + pair] = {'ok': os.path.exists(p), 'detail': p}
+ if not os.path.exists(p):
+ print('FAIL skill_' + pair + ': missing ' + p)
+ failed += 1
+
+ out = '/var/lib/webx11/newos4-maintenance/skill-audit.json'
+ os.makedirs(os.path.dirname(out), exist_ok=True)
+ with open(out, 'w', encoding='utf-8') as f:
+ json.dump(results, f, indent=2)
+ print(f'skill audit: {len(PROBES) - failed + len(PAIRS)}/{len(PROBES) + len(PAIRS)} pass')
+ return 1 if failed else 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())