aboutsummaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/course-daily-update.py138
-rw-r--r--scripts/course-insights.py57
-rw-r--r--scripts/course-validate.py95
-rw-r--r--scripts/gen-course-atlas.py113
-rw-r--r--scripts/newos4-maintain.py118
-rw-r--r--scripts/test-course.mjs59
6 files changed, 580 insertions, 0 deletions
diff --git a/scripts/course-daily-update.py b/scripts/course-daily-update.py
new file mode 100644
index 0000000..4f509d6
--- /dev/null
+++ b/scripts/course-daily-update.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Daily newOS4 course expansion. Grounded only: every command must exist in atlas.json."""
+import json, os, re, sys, subprocess
+from datetime import datetime, timezone
+import requests
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+COURSE = os.path.join(ROOT, 'course')
+ATLAS, FEED, FAQ = (os.path.join(COURSE, n) for n in ('atlas.json', 'feed.json', 'faq.json'))
+STATE = os.path.join(COURSE, '_state.json'); CHANGELOG = os.path.join(COURSE, 'changelog.jsonl')
+PENDING = os.path.join(COURSE, 'pending')
+SEARCH = 'https://webx11.duckdns.org/api/search'
+OLLAMA = 'http://127.0.0.1:11434/v1/chat/completions'
+MODEL = 'deepseek-v4.1-flash:cloud'
+TOPICS = ['terminal UX best practices command design', 'browser virtual filesystem design',
+ 'module lazy loading cache busting web apps', 'Ollama local LLM tool calling best practices',
+ 'WebDAV client sync reliability', 'developer onboarding interactive tutorial design',
+ 'AI agent tool safety confirmation patterns', 'cron automation in single page apps',
+ 'privacy preserving product analytics', 'prompt engineering grounded answers no hallucination',
+ 'web terminal accessibility keyboard UX', 'self-hosted AI assistant cost control',
+ 'spaced repetition micro learning programming', 'secure API key handling in browser apps']
+SYS_DRAFT = ('You write ONE micro-lesson for the newOS4 terminal course. Return STRICT JSON only (no fences):\n'
+ '{"title":"...","minutes":4,"blocks":[{"type":"text|code|cmd|tip|warn","text":"..."}],'
+ '"exercise":{"instruction":"...","accept":["cmd ..."],"check":{"type":"manual"},"hint":"..."},"sources":["https://..."]}\n'
+ 'Rules: 3-5 blocks; at least one cmd block; every command MUST appear verbatim in the ALLOWED list; '
+ 'only state facts present in the research snippets; sources must be URLs from the snippets.')
+
+def load(p, d):
+ try:
+ with open(p, encoding='utf-8') as f: return json.load(f)
+ except Exception: return d
+
+def save(p, obj):
+ tmp = p + '.tmp'
+ with open(tmp, 'w', encoding='utf-8') as f: json.dump(obj, f, ensure_ascii=False, indent=1)
+ os.replace(tmp, p); os.chmod(p, 0o644)
+
+def parse_json(txt):
+ txt = (txt or '').strip()
+ if txt.startswith('`' * 3): txt = txt.split('\n', 1)[1] if '\n' in txt else ''
+ txt = txt.strip()
+ if txt.endswith('`' * 3): txt = txt[:-3].strip()
+ m = re.search(r'\{[\s\S]*\}', txt)
+ try: return json.loads(m.group(0) if m else txt)
+ except Exception: return None
+
+def llm(system, user, n=900):
+ try:
+ r = requests.post(OLLAMA, json={'model': MODEL, 'stream': False, 'temperature': 0.4, 'max_tokens': n,
+ 'messages': [{'role': 'system', 'content': system}, {'role': 'user', 'content': user}]}, timeout=180)
+ if not r.ok: return None
+ return (r.json().get('choices') or [{}])[0].get('message', {}).get('content')
+ except Exception:
+ return None
+
+def research(q):
+ try:
+ r = requests.get(SEARCH, params={'q': q}, timeout=25)
+ if not r.ok: return []
+ return [{'title': (x.get('title') or '')[:160], 'url': x['url'], 'snippet': (x.get('content') or '')[:600]}
+ for x in (r.json().get('results') or [])[:4] if x.get('url')]
+ except Exception:
+ return []
+
+def validate_item(it, known, urls):
+ if not isinstance(it, dict): return False, 'not object'
+ if not (10 <= len(it.get('title') or '') <= 90): return False, 'title'
+ blocks = it.get('blocks') or []
+ if not (3 <= len(blocks) <= 6): return False, 'block count'
+ if any(b.get('type') not in ('text', 'code', 'cmd', 'tip', 'warn', 'table') or not (b.get('text') or '').strip() for b in blocks): return False, 'block'
+ if not any(b.get('type') == 'cmd' for b in blocks): return False, 'no cmd'
+ ex = it.get('exercise') or {}
+ if not ex.get('instruction') or not ex.get('accept'): return False, 'exercise'
+ for a in ex['accept']:
+ if a.split(' ')[0] not in known: return False, 'unknown command ' + a.split(' ')[0]
+ if not any(str(s) in urls for s in (it.get('sources') or []) if str(s).startswith('http')): return False, 'no source'
+ return True, ''
+
+def main():
+ dry = '--dry-run' in sys.argv
+ now = datetime.now(timezone.utc); day = now.strftime('%Y-%m-%d')
+ state = load(STATE, {'seen_commands': [], 'topic_index': 0, 'last_run': None})
+ if not dry: subprocess.run([sys.executable, os.path.join(ROOT, 'scripts', 'gen-course-atlas.py')], check=True)
+ atlas = load(ATLAS, {}); known = set(atlas.get('commands', {}))
+ seen = set(state.get('seen_commands') or known)
+ new = sorted(known - seen) if state.get('seen_commands') else []
+ feed = load(FEED, {'version': '', 'items': []}); added = []; rejected = []
+ if new:
+ feed['items'].insert(0, {'id': 'new-' + day, 'date': day, 'title': f'{len(new)} new command(s) in newOS4', 'minutes': 2,
+ 'generated': False, 'sources': ['atlas.json'],
+ 'blocks': [{'type': 'text', 'text': 'Run `help <cmd>` for details.'}, {'type': 'cmd', 'text': '\n'.join(new[:12])}]})
+ added.append('new-' + day)
+ if not any(i.get('id') == 'daily-' + day for i in feed['items']):
+ topic = TOPICS[state.get('topic_index', 0) % len(TOPICS)]
+ results = research(topic)
+ if results:
+ snippets = '\n'.join(f"- {r['title']} | {r['url']}\n {r['snippet']}" for r in results)
+ draft = parse_json(llm(SYS_DRAFT, f'TOPIC: {topic}\n\nSNIPPETS:\n{snippets}\n\nALLOWED COMMANDS:\n' + ', '.join(sorted(known))))
+ ok, why = validate_item(draft, known, {r['url'] for r in results}) if draft else (False, 'no JSON')
+ if ok:
+ draft.update({'id': 'daily-' + day, 'date': day, 'generated': True})
+ feed['items'].insert(0, draft); added.append('daily-' + day)
+ else:
+ os.makedirs(PENDING, exist_ok=True)
+ save(os.path.join(PENDING, f'daily-{day}.json'), {'draft': draft, 'why': why, 'topic': topic, 'results': results})
+ rejected.append('daily: ' + why)
+ else:
+ rejected.append('daily: no research results')
+ state['topic_index'] = state.get('topic_index', 0) + 1
+ feed['items'] = feed['items'][:60]; feed['version'] = day
+ if not dry: save(FEED, feed)
+ if not dry: subprocess.run([sys.executable, os.path.join(ROOT, 'scripts', 'course-insights.py')], check=True)
+ insights = load(os.path.join(COURSE, 'insights.json'), {})
+ faq = load(FAQ, {'items': []}); have = {f.get('q') for f in faq['items']}; faq_n = 0
+ for q in (insights.get('top_questions') or [])[:5]:
+ if q['q'] in have: continue
+ draft = parse_json(llm('Answer a repeated newOS4 question. Return STRICT JSON only: {"a":"<=400 chars","commands":["cmd"],"sources":["atlas.json#cmd"]}. Only use ALLOWED commands.', f'QUESTION: {q["q"]}\nALLOWED COMMANDS:\n' + ', '.join(sorted(known)), 500))
+ if draft and draft.get('a') and all(c in known for c in draft.get('commands', [])):
+ faq['items'].append({'q': q['q'], 'a': draft['a'][:400], 'commands': draft['commands'],
+ 'sources': draft.get('sources') or ['insights.json'], 'added': day})
+ have.add(q['q']); faq_n += 1
+ else: rejected.append('faq: ' + q['q'][:60])
+ if not dry: save(FAQ, faq)
+ state.update({'seen_commands': sorted(known), 'last_run': now.strftime('%Y-%m-%dT%H:%M:%SZ')})
+ if not dry:
+ save(STATE, state)
+ with open(CHANGELOG, 'a', encoding='utf-8') as f:
+ f.write(json.dumps({'date': day, 'added': added, 'faq': faq_n, 'rejected': rejected,
+ 'atlas': f'{len(known)} commands'}, ensure_ascii=False) + '\n')
+ print('newOS4 Course daily — ' + day)
+ print('atlas: ' + str(len(known)) + ' commands | new: ' + str(len(new)))
+ print('daily lesson: ' + next((i['title'] for i in feed['items'] if i.get('id') == 'daily-' + day), 'none added'))
+ print('faq added: ' + str(faq_n) + ' | feed: ' + str(len(feed['items'])) + '/60')
+ print('rejected: ' + ('; '.join(rejected) if rejected else 'none'))
+ print('needs OK: none')
+
+if __name__ == '__main__':
+ sys.exit(main() or 0)
diff --git a/scripts/course-insights.py b/scripts/course-insights.py
new file mode 100644
index 0000000..0ee891e
--- /dev/null
+++ b/scripts/course-insights.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Aggregate course events -> course/insights.json (counts + capped question text, no sids/IPs)."""
+import json, os, glob, re
+from collections import Counter
+from datetime import datetime, timezone
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DATA = os.path.join(ROOT, 'data', 'course')
+OUT = os.path.join(ROOT, 'course', 'insights.json')
+
+def norm_q(q):
+ q = re.sub(r'https?://\S+', '', q or '')
+ return re.sub(r'\s+', ' ', q).strip().lower()[:200]
+
+def main():
+ opens, completes, fails, cmds, questions = Counter(), Counter(), Counter(), Counter(), Counter()
+ projects = Counter()
+ files = sorted(glob.glob(os.path.join(DATA, 'events-*.jsonl')))[-2:]
+ for path in files:
+ with open(path, encoding='utf-8', errors='ignore') as f:
+ for line in f:
+ try: e = json.loads(line)
+ except Exception: continue
+ t, d = e.get('type'), e.get('detail') or {}
+ lesson = (d.get('lesson') or '')[:40]
+ if t == 'lesson_open' and lesson: opens[lesson] += 1
+ elif t == 'lesson_complete' and lesson: completes[lesson] += 1
+ elif t == 'exercise_fail':
+ if lesson: fails[lesson] += 1
+ c = (d.get('command') or '').strip().split(' ')[0][:40]
+ if c: cmds[c] += 1
+ elif t == 'question':
+ q = norm_q(d.get('q'))
+ if len(q) >= 8: questions[q] += 1
+ elif t in ('project_open', 'project_view'):
+ p = (d.get('project') or '')[:40]
+ if p: projects[p] += 1
+ lessons = {}
+ for lid in set(opens) | set(completes):
+ o, c = opens[lid], completes[lid]
+ lessons[lid] = {'opens': o, 'completes': c, 'failures': fails.get(lid, 0),
+ 'completion_rate': round(c / o, 2) if o else 0.0}
+ payload = {'generated': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+ 'window_files': [os.path.basename(p) for p in files],
+ 'lessons': dict(sorted(lessons.items())),
+ 'top_failed_commands': [{'cmd': c, 'n': n} for c, n in cmds.most_common(20)],
+ 'top_questions': [{'q': q, 'n': n} for q, n in questions.most_common(20) if n >= 2],
+ 'projects': dict(projects),
+ 'totals': {'opens': sum(opens.values()), 'completes': sum(completes.values()), 'failures': sum(fails.values())}}
+ os.makedirs(os.path.dirname(OUT), exist_ok=True)
+ tmp = OUT + '.tmp'
+ with open(tmp, 'w', encoding='utf-8') as f: json.dump(payload, f, ensure_ascii=False, indent=1)
+ os.replace(tmp, OUT); os.chmod(OUT, 0o644)
+ print(json.dumps({'lessons': len(lessons), 'questions': len(payload['top_questions']), 'totals': payload['totals']}))
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/course-validate.py b/scripts/course-validate.py
new file mode 100644
index 0000000..9b5dcf4
--- /dev/null
+++ b/scripts/course-validate.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Validate newOS4 course content. Exit 0 = publishable, 1 = errors printed."""
+import json, os, glob, sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+COURSE = os.path.join(ROOT, 'course')
+ATLAS = os.path.join(COURSE, 'atlas.json')
+errors, warnings = [], []
+
+def load(p):
+ try:
+ with open(p, encoding='utf-8') as f:
+ return json.load(f)
+ except Exception as e:
+ errors.append(f'{os.path.relpath(p, ROOT)}: invalid JSON: {e}')
+ return None
+
+def main():
+ manifest = load(os.path.join(COURSE, 'course.json'))
+ if not manifest:
+ return finish()
+ ids, lesson_ids = set(), set()
+ for ch in manifest.get('chapters', []):
+ cid = ch.get('id')
+ if not cid:
+ errors.append('course.json: chapter missing id'); continue
+ if cid in ids:
+ errors.append(f'course.json: duplicate chapter id {cid}')
+ ids.add(cid)
+ path = os.path.join(COURSE, 'chapters', cid + '.json')
+ if not os.path.exists(path):
+ errors.append(f'missing chapter file {cid}.json'); continue
+ chd = load(path)
+ if not chd: continue
+ man_ids = [x.get('id') for x in ch.get('lessons', [])]
+ file_ids = [x.get('id') for x in chd.get('lessons', [])]
+ if man_ids != file_ids:
+ errors.append(f'{cid}: manifest lessons {man_ids} != chapter file lessons {file_ids}')
+ for l in chd.get('lessons', []):
+ lid = l.get('id')
+ if not lid:
+ errors.append(f'{cid}: lesson missing id'); continue
+ if lid in lesson_ids:
+ errors.append(f'duplicate lesson id {lid}')
+ lesson_ids.add(lid)
+ for k in ('title', 'objective'):
+ if not l.get(k): errors.append(f'{lid}: missing {k}')
+ mins = l.get('minutes', 0)
+ if not isinstance(mins, int) or isinstance(mins, bool) or mins < 1:
+ errors.append(f'{lid}: minutes must be >=1')
+ blocks = l.get('blocks') or []
+ if len(blocks) < 3: errors.append(f'{lid}: needs >=3 blocks')
+ for b in blocks:
+ if b.get('type') not in ('text', 'code', 'cmd', 'tip', 'warn', 'table'):
+ errors.append(f'{lid}: bad block type {b.get("type")!r}')
+ ex = l.get('exercise')
+ if ex:
+ if not ex.get('instruction') or not ex.get('accept'):
+ errors.append(f'{lid}: exercise needs instruction+accept')
+ chk = ex.get('check', {})
+ if chk.get('type') not in ('command', 'file', 'output', 'manual'):
+ errors.append(f'{lid}: exercise check.type invalid')
+ if chk.get('type') == 'file' and not chk.get('path'):
+ errors.append(f'{lid}: file check requires path')
+ if chk.get('type') == 'output' and not chk.get('contains'):
+ errors.append(f'{lid}: output check requires contains')
+ if not ex.get('hint'): warnings.append(f'{lid}: exercise without hint')
+ if l.get('generated') and not l.get('sources'):
+ errors.append(f'{lid}: generated lesson needs sources')
+ if os.path.exists(ATLAS):
+ atlas = load(ATLAS) or {}
+ known = set(atlas.get('commands', {}))
+ for f in glob.glob(os.path.join(COURSE, 'chapters', '*.json')):
+ chd = load(f) or {}
+ for l in chd.get('lessons', []):
+ for c in l.get('commands', []):
+ if c not in known:
+ errors.append(f'{l.get("id")}: unknown command {c}')
+ for name, key in (('feed.json', 'items'), ('faq.json', 'items')):
+ p = os.path.join(COURSE, name)
+ if not os.path.exists(p): continue
+ d = load(p) or {}
+ for it in d.get(key, []):
+ if isinstance(it, dict) and it.get('date') is None:
+ warnings.append(f'{name}: item without date')
+ return finish()
+
+def finish():
+ for w in warnings: print(f'WARN {w}')
+ for e in errors: print(f'ERROR {e}')
+ print(f'course-validate: {len(errors)} error(s), {len(warnings)} warning(s)')
+ return 1 if errors else 0
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/gen-course-atlas.py b/scripts/gen-course-atlas.py
new file mode 100644
index 0000000..559fd4c
--- /dev/null
+++ b/scripts/gen-course-atlas.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Build course/atlas.json from newOS4 source. Ground truth for every lesson claim."""
+import json, os, re, glob, hashlib
+from datetime import datetime, timezone
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+OUT = os.path.join(ROOT, 'course', 'atlas.json')
+CORE_DIR, EXT_DIR = os.path.join(ROOT, 'modules', 'core'), os.path.join(ROOT, 'modules', 'ext')
+SKIP = {'core-bundle.js', 'critical-ext-bundle.js', 'sandbox-worker.js'}
+
+CATEGORY = [
+ ('module-fs', 'filesystem'), ('virtual-fs', 'filesystem'), ('module-aiassist', 'ai'), ('module-ollama', 'ai'),
+ ('module-memory-graph', 'ai'), ('module-smartctx', 'ai'), ('module-neural-vfs', 'ai'), ('module-world-model', 'ai'),
+ ('module-sandbox', 'ai'), ('module-searxng', 'web'), ('module-fetcher', 'web'), ('webdav', 'cloud'),
+ ('editor', 'editor'), ('filemgr', 'filesystem'), ('finder', 'filesystem'), ('diff', 'editor'),
+ ('cron', 'automation'), ('kanban', 'automation'), ('shell', 'shell'), ('busybox', 'shell'),
+ ('theme', 'ui'), ('floating', 'ui'), ('voice', 'ui'), ('three', 'ui'), ('console', 'dev'), ('debug-panel', 'dev'),
+ ('wpm', 'system'), ('tutorial', 'learning'), ('skill-pack-core', 'ai'), ('cryptogzip', 'system'),
+ ('module-agent-registry', 'agents'), ('module-orchestrator', 'agents'), ('module-leases', 'agents'),
+ ('module-chatstudio', 'ai'), ('module-servermon', 'system'), ('module-config', 'system'), ('module-system', 'system'),
+ ('course', 'learning'),
+]
+
+def category_for(filename, cmd):
+ for frag, cat in CATEGORY:
+ if frag in filename: return cat
+ if cmd.startswith('webdav'): return 'cloud'
+ if cmd.startswith('cron') or cmd.startswith('kanban'): return 'automation'
+ if cmd.startswith('ai-') or cmd in ('ai', 'ask'): return 'ai'
+ return 'general'
+
+def tier_lists():
+ rebuild = open(os.path.join(ROOT, 'scripts', 'rebuild-bundles.py')).read()
+ def listed(name):
+ m = re.search(name + r"\s*=\s*\[([^\]]+)\]", rebuild, re.S)
+ return set(re.findall(r"([a-z0-9-]+\.js)", m.group(1))) if m else set()
+ loader = open(os.path.join(ROOT, 'modules', 'core', 'module-loader.js')).read()
+ m = re.search(r'extKeys\s*=\s*\[([^\]]+)\]', loader)
+ ext = set(re.findall(r"'([a-z0-9-]+)'", m.group(1))) if m else set()
+ return listed('CORE_FILES'), listed('CRITICAL_EXT_FILES'), ext
+
+CORE_FILES, CRITICAL_EXT_FILES, EXT_KEYS = tier_lists()
+
+def module_tier(filename):
+ base = os.path.basename(filename)
+ if base in CRITICAL_EXT_FILES: return 'critical-ext-bundle'
+ if base in CORE_FILES: return 'core-bundle'
+ if base[:-3] in EXT_KEYS: return 'ext-auto'
+ return 'ext-manual'
+
+def desc_map(text):
+ out = {}
+ for m in re.finditer(r"'([a-z0-9][a-z0-9_-]{1,40})'\s*:\s*'((?:[^'\\]|\\.){8,300})'", text):
+ cmd, desc = m.group(1), m.group(2)
+ if len(desc) < 10 or ' ' not in desc or desc.endswith('.js'):
+ continue
+ out.setdefault(cmd, desc.replace("\\'", "'"))
+ return out
+
+def main():
+ commands, modules = {}, {}
+ files = sorted(glob.glob(os.path.join(CORE_DIR, '*.js')) + glob.glob(os.path.join(EXT_DIR, '*.js')))
+ for path in files:
+ base = os.path.basename(path)
+ if base in SKIP: continue
+ text = open(path, encoding='utf-8', errors='ignore').read()
+ cmds = sorted(set(re.findall(r"commands\['([A-Za-z0-9_-]+)'\]\s*=", text)))
+ if not cmds: continue
+ modname = base[:-3]
+ modules[modname] = {'file': os.path.relpath(path, ROOT), 'tier': module_tier(path),
+ 'commands': cmds, 'lines': text.count('\n') + 1}
+ dm = desc_map(text)
+ for c in cmds:
+ commands[c] = {'module': modname, 'file': os.path.relpath(path, ROOT), 'tier': module_tier(path),
+ 'category': category_for(base, c), 'desc': dm.get(c, ''),
+ 'builtin': False, 'example': c}
+ term = open(os.path.join(ROOT, 'modules', 'core', 'terminal.js'), encoding='utf-8').read()
+ for c in sorted(set(re.findall(r"cmd === '([a-z0-9-]+)'", term))):
+ if c in commands: continue
+ commands[c] = {'module': 'terminal', 'file': 'modules/core/terminal.js', 'tier': 'core-bundle',
+ 'category': 'system', 'desc': '', 'builtin': True, 'example': c}
+ idx_path = os.path.join(ROOT, 'modules-index.json')
+ mod_by_cmd = {}
+ if os.path.exists(idx_path):
+ idx = json.load(open(idx_path))
+ for m in idx.get('modules', {}).values():
+ for c in m.get('commands', []):
+ mod_by_cmd.setdefault(c, m)
+ for c, e in commands.items():
+ if not e['desc']:
+ m = mod_by_cmd.get(c) or {}
+ e['desc'] = (m.get('description') or f'{c} — command from {e["module"]}')[:300]
+ payload = {
+ 'version': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+ 'source_hash': hashlib.sha256(''.join(sorted(commands)).encode()).hexdigest()[:16],
+ 'commands': dict(sorted(commands.items())),
+ 'modules': dict(sorted(modules.items())),
+ 'coverage': {'commands': len(commands),
+ 'described': sum(1 for e in commands.values() if e['desc']),
+ 'modules': len(modules),
+ 'tiers': {t: sum(1 for e in commands.values() if e['tier'] == t)
+ for t in ('core-bundle', 'critical-ext-bundle', 'ext-auto', 'ext-manual')}},
+ }
+ os.makedirs(os.path.dirname(OUT), exist_ok=True)
+ tmp = OUT + '.tmp'
+ with open(tmp, 'w', encoding='utf-8') as f:
+ json.dump(payload, f, ensure_ascii=False, indent=1)
+ os.replace(tmp, OUT)
+ os.chmod(OUT, 0o644)
+ print(json.dumps(payload['coverage'], ensure_ascii=False))
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/newos4-maintain.py b/scripts/newos4-maintain.py
new file mode 100644
index 0000000..6321a3f
--- /dev/null
+++ b/scripts/newos4-maintain.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""newOS4 maintenance: drift, bundles, syntax, perms, data hygiene, live health.
+Safe fixes only with --fix. Never touches nginx, services, sudo, or user content."""
+import glob, json, os, re, subprocess, sys
+from datetime import datetime, timezone
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+WEB = 'https://webx11.duckdns.org/spaces/newos4'
+FIX = '--fix' in sys.argv
+DEEP = '--deep' in sys.argv
+issues = []
+def add(l, m): issues.append((l, m))
+def run(cmd, **kw): return subprocess.run(cmd, capture_output=True, text=True, **kw)
+def sh(cmd): return run(cmd, shell=True).stdout.strip()
+
+def check_versions():
+ html = open(os.path.join(ROOT, 'main.html'), encoding='utf-8').read()
+ loader = open(os.path.join(ROOT, 'modules/core/module-loader.js'), encoding='utf-8').read()
+ m = re.search(r"const LATEST_VER\s*=\s*'([^']+)'", loader)
+ ver = m.group(1) if m else '?'
+ n = html.count(ver)
+ add('PASS' if n == 3 else 'FAIL', f'version {ver} x{n} in main.html (expect 3)')
+
+def check_bundles():
+ paths = ('modules/core/core-bundle.js', 'modules/ext/critical-ext-bundle.js')
+ before = {p: sh(f'md5sum {p}') for p in paths}
+ if not FIX:
+ add('PASS', 'bundle check skipped (use --fix)'); return
+ r = run([sys.executable, 'scripts/rebuild-bundles.py'], cwd=ROOT)
+ after = {p: sh(f'md5sum {p}') for p in paths}
+ changed = [p for p in paths if before[p] != after[p]]
+ add('FAIL' if r.returncode else ('WARN' if changed else 'PASS'),
+ 'bundles ' + ('FAILED' if r.returncode else ('rebuilt: ' + ', '.join(changed) if changed else 'clean')))
+
+def check_syntax():
+ bad = []
+ for p in glob.glob(os.path.join(ROOT, 'modules/core/*.js')) + glob.glob(os.path.join(ROOT, 'modules/ext/*.js')):
+ if os.path.basename(p) in ('core-bundle.js', 'critical-ext-bundle.js'): continue
+ if run(['node', '--check', p]).returncode: bad.append(os.path.basename(p))
+ add('PASS' if not bad else 'FAIL', 'syntax ' + ('OK' if not bad else 'fail: ' + ', '.join(bad)))
+
+def check_perms():
+ targets = glob.glob(os.path.join(ROOT, 'course/*.json')) + glob.glob(os.path.join(ROOT, 'course/chapters/*.json')) \
+ + [os.path.join(ROOT, 'modules/ext/course.js'), os.path.join(ROOT, 'course.html'), os.path.join(ROOT, 'api/course/event.php')]
+ bad = []
+ for p in targets:
+ if not os.path.exists(p): continue
+ if (os.stat(p).st_mode & 0o777) != 0o644 or sh(f'stat -c %G {p}') != 'www-data':
+ if FIX:
+ os.chmod(p, 0o644); sh(f'chgrp www-data {p}')
+ else: bad.append(os.path.basename(p))
+ add('PASS' if not bad else ('WARN' if FIX else 'FAIL'), 'perms ' + ('clean' if not bad else ('fixed' if FIX else 'bad: ' + ', '.join(bad[:5]))))
+
+def check_index():
+ idxp = os.path.join(ROOT, 'modules-index.json')
+ atlas = json.load(open(os.path.join(ROOT, 'course/atlas.json')))
+ idx = json.load(open(idxp))
+ cur = len(set(c for m in idx.get('modules', {}).values() for c in m.get('commands', [])))
+ if cur == atlas['coverage']['commands']:
+ add('PASS', f'modules-index {cur} commands'); return
+ add('WARN', f'modules-index stale {cur} vs {atlas["coverage"]["commands"]}')
+ if not FIX: return
+ for name, m in atlas['modules'].items():
+ e = idx['modules'].get(name, {'name': name, 'file': m['file'], 'type': 'core' if m['tier'] == 'core-bundle' else 'ext',
+ 'class': '', 'auto_load': m['tier'] != 'ext-manual', 'description': ''})
+ e.update({'commands': m['commands'], 'file': m['file'], 'tier': m['tier']})
+ idx['modules'][name] = e
+ idx['version'] = datetime.now(timezone.utc).strftime('%Y-%m-%d') + '-atlas'
+ tmp = idxp + '.tmp'
+ with open(tmp, 'w', encoding='utf-8') as f: json.dump(idx, f, ensure_ascii=False, indent=1)
+ os.replace(tmp, idxp); os.chmod(idxp, 0o644); sh(f'chgrp www-data {idxp}')
+ add('PASS', 'modules-index regenerated')
+
+def check_data():
+ err = os.path.join(ROOT, 'data/debug/errors.json')
+ if os.path.exists(err) and os.path.getsize(err) > 5 * 1024 * 1024:
+ if FIX:
+ arc = os.path.join(ROOT, 'data/debug/archive'); os.makedirs(arc, exist_ok=True)
+ os.replace(err, os.path.join(arc, 'errors-' + datetime.now().strftime('%Y%m') + '.json'))
+ open(err, 'w').write('[]'); os.chmod(err, 0o664); sh(f'chgrp www-data {err}')
+ add('PASS', 'errors.json rotated')
+ else: add('WARN', 'errors.json >5MB')
+ else: add('PASS', 'debug data OK')
+
+def check_live():
+ for url in (f'{WEB}/main.html', f'{WEB}/course.html', f'{WEB}/course/course.json', f'{WEB}/modules/core/core-bundle.js'):
+ code = sh(f"curl -s -o /dev/null -w '%{{http_code}}' '{url}'")
+ add('PASS' if code == '200' else 'FAIL', f'HTTP {code} {url.replace(WEB, "")}')
+
+def check_course():
+ r = run([sys.executable, 'scripts/course-validate.py'], cwd=ROOT)
+ add('PASS' if r.returncode == 0 else 'FAIL', 'course ' + ('valid' if r.returncode == 0 else 'INVALID: ' + r.stdout[-160:]))
+
+def check_links():
+ if not DEEP: return
+ broken = []
+ for page in ('main.html', 'course.html', 'help.html', 'modules.html', 'guide.html'):
+ p = os.path.join(ROOT, page)
+ if not os.path.exists(p): continue
+ for href in re.findall(r'(?:href|src)="([^"#]+)"', open(p, encoding='utf-8').read()):
+ if any(x in href for x in ("'", '+', '{', '}')): continue
+ if href.startswith(('http', '//', 'mailto:', 'javascript:', 'data:', '/api/')): continue
+ target = os.path.normpath(os.path.join(ROOT, href.split('?')[0]))
+ if not os.path.exists(target): broken.append(f'{page}:{href}')
+ add('PASS' if not broken else 'WARN', 'links ' + ('OK' if not broken else f'{len(broken)} broken (first: {broken[0]})'))
+
+def main():
+ os.chdir(ROOT)
+ check_versions(); check_bundles(); check_syntax(); check_perms(); check_index(); check_data(); check_live(); check_course(); check_links()
+ fails = [m for l, m in issues if l == 'FAIL']
+ print('newOS4 maintenance — ' + datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%MZ') + (' [fix]' if FIX else '') + (' [deep]' if DEEP else ''))
+ for l, m in issues: print(f'{l}: {m}')
+ print('maintenance result: ' + ('FAIL' if fails else 'OK'))
+ print('needs OK: ' + ('none' if not fails else '; '.join(fails)[:400]))
+ return 1 if fails else 0
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/test-course.mjs b/scripts/test-course.mjs
new file mode 100644
index 0000000..7a64dc8
--- /dev/null
+++ b/scripts/test-course.mjs
@@ -0,0 +1,59 @@
+import fs from 'fs';
+import vm from 'vm';
+const SRC = '/var/www/webx11.duckdns.org/spaces/newos4/modules/ext/course.js';
+const MANIFEST = { version: 'test-1', updated: '2026-10-18', chapters: [
+ { id: '01-orientation', title: 'Orientation', lessons: [{ id: '1.1', title: 'T', objective: 'O', minutes: 5 }] }
+], projects: ['p1'] };
+const CHAPTER = { id: '01-orientation', title: 'Orientation', lessons: [{ id: '1.1', title: 'T', objective: 'O', minutes: 5,
+ blocks: [{ type: 'text', text: 'hello' }, { type: 'cmd', text: 'status' }, { type: 'tip', text: 'tip' }],
+ exercise: { instruction: 'touch ok.txt', accept: ['touch ok.txt'], check: { type: 'file', path: '/ok.txt' }, hint: 'touch ok.txt' } }] };
+let printed = [];
+const terminal = {
+ print(t) { printed.push(String(t)); },
+ fs: { _f: {}, readFile(p) { return Object.prototype.hasOwnProperty.call(this._f, p) ? this._f[p] : null; },
+ writeFile(p, c) { this._f[p] = c; }, mkdirRecursive() {}, fsReadJSON(p) { try { return JSON.parse(this._f[p] || 'null'); } catch (e) { return null; } },
+ fsWriteJSON(p, o) { this._f[p] = JSON.stringify(o); } },
+ _pushPrintCapture(cb) { this._cap = cb; }, _popPrintCapture() { this._cap && this._cap('output'); },
+ ollamaManager: { currentModel: 'test-model' }
+};
+const store = {};
+const sandbox = {
+ window: { webx11: { moduleManager: { registerModule() {} } } },
+ localStorage: { getItem: k => store[k] ?? null, setItem: (k, v) => { store[k] = String(v); }, removeItem: k => { delete store[k]; } },
+ fetch: async (url) => {
+ if (url.includes('course.json')) return { ok: true, json: async () => MANIFEST };
+ if (url.includes('atlas.json')) return { ok: true, json: async () => ({ commands: { status: { desc: 'Status', module: 'system', tier: 'core-bundle', category: 'system' }, touch: { desc: 'Touch', module: 'fs', tier: 'core-bundle', category: 'filesystem' }, ls: { desc: 'List', module: 'fs', tier: 'core-bundle', category: 'filesystem' } } }) };
+ if (url.includes('chapters/')) return { ok: true, json: async () => CHAPTER };
+ if (url.includes('event.php')) return { ok: true, json: async () => ({}) };
+ if (url.includes('/ai/')) return { ok: true, json: async () => ({ choices: [{ message: { content: 'answer' } }] }) };
+ return { ok: false, status: 404, json: async () => ({}) };
+ },
+ console, document: { body: {} }, setTimeout, clearTimeout, Date, Math, JSON, RegExp
+};
+sandbox.globalThis = sandbox;
+sandbox.webx11 = sandbox.window.webx11;
+vm.createContext(sandbox);
+vm.runInContext(fs.readFileSync(SRC, 'utf8') + '\n;globalThis.__CourseModule = CourseModule;', sandbox);
+const mod = new sandbox.__CourseModule();
+mod.terminal = terminal;
+let pass = 0, fail = 0;
+const check = (n, c, got) => { if (c) { pass++; console.log('PASS', n); } else { fail++; console.log('FAIL', n, got || ''); } };
+const run = async () => {
+ mod.registerCommands();
+ await mod._warm();
+ await mod.handle(['list']);
+ check('list mentions Orientation', printed.join('\n').includes('Orientation'));
+ await mod.handle(['open', '1.1']);
+ check('lesson printed', printed.join('\n').includes('Exercise'));
+ mod.onCommandStart('touch', ['ok.txt']); terminal.fs.writeFile('/ok.txt', 'x'); mod.onCommandExecuted('touch', ['ok.txt']);
+ check('file exercise completes', !!mod._progress().completed['1.1'], JSON.stringify(mod._progress()));
+ check('xp awarded', mod._progress().xp >= 10);
+ check('match exact', mod._match('ls', 'ls') && !mod._match('ls', 'ls -la'));
+ check('match wildcard', mod._match('touch *', 'touch a.txt'));
+ await mod.handle(['ask', 'what is status?']);
+ check('tutor answered', printed.join('\n').includes('answer'));
+ check('skill installed', !!terminal.fs._f['/skills/course-01-orientation.md']);
+ console.log(`RESULT ${pass} passed, ${fail} failed`);
+ process.exit(fail ? 1 : 0);
+};
+run().catch(e => { console.error(e); process.exit(1); });
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-06 16:37:05 +0000