feat(newos4): fix ledger + norm clustering — the loop now knows what a fix was worth

Lifetime counts ranked an 8-day-old outage above today's live crash, and nothing asserted that a
shipped fix actually held: a regression of a fixed bug looked exactly like a new bug.

- api/debug/ingest.php: norm_key() stores `norm` (host/port/scheme/`?v=`/`//` stripped) — the client
  hash covers the raw URL, so one outage became two signatures (`/ollama//api/tags` 18 203x and
  `/ollama/api/tags` 16 540x). PHP + Python implementations verified byte-identical on 5 real
  messages.
- scripts/newos4-ledger.py: data/maintenance/fixes.json maps norm -> {commit, fixed_at, note}, and
  verify() returns REGRESSED / WATCHING / CLOSED / UNSEEN. "Seen after the fix" alone is not a
  regression (a fix shipped 10d ago with one sighting 9d ago has been silent since) — regressed means
  seen after the fix AND recent; otherwise the silence wins.
- scripts/newos4-triage.py: clusters (grouped by norm, recency-ordered), active<24h / stale>7d counts,
  FIX| verdict lines; digest gains clusters + fixes and active/stale/cluster/regressed counters.
- api/debug/dashboard.php: Clusters / Active<24h / Stale>7d / Regressed columns and a red REGRESSED
  detail line, so a regression is visible without reading JSON.

Seeded with the two fixes whose signatures exist: 2855c63 (pipe crash, WATCHING 69.3h silent) and
563a9be (emptyTrash, UNSEEN). Verdict logic proven on fixtures: recurrence 5m after fix -> REGRESSED,
silent 9d -> CLOSED, last seen 2d -> REGRESSED, fixed 5m ago/last seen 3d ago -> WATCHING.
Loop verified end to end: triage via newos4-maintenance.sh prints ACTIVE/CLUSTER/FIX lines,
maintain OK, skill audit 7/7, dashboard ALL VIEWPORTS OK.

1 files changed, 225 insertions, 0 deletions

diff --git a/scripts/newos4-ledger.py b/scripts/newos4-ledger.py
new file mode 100755
index 0000000..7de993b
--- /dev/null
+++ b/scripts/newos4-ledger.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""newOS4 fix ledger — closes the loop between "a fix shipped" and "the error stopped".
+
+A signature is only *fixed* if it goes silent after the commit that claims to fix it. Nothing in the
+debug pipeline asserted that before this file: `errors.json` kept a 989-occurrence crash for three
+days after the report was filed, and a future regression of a fixed bug would look exactly like a new
+bug. The ledger stores `norm → {commit, fixed_at, note}` and re-checks each one on every triage run.
+
+Verdicts (per fix entry):
+  REGRESSED  the signature was seen AFTER fixed_at            -> the only alert worth acting on
+  WATCHING   silent, but younger than silent_days             -> no claim either way
+  CLOSED     silent for >= silent_days (default 7)            -> the report can be archived
+  UNSEEN     no matching signature at all right now           -> treated as silent (closed later)
+
+Usage:
+  python3 scripts/newos4-ledger.py add --commit 2855c63 --message "Cannot read properties of
+      undefined (reading 'toLowerCase')" [--norm <key>] [--hash <hash>] [--note "..."]
+  python3 scripts/newos4-ledger.py verify [--json]
+  python3 scripts/newos4-ledger.py close --norm <key>
+  python3 scripts/newos4-ledger.py list
+"""
+import argparse, json, os, re, sys
+from datetime import datetime, timezone
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+ERRORS = os.path.join(ROOT, 'data', 'debug', 'errors.json')
+LEDGER = os.path.join(ROOT, 'data', 'maintenance', 'fixes.json')
+SILENT_DAYS = 7
+
+
+def now_iso():
+    return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def parse_ts(s):
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(str(s).replace('Z', '+00:00')).astimezone(timezone.utc)
+    except Exception:
+        return None
+
+
+def norm_key(message, type_=None):
+    """Cross-language contract: MUST match norm_key() in api/debug/ingest.php.
+
+    The same failure reaches us with a different host, port, scheme or query on every deploy, and the
+    client hash includes the raw source URL — so one bug becomes N signatures (`/ollama//api/tags` and
+    `/ollama/api/tags` were two rows for one outage, 18 203 + 16 540 occurrences).
+    """
+    s = str(message or '').lower()
+    s = re.sub(r'https?://[^\s/]+', '', s)      # scheme + host + port
+    s = re.sub(r'/{2,}', '/', s)                # double slashes
+    s = re.sub(r'\?v=[\w.]+', '', s)            # cache-bust query
+    s = re.sub(r'\s+', ' ', s).strip()
+    return f"{str(type_ or '').lower().strip()}|{s}"[:400]
+
+
+def load(path, default):
+    try:
+        with open(path, encoding='utf-8') as f:
+            return json.load(f)
+    except Exception:
+        return default
+
+
+def save(path, obj):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    tmp = path + '.tmp'
+    with open(tmp, 'w', encoding='utf-8') as f:
+        json.dump(obj, f, indent=1)
+    os.replace(tmp, path)
+
+
+def load_ledger():
+    d = load(LEDGER, {'fixes': []})
+    if isinstance(d, dict) and isinstance(d.get('fixes'), list):
+        return d
+    return {'fixes': []}
+
+
+def rows_by_norm(errors):
+    """Group live signatures by normalised key; historical rows have no stored `norm`."""
+    out = {}
+    for e in errors:
+        key = e.get('norm') or norm_key(e.get('message'), e.get('type'))
+        r = out.setdefault(key, {'norm': key, 'count': 0, 'signatures': 0,
+                                 'first_seen': None, 'last_seen': None,
+                                 'messages': [], 'frame': None})
+        r['count'] += int(e.get('count') or 0)
+        r['signatures'] += 1
+        for f in ('first_seen', 'last_seen'):
+            t = parse_ts(e.get(f))
+            prev = parse_ts(r[f])
+            if t and (prev is None or (t < prev if f == 'first_seen' else t > prev)):
+                r[f] = e.get(f)
+        if len(r['messages']) < 3:
+            r['messages'].append(str(e.get('message') or '')[:120])
+        r['frame'] = r['frame'] or e.get('frame')
+    return out
+
+
+def verify(errors, ledger=None, silent_days=SILENT_DAYS):
+    """Compare each ledger entry against the live signatures."""
+    ledger = ledger if ledger is not None else load_ledger()
+    groups = rows_by_norm(errors)
+    by_hash = {e.get('hash'): e for e in errors if e.get('hash')}
+    now = datetime.now(timezone.utc)
+    results = []
+    for fx in ledger.get('fixes', []):
+        key = fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))
+        grp = groups.get(key)
+        fixed_at = parse_ts(fx.get('fixed_at')) or now
+        if not grp and fx.get('hash') and fx['hash'] in by_hash:
+            e = by_hash[fx['hash']]
+            grp = {'count': int(e.get('count') or 0), 'last_seen': e.get('last_seen'),
+                   'first_seen': e.get('first_seen'), 'signatures': 1, 'frame': e.get('frame'),
+                   'messages': [str(e.get('message') or '')[:120]]}
+        last = parse_ts((grp or {}).get('last_seen'))
+        silent_h = (now - last).total_seconds() / 3600.0 if last else None
+        # Order matters: "seen after the fix" alone is not a regression — a fix shipped 10 days ago
+        # with one sighting 9 days ago has been silent since. Regressed means seen after the fix AND
+        # still recent; otherwise the silence wins.
+        if grp and last and last > fixed_at and (silent_h is not None and silent_h <= silent_days * 24):
+            verdict = 'REGRESSED'
+            detail = f"seen {last.strftime('%Y-%m-%dT%H:%MZ')} after fix {fx.get('commit')}"
+        elif grp and silent_h is not None and silent_h >= silent_days * 24:
+            verdict = 'CLOSED'
+            detail = f"silent {silent_h / 24:.1f}d" + (f" (last sighting after fix: {last.strftime('%Y-%m-%d')})" if last and last > fixed_at else '')
+        elif grp:
+            verdict = 'WATCHING'
+            detail = f"silent {silent_h:.1f}h" if silent_h is not None else 'no timestamp'
+        else:
+            verdict, detail = 'UNSEEN', 'signature not in errors.json'
+        results.append({'norm': key, 'commit': fx.get('commit'), 'note': fx.get('note'),
+                        'fixed_at': fx.get('fixed_at'), 'status': fx.get('status', 'open'),
+                        'verdict': verdict, 'detail': detail,
+                        'count': (grp or {}).get('count', 0),
+                        'last_seen': (grp or {}).get('last_seen'),
+                        'frame': (grp or {}).get('frame'),
+                        'message': ((grp or {}).get('messages') or [fx.get('message', '')])[0]})
+    return results
+
+
+def summarise(results):
+    return {'regressed': [r for r in results if r['verdict'] == 'REGRESSED'],
+            'watching': [r for r in results if r['verdict'] == 'WATCHING'],
+            'closed': [r for r in results if r['verdict'] in ('CLOSED', 'UNSEEN')]}
+
+
+def print_verify(results):
+    s = summarise(results)
+    print(f"fix-ledger: {len(results)} entries | regressed={len(s['regressed'])} "
+          f"watching={len(s['watching'])} closed={len(s['closed'])}")
+    for r in s['regressed']:
+        print(f"  FIX| REGRESSED {r['commit']} {r['count']}x {r['message'][:90]} ({r['detail']})")
+    for r in s['watching']:
+        print(f"  FIX| watching  {r['commit']} {r['message'][:90]} ({r['detail']})")
+    for r in s['closed']:
+        print(f"  FIX| closed    {r['commit']} {r['message'][:90]}")
+
+
+def add(commit, message=None, norm=None, hash_=None, note=None, type_=None):
+    led = load_ledger()
+    key = norm or norm_key(message, type_)
+    for fx in led['fixes']:
+        if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == key:
+            fx.update({'commit': commit, 'fixed_at': now_iso(), 'note': note or fx.get('note'),
+                       'status': 'open', 'hash': hash_ or fx.get('hash')})
+            save(LEDGER, led)
+            return {'updated': key, 'commit': commit}
+    led['fixes'].append({'norm': key, 'hash': hash_, 'message': (message or '')[:300],
+                         'commit': commit, 'fixed_at': now_iso(), 'note': note,
+                         'status': 'open', 'silent_days': SILENT_DAYS})
+    save(LEDGER, led)
+    return {'added': key, 'commit': commit}
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description='newOS4 fix ledger')
+    sub = ap.add_subparsers(dest='cmd', required=True)
+
+    a = sub.add_parser('add')
+    a.add_argument('--commit', required=True)
+    a.add_argument('--message')
+    a.add_argument('--norm')
+    a.add_argument('--hash')
+    a.add_argument('--note')
+    a.add_argument('--type')
+
+    v = sub.add_parser('verify')
+    v.add_argument('--json', action='store_true')
+
+    c = sub.add_parser('close')
+    c.add_argument('--norm', required=True)
+
+    sub.add_parser('list')
+    args = ap.parse_args(argv)
+
+    if args.cmd == 'add':
+        print(json.dumps(add(args.commit, args.message, args.norm, args.hash, args.note, args.type)))
+    elif args.cmd == 'verify':
+        res = verify(load(ERRORS, []))
+        if args.json:
+            print(json.dumps({'results': res, **summarise(res)}, indent=1))
+        else:
+            print_verify(res)
+    elif args.cmd == 'close':
+        led = load_ledger()
+        hit = 0
+        for fx in led['fixes']:
+            if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == args.norm:
+                fx['status'] = 'closed'
+                hit += 1
+        save(LEDGER, led)
+        print(json.dumps({'closed': hit}))
+    elif args.cmd == 'list':
+        for fx in load_ledger()['fixes']:
+            print(f"{fx.get('commit','?'):<10} {fx.get('fixed_at','?')} {fx.get('status','open'):<7} "
+                  f"{(fx.get('message') or fx.get('norm'))[:90]}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())