feat(newos4): fix ledger + norm clustering — the loop now knows what a fix was worth

Lifetime counts ranked an 8-day-old outage above today's live crash, and nothing asserted that a
shipped fix actually held: a regression of a fixed bug looked exactly like a new bug.

- api/debug/ingest.php: norm_key() stores `norm` (host/port/scheme/`?v=`/`//` stripped) — the client
  hash covers the raw URL, so one outage became two signatures (`/ollama//api/tags` 18 203x and
  `/ollama/api/tags` 16 540x). PHP + Python implementations verified byte-identical on 5 real
  messages.
- scripts/newos4-ledger.py: data/maintenance/fixes.json maps norm -> {commit, fixed_at, note}, and
  verify() returns REGRESSED / WATCHING / CLOSED / UNSEEN. "Seen after the fix" alone is not a
  regression (a fix shipped 10d ago with one sighting 9d ago has been silent since) — regressed means
  seen after the fix AND recent; otherwise the silence wins.
- scripts/newos4-triage.py: clusters (grouped by norm, recency-ordered), active<24h / stale>7d counts,
  FIX| verdict lines; digest gains clusters + fixes and active/stale/cluster/regressed counters.
- api/debug/dashboard.php: Clusters / Active<24h / Stale>7d / Regressed columns and a red REGRESSED
  detail line, so a regression is visible without reading JSON.

Seeded with the two fixes whose signatures exist: 2855c63 (pipe crash, WATCHING 69.3h silent) and
563a9be (emptyTrash, UNSEEN). Verdict logic proven on fixtures: recurrence 5m after fix -> REGRESSED,
silent 9d -> CLOSED, last seen 2d -> REGRESSED, fixed 5m ago/last seen 3d ago -> WATCHING.
Loop verified end to end: triage via newos4-maintenance.sh prints ACTIVE/CLUSTER/FIX lines,
maintain OK, skill audit 7/7, dashboard ALL VIEWPORTS OK.

4 files changed, 300 insertions, 5 deletions

diff --git a/api/debug/dashboard.php b/api/debug/dashboard.php
index 9b717c8..a20271f 100644
--- a/api/debug/dashboard.php
+++ b/api/debug/dashboard.php
@@ -203,14 +203,15 @@ if (is_array($rawChecks)) {
   <?php else: ?>
     <div class="table-scroll">
       <table>
-        <tr><th>Signatures</th><th>Occurrences</th><th>Real</th><th>Noise</th><th>Self-ref</th><th>New reports</th><th>Generated</th></tr>
+        <tr><th>Signatures</th><th>Clusters</th><th>Active &lt;24h</th><th>Stale &gt;7d</th><th>Occurrences</th><th>Regressed</th><th>Generated</th></tr>
         <tr>
           <td><?= (int)($tri['records'] ?? 0) ?></td>
+          <td><?= (int)($tri['clusters'] ?? 0) ?></td>
+          <td><?= (int)($tri['active_records'] ?? 0) ?></td>
+          <td><?= (int)($tri['stale_records'] ?? 0) ?></td>
           <td><?= (int)($tri['occurrences'] ?? 0) ?></td>
-          <td><?= (int)($tri['real_records'] ?? 0) ?></td>
-          <td><?= (int)($tri['noise_records'] ?? 0) ?></td>
-          <td><?= (int)($tri['self_referential_records'] ?? 0) ?></td>
-          <td><?= (int)($tri['reports_new'] ?? 0) ?></td>
+          <td><?php $rg = (int)($tri['fixes_regressed'] ?? 0); ?>
+            <?php if ($rg > 0): ?><span class="status-pill degraded"><?= $rg ?> regressed</span><?php else: ?><span class="status-pill ok">none</span><?php endif; ?></td>
           <td><?= htmlspecialchars(substr($triage['generated_at'] ?? '', 5, 11)) ?>Z</td>
         </tr>
       </table>
@@ -221,6 +222,13 @@ if (is_array($rawChecks)) {
         <div class="thread"><div class="body" style="font-size:.82em;"><?= (int)($t['count'] ?? 0) ?>× <span class="tag"><?= htmlspecialchars($t['type'] ?? '?') ?></span> <?= htmlspecialchars(mb_substr($t['message'] ?? '', 0, 110)) ?></div></div>
       <?php endforeach; ?>
     <?php endif; ?>
+    <?php foreach (($triage['fixes']['regressed'] ?? []) as $fx): ?>
+      <div style="color:var(--error);font-size:.85em;margin-top:8px;">
+        <b>REGRESSED</b> <?= htmlspecialchars((string)($fx['commit'] ?? '?')) ?> —
+        <?= htmlspecialchars(substr((string)($fx['message'] ?? ''), 0, 90)) ?>
+        <span style="color:var(--muted)">(<?= htmlspecialchars((string)($fx['detail'] ?? '')) ?>)</span>
+      </div>
+    <?php endforeach; ?>
     <?php if ($triChecks): ?>
       <div style="margin-top:10px;font-size:.82em;color:var(--warn);">
         <?php foreach (array_slice($triChecks, 0, 4) as $c): ?><div class="body"><?= htmlspecialchars($c) ?></div><?php endforeach; ?>
diff --git a/api/debug/ingest.php b/api/debug/ingest.php
index 98354fe..dc425b8 100644
--- a/api/debug/ingest.php
+++ b/api/debug/ingest.php
@@ -94,6 +94,18 @@ function first_frame($stack) {
     return null;
 }
 
+// Cross-language contract: MUST match norm_key() in scripts/newos4-ledger.py.
+// One outage produced two signatures (`/ollama//api/tags` 18 203x, `/ollama/api/tags` 16 540x) because
+// the client hash covers the raw source URL. `norm` is what the triage and the fix ledger group on.
+function norm_key($message, $type = '') {
+    $s = mb_strtolower((string)$message);
+    $s = preg_replace('#https?://[^\s/]+#', '', $s);
+    $s = preg_replace('#/{2,}#', '/', $s);
+    $s = preg_replace('#\?v=[\w.]+#', '', $s);
+    $s = trim(preg_replace('#\s+#', ' ', $s));
+    return substr(mb_strtolower(trim((string)$type)) . '|' . $s, 0, 400);
+}
+
 $processed = 0;   // entries handled (NOT occurrences) — the client's truncation warning uses this
 foreach ($errors as $e) {
     $processed++;
@@ -119,6 +131,9 @@ foreach ($errors as $e) {
                 $allErrors[$idx]['stack'] = substr($e['stack'], 0, 2000);
                 $allErrors[$idx]['frame'] = first_frame($e['stack']) ?? ($allErrors[$idx]['frame'] ?? null);
             }
+            if (empty($allErrors[$idx]['norm']) && !empty($e['message'])) {
+                $allErrors[$idx]['norm'] = norm_key($e['message'], $e['type'] ?? '');
+            }
             $ingested += $delta;
         }
         continue;
@@ -134,6 +149,7 @@ foreach ($errors as $e) {
         'module' => $e['module'] ?? null,
         'stack' => isset($e['stack']) ? substr($e['stack'], 0, 2000) : null,
         'frame' => first_frame($e['stack'] ?? null),
+        'norm'  => norm_key($e['message'] ?? '', $e['type'] ?? ''),
         'session_id' => $sessionId,
         'count' => $delta,
         'first_seen' => norm_ts($e['firstSeen'] ?? null, $now),
diff --git a/scripts/newos4-ledger.py b/scripts/newos4-ledger.py
new file mode 100755
index 0000000..7de993b
--- /dev/null
+++ b/scripts/newos4-ledger.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""newOS4 fix ledger — closes the loop between "a fix shipped" and "the error stopped".
+
+A signature is only *fixed* if it goes silent after the commit that claims to fix it. Nothing in the
+debug pipeline asserted that before this file: `errors.json` kept a 989-occurrence crash for three
+days after the report was filed, and a future regression of a fixed bug would look exactly like a new
+bug. The ledger stores `norm → {commit, fixed_at, note}` and re-checks each one on every triage run.
+
+Verdicts (per fix entry):
+  REGRESSED  the signature was seen AFTER fixed_at            -> the only alert worth acting on
+  WATCHING   silent, but younger than silent_days             -> no claim either way
+  CLOSED     silent for >= silent_days (default 7)            -> the report can be archived
+  UNSEEN     no matching signature at all right now           -> treated as silent (closed later)
+
+Usage:
+  python3 scripts/newos4-ledger.py add --commit 2855c63 --message "Cannot read properties of
+      undefined (reading 'toLowerCase')" [--norm <key>] [--hash <hash>] [--note "..."]
+  python3 scripts/newos4-ledger.py verify [--json]
+  python3 scripts/newos4-ledger.py close --norm <key>
+  python3 scripts/newos4-ledger.py list
+"""
+import argparse, json, os, re, sys
+from datetime import datetime, timezone
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+ERRORS = os.path.join(ROOT, 'data', 'debug', 'errors.json')
+LEDGER = os.path.join(ROOT, 'data', 'maintenance', 'fixes.json')
+SILENT_DAYS = 7
+
+
+def now_iso():
+    return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def parse_ts(s):
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(str(s).replace('Z', '+00:00')).astimezone(timezone.utc)
+    except Exception:
+        return None
+
+
+def norm_key(message, type_=None):
+    """Cross-language contract: MUST match norm_key() in api/debug/ingest.php.
+
+    The same failure reaches us with a different host, port, scheme or query on every deploy, and the
+    client hash includes the raw source URL — so one bug becomes N signatures (`/ollama//api/tags` and
+    `/ollama/api/tags` were two rows for one outage, 18 203 + 16 540 occurrences).
+    """
+    s = str(message or '').lower()
+    s = re.sub(r'https?://[^\s/]+', '', s)      # scheme + host + port
+    s = re.sub(r'/{2,}', '/', s)                # double slashes
+    s = re.sub(r'\?v=[\w.]+', '', s)            # cache-bust query
+    s = re.sub(r'\s+', ' ', s).strip()
+    return f"{str(type_ or '').lower().strip()}|{s}"[:400]
+
+
+def load(path, default):
+    try:
+        with open(path, encoding='utf-8') as f:
+            return json.load(f)
+    except Exception:
+        return default
+
+
+def save(path, obj):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    tmp = path + '.tmp'
+    with open(tmp, 'w', encoding='utf-8') as f:
+        json.dump(obj, f, indent=1)
+    os.replace(tmp, path)
+
+
+def load_ledger():
+    d = load(LEDGER, {'fixes': []})
+    if isinstance(d, dict) and isinstance(d.get('fixes'), list):
+        return d
+    return {'fixes': []}
+
+
+def rows_by_norm(errors):
+    """Group live signatures by normalised key; historical rows have no stored `norm`."""
+    out = {}
+    for e in errors:
+        key = e.get('norm') or norm_key(e.get('message'), e.get('type'))
+        r = out.setdefault(key, {'norm': key, 'count': 0, 'signatures': 0,
+                                 'first_seen': None, 'last_seen': None,
+                                 'messages': [], 'frame': None})
+        r['count'] += int(e.get('count') or 0)
+        r['signatures'] += 1
+        for f in ('first_seen', 'last_seen'):
+            t = parse_ts(e.get(f))
+            prev = parse_ts(r[f])
+            if t and (prev is None or (t < prev if f == 'first_seen' else t > prev)):
+                r[f] = e.get(f)
+        if len(r['messages']) < 3:
+            r['messages'].append(str(e.get('message') or '')[:120])
+        r['frame'] = r['frame'] or e.get('frame')
+    return out
+
+
+def verify(errors, ledger=None, silent_days=SILENT_DAYS):
+    """Compare each ledger entry against the live signatures."""
+    ledger = ledger if ledger is not None else load_ledger()
+    groups = rows_by_norm(errors)
+    by_hash = {e.get('hash'): e for e in errors if e.get('hash')}
+    now = datetime.now(timezone.utc)
+    results = []
+    for fx in ledger.get('fixes', []):
+        key = fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))
+        grp = groups.get(key)
+        fixed_at = parse_ts(fx.get('fixed_at')) or now
+        if not grp and fx.get('hash') and fx['hash'] in by_hash:
+            e = by_hash[fx['hash']]
+            grp = {'count': int(e.get('count') or 0), 'last_seen': e.get('last_seen'),
+                   'first_seen': e.get('first_seen'), 'signatures': 1, 'frame': e.get('frame'),
+                   'messages': [str(e.get('message') or '')[:120]]}
+        last = parse_ts((grp or {}).get('last_seen'))
+        silent_h = (now - last).total_seconds() / 3600.0 if last else None
+        # Order matters: "seen after the fix" alone is not a regression — a fix shipped 10 days ago
+        # with one sighting 9 days ago has been silent since. Regressed means seen after the fix AND
+        # still recent; otherwise the silence wins.
+        if grp and last and last > fixed_at and (silent_h is not None and silent_h <= silent_days * 24):
+            verdict = 'REGRESSED'
+            detail = f"seen {last.strftime('%Y-%m-%dT%H:%MZ')} after fix {fx.get('commit')}"
+        elif grp and silent_h is not None and silent_h >= silent_days * 24:
+            verdict = 'CLOSED'
+            detail = f"silent {silent_h / 24:.1f}d" + (f" (last sighting after fix: {last.strftime('%Y-%m-%d')})" if last and last > fixed_at else '')
+        elif grp:
+            verdict = 'WATCHING'
+            detail = f"silent {silent_h:.1f}h" if silent_h is not None else 'no timestamp'
+        else:
+            verdict, detail = 'UNSEEN', 'signature not in errors.json'
+        results.append({'norm': key, 'commit': fx.get('commit'), 'note': fx.get('note'),
+                        'fixed_at': fx.get('fixed_at'), 'status': fx.get('status', 'open'),
+                        'verdict': verdict, 'detail': detail,
+                        'count': (grp or {}).get('count', 0),
+                        'last_seen': (grp or {}).get('last_seen'),
+                        'frame': (grp or {}).get('frame'),
+                        'message': ((grp or {}).get('messages') or [fx.get('message', '')])[0]})
+    return results
+
+
+def summarise(results):
+    return {'regressed': [r for r in results if r['verdict'] == 'REGRESSED'],
+            'watching': [r for r in results if r['verdict'] == 'WATCHING'],
+            'closed': [r for r in results if r['verdict'] in ('CLOSED', 'UNSEEN')]}
+
+
+def print_verify(results):
+    s = summarise(results)
+    print(f"fix-ledger: {len(results)} entries | regressed={len(s['regressed'])} "
+          f"watching={len(s['watching'])} closed={len(s['closed'])}")
+    for r in s['regressed']:
+        print(f"  FIX| REGRESSED {r['commit']} {r['count']}x {r['message'][:90]} ({r['detail']})")
+    for r in s['watching']:
+        print(f"  FIX| watching  {r['commit']} {r['message'][:90]} ({r['detail']})")
+    for r in s['closed']:
+        print(f"  FIX| closed    {r['commit']} {r['message'][:90]}")
+
+
+def add(commit, message=None, norm=None, hash_=None, note=None, type_=None):
+    led = load_ledger()
+    key = norm or norm_key(message, type_)
+    for fx in led['fixes']:
+        if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == key:
+            fx.update({'commit': commit, 'fixed_at': now_iso(), 'note': note or fx.get('note'),
+                       'status': 'open', 'hash': hash_ or fx.get('hash')})
+            save(LEDGER, led)
+            return {'updated': key, 'commit': commit}
+    led['fixes'].append({'norm': key, 'hash': hash_, 'message': (message or '')[:300],
+                         'commit': commit, 'fixed_at': now_iso(), 'note': note,
+                         'status': 'open', 'silent_days': SILENT_DAYS})
+    save(LEDGER, led)
+    return {'added': key, 'commit': commit}
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description='newOS4 fix ledger')
+    sub = ap.add_subparsers(dest='cmd', required=True)
+
+    a = sub.add_parser('add')
+    a.add_argument('--commit', required=True)
+    a.add_argument('--message')
+    a.add_argument('--norm')
+    a.add_argument('--hash')
+    a.add_argument('--note')
+    a.add_argument('--type')
+
+    v = sub.add_parser('verify')
+    v.add_argument('--json', action='store_true')
+
+    c = sub.add_parser('close')
+    c.add_argument('--norm', required=True)
+
+    sub.add_parser('list')
+    args = ap.parse_args(argv)
+
+    if args.cmd == 'add':
+        print(json.dumps(add(args.commit, args.message, args.norm, args.hash, args.note, args.type)))
+    elif args.cmd == 'verify':
+        res = verify(load(ERRORS, []))
+        if args.json:
+            print(json.dumps({'results': res, **summarise(res)}, indent=1))
+        else:
+            print_verify(res)
+    elif args.cmd == 'close':
+        led = load_ledger()
+        hit = 0
+        for fx in led['fixes']:
+            if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == args.norm:
+                fx['status'] = 'closed'
+                hit += 1
+        save(LEDGER, led)
+        print(json.dumps({'closed': hit}))
+    elif args.cmd == 'list':
+        for fx in load_ledger()['fixes']:
+            print(f"{fx.get('commit','?'):<10} {fx.get('fixed_at','?')} {fx.get('status','open'):<7} "
+                  f"{(fx.get('message') or fx.get('norm'))[:90]}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/scripts/newos4-triage.py b/scripts/newos4-triage.py
index f639359..ef05dc1 100755
--- a/scripts/newos4-triage.py
+++ b/scripts/newos4-triage.py
@@ -61,6 +61,19 @@ def age_hours(s):
     return None if d is None else round((datetime.now(timezone.utc) - d).total_seconds() / 3600, 1)
 
 
+def _ledger_module():
+    """Load scripts/newos4-ledger.py by path (the filename has a hyphen, so no plain import)."""
+    try:
+        import importlib.util
+        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'newos4-ledger.py')
+        spec = importlib.util.spec_from_file_location('newos4_ledger', path)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod
+    except Exception:
+        return None
+
+
 def check_runner():
     r = subprocess.run([sys.executable, 'scripts/newos4-maintain.py'], cwd=ROOT,
                        capture_output=True, text=True, timeout=300)
@@ -104,6 +117,22 @@ def build(apply_fix=False):
     for e in errors:
         buckets[classify(e)].append(e)
 
+    # A signature's lifetime count is not a priority: the four biggest entries in this file are one
+    # 8-day-old Ollama outage. Rank by recency + normalised clusters, and let the ledger say whether a
+    # shipped fix actually held.
+    ledger = _ledger_module()
+    groups = ledger.rows_by_norm(errors) if ledger else {}
+    fixes = ledger.verify(errors) if ledger else []
+    fix_summary = ledger.summarise(fixes) if ledger else {'regressed': [], 'watching': [], 'closed': []}
+    active = [e for e in errors if (age_hours(e.get('last_seen')) or 1e9) < 24]
+    stale = [e for e in errors if (age_hours(e.get('last_seen')) or 0) > 168]
+    clusters = sorted(
+        ({'norm': g['norm'], 'count': g['count'], 'signatures': g['signatures'],
+          'last_hours': age_hours(g['last_seen']), 'frame': g['frame'],
+          'message': safe((g['messages'] or [''])[0], 120)} for g in groups.values()),
+        key=lambda x: (x['last_hours'] if x['last_hours'] is not None else 1e9, -x['count'])
+    )[:8]
+
     digest = {
         'generated_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
         'counts': {
@@ -114,7 +143,15 @@ def build(apply_fix=False):
             'self_referential_records': len(buckets['self-referential']),
             'reports_total': len(reports),
             'reports_new': len(new_reports),
+            'active_records': len(active),
+            'stale_records': len(stale),
+            'clusters': len(groups),
+            'fixes_regressed': len(fix_summary['regressed']),
+            'fixes_watching': len(fix_summary['watching']),
+            'fixes_closed': len(fix_summary['closed']),
         },
+        'clusters': clusters,
+        'fixes': fix_summary,
         'top_real': sorted(
             ({'message': safe(e.get('message'), 200), 'type': e.get('type'),
               'count': int(e.get('count') or 1), 'age_hours': age_hours(e.get('last_seen')),
@@ -148,6 +185,15 @@ def build(apply_fix=False):
     print(f"newOS4 triage {digest['generated_at']}: records={c['records']} "
           f"occurrences={c['occurrences']} real={c['real_records']} noise={c['noise_records']} "
           f"self={c['self_referential_records']} reports={c['reports_total']} new={c['reports_new']}")
+    print(f"  ACTIVE| {c['active_records']} records seen in last 24h | stale>7d {c['stale_records']} "
+          f"| clusters {c['clusters']} of {c['records']} signatures")
+    for cl in digest['clusters'][:5]:
+        fr = f" @ {cl['frame']}" if cl.get('frame') else ''
+        print(f"  CLUSTER| {cl['count']}x ({cl['signatures']} sig) {safe(cl['message'], 95)}{fr}")
+    for fx in digest['fixes']['regressed']:
+        print(f"  FIX| REGRESSED {fx['commit']} {fx['count']}x {safe(fx['message'], 80)} ({fx['detail']})")
+    for fx in digest['fixes']['watching']:
+        print(f"  FIX| watching   {fx['commit']} {safe(fx['message'], 80)} ({fx['detail']})")
     for f in digest['checks']['findings']:
         print('check:', f)
     for e in digest['top_real'][:5]: