From 0c01cf48e83eecca953d0c6cc2dc949aed552f2f Mon Sep 17 00:00:00 2001 From: Agando Date: Fri, 23 Oct 2026 16:16:39 +0200 Subject: feat(newos4): fix ledger + norm clustering — the loop now knows what a fix was worth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifetime counts ranked an 8-day-old outage above today's live crash, and nothing asserted that a shipped fix actually held: a regression of a fixed bug looked exactly like a new bug. - api/debug/ingest.php: norm_key() stores `norm` (host/port/scheme/`?v=`/`//` stripped) — the client hash covers the raw URL, so one outage became two signatures (`/ollama//api/tags` 18 203x and `/ollama/api/tags` 16 540x). PHP + Python implementations verified byte-identical on 5 real messages. - scripts/newos4-ledger.py: data/maintenance/fixes.json maps norm -> {commit, fixed_at, note}, and verify() returns REGRESSED / WATCHING / CLOSED / UNSEEN. "Seen after the fix" alone is not a regression (a fix shipped 10d ago with one sighting 9d ago has been silent since) — regressed means seen after the fix AND recent; otherwise the silence wins. - scripts/newos4-triage.py: clusters (grouped by norm, recency-ordered), active<24h / stale>7d counts, FIX| verdict lines; digest gains clusters + fixes and active/stale/cluster/regressed counters. - api/debug/dashboard.php: Clusters / Active<24h / Stale>7d / Regressed columns and a red REGRESSED detail line, so a regression is visible without reading JSON. Seeded with the two fixes whose signatures exist: 2855c63 (pipe crash, WATCHING 69.3h silent) and 563a9be (emptyTrash, UNSEEN). Verdict logic proven on fixtures: recurrence 5m after fix -> REGRESSED, silent 9d -> CLOSED, last seen 2d -> REGRESSED, fixed 5m ago/last seen 3d ago -> WATCHING. Loop verified end to end: triage via newos4-maintenance.sh prints ACTIVE/CLUSTER/FIX lines, maintain OK, skill audit 7/7, dashboard ALL VIEWPORTS OK. --- scripts/newos4-ledger.py | 225 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100755 scripts/newos4-ledger.py (limited to 'scripts/newos4-ledger.py') diff --git a/scripts/newos4-ledger.py b/scripts/newos4-ledger.py new file mode 100755 index 0000000..7de993b --- /dev/null +++ b/scripts/newos4-ledger.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""newOS4 fix ledger — closes the loop between "a fix shipped" and "the error stopped". + +A signature is only *fixed* if it goes silent after the commit that claims to fix it. Nothing in the +debug pipeline asserted that before this file: `errors.json` kept a 989-occurrence crash for three +days after the report was filed, and a future regression of a fixed bug would look exactly like a new +bug. The ledger stores `norm → {commit, fixed_at, note}` and re-checks each one on every triage run. + +Verdicts (per fix entry): + REGRESSED the signature was seen AFTER fixed_at -> the only alert worth acting on + WATCHING silent, but younger than silent_days -> no claim either way + CLOSED silent for >= silent_days (default 7) -> the report can be archived + UNSEEN no matching signature at all right now -> treated as silent (closed later) + +Usage: + python3 scripts/newos4-ledger.py add --commit 2855c63 --message "Cannot read properties of + undefined (reading 'toLowerCase')" [--norm ] [--hash ] [--note "..."] + python3 scripts/newos4-ledger.py verify [--json] + python3 scripts/newos4-ledger.py close --norm + python3 scripts/newos4-ledger.py list +""" +import argparse, json, os, re, sys +from datetime import datetime, timezone + +ROOT = '/var/www/webx11.duckdns.org/spaces/newos4' +ERRORS = os.path.join(ROOT, 'data', 'debug', 'errors.json') +LEDGER = os.path.join(ROOT, 'data', 'maintenance', 'fixes.json') +SILENT_DAYS = 7 + + +def now_iso(): + return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def parse_ts(s): + if not s: + return None + try: + return datetime.fromisoformat(str(s).replace('Z', '+00:00')).astimezone(timezone.utc) + except Exception: + return None + + +def norm_key(message, type_=None): + """Cross-language contract: MUST match norm_key() in api/debug/ingest.php. + + The same failure reaches us with a different host, port, scheme or query on every deploy, and the + client hash includes the raw source URL — so one bug becomes N signatures (`/ollama//api/tags` and + `/ollama/api/tags` were two rows for one outage, 18 203 + 16 540 occurrences). + """ + s = str(message or '').lower() + s = re.sub(r'https?://[^\s/]+', '', s) # scheme + host + port + s = re.sub(r'/{2,}', '/', s) # double slashes + s = re.sub(r'\?v=[\w.]+', '', s) # cache-bust query + s = re.sub(r'\s+', ' ', s).strip() + return f"{str(type_ or '').lower().strip()}|{s}"[:400] + + +def load(path, default): + try: + with open(path, encoding='utf-8') as f: + return json.load(f) + except Exception: + return default + + +def save(path, obj): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + '.tmp' + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(obj, f, indent=1) + os.replace(tmp, path) + + +def load_ledger(): + d = load(LEDGER, {'fixes': []}) + if isinstance(d, dict) and isinstance(d.get('fixes'), list): + return d + return {'fixes': []} + + +def rows_by_norm(errors): + """Group live signatures by normalised key; historical rows have no stored `norm`.""" + out = {} + for e in errors: + key = e.get('norm') or norm_key(e.get('message'), e.get('type')) + r = out.setdefault(key, {'norm': key, 'count': 0, 'signatures': 0, + 'first_seen': None, 'last_seen': None, + 'messages': [], 'frame': None}) + r['count'] += int(e.get('count') or 0) + r['signatures'] += 1 + for f in ('first_seen', 'last_seen'): + t = parse_ts(e.get(f)) + prev = parse_ts(r[f]) + if t and (prev is None or (t < prev if f == 'first_seen' else t > prev)): + r[f] = e.get(f) + if len(r['messages']) < 3: + r['messages'].append(str(e.get('message') or '')[:120]) + r['frame'] = r['frame'] or e.get('frame') + return out + + +def verify(errors, ledger=None, silent_days=SILENT_DAYS): + """Compare each ledger entry against the live signatures.""" + ledger = ledger if ledger is not None else load_ledger() + groups = rows_by_norm(errors) + by_hash = {e.get('hash'): e for e in errors if e.get('hash')} + now = datetime.now(timezone.utc) + results = [] + for fx in ledger.get('fixes', []): + key = fx.get('norm') or norm_key(fx.get('message'), fx.get('type')) + grp = groups.get(key) + fixed_at = parse_ts(fx.get('fixed_at')) or now + if not grp and fx.get('hash') and fx['hash'] in by_hash: + e = by_hash[fx['hash']] + grp = {'count': int(e.get('count') or 0), 'last_seen': e.get('last_seen'), + 'first_seen': e.get('first_seen'), 'signatures': 1, 'frame': e.get('frame'), + 'messages': [str(e.get('message') or '')[:120]]} + last = parse_ts((grp or {}).get('last_seen')) + silent_h = (now - last).total_seconds() / 3600.0 if last else None + # Order matters: "seen after the fix" alone is not a regression — a fix shipped 10 days ago + # with one sighting 9 days ago has been silent since. Regressed means seen after the fix AND + # still recent; otherwise the silence wins. + if grp and last and last > fixed_at and (silent_h is not None and silent_h <= silent_days * 24): + verdict = 'REGRESSED' + detail = f"seen {last.strftime('%Y-%m-%dT%H:%MZ')} after fix {fx.get('commit')}" + elif grp and silent_h is not None and silent_h >= silent_days * 24: + verdict = 'CLOSED' + detail = f"silent {silent_h / 24:.1f}d" + (f" (last sighting after fix: {last.strftime('%Y-%m-%d')})" if last and last > fixed_at else '') + elif grp: + verdict = 'WATCHING' + detail = f"silent {silent_h:.1f}h" if silent_h is not None else 'no timestamp' + else: + verdict, detail = 'UNSEEN', 'signature not in errors.json' + results.append({'norm': key, 'commit': fx.get('commit'), 'note': fx.get('note'), + 'fixed_at': fx.get('fixed_at'), 'status': fx.get('status', 'open'), + 'verdict': verdict, 'detail': detail, + 'count': (grp or {}).get('count', 0), + 'last_seen': (grp or {}).get('last_seen'), + 'frame': (grp or {}).get('frame'), + 'message': ((grp or {}).get('messages') or [fx.get('message', '')])[0]}) + return results + + +def summarise(results): + return {'regressed': [r for r in results if r['verdict'] == 'REGRESSED'], + 'watching': [r for r in results if r['verdict'] == 'WATCHING'], + 'closed': [r for r in results if r['verdict'] in ('CLOSED', 'UNSEEN')]} + + +def print_verify(results): + s = summarise(results) + print(f"fix-ledger: {len(results)} entries | regressed={len(s['regressed'])} " + f"watching={len(s['watching'])} closed={len(s['closed'])}") + for r in s['regressed']: + print(f" FIX| REGRESSED {r['commit']} {r['count']}x {r['message'][:90]} ({r['detail']})") + for r in s['watching']: + print(f" FIX| watching {r['commit']} {r['message'][:90]} ({r['detail']})") + for r in s['closed']: + print(f" FIX| closed {r['commit']} {r['message'][:90]}") + + +def add(commit, message=None, norm=None, hash_=None, note=None, type_=None): + led = load_ledger() + key = norm or norm_key(message, type_) + for fx in led['fixes']: + if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == key: + fx.update({'commit': commit, 'fixed_at': now_iso(), 'note': note or fx.get('note'), + 'status': 'open', 'hash': hash_ or fx.get('hash')}) + save(LEDGER, led) + return {'updated': key, 'commit': commit} + led['fixes'].append({'norm': key, 'hash': hash_, 'message': (message or '')[:300], + 'commit': commit, 'fixed_at': now_iso(), 'note': note, + 'status': 'open', 'silent_days': SILENT_DAYS}) + save(LEDGER, led) + return {'added': key, 'commit': commit} + + +def main(argv=None): + ap = argparse.ArgumentParser(description='newOS4 fix ledger') + sub = ap.add_subparsers(dest='cmd', required=True) + + a = sub.add_parser('add') + a.add_argument('--commit', required=True) + a.add_argument('--message') + a.add_argument('--norm') + a.add_argument('--hash') + a.add_argument('--note') + a.add_argument('--type') + + v = sub.add_parser('verify') + v.add_argument('--json', action='store_true') + + c = sub.add_parser('close') + c.add_argument('--norm', required=True) + + sub.add_parser('list') + args = ap.parse_args(argv) + + if args.cmd == 'add': + print(json.dumps(add(args.commit, args.message, args.norm, args.hash, args.note, args.type))) + elif args.cmd == 'verify': + res = verify(load(ERRORS, [])) + if args.json: + print(json.dumps({'results': res, **summarise(res)}, indent=1)) + else: + print_verify(res) + elif args.cmd == 'close': + led = load_ledger() + hit = 0 + for fx in led['fixes']: + if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == args.norm: + fx['status'] = 'closed' + hit += 1 + save(LEDGER, led) + print(json.dumps({'closed': hit})) + elif args.cmd == 'list': + for fx in load_ledger()['fixes']: + print(f"{fx.get('commit','?'):<10} {fx.get('fixed_at','?')} {fx.get('status','open'):<7} " + f"{(fx.get('message') or fx.get('norm'))[:90]}") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.2.3