diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/newos4-ledger.py | 225 | ||||
| -rwxr-xr-x | scripts/newos4-triage.py | 46 |
2 files changed, 271 insertions, 0 deletions
diff --git a/scripts/newos4-ledger.py b/scripts/newos4-ledger.py new file mode 100755 index 0000000..7de993b --- /dev/null +++ b/scripts/newos4-ledger.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""newOS4 fix ledger — closes the loop between "a fix shipped" and "the error stopped". + +A signature is only *fixed* if it goes silent after the commit that claims to fix it. Nothing in the +debug pipeline asserted that before this file: `errors.json` kept a 989-occurrence crash for three +days after the report was filed, and a future regression of a fixed bug would look exactly like a new +bug. The ledger stores `norm → {commit, fixed_at, note}` and re-checks each one on every triage run. + +Verdicts (per fix entry): + REGRESSED the signature was seen AFTER fixed_at -> the only alert worth acting on + WATCHING silent, but younger than silent_days -> no claim either way + CLOSED silent for >= silent_days (default 7) -> the report can be archived + UNSEEN no matching signature at all right now -> treated as silent (closed later) + +Usage: + python3 scripts/newos4-ledger.py add --commit 2855c63 --message "Cannot read properties of + undefined (reading 'toLowerCase')" [--norm <key>] [--hash <hash>] [--note "..."] + python3 scripts/newos4-ledger.py verify [--json] + python3 scripts/newos4-ledger.py close --norm <key> + python3 scripts/newos4-ledger.py list +""" +import argparse, json, os, re, sys +from datetime import datetime, timezone + +ROOT = '/var/www/webx11.duckdns.org/spaces/newos4' +ERRORS = os.path.join(ROOT, 'data', 'debug', 'errors.json') +LEDGER = os.path.join(ROOT, 'data', 'maintenance', 'fixes.json') +SILENT_DAYS = 7 + + +def now_iso(): + return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def parse_ts(s): + if not s: + return None + try: + return datetime.fromisoformat(str(s).replace('Z', '+00:00')).astimezone(timezone.utc) + except Exception: + return None + + +def norm_key(message, type_=None): + """Cross-language contract: MUST match norm_key() in api/debug/ingest.php. + + The same failure reaches us with a different host, port, scheme or query on every deploy, and the + client hash includes the raw source URL — so one bug becomes N signatures (`/ollama//api/tags` and + `/ollama/api/tags` were two rows for one outage, 18 203 + 16 540 occurrences). + """ + s = str(message or '').lower() + s = re.sub(r'https?://[^\s/]+', '', s) # scheme + host + port + s = re.sub(r'/{2,}', '/', s) # double slashes + s = re.sub(r'\?v=[\w.]+', '', s) # cache-bust query + s = re.sub(r'\s+', ' ', s).strip() + return f"{str(type_ or '').lower().strip()}|{s}"[:400] + + +def load(path, default): + try: + with open(path, encoding='utf-8') as f: + return json.load(f) + except Exception: + return default + + +def save(path, obj): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + '.tmp' + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(obj, f, indent=1) + os.replace(tmp, path) + + +def load_ledger(): + d = load(LEDGER, {'fixes': []}) + if isinstance(d, dict) and isinstance(d.get('fixes'), list): + return d + return {'fixes': []} + + +def rows_by_norm(errors): + """Group live signatures by normalised key; historical rows have no stored `norm`.""" + out = {} + for e in errors: + key = e.get('norm') or norm_key(e.get('message'), e.get('type')) + r = out.setdefault(key, {'norm': key, 'count': 0, 'signatures': 0, + 'first_seen': None, 'last_seen': None, + 'messages': [], 'frame': None}) + r['count'] += int(e.get('count') or 0) + r['signatures'] += 1 + for f in ('first_seen', 'last_seen'): + t = parse_ts(e.get(f)) + prev = parse_ts(r[f]) + if t and (prev is None or (t < prev if f == 'first_seen' else t > prev)): + r[f] = e.get(f) + if len(r['messages']) < 3: + r['messages'].append(str(e.get('message') or '')[:120]) + r['frame'] = r['frame'] or e.get('frame') + return out + + +def verify(errors, ledger=None, silent_days=SILENT_DAYS): + """Compare each ledger entry against the live signatures.""" + ledger = ledger if ledger is not None else load_ledger() + groups = rows_by_norm(errors) + by_hash = {e.get('hash'): e for e in errors if e.get('hash')} + now = datetime.now(timezone.utc) + results = [] + for fx in ledger.get('fixes', []): + key = fx.get('norm') or norm_key(fx.get('message'), fx.get('type')) + grp = groups.get(key) + fixed_at = parse_ts(fx.get('fixed_at')) or now + if not grp and fx.get('hash') and fx['hash'] in by_hash: + e = by_hash[fx['hash']] + grp = {'count': int(e.get('count') or 0), 'last_seen': e.get('last_seen'), + 'first_seen': e.get('first_seen'), 'signatures': 1, 'frame': e.get('frame'), + 'messages': [str(e.get('message') or '')[:120]]} + last = parse_ts((grp or {}).get('last_seen')) + silent_h = (now - last).total_seconds() / 3600.0 if last else None + # Order matters: "seen after the fix" alone is not a regression — a fix shipped 10 days ago + # with one sighting 9 days ago has been silent since. Regressed means seen after the fix AND + # still recent; otherwise the silence wins. + if grp and last and last > fixed_at and (silent_h is not None and silent_h <= silent_days * 24): + verdict = 'REGRESSED' + detail = f"seen {last.strftime('%Y-%m-%dT%H:%MZ')} after fix {fx.get('commit')}" + elif grp and silent_h is not None and silent_h >= silent_days * 24: + verdict = 'CLOSED' + detail = f"silent {silent_h / 24:.1f}d" + (f" (last sighting after fix: {last.strftime('%Y-%m-%d')})" if last and last > fixed_at else '') + elif grp: + verdict = 'WATCHING' + detail = f"silent {silent_h:.1f}h" if silent_h is not None else 'no timestamp' + else: + verdict, detail = 'UNSEEN', 'signature not in errors.json' + results.append({'norm': key, 'commit': fx.get('commit'), 'note': fx.get('note'), + 'fixed_at': fx.get('fixed_at'), 'status': fx.get('status', 'open'), + 'verdict': verdict, 'detail': detail, + 'count': (grp or {}).get('count', 0), + 'last_seen': (grp or {}).get('last_seen'), + 'frame': (grp or {}).get('frame'), + 'message': ((grp or {}).get('messages') or [fx.get('message', '')])[0]}) + return results + + +def summarise(results): + return {'regressed': [r for r in results if r['verdict'] == 'REGRESSED'], + 'watching': [r for r in results if r['verdict'] == 'WATCHING'], + 'closed': [r for r in results if r['verdict'] in ('CLOSED', 'UNSEEN')]} + + +def print_verify(results): + s = summarise(results) + print(f"fix-ledger: {len(results)} entries | regressed={len(s['regressed'])} " + f"watching={len(s['watching'])} closed={len(s['closed'])}") + for r in s['regressed']: + print(f" FIX| REGRESSED {r['commit']} {r['count']}x {r['message'][:90]} ({r['detail']})") + for r in s['watching']: + print(f" FIX| watching {r['commit']} {r['message'][:90]} ({r['detail']})") + for r in s['closed']: + print(f" FIX| closed {r['commit']} {r['message'][:90]}") + + +def add(commit, message=None, norm=None, hash_=None, note=None, type_=None): + led = load_ledger() + key = norm or norm_key(message, type_) + for fx in led['fixes']: + if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == key: + fx.update({'commit': commit, 'fixed_at': now_iso(), 'note': note or fx.get('note'), + 'status': 'open', 'hash': hash_ or fx.get('hash')}) + save(LEDGER, led) + return {'updated': key, 'commit': commit} + led['fixes'].append({'norm': key, 'hash': hash_, 'message': (message or '')[:300], + 'commit': commit, 'fixed_at': now_iso(), 'note': note, + 'status': 'open', 'silent_days': SILENT_DAYS}) + save(LEDGER, led) + return {'added': key, 'commit': commit} + + +def main(argv=None): + ap = argparse.ArgumentParser(description='newOS4 fix ledger') + sub = ap.add_subparsers(dest='cmd', required=True) + + a = sub.add_parser('add') + a.add_argument('--commit', required=True) + a.add_argument('--message') + a.add_argument('--norm') + a.add_argument('--hash') + a.add_argument('--note') + a.add_argument('--type') + + v = sub.add_parser('verify') + v.add_argument('--json', action='store_true') + + c = sub.add_parser('close') + c.add_argument('--norm', required=True) + + sub.add_parser('list') + args = ap.parse_args(argv) + + if args.cmd == 'add': + print(json.dumps(add(args.commit, args.message, args.norm, args.hash, args.note, args.type))) + elif args.cmd == 'verify': + res = verify(load(ERRORS, [])) + if args.json: + print(json.dumps({'results': res, **summarise(res)}, indent=1)) + else: + print_verify(res) + elif args.cmd == 'close': + led = load_ledger() + hit = 0 + for fx in led['fixes']: + if (fx.get('norm') or norm_key(fx.get('message'), fx.get('type'))) == args.norm: + fx['status'] = 'closed' + hit += 1 + save(LEDGER, led) + print(json.dumps({'closed': hit})) + elif args.cmd == 'list': + for fx in load_ledger()['fixes']: + print(f"{fx.get('commit','?'):<10} {fx.get('fixed_at','?')} {fx.get('status','open'):<7} " + f"{(fx.get('message') or fx.get('norm'))[:90]}") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/newos4-triage.py b/scripts/newos4-triage.py index f639359..ef05dc1 100755 --- a/scripts/newos4-triage.py +++ b/scripts/newos4-triage.py @@ -61,6 +61,19 @@ def age_hours(s): return None if d is None else round((datetime.now(timezone.utc) - d).total_seconds() / 3600, 1) +def _ledger_module(): + """Load scripts/newos4-ledger.py by path (the filename has a hyphen, so no plain import).""" + try: + import importlib.util + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'newos4-ledger.py') + spec = importlib.util.spec_from_file_location('newos4_ledger', path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + except Exception: + return None + + def check_runner(): r = subprocess.run([sys.executable, 'scripts/newos4-maintain.py'], cwd=ROOT, capture_output=True, text=True, timeout=300) @@ -104,6 +117,22 @@ def build(apply_fix=False): for e in errors: buckets[classify(e)].append(e) + # A signature's lifetime count is not a priority: the four biggest entries in this file are one + # 8-day-old Ollama outage. Rank by recency + normalised clusters, and let the ledger say whether a + # shipped fix actually held. + ledger = _ledger_module() + groups = ledger.rows_by_norm(errors) if ledger else {} + fixes = ledger.verify(errors) if ledger else [] + fix_summary = ledger.summarise(fixes) if ledger else {'regressed': [], 'watching': [], 'closed': []} + active = [e for e in errors if (age_hours(e.get('last_seen')) or 1e9) < 24] + stale = [e for e in errors if (age_hours(e.get('last_seen')) or 0) > 168] + clusters = sorted( + ({'norm': g['norm'], 'count': g['count'], 'signatures': g['signatures'], + 'last_hours': age_hours(g['last_seen']), 'frame': g['frame'], + 'message': safe((g['messages'] or [''])[0], 120)} for g in groups.values()), + key=lambda x: (x['last_hours'] if x['last_hours'] is not None else 1e9, -x['count']) + )[:8] + digest = { 'generated_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), 'counts': { @@ -114,7 +143,15 @@ def build(apply_fix=False): 'self_referential_records': len(buckets['self-referential']), 'reports_total': len(reports), 'reports_new': len(new_reports), + 'active_records': len(active), + 'stale_records': len(stale), + 'clusters': len(groups), + 'fixes_regressed': len(fix_summary['regressed']), + 'fixes_watching': len(fix_summary['watching']), + 'fixes_closed': len(fix_summary['closed']), }, + 'clusters': clusters, + 'fixes': fix_summary, 'top_real': sorted( ({'message': safe(e.get('message'), 200), 'type': e.get('type'), 'count': int(e.get('count') or 1), 'age_hours': age_hours(e.get('last_seen')), @@ -148,6 +185,15 @@ def build(apply_fix=False): print(f"newOS4 triage {digest['generated_at']}: records={c['records']} " f"occurrences={c['occurrences']} real={c['real_records']} noise={c['noise_records']} " f"self={c['self_referential_records']} reports={c['reports_total']} new={c['reports_new']}") + print(f" ACTIVE| {c['active_records']} records seen in last 24h | stale>7d {c['stale_records']} " + f"| clusters {c['clusters']} of {c['records']} signatures") + for cl in digest['clusters'][:5]: + fr = f" @ {cl['frame']}" if cl.get('frame') else '' + print(f" CLUSTER| {cl['count']}x ({cl['signatures']} sig) {safe(cl['message'], 95)}{fr}") + for fx in digest['fixes']['regressed']: + print(f" FIX| REGRESSED {fx['commit']} {fx['count']}x {safe(fx['message'], 80)} ({fx['detail']})") + for fx in digest['fixes']['watching']: + print(f" FIX| watching {fx['commit']} {safe(fx['message'], 80)} ({fx['detail']})") for f in digest['checks']['findings']: print('check:', f) for e in digest['top_real'][:5]: |