From 0c01cf48e83eecca953d0c6cc2dc949aed552f2f Mon Sep 17 00:00:00 2001 From: Agando Date: Fri, 23 Oct 2026 16:16:39 +0200 Subject: feat(newos4): fix ledger + norm clustering — the loop now knows what a fix was worth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifetime counts ranked an 8-day-old outage above today's live crash, and nothing asserted that a shipped fix actually held: a regression of a fixed bug looked exactly like a new bug. - api/debug/ingest.php: norm_key() stores `norm` (host/port/scheme/`?v=`/`//` stripped) — the client hash covers the raw URL, so one outage became two signatures (`/ollama//api/tags` 18 203x and `/ollama/api/tags` 16 540x). PHP + Python implementations verified byte-identical on 5 real messages. - scripts/newos4-ledger.py: data/maintenance/fixes.json maps norm -> {commit, fixed_at, note}, and verify() returns REGRESSED / WATCHING / CLOSED / UNSEEN. "Seen after the fix" alone is not a regression (a fix shipped 10d ago with one sighting 9d ago has been silent since) — regressed means seen after the fix AND recent; otherwise the silence wins. - scripts/newos4-triage.py: clusters (grouped by norm, recency-ordered), active<24h / stale>7d counts, FIX| verdict lines; digest gains clusters + fixes and active/stale/cluster/regressed counters. - api/debug/dashboard.php: Clusters / Active<24h / Stale>7d / Regressed columns and a red REGRESSED detail line, so a regression is visible without reading JSON. Seeded with the two fixes whose signatures exist: 2855c63 (pipe crash, WATCHING 69.3h silent) and 563a9be (emptyTrash, UNSEEN). Verdict logic proven on fixtures: recurrence 5m after fix -> REGRESSED, silent 9d -> CLOSED, last seen 2d -> REGRESSED, fixed 5m ago/last seen 3d ago -> WATCHING. Loop verified end to end: triage via newos4-maintenance.sh prints ACTIVE/CLUSTER/FIX lines, maintain OK, skill audit 7/7, dashboard ALL VIEWPORTS OK. --- scripts/newos4-triage.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'scripts/newos4-triage.py') diff --git a/scripts/newos4-triage.py b/scripts/newos4-triage.py index f639359..ef05dc1 100755 --- a/scripts/newos4-triage.py +++ b/scripts/newos4-triage.py @@ -61,6 +61,19 @@ def age_hours(s): return None if d is None else round((datetime.now(timezone.utc) - d).total_seconds() / 3600, 1) +def _ledger_module(): + """Load scripts/newos4-ledger.py by path (the filename has a hyphen, so no plain import).""" + try: + import importlib.util + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'newos4-ledger.py') + spec = importlib.util.spec_from_file_location('newos4_ledger', path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + except Exception: + return None + + def check_runner(): r = subprocess.run([sys.executable, 'scripts/newos4-maintain.py'], cwd=ROOT, capture_output=True, text=True, timeout=300) @@ -104,6 +117,22 @@ def build(apply_fix=False): for e in errors: buckets[classify(e)].append(e) + # A signature's lifetime count is not a priority: the four biggest entries in this file are one + # 8-day-old Ollama outage. Rank by recency + normalised clusters, and let the ledger say whether a + # shipped fix actually held. + ledger = _ledger_module() + groups = ledger.rows_by_norm(errors) if ledger else {} + fixes = ledger.verify(errors) if ledger else [] + fix_summary = ledger.summarise(fixes) if ledger else {'regressed': [], 'watching': [], 'closed': []} + active = [e for e in errors if (age_hours(e.get('last_seen')) or 1e9) < 24] + stale = [e for e in errors if (age_hours(e.get('last_seen')) or 0) > 168] + clusters = sorted( + ({'norm': g['norm'], 'count': g['count'], 'signatures': g['signatures'], + 'last_hours': age_hours(g['last_seen']), 'frame': g['frame'], + 'message': safe((g['messages'] or [''])[0], 120)} for g in groups.values()), + key=lambda x: (x['last_hours'] if x['last_hours'] is not None else 1e9, -x['count']) + )[:8] + digest = { 'generated_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), 'counts': { @@ -114,7 +143,15 @@ def build(apply_fix=False): 'self_referential_records': len(buckets['self-referential']), 'reports_total': len(reports), 'reports_new': len(new_reports), + 'active_records': len(active), + 'stale_records': len(stale), + 'clusters': len(groups), + 'fixes_regressed': len(fix_summary['regressed']), + 'fixes_watching': len(fix_summary['watching']), + 'fixes_closed': len(fix_summary['closed']), }, + 'clusters': clusters, + 'fixes': fix_summary, 'top_real': sorted( ({'message': safe(e.get('message'), 200), 'type': e.get('type'), 'count': int(e.get('count') or 1), 'age_hours': age_hours(e.get('last_seen')), @@ -148,6 +185,15 @@ def build(apply_fix=False): print(f"newOS4 triage {digest['generated_at']}: records={c['records']} " f"occurrences={c['occurrences']} real={c['real_records']} noise={c['noise_records']} " f"self={c['self_referential_records']} reports={c['reports_total']} new={c['reports_new']}") + print(f" ACTIVE| {c['active_records']} records seen in last 24h | stale>7d {c['stale_records']} " + f"| clusters {c['clusters']} of {c['records']} signatures") + for cl in digest['clusters'][:5]: + fr = f" @ {cl['frame']}" if cl.get('frame') else '' + print(f" CLUSTER| {cl['count']}x ({cl['signatures']} sig) {safe(cl['message'], 95)}{fr}") + for fx in digest['fixes']['regressed']: + print(f" FIX| REGRESSED {fx['commit']} {fx['count']}x {safe(fx['message'], 80)} ({fx['detail']})") + for fx in digest['fixes']['watching']: + print(f" FIX| watching {fx['commit']} {safe(fx['message'], 80)} ({fx['detail']})") for f in digest['checks']['findings']: print('check:', f) for e in digest['top_real'][:5]: -- cgit v1.2.3