From 97f55bf200c74a1b8ca0068ff70cbfeda77916cf Mon Sep 17 00:00:00 2001
From: Agando <roduccomm@gmail.com>
Date: Fri, 23 Oct 2026 14:39:30 +0200
Subject: fix(newos4-debug): idempotent ingest, admin-gated reads, honest
 counts (v4.6.52-53)

Root cause: _syncToBackend() re-sent the whole error DB with cumulative counts every
300s, and ingest.php did count += client count per batch. 33 real signatures had
accumulated 116,696 "occurrences" (one self-referential row alone: 46,133).

Backend
- ingest.php: per-(session,hash) synced map, adds only the delta, resets prev when the
  client count shrank (cleared localStorage), caps delta at 5000, bounds synced at 500
  hashes and sessions at 200, normalises timestamps to +00:00, 30-day retention that
  also purges dropped hashes from every synced map, returns `accepted` (entries) next
  to `ingested` (occurrences).
- stats.php/dashboard.php: total_errors = occurrences, unique_errors = signatures,
  last-ingest card. The two numbers were inconsistent by ~3,500x.
- health.php: reports self_referential count and status=degraded instead of ok.
- Pruned live data: 33 -> 23 records, 116,696 -> 58,759 occurrences, max cluster
  110,226 -> 52,387 (archive kept in data/debug/archive/).
- nginx: stats/errors/clusters/dashboard/agent-replies now behind
  auth_request /_wx11_admin_auth (302 without a session); ingest/report/health stay
  public for telemetry. Verified: reads 302, ingest 200, report 400.

Client
- module-debug-supervisor.js: sends only hashes whose count changed (persisted
  webx11_debug_synced), records synced counts only after success, stops auto-filing
  aborts/timeouts and debug-pipeline requests as bug reports (2 of the last 5 reports
  were "user aborted a request").

Verified with a scratch php -S copy: same batch POSTed 3x -> count stays 5;
count=99999 -> clamped to 1000; retention dropped 10 legacy rows.
---
 scripts/newos4-prune-debug.py | 92 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100755 scripts/newos4-prune-debug.py

(limited to 'scripts/newos4-prune-debug.py')

diff --git a/scripts/newos4-prune-debug.py b/scripts/newos4-prune-debug.py
new file mode 100755
index 0000000..7fe0fcb
--- /dev/null
+++ b/scripts/newos4-prune-debug.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Prune self-referential / stale debug rows from newOS4 telemetry.
+
+Read-only unless --apply. Coordinates with PHP via the SAME flock on errors.json
+(ingest.php opens it 'c' + LOCK_EX), and always archives before rewriting.
+"""
+import argparse, fcntl, json, os, re, time
+
+ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
+DEBUG = os.path.join(ROOT, 'data', 'debug')
+ERRORS = os.path.join(DEBUG, 'errors.json')
+ARCHIVE = os.path.join(DEBUG, 'archive')
+SELF = re.compile(r'/api/debug/(ingest|report)\.php', re.I)
+
+
+def iso_to_epoch(s):
+    """Parse '2026-10-20T16:57:31.589Z' or '...+00:00'. Returns None when unparseable."""
+    if not s:
+        return None
+    t = str(s).strip().replace('Z', '+00:00')
+    if '.' in t:
+        head, rest = t.split('.', 1)
+        frac = rest[:6].ljust(6, '0')
+        tail = rest[6:]
+        t = f'{head}.{frac}{tail}'
+    try:
+        import datetime
+        d = datetime.datetime.fromisoformat(t)
+        if d.tzinfo is None:
+            d = d.replace(tzinfo=datetime.timezone.utc)
+        return d.timestamp()
+    except Exception:
+        return None
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--apply', action='store_true')
+    ap.add_argument('--max-age-days', type=int, default=30)
+    a = ap.parse_args()
+    cutoff = time.time() - a.max_age_days * 86400
+
+    fp = open(ERRORS, 'r+', encoding='utf-8')
+    fcntl.flock(fp, fcntl.LOCK_EX)
+    try:
+        errors = json.load(fp) or []
+        keep, drop = [], []
+        for e in errors:
+            msg = str(e.get('message', ''))
+            age_ref = iso_to_epoch(e.get('last_seen')) or iso_to_epoch(e.get('ingested_at'))
+            if SELF.search(msg):
+                drop.append((e, 'self-referential'))
+            elif age_ref is not None and age_ref < cutoff:
+                drop.append((e, f'older than {a.max_age_days}d'))
+            else:
+                keep.append(e)
+
+        print(f'records={len(errors)} keep={len(keep)} drop={len(drop)}')
+        reasons = {}
+        for e, why in drop:
+            reasons[why] = reasons.get(why, 0) + 1
+        for why, n in sorted(reasons.items()):
+            print(f'  drop {n:>2} reason={why}')
+        for e, why in drop[:8]:
+            print(f"    count={e.get('count')} {why}: {str(e.get('message'))[:80]}")
+        print(f"occurrences before={sum(int(e.get('count') or 1) for e in errors)} "
+              f"after={sum(int(e.get('count') or 1) for e in keep)}")
+
+        if not a.apply:
+            print('dry-run; re-run with --apply to write')
+            return
+
+        os.makedirs(ARCHIVE, exist_ok=True)
+        archive = os.path.join(ARCHIVE, f'errors-{time.strftime("%Y%m%d_%H%M%S")}.json')
+        with open(archive, 'w', encoding='utf-8') as af:
+            json.dump(errors, af, indent=2)
+        os.chmod(archive, 0o644)
+
+        fp.seek(0)
+        fp.truncate()
+        json.dump(keep, fp, indent=2)
+        fp.flush()
+        os.fsync(fp.fileno())
+        open(os.path.join(DEBUG, '.rebuild_clusters'), 'w').write(str(int(time.time())))
+        print(f'applied: {archive} written, errors.json rewritten, cluster rebuild flagged')
+    finally:
+        fcntl.flock(fp, fcntl.LOCK_UN)
+        fp.close()
+
+
+if __name__ == '__main__':
+    main()
-- 
cgit v1.2.3