1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
#!/usr/bin/env python3
"""Prune self-referential / stale debug rows from newOS4 telemetry.
Read-only unless --apply. Coordinates with PHP via the SAME flock on errors.json
(ingest.php opens it 'c' + LOCK_EX), and always archives before rewriting.
"""
import argparse, fcntl, json, os, re, time
ROOT = '/var/www/webx11.duckdns.org/spaces/newos4'
DEBUG = os.path.join(ROOT, 'data', 'debug')
ERRORS = os.path.join(DEBUG, 'errors.json')
ARCHIVE = os.path.join(DEBUG, 'archive')
SELF = re.compile(r'/api/debug/(ingest|report)\.php', re.I)
def iso_to_epoch(s):
"""Parse '2026-10-20T16:57:31.589Z' or '...+00:00'. Returns None when unparseable."""
if not s:
return None
t = str(s).strip().replace('Z', '+00:00')
if '.' in t:
head, rest = t.split('.', 1)
frac = rest[:6].ljust(6, '0')
tail = rest[6:]
t = f'{head}.{frac}{tail}'
try:
import datetime
d = datetime.datetime.fromisoformat(t)
if d.tzinfo is None:
d = d.replace(tzinfo=datetime.timezone.utc)
return d.timestamp()
except Exception:
return None
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--apply', action='store_true')
ap.add_argument('--max-age-days', type=int, default=30)
a = ap.parse_args()
cutoff = time.time() - a.max_age_days * 86400
fp = open(ERRORS, 'r+', encoding='utf-8')
fcntl.flock(fp, fcntl.LOCK_EX)
try:
errors = json.load(fp) or []
keep, drop = [], []
for e in errors:
msg = str(e.get('message', ''))
age_ref = iso_to_epoch(e.get('last_seen')) or iso_to_epoch(e.get('ingested_at'))
if SELF.search(msg):
drop.append((e, 'self-referential'))
elif age_ref is not None and age_ref < cutoff:
drop.append((e, f'older than {a.max_age_days}d'))
else:
keep.append(e)
print(f'records={len(errors)} keep={len(keep)} drop={len(drop)}')
reasons = {}
for e, why in drop:
reasons[why] = reasons.get(why, 0) + 1
for why, n in sorted(reasons.items()):
print(f' drop {n:>2} reason={why}')
for e, why in drop[:8]:
print(f" count={e.get('count')} {why}: {str(e.get('message'))[:80]}")
print(f"occurrences before={sum(int(e.get('count') or 1) for e in errors)} "
f"after={sum(int(e.get('count') or 1) for e in keep)}")
if not a.apply:
print('dry-run; re-run with --apply to write')
return
os.makedirs(ARCHIVE, exist_ok=True)
archive = os.path.join(ARCHIVE, f'errors-{time.strftime("%Y%m%d_%H%M%S")}.json')
with open(archive, 'w', encoding='utf-8') as af:
json.dump(errors, af, indent=2)
os.chmod(archive, 0o644)
fp.seek(0)
fp.truncate()
json.dump(keep, fp, indent=2)
fp.flush()
os.fsync(fp.fileno())
open(os.path.join(DEBUG, '.rebuild_clusters'), 'w').write(str(int(time.time())))
print(f'applied: {archive} written, errors.json rewritten, cluster rebuild flagged')
finally:
fcntl.flock(fp, fcntl.LOCK_UN)
fp.close()
if __name__ == '__main__':
main()
|
