From 97f55bf200c74a1b8ca0068ff70cbfeda77916cf Mon Sep 17 00:00:00 2001 From: Agando Date: Fri, 23 Oct 2026 14:39:30 +0200 Subject: fix(newos4-debug): idempotent ingest, admin-gated reads, honest counts (v4.6.52-53) Root cause: _syncToBackend() re-sent the whole error DB with cumulative counts every 300s, and ingest.php did count += client count per batch. 33 real signatures had accumulated 116,696 "occurrences" (one self-referential row alone: 46,133). Backend - ingest.php: per-(session,hash) synced map, adds only the delta, resets prev when the client count shrank (cleared localStorage), caps delta at 5000, bounds synced at 500 hashes and sessions at 200, normalises timestamps to +00:00, 30-day retention that also purges dropped hashes from every synced map, returns `accepted` (entries) next to `ingested` (occurrences). - stats.php/dashboard.php: total_errors = occurrences, unique_errors = signatures, last-ingest card. The two numbers were inconsistent by ~3,500x. - health.php: reports self_referential count and status=degraded instead of ok. - Pruned live data: 33 -> 23 records, 116,696 -> 58,759 occurrences, max cluster 110,226 -> 52,387 (archive kept in data/debug/archive/). - nginx: stats/errors/clusters/dashboard/agent-replies now behind auth_request /_wx11_admin_auth (302 without a session); ingest/report/health stay public for telemetry. Verified: reads 302, ingest 200, report 400. Client - module-debug-supervisor.js: sends only hashes whose count changed (persisted webx11_debug_synced), records synced counts only after success, stops auto-filing aborts/timeouts and debug-pipeline requests as bug reports (2 of the last 5 reports were "user aborted a request"). Verified with a scratch php -S copy: same batch POSTed 3x -> count stays 5; count=99999 -> clamped to 1000; retention dropped 10 legacy rows. --- api/debug/dashboard.php | 10 ++- api/debug/health.php | 17 ++++- api/debug/ingest.php | 91 ++++++++++++++++++++++----- api/debug/stats.php | 9 ++- main.html | 6 +- modules/core/core-bundle.js | 107 +++++++++++++++++++++++++++++--- modules/core/module-debug-supervisor.js | 67 +++++++++++++++++++- modules/core/module-loader.js | 2 +- scripts/newos4-prune-debug.py | 92 +++++++++++++++++++++++++++ 9 files changed, 365 insertions(+), 36 deletions(-) create mode 100755 scripts/newos4-prune-debug.py diff --git a/api/debug/dashboard.php b/api/debug/dashboard.php index c235ec0..74ba3af 100644 --- a/api/debug/dashboard.php +++ b/api/debug/dashboard.php @@ -5,7 +5,9 @@ header('Content-Type: text/html; charset=utf-8'); $errors = load_json(ERRORS_FILE); $sessions = load_json(SESSIONS_FILE); -$totalErrors = count($errors); +$uniqueErrors = count($errors); +$totalOccurrences = array_sum(array_map(fn($e) => (int)($e['count'] ?? 1), $errors)); +$lastIngest = $errors ? max(array_column($errors, 'ingested_at')) : null; $totalSessions = count($sessions); $byType = []; $bySeverity = []; @@ -91,8 +93,10 @@ if ($lastSync): ?>
-

Total Errors

-
+

Total Occurrences

+
+
unique signatures
+
last ingest:

Sessions

diff --git a/api/debug/health.php b/api/debug/health.php index 5a0048f..883588f 100644 --- a/api/debug/health.php +++ b/api/debug/health.php @@ -6,11 +6,24 @@ $sessions = load_json(SESSIONS_FILE); $lastError = !empty($errors) ? end($errors) : null; +// Self-referential rows mean the debug pipeline is counting itself: the failure mode that +// turned 33 real errors into 116,696 "occurrences" before 2026-10-23. Surface it instead of +// reporting a poisoned dataset as healthy. +$selfRef = 0; +$occurrences = 0; +foreach ($errors as $e) { + if (stripos($e['message'] ?? '', '/api/debug/') !== false) $selfRef++; + $occurrences += (int)($e['count'] ?? 1); +} + json_response([ 'success' => true, - 'status' => 'ok', + 'status' => $selfRef > 0 ? 'degraded' : 'ok', 'uptime' => filemtime(__FILE__) ? (time() - filemtime(__FILE__)) : 0, - 'total_errors' => count($errors), + 'total_errors' => $occurrences, + 'total_occurrences' => $occurrences, + 'unique_errors' => count($errors), + 'self_referential' => $selfRef, 'total_sessions' => count($sessions), 'errors_file_size_kb' => file_exists(ERRORS_FILE) ? round(filesize(ERRORS_FILE) / 1024, 1) : 0, 'sessions_file_size_kb' => file_exists(SESSIONS_FILE) ? round(filesize(SESSIONS_FILE) / 1024, 1) : 0, diff --git a/api/debug/ingest.php b/api/debug/ingest.php index b513593..7388bce 100644 --- a/api/debug/ingest.php +++ b/api/debug/ingest.php @@ -64,23 +64,51 @@ foreach ($allErrors as $idx => $existing) { } } -// Ingest each error (dedup by hash) +// Ingest each error (dedup by hash). +// The client re-sends its whole error DB every 5 min with a CUMULATIVE count per hash. +// Store what the client already reported for this session/hash and only add the delta, +// otherwise every sync multiplies the occurrence count (measured 2026-10-23: +// 33 real records had accumulated 116,696 "occurrences"). +if (!isset($sessions[$sessionId]['synced']) || !is_array($sessions[$sessionId]['synced'])) { + $sessions[$sessionId]['synced'] = []; +} + +function norm_ts($s, $fallback) { + if (!is_string($s) || $s === '') return $fallback; + $t = strtotime($s); + if ($t === false) return $fallback; + return gmdate('c', $t); // always +00:00, never Z +} + +$processed = 0; // entries handled (NOT occurrences) — the client's truncation warning uses this foreach ($errors as $e) { + $processed++; $hash = $e['hash'] ?? md5(($e['message'] ?? '') . '|' . ($e['source'] ?? '') . '|' . ($e['type'] ?? '')); - + $clientCount = clamp_count($e['count'] ?? 1); + $prevCount = (int)($sessions[$sessionId]['synced'][$hash] ?? 0); + // A client whose localStorage was cleared (or a replayed payload) can present a huge + // cumulative count against prev=0. Cap the accepted delta per sync so one replay cannot + // reintroduce the 46k-style inflation. + if ($prevCount > $clientCount) $prevCount = 0; + $delta = min(5000, max(0, $clientCount - $prevCount)); + if ($clientCount > $prevCount) { + $sessions[$sessionId]['synced'][$hash] = $clientCount; + } + if (isset($existingHashes[$hash])) { - // Update existing entry $idx = $existingHashes[$hash]; - $allErrors[$idx]['count'] += clamp_count($e['count'] ?? 1); - $allErrors[$idx]['last_seen'] = $e['lastSeen'] ?? $now; - $allErrors[$idx]['ingested_at'] = $now; - if (!empty($e['stack'])) { - $allErrors[$idx]['stack'] = substr($e['stack'], 0, 2000); + if ($delta > 0) { + $allErrors[$idx]['count'] += $delta; + $allErrors[$idx]['last_seen'] = norm_ts($e['lastSeen'] ?? null, $now); + $allErrors[$idx]['ingested_at'] = $now; + if (!empty($e['stack'])) { + $allErrors[$idx]['stack'] = substr($e['stack'], 0, 2000); + } + $ingested += $delta; } - $ingested++; continue; } - + $entry = [ 'hash' => $hash, 'message' => substr($e['message'] ?? '', 0, 500), @@ -91,15 +119,38 @@ foreach ($errors as $e) { 'module' => $e['module'] ?? null, 'stack' => isset($e['stack']) ? substr($e['stack'], 0, 2000) : null, 'session_id' => $sessionId, - 'count' => clamp_count($e['count'] ?? 1), - 'first_seen' => $e['firstSeen'] ?? $now, - 'last_seen' => $e['lastSeen'] ?? $now, + 'count' => $delta, + 'first_seen' => norm_ts($e['firstSeen'] ?? null, $now), + 'last_seen' => norm_ts($e['lastSeen'] ?? null, $now), 'ingested_at' => $now, ]; - + $allErrors[] = $entry; $existingHashes[$hash] = count($allErrors) - 1; - $ingested++; + $ingested += $delta; +} + +// Bound the per-session counter map so sessions.json cannot grow without limit. +if (count($sessions[$sessionId]['synced']) > 500) { + $sessions[$sessionId]['synced'] = array_slice($sessions[$sessionId]['synced'], -500, 500, true); +} + +// Retention: drop signatures unseen for 30 days (occurrence history is in the archive). +// A dropped hash MUST also leave every sessions[*].synced map, otherwise the next client +// sync recreates the record with delta=0 and the retention pass never converges. +$retentionCut = time() - 30 * 86400; +$survivors = []; +$allErrors = array_values(array_filter($allErrors, function ($e) use ($retentionCut, &$survivors) { + $t = strtotime($e['last_seen'] ?? ''); + if ($t === false) { $survivors[$e['hash'] ?? ''] = true; return true; } + if ($t > $retentionCut) { $survivors[$e['hash'] ?? ''] = true; return true; } + return false; +})); +foreach ($sessions as $sid => $sinfo) { + if (empty($sinfo['synced'])) continue; + foreach (array_keys($sinfo['synced']) as $h) { + if (!isset($survivors[$h])) unset($sessions[$sid]['synced'][$h]); + } } // Trim to last 10,000 errors (ring buffer) @@ -107,6 +158,15 @@ if (count($allErrors) > 10000) { $allErrors = array_slice($allErrors, -10000); } +// Bound the session map itself: keep the 200 most recently seen sessions. +if (count($sessions) > 200) { + uasort($sessions, fn($a, $b) => strcmp($a['last_seen'] ?? '', $b['last_seen'] ?? '')); + $sessions = array_slice($sessions, -200, 200, true); +} + +// Write order matters: errors first, then sessions. `synced` is what suppresses a re-add, so +// recording it before the count lands would permanently lose occurrences; the reverse loses at +// most one batch of duplicates (harmless). Both files use flock inside save_json(). save_json(ERRORS_FILE, $allErrors); save_json(SESSIONS_FILE, $sessions); @@ -117,6 +177,7 @@ file_put_contents($rebuildFlag, time()); json_response([ 'success' => true, 'ingested' => $ingested, + 'accepted' => $processed, 'session_id' => $sessionId, 'total_errors' => count($allErrors), 'total_sessions' => count($sessions), diff --git a/api/debug/stats.php b/api/debug/stats.php index dd0b0db..6e4feb0 100644 --- a/api/debug/stats.php +++ b/api/debug/stats.php @@ -37,13 +37,20 @@ foreach ($errors as $e) { } } +$totalOccurrences = 0; +foreach ($errors as $e) { $totalOccurrences += (int)($e['count'] ?? 1); } + usort($topErrors, fn($a, $b) => $b['count'] - $a['count']); $topErrors = array_slice($topErrors, 0, 20); ksort($byDay); json_response([ 'success' => true, - 'total_errors' => $total, + // total_errors = OCCURRENCES (sum of counts); unique_errors = distinct signatures. + // Before 2026-10-23 the key held the record count while by_type held occurrences, + // so two numbers on the same screen disagreed by ~3,500x. + 'total_errors' => $totalOccurrences, + 'unique_errors' => $total, 'total_sessions' => count($sessions), 'by_type' => $byType, 'by_severity' => $bySeverity, diff --git a/main.html b/main.html index f41b12d..9701164 100644 --- a/main.html +++ b/main.html @@ -1349,13 +1349,13 @@ - + - + - +