diff options
| author | Agando <roduccomm@gmail.com> | 2026-10-23 14:39:30 +0200 |
|---|---|---|
| committer | Agando <roduccomm@gmail.com> | 2026-10-23 14:39:30 +0200 |
| commit | 97f55bf200c74a1b8ca0068ff70cbfeda77916cf (patch) | |
| tree | f27fdfda0031d716bdc3717301eb2e250dbda9f8 /api/debug/ingest.php | |
| parent | 3bbaa9df177fcab2ee9dd04857d16d92c4812780 (diff) | |
fix(newos4-debug): idempotent ingest, admin-gated reads, honest counts (v4.6.52-53)
Root cause: _syncToBackend() re-sent the whole error DB with cumulative counts every
300s, and ingest.php did count += client count per batch. 33 real signatures had
accumulated 116,696 "occurrences" (one self-referential row alone: 46,133).
Backend
- ingest.php: per-(session,hash) synced map, adds only the delta, resets prev when the
client count shrank (cleared localStorage), caps delta at 5000, bounds synced at 500
hashes and sessions at 200, normalises timestamps to +00:00, 30-day retention that
also purges dropped hashes from every synced map, returns `accepted` (entries) next
to `ingested` (occurrences).
- stats.php/dashboard.php: total_errors = occurrences, unique_errors = signatures,
last-ingest card. The two numbers were inconsistent by ~3,500x.
- health.php: reports self_referential count and status=degraded instead of ok.
- Pruned live data: 33 -> 23 records, 116,696 -> 58,759 occurrences, max cluster
110,226 -> 52,387 (archive kept in data/debug/archive/).
- nginx: stats/errors/clusters/dashboard/agent-replies now behind
auth_request /_wx11_admin_auth (302 without a session); ingest/report/health stay
public for telemetry. Verified: reads 302, ingest 200, report 400.
Client
- module-debug-supervisor.js: sends only hashes whose count changed (persisted
webx11_debug_synced), records synced counts only after success, stops auto-filing
aborts/timeouts and debug-pipeline requests as bug reports (2 of the last 5 reports
were "user aborted a request").
Verified with a scratch php -S copy: same batch POSTed 3x -> count stays 5;
count=99999 -> clamped to 1000; retention dropped 10 legacy rows.
Diffstat (limited to 'api/debug/ingest.php')
| -rw-r--r-- | api/debug/ingest.php | 91 |
1 files changed, 76 insertions, 15 deletions
diff --git a/api/debug/ingest.php b/api/debug/ingest.php index b513593..7388bce 100644 --- a/api/debug/ingest.php +++ b/api/debug/ingest.php @@ -64,23 +64,51 @@ foreach ($allErrors as $idx => $existing) { } } -// Ingest each error (dedup by hash) +// Ingest each error (dedup by hash). +// The client re-sends its whole error DB every 5 min with a CUMULATIVE count per hash. +// Store what the client already reported for this session/hash and only add the delta, +// otherwise every sync multiplies the occurrence count (measured 2026-10-23: +// 33 real records had accumulated 116,696 "occurrences"). +if (!isset($sessions[$sessionId]['synced']) || !is_array($sessions[$sessionId]['synced'])) { + $sessions[$sessionId]['synced'] = []; +} + +function norm_ts($s, $fallback) { + if (!is_string($s) || $s === '') return $fallback; + $t = strtotime($s); + if ($t === false) return $fallback; + return gmdate('c', $t); // always +00:00, never Z +} + +$processed = 0; // entries handled (NOT occurrences) — the client's truncation warning uses this foreach ($errors as $e) { + $processed++; $hash = $e['hash'] ?? md5(($e['message'] ?? '') . '|' . ($e['source'] ?? '') . '|' . ($e['type'] ?? '')); - + $clientCount = clamp_count($e['count'] ?? 1); + $prevCount = (int)($sessions[$sessionId]['synced'][$hash] ?? 0); + // A client whose localStorage was cleared (or a replayed payload) can present a huge + // cumulative count against prev=0. Cap the accepted delta per sync so one replay cannot + // reintroduce the 46k-style inflation. + if ($prevCount > $clientCount) $prevCount = 0; + $delta = min(5000, max(0, $clientCount - $prevCount)); + if ($clientCount > $prevCount) { + $sessions[$sessionId]['synced'][$hash] = $clientCount; + } + if (isset($existingHashes[$hash])) { - // Update existing entry $idx = $existingHashes[$hash]; - $allErrors[$idx]['count'] += clamp_count($e['count'] ?? 1); - $allErrors[$idx]['last_seen'] = $e['lastSeen'] ?? $now; - $allErrors[$idx]['ingested_at'] = $now; - if (!empty($e['stack'])) { - $allErrors[$idx]['stack'] = substr($e['stack'], 0, 2000); + if ($delta > 0) { + $allErrors[$idx]['count'] += $delta; + $allErrors[$idx]['last_seen'] = norm_ts($e['lastSeen'] ?? null, $now); + $allErrors[$idx]['ingested_at'] = $now; + if (!empty($e['stack'])) { + $allErrors[$idx]['stack'] = substr($e['stack'], 0, 2000); + } + $ingested += $delta; } - $ingested++; continue; } - + $entry = [ 'hash' => $hash, 'message' => substr($e['message'] ?? '', 0, 500), @@ -91,15 +119,38 @@ foreach ($errors as $e) { 'module' => $e['module'] ?? null, 'stack' => isset($e['stack']) ? substr($e['stack'], 0, 2000) : null, 'session_id' => $sessionId, - 'count' => clamp_count($e['count'] ?? 1), - 'first_seen' => $e['firstSeen'] ?? $now, - 'last_seen' => $e['lastSeen'] ?? $now, + 'count' => $delta, + 'first_seen' => norm_ts($e['firstSeen'] ?? null, $now), + 'last_seen' => norm_ts($e['lastSeen'] ?? null, $now), 'ingested_at' => $now, ]; - + $allErrors[] = $entry; $existingHashes[$hash] = count($allErrors) - 1; - $ingested++; + $ingested += $delta; +} + +// Bound the per-session counter map so sessions.json cannot grow without limit. +if (count($sessions[$sessionId]['synced']) > 500) { + $sessions[$sessionId]['synced'] = array_slice($sessions[$sessionId]['synced'], -500, 500, true); +} + +// Retention: drop signatures unseen for 30 days (occurrence history is in the archive). +// A dropped hash MUST also leave every sessions[*].synced map, otherwise the next client +// sync recreates the record with delta=0 and the retention pass never converges. +$retentionCut = time() - 30 * 86400; +$survivors = []; +$allErrors = array_values(array_filter($allErrors, function ($e) use ($retentionCut, &$survivors) { + $t = strtotime($e['last_seen'] ?? ''); + if ($t === false) { $survivors[$e['hash'] ?? ''] = true; return true; } + if ($t > $retentionCut) { $survivors[$e['hash'] ?? ''] = true; return true; } + return false; +})); +foreach ($sessions as $sid => $sinfo) { + if (empty($sinfo['synced'])) continue; + foreach (array_keys($sinfo['synced']) as $h) { + if (!isset($survivors[$h])) unset($sessions[$sid]['synced'][$h]); + } } // Trim to last 10,000 errors (ring buffer) @@ -107,6 +158,15 @@ if (count($allErrors) > 10000) { $allErrors = array_slice($allErrors, -10000); } +// Bound the session map itself: keep the 200 most recently seen sessions. +if (count($sessions) > 200) { + uasort($sessions, fn($a, $b) => strcmp($a['last_seen'] ?? '', $b['last_seen'] ?? '')); + $sessions = array_slice($sessions, -200, 200, true); +} + +// Write order matters: errors first, then sessions. `synced` is what suppresses a re-add, so +// recording it before the count lands would permanently lose occurrences; the reverse loses at +// most one batch of duplicates (harmless). Both files use flock inside save_json(). save_json(ERRORS_FILE, $allErrors); save_json(SESSIONS_FILE, $sessions); @@ -117,6 +177,7 @@ file_put_contents($rebuildFlag, time()); json_response([ 'success' => true, 'ingested' => $ingested, + 'accepted' => $processed, 'session_id' => $sessionId, 'total_errors' => count($allErrors), 'total_sessions' => count($sessions), |