feat(newos4): ingest report-like chat messages into the feedback queue

The chat plane (api/chat/post.php -> data/chat/*.json) was read by nothing in the
feedback pipeline: 22 report-like messages over 11 days produced 0 queue items.
This adds scripts/newos4-chat-ingest.py - a read-only scanner by default, --apply
writes RPT-* files in the report.php schema with diagnostics.auto=false so the
auto-reviewer leaves them for a human, deduped on (thread, msg_id).

First apply filed 5 items (2 x 2026-10-28, 3 x 2026-11-02); snapshot went
reports_total 23 -> 28, awaiting_decision 0 -> 5. Second run is silent.

1 files changed, 186 insertions, 0 deletions

diff --git a/scripts/newos4-chat-ingest.py b/scripts/newos4-chat-ingest.py
new file mode 100755
index 0000000..33409fa
--- /dev/null
+++ b/scripts/newos4-chat-ingest.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""newos4-chat-ingest.py — turn report-like chat messages into RPT-* queue items.
+
+The chat bridge (api/chat/post.php) stores threads in data/chat/<thread>.json. Nothing in the
+feedback pipeline reads that directory, so a bug report posted through the in-page agent never
+becomes a card (measured 2026-11-02: 22 report-like messages, 0 reports).
+
+This scanner is read-only by default. With --apply it writes one RPT-*.json per new message, in the
+same schema api/debug/report.php produces, with diagnostics.auto = false so the auto-reviewer leaves
+it for a human (newos4-feedback-autoreview.py:165).
+
+Dedupe: data/feedback/chat-ingest-state.json (thread:msg_id -> report id) AND an existing report
+carrying the same diagnostics.chat.msg_id, so losing the state file cannot duplicate an item.
+"""
+import argparse
+import glob
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+
+ROOT = '/var/www/webx11.duckdns.org'
+CHAT = f'{ROOT}/spaces/newos4/data/chat'
+REPORTS = f'{ROOT}/spaces/newos4/data/reports'
+STATE = f'{ROOT}/spaces/newos4/data/feedback/chat-ingest-state.json'
+GENERATOR = f'{ROOT}/spaces/newos4/scripts/gen-bug-report.py'
+
+# A message is a report when it opens a section with one of these headers. Chat smalltalk, the
+# relay's own replies and one-line questions never match.
+REPORTISH = re.compile(r'(?m)^\s*(BUG|FEEDBACK|SYMPTOM|ROOT CAUSE|SECOND DEFECT|REPRO|MODULE)\b')
+# Client-blip traffic that the ingest filter (api/debug/report.php) already refuses to card.
+NOISE = re.compile(r'(poll\.php|networkerror|failed to fetch|signal is aborted|ERR_NETWORK)', re.I)
+# Work orders and replies, not defects: the 2026-10-23 thread carries two DEPLOY SPECs that would
+# otherwise enter the defect queue (dry run 2026-11-02 filed them as 2 of 7 candidates).
+NOT_A_REPORT = re.compile(r'(?m)^\s*(DEPLOY SPEC|ACKNOWLEDGED|RETRACTED|THANK YOU)\b')
+SKIP_AUTHORS = {'hermes'}   # our own replies (the relay's answers are not reports)
+MIN_LEN = 200                      # a real report is never a one-liner
+
+
+def load(path, default):
+    try:
+        with open(path, encoding='utf-8') as fh:
+            return json.load(fh)
+    except Exception:
+        return default
+
+
+def save_json(path, obj):
+    tmp = path + '.tmp'
+    with open(tmp, 'w', encoding='utf-8') as fh:
+        json.dump(obj, fh, indent=1, ensure_ascii=False)
+    os.replace(tmp, path)
+
+
+def messages(path):
+    d = load(path, [])
+    return d if isinstance(d, list) else (d.get('messages') or [])
+
+
+def version_of(text):
+    m = re.search(r'\bv(\d+\.\d+\.\d+)\b', text)
+    return 'v' + m.group(1) if m else 'unknown'
+
+
+def is_report(msg):
+    if str(msg.get('author')) in SKIP_AUTHORS:
+        return False
+    text = str(msg.get('text') or '')
+    if len(text) < MIN_LEN or not REPORTISH.search(text):
+        return False
+    if NOT_A_REPORT.match(text.strip()):
+        return False
+    if NOISE.search(text):
+        # a report may quote a fetch failure; what disqualifies it is *being* one
+        return bool(re.search(r'(?m)^\s*(ROOT CAUSE|BUG)\b', text))
+    return True
+
+
+def title_for(msg):
+    first = str(msg.get('text') or '').strip().splitlines()[0]
+    return ('Chat: ' + first)[:180]
+
+
+def fingerprint(title, version):
+    # mirror api/debug/report.php:146 — sha1(title|json_encode(errKey)|version); chat reports carry
+    # no errors array, so errKey is [] and PHP's json_encode([]) is "[]".
+    return hashlib.sha1((title + '|[]|' + version).encode('utf-8')).hexdigest()
+
+
+def existing_msg_ids():
+    seen = {}
+    for p in glob.glob(os.path.join(REPORTS, 'RPT-*.json')):
+        r = load(p, {})
+        chat = ((r.get('diagnostics') or {}).get('chat') or {})
+        mid = chat.get('msg_id')
+        if mid:
+            seen[mid] = r.get('id')
+    return seen
+
+
+def build_report(thread, msg):
+    text = str(msg.get('text') or '')[:8000]
+    version = version_of(text)
+    title = title_for(msg)
+    now = msg.get('ts') or ''
+    stamp = re.sub(r'[^0-9]', '', now)[:14] or '19700101T000000'
+    rid = 'RPT-%s-%s-%s' % (stamp[:8] + '-' + (stamp[8:14] or '000000'),
+                            'chat', hashlib.sha1((thread + ':' + str(msg.get('id'))).encode()).hexdigest()[:8])
+    return {
+        'id': rid,
+        'created_at': now,
+        'fingerprint': fingerprint(title, version),
+        'title': title,
+        'body': text,
+        'diagnostics': {
+            'version': version,
+            'url': '',
+            'ua': 'chat-bridge',
+            'auto': False,                 # -> the auto-reviewer leaves it for the operator
+            'errors': [],
+            'chat': {'thread': thread, 'msg_id': msg.get('id'),
+                     'author': msg.get('author'), 'ts': now},
+        },
+        'thread': '',
+        'kind': 'user',
+        'meta': {'ip': '', 'ua': 'chat-bridge'},
+    }
+
+
+def scan(include_seen=False):
+    state = load(STATE, {'v': 1, 'seen': {}})
+    seen = state.get('seen', {})
+    already = existing_msg_ids()
+    out = []
+    for path in sorted(glob.glob(os.path.join(CHAT, '*.json'))):
+        thread = os.path.basename(path)[:-5]
+        if thread.startswith('.'):
+            continue                        # .rate-* sidecars
+        for msg in messages(path):
+            mid = str(msg.get('id') or '')
+            if not mid or not is_report(msg):
+                continue
+            if not include_seen and (mid in seen or mid in already):
+                continue
+            out.append((thread, msg))
+    return out, state
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--apply', action='store_true', help='write the RPT files (default: dry run)')
+    ap.add_argument('--include-seen', action='store_true', help='show already-ingested messages too')
+    a = ap.parse_args()
+
+    found, state = scan(a.include_seen)
+    if not found:
+        return 0                            # silent: nothing to ingest (no_agent cron contract)
+
+    filed = []
+    for thread, msg in found:
+        rec = build_report(thread, msg)
+        print('%s  %s  %s  %s' % ('FILE' if a.apply else 'DRY ', rec['id'],
+                                  rec['diagnostics']['chat']['msg_id'][:28], rec['title'][:80]))
+        if not a.apply:
+            continue
+        path = os.path.join(REPORTS, rec['id'] + '.json')
+        if os.path.exists(path):
+            continue                        # id collision cannot happen, but never overwrite
+        save_json(path, rec)
+        os.chmod(path, 0o664)
+        state.setdefault('seen', {})[rec['diagnostics']['chat']['msg_id']] = rec['id']
+        filed.append(rec['id'])
+
+    if a.apply and filed:
+        save_json(STATE, state)
+        if os.path.exists(GENERATOR):
+            subprocess.run(['/usr/bin/python3', GENERATOR], capture_output=True, timeout=120)
+        for rid in filed:
+            print('filed ' + rid)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())