#!/usr/bin/env python3
"""newos4-chat-ingest.py — turn report-like chat messages into RPT-* queue items.

The chat bridge (api/chat/post.php) stores threads in data/chat/<thread>.json. Nothing in the
feedback pipeline reads that directory, so a bug report posted through the in-page agent never
becomes a card (measured 2026-11-02: 22 report-like messages, 0 reports).

This scanner is read-only by default. With --apply it writes one RPT-*.json per new message, in the
same schema api/debug/report.php produces, with diagnostics.auto = false so the auto-reviewer leaves
it for a human (newos4-feedback-autoreview.py:165).

Dedupe: data/feedback/chat-ingest-state.json (thread:msg_id -> report id) AND an existing report
carrying the same diagnostics.chat.msg_id, so losing the state file cannot duplicate an item.
"""
import argparse
import glob
import hashlib
import json
import os
import re
import subprocess
import sys

ROOT = '/var/www/webx11.duckdns.org'
CHAT = f'{ROOT}/spaces/newos4/data/chat'
REPORTS = f'{ROOT}/spaces/newos4/data/reports'
STATE = f'{ROOT}/spaces/newos4/data/feedback/chat-ingest-state.json'
GENERATOR = f'{ROOT}/spaces/newos4/scripts/gen-bug-report.py'

# A message is a report when it opens a section with one of these headers. Chat smalltalk, the
# relay's own replies and one-line questions never match.
REPORTISH = re.compile(r'(?m)^\s*(BUG|FEEDBACK|SYMPTOM|ROOT CAUSE|SECOND DEFECT|REPRO|MODULE)\b')
# Client-blip traffic that the ingest filter (api/debug/report.php) already refuses to card.
NOISE = re.compile(r'(poll\.php|networkerror|failed to fetch|signal is aborted|ERR_NETWORK)', re.I)
# Work orders and replies, not defects: the 2026-10-23 thread carries two DEPLOY SPECs that would
# otherwise enter the defect queue (dry run 2026-11-02 filed them as 2 of 7 candidates).
NOT_A_REPORT = re.compile(r'(?m)^\s*(DEPLOY SPEC|ACKNOWLEDGED|RETRACTED|THANK YOU)\b')
SKIP_AUTHORS = {'hermes'}   # our own replies (the relay's answers are not reports)
MIN_LEN = 200                      # a real report is never a one-liner


def load(path, default):
    try:
        with open(path, encoding='utf-8') as fh:
            return json.load(fh)
    except Exception:
        return default


def save_json(path, obj):
    tmp = path + '.tmp'
    with open(tmp, 'w', encoding='utf-8') as fh:
        json.dump(obj, fh, indent=1, ensure_ascii=False)
    os.replace(tmp, path)


def messages(path):
    d = load(path, [])
    return d if isinstance(d, list) else (d.get('messages') or [])


def version_of(text):
    m = re.search(r'\bv(\d+\.\d+\.\d+)\b', text)
    return 'v' + m.group(1) if m else 'unknown'


def is_report(msg):
    if str(msg.get('author')) in SKIP_AUTHORS:
        return False
    text = str(msg.get('text') or '')
    if len(text) < MIN_LEN or not REPORTISH.search(text):
        return False
    if NOT_A_REPORT.match(text.strip()):
        return False
    if NOISE.search(text):
        # a report may quote a fetch failure; what disqualifies it is *being* one
        return bool(re.search(r'(?m)^\s*(ROOT CAUSE|BUG)\b', text))
    return True


def title_for(msg):
    first = str(msg.get('text') or '').strip().splitlines()[0]
    return ('Chat: ' + first)[:180]


def fingerprint(title, version):
    # mirror api/debug/report.php:146 — sha1(title|json_encode(errKey)|version); chat reports carry
    # no errors array, so errKey is [] and PHP's json_encode([]) is "[]".
    return hashlib.sha1((title + '|[]|' + version).encode('utf-8')).hexdigest()


def existing_msg_ids():
    seen = {}
    for p in glob.glob(os.path.join(REPORTS, 'RPT-*.json')):
        r = load(p, {})
        chat = ((r.get('diagnostics') or {}).get('chat') or {})
        mid = chat.get('msg_id')
        if mid:
            seen[mid] = r.get('id')
    return seen


def build_report(thread, msg):
    text = str(msg.get('text') or '')[:8000]
    version = version_of(text)
    title = title_for(msg)
    now = msg.get('ts') or ''
    stamp = re.sub(r'[^0-9]', '', now)[:14] or '19700101T000000'
    rid = 'RPT-%s-%s-%s' % (stamp[:8] + '-' + (stamp[8:14] or '000000'),
                            'chat', hashlib.sha1((thread + ':' + str(msg.get('id'))).encode()).hexdigest()[:8])
    return {
        'id': rid,
        'created_at': now,
        'fingerprint': fingerprint(title, version),
        'title': title,
        'body': text,
        'diagnostics': {
            'version': version,
            'url': '',
            'ua': 'chat-bridge',
            'auto': False,                 # -> the auto-reviewer leaves it for the operator
            'errors': [],
            'chat': {'thread': thread, 'msg_id': msg.get('id'),
                     'author': msg.get('author'), 'ts': now},
        },
        'thread': '',
        'kind': 'user',
        'meta': {'ip': '', 'ua': 'chat-bridge'},
    }


def scan(include_seen=False):
    state = load(STATE, {'v': 1, 'seen': {}})
    seen = state.get('seen', {})
    already = existing_msg_ids()
    out = []
    for path in sorted(glob.glob(os.path.join(CHAT, '*.json'))):
        thread = os.path.basename(path)[:-5]
        if thread.startswith('.'):
            continue                        # .rate-* sidecars
        for msg in messages(path):
            mid = str(msg.get('id') or '')
            if not mid or not is_report(msg):
                continue
            if not include_seen and (mid in seen or mid in already):
                continue
            out.append((thread, msg))
    return out, state


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--apply', action='store_true', help='write the RPT files (default: dry run)')
    ap.add_argument('--include-seen', action='store_true', help='show already-ingested messages too')
    a = ap.parse_args()

    found, state = scan(a.include_seen)
    if not found:
        return 0                            # silent: nothing to ingest (no_agent cron contract)

    filed = []
    for thread, msg in found:
        rec = build_report(thread, msg)
        print('%s  %s  %s  %s' % ('FILE' if a.apply else 'DRY ', rec['id'],
                                  rec['diagnostics']['chat']['msg_id'][:28], rec['title'][:80]))
        if not a.apply:
            continue
        path = os.path.join(REPORTS, rec['id'] + '.json')
        if os.path.exists(path):
            continue                        # id collision cannot happen, but never overwrite
        save_json(path, rec)
        os.chmod(path, 0o664)
        state.setdefault('seen', {})[rec['diagnostics']['chat']['msg_id']] = rec['id']
        filed.append(rec['id'])

    if a.apply and filed:
        save_json(STATE, state)
        if os.path.exists(GENERATOR):
            subprocess.run(['/usr/bin/python3', GENERATOR], capture_output=True, timeout=120)
        for rid in filed:
            print('filed ' + rid)
    return 0


if __name__ == '__main__':
    sys.exit(main())