1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
#!/usr/bin/env python3
"""Snapshot agent-side review items for /admin/feedback.html.
Runs as darkbot (Hermes cron): .hermes/plans/*.md are 600 darkbot, unreadable to php-fpm,
so the snapshot is the only way the page can see them. Writes atomically, 644 www-data.
Read-only with respect to every source file.
"""
import glob, json, os, re, time
from datetime import datetime, timezone
ROOT = '/var/www/webx11.duckdns.org'
OUT_DIR = os.path.join(ROOT, 'admin/docs/feedback')
OUT = os.path.join(OUT_DIR, 'agent-items.json')
PLANS = os.path.join(ROOT, '.hermes/plans')
TRIAGE = os.path.join(ROOT, 'admin/docs/newos4-triage-latest.json')
STEER = os.path.join(ROOT, 'spaces/newos4/data/maintenance/steering.json')
DECISIONS = os.path.join(ROOT, 'spaces/newos4/data/feedback/decisions.jsonl')
APPLIED = os.path.join(ROOT, 'spaces/newos4/data/feedback/applied.json') # ledger written by the applier
NETFAIL = os.path.join(ROOT, 'spaces/newos4/data/debug/network_failures.json') # ingest-filter counter
AUTOREVIEW = os.path.join(ROOT, 'spaces/newos4/data/feedback/autoreview-latest.json')
# A finished plan keeps its '<!-- autonomy: review -->' marker (the applier only flips
# review<->safe), so the marker alone overstates the queue. The body is the evidence: a Status /
# Implementation heading followed by 'shipped'/'executed' plus a version, a commit ref or a dated
# stamp. Order-free on purpose — "executed 2026-10-26" and "shipped v4.6.128 (75c5113)" both count.
STAMP_WORD = re.compile(r'(?i)\b(?:shipped|executed)\b')
STAMP_REF = re.compile(r'\b(?:[vV]\d+\.\d+(?:\.\d+)?|[0-9a-f]{7,40}|20\d\d-\d\d-\d\d)\b')
def execution_stamp(lines):
"""True when a Status/Implementation heading carries an execution stamp.
Looks at the heading plus the ~8 lines under it. Truncated or empty files simply fail the
test; this never raises.
"""
for i, line in enumerate(lines):
if not (line.startswith('## Status') or line.startswith('## Implementation')):
continue
block = '\n'.join(lines[i:i + 9])
if STAMP_WORD.search(block) and STAMP_REF.search(block):
return True
return False
def load(path, default):
try:
with open(path, encoding='utf-8') as f:
return json.load(f)
except Exception:
return default
def decisions():
"""Latest decision per item id, from the append-only log written by php."""
out = {}
if not os.path.exists(DECISIONS):
return out
try:
with open(DECISIONS, encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
try:
d = json.loads(line)
except Exception:
continue # torn line: ignore, never a crash
out[d.get('item_id', '')] = d
except Exception:
pass
return out
def plan_key(name):
# The ledger has both spellings (2026-10-23..27 rows carry no `.md`), so a strict
# `dec.get('plan:' + name)` misses five decisions and the page then shows decided plans
# as awaiting. Normalise on READ — the ledger is an audit log, never rewritten.
return 'plan:' + (name if name.endswith('.md') else name + '.md')
def plans_needing_decision(dec):
items = []
for p in sorted(glob.glob(os.path.join(PLANS, '*.md'))):
try:
with open(p, encoding='utf-8', errors='ignore') as fh:
head = ''.join(fh.readline() for _ in range(40))
except Exception:
continue
# Exact-line match: the analyser contract puts the marker on the line after the
# title. A substring test also matches plans that merely *document* the marker
# (measured 2026-10-26: this plan's own audit table inflated the count to 4).
if not any(l.strip() == '<!-- autonomy: review -->' for l in head.splitlines()):
continue
name = os.path.basename(p)
title = next((l.lstrip('# ').strip() for l in head.splitlines() if l.startswith('# ')), name)
try:
body = open(p, encoding='utf-8', errors='ignore').read()
except Exception:
body = ''
lines = body.splitlines()
# A decision turns on the plan's own text: the opening 60 lines carry Goal/Approach and the
# task headings show the shape of the work. The full file is 20-40 KB per plan — too much.
excerpt = '\n'.join(lines[:60])[:2400]
tasks = [l.lstrip('#').strip() for l in lines if re.match(r'^#{2,3}\s+Task\b', l)][:20]
status = [l.strip() for l in lines if l.startswith('## Status') or l.startswith('## Implementation')][:2]
done = execution_stamp(lines)
d = dec.get(plan_key(name), {})
items.append({
'id': 'plan:' + name, 'kind': 'plan', 'title': title, 'path': p,
'size': os.path.getsize(p), 'mtime': datetime.fromtimestamp(
os.path.getmtime(p), timezone.utc).isoformat(timespec='seconds'),
'excerpt': excerpt, 'tasks': tasks,
# 'done' when the body records the execution; otherwise the raw heading lines, as before.
# Done plans stay in this list so the page can show them as shipped, but they no longer
# count as a pending review — see the plans_review count in main().
'status': 'done' if done else status, 'status_lines': status,
'decision': d.get('decision'), 'decided_at': d.get('decided_at') or d.get('ts'),
'note': d.get('note', ''), 'actor': d.get('actor', ''),
})
return items
def reports_needing_decision(dec):
triage = load(TRIAGE, {}) or {}
new_ids = {r.get('id') for r in (triage.get('new_reports') or [])}
items = []
for p in sorted(glob.glob(os.path.join(ROOT, 'spaces/newos4/data/reports/*.json'))):
try:
r = json.load(open(p, encoding='utf-8'))
except Exception:
continue
rid = r.get('id') or os.path.basename(p)[:-5]
diag = r.get('diagnostics') or {}
errs = diag.get('errors') or []
d = dec.get('report:' + rid, {})
items.append({
'id': 'report:' + rid, 'kind': 'report', 'title': (r.get('title') or rid)[:180],
'created_at': r.get('created_at'), 'auto': bool(diag.get('auto')),
'new': rid in new_ids, 'decision': d.get('decision'),
'decided_at': d.get('decided_at') or d.get('ts'), 'note': d.get('note', ''),
'actor': d.get('actor', ''),
# A decision written in the 2026-10-26 bulk pass is not the same evidence as one
# written today; the card marks it so the provenance is visible at a glance.
'bulk_pass': bool(str(d.get('ts', '')).startswith('2026-10-26')),
'message': (r.get('body') or '')[:1500], 'page': diag.get('url') or '',
'version': diag.get('version') or '', 'ua': (diag.get('ua') or '')[:120],
'errors': [str(e)[:300] for e in errs[:3]],
})
items.sort(key=lambda x: (x.get('created_at') or ''), reverse=True)
return items
def recent_apply(n):
"""Last n decision outcomes, newest first — failures included, so the dashboard can show them."""
try:
with open(APPLIED, encoding='utf-8') as f:
d = json.load(f)
rows = [v for v in d.values() if isinstance(v, dict)]
rows.sort(key=lambda r: str(r.get('applied_at') or ''), reverse=True)
return rows[:n]
except Exception:
return []
def main():
dec = decisions()
steer_raw = load(STEER, [])
steer_ok = isinstance(steer_raw, list)
steer = [s for s in (steer_raw if steer_ok else []) if isinstance(s, dict) and not s.get('done')]
triage = load(TRIAGE, {}) or {}
plans = plans_needing_decision(dec)
reports = reports_needing_decision(dec)
# A plan whose body records its execution is shipped, not pending: it stays in `plans` (so the
# page can show it) but must not sit in the operator's "awaiting a decision" queue.
awaiting = [x for x in plans + reports if not x.get('decision') and x.get('status') != 'done']
net = load(NETFAIL, {}) or {}
net_top = sorted(((net.get('by_source') or {}) if isinstance(net.get('by_source'), dict) else {}).items(),
key=lambda kv: -int(kv[1]))[:3]
doc = {
'network_failures': {'day': net.get('day'), 'total': int(net.get('total') or 0),
'top': [[str(k), int(v)] for k, v in net_top]},
'generated_at': datetime.now(timezone.utc).isoformat(timespec='seconds'),
'counts': {
# Only plans still awaiting a decision: a review marker whose body already records the
# execution is a shipped plan, not an open question. It stays in 'plans' below.
'plans_review': sum(1 for p in plans if p.get('status') != 'done'),
'reports_total': len(reports),
'awaiting_decision': len(awaiting),
'steering_pending': len(steer) if steer_ok else -1, # -1 = steering.json unreadable/wrong shape
'triage_reports_new': (triage.get('counts') or {}).get('reports_new', 0),
# The chat plane used to be visible nowhere; this tile is the count of items that
# arrived through it (scripts/newos4-chat-ingest.py).
'chat_total': sum(1 for r in reports if '-chat-' in r['id']),
},
'plans': plans, 'reports': reports, 'steering': steer[:20],
'recent_apply': recent_apply(8),
'autoreview': {k: (load(AUTOREVIEW, {}) or {}).get(k) for k in ('generated_at', 'mode', 'decided', 'skipped')},
'triage_checks': (triage.get('checks') or {}),
'triage_generated_at': triage.get('generated_at'),
}
os.makedirs(OUT_DIR, exist_ok=True)
tmp = OUT + '.tmp'
with open(tmp, 'w', encoding='utf-8') as f:
json.dump(doc, f, ensure_ascii=False, indent=1)
os.replace(tmp, OUT)
os.chmod(OUT, 0o644)
try:
import grp
os.chown(OUT, -1, grp.getgrnam('www-data').gr_gid)
except Exception:
pass
print(f"feedback snapshot: plans={len(plans)} reports={len(reports)} "
f"awaiting={len(awaiting)} steering={len(steer)} -> {OUT}")
return 0
if __name__ == '__main__':
raise SystemExit(main())
|
