#!/usr/bin/env python3 """Regenerate the newOS4 changelog page from git history. Why git: the page used to be built by hand from VFS documents, so it had holes (v4.6.59 - v4.6.100 were "unknown"). Git is the one source that has every change, with dates and hashes, and it cannot drift from what shipped. The page is its own template: everything visible comes from the JSON island (`#changelog-data`), so regenerating means swapping that island and refreshing the two prose blocks that described the old data source. No design duplication, no second copy of the CSS or the renderer. Run: python3 scripts/build-changelog.py # write page + CHANGELOG.md + json + atom python3 scripts/build-changelog.py --check # exit 1 if stale, write nothing Scale: the page is one file while it stays under ~150 builds (measured 2026-10-27: 68 builds = 63 KB, so ~200 KB lands near 200 builds). Past that, split by calendar year -- changelog.html keeps the newest 40 builds, changelog-.html takes the rest, and this script links them. Trigger the work when len(page['entries']) >= 150 stops being unusual, not when the page feels slow in a browser.""" import html import json import os import re import subprocess import sys from collections import Counter, OrderedDict from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) APP = os.path.dirname(HERE) # spaces/newos4 ROOT = os.path.dirname(os.path.dirname(APP)) # repo root PAGE = os.path.join(ROOT, 'spaces/agando/downloads/changelog.html') MD = os.path.join(APP, 'CHANGELOG.md') LOADER = os.path.join('spaces/newos4/modules/core/module-loader.js') APP_REL = 'spaces/newos4' # commit prefix -> the page's own kind vocabulary (KIND_LABEL in the renderer) KIND = OrderedDict([ ('feat', 'release'), ('fix', 'fix'), ('perf', 'fix'), ('revert', 'fix'), ('test', 'review'), ('refactor', 'project'), ('docs', 'docs'), ('style', 'project'), ('chore', 'project'), ('build', 'project'), ('ci', 'project'), ]) MAX_ITEMS = 14 # per release; the rest is counted, not hidden def git(*args): # encoding is pinned: the cron runs with an unset locale and commit subjects carry non-ASCII # (em dashes, ellipses). Without it a decode error would kill every 07:15 regeneration. return subprocess.run(['git', '-C', ROOT] + list(args), capture_output=True, text=True, encoding='utf-8', check=True).stdout def commits(): """Oldest first: [{sha, date, subject, body, files:[...]}] `%b` is read, not just `%s`: the shipped fixes carry their symptom -> cause -> fix -> verified prose in the commit BODY (5b55af9 is a typical one), so a subject-only changelog rendered "one-line summaries" for every entry and made README's claim about this file false as served. The body is terminated by %x02 because it is multi-line and the parser below is line-based; without a terminator a body line is indistinguishable from a path. """ raw = git('log', '--reverse', '--date=iso-strict', '--format=%x01%H%x01%aI%x01%s%x01%b%x02', '--name-only', '--', APP_REL) out, cur, where = [], None, 'body' for line in raw.split('\n'): if line.startswith('\x01'): parts = line.split('\x01', 4) if len(parts) < 5: continue _, sha, iso, subject, line = parts cur = {'sha': sha, 'iso': iso, 'date': iso[:10], 'subject': subject, 'body': [], 'files': []} out.append(cur) where = 'body' if cur is None: continue if where == 'body': if '\x02' in line: head = line.split('\x02', 1)[0] if head.strip(): cur['body'].append(head) where = 'files' else: # Blank lines are kept: they are the paragraph boundary the renderer splits on. cur['body'].append(line.rstrip()) elif line.strip(): cur['files'].append(line.strip()) for row in out: row['body'] = '\n'.join(row['body']).strip() return out _TAGMAP = None def tag_map(): """commit -> build tag, in ONE git call. `git describe --tags --abbrev=0 ` per commit is correct but costs a history walk each time: measured 2.5 s for 158 commits versus 1.0 s for plain tree reads. One for-each-ref pass gives the same authority (a tag is a ref, not an inference) at no per-commit cost. `%(*objectname)` dereferences annotated tags to the commit they point at. """ global _TAGMAP if _TAGMAP is None: _TAGMAP = {} try: out = git('for-each-ref', 'refs/tags', '--format=%(refname:short) %(*objectname)') except subprocess.CalledProcessError: out = '' for line in out.splitlines(): parts = line.split() if len(parts) == 2 and re.match(r'^v?\d+(\.\d+)+$', parts[0]): _TAGMAP[parts[1]] = parts[0] return _TAGMAP def version_at(sha): # A tag is the authority; the tree read is only needed for commits that predate tagging. tagged = tag_map().get(sha) if tagged: return tagged try: src = git('show', f'{sha}:{LOADER}') except subprocess.CalledProcessError: return None m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", src) if not m: return None # Only a version-shaped stamp is a build id; early builds used prose/date strings, which would # otherwise become nonsense chip labels like "2026-06-02-fontfix3". return m.group(1) if re.match(r'^v?\d+(\.\d+)+$', m.group(1).strip()) else None def current_version(): with open(os.path.join(ROOT, LOADER), encoding='utf-8') as fh: m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", fh.read()) if not m: raise SystemExit('module-loader.js has no LATEST_VER - refusing to guess a version') return m.group(1) def kind_of(subject): head = subject.split(':', 1)[0].split('(', 1)[0].strip().lower() return KIND.get(head, 'project') def clean(subject): """Drop the conventional prefix; the kind is shown as a chip already. Subjects written by this project use `backticks` for identifiers, and the page injects item text as author-controlled HTML, so turn each pair into . Everything is escaped first - a commit subject is still untrusted text. """ body = subject.split(':', 1)[1].strip() if ':' in subject else subject body = body[0].upper() + body[1:] if body else subject return inline(body) def inline(text): """Escape, then backticks -> . Shared by subjects and body prose. Escape ONCE, before wrapping: escaping the captured group again double-encodes any `&`/`<`/`>` that sits inside a code span (measured: the "run `git show `" marker shipped as &lt;sha&gt; and reached CHANGELOG.md as <sha>). """ return re.sub(r'`([^`]+)`', lambda m: '' + m.group(1) + '', html.escape(text)) # A commit body can run to 2 KB and this page lists every build, so body detail is capped and # the cut is MARKED: a reader must be able to tell a summary from a full transcript. DETAIL_MAX_CHARS = 900 DETAIL_MAX_BLOCKS = 8 def detail_blocks(text): """Commit body -> [('p'|'li', plain text)], blank-line separated. Bullet runs become list items (they read as the symptom/cause/fix/verified lines this project writes); wrapped prose lines are rejoined into one paragraph. Without %b the changelog could only ever show a subject, whatever the commit actually documented. """ blocks = [] for chunk in re.split(r'\n\s*\n', (text or '').strip()): lines = [ln.strip() for ln in chunk.splitlines() if ln.strip()] if not lines: continue if all(re.match(r'^[-*\u2022]\s+', ln) for ln in lines): for ln in lines: blocks.append(('li', re.sub(r'^[-*\u2022]\s+', '', ln))) continue prose = [] for ln in lines: if re.match(r'^[-*\u2022]\s+', ln): if prose: blocks.append(('p', ' '.join(prose))) prose = [] blocks.append(('li', re.sub(r'^[-*\u2022]\s+', '', ln))) else: prose.append(ln) if prose: blocks.append(('p', ' '.join(prose))) out, used = [], 0 for kind, body in blocks: if len(out) >= DETAIL_MAX_BLOCKS or used + len(body) > DETAIL_MAX_CHARS: out.append(('p', '\u2026 run `git show ` for the rest')) break out.append((kind, body)) used += len(body) return out def detail_html(blocks): """Collapsible detail inside the list item: native
, so the page's renderer (item.innerHTML = html) needs no change and a reader can ignore it.""" if not blocks: return '' out, in_ul = ['
detail'], False for kind, body in blocks: if kind == 'li': if not in_ul: out.append('
    ') in_ul = True out.append('
  • %s
  • ' % inline(body)) else: if in_ul: out.append('
') in_ul = False out.append('

%s

' % inline(body)) if in_ul: out.append('') out.append('
') return ''.join(out) def build(): rows = commits() if not rows: raise SystemExit('no commits touch ' + APP_REL + ' - nothing to build') # Each commit is stamped with the version that was live *at that commit*, so a # release group is "everything that landed while this version was the build". for row in rows: row['version'] = version_at(row['sha']) version = None releases = OrderedDict() for row in rows: if row['version']: version = row['version'] if version: releases.setdefault(version, []).append(row) else: releases.setdefault('\x00pre', []).append(row) ver_now = current_version() first_known = next((v for v in releases if v != '\x00pre'), ver_now) if '\x00pre' in releases: # Commits that predate the first LATEST_VER stamp share one honest label, and it has to # stay FIRST: the entries list is reversed for display, so order here is chronological. fixed = OrderedDict() for key, group in releases.items(): fixed[('before ' + first_known) if key == '\x00pre' else key] = group releases = fixed entries = [] for ver, group in releases.items(): items = [] # The NEWEST commits of a build, not the oldest: group is chronological, so group[:N] # showed the beginning of a 17-commit build and dropped the fixes that shipped last — # measured 2026-11-06 when this fix's own commit was cut off its own entry. shown = group[-MAX_ITEMS:] if len(group) > MAX_ITEMS: items.append('…and %d earlier commit(s) in this build' % (len(group) - MAX_ITEMS)) for row in shown: items.append('%s %s%s' % ( clean(row['subject']), html.escape(row['sha'][:7]), detail_html(detail_blocks(row.get('body'))))) kinds = Counter(kind_of(r['subject']) for r in group) # The dominant kind makes the chip honest: a build that was 9 fixes and 1 # feature is a fix build, not a feature release. kind = kinds.most_common(1)[0][0] files = [f for r in group for f in r['files']] areas = Counter(f.split('/')[2] for f in files if f.count('/') >= 2) entries.append({ 'kind': kind, 'version': ver, 'date': group[-1]['date'], 'updated': group[-1]['iso'], 'title': '%s — %d commit%s' % (ver, len(group), '' if len(group) == 1 else 's'), 'items': items, 'note': 'Touches ' + ', '.join('%s (%d)' % (a, n) for a, n in areas.most_common(4)) + ' · ' + ', '.join('%s %d' % (k, n) for k, n in kinds.most_common(3)), 'sources': sorted({f for r in group for f in r['files'][:1]})[:4], 'commits': [r['sha'][:7] for r in shown], }) entries.reverse() # newest build first first_date = rows[0]['date'] repo_first = (git('log', '--max-parents=0', '--date=short', '--format=%ad').split() or ['?'])[0] gaps = [] if repo_first < first_date: gaps.append({'range': 'before ' + first_date, 'why': 'The repository holds commits from ' + repo_first + ', but nothing that ' 'touches ' + APP_REL + ' before this date - earlier work predates the ' 'directory or was never committed here.'}) gaps.append({'range': 'missing build numbers', 'why': 'A version bumped twice before anything was committed leaves no trace: only ' 'builds whose tree reached a commit can appear here. Read a jump in the ' 'version chip as "those bumps were superseded before commit".'}) gaps.append({'range': 'every build', 'why': 'Entries carry the commit subject and, where the commit body has ' 'substance, its symptom / cause / fix / verification text (folded open ' 'per line on the page, nested under the line in CHANGELOG.md). A build ' 'whose work was squashed into one commit still shows one line; the diff ' 'itself is always `git show `.'}) now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') prev = existing_island() # `generated` is the last time the CONTENT changed; `checked` is the last time this ran. Two # fields because one cannot serve both: a daily rewrite would churn 65 KB of git for nothing, # while a content-only stamp cannot tell a live cron from a dead one. content_same = bool(prev) and prev.get('entries') == entries page = { 'generated': (prev or {}).get('generated', now) if content_same else now, 'checked': now, 'currentVersion': ver_now, 'stats': { 'entries': len(entries), 'versions': '%s – %s' % (first_known, ver_now), 'records': len(rows), }, 'gaps': gaps, 'entries': entries, } return page, rows DEK = ('Every change that has a commit behind it. Each line names the build it landed in and the ' 'commit it came from, and folds open to the commit body — the symptom, cause, fix and how ' 'it was verified — so any claim here can be checked with git show. Where ' 'the repository has no record, the gap is written down instead of filled in.') GAP_INTRO = ('This page is generated from the repository\'s own history for ' 'spaces/newos4 — the same source the running build comes from, so ' 'there are no unexplained stretches in the release line. What it cannot tell you is ' 'the shape of a squash-merged change.') def write_atomic(path, text): """Temp + rename: nginx serves these files straight off disk, so an in-place truncate can hand a visitor a half-written page (or leave one after a crash).""" tmp = path + '.tmp' with open(tmp, 'w', encoding='utf-8') as fh: fh.write(text) os.replace(tmp, path) def existing_island(): try: with open(PAGE, encoding='utf-8') as fh: return json.loads(fh.read().split('id="changelog-data">')[1].split('')[0]) except Exception: return None # Rendered as native
inside each list item, so the page's own JS needs no change. # Injected once (sentinel-guarded) because the page is a static document: a builder that only # rewrites its JSON island cannot restyle anything else. DETAIL_CSS = """ /* wx11-changelog-detail:BEGIN */ .entry li>details{margin:.3rem 0 0} .entry li>details>summary{ cursor:pointer; font-family:var(--mono); font-size:.7rem; letter-spacing:.08em; text-transform:uppercase; color:var(--ink-3); padding:.45rem 0; } .entry li>details>summary:hover{color:var(--cyan)} .entry li>details p{margin:.35rem 0 0; font-size:.88rem; color:var(--ink-2); line-height:1.55; overflow-wrap:anywhere} .entry li>details ul{margin:.3rem 0 0; padding-left:1rem; font-size:.88rem; color:var(--ink-2); line-height:1.55; overflow-wrap:anywhere} /* wx11-changelog-detail:END */ """ DETAIL_CSS_RX = re.compile(r'/\* wx11-changelog-detail:BEGIN \*/[\s\S]*?/\* wx11-changelog-detail:END \*/\n?') # The first version shipped a single-line sentinel and no end marker; strip it or the page # would keep both copies forever. LEGACY_CSS_RX = re.compile(r'\s*/\* wx11-changelog-detail: collapsible commit body \*/' r'[\s\S]*?\.entry li>details ul\{[^}]*\}\n') def write_page(page): with open(PAGE, encoding='utf-8') as fh: doc = fh.read() # json.dumps leaves " would end the island # early and truncate the page. Escape it; JSON.parse reads "<\/" back as ")[\s\S]*?()', lambda m: m.group(1) + '\n' + island + '\n' + m.group(2), doc, count=1) assert n == 1, 'changelog JSON island not found - page layout changed' # Prose that described the old VFS-only source; replaced by pattern so re-runs are idempotent. new = re.sub(r'(

)[\s\S]*?(

)', lambda m: m.group(1) + DEK + m.group(2), new, count=1) new = re.sub(r'(
[\s\S]*?

[^<]*

\s*

)[\s\S]*?(

)', lambda m: m.group(1) + GAP_INTRO + m.group(2), new, count=1) new = re.sub(r'()[^<]*()', lambda m: m.group(1) + html.escape(page['currentVersion']) + m.group(2), new, count=1) new = LEGACY_CSS_RX.sub('\n', new, count=1) if DETAIL_CSS_RX.search(new): # Owned block: re-emitted every run, so the rules can be edited here (the page is # static; a stale copy on disk would silently win). new = DETAIL_CSS_RX.sub(lambda m: DETAIL_CSS, new, count=1) else: new, n = re.subn(r'(\n\s*)', lambda m: '\n' + DETAIL_CSS + m.group(1), new, count=1) assert n == 1, 'no to inject the detail CSS before - page layout changed' write_atomic(PAGE, new) return len(new) PREAMBLE = ('# newOS4 changelog\n\n' 'Generated from git history by `scripts/build-changelog.py` — do not edit by hand ' 'between the markers.\n\n') # The preamble is emitted EXACTLY once. Before 2026-11-05 it was part of the generated body *and* # old[:m.start()] was preserved on top of it, so each run prepended another copy: 44 runs left 44 # copies of the H1 at the top of the file (and pushed the real first release from line 6 to 179). PREAMBLE_RE = re.compile(re.escape('# newOS4 changelog') + r'\n\nGenerated from git history by `scripts/build-changelog\.py` — ' r'do not edit by hand between the markers\.\n+') DETAIL_RX = re.compile(r']*>\s*[^<]*(.*?)
\s*$', re.S) def md_inline(markup): """Our own generated item markup -> markdown. The old line replaced EVERY tag with a backtick, so `` became a stray tick and a body's paragraph tags would have shredded the file. Explicit rules beat a generic strip: / survive as markdown, everything else becomes a space. """ text = re.sub(r'(.*?)', lambda m: '`' + m.group(1) + '`', markup, flags=re.S) text = re.sub(r'(.*?)', lambda m: '**' + m.group(1) + '**', text, flags=re.S) text = re.sub(r'<[^>]+>', ' ', text) return re.sub(r'\s+', ' ', html.unescape(text)).strip() def md_item(item): """One changelog line, with the commit body as nested bullets. This is the file the public mirror serves, so the README claim "symptom -> cause -> fix -> verified" has to be true HERE, not only on the rendered page. """ m = DETAIL_RX.search(item) lines = ['- ' + md_inline(item[:m.start()] if m else item)] if m: for _tag, body in re.findall(r'<(p|li)>(.*?)', m.group(1), re.S): body = md_inline(body) if body: lines.append(' - ' + body) return lines def write_md(page, rows): """App-side Keep-a-Changelog history; only the region between the markers is generated.""" region = ['', ''] for entry in page['entries']: region.append('## %s — %s' % (entry['version'], entry['date'])) region.append('') if entry['note']: region.extend(['_' + re.sub(r'<[^>]+>', '', entry['note']) + '_', '']) for item in entry['items']: region.extend(md_item(item)) region.append('') region.extend(['', '']) region_text = '\n'.join(region) if os.path.exists(MD): with open(MD, encoding='utf-8') as fh: old = fh.read() m = re.search(r'[\s\S]*?\n?', old) if not m: # Refuse rather than replace: without the markers the file may hold prose that a # hand-edit added, and silently dropping it is worse than failing loudly. raise SystemExit('CHANGELOG.md exists but has no generated:start/end markers - ' 'refusing to overwrite it') head, tail = old[:m.start()], old[m.end():] copies = len(PREAMBLE_RE.findall(head)) if copies != 1: rest = PREAMBLE_RE.sub('', head).strip('\n') head = PREAMBLE + (rest + '\n\n' if rest else '') print('changelog: collapsed %d preamble copies in CHANGELOG.md' % copies) body = head + region_text + tail else: body = PREAMBLE + region_text # Guard against the 2026-11-05 regression class: the generated region may appear exactly once. # The old code concatenated the preamble into `body` and kept old[:m.start()] as well, so every # run appended another copy — 44 runs, 44 H1s, and the first release pushed to line 179. assert body.count('') == 1, 'CHANGELOG.md would contain a duplicated region' write_atomic(MD, body) return len(body) def write_feed(page): """Same data, two machine shapes: JSON for tools, Atom for readers. Entry dates are day-precision, so gets T00:00:00Z appended rather than pretending to a time of day. Item markup is stripped: a feed reader gets text, not HTML. """ base = os.path.dirname(PAGE) write_atomic(os.path.join(base, 'changelog.json'), json.dumps(page, ensure_ascii=False, indent=1)) entries = [] for e in page['entries']: title = '%s — %s' % (e['version'], e['date']) body = re.sub(r'\s+', ' ', ' '.join(re.sub(r'<[^>]+>', ' ', i).strip() for i in e['items'])).strip() # Truncate the PLAIN text, then escape: slicing an escaped string can cut "<" in half # and leave garbage at the tail of every long summary. if len(body) > 900: body = body[:900].rsplit(' ', 1)[0] + ' …' # A tag: URI cannot contain spaces ("before v4.6.10") and must not hardcode the year. year = (e['date'] or '')[:4] or '2026' slug = re.sub(r'[^A-Za-z0-9._-]+', '-', e['version']) anchor = re.sub(r'[^a-z0-9]+', '-', e['version'].lower()) entries.append( ' \n' ' %s\n' ' tag:webx11.duckdns.org,%s:%s\n' ' \n' ' %s\n' ' %s\n' ' ' % (html.escape(title), year, slug, anchor, e.get('updated') or (e['date'] + 'T00:00:00Z'), html.escape(body)) ) feed = ('\n' '\n' ' newOS4 changelog\n' ' tag:webx11.duckdns.org,2026:changelog\n' ' %s\n WebX11\n%s\n\n' % (page['generated'], '\n'.join(entries))) write_atomic(os.path.join(base, 'feed.xml'), feed) return len(feed) def main(): page, rows = build() if '--check' in sys.argv: with open(PAGE, encoding='utf-8') as fh: doc = fh.read() m = re.search(r'"currentVersion"\s*:\s*"([^"]+)"', doc) have = m.group(1) if m else 'none' if have != page['currentVersion']: print('changelog stale: page says %s, build is %s' % (have, page['currentVersion'])) return 1 print('changelog current: %s (%d entries)' % (have, len(page['entries']))) return 0 size = write_page(page) mdsize = write_md(page, rows) fsize = write_feed(page) print('changelog: %d entries, %d commits, %s -> page %d bytes, md %d bytes, feed %d bytes' % (len(page['entries']), len(rows), page['stats']['versions'], size, mdsize, fsize)) return 0 if __name__ == '__main__': sys.exit(main())