path: root/scripts/build-changelog.py

#!/usr/bin/env python3
"""Regenerate the newOS4 changelog page from git history.

Why git: the page used to be built by hand from VFS documents, so it had holes
(v4.6.59 - v4.6.100 were "unknown"). Git is the one source that has every
change, with dates and hashes, and it cannot drift from what shipped.

The page is its own template: everything visible comes from the JSON island
(`#changelog-data`), so regenerating means swapping that island and refreshing
the two prose blocks that described the old data source. No design duplication,
no second copy of the CSS or the renderer.

Run:
    python3 scripts/build-changelog.py            # write page + CHANGELOG.md + json + atom
    python3 scripts/build-changelog.py --check    # exit 1 if stale, write nothing

Scale: the page is one file while it stays under ~150 builds (measured 2026-10-27: 68 builds =
63 KB, so ~200 KB lands near 200 builds). Past that, split by calendar year -- changelog.html
keeps the newest 40 builds, changelog-<year>.html takes the rest, and this script links them.
Trigger the work when len(page['entries']) >= 150 stops being unusual, not when the page feels
slow in a browser."""

import html
import json
import os
import re
import subprocess
import sys
from collections import Counter, OrderedDict
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
APP = os.path.dirname(HERE)                       # spaces/newos4
ROOT = os.path.dirname(os.path.dirname(APP))      # repo root
PAGE = os.path.join(ROOT, 'spaces/agando/downloads/changelog.html')
MD = os.path.join(APP, 'CHANGELOG.md')
LOADER = os.path.join('spaces/newos4/modules/core/module-loader.js')
APP_REL = 'spaces/newos4'

# commit prefix -> the page's own kind vocabulary (KIND_LABEL in the renderer)
KIND = OrderedDict([
    ('feat', 'release'), ('fix', 'fix'), ('perf', 'fix'), ('revert', 'fix'),
    ('test', 'review'), ('refactor', 'project'), ('docs', 'docs'),
    ('style', 'project'), ('chore', 'project'), ('build', 'project'), ('ci', 'project'),
])
MAX_ITEMS = 14          # per release; the rest is counted, not hidden


def git(*args):
    # encoding is pinned: the cron runs with an unset locale and commit subjects carry non-ASCII
    # (em dashes, ellipses). Without it a decode error would kill every 07:15 regeneration.
    return subprocess.run(['git', '-C', ROOT] + list(args),
                          capture_output=True, text=True, encoding='utf-8', check=True).stdout


def commits():
    """Oldest first: [{sha, date, subject, files:[...]}]"""
    raw = git('log', '--reverse', '--date=iso-strict',
              '--format=%x01%H%x01%aI%x01%s', '--name-only', '--', APP_REL)
    out, cur = [], None
    for line in raw.split('\n'):
        if line.startswith('\x01'):
            _, sha, iso, subject = line.split('\x01', 3)
            cur = {'sha': sha, 'iso': iso, 'date': iso[:10], 'subject': subject, 'files': []}
            out.append(cur)
        elif line.strip() and cur is not None:
            cur['files'].append(line.strip())
    return out


_TAGMAP = None


def tag_map():
    """commit -> build tag, in ONE git call.

    `git describe --tags --abbrev=0 <sha>` per commit is correct but costs a history walk each
    time: measured 2.5 s for 158 commits versus 1.0 s for plain tree reads. One for-each-ref pass
    gives the same authority (a tag is a ref, not an inference) at no per-commit cost.
    `%(*objectname)` dereferences annotated tags to the commit they point at.
    """
    global _TAGMAP
    if _TAGMAP is None:
        _TAGMAP = {}
        try:
            out = git('for-each-ref', 'refs/tags', '--format=%(refname:short) %(*objectname)')
        except subprocess.CalledProcessError:
            out = ''
        for line in out.splitlines():
            parts = line.split()
            if len(parts) == 2 and re.match(r'^v?\d+(\.\d+)+$', parts[0]):
                _TAGMAP[parts[1]] = parts[0]
    return _TAGMAP


def version_at(sha):
    # A tag is the authority; the tree read is only needed for commits that predate tagging.
    tagged = tag_map().get(sha)
    if tagged:
        return tagged
    try:
        src = git('show', f'{sha}:{LOADER}')
    except subprocess.CalledProcessError:
        return None
    m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", src)
    if not m:
        return None
    # Only a version-shaped stamp is a build id; early builds used prose/date strings, which would
    # otherwise become nonsense chip labels like "2026-06-02-fontfix3".
    return m.group(1) if re.match(r'^v?\d+(\.\d+)+$', m.group(1).strip()) else None


def current_version():
    with open(os.path.join(ROOT, LOADER), encoding='utf-8') as fh:
        m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", fh.read())
    if not m:
        raise SystemExit('module-loader.js has no LATEST_VER - refusing to guess a version')
    return m.group(1)


def kind_of(subject):
    head = subject.split(':', 1)[0].split('(', 1)[0].strip().lower()
    return KIND.get(head, 'project')


def clean(subject):
    """Drop the conventional prefix; the kind is shown as a chip already.

    Subjects written by this project use `backticks` for identifiers, and the page
    injects item text as author-controlled HTML, so turn each pair into <code>.
    Everything is escaped first - a commit subject is still untrusted text.
    """
    body = subject.split(':', 1)[1].strip() if ':' in subject else subject
    body = body[0].upper() + body[1:] if body else subject
    return re.sub(r'`([^`]+)`', lambda m: '<code>' + html.escape(m.group(1)) + '</code>',
                  html.escape(body))


def build():
    rows = commits()
    if not rows:
        raise SystemExit('no commits touch ' + APP_REL + ' - nothing to build')

    # Each commit is stamped with the version that was live *at that commit*, so a
    # release group is "everything that landed while this version was the build".
    for row in rows:
        row['version'] = version_at(row['sha'])
    version = None
    releases = OrderedDict()
    for row in rows:
        if row['version']:
            version = row['version']
        if version:
            releases.setdefault(version, []).append(row)
        else:
            releases.setdefault('\x00pre', []).append(row)

    ver_now = current_version()
    first_known = next((v for v in releases if v != '\x00pre'), ver_now)
    if '\x00pre' in releases:
        # Commits that predate the first LATEST_VER stamp share one honest label, and it has to
        # stay FIRST: the entries list is reversed for display, so order here is chronological.
        fixed = OrderedDict()
        for key, group in releases.items():
            fixed[('before ' + first_known) if key == '\x00pre' else key] = group
        releases = fixed

    entries = []
    for ver, group in releases.items():
        items = []
        for row in group[:MAX_ITEMS]:
            items.append('<strong>%s</strong> <code>%s</code>' % (
                clean(row['subject']), html.escape(row['sha'][:7])))
        if len(group) > MAX_ITEMS:
            items.append('…and %d more commit(s) in this build' % (len(group) - MAX_ITEMS))
        kinds = Counter(kind_of(r['subject']) for r in group)
        # The dominant kind makes the chip honest: a build that was 9 fixes and 1
        # feature is a fix build, not a feature release.
        kind = kinds.most_common(1)[0][0]
        files = [f for r in group for f in r['files']]
        areas = Counter(f.split('/')[2] for f in files if f.count('/') >= 2)
        entries.append({
            'kind': kind,
            'version': ver,
            'date': group[-1]['date'],
            'updated': group[-1]['iso'],
            'title': '%s — %d commit%s' % (ver, len(group), '' if len(group) == 1 else 's'),
            'items': items,
            'note': 'Touches ' + ', '.join('%s (%d)' % (a, n) for a, n in areas.most_common(4))
                    + ' · ' + ', '.join('%s %d' % (k, n) for k, n in kinds.most_common(3)),
            'sources': sorted({f for r in group for f in r['files'][:1]})[:4],
            'commits': [r['sha'][:7] for r in group[:MAX_ITEMS]],
        })
    entries.reverse()                                   # newest build first

    first_date = rows[0]['date']
    repo_first = (git('log', '--max-parents=0', '--date=short', '--format=%ad').split() or ['?'])[0]
    gaps = []
    if repo_first < first_date:
        gaps.append({'range': 'before ' + first_date,
                     'why': 'The repository holds commits from ' + repo_first + ', but nothing that '
                            'touches ' + APP_REL + ' before this date - earlier work predates the '
                            'directory or was never committed here.'})
    gaps.append({'range': 'missing build numbers',
                 'why': 'A version bumped twice before anything was committed leaves no trace: only '
                        'builds whose tree reached a commit can appear here. Read a jump in the '
                        'version chip as "those bumps were superseded before commit".'})
    gaps.append({'range': 'every build',
                 'why': 'Entries summarise commit subjects, not diffs. A build with one squashed '
                        'commit shows one line even if it changed ten files; run '
                        '`git show <sha>` for the change itself.'})

    now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
    prev = existing_island()
    # `generated` is the last time the CONTENT changed; `checked` is the last time this ran. Two
    # fields because one cannot serve both: a daily rewrite would churn 65 KB of git for nothing,
    # while a content-only stamp cannot tell a live cron from a dead one.
    content_same = bool(prev) and prev.get('entries') == entries
    page = {
        'generated': (prev or {}).get('generated', now) if content_same else now,
        'checked': now,
        'currentVersion': ver_now,
        'stats': {
            'entries': len(entries),
            'versions': '%s – %s' % (first_known, ver_now),
            'records': len(rows),
        },
        'gaps': gaps,
        'entries': entries,
    }
    return page, rows


DEK = ('Every change that has a commit behind it. Each line names the build it landed in and the '
       'commit it came from, so any claim here can be checked with <code>git show</code>. Where '
       'the repository has no record, the gap is written down instead of filled in.')
GAP_INTRO = ('This page is generated from the repository\'s own history for '
             '<code>spaces/newos4</code> — the same source the running build comes from, so '
             'there are no unexplained stretches in the release line. What it cannot tell you is '
             'the shape of a squash-merged change.')


def write_atomic(path, text):
    """Temp + rename: nginx serves these files straight off disk, so an in-place truncate can
    hand a visitor a half-written page (or leave one after a crash)."""
    tmp = path + '.tmp'
    with open(tmp, 'w', encoding='utf-8') as fh:
        fh.write(text)
    os.replace(tmp, path)


def existing_island():
    try:
        with open(PAGE, encoding='utf-8') as fh:
            return json.loads(fh.read().split('id="changelog-data">')[1].split('</script>')[0])
    except Exception:
        return None


def write_page(page):
    with open(PAGE, encoding='utf-8') as fh:
        doc = fh.read()
    # json.dumps leaves "</" intact, so a commit subject containing </script> would end the island
    # early and truncate the page. Escape it; JSON.parse reads "<\/" back as "</".
    island = json.dumps(page, ensure_ascii=False, indent=2).replace('</', '<\\/')
    new, n = re.subn(r'(<script type="application/json" id="changelog-data">)[\s\S]*?(</script>)',
                     lambda m: m.group(1) + '\n' + island + '\n' + m.group(2), doc, count=1)
    assert n == 1, 'changelog JSON island not found - page layout changed'
    # Prose that described the old VFS-only source; replaced by pattern so re-runs are idempotent.
    new = re.sub(r'(<p class="dek">)[\s\S]*?(</p>)', lambda m: m.group(1) + DEK + m.group(2), new, count=1)
    new = re.sub(r'(<section class="panel" aria-labelledby="gaps-h">[\s\S]*?<h2 id="gaps-h">[^<]*</h2>\s*<p>)[\s\S]*?(</p>)',
                 lambda m: m.group(1) + GAP_INTRO + m.group(2), new, count=1)
    new = re.sub(r'(<b id="cur-ver">)[^<]*(</b>)', lambda m: m.group(1) + html.escape(page['currentVersion']) + m.group(2), new, count=1)
    write_atomic(PAGE, new)
    return len(new)


PREAMBLE = ('# newOS4 changelog\n\n'
            'Generated from git history by `scripts/build-changelog.py` — do not edit by hand '
            'between the markers.\n\n')
# The preamble is emitted EXACTLY once. Before 2026-11-05 it was part of the generated body *and*
# old[:m.start()] was preserved on top of it, so each run prepended another copy: 44 runs left 44
# copies of the H1 at the top of the file (and pushed the real first release from line 6 to 179).
PREAMBLE_RE = re.compile(re.escape('# newOS4 changelog') +
                         r'\n\nGenerated from git history by `scripts/build-changelog\.py` — '
                         r'do not edit by hand between the markers\.\n+')


def write_md(page, rows):
    """App-side Keep-a-Changelog history; only the region between the markers is generated."""
    region = ['<!-- generated:start -->', '']
    for entry in page['entries']:
        region.append('## %s — %s' % (entry['version'], entry['date']))
        region.append('')
        if entry['note']:
            region.extend(['_' + re.sub(r'<[^>]+>', '', entry['note']) + '_', ''])
        for item in entry['items']:
            region.append('- ' + re.sub(r'<[^>]+>', '`', item).replace('``', '`').replace('``', '`'))
        region.append('')
    region.extend(['<!-- generated:end -->', ''])
    region_text = '\n'.join(region)

    if os.path.exists(MD):
        with open(MD, encoding='utf-8') as fh:
            old = fh.read()
        m = re.search(r'<!-- generated:start -->[\s\S]*?<!-- generated:end -->\n?', old)
        if not m:
            # Refuse rather than replace: without the markers the file may hold prose that a
            # hand-edit added, and silently dropping it is worse than failing loudly.
            raise SystemExit('CHANGELOG.md exists but has no generated:start/end markers - '
                             'refusing to overwrite it')
        head, tail = old[:m.start()], old[m.end():]
        copies = len(PREAMBLE_RE.findall(head))
        if copies != 1:
            rest = PREAMBLE_RE.sub('', head).strip('\n')
            head = PREAMBLE + (rest + '\n\n' if rest else '')
            print('changelog: collapsed %d preamble copies in CHANGELOG.md' % copies)
        body = head + region_text + tail
    else:
        body = PREAMBLE + region_text
    # Guard against the 2026-11-05 regression class: the generated region may appear exactly once.
    # The old code concatenated the preamble into `body` and kept old[:m.start()] as well, so every
    # run appended another copy — 44 runs, 44 H1s, and the first release pushed to line 179.
    assert body.count('<!-- generated:start -->') == 1, 'CHANGELOG.md would contain a duplicated region'
    write_atomic(MD, body)
    return len(body)


def write_feed(page):
    """Same data, two machine shapes: JSON for tools, Atom for readers.

    Entry dates are day-precision, so <updated> gets T00:00:00Z appended rather than
    pretending to a time of day. Item markup is stripped: a feed reader gets text, not HTML.
    """
    base = os.path.dirname(PAGE)
    write_atomic(os.path.join(base, 'changelog.json'),
                 json.dumps(page, ensure_ascii=False, indent=1))
    entries = []
    for e in page['entries']:
        title = '%s — %s' % (e['version'], e['date'])
        body = re.sub(r'\s+', ' ', ' '.join(re.sub(r'<[^>]+>', ' ', i).strip()
                                             for i in e['items'])).strip()
        # Truncate the PLAIN text, then escape: slicing an escaped string can cut "&lt;" in half
        # and leave garbage at the tail of every long summary.
        if len(body) > 900:
            body = body[:900].rsplit(' ', 1)[0] + ' …'
        # A tag: URI cannot contain spaces ("before v4.6.10") and must not hardcode the year.
        year = (e['date'] or '')[:4] or '2026'
        slug = re.sub(r'[^A-Za-z0-9._-]+', '-', e['version'])
        anchor = re.sub(r'[^a-z0-9]+', '-', e['version'].lower())
        entries.append(
            '  <entry>\n'
            '    <title>%s</title>\n'
            '    <id>tag:webx11.duckdns.org,%s:%s</id>\n'
            '    <link href="https://webx11.duckdns.org/spaces/agando/downloads/changelog.html#%s"/>\n'
            '    <updated>%s</updated>\n'
            '    <summary>%s</summary>\n'
            '  </entry>' % (html.escape(title), year, slug, anchor,
                            e.get('updated') or (e['date'] + 'T00:00:00Z'), html.escape(body))
        )
    feed = ('<?xml version="1.0" encoding="utf-8"?>\n'
            '<feed xmlns="http://www.w3.org/2005/Atom">\n'
            '  <title>newOS4 changelog</title>\n'
            '  <id>tag:webx11.duckdns.org,2026:changelog</id>\n'
            '  <updated>%s</updated>\n  <author><name>WebX11</name></author>\n%s\n</feed>\n'
            % (page['generated'], '\n'.join(entries)))
    write_atomic(os.path.join(base, 'feed.xml'), feed)
    return len(feed)


def main():
    page, rows = build()
    if '--check' in sys.argv:
        with open(PAGE, encoding='utf-8') as fh:
            doc = fh.read()
        m = re.search(r'"currentVersion"\s*:\s*"([^"]+)"', doc)
        have = m.group(1) if m else 'none'
        if have != page['currentVersion']:
            print('changelog stale: page says %s, build is %s' % (have, page['currentVersion']))
            return 1
        print('changelog current: %s (%d entries)' % (have, len(page['entries'])))
        return 0
    size = write_page(page)
    mdsize = write_md(page, rows)
    fsize = write_feed(page)
    print('changelog: %d entries, %d commits, %s -> page %d bytes, md %d bytes, feed %d bytes'
          % (len(page['entries']), len(rows), page['stats']['versions'], size, mdsize, fsize))
    return 0


if __name__ == '__main__':
    sys.exit(main())