path: root/scripts/build-changelog.py

#!/usr/bin/env python3
"""Regenerate the newOS4 changelog page from git history.

Why git: the page used to be built by hand from VFS documents, so it had holes
(v4.6.59 - v4.6.100 were "unknown"). Git is the one source that has every
change, with dates and hashes, and it cannot drift from what shipped.

The page is its own template: everything visible comes from the JSON island
(`#changelog-data`), so regenerating means swapping that island and refreshing
the two prose blocks that described the old data source. No design duplication,
no second copy of the CSS or the renderer.

Run:
    python3 scripts/build-changelog.py            # write page + CHANGELOG.md + json + atom
    python3 scripts/build-changelog.py --check    # exit 1 if stale, write nothing

Scale: the page is one file while it stays under ~150 builds (measured 2026-10-27: 68 builds =
63 KB, so ~200 KB lands near 200 builds). Past that, split by calendar year -- changelog.html
keeps the newest 40 builds, changelog-<year>.html takes the rest, and this script links them.
Trigger the work when len(page['entries']) >= 150 stops being unusual, not when the page feels
slow in a browser."""

import html
import json
import os
import re
import subprocess
import sys
from collections import Counter, OrderedDict
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
APP = os.path.dirname(HERE)                       # spaces/newos4
ROOT = os.path.dirname(os.path.dirname(APP))      # repo root
PAGE = os.path.join(ROOT, 'spaces/agando/downloads/changelog.html')
MD = os.path.join(APP, 'CHANGELOG.md')
LOADER = os.path.join('spaces/newos4/modules/core/module-loader.js')
APP_REL = 'spaces/newos4'

# commit prefix -> the page's own kind vocabulary (KIND_LABEL in the renderer)
KIND = OrderedDict([
    ('feat', 'release'), ('fix', 'fix'), ('perf', 'fix'), ('revert', 'fix'),
    ('test', 'review'), ('refactor', 'project'), ('docs', 'docs'),
    ('style', 'project'), ('chore', 'project'), ('build', 'project'), ('ci', 'project'),
])
KIND_ORDER = ['current', 'release', 'fix', 'review', 'docs', 'project', 'bug']
MAX_ITEMS = 14          # per release; the rest is counted, not hidden


def git(*args):
    return subprocess.run(['git', '-C', ROOT] + list(args),
                          capture_output=True, text=True, check=True).stdout


def commits():
    """Oldest first: [{sha, date, subject, files:[...]}]"""
    raw = git('log', '--reverse', '--date=short',
              '--format=%x01%H%x01%ad%x01%s', '--name-only', '--', APP_REL)
    out, cur = [], None
    for line in raw.split('\n'):
        if line.startswith('\x01'):
            _, sha, date, subject = line.split('\x01', 3)
            cur = {'sha': sha, 'date': date, 'subject': subject, 'files': []}
            out.append(cur)
        elif line.strip() and cur is not None:
            cur['files'].append(line.strip())
    return out


_TAGMAP = None


def tag_map():
    """commit -> build tag, in ONE git call.

    `git describe --tags --abbrev=0 <sha>` per commit is correct but costs a history walk each
    time: measured 2.5 s for 158 commits versus 1.0 s for plain tree reads. One for-each-ref pass
    gives the same authority (a tag is a ref, not an inference) at no per-commit cost.
    `%(*objectname)` dereferences annotated tags to the commit they point at.
    """
    global _TAGMAP
    if _TAGMAP is None:
        _TAGMAP = {}
        try:
            out = git('for-each-ref', 'refs/tags', '--format=%(refname:short) %(*objectname)')
        except subprocess.CalledProcessError:
            out = ''
        for line in out.splitlines():
            parts = line.split()
            if len(parts) == 2 and re.match(r'^v?\d+(\.\d+)+$', parts[0]):
                _TAGMAP[parts[1]] = parts[0]
    return _TAGMAP


def version_at(sha):
    # A tag is the authority; the tree read is only needed for commits that predate tagging.
    tagged = tag_map().get(sha)
    if tagged:
        return tagged
    try:
        src = git('show', f'{sha}:{LOADER}')
    except subprocess.CalledProcessError:
        return None
    m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", src)
    if not m:
        return None
    # Only a version-shaped stamp is a build id; early builds used prose/date strings, which would
    # otherwise become nonsense chip labels like "2026-06-02-fontfix3".
    return m.group(1) if re.match(r'^v?\d+(\.\d+)+$', m.group(1).strip()) else None


def current_version():
    with open(os.path.join(ROOT, LOADER), encoding='utf-8') as fh:
        m = re.search(r"LATEST_VER\s*=\s*'([^']+)'", fh.read())
    if not m:
        raise SystemExit('module-loader.js has no LATEST_VER - refusing to guess a version')
    return m.group(1)


def kind_of(subject):
    head = subject.split(':', 1)[0].split('(', 1)[0].strip().lower()
    return KIND.get(head, 'project')


def clean(subject):
    """Drop the conventional prefix; the kind is shown as a chip already.

    Subjects written by this project use `backticks` for identifiers, and the page
    injects item text as author-controlled HTML, so turn each pair into <code>.
    Everything is escaped first - a commit subject is still untrusted text.
    """
    body = subject.split(':', 1)[1].strip() if ':' in subject else subject
    body = body[0].upper() + body[1:] if body else subject
    return re.sub(r'`([^`]+)`', lambda m: '<code>' + html.escape(m.group(1)) + '</code>',
                  html.escape(body))


def build():
    rows = commits()
    if not rows:
        raise SystemExit('no commits touch ' + APP_REL + ' - nothing to build')

    # Each commit is stamped with the version that was live *at that commit*, so a
    # release group is "everything that landed while this version was the build".
    for row in rows:
        row['version'] = version_at(row['sha'])
    version = None
    releases = OrderedDict()
    for row in rows:
        if row['version']:
            version = row['version']
        if version:
            releases.setdefault(version, []).append(row)
        else:
            releases.setdefault('\x00pre', []).append(row)

    ver_now = current_version()
    first_known = next((v for v in releases if v != '\x00pre'), ver_now)
    if '\x00pre' in releases:
        # Commits that predate the first LATEST_VER stamp share one honest label, and it has to
        # stay FIRST: the entries list is reversed for display, so order here is chronological.
        fixed = OrderedDict()
        for key, group in releases.items():
            fixed[('before ' + first_known) if key == '\x00pre' else key] = group
        releases = fixed

    entries = []
    for ver, group in releases.items():
        items = []
        for row in group[:MAX_ITEMS]:
            items.append('<strong>%s</strong> <code>%s</code>' % (
                clean(row['subject']), html.escape(row['sha'][:7])))
        if len(group) > MAX_ITEMS:
            items.append('…and %d more commit(s) in this build' % (len(group) - MAX_ITEMS))
        kinds = Counter(kind_of(r['subject']) for r in group)
        # The dominant kind makes the chip honest: a build that was 9 fixes and 1
        # feature is a fix build, not a feature release.
        kind = kinds.most_common(1)[0][0]
        files = [f for r in group for f in r['files']]
        areas = Counter(f.split('/')[2] for f in files if f.count('/') >= 2)
        entries.append({
            'kind': kind,
            'version': ver,
            'date': group[-1]['date'],
            'title': '%s — %d commit%s' % (ver, len(group), '' if len(group) == 1 else 's'),
            'items': items,
            'note': 'Touches ' + ', '.join('%s (%d)' % (a, n) for a, n in areas.most_common(4))
                    + ' · ' + ', '.join('%s %d' % (k, n) for k, n in kinds.most_common(3)),
            'sources': sorted({f for r in group for f in r['files'][:1]})[:4],
            'commits': [r['sha'][:7] for r in group[:MAX_ITEMS]],
        })
    entries.reverse()                                   # newest build first

    first_date = rows[0]['date']
    repo_first = (git('log', '--max-parents=0', '--date=short', '--format=%ad').split() or ['?'])[0]
    gaps = []
    if repo_first < first_date:
        gaps.append({'range': 'before ' + first_date,
                     'why': 'The repository holds commits from ' + repo_first + ', but nothing that '
                            'touches ' + APP_REL + ' before this date - earlier work predates the '
                            'directory or was never committed here.'})
    gaps.append({'range': 'missing build numbers',
                 'why': 'A version bumped twice before anything was committed leaves no trace: only '
                        'builds whose tree reached a commit can appear here. Read a jump in the '
                        'version chip as "those bumps were superseded before commit".'})
    gaps.append({'range': 'every build',
                 'why': 'Entries summarise commit subjects, not diffs. A build with one squashed '
                        'commit shows one line even if it changed ten files; run '
                        '`git show <sha>` for the change itself.'})

    page = {
        'generated': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'currentVersion': ver_now,
        'currentSynthesized': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'stats': {
            'entries': len(entries),
            'versions': '%s – %s' % (first_known, ver_now),
            'records': len(rows),
            'sourcesRead': 1,
        },
        'gaps': gaps,
        'entries': entries,
    }
    return page, rows


DEK = ('Every change that has a commit behind it. Each line names the build it landed in and the '
       'commit it came from, so any claim here can be checked with <code>git show</code>. Where '
       'the repository has no record, the gap is written down instead of filled in.')
GAP_INTRO = ('This page is generated from the repository\'s own history for '
             '<code>spaces/newos4</code> — the same source the running build comes from, so '
             'there are no unexplained stretches in the release line. What it cannot tell you is '
             'the shape of a squash-merged change.')


def write_page(page):
    with open(PAGE, encoding='utf-8') as fh:
        doc = fh.read()
    island = json.dumps(page, ensure_ascii=False, indent=2)
    new, n = re.subn(r'(<script type="application/json" id="changelog-data">)[\s\S]*?(</script>)',
                     lambda m: m.group(1) + '\n' + island + '\n' + m.group(2), doc, count=1)
    assert n == 1, 'changelog JSON island not found - page layout changed'
    # Prose that described the old VFS-only source; replaced by pattern so re-runs are idempotent.
    new = re.sub(r'(<p class="dek">)[\s\S]*?(</p>)', lambda m: m.group(1) + DEK + m.group(2), new, count=1)
    new = re.sub(r'(<section class="panel" aria-labelledby="gaps-h">[\s\S]*?<h2 id="gaps-h">[^<]*</h2>\s*<p>)[\s\S]*?(</p>)',
                 lambda m: m.group(1) + GAP_INTRO + m.group(2), new, count=1)
    new = re.sub(r'(<b id="cur-ver">)[^<]*(</b>)', lambda m: m.group(1) + html.escape(page['currentVersion']) + m.group(2), new, count=1)
    with open(PAGE, 'w', encoding='utf-8') as fh:
        fh.write(new)
    return len(new)


def write_md(page, rows):
    """App-side Keep-a-Changelog history; generated region is delimited."""
    order = {'release': 0, 'fix': 1, 'review': 2, 'docs': 3, 'project': 4, 'bug': 5}
    out = ['# newOS4 changelog', '',
           'Generated from git history by `scripts/build-changelog.py` — do not edit by hand '
           'between the markers.', '', '<!-- generated:start -->', '']
    for entry in page['entries']:
        out.append('## %s — %s' % (entry['version'], entry['date']))
        out.append('')
        if entry['note']:
            out.extend(['_' + re.sub(r'<[^>]+>', '', entry['note']) + '_', ''])
        for item in entry['items']:
            out.append('- ' + re.sub(r'<[^>]+>', '`', item).replace('``', '`').replace('``', '`'))
        out.append('')
    out.extend(['<!-- generated:end -->', ''])
    body = '\n'.join(out)
    if os.path.exists(MD):
        old = open(MD, encoding='utf-8').read()
        m = re.search(r'<!-- generated:start -->[\s\S]*?<!-- generated:end -->\n?', old)
        if m:
            body = old[:m.start()] + body + old[m.end():]
    with open(MD, 'w', encoding='utf-8') as fh:
        fh.write(body)
    return len(body)


def write_feed(page):
    """Same data, two machine shapes: JSON for tools, Atom for readers.

    Entry dates are day-precision, so <updated> gets T00:00:00Z appended rather than
    pretending to a time of day. Item markup is stripped: a feed reader gets text, not HTML.
    """
    base = os.path.dirname(PAGE)
    with open(os.path.join(base, 'changelog.json'), 'w', encoding='utf-8') as fh:
        json.dump(page, fh, ensure_ascii=False, indent=1)
    entries = []
    for e in page['entries']:
        title = '%s — %s' % (e['version'], e['date'])
        body = ' '.join(re.sub(r'<[^>]+>', ' ', i).strip() for i in e['items'])
        entries.append(
            '  <entry>\n'
            '    <title>%s</title>\n'
            '    <id>tag:webx11.duckdns.org,2026:%s</id>\n'
            '    <updated>%sT00:00:00Z</updated>\n'
            '    <summary>%s</summary>\n'
            '  </entry>' % (html.escape(title), html.escape(e['version']),
                            e['date'], html.escape(re.sub(r'\s+', ' ', body)[:900]))
        )
    feed = ('<?xml version="1.0" encoding="utf-8"?>\n'
            '<feed xmlns="http://www.w3.org/2005/Atom">\n'
            '  <title>newOS4 changelog</title>\n'
            '  <id>tag:webx11.duckdns.org,2026:changelog</id>\n'
            '  <updated>%s</updated>\n  <author><name>WebX11</name></author>\n%s\n</feed>\n'
            % (page['generated'], '\n'.join(entries)))
    path = os.path.join(base, 'feed.xml')
    with open(path, 'w', encoding='utf-8') as fh:
        fh.write(feed)
    return len(feed)


def main():
    page, rows = build()
    if '--check' in sys.argv:
        with open(PAGE, encoding='utf-8') as fh:
            doc = fh.read()
        m = re.search(r'"currentVersion"\s*:\s*"([^"]+)"', doc)
        have = m.group(1) if m else 'none'
        if have != page['currentVersion']:
            print('changelog stale: page says %s, build is %s' % (have, page['currentVersion']))
            return 1
        print('changelog current: %s (%d entries)' % (have, len(page['entries'])))
        return 0
    size = write_page(page)
    mdsize = write_md(page, rows)
    fsize = write_feed(page)
    print('changelog: %d entries, %d commits, %s -> page %d bytes, md %d bytes, feed %d bytes'
          % (len(page['entries']), len(rows), page['stats']['versions'], size, mdsize, fsize))
    return 0


if __name__ == '__main__':
    sys.exit(main())