ci: separate dev deployment, main checks and GitHub mirroring

scripts/ci-check.sh

@@ -4,6 +4,8 @@ set -eu
 
 mode="${1:?Expected development or release}"
 case "$mode" in development|release) ;; *) exit 2 ;; esac
+if [ "$mode" = development ]; then export RAWMARK_BUILD_CHANNEL=development; fi
+export RAWMARK_BUILD_VERSION="$(python3 scripts/release.py version)"
 export CGO_ENABLED=0
 corepack enable
 corepack prepare [email protected] --activate

scripts/ci-package.sh

@@ -7,6 +7,8 @@ case "$mode" in
   release) targets='linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64' ;;
   *) exit 2 ;;
 esac
+if [ "$mode" = development ]; then export RAWMARK_BUILD_CHANNEL=development; fi
+export RAWMARK_BUILD_VERSION="$(python3 scripts/release.py version)"
 export CGO_ENABLED=0
 mkdir -p package/licenses downloads
 cp LICENSE package/
@@ -19,7 +21,7 @@ for target in $targets; do
   if [ "$GOOS" = windows ]; then extension='.exe'; fi
   rm -f package/rawmark package/rawmark.exe
   go vet ./...
-  go build -trimpath -ldflags="-s -w -X main.releaseVersion=$(cat VERSION)" \
+  go build -trimpath -ldflags="-s -w -X main.releaseVersion=$(python3 scripts/release.py version)" \
     -o "package/rawmark$extension" ./cmd/rawmark
   if [ "$target" = linux/amd64 ]; then python3 scripts/smoke-release.py package/rawmark; fi
   python3 scripts/package-release.py "$GOOS" "$GOARCH"

scripts/deploy.py

@@ -14,7 +14,7 @@ import tempfile
 import time
 import urllib.request
 
-from release import API, GITEA_IMAGE, GITHUB_IMAGE, release_receipt, require, run
+from release import API, GITEA_DEV_IMAGE, GITHUB_IMAGE, release_receipt, require, run
 
 
 def settings(directory, channel):
@@ -49,7 +49,7 @@ def save(path, data):
 def deployment(directory, channel, image, commit, version, client_container=None):
     directory = Path(directory).resolve(strict=True)
     cfg = settings(directory, channel)
-    registry = GITEA_IMAGE if channel == 'development' else GITHUB_IMAGE
+    registry = GITEA_DEV_IMAGE if channel == 'development' else GITHUB_IMAGE
     require(re.fullmatch(re.escape(registry) + r'@sha256:[0-9a-f]{64}', image), 'Expected an immutable registry digest')
     require(re.fullmatch(r'[0-9a-f]{40}', commit), 'Expected a full commit SHA')
     # Serializes actual deployments, including manually started jobs and CLI use.
@@ -67,8 +67,8 @@ def deployment(directory, channel, image, commit, version, client_container=None
                 'Executable version mismatch')
         if channel == 'development':
             api = API(os.environ['GITEA_API_URL'], os.environ['GITEA_RELEASE_TOKEN'])
-            if api.repo('/branches/main')['commit']['id'] != commit:
-                print('Skipped development deployment: main has advanced')
+            if api.repo('/branches/dev')['commit']['id'] != commit:
+                print('Skipped development deployment: dev has advanced')
                 return
         identifiers = run('docker', 'ps', '-aq', '--filter', 'label=com.docker.compose.project=' + cfg['RAWMARK_PROJECT'],
                           '--filter', 'label=com.docker.compose.service=rawmark').split()
@@ -87,6 +87,7 @@ def deployment(directory, channel, image, commit, version, client_container=None
         backup = None
         if previous:
             run('docker', 'stop', '--time', '15', previous['Id'])
+        if any((directory / 'data').iterdir()):
             try:
                 backups = directory / 'backups'
                 backups.mkdir(mode=0o700, exist_ok=True)
@@ -101,7 +102,7 @@ def deployment(directory, channel, image, commit, version, client_container=None
                                 archive.add(directory / name, arcname=name)
             except BaseException:
                 # No new executable has touched the data; restarting the old container is safe here.
-                if previous['State']['Running']:
+                if previous and previous['State']['Running']:
                     run('docker', 'start', previous['Id'])
                 raise
         pending = {'image': image, 'commit': commit, 'version': version, 'channel': channel,

scripts/mirror.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Mirror only business Git references under one persistent repository lock."""
+
+import argparse
+import fcntl
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+
+from release import API, git_env, require
+
+
+def git(directory, *args, env=None):
+    return subprocess.check_output(['git', '-C', str(directory), *args], env=env, text=True).strip()
+
+
+def references(directory, remote=None, env=None):
+    command = ['ls-remote', remote, 'refs/heads/*', 'refs/tags/*'] if remote else [
+        'for-each-ref', '--format=%(objectname)\t%(refname)', 'refs/heads/', 'refs/tags/']
+    output = git(directory, *command, env=env)
+    return {ref: sha for line in output.splitlines() for sha, ref in [line.split()] if not ref.endswith('^{}')}
+
+
+def synchronize(directory, source, target, environment):
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    with (directory / '.mirror.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        repo = directory / 'repository.git'
+        if not repo.exists():
+            subprocess.run(['git', 'init', '--bare', str(repo)], check=True, stdout=subprocess.DEVNULL)
+        for _ in range(3):
+            git(repo, 'fetch', '--prune', '--no-tags', source,
+                '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', env=environment)
+            snapshot = references(repo)
+            require('refs/heads/main' in snapshot, 'Source main is missing; refusing destructive synchronization')
+            # Fetch every LFS object before updating destination references when LFS is present.
+            if git(repo, 'log', '--all', '--format=', '--name-only', '--', '.gitattributes', '**/.gitattributes').strip():
+                subprocess.run(['git', '-C', str(repo), 'lfs', 'fetch', '--all', source], env=environment, check=True)
+                subprocess.run(['git', '-C', str(repo), 'lfs', 'push', '--all', target], env=environment, check=True)
+            git(repo, 'push', '--atomic', '--prune', target,
+                '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', env=environment)
+            require(references(repo, target, environment) == snapshot, 'Destination references differ')
+            if references(repo, source, environment) == snapshot:
+                receipt = {'source': source, 'target': target, 'refs': snapshot, 'verified_at': int(time.time())}
+                (directory / 'last-sync.json').write_text(json.dumps(receipt, indent=2) + '\n')
+                return receipt
+        raise RuntimeError('Source kept changing; retry synchronization')
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--directory', required=True)
+    args = parser.parse_args()
+    api = API('https://api.github.com', os.environ['GITHUB_MIRROR_TOKEN'], github=True)
+    synchronize(args.directory, 'https://git.134333.xyz/eezd/RawMark.git',
+                'https://github.com/eezd/RawMark.git', git_env(api))
+    print('GitHub mirror verified')
+
+
+if __name__ == '__main__':
+    main()

scripts/package-release.py

@@ -7,8 +7,12 @@ import tarfile
 import zipfile
 
 system, arch = sys.argv[1:]
-version = Path('VERSION').read_text().strip()
-name = f'rawmark_{version}_{system}_{arch}'
+from release import build_version
+
+version = build_version()
+from release import archive_name
+
+name = archive_name(version, system + '/' + arch).removesuffix('.zip').removesuffix('.tar.gz')
 package = Path('package')
 files = sorted(p for p in package.rglob('*') if p.is_file())
 if system == 'windows':

scripts/release.py

@@ -3,6 +3,7 @@
 
 import argparse
 import base64
+import datetime
 import hashlib
 import io
 import json
@@ -22,6 +23,7 @@ import zipfile
 
 REPOSITORY = 'eezd/RawMark'
 GITEA_IMAGE = 'git.134333.xyz/eezd/rawmark'
+GITEA_DEV_IMAGE = 'git.134333.xyz/eezd/dev/rawmark'
 GITHUB_IMAGE = 'ghcr.io/eezd/rawmark'
 PLATFORMS = ('linux/amd64', 'linux/arm64', 'windows/amd64', 'darwin/amd64', 'darwin/arm64')
 UPLOAD_LIMIT = 90_000_000
@@ -52,13 +54,23 @@ def output(name, value):
             target.write(f'{name}={value}\n')
 
 
+def build_version():
+    if os.environ.get('RAWMARK_BUILD_CHANNEL') != 'development':
+        return Path('VERSION').read_text().strip()
+    commit = run('git', 'rev-parse', 'HEAD')
+    timestamp = int(run('git', 'show', '-s', '--format=%ct', commit))
+    date = datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).strftime('%Y%m%d')
+    return date + '-' + commit[:8]
+
+
 def archive_name(version, platform):
     extension = '.zip' if platform.startswith('windows/') else '.tar.gz'
-    return f'rawmark_{version}_{platform.replace("/", "_")}{extension}'
+    prefix = 'RawMark' if re.fullmatch(r'[0-9]{8}-[0-9a-f]{8}', version) else 'rawmark'
+    return f'{prefix}_{version}_{platform.replace("/", "_")}{extension}'
 
 
 def bundle(mode):
-    version = Path('VERSION').read_text().strip()
+    version = build_version()
     commit = run('git', 'rev-parse', 'HEAD')
     platforms = PLATFORMS if mode == 'release' else PLATFORMS[:1]
     files = {}
@@ -91,7 +103,7 @@ def bundle(mode):
 
 def validate_build(build):
     require(build['schema'] == 1, 'Unsupported build receipt')
-    require(build['version'] == Path('VERSION').read_text().strip(), 'Build version mismatch')
+    require(build['version'] == build_version(), 'Build version mismatch')
     require(build['commit'] == run('git', 'rev-parse', 'HEAD'), 'Build commit mismatch')
     require(build['mode'] in ('development', 'release'), 'Unknown build mode')
     expected = PLATFORMS if build['mode'] == 'release' else PLATFORMS[:1]
@@ -423,7 +435,19 @@ def promote_development(local_image):
     token = os.environ['GITEA_RELEASE_TOKEN']
     subprocess.run(['docker', 'login', 'git.134333.xyz', '--username', 'eezd', '--password-stdin'],
                    input=token, text=True, check=True)
-    candidate = GITEA_IMAGE + ':build-' + build['commit'] + '-' + os.environ['CI_RUN_ID'] + '-' + os.environ['CI_RUN_ATTEMPT']
+    existing = registry_digest(GITEA_DEV_IMAGE, build['version'])
+    if existing:
+        reference = GITEA_DEV_IMAGE + '@' + existing
+        run('docker', 'pull', reference)
+        info = json.loads(run('docker', 'image', 'inspect', reference))[0]
+        labels = info['Config'].get('Labels') or {}
+        require(labels.get('org.opencontainers.image.revision') == build['commit']
+                and labels.get('org.opencontainers.image.version') == build['version'],
+                'Existing development version belongs to another commit')
+        output('image', reference)
+        output('version', build['version'])
+        return
+    candidate = GITEA_DEV_IMAGE + ':build-' + build['commit'] + '-' + os.environ['CI_RUN_ID'] + '-' + os.environ['CI_RUN_ATTEMPT']
     # Uncompressed layer sizes provide a conservative bound before any CDN upload.
     with tempfile.TemporaryFile() as target:
         subprocess.run(['docker', 'image', 'save', local_image], stdout=target, check=True)
@@ -434,15 +458,15 @@ def promote_development(local_image):
     run('docker', 'tag', local_image, candidate)
     try:
         run('docker', 'push', candidate)
-        value = registry_digest(GITEA_IMAGE, candidate.rsplit(':', 1)[1])
-        tag = 'dev-' + build['commit']
-        existing = registry_digest(GITEA_IMAGE, tag)
+        value = registry_digest(GITEA_DEV_IMAGE, candidate.rsplit(':', 1)[1])
+        tag = build['version']
+        existing = registry_digest(GITEA_DEV_IMAGE, tag)
         require(existing in (None, value), 'Immutable development tag already has different content')
         if not existing:
             run('docker', 'buildx', 'imagetools', 'create', '--prefer-index=false', '--tag',
-                GITEA_IMAGE + ':' + tag, GITEA_IMAGE + '@' + value)
-        require(registry_digest(GITEA_IMAGE, tag) == value, 'Development tag digest changed')
-        output('image', GITEA_IMAGE + '@' + value)
+                GITEA_DEV_IMAGE + ':' + tag, GITEA_DEV_IMAGE + '@' + value)
+        require(registry_digest(GITEA_DEV_IMAGE, tag) == value, 'Development tag digest changed')
+        output('image', GITEA_DEV_IMAGE + '@' + value)
         output('version', build['version'])
     finally:
         run('docker', 'image', 'rm', candidate)
@@ -610,6 +634,7 @@ def main():
     sub = parser.add_subparsers(dest='command', required=True)
     sub.add_parser('bundle').add_argument('mode', choices=('development', 'release'))
     sub.add_parser('prepare-image')
+    sub.add_parser('version')
     pre = sub.add_parser('preflight')
     pre.add_argument('version')
     pre.add_argument('commit')
@@ -617,7 +642,9 @@ def main():
     sub.add_parser('publish')
     sub.add_parser('promote-development').add_argument('image')
     args = parser.parse_args()
-    if args.command == 'bundle':
+    if args.command == 'version':
+        print(build_version())
+    elif args.command == 'bundle':
         bundle(args.mode)
     elif args.command == 'prepare-image':
         prepare_image()

scripts/smoke-docker.py

@@ -18,7 +18,9 @@ parser.add_argument('image')
 parser.add_argument('--client-container', help='CI job container to connect to the test network')
 parser.add_argument('--resource-label', help='Label temporary resources for cancellation cleanup')
 args = parser.parse_args()
-version = Path('VERSION').read_text().strip()
+from release import build_version
+
+version = build_version()
 name = 'rawmark-ci-' + secrets.token_hex(8)
 labels = ('--label', args.resource_label) if args.resource_label else ()
 

scripts/smoke-release.py

@@ -18,7 +18,9 @@ import urllib.request
 
 
 binary = str(Path(sys.argv[1]).resolve())
-version = Path('VERSION').read_text().strip()
+from release import build_version
+
+version = build_version()
 assert subprocess.check_output([binary, '--version'], text=True).strip() == f'RawMark {version}'
 
 

scripts/tests/test_cicd.py

@@ -204,7 +204,7 @@ class DeploymentTests(unittest.TestCase):
         (self.directory / '.env').write_text(
             f'RAWMARK_CHANNEL=development\nRAWMARK_PROJECT=rawmark-ci-deploy\n'
             f'RAWMARK_BIND=127.0.0.1\nRAWMARK_PORT=21899\nRAWMARK_DATA_DIR={self.directory}/data\n')
-        self.image = release.GITEA_IMAGE + '@' + DIGEST
+        self.image = release.GITEA_DEV_IMAGE + '@' + DIGEST
         self.previous = {'Id': 'old-id', 'Image': 'old-image', 'State': {'Running': True},
                          'Mounts': [{'Destination': '/data', 'Type': 'bind', 'Source': str(self.directory / 'data')}]}
         self.image_info = {'Id': 'new-image', 'Config': {'User': '65532:65532', 'Labels': {
@@ -249,7 +249,7 @@ class DeploymentTests(unittest.TestCase):
                 deploy.deployment(self.directory, 'development', self.image, SHA, '0.1.1')
             docker.assert_not_called()
 
-    def test_stale_main_skips_without_stopping_existing_service(self):
+    def test_stale_dev_skips_without_stopping_existing_service(self):
         with patch.object(deploy, 'API') as api, patch.object(deploy, 'run', side_effect=self.command):
             api.return_value.repo.return_value = {'commit': {'id': 'c' * 40}}
             deploy.deployment(self.directory, 'development', self.image, SHA, '0.1.1')
@@ -278,6 +278,21 @@ class DeploymentTests(unittest.TestCase):
         self.assertLess(stop, start)
         self.assertIn('RAWMARK_IMAGE=' + self.image, (self.directory / '.env').read_text())
 
+    def test_existing_data_is_backed_up_without_a_container(self):
+        def without_container(*args, **kwargs):
+            if args[:2] == ('docker', 'ps'):
+                return ''
+            return self.command(*args, **kwargs)
+        opener = Mock()
+        opener.open.side_effect = [contextlib.nullcontext(io.BytesIO(b'<html>RawMark</html>')),
+                                   contextlib.nullcontext(io.BytesIO(b'{"code":0}'))]
+        with patch.object(deploy, 'run', side_effect=without_container), patch.object(
+                deploy.urllib.request, 'build_opener', return_value=opener):
+            deploy.deployment(self.directory, 'development', self.image, SHA, '0.1.1')
+        receipt = json.loads((self.directory / 'deployment.json').read_text())
+        self.assertTrue(Path(receipt['backup']).exists())
+        self.assertIsNone(receipt['previous_image'])
+
     def test_failed_replacement_keeps_backup_and_does_not_downgrade_data(self):
         def failure(*args, **kwargs):
             if args[:2] == ('docker', 'compose') and 'up' in args:
@@ -293,6 +308,15 @@ class DeploymentTests(unittest.TestCase):
         self.assertNotIn('RAWMARK_IMAGE=', (self.directory / '.env').read_text())
 
 
+class DevelopmentVersionTests(unittest.TestCase):
+    def test_version_uses_utc_commit_date_and_exact_eight_character_hash(self):
+        with patch.dict(os.environ, {'RAWMARK_BUILD_CHANNEL': 'development'}), patch.object(
+                release, 'run', side_effect=[SHA, '0']):
+            self.assertEqual(release.build_version(), '19700101-aaaaaaaa')
+        self.assertEqual(release.archive_name('19700101-aaaaaaaa', 'linux/amd64'),
+                         'RawMark_19700101-aaaaaaaa_linux_amd64.tar.gz')
+
+
 class CleanupTests(unittest.TestCase):
     def test_cleanup_rejects_broad_filter(self):
         with patch('subprocess.check_output') as docker:

scripts/tests/test_mirror.py

@@ -0,0 +1,36 @@
+"""Verify strict mirroring with isolated local repositories."""
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from mirror import git, references, synchronize
+
+
+class MirrorTests(unittest.TestCase):
+    def test_full_history_tags_deletions_and_failed_source(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source, target = root / 'source', root / 'target.git'
+            subprocess.run(['git', 'init', '-b', 'main', str(source)], check=True, capture_output=True)
+            subprocess.run(['git', 'init', '--bare', str(target)], check=True, capture_output=True)
+            (source / 'note.md').write_text('synthetic note')
+            git(source, 'add', '.')
+            git(source, '-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-m', 'test')
+            git(source, 'branch', 'dev')
+            git(source, '-c', 'user.name=Test', '-c', '[email protected]', 'tag', '-a', 'test-tag', '-m', 'test')
+            state = root / 'state'
+            synchronize(state, str(source), str(target), os.environ.copy())
+            self.assertEqual(references(source), references(target))
+            git(source, 'branch', '-D', 'dev')
+            git(source, 'tag', '-d', 'test-tag')
+            git(target, 'branch', 'target-only', 'main')
+            synchronize(state, str(source), str(target), os.environ.copy())
+            self.assertEqual(references(source), references(target))
+            before = references(target)
+            with self.assertRaises(subprocess.CalledProcessError):
+                synchronize(state, str(root / 'missing'), str(target), os.environ.copy())
+            self.assertEqual(before, references(target))