From ef39f9d120af0c26ea32f68eb2b11754b68d9dfe Mon Sep 17 00:00:00 2001 From: eezd Date: Fri, 30 Oct 2026 13:29:02 +0800 Subject: [PATCH] Keep DEV as the sole instance and finish releases after publication --- .gitea/workflows/deploy-production.yml | 53 -------------------------- .gitea/workflows/release.yml | 32 ---------------- AGENTS.md | 4 +- scripts/deploy.py | 32 +++------------- scripts/release.py | 15 ++++++-- scripts/tests/test_cicd.py | 51 ++++++++++++++++++++++++- 6 files changed, 70 insertions(+), 117 deletions(-) delete mode 100644 .gitea/workflows/deploy-production.yml diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml deleted file mode 100644 index 3cec9c5..0000000 --- a/.gitea/workflows/deploy-production.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Deploy production manually - -on: - workflow_dispatch: - inputs: - version: - description: Completed date-hash release version - required: true - type: string - -permissions: - contents: read - -concurrency: - group: rawmark-production - cancel-in-progress: false - -jobs: - deploy: - if: gitea.ref == 'refs/heads/main' - runs-on: docker - timeout-minutes: 20 - container: - image: docker:29-cli - volumes: - - /docker/gitea/main/RawMark:/docker/gitea/main/RawMark - - /docker/gitea/mirrors/RawMark:/docker/gitea/mirrors/RawMark - env: - GITEA_API_URL: ${{ gitea.server_url }}/api/v1 - GITEA_READ_TOKEN: ${{ gitea.token }} - RELEASE_VERSION: ${{ inputs.version }} - DOCKER_CONFIG: /tmp/rawmark-anonymous-docker - defaults: - run: - shell: sh - steps: - - name: Install deployment runtimes - run: apk add --no-cache nodejs python3 git docker-cli-compose - - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - ref: main - fetch-depth: 0 - persist-credentials: false - - name: Verify completed release, back up and deploy only RawMark - run: | - set -eu - mkdir -p "$DOCKER_CONFIG" - chmod 700 "$DOCKER_CONFIG" - python3 scripts/deploy.py production --directory /docker/gitea/main/RawMark \ - --version "$RELEASE_VERSION" --client-container "$(hostname)" - - name: Remove job configuration - if: always() - run: rm -rf "$DOCKER_CONFIG" diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index c18dd02..a32e731 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -182,35 +182,3 @@ jobs: run: | python3 scripts/ci-cleanup.py "$CI_RESOURCE_LABEL" --client-container "$(hostname)" rm -rf "$DOCKER_CONFIG" - - deploy: - needs: [build, publish] - if: always() && needs.build.result == 'success' && (needs.publish.result == 'success' || needs.publish.result == 'skipped') - runs-on: docker - timeout-minutes: 20 - container: - image: docker:29-cli - volumes: - - /docker/gitea/mirrors/RawMark:/docker/gitea/mirrors/RawMark - - /docker/gitea/main/RawMark:/docker/gitea/main/RawMark - env: - GITEA_READ_TOKEN: ${{ secrets.RAWMARK_GITEA_TOKEN }} - DOCKER_CONFIG: /tmp/rawmark-anonymous-docker - steps: - - name: Install deployment runtimes - run: apk add --no-cache nodejs python3 git docker-cli-compose - - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - ref: ${{ gitea.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Deploy the completed release and finish its reservation - run: | - set -eu - mkdir -p "$DOCKER_CONFIG" - python3 scripts/deploy.py production --directory /docker/gitea/main/RawMark \ - --version "$(python3 -c 'import sys;sys.path.insert(0,"scripts");import release;print(release.selected_state()["version"])')" \ - --client-container "$(hostname)" - - name: Remove temporary Docker configuration - if: always() - run: rm -rf "$DOCKER_CONFIG" diff --git a/AGENTS.md b/AGENTS.md index 1cab319..2d03ca4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,8 +133,8 @@ Gitea 为公开主仓库，GitHub 为单向公开镜像；默认分支均为 mai - dev push：容器内检查、Linux amd64 打包与隔离冒烟，通过后发布 `git.134333.xyz/eezd/dev/rawmark:YYYYMMDD-<8位SHA>`，按摘要部署 DEV。日期取提交时间 UTC，完整 SHA 保留在回执；同名异提交拒绝，已验证镜像重试复用。 - main push：独立检查实际合并提交、Linux amd64 构建及容器冒烟，不发布镜像、不部署。仓库内受信 PR 只检查；外部 fork PR 不自动进入共享 Docker runner，不注入发布凭据。 -- DEV 为 `/docker/gitea/dev/RawMark`，容器 `RawMark-dev`，Compose 项目 `rawmark-dev`，监听 `127.0.0.1:21001`，域名 `raw-dev.134333.xyz`。数据独立；所有部署共用宿主实例 `.deploy.lock`，锁内核对 dev 最新 SHA，pending 未处理时拒绝覆盖。已有数据即使无旧容器也必须备份。 +- DEV 为 `/docker/gitea/dev/RawMark`，容器 `RawMark-dev`，Compose 项目 `rawmark-dev`，监听 `127.0.0.1:21001`，域名 `raw.134333.xyz`，raw-dev 域名停用。DEV 是唯一常驻实例，保存个人真实数据；CI 与冒烟使用隔离合成数据。数据独立；所有部署共用宿主实例 `.deploy.lock`，锁内核对 dev 最新 SHA，pending 未处理时拒绝覆盖。已有数据即使无旧容器也必须备份。 - Git 镜像独立于 CI 成败：push/create/delete、每小时对账和手动重试，共用 `/docker/gitea/mirrors/RawMark/.mirror.lock`，仅同步业务分支和 TAG 及完整历史，跟随源端删除。同步代码从 main 读取；`RAWMARK_MIRROR_ENABLED=true` 后启用，凭据统一使用 Gitea 账号级 Secret `ALL_GITHUB_TOKEN`（头像 → 设置 → Actions → 密钥），接入前配置并验证实际权限，仓库级同名项会覆盖账号级值。DEV 镜像不上传 GHCR。main 切换前保持该变量关闭。 - CI 凭据 `RAWMARK_GITEA_TOKEN` 需要仓库和 Packages 写权限。runner 保持容量一、`sweep: false`，只运行受信代码；工具链安装在任务容器。资源按任务标签清理，禁止全局 prune。 - 开发构建通过 `RAWMARK_BUILD_CHANNEL=development` 派生版本，前端与二进制一致；正式 `VERSION` 不因开发构建改变。开发包为 `RawMark_<版本>_linux_amd64.tar.gz`；历史正式产物不改名。安装包及单次传输保持既有 90,000,000 字节预算。 -- 正式发布从受信 main 手动触发，使用日期哈希注解 TAG、六平台程序包与 Linux 双架构镜像；发布 ID 和候选 SHA 固定，发布记录与 .release.lock 位于持久镜像目录。GitHub 引用同步复用镜像锁，Pages Demo 不切换。双端发布和 GHCR 公开拉取验收后自动部署 MAIN：/docker/gitea/main/RawMark、RawMark-main、rawmark-main，127.0.0.1:21000，raw.134333.xyz。自动/手动/批量启停共用实例 .deploy.lock，旧版本不得覆盖新版；生产数据保留并先备份，不随开发推送更新。 +- 正式发布从受信 main 手动触发，使用日期哈希注解 TAG、六平台程序包与 Linux 双架构镜像；发布 ID 和候选 SHA 固定，发布记录与 .release.lock 位于持久镜像目录。GitHub 引用同步复用镜像锁，Pages Demo 不切换。双端发布和 GHCR 公开拉取验收后结束发布并标记预约完成，不部署 MAIN、不更新 DEV。DEV 批量管理入口为 `/docker/gitea/dev/manage.sh start|stop`，与 CD 共用实例锁，pending 阻止启动。 diff --git a/scripts/deploy.py b/scripts/deploy.py index efe78ef..7d4b420 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -14,7 +14,7 @@ import tempfile import time import urllib.request -from release import API, GITEA_DEV_IMAGE, GITHUB_IMAGE, release_receipt, require, run, release_state, save_state +from release import API, GITEA_DEV_IMAGE, require, run def settings(directory, channel): @@ -48,16 +48,14 @@ def save(path, data): def deployment(directory, channel, image, commit, version, client_container=None): directory = Path(directory).resolve(strict=True) + require(channel == 'development', 'Only the persistent DEV instance supports deployment') cfg = settings(directory, channel) - registry = GITEA_DEV_IMAGE if channel == 'development' else GITHUB_IMAGE + registry = GITEA_DEV_IMAGE require(re.fullmatch(re.escape(registry) + r'@sha256:[0-9a-f]{64}', image), 'Expected an immutable registry digest') require(re.fullmatch(r'[0-9a-f]{40}', commit), 'Expected a full commit SHA') # Serializes actual deployments, including manually started jobs and CLI use. with (directory / '.deploy.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) - if channel == 'production' and (directory / 'deployment.json').exists(): - deployed = json.loads((directory / 'deployment.json').read_text()) - run('git', 'merge-base', '--is-ancestor', deployed['commit'], commit) require(not (directory / 'deployment.pending.json').exists(), 'Resolve the previous pending deployment before attempting another update') run('docker', 'pull', image) @@ -143,33 +141,15 @@ def deployment(directory, channel, image, commit, version, client_container=None def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('channel', choices=('development', 'production')) + parser.add_argument('channel', choices=('development',)) parser.add_argument('--directory', required=True) parser.add_argument('--image') parser.add_argument('--commit') parser.add_argument('--version', required=True) parser.add_argument('--client-container') args = parser.parse_args() - if args.channel == 'production': - require(not args.image and not args.commit, 'Production accepts a completed version, not an arbitrary image') - gitea = API(os.environ['GITEA_API_URL'], os.environ['GITEA_READ_TOKEN']) - github = API('https://api.github.com', '', github=True) - build, receipt = release_receipt(gitea, github, args.version) - image, commit = GITHUB_IMAGE + '@' + receipt['image_digest'], build['commit'] - else: - image, commit = args.image, args.commit - require(image and commit, 'Development requires an image digest and commit') - if args.channel == 'production': - with release_state() as state: - if state and state.get('status') != 'complete': - require(state.get('commit') == commit and state.get('version') == args.version, - 'Another release is pending') - deployment(args.directory, args.channel, image, commit, args.version, args.client_container) - if state and state.get('commit') == commit: - state['status'] = 'complete' - save_state(state) - else: - deployment(args.directory, args.channel, image, commit, args.version, args.client_container) + require(args.image and args.commit, 'Development requires an image digest and commit') + deployment(args.directory, args.channel, args.image, args.commit, args.version, args.client_container) if __name__ == '__main__': diff --git a/scripts/release.py b/scripts/release.py index ac6c87e..13da40b 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -332,8 +332,7 @@ def preflight(version, commit): 'GitHub classic PAT needs repository access, workflow and write:packages') current = gitea.release(version) if current and not current['draft']: - build, _ = release_receipt(gitea, github, version) - require(build['commit'] == commit, 'Published version belongs to another commit') + complete_publication(gitea, github, version, commit) output('completed', 'true') output('reuse', 'true') return @@ -386,6 +385,16 @@ def save_state(state): temporary.replace(path) +def complete_publication(gitea, github, version, commit): + """Finish the matching reservation only after both published receipts validate.""" + state = selected_state() + require(state['version'] == version and state['commit'] == commit, 'Publication candidate mismatch') + build, _ = release_receipt(gitea, github, version) + require(build['commit'] == commit, 'Published version belongs to another commit') + state['status'] = 'complete' + save_state(state) + + def reserve(dev): require(re.fullmatch(r'[0-9a-f]{40}', dev), 'Expected the accepted dev SHA') with release_state() as current: @@ -651,7 +660,7 @@ def publish(): run('docker', 'buildx', 'imagetools', 'create', '--tag', image + ':' + stable_tag, image + '@' + value) require(registry_digest(image, stable_tag) == value, 'Promoted image digest differs') gitea.repo(f'/releases/{release["id"]}', method='PATCH', data={'draft': False}) - release_receipt(gitea, github, tag) + complete_publication(gitea, github, tag, build['commit']) print(f'Published RawMark {tag}: {build["commit"]}, {value}') diff --git a/scripts/tests/test_cicd.py b/scripts/tests/test_cicd.py index d4c844b..d89c687 100644 --- a/scripts/tests/test_cicd.py +++ b/scripts/tests/test_cicd.py @@ -239,7 +239,7 @@ class DeploymentTests(unittest.TestCase): def test_channel_mismatch_cannot_touch_docker(self): with patch.object(deploy, 'run') as docker: with self.assertRaisesRegex(ValueError, 'channel'): - deploy.deployment(self.directory, 'production', self.image, SHA, '0.1.1') + deploy.settings(self.directory, 'production') docker.assert_not_called() def test_unresolved_pending_deployment_cannot_be_overwritten(self): @@ -380,3 +380,52 @@ class MainAdmissionTests(unittest.TestCase): {'head_sha': SHA, 'event': 'push', 'path': 'dev.yml@refs/heads/dev', 'conclusion': 'success'}, ]}, {'workflow_runs': []}] self.assertFalse(release.successful_main_ci(api, SHA)) + + +class PublicationCompletionTests(unittest.TestCase): + def test_success_finishes_reservation_without_deployment(self): + state = {'version': 'version', 'commit': SHA, 'status': 'selected'} + with patch.object(release, 'selected_state', return_value=state), \ + patch.object(release, 'release_receipt', return_value=({'commit': SHA}, {})), \ + patch.object(release, 'save_state') as save: + release.complete_publication(None, None, 'version', SHA) + self.assertEqual(state['status'], 'complete') + save.assert_called_once_with(state) + + def test_partial_publication_remains_pending(self): + state = {'version': 'version', 'commit': SHA, 'status': 'selected'} + with patch.object(release, 'selected_state', return_value=state), \ + patch.object(release, 'release_receipt', side_effect=ValueError('Incomplete mirror')), \ + patch.object(release, 'save_state') as save: + with self.assertRaisesRegex(ValueError, 'Incomplete mirror'): + release.complete_publication(None, None, 'version', SHA) + self.assertEqual(state['status'], 'selected') + save.assert_not_called() + + def test_published_retry_repairs_pending_reservation(self): + state = {'version': 'version', 'commit': SHA, 'status': 'selected'} + gitea, github = Mock(), Mock() + gitea.request.return_value = github.request.return_value = {'login': 'eezd'} + github.repo.return_value = {'permissions': {'push': True}} + github.headers = {'X-OAuth-Scopes': 'repo, workflow, write:packages'} + gitea.release.return_value = {'draft': False} + with patch.dict(os.environ, {'CI_RUN_ID': '42'}), \ + patch.object(release, 'selected_state', return_value=state), \ + patch.object(release, 'clients', return_value=(gitea, github)), \ + patch.object(release, 'validate_identity'), \ + patch.object(release, 'release_receipt', return_value=({'commit': SHA}, {})), \ + patch.object(release, 'save_state'), patch.object(release, 'run') as run, \ + patch.object(release, 'output') as output: + release.preflight('version', SHA) + self.assertEqual(state['status'], 'complete') + output.assert_any_call('completed', 'true') + run.assert_not_called() + + def test_wrong_candidate_cannot_finish_reservation(self): + state = {'version': 'version', 'commit': SHA, 'status': 'selected'} + with patch.object(release, 'selected_state', return_value=state), \ + patch.object(release, 'release_receipt', return_value=({'commit': 'c' * 40}, {})), \ + patch.object(release, 'save_state') as save: + with self.assertRaisesRegex(ValueError, 'another commit'): + release.complete_publication(None, None, 'version', SHA) + save.assert_not_called()