Use DEV as the sole personal instance #4

Merged
eezd merged 2 commits from dev into main 2026-10-30 13:40:02 +08:00
6 changed files with 70 additions and 117 deletions
Showing only changes of commit ef39f9d120 - Show all commits

No files matched your search

-53
View File
@@ -1,53 +0,0 @@
name: Deploy production manually
on:
workflow_dispatch:
inputs:
version:
description: Completed date-hash release version
required: true
type: string
permissions:
contents: read
concurrency:
group: rawmark-production
cancel-in-progress: false
jobs:
deploy:
if: gitea.ref == 'refs/heads/main'
runs-on: docker
timeout-minutes: 20
container:
image: docker:29-cli
volumes:
- /docker/gitea/main/RawMark:/docker/gitea/main/RawMark
- /docker/gitea/mirrors/RawMark:/docker/gitea/mirrors/RawMark
env:
GITEA_API_URL: ${{ gitea.server_url }}/api/v1
GITEA_READ_TOKEN: ${{ gitea.token }}
RELEASE_VERSION: ${{ inputs.version }}
DOCKER_CONFIG: /tmp/rawmark-anonymous-docker
defaults:
run:
shell: sh
steps:
- name: Install deployment runtimes
run: apk add --no-cache nodejs python3 git docker-cli-compose
- uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
ref: main
fetch-depth: 0
persist-credentials: false
- name: Verify completed release, back up and deploy only RawMark
run: |
set -eu
mkdir -p "$DOCKER_CONFIG"
chmod 700 "$DOCKER_CONFIG"
python3 scripts/deploy.py production --directory /docker/gitea/main/RawMark \
--version "$RELEASE_VERSION" --client-container "$(hostname)"
- name: Remove job configuration
if: always()
run: rm -rf "$DOCKER_CONFIG"
-32
View File
@@ -182,35 +182,3 @@ jobs:
run: | run: |
python3 scripts/ci-cleanup.py "$CI_RESOURCE_LABEL" --client-container "$(hostname)" python3 scripts/ci-cleanup.py "$CI_RESOURCE_LABEL" --client-container "$(hostname)"
rm -rf "$DOCKER_CONFIG" rm -rf "$DOCKER_CONFIG"
deploy:
needs: [build, publish]
if: always() && needs.build.result == 'success' && (needs.publish.result == 'success' || needs.publish.result == 'skipped')
runs-on: docker
timeout-minutes: 20
container:
image: docker:29-cli
volumes:
- /docker/gitea/mirrors/RawMark:/docker/gitea/mirrors/RawMark
- /docker/gitea/main/RawMark:/docker/gitea/main/RawMark
env:
GITEA_READ_TOKEN: ${{ secrets.RAWMARK_GITEA_TOKEN }}
DOCKER_CONFIG: /tmp/rawmark-anonymous-docker
steps:
- name: Install deployment runtimes
run: apk add --no-cache nodejs python3 git docker-cli-compose
- uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262
with:
ref: ${{ gitea.sha }}
fetch-depth: 0
persist-credentials: false
- name: Deploy the completed release and finish its reservation
run: |
set -eu
mkdir -p "$DOCKER_CONFIG"
python3 scripts/deploy.py production --directory /docker/gitea/main/RawMark \
--version "$(python3 -c 'import sys;sys.path.insert(0,"scripts");import release;print(release.selected_state()["version"])')" \
--client-container "$(hostname)"
- name: Remove temporary Docker configuration
if: always()
run: rm -rf "$DOCKER_CONFIG"
+2 -2
View File
@@ -133,8 +133,8 @@ Gitea 为公开主仓库,GitHub 为单向公开镜像;默认分支均为 mai
- dev push:容器内检查、Linux amd64 打包与隔离冒烟,通过后发布 `git.134333.xyz/eezd/dev/rawmark:YYYYMMDD-<8位SHA>`,按摘要部署 DEV。日期取提交时间 UTC,完整 SHA 保留在回执;同名异提交拒绝,已验证镜像重试复用。 - dev push:容器内检查、Linux amd64 打包与隔离冒烟,通过后发布 `git.134333.xyz/eezd/dev/rawmark:YYYYMMDD-<8位SHA>`,按摘要部署 DEV。日期取提交时间 UTC,完整 SHA 保留在回执;同名异提交拒绝,已验证镜像重试复用。
- main push:独立检查实际合并提交、Linux amd64 构建及容器冒烟,不发布镜像、不部署。仓库内受信 PR 只检查;外部 fork PR 不自动进入共享 Docker runner,不注入发布凭据。 - main push:独立检查实际合并提交、Linux amd64 构建及容器冒烟,不发布镜像、不部署。仓库内受信 PR 只检查;外部 fork PR 不自动进入共享 Docker runner,不注入发布凭据。
- DEV 为 `/docker/gitea/dev/RawMark`,容器 `RawMark-dev`,Compose 项目 `rawmark-dev`,监听 `127.0.0.1:21001`,域名 `raw-dev.134333.xyz`。数据独立;所有部署共用宿主实例 `.deploy.lock`,锁内核对 dev 最新 SHA,pending 未处理时拒绝覆盖。已有数据即使无旧容器也必须备份。 - DEV 为 `/docker/gitea/dev/RawMark`,容器 `RawMark-dev`,Compose 项目 `rawmark-dev`,监听 `127.0.0.1:21001`,域名 `raw.134333.xyz`,raw-dev 域名停用。DEV 是唯一常驻实例,保存个人真实数据;CI 与冒烟使用隔离合成数据。数据独立;所有部署共用宿主实例 `.deploy.lock`,锁内核对 dev 最新 SHA,pending 未处理时拒绝覆盖。已有数据即使无旧容器也必须备份。
- Git 镜像独立于 CI 成败:push/create/delete、每小时对账和手动重试,共用 `/docker/gitea/mirrors/RawMark/.mirror.lock`,仅同步业务分支和 TAG 及完整历史,跟随源端删除。同步代码从 main 读取;`RAWMARK_MIRROR_ENABLED=true` 后启用,凭据统一使用 Gitea 账号级 Secret `ALL_GITHUB_TOKEN`(头像 → 设置 → Actions → 密钥),接入前配置并验证实际权限,仓库级同名项会覆盖账号级值。DEV 镜像不上传 GHCR。main 切换前保持该变量关闭。 - Git 镜像独立于 CI 成败:push/create/delete、每小时对账和手动重试,共用 `/docker/gitea/mirrors/RawMark/.mirror.lock`,仅同步业务分支和 TAG 及完整历史,跟随源端删除。同步代码从 main 读取;`RAWMARK_MIRROR_ENABLED=true` 后启用,凭据统一使用 Gitea 账号级 Secret `ALL_GITHUB_TOKEN`(头像 → 设置 → Actions → 密钥),接入前配置并验证实际权限,仓库级同名项会覆盖账号级值。DEV 镜像不上传 GHCR。main 切换前保持该变量关闭。
- CI 凭据 `RAWMARK_GITEA_TOKEN` 需要仓库和 Packages 写权限。runner 保持容量一、`sweep: false`,只运行受信代码;工具链安装在任务容器。资源按任务标签清理,禁止全局 prune。 - CI 凭据 `RAWMARK_GITEA_TOKEN` 需要仓库和 Packages 写权限。runner 保持容量一、`sweep: false`,只运行受信代码;工具链安装在任务容器。资源按任务标签清理,禁止全局 prune。
- 开发构建通过 `RAWMARK_BUILD_CHANNEL=development` 派生版本,前端与二进制一致;正式 `VERSION` 不因开发构建改变。开发包为 `RawMark_<版本>_linux_amd64.tar.gz`;历史正式产物不改名。安装包及单次传输保持既有 90,000,000 字节预算。 - 开发构建通过 `RAWMARK_BUILD_CHANNEL=development` 派生版本,前端与二进制一致;正式 `VERSION` 不因开发构建改变。开发包为 `RawMark_<版本>_linux_amd64.tar.gz`;历史正式产物不改名。安装包及单次传输保持既有 90,000,000 字节预算。
- 正式发布从受信 main 手动触发,使用日期哈希注解 TAG、六平台程序包与 Linux 双架构镜像;发布 ID 和候选 SHA 固定,发布记录与 .release.lock 位于持久镜像目录。GitHub 引用同步复用镜像锁,Pages Demo 不切换。双端发布和 GHCR 公开拉取验收后自动部署 MAIN:/docker/gitea/main/RawMark、RawMark-main、rawmark-main,127.0.0.1:21000,raw.134333.xyz。自动/手动/批量启停共用实例 .deploy.lock,旧版本不得覆盖新版;生产数据保留并先备份,不随开发推送更新。 - 正式发布从受信 main 手动触发,使用日期哈希注解 TAG、六平台程序包与 Linux 双架构镜像;发布 ID 和候选 SHA 固定,发布记录与 .release.lock 位于持久镜像目录。GitHub 引用同步复用镜像锁,Pages Demo 不切换。双端发布和 GHCR 公开拉取验收后结束发布并标记预约完成,不部署 MAIN、不更新 DEV。DEV 批量管理入口为 `/docker/gitea/dev/manage.sh start|stop`,与 CD 共用实例锁,pending 阻止启动。
+6 -26
View File
@@ -14,7 +14,7 @@ import tempfile
import time import time
import urllib.request import urllib.request
from release import API, GITEA_DEV_IMAGE, GITHUB_IMAGE, release_receipt, require, run, release_state, save_state from release import API, GITEA_DEV_IMAGE, require, run
def settings(directory, channel): def settings(directory, channel):
@@ -48,16 +48,14 @@ def save(path, data):
def deployment(directory, channel, image, commit, version, client_container=None): def deployment(directory, channel, image, commit, version, client_container=None):
directory = Path(directory).resolve(strict=True) directory = Path(directory).resolve(strict=True)
require(channel == 'development', 'Only the persistent DEV instance supports deployment')
cfg = settings(directory, channel) cfg = settings(directory, channel)
registry = GITEA_DEV_IMAGE if channel == 'development' else GITHUB_IMAGE registry = GITEA_DEV_IMAGE
require(re.fullmatch(re.escape(registry) + r'@sha256:[0-9a-f]{64}', image), 'Expected an immutable registry digest') require(re.fullmatch(re.escape(registry) + r'@sha256:[0-9a-f]{64}', image), 'Expected an immutable registry digest')
require(re.fullmatch(r'[0-9a-f]{40}', commit), 'Expected a full commit SHA') require(re.fullmatch(r'[0-9a-f]{40}', commit), 'Expected a full commit SHA')
# Serializes actual deployments, including manually started jobs and CLI use. # Serializes actual deployments, including manually started jobs and CLI use.
with (directory / '.deploy.lock').open('a') as lock: with (directory / '.deploy.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX) fcntl.flock(lock, fcntl.LOCK_EX)
if channel == 'production' and (directory / 'deployment.json').exists():
deployed = json.loads((directory / 'deployment.json').read_text())
run('git', 'merge-base', '--is-ancestor', deployed['commit'], commit)
require(not (directory / 'deployment.pending.json').exists(), require(not (directory / 'deployment.pending.json').exists(),
'Resolve the previous pending deployment before attempting another update') 'Resolve the previous pending deployment before attempting another update')
run('docker', 'pull', image) run('docker', 'pull', image)
@@ -143,33 +141,15 @@ def deployment(directory, channel, image, commit, version, client_container=None
def main(): def main():
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('channel', choices=('development', 'production')) parser.add_argument('channel', choices=('development',))
parser.add_argument('--directory', required=True) parser.add_argument('--directory', required=True)
parser.add_argument('--image') parser.add_argument('--image')
parser.add_argument('--commit') parser.add_argument('--commit')
parser.add_argument('--version', required=True) parser.add_argument('--version', required=True)
parser.add_argument('--client-container') parser.add_argument('--client-container')
args = parser.parse_args() args = parser.parse_args()
if args.channel == 'production': require(args.image and args.commit, 'Development requires an image digest and commit')
require(not args.image and not args.commit, 'Production accepts a completed version, not an arbitrary image') deployment(args.directory, args.channel, args.image, args.commit, args.version, args.client_container)
gitea = API(os.environ['GITEA_API_URL'], os.environ['GITEA_READ_TOKEN'])
github = API('https://api.github.com', '', github=True)
build, receipt = release_receipt(gitea, github, args.version)
image, commit = GITHUB_IMAGE + '@' + receipt['image_digest'], build['commit']
else:
image, commit = args.image, args.commit
require(image and commit, 'Development requires an image digest and commit')
if args.channel == 'production':
with release_state() as state:
if state and state.get('status') != 'complete':
require(state.get('commit') == commit and state.get('version') == args.version,
'Another release is pending')
deployment(args.directory, args.channel, image, commit, args.version, args.client_container)
if state and state.get('commit') == commit:
state['status'] = 'complete'
save_state(state)
else:
deployment(args.directory, args.channel, image, commit, args.version, args.client_container)
if __name__ == '__main__': if __name__ == '__main__':
+12 -3
View File
@@ -332,8 +332,7 @@ def preflight(version, commit):
'GitHub classic PAT needs repository access, workflow and write:packages') 'GitHub classic PAT needs repository access, workflow and write:packages')
current = gitea.release(version) current = gitea.release(version)
if current and not current['draft']: if current and not current['draft']:
build, _ = release_receipt(gitea, github, version) complete_publication(gitea, github, version, commit)
require(build['commit'] == commit, 'Published version belongs to another commit')
output('completed', 'true') output('completed', 'true')
output('reuse', 'true') output('reuse', 'true')
return return
@@ -386,6 +385,16 @@ def save_state(state):
temporary.replace(path) temporary.replace(path)
def complete_publication(gitea, github, version, commit):
"""Finish the matching reservation only after both published receipts validate."""
state = selected_state()
require(state['version'] == version and state['commit'] == commit, 'Publication candidate mismatch')
build, _ = release_receipt(gitea, github, version)
require(build['commit'] == commit, 'Published version belongs to another commit')
state['status'] = 'complete'
save_state(state)
def reserve(dev): def reserve(dev):
require(re.fullmatch(r'[0-9a-f]{40}', dev), 'Expected the accepted dev SHA') require(re.fullmatch(r'[0-9a-f]{40}', dev), 'Expected the accepted dev SHA')
with release_state() as current: with release_state() as current:
@@ -651,7 +660,7 @@ def publish():
run('docker', 'buildx', 'imagetools', 'create', '--tag', image + ':' + stable_tag, image + '@' + value) run('docker', 'buildx', 'imagetools', 'create', '--tag', image + ':' + stable_tag, image + '@' + value)
require(registry_digest(image, stable_tag) == value, 'Promoted image digest differs') require(registry_digest(image, stable_tag) == value, 'Promoted image digest differs')
gitea.repo(f'/releases/{release["id"]}', method='PATCH', data={'draft': False}) gitea.repo(f'/releases/{release["id"]}', method='PATCH', data={'draft': False})
release_receipt(gitea, github, tag) complete_publication(gitea, github, tag, build['commit'])
print(f'Published RawMark {tag}: {build["commit"]}, {value}') print(f'Published RawMark {tag}: {build["commit"]}, {value}')
+50 -1
View File
@@ -239,7 +239,7 @@ class DeploymentTests(unittest.TestCase):
def test_channel_mismatch_cannot_touch_docker(self): def test_channel_mismatch_cannot_touch_docker(self):
with patch.object(deploy, 'run') as docker: with patch.object(deploy, 'run') as docker:
with self.assertRaisesRegex(ValueError, 'channel'): with self.assertRaisesRegex(ValueError, 'channel'):
deploy.deployment(self.directory, 'production', self.image, SHA, '0.1.1') deploy.settings(self.directory, 'production')
docker.assert_not_called() docker.assert_not_called()
def test_unresolved_pending_deployment_cannot_be_overwritten(self): def test_unresolved_pending_deployment_cannot_be_overwritten(self):
@@ -380,3 +380,52 @@ class MainAdmissionTests(unittest.TestCase):
{'head_sha': SHA, 'event': 'push', 'path': 'dev.yml@refs/heads/dev', 'conclusion': 'success'}, {'head_sha': SHA, 'event': 'push', 'path': 'dev.yml@refs/heads/dev', 'conclusion': 'success'},
]}, {'workflow_runs': []}] ]}, {'workflow_runs': []}]
self.assertFalse(release.successful_main_ci(api, SHA)) self.assertFalse(release.successful_main_ci(api, SHA))
class PublicationCompletionTests(unittest.TestCase):
def test_success_finishes_reservation_without_deployment(self):
state = {'version': 'version', 'commit': SHA, 'status': 'selected'}
with patch.object(release, 'selected_state', return_value=state), \
patch.object(release, 'release_receipt', return_value=({'commit': SHA}, {})), \
patch.object(release, 'save_state') as save:
release.complete_publication(None, None, 'version', SHA)
self.assertEqual(state['status'], 'complete')
save.assert_called_once_with(state)
def test_partial_publication_remains_pending(self):
state = {'version': 'version', 'commit': SHA, 'status': 'selected'}
with patch.object(release, 'selected_state', return_value=state), \
patch.object(release, 'release_receipt', side_effect=ValueError('Incomplete mirror')), \
patch.object(release, 'save_state') as save:
with self.assertRaisesRegex(ValueError, 'Incomplete mirror'):
release.complete_publication(None, None, 'version', SHA)
self.assertEqual(state['status'], 'selected')
save.assert_not_called()
def test_published_retry_repairs_pending_reservation(self):
state = {'version': 'version', 'commit': SHA, 'status': 'selected'}
gitea, github = Mock(), Mock()
gitea.request.return_value = github.request.return_value = {'login': 'eezd'}
github.repo.return_value = {'permissions': {'push': True}}
github.headers = {'X-OAuth-Scopes': 'repo, workflow, write:packages'}
gitea.release.return_value = {'draft': False}
with patch.dict(os.environ, {'CI_RUN_ID': '42'}), \
patch.object(release, 'selected_state', return_value=state), \
patch.object(release, 'clients', return_value=(gitea, github)), \
patch.object(release, 'validate_identity'), \
patch.object(release, 'release_receipt', return_value=({'commit': SHA}, {})), \
patch.object(release, 'save_state'), patch.object(release, 'run') as run, \
patch.object(release, 'output') as output:
release.preflight('version', SHA)
self.assertEqual(state['status'], 'complete')
output.assert_any_call('completed', 'true')
run.assert_not_called()
def test_wrong_candidate_cannot_finish_reservation(self):
state = {'version': 'version', 'commit': SHA, 'status': 'selected'}
with patch.object(release, 'selected_state', return_value=state), \
patch.object(release, 'release_receipt', return_value=({'commit': 'c' * 40}, {})), \
patch.object(release, 'save_state') as save:
with self.assertRaisesRegex(ValueError, 'another commit'):
release.complete_publication(None, None, 'version', SHA)
save.assert_not_called()