diff --git a/.gitea/workflows/mirror.yml b/.gitea/workflows/mirror.yml index f1b7cfc..684bcda 100644 --- a/.gitea/workflows/mirror.yml +++ b/.gitea/workflows/mirror.yml @@ -30,5 +30,5 @@ jobs: persist-credentials: false - name: Reconcile the current source references env: - ALPHAPULSE_GITHUB_SSH_KEY: ${{ secrets.ALPHAPULSE_GITHUB_SSH_KEY }} + ALL_GITHUB_TOKEN: ${{ secrets.ALL_GITHUB_TOKEN }} run: python3 scripts/mirror.py diff --git a/deploy/README.md b/deploy/README.md index a682eb1..d9d81eb 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -25,7 +25,7 @@ Automatic and manual updates use the same command and persistent lock: python3 scripts/deploy.py update --build /path/to/accepted/build.json ``` -The command checks image identities, current dev SHA and database migration compatibility before updating. It waits for active Chat/trading analysis, records pending state, then starts API/web before the workers. An outdated dev build is skipped. A pending deployment blocks updates. Do not invoke `docker compose up` directly to bypass the lock. +The command checks image identities, current dev SHA and database migration compatibility before updating. It waits for active Chat/trading analysis, records pending state, then starts API/web before the workers. An outdated dev build is skipped. A pending deployment blocks updates. Preparation is recorded before pausing the trader; recovery resumes a paused trader without restarting other services if updates have not begun. Do not invoke `docker compose up` directly to bypass the lock. A failed update with a previous deployment restores the prior images/configuration while retaining data. With no previous deployment, the pending marker remains for diagnosis. Recovery and rollback are explicit operator commands: @@ -38,7 +38,7 @@ Neither command restores database contents or resets trading orders. Schema migr ## Credentials and mirroring -Gitea secrets are `ALPHAPULSE_GITEA_TOKEN` (image publication) and `ALPHAPULSE_GITHUB_SSH_KEY` (a write-enabled deploy key restricted to the GitHub mirror). Ordinary checks and the deployment job do not receive publication secrets. Credentials are removed from job-local files on exit. +Gitea secrets are `ALPHAPULSE_GITEA_TOKEN` (image publication) and `ALL_GITHUB_TOKEN` (the account-level GitHub PAT classic with repository write permission; no repository-level override). Ordinary checks and the deployment job do not receive publication secrets. The GitHub token is read from the task environment, never stored in Git URLs or credential files. The trusted mirror task verifies the GitHub identity and repository write permission before synchronization. Git mirroring is independent of build results. Push/create/delete events, hourly reconciliation and manual dispatch use the same `.mirror.lock` in `/docker/gitea/mirrors/AlphaPulse`. Only business heads/tags are managed; internal PR references are excluded. Missing or incomplete source reads fail rather than imply deletion. GitHub is not an independent development source. GitHub Actions is disabled on the mirror. diff --git a/scripts/deploy.py b/scripts/deploy.py index 6b86333..cd54164 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -89,7 +89,8 @@ def health(command, build): containers = inspect_services(command) require({c['Config']['Labels']['com.docker.compose.service'] for c in containers} == set(SERVICES), 'Missing services') for container in containers: - require(container['State']['Running'] and container['RestartCount'] == 0, 'Service exited or restarted') + require(container['State']['Running'] and not container['State']['Paused'] and container['RestartCount'] == 0, + 'Service exited, paused or restarted') part = 'web' if container['Name'] == '/AlphaPulse-dev' else 'backend' expected = json.loads(run('docker', 'image', 'inspect', build['images'][part]['digest']))[0]['Id'] require(container['Image'] == expected, 'Running image differs from the accepted digest') @@ -119,7 +120,9 @@ def deploy(build, directory=INSTANCE): previous_running = [] if previous: old_command = compose(directory, previous) - previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in inspect_services(old_command) if c['State']['Running']] + containers = inspect_services(old_command) + require(not any(c['State'].get('Paused') for c in containers), 'Recover paused services before deploying') + previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in containers if c['State']['Running']] for _ in range(60): idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE)) if not idle['busy']: @@ -131,18 +134,17 @@ def deploy(build, directory=INSTANCE): if branch_head() != build['commit']: print('Skipped: dev advanced while preparing deployment') return - if previous and 'trader' in previous_running: - run('docker', 'pause', 'AlphaPulse-dev-trader') - try: - idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE)) - require(not idle['busy'], 'Analysis became active during preparation; retry after it completes') - except BaseException: - run('docker', 'unpause', 'AlphaPulse-dev-trader') - raise pending = {'build': build, 'previous': previous, 'previous_compose': old_compose, - 'previous_running': previous_running, 'started_at': time.time()} + 'previous_running': previous_running, 'started_at': time.time(), 'phase': 'preparing'} save(directory / 'deployment.pending.json', pending) try: + if previous and 'trader' in previous_running: + run('docker', 'pause', 'AlphaPulse-dev-trader') + idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE)) + require(not idle['busy'], 'Analysis became active during preparation; retry after it completes') + # Persist the transition before stopping any existing service. + save(directory / 'deployment.pending.json', {**pending, 'phase': 'updating'}) + pending['phase'] = 'updating' if previous: # Docker unpauses a paused container before delivering the stop signal. run(*old_command, 'stop', 'trader') @@ -160,6 +162,9 @@ def deploy(build, directory=INSTANCE): save(previous_file, build) (directory / 'deployment.pending.json').unlink() except BaseException: + if pending['phase'] == 'preparing': + restore(directory, pending) + raise # Never roll back or reset business data. A pending marker blocks subsequent jobs. subprocess.run(['docker', 'compose', '--project-name', 'alphapulse-dev', '--env-file', str(directory / 'images.env'), '-f', str(directory / 'compose.yml'), 'stop'], capture_output=True) @@ -172,6 +177,16 @@ def deploy(build, directory=INSTANCE): def restore(directory, pending): previous = pending['previous'] require(previous is not None, 'No previous deployment exists; inspect the pending first-start failure') + trader = run('docker', 'ps', '-aq', '--filter', 'name=^/AlphaPulse-dev-trader$') + if trader: + state = json.loads(run('docker', 'inspect', trader))[0]['State'] + if state['Paused']: + run('docker', 'unpause', trader) + if pending.get('phase') == 'preparing': + # No service has been stopped or replaced during preparation. + (directory / 'deployment.pending.json').unlink() + print('Restored preparation state; existing services were retained', flush=True) + return validate_images(previous) (directory / 'compose.yml').write_text(pending['previous_compose']) command = compose(directory, previous) diff --git a/scripts/mirror.py b/scripts/mirror.py index 66e40e0..8cbd8e7 100644 --- a/scripts/mirror.py +++ b/scripts/mirror.py @@ -4,13 +4,12 @@ import argparse import os from pathlib import Path -import shlex import tempfile import time from ci import SOURCE, locked, request_json, require, run, save -TARGET = 'git@github.com:eezd/AlphaPulse.git' +TARGET = 'https://github.com/eezd/AlphaPulse.git' DIRECTORY = Path('/docker/gitea/mirrors/AlphaPulse') @@ -47,16 +46,22 @@ def synchronize(directory, source, target, env=None): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.parse_args() - # A write-enabled deploy key limits GitHub access to this repository. + token = os.environ.get('ALL_GITHUB_TOKEN') + require(token, 'Missing account-level ALL_GITHUB_TOKEN') + identity = request_json('https://api.github.com/user', token=token) + require(identity['login'] == 'eezd', 'GitHub token belongs to an unexpected account') + repository = request_json('https://api.github.com/repos/eezd/AlphaPulse', token=token) + require(repository.get('permissions', {}).get('push'), 'GitHub repository write permission is required') + # The helper contains no credentials; Git reads the token from the task environment. with tempfile.TemporaryDirectory(prefix='ap-mirror-auth-') as temporary: - key = Path(temporary) / 'key' - key.write_text(os.environ['ALPHAPULSE_GITHUB_SSH_KEY']) - key.chmod(0o600) - hosts = Path(temporary) / 'known_hosts' - hosts.write_text(''.join('github.com ' + entry + '\n' for entry in request_json('https://api.github.com/meta')['ssh_keys'])) - command = ['ssh', '-i', str(key), '-o', 'IdentitiesOnly=yes', '-o', 'StrictHostKeyChecking=yes', - '-o', 'UserKnownHostsFile=' + str(hosts)] - env = {**os.environ, 'GIT_SSH_COMMAND': shlex.join(command), 'GIT_TERMINAL_PROMPT': '0'} + helper = Path(temporary) / 'askpass' + helper.write_text('#!/bin/sh\ncase "$1" in\n' + ' *github.com*) case "$1" in *Username*) printf "%s" "x-access-token" ;; ' + '*) printf "%s" "$ALL_GITHUB_TOKEN" ;; esac ;;\n' + ' *) exit 1 ;;\nesac\n') + helper.chmod(0o700) + env = {**os.environ, 'GIT_ASKPASS': str(helper), 'GIT_TERMINAL_PROMPT': '0', + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'credential.helper', 'GIT_CONFIG_VALUE_0': ''} synchronize(DIRECTORY, SOURCE, TARGET, env) print('GitHub business references and history verified') diff --git a/scripts/tests/test_cicd.py b/scripts/tests/test_cicd.py index bd2906d..a6435aa 100644 --- a/scripts/tests/test_cicd.py +++ b/scripts/tests/test_cicd.py @@ -81,6 +81,85 @@ class BuildTests(unittest.TestCase): fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) +class RecoveryTests(unittest.TestCase): + def test_pending_write_failure_never_pauses_trader(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + previous = {'images': {part: {'digest': 'synthetic'} for part in ci.IMAGES}} + ci.save(directory / 'deployment.json', previous) + (directory / 'compose.yml').write_text('synthetic') + running = [{'Config': {'Labels': {'com.docker.compose.service': 'trader'}}, 'State': {'Running': True}}] + with patch.object(deploy, 'INSTANCE', directory), patch.object(deploy, 'branch_head', return_value='a' * 40), \ + patch.object(deploy, 'validate_images'), patch.object(deploy, 'inspect_services', return_value=running), \ + patch.object(deploy, 'run', return_value='{"busy":0}') as run, \ + patch.object(deploy, 'save', side_effect=OSError('disk failure')): + with self.assertRaises(OSError): + deploy.deploy({**previous, 'commit': 'a' * 40}, directory) + self.assertFalse(any(call.args[:2] == ('docker', 'pause') for call in run.call_args_list)) + + def test_busy_recheck_recovers_without_stopping_services(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + previous = {'images': {part: {'digest': 'synthetic'} for part in ci.IMAGES}} + ci.save(directory / 'deployment.json', previous) + (directory / 'compose.yml').write_text('synthetic') + running = [{'Config': {'Labels': {'com.docker.compose.service': 'trader'}}, 'State': {'Running': True}}] + idle = iter(['{"busy":0}', '{"busy":1}']) + def command(*args, **kwargs): + if kwargs.get('input') == deploy.IDLE: + return next(idle) + if args[:2] == ('docker', 'inspect'): + return '[{"State":{"Paused":true}}]' + if args[:2] == ('docker', 'ps'): + return 'trader-id' + return '{}' + with patch.object(deploy, 'INSTANCE', directory), patch.object(deploy, 'branch_head', return_value='a' * 40), \ + patch.object(deploy, 'validate_images'), patch.object(deploy, 'inspect_services', return_value=running), \ + patch.object(deploy, 'run', side_effect=command) as run: + with self.assertRaisesRegex(RuntimeError, 'Analysis became active'): + deploy.deploy({**previous, 'commit': 'a' * 40}, directory) + self.assertFalse(any('stop' in call.args for call in run.call_args_list)) + self.assertTrue(any(call.args[:2] == ('docker', 'unpause') for call in run.call_args_list)) + self.assertFalse((directory / 'deployment.pending.json').exists()) + + def test_preparation_recovery_unpauses_without_replacing_services(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + ci.save(directory / 'deployment.pending.json', {}) + with patch.object(deploy, 'run', side_effect=['trader-id', '[{"State":{"Paused":true}}]', '']) as run, \ + patch.object(deploy, 'validate_images') as validate: + deploy.restore(directory, {'previous': {}, 'phase': 'preparing'}) + run.assert_called_with('docker', 'unpause', 'trader-id') + validate.assert_not_called() + self.assertFalse((directory / 'deployment.pending.json').exists()) + + def test_failed_unpause_preserves_pending(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + ci.save(directory / 'deployment.pending.json', {}) + with patch.object(deploy, 'run', side_effect=['trader-id', '[{"State":{"Paused":true}}]', RuntimeError('unpause failed')]): + with self.assertRaisesRegex(RuntimeError, 'unpause failed'): + deploy.restore(directory, {'previous': {}, 'phase': 'preparing'}) + self.assertTrue((directory / 'deployment.pending.json').exists()) + + def test_missing_or_wrong_github_identity_never_synchronizes(self): + for token, identity in [('', None), ('synthetic', {'login': 'other'})]: + with self.subTest(token=bool(token)), patch.dict(os.environ, {'ALL_GITHUB_TOKEN': token}), \ + patch.object(sys, 'argv', ['mirror.py']), patch.object(mirror, 'request_json', return_value=identity), \ + patch.object(mirror, 'synchronize') as sync: + with self.assertRaises(RuntimeError): + mirror.main() + sync.assert_not_called() + + def test_github_read_only_token_never_synchronizes(self): + with patch.dict(os.environ, {'ALL_GITHUB_TOKEN': 'synthetic'}), patch.object(sys, 'argv', ['mirror.py']), \ + patch.object(mirror, 'request_json', side_effect=[{'login': 'eezd'}, {'permissions': {'push': False}}]), \ + patch.object(mirror, 'synchronize') as sync: + with self.assertRaisesRegex(RuntimeError, 'write permission'): + mirror.main() + sync.assert_not_called() + + class MirrorTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory()