Fix deployment preparation recovery and unify GitHub authentication

scripts/deploy.py

@@ -89,7 +89,8 @@ def health(command, build):
     containers = inspect_services(command)
     require({c['Config']['Labels']['com.docker.compose.service'] for c in containers} == set(SERVICES), 'Missing services')
     for container in containers:
-        require(container['State']['Running'] and container['RestartCount'] == 0, 'Service exited or restarted')
+        require(container['State']['Running'] and not container['State']['Paused'] and container['RestartCount'] == 0,
+                'Service exited, paused or restarted')
         part = 'web' if container['Name'] == '/AlphaPulse-dev' else 'backend'
         expected = json.loads(run('docker', 'image', 'inspect', build['images'][part]['digest']))[0]['Id']
         require(container['Image'] == expected, 'Running image differs from the accepted digest')
@@ -119,7 +120,9 @@ def deploy(build, directory=INSTANCE):
         previous_running = []
         if previous:
             old_command = compose(directory, previous)
-            previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in inspect_services(old_command) if c['State']['Running']]
+            containers = inspect_services(old_command)
+            require(not any(c['State'].get('Paused') for c in containers), 'Recover paused services before deploying')
+            previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in containers if c['State']['Running']]
             for _ in range(60):
                 idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
                 if not idle['busy']:
@@ -131,18 +134,17 @@ def deploy(build, directory=INSTANCE):
         if branch_head() != build['commit']:
             print('Skipped: dev advanced while preparing deployment')
             return
-        if previous and 'trader' in previous_running:
-            run('docker', 'pause', 'AlphaPulse-dev-trader')
-            try:
-                idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
-                require(not idle['busy'], 'Analysis became active during preparation; retry after it completes')
-            except BaseException:
-                run('docker', 'unpause', 'AlphaPulse-dev-trader')
-                raise
         pending = {'build': build, 'previous': previous, 'previous_compose': old_compose,
-                   'previous_running': previous_running, 'started_at': time.time()}
+                   'previous_running': previous_running, 'started_at': time.time(), 'phase': 'preparing'}
         save(directory / 'deployment.pending.json', pending)
         try:
+            if previous and 'trader' in previous_running:
+                run('docker', 'pause', 'AlphaPulse-dev-trader')
+                idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
+                require(not idle['busy'], 'Analysis became active during preparation; retry after it completes')
+            # Persist the transition before stopping any existing service.
+            save(directory / 'deployment.pending.json', {**pending, 'phase': 'updating'})
+            pending['phase'] = 'updating'
             if previous:
                 # Docker unpauses a paused container before delivering the stop signal.
                 run(*old_command, 'stop', 'trader')
@@ -160,6 +162,9 @@ def deploy(build, directory=INSTANCE):
             save(previous_file, build)
             (directory / 'deployment.pending.json').unlink()
         except BaseException:
+            if pending['phase'] == 'preparing':
+                restore(directory, pending)
+                raise
             # Never roll back or reset business data. A pending marker blocks subsequent jobs.
             subprocess.run(['docker', 'compose', '--project-name', 'alphapulse-dev', '--env-file', str(directory / 'images.env'),
                             '-f', str(directory / 'compose.yml'), 'stop'], capture_output=True)
@@ -172,6 +177,16 @@ def deploy(build, directory=INSTANCE):
 def restore(directory, pending):
     previous = pending['previous']
     require(previous is not None, 'No previous deployment exists; inspect the pending first-start failure')
+    trader = run('docker', 'ps', '-aq', '--filter', 'name=^/AlphaPulse-dev-trader$')
+    if trader:
+        state = json.loads(run('docker', 'inspect', trader))[0]['State']
+        if state['Paused']:
+            run('docker', 'unpause', trader)
+    if pending.get('phase') == 'preparing':
+        # No service has been stopped or replaced during preparation.
+        (directory / 'deployment.pending.json').unlink()
+        print('Restored preparation state; existing services were retained', flush=True)
+        return
     validate_images(previous)
     (directory / 'compose.yml').write_text(pending['previous_compose'])
     command = compose(directory, previous)

scripts/mirror.py

@@ -4,13 +4,12 @@
 import argparse
 import os
 from pathlib import Path
-import shlex
 import tempfile
 import time
 
 from ci import SOURCE, locked, request_json, require, run, save
 
-TARGET = 'git@github.com:eezd/AlphaPulse.git'
+TARGET = 'https://github.com/eezd/AlphaPulse.git'
 DIRECTORY = Path('/docker/gitea/mirrors/AlphaPulse')
 
 
@@ -47,16 +46,22 @@ def synchronize(directory, source, target, env=None):
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.parse_args()
-    # A write-enabled deploy key limits GitHub access to this repository.
+    token = os.environ.get('ALL_GITHUB_TOKEN')
+    require(token, 'Missing account-level ALL_GITHUB_TOKEN')
+    identity = request_json('https://api.github.com/user', token=token)
+    require(identity['login'] == 'eezd', 'GitHub token belongs to an unexpected account')
+    repository = request_json('https://api.github.com/repos/eezd/AlphaPulse', token=token)
+    require(repository.get('permissions', {}).get('push'), 'GitHub repository write permission is required')
+    # The helper contains no credentials; Git reads the token from the task environment.
     with tempfile.TemporaryDirectory(prefix='ap-mirror-auth-') as temporary:
-        key = Path(temporary) / 'key'
-        key.write_text(os.environ['ALPHAPULSE_GITHUB_SSH_KEY'])
-        key.chmod(0o600)
-        hosts = Path(temporary) / 'known_hosts'
-        hosts.write_text(''.join('github.com ' + entry + '\n' for entry in request_json('https://api.github.com/meta')['ssh_keys']))
-        command = ['ssh', '-i', str(key), '-o', 'IdentitiesOnly=yes', '-o', 'StrictHostKeyChecking=yes',
-                   '-o', 'UserKnownHostsFile=' + str(hosts)]
-        env = {**os.environ, 'GIT_SSH_COMMAND': shlex.join(command), 'GIT_TERMINAL_PROMPT': '0'}
+        helper = Path(temporary) / 'askpass'
+        helper.write_text('#!/bin/sh\ncase "$1" in\n'
+                          '  *github.com*) case "$1" in *Username*) printf "%s" "x-access-token" ;; '
+                          '*) printf "%s" "$ALL_GITHUB_TOKEN" ;; esac ;;\n'
+                          '  *) exit 1 ;;\nesac\n')
+        helper.chmod(0o700)
+        env = {**os.environ, 'GIT_ASKPASS': str(helper), 'GIT_TERMINAL_PROMPT': '0',
+               'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'credential.helper', 'GIT_CONFIG_VALUE_0': ''}
         synchronize(DIRECTORY, SOURCE, TARGET, env)
     print('GitHub business references and history verified')
 

scripts/tests/test_cicd.py

@@ -81,6 +81,85 @@ class BuildTests(unittest.TestCase):
                 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
 
 
+class RecoveryTests(unittest.TestCase):
+    def test_pending_write_failure_never_pauses_trader(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            directory = Path(temporary)
+            previous = {'images': {part: {'digest': 'synthetic'} for part in ci.IMAGES}}
+            ci.save(directory / 'deployment.json', previous)
+            (directory / 'compose.yml').write_text('synthetic')
+            running = [{'Config': {'Labels': {'com.docker.compose.service': 'trader'}}, 'State': {'Running': True}}]
+            with patch.object(deploy, 'INSTANCE', directory), patch.object(deploy, 'branch_head', return_value='a' * 40), \
+                    patch.object(deploy, 'validate_images'), patch.object(deploy, 'inspect_services', return_value=running), \
+                    patch.object(deploy, 'run', return_value='{"busy":0}') as run, \
+                    patch.object(deploy, 'save', side_effect=OSError('disk failure')):
+                with self.assertRaises(OSError):
+                    deploy.deploy({**previous, 'commit': 'a' * 40}, directory)
+                self.assertFalse(any(call.args[:2] == ('docker', 'pause') for call in run.call_args_list))
+
+    def test_busy_recheck_recovers_without_stopping_services(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            directory = Path(temporary)
+            previous = {'images': {part: {'digest': 'synthetic'} for part in ci.IMAGES}}
+            ci.save(directory / 'deployment.json', previous)
+            (directory / 'compose.yml').write_text('synthetic')
+            running = [{'Config': {'Labels': {'com.docker.compose.service': 'trader'}}, 'State': {'Running': True}}]
+            idle = iter(['{"busy":0}', '{"busy":1}'])
+            def command(*args, **kwargs):
+                if kwargs.get('input') == deploy.IDLE:
+                    return next(idle)
+                if args[:2] == ('docker', 'inspect'):
+                    return '[{"State":{"Paused":true}}]'
+                if args[:2] == ('docker', 'ps'):
+                    return 'trader-id'
+                return '{}'
+            with patch.object(deploy, 'INSTANCE', directory), patch.object(deploy, 'branch_head', return_value='a' * 40), \
+                    patch.object(deploy, 'validate_images'), patch.object(deploy, 'inspect_services', return_value=running), \
+                    patch.object(deploy, 'run', side_effect=command) as run:
+                with self.assertRaisesRegex(RuntimeError, 'Analysis became active'):
+                    deploy.deploy({**previous, 'commit': 'a' * 40}, directory)
+                self.assertFalse(any('stop' in call.args for call in run.call_args_list))
+                self.assertTrue(any(call.args[:2] == ('docker', 'unpause') for call in run.call_args_list))
+            self.assertFalse((directory / 'deployment.pending.json').exists())
+
+    def test_preparation_recovery_unpauses_without_replacing_services(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            directory = Path(temporary)
+            ci.save(directory / 'deployment.pending.json', {})
+            with patch.object(deploy, 'run', side_effect=['trader-id', '[{"State":{"Paused":true}}]', '']) as run, \
+                    patch.object(deploy, 'validate_images') as validate:
+                deploy.restore(directory, {'previous': {}, 'phase': 'preparing'})
+                run.assert_called_with('docker', 'unpause', 'trader-id')
+                validate.assert_not_called()
+            self.assertFalse((directory / 'deployment.pending.json').exists())
+
+    def test_failed_unpause_preserves_pending(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            directory = Path(temporary)
+            ci.save(directory / 'deployment.pending.json', {})
+            with patch.object(deploy, 'run', side_effect=['trader-id', '[{"State":{"Paused":true}}]', RuntimeError('unpause failed')]):
+                with self.assertRaisesRegex(RuntimeError, 'unpause failed'):
+                    deploy.restore(directory, {'previous': {}, 'phase': 'preparing'})
+            self.assertTrue((directory / 'deployment.pending.json').exists())
+
+    def test_missing_or_wrong_github_identity_never_synchronizes(self):
+        for token, identity in [('', None), ('synthetic', {'login': 'other'})]:
+            with self.subTest(token=bool(token)), patch.dict(os.environ, {'ALL_GITHUB_TOKEN': token}), \
+                    patch.object(sys, 'argv', ['mirror.py']), patch.object(mirror, 'request_json', return_value=identity), \
+                    patch.object(mirror, 'synchronize') as sync:
+                with self.assertRaises(RuntimeError):
+                    mirror.main()
+                sync.assert_not_called()
+
+    def test_github_read_only_token_never_synchronizes(self):
+        with patch.dict(os.environ, {'ALL_GITHUB_TOKEN': 'synthetic'}), patch.object(sys, 'argv', ['mirror.py']), \
+                patch.object(mirror, 'request_json', side_effect=[{'login': 'eezd'}, {'permissions': {'push': False}}]), \
+                patch.object(mirror, 'synchronize') as sync:
+            with self.assertRaisesRegex(RuntimeError, 'write permission'):
+                mirror.main()
+            sync.assert_not_called()
+
+
 class MirrorTests(unittest.TestCase):
     def setUp(self):
         self.temporary = tempfile.TemporaryDirectory()