Remove CLS collection and support a full news AI reset

scripts/ci.py

@@ -184,6 +184,8 @@ def smoke(build, label):
                   input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
         print(run('docker', 'exec', '-i', names['api'], 'python', '-',
                   input=(ROOT / 'scripts/smoke_calendar.py').read_text()), flush=True)
+        print(run('docker', 'exec', '-i', names['api'], 'python', '-',
+                  input=(ROOT / 'scripts/smoke_remove_cls.py').read_text()), flush=True)
         output = run('docker', 'exec', '-i', '-e', 'SMOKE_WEB=' + names['web'], names['api'], 'python', '-',
                      input=(ROOT / 'scripts/smoke_app.py').read_text())
         print(output, flush=True)

scripts/deploy.py

@@ -172,13 +172,13 @@ def deploy(build, directory=INSTANCE):
             containers = inspect_services(old_command)
             require(not any(c['State'].get('Paused') for c in containers), 'Recover paused services before deploying')
             previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in containers if c['State']['Running']]
-            for _ in range(60):
+            for _ in range(60) if previous_running else range(0):
                 idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
                 if not idle['busy']:
                     break
                 time.sleep(5)
             else:
-                raise RuntimeError('Chat or trading analysis did not become idle within five minutes')
+                require(not previous_running, 'Chat or trading analysis did not become idle within five minutes')
         # Recheck after pulls and waiting; only a current dev commit can mutate the instance.
         if branch_head() != build['commit']:
             print('Skipped: dev advanced while preparing deployment')
@@ -259,6 +259,37 @@ def restore(directory, pending):
 
 
 
+def remove_cls(build, execute=False):
+    """Retire CLS and reset all news AI using an accepted image and the instance lock."""
+    with locked(INSTANCE / '.deploy.lock'):
+        require(not (INSTANCE / 'deployment.pending.json').exists(), 'Recover the pending deployment first')
+        require(branch_head() == build['commit'], 'Expected the current dev build')
+        validate_images(build)
+        command = ['docker', 'run', '--rm', '-i', '--network', 'dev_default', '--env-file', str(INSTANCE / '.env'),
+                   IMAGES['backend'] + ':' + build['version']]
+        if execute:
+            current = json.loads((INSTANCE / 'deployment.json').read_text())
+            old_command = compose(INSTANCE, current)
+            for _ in range(60):
+                idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
+                if not idle['busy']:
+                    break
+                time.sleep(5)
+            else:
+                raise RuntimeError('Chat or trading analysis remained busy')
+            run('docker', 'pause', 'AlphaPulse-dev-trader')
+            idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE))
+            if idle['busy']:
+                run('docker', 'unpause', 'AlphaPulse-dev-trader')
+                raise RuntimeError('Analysis became active; retry after it completes')
+            run(*old_command, 'stop')
+        print(run(*command, 'python', '-m', 'alphapulse.crawler.remove_cls', *(['--execute'] if execute else [])), flush=True)
+        if execute:
+            print(run(*command, 'alembic', 'upgrade', 'head'), flush=True)
+    if execute:
+        deploy(build)
+
+
 def reset_news_ai(build, confirmed=False):
     """Preview or clear only legacy news AI records under the deployment lock."""
     validate_images(build)
@@ -315,14 +346,17 @@ def reset_news_derived(build, execute=False, acceptance=None, baseline=None):
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('command', choices=('update', 'recover', 'rollback', 'reset-news-ai', 'reset-news-derived', 'install-entrypoints'))
+    parser.add_argument('command', choices=('update', 'recover', 'rollback', 'reset-news-ai', 'reset-news-derived', 'install-entrypoints', 'remove-cls'))
+    parser.add_argument('--execute', action='store_true')
     parser.add_argument('--build', type=Path, default=ROOT / 'build.json')
     parser.add_argument('--confirm-delete-old-news-ai', action='store_true')
     parser.add_argument('--confirm-reset-news-derived', action='store_true')
     parser.add_argument('--acceptance', type=Path)
     parser.add_argument('--baseline', type=Path)
     args = parser.parse_args()
-    if args.command == 'install-entrypoints':
+    if args.command == 'remove-cls':
+        remove_cls(json.loads(args.build.read_text()), args.execute)
+    elif args.command == 'install-entrypoints':
         with locked(INSTANCE / '.deploy.lock'):
             require(not (INSTANCE / 'deployment.pending.json').exists(), 'Recover the pending deployment first')
             install_entrypoints(INSTANCE)

scripts/smoke_remove_cls.py

@@ -0,0 +1,80 @@
+"""Verify retirement and full news-AI reset only inside the disposable CI database."""
+import asyncio
+from datetime import UTC, datetime
+
+from sqlalchemy import DateTime, Integer, MetaData, String, insert, text
+from sqlalchemy.dialects.postgresql import JSONB
+
+from alphapulse.core.config import get_settings
+from alphapulse.core.db import Base, engine
+from alphapulse.crawler.remove_cls import AI_TABLES, OPTIONAL_AI_TABLES, cleanup
+from alphapulse.models import NewsItem, Source
+import alphapulse.models.risk  # noqa: F401
+import alphapulse.news_research.models  # noqa: F401
+
+
+async def main():
+    assert get_settings().database_url.startswith('postgresql+asyncpg://ci:ci-only@')
+    assert get_settings().database_url.endswith('/ci')
+    try:
+        async with engine.connect() as connection:
+            transaction = await connection.begin()
+            try:
+                assert not await connection.scalar(text("SELECT count(*) FROM sources WHERE code IN ('cls_depth','cls_telegraph')"))
+                await connection.execute(insert(Source).values(code='cls_depth', name='CI CLS', list_url='https://example.com', interval_seconds=120))
+                for identity, source in ((9000001, 'cls_depth'), (9000002, 'wscn_live')):
+                    await connection.execute(insert(NewsItem).values(id=identity, source_code=source, external_id='cleanup-ci',
+                        url='https://example.com', published_at=datetime.now(UTC), content_hash=str(identity)))
+                assert set((await cleanup(connection))['absent_legacy_tables']) == set(OPTIONAL_AI_TABLES)
+                for name in OPTIONAL_AI_TABLES:
+                    await connection.run_sync(lambda sync, name=name: Base.metadata.tables[name].create(sync))
+                reflected = MetaData()
+                await connection.run_sync(lambda sync: reflected.reflect(sync, only=list((*AI_TABLES, *OPTIONAL_AI_TABLES))))
+                for table in reflected.sorted_tables:
+                    if table.name not in (*AI_TABLES, *OPTIONAL_AI_TABLES):
+                        continue
+                    values = {}
+                    for column in table.columns:
+                        if not column.primary_key and (column.nullable or column.default is not None or column.server_default is not None):
+                            continue
+                        if isinstance(column.type, Integer):
+                            values[column.name] = 9000001
+                        elif isinstance(column.type, DateTime):
+                            values[column.name] = datetime.now(UTC)
+                        elif isinstance(column.type, String):
+                            values[column.name] = 'ci'
+                        elif isinstance(column.type, JSONB):
+                            values[column.name] = {}
+                        else:
+                            raise AssertionError(column)
+                    await connection.execute(table.insert().values(**values))
+                await connection.execute(text("INSERT INTO raw_bodies VALUES (:shared,:body),(:exclusive,:body)"),
+                    {'shared': 'a'*64, 'exclusive': 'b'*64, 'body': b'ci'})
+                for identity, source, digest in ((9000001,'cls_depth','a'*64),(9000002,'wscn_live','a'*64),(9000003,'cls_depth','b'*64)):
+                    await connection.execute(text('''INSERT INTO raw_responses
+                        (id,source_code,url,method,status,headers,body_sha256) VALUES
+                        (:id,:source,'https://example.com','GET',200,'{}',:digest)'''),
+                        {'id':identity,'source':source,'digest':digest})
+                preview = await cleanup(connection)
+                assert not preview['executed'] and preview['counts']['sources'] == 1
+                savepoint = await connection.begin_nested()
+                await cleanup(connection, True)
+                await savepoint.rollback()
+                assert await connection.scalar(text("SELECT count(*) FROM news_items WHERE id=9000001")) == 1
+                result = await cleanup(connection, True)
+                assert result['executed']
+                for table in (*AI_TABLES, *OPTIONAL_AI_TABLES):
+                    assert await connection.scalar(text('SELECT count(*) FROM '+table)) == 0, table
+                assert await connection.scalar(text("SELECT count(*) FROM news_items WHERE id=9000001")) == 0
+                assert await connection.scalar(text("SELECT count(*) FROM news_items WHERE id=9000002")) == 1
+                assert await connection.scalar(text("SELECT count(*) FROM raw_bodies WHERE sha256=:digest"), {'digest':'a'*64}) == 1
+                assert await connection.scalar(text("SELECT count(*) FROM raw_bodies WHERE sha256=:digest"), {'digest':'b'*64}) == 0
+                assert (await cleanup(connection, True))['counts']['sources'] == 0
+                print('CLS removal: preview, rollback, shared bodies, retained originals, 26 AI tables and repeat execution passed')
+            finally:
+                await transaction.rollback()
+    finally:
+        await engine.dispose()
+
+
+asyncio.run(main())

scripts/tests/test_remove_cls.py

@@ -0,0 +1,56 @@
+"""Exercise retirement orchestration with synthetic configuration only."""
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import deploy
+
+
+class RemoveClsTests(unittest.TestCase):
+    def exercise(self, execute, failure=False):
+        with tempfile.TemporaryDirectory() as temporary:
+            directory = Path(temporary)
+            build = {'commit': 'a'*40, 'version': '20261104-aaaaaaaa'}
+            (directory / 'deployment.json').write_text(json.dumps(build))
+            calls = []
+
+            def command(*args, **kwargs):
+                calls.append(args)
+                if kwargs.get('input') == deploy.IDLE:
+                    return '{"busy":0}'
+                if failure and '--execute' in args:
+                    raise RuntimeError('cleanup failed')
+                return '{}'
+
+            with patch.object(deploy, 'INSTANCE', directory), patch.object(deploy, 'validate_images'), \
+                    patch.object(deploy, 'branch_head', return_value=build['commit']), \
+                    patch.object(deploy, 'run', side_effect=command), patch.object(deploy, 'deploy') as update:
+                if failure:
+                    with self.assertRaisesRegex(RuntimeError, 'cleanup failed'):
+                        deploy.remove_cls(build, execute)
+                    update.assert_not_called()
+                else:
+                    deploy.remove_cls(build, execute)
+                    self.assertEqual(update.call_count, int(execute))
+            return calls
+
+    def test_preview_never_stops_migrates_or_deploys(self):
+        calls = self.exercise(False)
+        self.assertEqual(len(calls), 1)
+        self.assertNotIn('--execute', calls[0])
+        self.assertNotIn('alembic', calls[0])
+
+    def test_execute_stops_before_cleanup_and_migration(self):
+        calls = self.exercise(True)
+        stop = next(i for i, args in enumerate(calls) if 'stop' in args)
+        cleanup = next(i for i, args in enumerate(calls) if '--execute' in args)
+        migration = next(i for i, args in enumerate(calls) if 'alembic' in args)
+        self.assertLess(stop, cleanup)
+        self.assertLess(cleanup, migration)
+
+    def test_cleanup_failure_leaves_writers_stopped(self):
+        calls = self.exercise(True, True)
+        self.assertTrue(any('stop' in args for args in calls))
+        self.assertFalse(any('up' in args or 'alembic' in args for args in calls))