feat(news): restrict research to local sources and process frozen batches

scripts/accept_news_research.py

@@ -75,10 +75,6 @@ async def main(args):
     if len(news) != (12 if args.synthetic else 30):
         raise ValueError("样本数量不符合冻结范围")
     credentials = json.loads(args.credentials.read_text())
-    if not args.synthetic and credentials.get("searxng_url"):
-        from alphapulse.core.config import get_settings
-
-        get_settings().searxng_url = credentials["searxng_url"]
     config = {"isolated_acceptance": True, "concurrency": 3}
     async with SessionLocal() as session:
         if await session.scalar(select(func.count()).select_from(NewsItem)):
@@ -140,19 +136,11 @@ async def main(args):
         async with semaphore:
             await service.process_event(identifier, config, force=True)
 
-    async def controlled_web(self, name, arguments):
-        if name in ("search_web", "read_web"):
-            return {"status": "unavailable", "reason": "虚构验收组不使用真实外部网络"}
-        return await original_tool(self, name, arguments)
-
-    original_tool = service.Research.tool
     with ExitStack() as stack:
         for name in ("_stream_openai", "_stream_anthropic"):
             stack.enter_context(
                 patch.object(llm, name, budget.wrap(getattr(llm, name)))
             )
-        if args.synthetic:
-            stack.enter_context(patch.object(service.Research, "tool", controlled_web))
         for batch in batches:
             if args.budget.exists() and int(args.budget.read_text() or "0") >= 240:
                 break

scripts/accept_news_scenario.py

@@ -14,7 +14,7 @@ from alphapulse.ai import llm
 from alphapulse.ai.skills import create_node, add_version
 from alphapulse.core.crypto import encrypt, decrypt
 from alphapulse.core.db import SessionLocal, engine
-from alphapulse.models import AiProvider, NewsItem, SkillNode, SkillVersion
+from alphapulse.models import AiProvider, NewsItem, SkillNode
 from alphapulse.news_research import service
 from alphapulse.news_research.contracts import INDEX_SKILL, ASSIGN_SKILL, RESEARCH_SKILL
 from alphapulse.news_research.models import Package, Attempt, Event, Version, Membership, Reference, VocabularyVersion
@@ -129,39 +129,8 @@ def assertions(manifest, tables, checkpoints):
     return checks
 
 
-def official_assertions(manifest, tables, external):
-    packages = {p['id']: p['news_id'] for p in tables['news_research_packages']}
-    links = {n: {m['event_id'] for m in tables['news_research_memberships']
-                 if packages[m['package_id']] == n} for n in manifest['external_news_ids']}
-    checks = []
-    for news_id, source in zip(manifest['external_news_ids'], manifest['official_sources'], strict=True):
-        valid = False
-        for version in tables['news_research_versions']:
-            if version['state'] != 'done' or version['event_id'] not in links[news_id] or version['event_id'] not in external:
-                continue
-            output = version.get('output') or {}
-            refs = [ref for claim in output.get('claims', []) if claim.get('verdict') in ('supported', 'partial')
-                    for ref in claim.get('evidence_refs', [])]
-            for record in version['evidence']:
-                body = record.get('result', {})
-                if record['tool'] != 'read_web' or body.get('error') or not body.get('content'):
-                    continue
-                if urlparse(body.get('url', '')).hostname != urlparse(source['url']).hostname:
-                    continue
-                if not all(a.casefold() in body['content'].casefold() for a in source['anchors']):
-                    continue
-                if any(ref['article_id'] == body.get('article_id') and ref['quote'].strip()
-                       and ref['quote'] in body['content'] for ref in refs):
-                    valid = True
-        checks.append({'name': 'official_claim_evidence', 'news_id': news_id,
-                       'domain': urlparse(source['url']).hostname, 'passed': valid})
-    return checks
 
 
-def official_hints(manifest, news_ids):
-    return '\n验收提供的公开来源线索（链接不是证据）：' + '\n'.join(
-        source['url'] for news_id, source in zip(manifest['external_news_ids'], manifest['official_sources'], strict=True)
-        if news_id in news_ids)
 
 
 async def reference_api_assertion(manifest, tables):
@@ -194,18 +163,7 @@ async def _run(args):
     baseline = Path(manifest['baseline_counter'])
     if int(baseline.read_text()) != 199 or hashlib.sha256(baseline.read_bytes()).hexdigest() != manifest['baseline_counter_sha256']:
         raise ValueError('历史计数不匹配')
-    # Preflight precedes all paid calls, using the application's public reader.
-    from alphapulse.news_research.tools import fetch_public
-    preflight = []
-    for source in manifest['official_sources']:
-        result = await fetch_public(source['url'])
-        if not all(anchor.lower() in result['content'].lower() for anchor in source['anchors']):
-            raise ValueError('官方正文缺少预期事实锚点')
-        preflight.append({**result, 'sha256': digest(result['content'])})
     credentials = json.loads(args.credentials.read_text())
-    if credentials.get('searxng_url'):
-        from alphapulse.core.config import get_settings
-        get_settings().searxng_url = credentials['searxng_url']
     state_path = args.output / 'state.json'
     if args.resume:
         state = json.loads(state_path.read_text())
@@ -223,10 +181,9 @@ async def _run(args):
         args.output.mkdir(mode=0o700)
         config = await initialize(credentials)
         state = {'manifest_sha256': digest(manifest), 'credentials_sha256': digest(credentials), 'config': config,
-                 'rules': await freeze_config(config), 'batch': 0, 'checkpoints': [], 'requests': 0, 'external': []}
+                 'rules': await freeze_config(config), 'batch': 0, 'checkpoints': [], 'requests': 0}
         args.budget.write_text('0'); args.budget.chmod(0o600)
         save(state_path, state)
-    save(args.output / 'preflight.json', preflight)
     async with SessionLocal() as session:
         for stage in ('index', 'research'):
             provider = await session.get(AiProvider, state['config'][stage]['provider_id'])
@@ -266,13 +223,6 @@ async def _run(args):
         state['request_limit'] = limit
         save(state_path, state)
         budget = Budget(args.budget, limit)
-        original_tool = service.Research.tool
-        async def chronological_tool(self, name, arguments):
-            if name in ('search_web', 'read_web'):
-                result = {'status': 'unavailable', 'error': '历史顺序回放不读取当前互联网，最终单独核验'}
-                self.records.append({'tool': name, 'arguments': arguments, 'result': result})
-                return result
-            return await original_tool(self, name, arguments)
         async def process(identifier):
             await service.process_package(identifier, config)
         async def research_pending():
@@ -284,7 +234,7 @@ async def _run(args):
         with ExitStack() as stack:
             for name in ('_stream_openai', '_stream_anthropic'):
                 stack.enter_context(patch.object(llm, name, budget.wrap(getattr(llm, name))))
-            with patch.object(service.Research, 'tool', chronological_tool):
+            with ExitStack():
                 if args.repair_references:
                     if state['batch'] >= len(manifest['batches']):
                         raise ValueError('引用修复须在最终自然更新批次之前执行')
@@ -350,39 +300,10 @@ async def _run(args):
                     print(json.dumps({'batch': state['batch'], 'requests': state['requests'], 'complete': complete}), flush=True)
                     if not complete:
                         break
-            if state['batch'] == len(manifest['batches']) and int(args.budget.read_text()) < limit:
-                async with SessionLocal() as session:
-                    targets = list(await session.scalars(select(Membership.event_id).join(Package, Package.id == Membership.package_id).where(Package.news_id.in_(manifest['external_news_ids'])).distinct().order_by(Membership.event_id)))
-                for identifier in targets:
-                    if identifier in state['external'] or int(args.budget.read_text()) >= limit:
-                        continue
-                    async with SessionLocal() as session:
-                        # Supply only this event's official leads, never unrelated sources.
-                        news_ids = set(await session.scalars(select(Package.news_id).join(Membership).where(Membership.event_id == identifier)))
-                        content = RESEARCH_SKILL + official_hints(manifest, news_ids)
-                        node = await session.get(SkillNode, config['research']['skill_id'])
-                        version = await session.scalar(select(SkillVersion).where(SkillVersion.skill_id == node.id, SkillVersion.content == content).order_by(SkillVersion.version_no.desc()).limit(1))
-                        if version is None:
-                            version = await add_version(session, node, content, 'user', 'Event-specific official leads for final live verification')
-                        node.published_version_id = version.id
-                        event = await session.get(Event, identifier)
-                        if event.state == 'done':
-                            event.revision += 1
-                        event.state = 'pending'
-                        await session.commit()
-                    state['rules'] = await freeze_config(config)
-                    save(state_path, state)
-                    await service.process_event(identifier, config, force=True)
-                    async with SessionLocal() as session:
-                        event = await session.get(Event, identifier)
-                        if event.state == 'done':
-                            state['external'].append(identifier)
-                    state['requests'] = int(args.budget.read_text())
-                    save(state_path, state)
         tables = await snapshot()
         checks = assertions(manifest, tables, state['checkpoints'])
         checks.extend(await reference_api_assertion(manifest, tables))
-        checks.extend(official_assertions(manifest, tables, state['external']))
+        checks.append({'name': 'station_only_tools', 'passed': all(r['tool'] not in ('search_web', 'read_web') for v in tables['news_research_versions'] for r in v.get('evidence', []))})
         report = {'quality_accepted': False, 'long_running_quality_verified': False, 'automated_passed': all(c['passed'] for c in checks),
                   'assertions': checks, 'requests_new': int(args.budget.read_text()), 'requests_historical': 199,
                   'requests_cumulative': 199 + int(args.budget.read_text()), 'new_cap': limit, 'manifest': manifest,

scripts/ci.py

@@ -180,6 +180,8 @@ def smoke(build, label):
         run('docker', 'run', '-d', '--name', names['web'], '--network-alias', 'web', *common, build['images']['web']['local'])
         wait_command('docker', 'exec', names['api'], 'python', '-c',
                      'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)')
+        print(run('docker', 'exec', '-i', names['api'], 'python', '-',
+                  input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True)
         print(run('docker', 'exec', '-i', names['api'], 'python', '-',
                   input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
         print(run('docker', 'exec', '-i', names['api'], 'python', '-',

scripts/deploy.py

@@ -3,7 +3,6 @@
 
 import argparse
 import hashlib
-import secrets
 import json
 from pathlib import Path
 import re
@@ -16,7 +15,7 @@ SEARXNG_IMAGE = 'docker.io/searxng/searxng:2026.10.30-ffa88e3ba'
 SEARXNG_DIGEST = 'sha256:f4aa660c54d062dedb268be84e821ea24e39f48c7a496af8dd02cc9eedb93847'
 
 INSTANCE = Path('/docker/gitea/dev/AlphaPulse')
-SERVICES = ('api', 'web', 'market', 'crawler', 'calendar', 'trader', 'news-risk', 'jin10-calendar', 'searxng')
+SERVICES = ('api', 'web', 'market', 'crawler', 'calendar', 'trader', 'news-risk', 'jin10-calendar')
 
 # Recognize the unmodified skill templates installed before entrypoint versioning.
 LEGACY_ENTRIES = {
@@ -167,8 +166,6 @@ def deploy_locked(build, directory, maintenance=None):
         print('Skipped: dev has advanced')
         return
     validate_images(build)
-    run('docker', 'pull', SEARXNG_IMAGE)
-    validate_search_image()
     # Only this read-only schema guard receives the existing application environment.
     print(run('docker', 'run', '--rm', '-i', '--network', 'dev_default', '--env-file', directory / '.env',
               IMAGES['backend'] + ':' + build['version'], 'python', '-', input=SCHEMA), flush=True)
@@ -214,14 +211,10 @@ def deploy_locked(build, directory, maintenance=None):
             run(*old_command, 'stop', 'trader')
             run(*old_command, 'stop', *[service for service in previous_running if service != 'trader'])
         (directory / 'compose.yml').write_text((ROOT / 'deploy/docker-compose.yml').read_text())
-        search_config = directory / 'searxng.yml'
-        if not search_config.exists():
-            search_config.write_text((ROOT / 'deploy/searxng.yml').read_text().replace('@SEARXNG_SECRET@', secrets.token_hex(32)))
-            search_config.chmod(0o644)
         command = compose(directory, build)
         run(*command, 'config', '--quiet')
-        run(*command, 'up', '-d', '--no-build', '--pull', 'never', '--wait', '--wait-timeout', '120', 'api', 'web')
-        run(*command, 'up', '-d', '--no-build', '--pull', 'never', 'market', 'crawler', 'calendar', 'trader', 'news-risk', 'jin10-calendar', 'searxng')
+        run(*command, 'up', '-d', '--no-build', '--pull', 'never', '--remove-orphans', '--wait', '--wait-timeout', '120', 'api', 'web')
+        run(*command, 'up', '-d', '--no-build', '--pull', 'never', 'market', 'crawler', 'calendar', 'trader', 'news-risk', 'jin10-calendar')
         time.sleep(5)
         health(command, build)
         if previous:

scripts/smoke_news_batches.py

@@ -0,0 +1,91 @@
+"""Exercise durable batch barriers in the disposable CI database without model calls."""
+import asyncio
+import os
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from urllib.parse import urlparse
+from sqlalchemy import select, delete
+from alphapulse.core.db import SessionLocal, engine
+from alphapulse.models import NewsItem
+from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership
+from alphapulse.news_research import batches
+
+
+async def main():
+    url = urlparse(os.environ['DATABASE_URL'])
+    assert (url.username, url.password, url.path) == ('ci', 'ci-only', '/ci')
+    now = datetime.now(UTC)
+    settings = SimpleNamespace(start_at=now - timedelta(days=1))
+    async with SessionLocal() as s:
+        assert await s.scalar(select(Batch.id).limit(1)) is None
+        def news(key):
+            return NewsItem(source_code='wscn_news', external_id='batch-'+key,
+                title=key, summary='', content_text=key, content_hash='a'*64,
+                url='https://example.invalid/'+key, published_at=now)
+        originals = [news(str(i)) for i in range(3)]
+        s.add_all(originals)
+        await s.commit()
+        batch = await batches.freeze(s, settings)
+        identifier = batch.id
+        assert batch.phase == 'assigning'
+        packages = list(await s.scalars(select(Package).order_by(Package.id)))
+        assert len(packages) == 3
+        later = news('later')
+        s.add(later)
+        await s.commit()
+        actions = await batches.actions(s, batch, set(), 8)
+        assert len(actions) == 3 and all(stage == 'index' for stage, _ in actions)
+        for p in packages:
+            p.state = 'assign_pending'
+        await s.commit()
+        actions = await batches.actions(s, batch, set(), 8)
+        assert len(actions) == 1 and actions[0][0] == 'assign'
+        assert not await batches.actions(s, batch, {actions[0]}, 8)
+        event = Event(title='One shared event', tags=[], revision=3, state='pending', due_at=now)
+        s.add(event)
+        await s.flush()
+        for p in packages[:2]:
+            p.state = 'done'
+            s.add(Membership(event_id=event.id, package_id=p.id, evidence='original', reason='same event', change_kind='new_fact'))
+        packages[2].state = 'index_failed'
+        await s.commit()
+        assert not await batches.actions(s, batch, {('assign', packages[0].id)}, 8)
+        assert batch.phase == 'assigning'
+        assert not await batches.actions(s, batch, set(), 8)
+        await s.commit()
+        assert batch.phase == 'researching'
+        # Reopen the session to prove the admission barrier is durable.
+    async with SessionLocal() as s:
+        batch = await s.get(Batch, identifier)
+        actions = await batches.actions(s, batch, set(), 8)
+        assert actions == [('event', event.id)]
+        await s.commit()
+        assert not await batches.actions(s, batch, set(actions), 8)
+        e = await s.get(Event, event.id)
+        e.state = 'done'
+        await s.commit()
+        assert not await batches.actions(s, batch, set(), 8)
+        await s.commit()
+        assert batch.phase == 'partial'
+        progress = await batches.progress(s)
+        assert progress['packages_failed'] == 1 and progress['events_done'] == 1
+        next_batch = await batches.freeze(s, settings)
+        items = list(await s.scalars(select(BatchPackage).where(BatchPackage.batch_id == next_batch.id)))
+        assert len(items) == 1
+        p = await s.get(Package, items[0].package_id)
+        assert p.news_id == later.id
+        original = await s.get(NewsItem, later.id)
+        original.content_hash = 'b'*64
+        await s.commit()
+        assert not await batches.actions(s, next_batch, set(), 8)
+        await s.commit()
+        assert items[0].state == 'deferred'
+        # Leave no fixtures for other isolated acceptance scripts.
+        for model in (BatchEvent, BatchPackage, Batch, Membership, Package, Event, NewsItem):
+            await s.execute(delete(model))
+        await s.commit()
+    await engine.dispose()
+    print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
+
+
+asyncio.run(main())

scripts/smoke_news_research.py

@@ -229,7 +229,7 @@ async def main():
                     "SELECT count(*) FROM information_schema.tables WHERE table_name LIKE 'news_research_%'"
                 )
             )
-            == 8
+            == 11
         )
     # Exercise actual routes and published-Skill resolution against isolated storage.
     import httpx

scripts/smoke_remove_cls.py

@@ -76,7 +76,7 @@ async def main():
                 assert await connection.scalar(text("SELECT count(*) FROM raw_bodies WHERE sha256=:digest"), {'digest':'a'*64}) == 1
                 assert await connection.scalar(text("SELECT count(*) FROM raw_bodies WHERE sha256=:digest"), {'digest':'b'*64}) == 0
                 assert (await cleanup(connection, True))['counts']['sources'] == 0
-                print('CLS removal: preview, rollback, shared bodies, retained originals, 26 AI tables and repeat execution passed')
+                print('CLS removal: preview, rollback, shared bodies, retained originals, 29 AI tables and repeat execution passed')
             finally:
                 await transaction.rollback()
     finally:

scripts/tests/test_instance.py

@@ -99,10 +99,6 @@ class EntrypointTests(unittest.TestCase):
         self.assertEqual(up[up.index('--env-file') + 1], str(self.directory / 'images.env'))
         self.assertEqual(calls[-1]['args'][0], 'inspect')
 
-    def test_replaced_search_tag_never_starts_any_service(self):
-        self.fixture['images'][instance.SEARCH_IMAGE]['RepoDigests'] = []
-        calls = self.invoke(code=1)
-        self.assertFalse(any('up' in c['args'] for c in calls))
 
     def test_missing_or_mismatched_images_never_start_any_service(self):
         backend = deploy.IMAGES['backend'] + ':' + self.receipt['version']