Add resumable US policy news collectors and provenance

Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled.

Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.

scripts/capture_congress.py

@@ -0,0 +1,179 @@
+"""Capture bounded, resumable Congress acceptance evidence without production I/O."""
+import argparse
+import base64
+import gzip
+import json
+import os
+from pathlib import Path
+import time
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import httpx
+
+CREDENTIAL_FIELDS = {'apikey', 'api_key', 'token', 'key'}
+
+
+def clean_url(url):
+    parts = urlsplit(url)
+    query = [(k, v) for k, v in parse_qsl(parts.query) if k.lower() not in CREDENTIAL_FIELDS]
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def redact(value):
+    if isinstance(value, dict):
+        return {k: redact(v) for k, v in value.items() if k.lower() not in CREDENTIAL_FIELDS}
+    if isinstance(value, list):
+        return [redact(v) for v in value]
+    if isinstance(value, str) and value.startswith('https://'):
+        return clean_url(value)
+    return value
+
+
+def save(path, data):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + '.tmp')
+    with gzip.open(temporary, 'wt') as file:
+        json.dump(data, file)
+    temporary.replace(path)
+
+
+class CaptureBlocked(RuntimeError):
+    pass
+
+
+class Capture:
+    def __init__(self, path, key, limit=60, client=None):
+        self.path, self.key, self.limit = path, key, limit
+        self.client = client or httpx.Client(timeout=25, follow_redirects=False)
+        self.records = json.load(gzip.open(path, 'rt')) if path.exists() else {}
+        self.requests = 0
+        self.used_urls = set()
+
+    def fetch(self, url):
+        url = clean_url(url)
+        from alphapulse.core.us_news import official_url
+        official_url('congress', url)
+        self.used_urls.add(url)
+        if url in self.records:
+            return base64.b64decode(self.records[url]['body'])
+        if self.requests >= self.limit:
+            raise CaptureBlocked('本轮请求预算耗尽；成功响应已保存，下次可恢复')
+        self.requests += 1
+        headers = {'X-Api-Key': self.key} if urlsplit(url).hostname == 'api.congress.gov' else {}
+        response = self.client.get(url, headers=headers)
+        if response.status_code == 429:
+            raise CaptureBlocked(f"HTTP 429；Retry-After={response.headers.get('retry-after', 'unknown')}；已停止请求")
+        if response.status_code != 200:
+            raise CaptureBlocked(f'HTTP {response.status_code}；未保存错误响应')
+        body = response.content
+        if 'json' in response.headers.get('content-type', ''):
+            body = json.dumps(redact(response.json())).encode()
+        self.records[url] = {'body': base64.b64encode(body).decode()}
+        save(self.path, self.records)
+        time.sleep(1)
+        return body
+
+
+async def run(capture, item_sink=None, meeting_url=None):
+    # Override every application I/O setting before importing the parser modules.
+    os.environ.update(DATABASE_URL='postgresql+asyncpg://test:[email protected]:1/test', REDIS_URL='redis://127.0.0.1:1/0', JWT_SECRET='test-only', ADMIN_USERNAME='test', SECRET_ENCRYPTION_KEY='MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=')
+    from datetime import UTC, datetime
+    from types import SimpleNamespace
+    from unittest.mock import patch
+    import uuid
+    from scrapy import Request
+    from scrapy.http import Response
+    from alphapulse.crawler.spiders import us_official
+    from alphapulse.crawler.spiders.base import RunState
+    from alphapulse.core.us_news import CRYPTO
+    report = {'accepted': False, 'cases': [], 'requests': 0}
+    roots = [('GENIUS', 'bill/119/s/1582'), ('CLARITY', 'bill/119/hr/3633'), ('STABLE', 'bill/119/hr/2392'), ('meeting', 'committee-meeting/119')]
+    with patch.object(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key=capture.key, crawler_contact_email='[email protected]')):
+        for label, route in roots:
+            obj = us_official.USOfficialSpider(name='congress')
+            now = datetime.now(UTC)
+            obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025,1,1,tzinfo=UTC), history_deadline=float('inf'))
+            url = f'https://api.congress.gov/v3/{route}?format=json&limit=250'
+            obj.cursor = {'history': {'entry': 0, 'pending': []}}
+            case = {'name': label, 'items': 0, 'article_items': 0, 'partial_items': 0, 'complete': False}
+            report['cases'].append(case)
+            try:
+                if label == 'meeting' and meeting_url:
+                    from alphapulse.core.us_news import official_url
+                    selected = official_url('congress', meeting_url)
+                    if urlsplit(selected).hostname != 'api.congress.gov' or '/committee-meeting/119/' not in urlsplit(selected).path:
+                        raise CaptureBlocked('会议地址必须是119届Congress API单场会议地址')
+                    obj.cursor['history']['pending'] = [{'url': us_official.configure_url(selected, format='json'), 'kind': 'meeting'}]
+                elif label == 'meeting':
+                    listing = json.loads(capture.fetch(url))
+                    obj.cursor['history']['pending'] = [{'url': us_official.configure_url(entry['url'], format='json'), 'kind': 'meeting'} for entry in listing['committeeMeetings']]
+                    if not obj.cursor['history']['pending']:
+                        raise CaptureBlocked('会议列表为空，不能认定验收完成')
+                else:
+                    obj.cursor['history']['pending'] = [{'url':url, 'kind':'bill'}]
+                request = None
+                while obj.cursor['history']['pending']:
+                    if request is None:
+                        request = obj.advance('history')[0]
+                    body = capture.fetch(request.url)
+                    if label == 'meeting' and request.callback == obj.parse_document:
+                        meeting = json.loads(body)['committeeMeeting']
+                        if not CRYPTO.search(meeting['title'] + json.dumps(meeting.get('relatedItems', {}))):
+                            obj.cursor['history']['pending'].pop(0)
+                            request = None
+                            continue
+                    request.meta['raw_response_id'] = None
+                    response = Response(request.url, body=body, request=request)
+                    if request.callback == obj.parse_attachment:
+                        output = list(obj.parse_attachment(response))
+                    else:
+                        from scrapy.http import HtmlResponse
+                        output = [v async for v in obj.parse_document(HtmlResponse(response.url, body=body, encoding='utf-8', request=request))]
+                    requests = [v for v in output if isinstance(v, Request) and v.meta.get('document')]
+                    # Attachment requests hold parsed state rather than a document.
+                    requests += [v for v in output if isinstance(v, Request) and v.meta.get('parsed')]
+                    for item in (v for v in output if not isinstance(v, Request)):
+                        if item_sink is not None:
+                            item_sink.append(dict(item))
+                        case['items'] += 1
+                        case['article_items'] += int(item['extra']['content_scope'] == 'article')
+                        case['partial_items'] += int(item['extra']['content_scope'] == 'partial')
+                    if label == 'meeting' and case['article_items']:
+                        break
+                    request = requests[0] if requests else None
+                    if request is None and not obj.cursor['history']['pending']:
+                        break
+                case['traversal_complete'] = not obj.cursor['history']['pending'] if label != 'meeting' else bool(case['article_items'] + case['partial_items'])
+                case['parser_accepted'] = case['traversal_complete'] and bool(case['article_items'] + case['partial_items'])
+                case['unresolved_bills'] = obj.run.stats.get('unresolved_bills', 0)
+                case['complete'] = bool(case['article_items']) and not case['unresolved_bills'] and not obj.cursor.get('attachment_retries')
+            except Exception as exc:
+                # Error text may contain a credential-bearing upstream URL.
+                case['error'] = str(exc) if isinstance(exc, CaptureBlocked) else type(exc).__name__
+                break
+    report['requests'] = capture.requests
+    report['accepted'] = len(report['cases']) == len(roots) and all(c['complete'] for c in report['cases'])
+    return report
+
+
+def main():
+    import asyncio
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--cache', type=Path, required=True)
+    parser.add_argument('--meeting-url', help='Official Congress API URL for a known crypto hearing')
+    parser.add_argument('--max-requests', type=int, default=60)
+    args = parser.parse_args()
+    if not 1 <= args.max_requests <= 100:
+        parser.error('--max-requests must be between 1 and 100')
+    capture = Capture(args.cache, os.environ.get('CONGRESS_TEST_API_KEY') or 'DEMO_KEY', args.max_requests)
+    try:
+        report = asyncio.run(run(capture, meeting_url=args.meeting_url))
+        save(args.cache.with_suffix('.report.gz'), report)
+        print(json.dumps(report, ensure_ascii=False))
+    finally:
+        capture.client.close()
+    raise SystemExit(0 if report['accepted'] else 1)
+
+
+if __name__ == '__main__':
+    main()

scripts/smoke_us_news.py

@@ -0,0 +1,242 @@
+"""Validate source migration and real public-document idempotency in an isolated database."""
+import asyncio
+import copy
+import gzip
+import json
+import os
+from pathlib import Path
+import runpy
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+from urllib.parse import urlsplit
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import func, select, text
+
+from alphapulse.core.db import SessionLocal, engine
+from alphapulse.core.us_news import SOURCES, article, identity
+from alphapulse.crawler.spiders.base import RunState, html_to_text
+from alphapulse.crawler.spiders.us_official import USOfficialSpider
+from alphapulse.crawler import pipelines
+from alphapulse.models import CrawlRun, NewsItem, NewsRevision, RawBody, RawResponse, Source
+
+ROOT = Path(__file__).resolve().parents[1]
+BACKEND_ROOT = Path(pipelines.__file__).resolve().parents[3]
+
+
+async def public_api_items():
+    """Replay captured public API bodies and PDFs without network or database I/O."""
+    import base64
+    import time
+    import uuid
+    from datetime import UTC, datetime
+    from scrapy import Request
+    from scrapy.http import HtmlResponse, Response
+    from alphapulse.crawler.spiders.base import RunState
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/api_samples.json.gz', 'rt'))
+    pdfs = {s['url']: base64.b64decode(s['base64']) for s in samples if s['kind'] == 'pdf'}
+    items = []
+    for sample in (s for s in samples if s['kind'] != 'pdf'):
+        obj = USOfficialSpider(name=sample['source'])
+        now = datetime.now(UTC)
+        obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025, 1, 1, tzinfo=UTC), history_deadline=time.monotonic()-1)
+        doc = {k: sample[k] for k in ('url', 'kind')}
+        if doc['kind'] == 'register_issue':
+            doc.update(day='2025-07-18', count=106)
+        obj.cursor = {'history': {'entry': 0, 'pending': [doc], 'day': '2025-07-18'}}
+        req = Request(doc['url'], meta={'lane': 'history', 'document': doc, 'raw_response_id': None})
+        output = [v async for v in obj.parse_document(HtmlResponse(req.url, body=sample['body'].encode(), encoding='utf-8', request=req))]
+        if doc['kind'] == 'register_issue':
+            assert len(obj.cursor['history']['pending']) == 106
+            continue
+        if sample.get('excluded'):
+            assert not output
+            continue
+        while output and isinstance(output[0], Request):
+            req = output[0]
+            req.meta['raw_response_id'] = None
+            output = list(obj.parse_attachment(Response(req.url, body=pdfs[req.url], request=req)))
+        assert len(output) == 1 and output[0]['extra']['content_scope'] == 'article'
+        assert output[0]['extra']['attachments']
+        items += output
+    assert len(items) == 3
+    return items
+
+
+async def congress_reference_items():
+    """Cross-check public GovInfo XML through the text adapter, not the API adapter."""
+    import time
+    import uuid
+    from datetime import UTC, datetime
+    from lxml import etree
+    from scrapy import Request
+    from scrapy.http import HtmlResponse
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/congress_billstatus.json.gz', 'rt'))
+    items = []
+    assert len(samples) == 3
+    for sample in samples:
+        root = etree.fromstring(sample['body'].encode(), etree.XMLParser(resolve_entities=False, no_network=True))
+        versions = root.findall('.//textVersions/item')
+        if sample['bill'] == 's1582':
+            assert any(v.findtext('type') == 'Enrolled Bill' and not v.findtext('date') for v in versions)
+        obj = USOfficialSpider(name='congress')
+        now = datetime.now(UTC)
+        obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025,1,1,tzinfo=UTC), history_deadline=time.monotonic()-1)
+        text = sample['text']
+        doc = {'url': text['url'], 'kind': 'text', 'title': sample['title'], 'at': text['date']}
+        obj.cursor = {'history': {'entry': 0, 'pending': [doc]}}
+        req = Request(doc['url'], meta={'document': doc, 'lane': 'history', 'raw_response_id': None})
+        output = [v async for v in obj.parse_document(HtmlResponse(req.url, body=text['body'].encode(), encoding='utf-8', request=req))]
+        assert len(output) == 1 and output[0]['extra']['selection_reasons']
+        items += output
+    return items
+
+
+async def congress_api_items():
+    """Replay captured Congress API evidence with zero network requests."""
+    namespace = runpy.run_path(str(ROOT / 'scripts/capture_congress.py'))
+    cache = BACKEND_ROOT / 'tests/fixtures/us_news/congress_api.json.gz'
+    capture = namespace['Capture'](cache, 'offline-test-only', limit=0)
+    items = []
+    try:
+        report = await namespace['run'](capture, item_sink=items, meeting_url='https://api.congress.gov/v3/committee-meeting/119/house/119541')
+    finally:
+        capture.client.close()
+    assert capture.requests == 0
+    assert len(report['cases']) == 4
+    assert all(case.get('parser_accepted') for case in report['cases'])
+    assert report['cases'][0]['unresolved_bills'] == 1
+    assert report['accepted'] is False  # Undated GENIUS text and an unreadable meeting PDF remain gaps.
+    assert len(items) == sum(case['items'] for case in report['cases'])
+    return items
+
+
+async def main():
+    url = urlsplit(os.environ['DATABASE_URL'])
+    assert url.username == 'ci' and url.password == 'ci-only' and url.path == '/ci'
+    assert url.hostname == '127.0.0.1'
+    tables = [model.__table__ for model in (Source, CrawlRun, RawBody, RawResponse, NewsItem, NewsRevision)]
+    async with engine.begin() as connection:
+        await connection.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
+        await connection.run_sync(lambda conn: Source.metadata.create_all(conn, tables=tables))
+        for table, column in [('sources', 'history_cursor'), ('crawl_runs', 'stats'), ('news_revisions', 'extra')]:
+            await connection.execute(text(f'ALTER TABLE {table} DROP COLUMN {column}'))
+        namespace = runpy.run_path(str(BACKEND_ROOT / 'alembic/versions/20261104_da1234567890_us_official_news.py'))
+        def migrate(conn):
+            with Operations.context(MigrationContext.configure(conn)):
+                namespace['upgrade']()
+        await connection.run_sync(migrate)
+    async with SessionLocal() as session:
+        sources = list(await session.scalars(select(Source)))
+        assert len(sources) == 10 and all(not s.enabled for s in sources)
+        session.add(Source(code='fed', name='Fed', list_url=SOURCES['fed'].entries[0], enabled=False, interval_seconds=60))
+        await session.commit()
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/public_samples.json.gz', 'rt'))
+    assert len(samples) == len({s['url'] for s in samples}) == 50
+    spider = SimpleNamespace(run=SimpleNamespace(items_seen=0, items_new=0, items_revised=0))
+    pipeline = pipelines.NewsPipeline(SimpleNamespace(spider=spider))
+    async def ingest(sample):
+        data = article(sample['source'], sample['body'], sample['url'])
+        return await pipeline.process_item(dict(source_code=sample['source'], external_id=identity(sample['url']), url=sample['url'], title=data['title'], summary='', content_html=data['content_html'], content_text=html_to_text(data['content_html']), author=None, tags=[], is_paywalled=False, published_at=data['published_at'], source_updated_at=None, extra=data['extra'], raw_response_id=None))
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        for sample in samples:
+            await ingest(sample)
+        for sample in samples:
+            await ingest(sample)
+        assert spider.run.items_new == 50 and spider.run.items_revised == 0
+        sample = samples[0]
+        changed = {**sample, 'body': sample['body'].replace('</h1>', ' Updated</h1>', 1)}
+        await ingest(changed)
+        assert spider.run.items_revised == 1
+    async with SessionLocal() as session:
+        assert await session.scalar(select(func.count()).select_from(NewsItem)) == 50
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == 51
+        assert await session.scalar(select(func.count()).select_from(NewsRevision).where(NewsRevision.extra.is_not(None))) == 51
+    # Preserve the legacy RSS identity while enriching from the same public URL.
+    from datetime import UTC, datetime, timedelta
+    import time
+    import uuid
+    from scrapy import Request
+    from scrapy.http import HtmlResponse
+    sample = next(s for s in samples if s['source'] == 'fed')
+    async with SessionLocal() as session:
+        news = await session.scalar(select(NewsItem).where(NewsItem.url == sample['url']))
+        news.external_id = 'legacy-rss-guid'
+        baseline = (news.content_hash, news.revision_no)
+        await session.commit()
+    obj = USOfficialSpider(name='fed')
+    now = datetime.now(UTC)
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now-timedelta(hours=72), history_target=datetime(2025, 1, 1, tzinfo=UTC), history_position=now, history_deadline=time.monotonic()-1)
+    obj.cursor = {'history': {'entry': 0, 'pending': []}}
+    captured = []
+    obj.attach_or_finish = lambda parsed, lane: captured.append(parsed) or []
+    doc = {'url': sample['url'], 'kind': 'article'}
+    request = Request(sample['url'], meta={'document': doc, 'lane': 'history', 'raw_response_id': None})
+    valid = HtmlResponse(sample['url'], body=sample['body'].encode(), encoding='utf-8', request=request)
+    assert not [v async for v in obj.parse_document(valid)]
+    assert captured[0]['external_id'] == 'legacy-rss-guid'
+    captured.clear()
+    invalid = HtmlResponse(sample['url'], body=b'<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>', encoding='utf-8', request=request)
+    try:
+        async for item in obj.parse_document(invalid):
+            await pipeline.process_item(item)
+    except ValueError:
+        pass
+    else:
+        raise AssertionError('Challenge must fail before the pipeline')
+    async with SessionLocal() as session:
+        news = await session.scalar(select(NewsItem).where(NewsItem.external_id == 'legacy-rss-guid'))
+        assert (news.content_hash, news.revision_no) == baseline
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == 51
+    # A failed checkpoint commit must recover before a new round can read it.
+    async with SessionLocal() as session:
+        source = await session.get(Source, 'ofac')
+        source.enabled = True
+        await session.commit()
+    obj = USOfficialSpider(name='ofac')
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now-timedelta(hours=72), official_cursor={'checkpoint_probe': 'committed'})
+    obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=lambda request: None))
+    finish = obj._finish_run
+    attempts = []
+    async def transient_failure(run):
+        attempts.append(run.id)
+        if len(attempts) == 1:
+            raise RuntimeError('isolated checkpoint failure')
+        await finish(run)
+    obj._finish_run = transient_failure
+    original_id = obj.run.id
+    obj._close_run()
+    await asyncio.gather(obj.finish_task, return_exceptions=True)
+    await obj.start_round('schedule')
+    await asyncio.gather(*obj._tasks)
+    assert attempts == [original_id, original_id] and obj.run is not None
+    assert obj.cursor['checkpoint_probe'] == 'committed'
+    async with SessionLocal() as session:
+        saved = await session.get(CrawlRun, original_id)
+        assert saved.status == 'success'
+        assert (await session.get(Source, 'ofac')).history_cursor['checkpoint_probe'] == 'committed'
+    # API/PDF packages use the real parser and database revision pipeline too.
+    api_items = await public_api_items() + await congress_reference_items() + await congress_api_items()
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        for item in api_items:
+            await pipeline.process_item(dict(item))
+        async with SessionLocal() as session:
+            baseline_count = await session.scalar(select(func.count()).select_from(NewsRevision))
+        for item in api_items:
+            await pipeline.process_item(dict(item))
+        async with SessionLocal() as session:
+            assert await session.scalar(select(func.count()).select_from(NewsItem)) == 50 + len(api_items)
+            assert await session.scalar(select(func.count()).select_from(NewsRevision)) == baseline_count
+    revised = copy.deepcopy(api_items[0])
+    revised['title'] += ' Updated'
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        await pipeline.process_item(dict(revised))
+    async with SessionLocal() as session:
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == baseline_count + 1
+    await engine.dispose()
+    print('US source migration, HTML, API/PDF, GovInfo and Congress packages, API/PDF replay and revision, negative screening, daily XML, RSS identity, challenge rejection and checkpoint recovery passed')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())