Add resumable US policy news collectors and provenance

Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled.

Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.

backend/tests/test_congress_capture.py

@@ -0,0 +1,93 @@
+import asyncio
+import base64
+import gzip
+import json
+from pathlib import Path
+import runpy
+
+import httpx
+import pytest
+
+
+SCRIPT = Path(__file__).resolve().parents[2] / 'scripts/capture_congress.py'
+
+
+def test_capture_redacts_and_reuses_successful_response(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    seen = []
+    def handle(request):
+        seen.append(request)
+        return httpx.Response(200, json={'request': {'apiKey': 'private-test-secret'}, 'pagination': {'next': 'https://api.congress.gov/v3/bill?page=2&api_key=private-test-secret'}})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        url = 'https://api.congress.gov/v3/bill?format=json'
+        first = capture.fetch(url)
+        assert capture.fetch(url) == first and len(seen) == 1
+        loaded = json.load(gzip.open(path, 'rt'))
+        assert 'private-test-secret' not in base64.b64decode(loaded[url]['body']).decode()
+        assert seen[0].headers['X-Api-Key'] == 'private-test-secret'
+        capture.fetch('https://www.govinfo.gov/test.pdf')
+        assert 'X-Api-Key' not in seen[-1].headers
+
+
+def test_capture_stops_after_rate_limit_and_reports_incomplete(trading, tmp_path):
+    module = runpy.run_path(str(SCRIPT))
+    calls = []
+    def handle(request):
+        calls.append(request)
+        return httpx.Response(429, headers={'Retry-After': '123'})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        report = asyncio.run(module['run'](capture))
+    assert len(calls) == 1 and not path.exists()
+    assert report['accepted'] is False
+    assert len(report['cases']) == 1
+    assert '123' in report['cases'][0]['error']
+    assert 'private-test-secret' not in json.dumps(report)
+
+
+def test_capture_keeps_progress_on_budget_exhaustion(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b'body'))) as client:
+        capture = module['Capture'](path, 'test', limit=1, client=client)
+        capture.fetch('https://www.govinfo.gov/first.pdf')
+        with pytest.raises(module['CaptureBlocked'], match='预算'):
+            capture.fetch('https://www.govinfo.gov/second.pdf')
+        resumed = module['Capture'](path, 'test', limit=1, client=client)
+        assert resumed.fetch('https://www.govinfo.gov/first.pdf') == b'body'
+        assert resumed.requests == 0
+        assert resumed.fetch('https://www.govinfo.gov/second.pdf') == b'body'
+
+
+def test_capture_replays_all_case_callbacks_without_external_io(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    def handle(request):
+        path = request.url.path
+        if request.url.host == 'www.congress.gov':
+            return httpx.Response(200, text='<p>The law establishes regulatory requirements for stablecoins and digital assets.</p>')
+        if path == '/v3/committee-meeting/119':
+            data = {'committeeMeetings': [{'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1'}]}
+        elif '/committee-meeting/' in path:
+            data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z'}}
+        elif path.endswith('/text'):
+            data = {'textVersions': [{'date': '2025-01-01', 'type': 'Introduced', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov' + path + '.html'}]}]}
+        elif path.endswith('/actions'):
+            data = {'actions': [{'actionDate': '2025-01-01', 'text': 'Passed'}]}
+        else:
+            data = {'bill': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01'}}
+        return httpx.Response(200, json=data)
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](tmp_path / 'cache.gz', 'test', client=client)
+        report = asyncio.run(module['run'](capture))
+        assert report['accepted'] is True
+        assert len(report['cases']) == 4
+        assert sum(case['items'] for case in report['cases']) == 7
+        resumed = module['Capture'](capture.path, 'test', client=client)
+        assert asyncio.run(module['run'](resumed))['accepted'] is True
+        assert resumed.requests == 0

backend/tests/test_news_sources.py

@@ -33,7 +33,7 @@ ITEM = """<item><title>Bitcoin &amp; Ethereum</title>
 <dc:creator>News desk</dc:creator><category>Markets</category></item>"""
 
 
-@pytest.mark.parametrize("source", ["CoinDeskSpider", "FedSpider", "BeaSpider"])
+@pytest.mark.parametrize("source", ["CoinDeskSpider", "BeaSpider"])
 def test_rss_preserves_original_summary_time_and_provenance(trading, source):
     from alphapulse.crawler.spiders import rss
     spider = make_spider(getattr(rss, source))
@@ -53,8 +53,8 @@ def test_rss_preserves_original_summary_time_and_provenance(trading, source):
 
 
 def test_rss_identity_survives_revisions_and_different_subfeeds(trading):
-    from alphapulse.crawler.spiders.rss import FedSpider
-    spider = make_spider(FedSpider)
+    from alphapulse.crawler.spiders.rss import CoinDeskSpider
+    spider = make_spider(CoinDeskSpider)
     original, = spider.parse_feed(feed_response(ITEM))
     revised, = spider.parse_feed(feed_response(ITEM.replace("Bitcoin &amp; Ethereum", "Revised headline"), "https://example.invalid/other.xml"))
     assert original["external_id"] == revised["external_id"]

backend/tests/test_us_official_news.py

@@ -0,0 +1,659 @@
+import asyncio
+import copy
+import json
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+
+import pytest
+from scrapy import Request
+from scrapy.http import HtmlResponse
+
+from alphapulse.core.us_news import SOURCES, article, official_url, published, selection
+
+
+def spider(trading, source='ofac'):
+    from alphapulse.crawler.spiders.base import RunState
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    obj = USOfficialSpider(name=source)
+    obj.name = source
+    now = datetime(2026, 11, 4, tzinfo=UTC)
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now - timedelta(hours=72))
+    obj.run.history_target = datetime(2025, 1, 1, tzinfo=UTC)
+    obj.run.history_deadline = time.monotonic() + 20
+    obj.cursor = {'history': {'entry': 0, 'pending': []}, 'realtime': {'entry': 0, 'pending': []}}
+    obj.lanes = {}
+    return obj
+
+
+def response(url, body, **meta):
+    return HtmlResponse(url, body=body.encode(), encoding='utf-8', request=Request(url, meta={'raw_response_id': 12, **meta}))
+
+
+@pytest.mark.parametrize('source,title,text,expected', [
+    ('doj', 'Man sentenced for bitcoin theft', 'Personal bitcoin theft worth $300', False),
+    ('doj', 'Exchange charged', 'Cryptocurrency exchange charged with money laundering', True),
+    ('fincen', 'Digital asset rule', 'FinCEN withdraws proposed digital asset rule', True),
+    ('whitehouse', 'Tariffs', 'President adjusts import tariffs', True),
+    ('treasury', 'Meeting with delegation', 'Greetings and cooperation', False),
+    ('ofac', 'New sanctions', 'Sanctions against a financial institution', True),
+    ('sec', 'Ethernet equipment', 'Method and hardware', False),
+])
+def test_selection_boundaries(source, title, text, expected):
+    assert bool(selection(source, title, text)) is expected
+
+
+@pytest.mark.parametrize('url', ['http://ofac.treasury.gov/recent-actions/20261104', 'https://evil.example/a.pdf', 'https://[email protected]/a', 'https://ofac.treasury.gov/a?api_key=secret'])
+def test_official_links_reject_external_and_credentials(url):
+    with pytest.raises(ValueError):
+        official_url('ofac', url)
+
+
+def test_dates_do_not_fabricate_publisher_time():
+    assert published('2025-01-01') == (datetime(2025, 1, 1, tzinfo=UTC), 'date')
+    assert published('2026-11-01T01:30:00-04:00')[0] == datetime(2026, 11, 1, 5, 30, tzinfo=UTC)
+    with pytest.raises(ValueError):
+        published('yesterday')
+
+
+def test_ofac_original_body_and_attachment_boundaries():
+    data = article('ofac', '''<h1>Crypto sanctions</h1><main><div class="field--name-body"><p>New sanctions against a virtual currency exchange institution.</p>
+    <a href="/media/123/download">License</a><a href="https://evil.example/a.pdf">external</a></div></main>''', 'https://ofac.treasury.gov/recent-actions/20250101')
+    assert data['published_at'] == datetime(2025, 1, 1, tzinfo=UTC)
+    assert data['extra']['published_at_precision'] == 'date'
+    assert len(data['extra']['attachments']) == 2
+    assert data['extra']['attachments'][1]['status'] == 'unsupported_host'
+
+
+def test_challenge_is_failure_not_empty_success(trading):
+    obj = spider(trading)
+    with pytest.raises(ValueError, match='未识别'):
+        list(obj.parse_listing(response(SOURCES['ofac'].entries[0], '<html>Please wait</html>', lane='history')))
+
+
+def test_history_pending_documents_survive_deadline_and_resume(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.run.history_deadline = time.monotonic() - 1
+    assert obj.advance('history') == []
+    saved = copy.deepcopy(obj.cursor)
+    resumed = spider(trading)
+    resumed.cursor = saved
+    request, = resumed.advance('history')
+    assert request.url == doc['url'] and resumed.cursor['history']['pending'] == [doc]
+    assert not resumed.run.history_done
+
+
+@pytest.mark.parametrize('count', [1999, 2000, 2001])
+def test_register_oversized_day_uses_xml(trading, count):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-01'
+    req, = obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': count, 'results': [{'document_number': '2025-1'}]}), lane='history'))
+    assert ('govinfo.gov' in req.url) is (count > 2000)
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_keys_only_sent_to_congress_api(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test-key', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    assert obj.request_for('https://api.congress.gov/v3/bill', lambda x: x, 'history').headers['X-Api-Key'] == b'test-key'
+    assert 'X-Api-Key' not in obj.request_for('https://www.congress.gov/text', lambda x: x, 'history').headers
+
+
+def test_register_body_and_pdf_are_part_of_the_same_package(trading):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/api/v1/documents/2025-1.json', 'kind': 'register'}
+    obj.cursor['history']['pending'] = [doc]
+    res = response(doc['url'], json.dumps({'title': 'Digital asset rule', 'publication_date': '2025-01-01', 'document_number': '2025-1', 'body_html_url': 'https://www.federalregister.gov/documents/full_text/html/2025/01/01/2025-1.html', 'html_url': 'https://www.federalregister.gov/documents/2025/01/01/2025-1/a', 'pdf_url': 'https://www.govinfo.gov/a.pdf'}), lane='history', document=doc)
+    async def collect():
+        return [v async for v in obj.parse_document(res)]
+    request, = asyncio.run(collect())
+    assert request.meta['document']['pdf'] == 'https://www.govinfo.gov/a.pdf'
+
+
+def test_summary_scope_tests_do_not_hide_fed_enrichment(trading):
+    from alphapulse.crawler.spiders.rss import FedSpider
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    assert issubclass(FedSpider, USOfficialSpider)
+    assert len(SOURCES['fed'].entries) >= 8
+
+
+def test_fifty_real_public_documents_parse_without_io():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    assert len(samples) == 50
+    assert len({s['source'] for s in samples}) == 8
+    for sample in samples:
+        parsed = article(sample['source'], sample['body'], sample['url'])
+        assert parsed['title'] == sample['expected_title']
+        assert len(parsed['content_html']) > 40
+        assert parsed['published_at'].year >= 2025
+        assert parsed['extra']['published_at_precision'] in ('date', 'datetime')
+
+
+def test_daily_register_partitions_resume_without_marking_done(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-02'
+    requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': 0, 'results': []}), lane='history')))
+    assert len(requests) == 1
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_h41_uses_publisher_release_dates_not_assumed_thursdays(trading):
+    obj = spider(trading, 'fed')
+    data = [{'yearValue': '2025', 'Months': [{'MonthName': 'January', 'Dates': ['20250103', '20250109']}]}]
+    req, = obj.parse_listing(response('https://www.federalreserve.gov/releases/h41/releaseDates.json', json.dumps(data), lane='history'))
+    assert req.url == 'https://www.federalreserve.gov/releases/h41/20250109/'
+    assert obj.cursor['history']['pending'][1]['url'].endswith('20250103/')
+
+
+def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
+    obj = spider(trading)
+    obj.run.history_deadline = time.monotonic() - 1
+    url = 'https://ofac.treasury.gov/recent-actions/20250101'
+    doc = {'url': url, 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    parsed = {'url': url, 'title': 'Crypto sanctions', 'external_id': 'x', 'content_html': '<p>Virtual currency exchange sanctions.</p>', 'published_at': datetime(2025, 1, 1, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
+              'extra': {'attachments': [{'url': 'https://ofac.treasury.gov/a.pdf', 'title': 'License', 'status': 'unavailable'}]}}
+    item, = obj.attach_or_finish(parsed, 'history')
+    assert item['extra']['content_scope'] == 'partial'
+    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
+    assert obj.cursor['attachment_retries'][url]['document'] == doc
+
+
+@pytest.mark.parametrize('canceled,failed', [(False, False), (True, False), (False, True)])
+def test_checkpoint_does_not_advance_failed_or_canceled_history(trading, monkeypatch, canceled, failed):
+    from alphapulse.crawler.spiders import base
+    now = datetime.now(UTC)
+    target = now - timedelta(days=300)
+    source = SimpleNamespace(history_target=None if canceled else target, history_cursor={'history': {'entry': 0}}, consecutive_failures=0)
+    class Session:
+        async def __aenter__(self):
+            return self
+        async def __aexit__(self, *args):
+            pass
+        async def execute(self, *args):
+            pass
+        async def get(self, *args, **kwargs):
+            return source
+        async def commit(self):
+            pass
+    monkeypatch.setattr(base, 'SessionLocal', Session)
+    obj = spider(trading)
+    run = base.RunState(uuid.uuid4(), 'schedule', now, target, history_target=target, history_position=now, official_cursor={'history': {'entry': 5}, 'realtime': {'entry': 2}})
+    if failed:
+        run.errors.append('write failed')
+    asyncio.run(obj._finish_run(run))
+    assert source.history_cursor['history']['entry'] == (0 if canceled or failed else 5)
+
+
+def test_congress_action_links_use_public_bill_type(trading):
+    from alphapulse.crawler.spiders.us_official import congress_bill_url
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/hr/1') == 'https://www.congress.gov/bill/119th-congress/house-bill/1'
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/s/1582') == 'https://www.congress.gov/bill/119th-congress/senate-bill/1582'
+
+
+def test_tick_during_active_round_does_not_replace_its_cursors(trading):
+    obj = spider(trading)
+    obj.crawler = SimpleNamespace(engine=SimpleNamespace(spider_is_idle=lambda: False))
+    before = copy.deepcopy(obj.cursor)
+    assert obj.make_request_from_data(json.dumps({'trigger': 'schedule', 'official_cursor': {'history': {'entry': 99}}}).encode()) == []
+    assert obj.cursor == before
+
+
+def test_realtime_keeps_next_page(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20261104">Crypto</a><a rel="next" href="?page=1">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] == url + '?page=1'
+
+
+@pytest.mark.parametrize('body', ['', '<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>'])
+def test_text_challenge_never_emits_or_advances(trading, body):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    with pytest.raises(ValueError):
+        asyncio.run(collect())
+    assert obj.cursor['history']['pending'] == [doc]
+
+
+def test_genius_bill_requires_evidence_before_exclusion(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/bill/119/s/1582', 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'bill': {'title': 'GENIUS Act', 'policyArea': {'name': 'Finance and Financial Sector'}}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert '/subjects?' in req.url
+    assert not obj.run.stats.get('excluded')
+
+
+def test_real_titles_are_article_headlines():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    for s in samples:
+        if s['source'] == 'fed' and '2026-press.htm' in s['url']:
+            continue
+        title = article(s['source'], s['body'], s['url'])['title']
+        assert title not in ('Board of Governors of the Federal Reserve System', 'U.S. Department of the Treasury')
+        assert not title.startswith('Release Number ')
+
+
+@pytest.mark.parametrize('enabled,history_enabled,save_failed', [(True, True, False), (False, True, False), (True, False, False), (True, True, True)])
+def test_round_waits_for_commit_and_reads_fresh_state(trading, monkeypatch, enabled, history_enabled, save_failed):
+    from alphapulse.crawler.spiders import us_official
+    fresh = {'history': {'entry': 0, 'target': '2025-01-01T00:00:00+00:00', 'pending': [{'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}]}}
+    source = SimpleNamespace(enabled=enabled, paused_until=None, history_cursor=fresh, history_target=datetime(2025, 1, 1, tzinfo=UTC) if history_enabled else None, history_done=False, history_position=None, backfill_hours=72)
+    reads = []
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args):
+            reads.append(True)
+            return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def run():
+        obj = spider(trading)
+        obj.run = None
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=reads.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        saved = asyncio.Event()
+        async def finish():
+            await saved.wait()
+            if save_failed: raise RuntimeError('test write failure')
+        obj.finish_task = asyncio.create_task(finish())
+        boot = asyncio.create_task(obj.start_round('schedule'))
+        await asyncio.sleep(0)
+        assert not reads
+        saved.set()
+        await boot
+        requests = [r for r in reads if isinstance(r, Request)]
+        if enabled and not save_failed:
+            assert bool([r for r in requests if r.meta['lane'] == 'history']) is history_enabled
+            if history_enabled:
+                assert requests[-1].url.endswith('20250101')
+        else:
+            assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(run())
+
+
+def test_register_realtime_finishes_all_pages_before_changing_day(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['realtime']['day'] = '2026-11-04'
+    url = SOURCES[obj.name].entries[0]
+    list(obj.parse_listing(response(url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1234'}], 'next_page_url': url + '?page=2'}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    req, = obj.advance('realtime')
+    assert req.url.endswith('?page=2')
+    list(obj.parse_listing(response(req.url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1235'}]}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    obj.advance('realtime')
+    assert obj.cursor['realtime']['day'] == '2026-11-03'
+
+
+def test_pagination_cycle_does_not_claim_success(trading):
+    obj = spider(trading)
+    url = SOURCES[obj.name].entries[0]
+    obj.cursor['history']['visited_pages'] = [url]
+    with pytest.raises(ValueError, match='循环'):
+        list(obj.parse_listing(response(url + '?page=2', f'<h1>News</h1><a href="/recent-actions/20261104">News</a><a rel="next" href="{url}">Next</a>', lane='history')))
+    assert not obj.run.history_done
+
+
+def test_fragment_is_valid_without_page_heading(trading):
+    obj = spider(trading, 'federal_register')
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>This digital asset regulation establishes requirements for stablecoin issuers.</p>', lane='history', document=doc))]
+    item, = asyncio.run(collect())
+    assert item['extra']['content_scope'] == 'article'
+
+
+def test_bill_full_text_can_supply_relevance_and_resume(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(body):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    asyncio.run(parse(json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': '2026-11-04'}})))
+    asyncio.run(parse(json.dumps({'subjects': {'legislativeSubjects': []}})))
+    asyncio.run(parse(json.dumps({'summaries': []})))
+    asyncio.run(parse(json.dumps({'textVersions': [{'type': 'Enrolled', 'date': '2026-11-04', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov/text'}]}]})))
+    obj.cursor = json.loads(json.dumps(obj.cursor))
+    output = asyncio.run(parse('<p>The bill establishes the regulatory framework for payment stablecoins and digital assets.</p>'))
+    assert output[0]['title'] == 'GENIUS Act — Enrolled'
+    assert '/actions?' in output[1].url
+    output = asyncio.run(parse(json.dumps({'actions': [{'actionDate': '2026-11-04', 'text': 'Became law'}]})))
+    assert output[0]['extra']['content_scope'] == 'official_action'
+    assert not obj.cursor.get('bill_retries')
+
+
+def test_missing_bill_material_is_a_retryable_gap(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(data):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), lane='history', document=doc))]
+    asyncio.run(parse({'bill': {'title': 'GENIUS Act'}}))
+    asyncio.run(parse({'subjects': {}}))
+    asyncio.run(parse({'summaries': []}))
+    asyncio.run(parse({'textVersions': []}))
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+    assert not obj.run.stats.get('excluded')
+
+
+@pytest.mark.parametrize('attempts,version,expected_kind', [(3, 'v1', 'listing'), (1, 'v1', 'listing'), (3, 'v2', 'subjects')])
+def test_bill_retries_are_bounded_until_official_update(trading, monkeypatch, attempts, version, expected_kind):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    url = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': url, 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.cursor['bill_retries'] = {url: {'version': 'v1', 'attempts': attempts, 'next_at': (obj.run.started_at + timedelta(minutes=30)).timestamp()}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(url, json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': version}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert ('/subjects?' in req.url) is (expected_kind == 'subjects')
+
+
+def test_fed_archive_is_not_discovered_as_article(trading):
+    obj = spider(trading, 'fed')
+    url = SOURCES['fed'].entries[0]
+    req, = obj.parse_listing(response(url, '<h1>Press releases</h1><a href="2026-press.htm#content">Archive</a><a href="monetary20261104a.htm">News</a>', lane='history'))
+    assert req.url.endswith('monetary20261104a.htm')
+
+
+def test_ofac_entire_old_page_can_end_realtime_window(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20250101">Old</a><a rel="next" href="?page=2">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] is None
+
+
+@pytest.mark.parametrize('body', ['<title>Page not found</title><p>The requested official document could not be located.</p>', '<?xml version="1.0"?><bill><section>Digital asset regulation establishes requirements for banks.'])
+def test_error_html_and_malformed_xml_are_rejected(body):
+    from alphapulse.core.us_news import validate_body
+    with pytest.raises(ValueError):
+        validate_body(body)
+
+
+def test_later_bill_version_recalls_earlier_version(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    first = {'url': 'https://www.congress.gov/first', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2026-11-04', 'bill': base}
+    second = {**first, 'url': 'https://www.congress.gov/second'}
+    obj.cursor['history'].update(bill={'url': base, 'reasons': [], 'title': 'GENIUS Act', 'bodies': 0}, pending=[first, second])
+    async def parse(doc, body):
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert len(output) == 1 and isinstance(output[0], Request)
+    output = asyncio.run(parse(second, '<p>The legislation creates a regulatory framework for stablecoins and digital assets.</p>'))
+    assert output[0]['extra']['selection_reasons']
+    assert output[1].url == first['url']
+    obj.run.history_deadline = time.monotonic() - 1
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert output[0]['title'] == 'GENIUS Act'
+
+
+def test_bill_retry_keeps_historical_text_outside_realtime_window(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': 'https://www.congress.gov/old', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2025-01-01', 'bill': base}
+    obj.cursor['realtime'].update(bill={'url': base, 'title': 'GENIUS Act', 'reasons': ['crypto_policy'], 'bodies': 0, 'retry': True}, pending=[doc])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>The law establishes requirements for stablecoin issuers and digital assets.</p>', lane='realtime', document=doc))]
+    assert asyncio.run(collect())[0]['published_at'].year == 2025
+
+
+def test_failed_checkpoint_save_recovers_on_later_trigger(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=None, history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        requests, attempts = [], []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        original = obj.run
+        async def finish(run):
+            attempts.append(run)
+            if len(attempts) == 1:
+                raise RuntimeError('temporary checkpoint failure')
+        obj._finish_run = finish
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        await obj.start_round('schedule')
+        assert attempts == [original, original]
+        assert requests and obj.run is not None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_queued_tick_obeys_newly_saved_backoff(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=datetime.now(UTC) + timedelta(minutes=5), history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        obj.run = None
+        requests = []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        await obj.start_round('schedule')
+        assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_checkpoint_retry_budget_preserves_original_round(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'SessionLocal', lambda: pytest.fail('Uncommitted checkpoint must block source reads'))
+    async def check():
+        obj = spider(trading)
+        original, attempts = obj.run, []
+        async def fail(run):
+            attempts.append(run)
+            raise RuntimeError('persistent checkpoint failure')
+        obj._finish_run = fail
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        for _ in range(5):
+            await obj.start_round('schedule')
+        assert attempts == [original] * 3
+        assert obj.run is None and not obj._starting
+    asyncio.run(check())
+
+
+@pytest.mark.parametrize('failure', ['', 'count', 'date', 'malformed'])
+def test_register_xml_validates_and_resumes(trading, failure):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.govinfo.gov/issue.xml', 'kind': 'register_issue', 'day': '2025-01-01', 'count': 2001}
+    obj.cursor['history'].update(day=doc['day'], pending=[doc])
+    xml = '<FEDREG><DATE>January 1, 2025</DATE>' + ''.join(f'<FRDOC>[FR Doc. 2025-{n} Filed 1-1-25]</FRDOC>' for n in range(2001)) + '<FRDOC>[FR Doc. 2025-0 Filed 1-1-25]</FRDOC></FEDREG>'
+    if failure == 'count': doc['count'] += 1
+    if failure == 'date': doc['day'] = '2025-01-02'
+    if failure == 'malformed': xml = xml[:-10]
+    before = copy.deepcopy(obj.cursor)
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], xml, lane='history', document=doc))]
+    if failure:
+        from lxml import etree
+        with pytest.raises((ValueError, etree.XMLSyntaxError)):
+            asyncio.run(collect())
+        assert obj.cursor == before
+    else:
+        req, = asyncio.run(collect())
+        assert len(obj.cursor['history']['pending']) == 2001
+        assert req.meta['document']['kind'] == 'register'
+        assert obj.cursor['history']['day'] == '2025-01-01'
+        obj.run.history_deadline = time.monotonic() - 1
+        assert not obj.complete('history')
+        resumed = spider(trading, 'federal_register')
+        resumed.cursor = json.loads(json.dumps(obj.cursor))
+        assert resumed.advance('history')[0].meta['document'] == obj.cursor['history']['pending'][0]
+
+
+def test_doj_api_pagination_and_old_day_stop(trading):
+    obj = spider(trading, 'doj')
+    url = SOURCES['doj'].entries[0]
+    def data(day):
+        return {'metadata': {'responseInfo': {'status': 200}, 'resultset': {'count': '100', 'page': 0, 'pagesize': 50}}, 'results': [{'uuid': str(uuid.uuid4()), 'date': str(int(day.timestamp()))}]}
+    req, = obj.parse_listing(response(url, json.dumps(data(obj.run.started_at)), lane='realtime'))
+    assert req.meta['document']['kind'] == 'doj'
+    assert 'page=1' in obj.cursor['realtime']['next']
+    obj = spider(trading, 'doj')
+    list(obj.parse_listing(response(url, json.dumps(data(datetime(2024,1,1,tzinfo=UTC))), lane='history')))
+    assert obj.cursor['history']['next'] is None
+
+
+@pytest.mark.parametrize('bad_body', [False, True])
+def test_doj_body_identity_date_and_validation(trading, bad_body):
+    obj = spider(trading, 'doj')
+    obj.run.history_deadline = time.monotonic() - 1
+    identifier = str(uuid.uuid4())
+    doc = {'url': f'https://www.justice.gov/api/v1/press_releases/{identifier}.json', 'kind': 'doj'}
+    obj.cursor['history']['pending'] = [doc]
+    body = '<h1>Just a moment...</h1>' if bad_body else '<p>The cryptocurrency exchange pleaded guilty to violations of financial regulations.</p>'
+    record = {'uuid': identifier, 'url': 'https://justice.gov/opa/pr/test', 'date': '1735732800', 'title': 'Exchange &amp; settlement', 'body': body, 'changed': '<time datetime="2025-01-02T12:00:00Z">x</time>'}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'metadata': {'responseInfo': {'status': 200}}, 'results': [record]}), lane='history', document=doc))]
+    if bad_body:
+        with pytest.raises(ValueError): asyncio.run(collect())
+        assert obj.cursor['history']['pending'] == [doc]
+    else:
+        item, = asyncio.run(collect())
+        assert item['url'] == 'https://www.justice.gov/opa/pr/test'
+        assert item['title'] == 'Exchange & settlement'
+        assert item['extra']['published_at_precision'] == 'date'
+        assert item['source_updated_at'] == datetime(2025,1,2,12,tzinfo=UTC)
+
+
+def test_doj_old_discovery_cursor_is_rebuilt(trading):
+    obj = spider(trading, 'doj')
+    obj.cursor.update(checked={'old': 1}, history={'entry': 2}, attachment_retries={})
+    obj.first_requests()
+    assert obj.cursor['transport'] == 'doj-api-v1'
+    assert 'history' not in obj.cursor and obj.cursor['realtime']['entry'] == 0
+    assert obj.run.history_target == datetime(2025,1,1,tzinfo=UTC)
+
+
+def test_bill_undated_text_is_visible_retry_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    finish = {'url': base, 'kind': 'bill_finish'}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, finish])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': None, 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+def test_public_api_and_pdf_fixtures_replay_without_io(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    items = asyncio.run(script['public_api_items']())
+    assert len(items) == 3
+    assert all(item['extra']['content_scope'] == 'article' for item in items)
+
+
+
+def test_congress_official_xml_crosscheck(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    assert len(asyncio.run(script['congress_reference_items']())) == 3
+
+
+def test_congress_meeting_includes_deduplicated_witness_pdfs(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1', 'kind': 'meeting'}
+    obj.cursor['history']['pending'] = [doc]
+    pdf = {'url': 'https://www.congress.gov/witness.pdf', 'format': 'PDF', 'documentType': 'Witness Statement'}
+    data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z', 'meetingDocuments': [pdf], 'witnessDocuments': [pdf]}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), document=doc, lane='history'))]
+    req, = asyncio.run(collect())
+    assert req.url == pdf['url']
+    assert len(req.meta['parsed']['extra']['attachments']) == 1
+    assert req.meta['parsed']['extra']['attachments'][0]['title'] == 'Witness Statement'
+
+
+def test_dated_but_unavailable_bill_text_remains_a_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, {'url': base, 'kind': 'bill_finish'}])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': '2025-01-01', 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+
+def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_path):
+    import runpy
+    import shutil
+    from pathlib import Path
+    scripts = tmp_path / 'scripts'
+    scripts.mkdir()
+    original = Path(__file__).resolve().parents[2] / 'scripts'
+    for name in ('smoke_us_news.py', 'capture_congress.py'):
+        shutil.copyfile(original / name, scripts / name)
+    namespace = runpy.run_path(str(scripts / 'smoke_us_news.py'))
+    items = asyncio.run(namespace['congress_api_items']())
+    assert len(items) == 100
+    meetings = [item for item in items if item['extra'].get('official_meeting')]
+    assert len(meetings) == 1
+    assert meetings[0]['extra']['content_scope'] == 'partial'
+    attachments = meetings[0]['extra']['attachments']
+    assert len(attachments) == 15
+    assert len({a['url'] for a in attachments}) == 15
+    assert sum(a['status'] == 'complete' for a in attachments) == 14
+    assert sum(a['status'] == 'unavailable' for a in attachments) == 1
+    assert any(item['extra']['content_scope'] == 'official_action' for item in items)