Fix Fed speech archive discovery path

Use the official singular speech directory and replay all eight captured Fed discovery responses to catch invalid archive URLs and article discovery failures.

backend/src/alphapulse/core/us_news.py

@@ -30,7 +30,7 @@ SOURCES = {
     'federal_register': OfficialSource('Federal Register·规则', ('https://www.federalregister.gov/api/v1/documents.json',), r'/documents/', '', ('www.federalregister.gov', 'public-inspection.federalregister.gov', 'www.govinfo.gov'), 900),
     'occ': OfficialSource('OCC·银行与稳定币', tuple(f'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/{kind}/{year}-{kind}.html' for year in range(datetime.now(UTC).year, 2024, -1) for kind in ('news-releases', 'bulletins')), r'/news-issuances/(?:news-releases|bulletins|interpretations-and-precedents)/202[5-9]/[^/]+', '.occgov-issuance-content', ('www.occ.gov', 'occ.gov', 'www.occ.treas.gov', 'occ.treas.gov')),
     'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)),
-    'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speeches', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
+    'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speech', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
 }
 RULE_VERSION = 'us-policy-v1'
 CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I)

backend/tests/test_us_official_news.py

@@ -122,6 +122,23 @@ def test_summary_scope_tests_do_not_hide_fed_enrichment(trading):
     assert len(SOURCES['fed'].entries) >= 8
 
 
+def test_fed_entries_match_real_archives_and_discover_documents(trading):
+    import gzip
+    from pathlib import Path
+    samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes()))
+    for sample in samples:
+        assert sample['url'] in SOURCES['fed'].entries
+        obj = spider(trading, 'fed')
+        obj.cursor['history']['entry'] = SOURCES['fed'].entries.index(sample['url'])
+        request, = obj.advance('history')
+        assert request.url == sample['url']
+        document, = obj.parse_listing(response(request.url, sample['body'], lane='history'))
+        assert document.meta['document']['kind'] == 'article'
+        assert document.url not in SOURCES['fed'].entries
+        assert document.url.startswith('https://www.federalreserve.gov/')
+        assert obj.run.stats['discovered'] > 0
+
+
 def test_fifty_real_public_documents_parse_without_io():
     import gzip
     from pathlib import Path