diff --git a/backend/src/alphapulse/core/us_news.py b/backend/src/alphapulse/core/us_news.py index 1216246..1383b92 100644 --- a/backend/src/alphapulse/core/us_news.py +++ b/backend/src/alphapulse/core/us_news.py @@ -30,7 +30,7 @@ SOURCES = { 'federal_register': OfficialSource('Federal Register·规则', ('https://www.federalregister.gov/api/v1/documents.json',), r'/documents/', '', ('www.federalregister.gov', 'public-inspection.federalregister.gov', 'www.govinfo.gov'), 900), 'occ': OfficialSource('OCC·银行与稳定币', tuple(f'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/{kind}/{year}-{kind}.html' for year in range(datetime.now(UTC).year, 2024, -1) for kind in ('news-releases', 'bulletins')), r'/news-issuances/(?:news-releases|bulletins|interpretations-and-precedents)/202[5-9]/[^/]+', '.occgov-issuance-content', ('www.occ.gov', 'occ.gov', 'www.occ.treas.gov', 'occ.treas.gov')), 'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)), - 'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speeches', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60), + 'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speech', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60), } RULE_VERSION = 'us-policy-v1' CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I) diff --git a/backend/tests/fixtures/us_news/fed_archives.json.gz b/backend/tests/fixtures/us_news/fed_archives.json.gz new file mode 100644 index 0000000..2103451 Binary files /dev/null and b/backend/tests/fixtures/us_news/fed_archives.json.gz differ diff --git a/backend/tests/test_us_official_news.py b/backend/tests/test_us_official_news.py index 37323e3..6efca83 100644 --- a/backend/tests/test_us_official_news.py +++ b/backend/tests/test_us_official_news.py @@ -122,6 +122,23 @@ def test_summary_scope_tests_do_not_hide_fed_enrichment(trading): assert len(SOURCES['fed'].entries) >= 8 +def test_fed_entries_match_real_archives_and_discover_documents(trading): + import gzip + from pathlib import Path + samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes())) + for sample in samples: + assert sample['url'] in SOURCES['fed'].entries + obj = spider(trading, 'fed') + obj.cursor['history']['entry'] = SOURCES['fed'].entries.index(sample['url']) + request, = obj.advance('history') + assert request.url == sample['url'] + document, = obj.parse_listing(response(request.url, sample['body'], lane='history')) + assert document.meta['document']['kind'] == 'article' + assert document.url not in SOURCES['fed'].entries + assert document.url.startswith('https://www.federalreserve.gov/') + assert obj.run.stats['discovered'] > 0 + + def test_fifty_real_public_documents_parse_without_io(): import gzip from pathlib import Path diff --git a/deploy/README.md b/deploy/README.md index 56ffc87..2998c4f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -284,8 +284,12 @@ Validation without production credentials: `ci`, password `ci-only`, database `ci`, host `127.0.0.1`; it creates its own news tables and must run in an empty disposable database. Redis publication is mocked. -The 50 public HTML fixtures cover eight publishers. Additional API/PDF fixtures -cover three 2025 DOJ institutional cases (OPA and USAO), their four attachments, +The 50 public HTML fixtures cover eight publishers. Eight additional Fed +discovery responses were captured on 2026-11-04: the 2025/2026 +press, speech and testimony archives, FOMC calendars and H.4.1 release dates. +Their offline regression checks configured entry URLs and article discovery; +Fed speech archives use the singular `speech/` directory. +Additional API/PDF fixtures cover three 2025 DOJ institutional cases (OPA and USAO), their four attachments, one excluded personal crime case, and a Federal Register daily XML containing 106 unique document numbers. The isolated database smoke checks idempotency and revisions after API-package edits. Three additional