diff --git a/backend/src/alphapulse/core/us_news.py b/backend/src/alphapulse/core/us_news.py
index 1383b92..51f2e0d 100644
--- a/backend/src/alphapulse/core/us_news.py
+++ b/backend/src/alphapulse/core/us_news.py
@@ -105,11 +105,11 @@ def validate_body(content: str) -> None:
raise ValueError('官方正文为空或过短')
-def article(source: str, html: str, url: str) -> dict:
+def article(source: str, html: str, url: str, *, release_date: str | None = None) -> dict:
sel = Selector(html)
validate_body(html)
selectors = {
- 'fed': ('#article h3.title', '#article h3', '#article h2'),
+ 'fed': ('#article h3.title', '#article h3', '#article h2', '#content .heading h3', '#content .col-xs-12.col-sm-8.col-md-8 h3'),
'treasury': ('h2.uswds-page-title',),
'cftc': ('article h1:not(.press-release-title)',),
}.get(source, ('.main-content__main h1', '#main_content h1', 'main h1', 'h1'))
@@ -133,6 +133,10 @@ def article(source: str, html: str, url: str) -> dict:
values.insert(0, '-'.join(m.groups()))
if source == 'ofac' and (m := re.search(r'/recent-actions/(\d{4})(\d{2})(\d{2})', url)):
values.insert(0, '-'.join(m.groups()))
+ if source == 'fed' and '/monetarypolicy/fomcminutes' in url:
+ if release_date is None:
+ raise ValueError('FOMC 纪要缺少官方日历发布日期')
+ values = [release_date]
at = precision = None
for value in values:
try:
diff --git a/backend/src/alphapulse/crawler/spiders/us_official.py b/backend/src/alphapulse/crawler/spiders/us_official.py
index 94dc9ea..9d30fa1 100644
--- a/backend/src/alphapulse/crawler/spiders/us_official.py
+++ b/backend/src/alphapulse/crawler/spiders/us_official.py
@@ -237,12 +237,30 @@ class USOfficialSpider(NewsSpider):
except ValueError:
continue
if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
- docs.append({'url': url, 'kind': 'article'})
+ doc = {'url': url, 'kind': 'article'}
+ if self.name == 'fed' and '/monetarypolicy/fomcminutes' in url:
+ # The URL identifies the meeting; the calendar gives the release day.
+ match = re.search(r'Released\s+([A-Za-z]+ \d{1,2}, \d{4})', a.xpath('normalize-space(parent::*)').get() or '')
+ if not match:
+ raise ValueError('FOMC 纪要链接缺少官方发布日期')
+ doc.update(release_date=published(match[1])[0].date().isoformat(), release_date_source=response.url)
+ docs.append(doc)
if a.attrib.get('rel') == 'next' or re.match(r'^(?:Next|Next page|下一页)', a.xpath('normalize-space(.)').get() or '', re.I):
next_url = url
docs = list({d['url']: d for d in docs}.values())
if not docs:
raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+ if self.name == 'fed':
+ begin = self.run.history_target if lane == 'history' else self.run.cutoff
+ def in_range(doc):
+ if doc.get('release_date'):
+ day = published(doc['release_date'])[0].date()
+ elif match := re.search(r'((?:19|20)\d{6})', doc['url']):
+ day = datetime.strptime(match[1], '%Y%m%d').date()
+ else:
+ return True
+ return day >= begin.date()
+ docs = [doc for doc in docs if in_range(doc)]
if lane == 'realtime':
# OFAC's recent-actions entry is a descending day-indexed feed.
if self.name == 'ofac' and docs:
@@ -253,11 +271,6 @@ class USOfficialSpider(NewsSpider):
threshold = (self.run.started_at - timedelta(hours=24)).timestamp()
self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
- if self.name == 'fed':
- def recent(d):
- found = re.search(r'(202[5-9]\d{4})', d['url'])
- return not found or datetime.strptime(found[1], '%Y%m%d').replace(tzinfo=UTC) >= self.run.cutoff
- docs = [d for d in docs if recent(d)]
if next_url and official_url(self.name, next_url, response.url) == response.url:
raise ValueError('官方列表翻页重复当前页面，未推进断点')
visited = state.setdefault('visited_pages', [])
@@ -431,7 +444,9 @@ class USOfficialSpider(NewsSpider):
if doc.get('pdf'):
parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
else:
- parsed = article(self.name, response.text, response.url)
+ parsed = article(self.name, response.text, response.url, release_date=doc.get('release_date'))
+ if doc.get('release_date'):
+ parsed['extra'].update(published_at_basis='official_calendar_release', published_at_source=doc['release_date_source'])
if doc.get('bill'):
context = self.cursor[lane]['bill']
context['bodies'] += 1
@@ -446,7 +461,8 @@ class USOfficialSpider(NewsSpider):
begin = self.run.history_target if lane == 'history' else self.run.cutoff
if doc.get('attachment_retry') or (doc.get('bill') and self.cursor[lane]['bill'].get('retry')):
begin = datetime(2025, 1, 1, tzinfo=UTC)
- if parsed['published_at'] < begin:
+ outside_range = parsed['published_at'].date() < begin.date() if parsed['extra'].get('published_at_precision') == 'date' else parsed['published_at'] < begin
+ if outside_range:
self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
for result in self.complete(lane):
yield result
diff --git a/backend/tests/fixtures/us_news/fed_materials.json.gz b/backend/tests/fixtures/us_news/fed_materials.json.gz
new file mode 100644
index 0000000..69fbd8c
Binary files /dev/null and b/backend/tests/fixtures/us_news/fed_materials.json.gz differ
diff --git a/backend/tests/test_us_official_news.py b/backend/tests/test_us_official_news.py
index 6efca83..d367e1b 100644
--- a/backend/tests/test_us_official_news.py
+++ b/backend/tests/test_us_official_news.py
@@ -139,6 +139,99 @@ def test_fed_entries_match_real_archives_and_discover_documents(trading):
assert obj.run.stats['discovered'] > 0
+def test_fed_realtime_completes_all_real_discovery_entries(trading, monkeypatch):
+ from dataclasses import replace
+ import gzip
+ from pathlib import Path
+ samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes()))
+ bodies = {s['url']: s['body'] for s in samples}
+ monkeypatch.setitem(SOURCES, 'fed', replace(SOURCES['fed'], entries=tuple(bodies)))
+ obj = spider(trading, 'fed')
+ requests = obj.first_requests()
+ visited = []
+ while requests:
+ request = requests.pop()
+ if request.meta.get('document'):
+ assert '2024' not in request.url
+ requests.extend(obj.complete('realtime'))
+ else:
+ assert request.url not in visited
+ visited.append(request.url)
+ requests.extend(obj.parse_listing(response(request.url, bodies[request.url], lane='realtime')))
+ assert visited == list(SOURCES['fed'].entries)
+ assert obj.cursor['realtime'] == {'entry': 0}
+ assert not obj.run.errors
+
+
+@pytest.mark.parametrize('lane', ['history', 'realtime'])
+def test_fed_discovery_respects_old_and_boundary_dates(trading, lane):
+ obj = spider(trading, 'fed')
+ obj.run.cutoff = datetime(2026, 11, 1, 15, tzinfo=UTC)
+ body = 'FOMC
' + ''.join(
+ f'Note'
+ for day in ('20240131', '20251031', '20261101')
+ )
+ list(obj.parse_listing(response('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', body, lane=lane)))
+ pending = obj.cursor[lane]['pending']
+ assert all('20240131' not in doc['url'] for doc in pending)
+ assert any('20261101' in doc['url'] for doc in pending)
+ assert any('20251031' in doc['url'] for doc in pending) is (lane == 'history')
+
+
+@pytest.mark.parametrize('lane', ['history', 'realtime'])
+def test_fed_minutes_use_official_release_day_instead_of_meeting_day(trading, monkeypatch, lane):
+ import gzip
+ from pathlib import Path
+ samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes()))
+ sample = next(s for s in samples if 'fomccalendars' in s['url'])
+ obj = spider(trading, 'fed')
+ obj.run.cutoff = obj.run.history_target = datetime(2026, 2, 18, 15, tzinfo=UTC)
+ list(obj.parse_listing(response(sample['url'], sample['body'], lane=lane)))
+ doc = next(d for d in obj.cursor[lane]['pending'] if 'fomcminutes20260128.htm' in d['url'])
+ assert doc['release_date'] == '2026-02-18'
+ assert doc['release_date_source'] == sample['url']
+ bodies = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_materials.json.gz').read_bytes()))
+ body = next(s['body'] for s in bodies if s['url'] == doc['url'])
+ from alphapulse.crawler.spiders import us_official
+ class Session:
+ async def __aenter__(self): return self
+ async def __aexit__(self, *args): pass
+ async def scalar(self, query): return None
+ monkeypatch.setattr(us_official, 'SessionLocal', Session)
+ monkeypatch.setattr(obj, 'attach_or_finish', lambda parsed, lane: iter([parsed]))
+ async def collect():
+ return [p async for p in obj.parse_document(response(doc['url'], body, lane=lane, document=doc))]
+ parsed, = asyncio.run(collect())
+ assert parsed['published_at'] == datetime(2026, 2, 18, tzinfo=UTC)
+ assert parsed['extra']['published_at_basis'] == 'official_calendar_release'
+ assert parsed['extra']['published_at_source'] == sample['url']
+
+
+def test_fed_minutes_missing_release_date_fail_explicitly(trading):
+ obj = spider(trading, 'fed')
+ body = 'FOMC
HTML'
+ with pytest.raises(ValueError, match='官方发布日期'):
+ list(obj.parse_listing(response(SOURCES['fed'].entries[-2], body, lane='history')))
+
+
+def test_fed_real_policy_bodies_have_titles_and_publication_dates():
+ import gzip
+ from pathlib import Path
+ samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_materials.json.gz').read_bytes()))
+ for sample in samples:
+ is_minutes = '/fomcminutes' in sample['url']
+ if is_minutes:
+ with pytest.raises(ValueError, match='官方日历发布日期'):
+ article('fed', sample['body'], sample['url'])
+ with pytest.raises(ValueError, match='可识别的发布时间'):
+ article('fed', sample['body'], sample['url'], release_date='invalid')
+ parsed = article('fed', sample['body'], sample['url'], release_date=sample['published_date'] if is_minutes else None)
+ assert parsed['title'] == sample['title']
+ assert parsed['published_at'].date().isoformat() == sample['published_date']
+ assert parsed['extra']['published_at_precision'] == 'date'
+ assert len(parsed['content_html']) > 1000
+
+
def test_fifty_real_public_documents_parse_without_io():
import gzip
from pathlib import Path
diff --git a/deploy/README.md b/deploy/README.md
index 2998c4f..948bfaa 100644
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -289,6 +289,11 @@ discovery responses were captured on 2026-11-04: the 2025/2026
press, speech and testimony archives, FOMC calendars and H.4.1 release dates.
Their offline regression checks configured entry URLs and article discovery;
Fed speech archives use the singular `speech/` directory.
+Three captured policy bodies cover implementation notes, economic projections
+and meeting minutes. Minutes use the calendar's explicit release date, with its
+source URL recorded, rather than the meeting date encoded in the document URL.
+Date-only documents remain eligible on the cutoff day; older discovery links
+are filtered before fetching their bodies in both realtime and history lanes.
Additional API/PDF fixtures cover three 2025 DOJ institutional cases (OPA and USAO), their four attachments,
one excluded personal crime case, and a Federal Register daily XML containing 106
unique document numbers. The isolated database smoke checks idempotency and