Respect Fed release dates and policy document layouts

Filter discovery links against both realtime and historical date boundaries, keep date-only cutoff days, and use the official calendar release date for FOMC minutes. Parse implementation note and projection headings and replay real Fed discovery and policy bodies without external I/O.

backend/src/alphapulse/core/us_news.py

@@ -105,11 +105,11 @@ def validate_body(content: str) -> None:
         raise ValueError('官方正文为空或过短')
 
 
-def article(source: str, html: str, url: str) -> dict:
+def article(source: str, html: str, url: str, *, release_date: str | None = None) -> dict:
     sel = Selector(html)
     validate_body(html)
     selectors = {
-        'fed': ('#article h3.title', '#article h3', '#article h2'),
+        'fed': ('#article h3.title', '#article h3', '#article h2', '#content .heading h3', '#content .col-xs-12.col-sm-8.col-md-8 h3'),
         'treasury': ('h2.uswds-page-title',),
         'cftc': ('article h1:not(.press-release-title)',),
     }.get(source, ('.main-content__main h1', '#main_content h1', 'main h1', 'h1'))
@@ -133,6 +133,10 @@ def article(source: str, html: str, url: str) -> dict:
         values.insert(0, '-'.join(m.groups()))
     if source == 'ofac' and (m := re.search(r'/recent-actions/(\d{4})(\d{2})(\d{2})', url)):
         values.insert(0, '-'.join(m.groups()))
+    if source == 'fed' and '/monetarypolicy/fomcminutes' in url:
+        if release_date is None:
+            raise ValueError('FOMC 纪要缺少官方日历发布日期')
+        values = [release_date]
     at = precision = None
     for value in values:
         try:

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -237,12 +237,30 @@ class USOfficialSpider(NewsSpider):
                 except ValueError:
                     continue
                 if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
-                    docs.append({'url': url, 'kind': 'article'})
+                    doc = {'url': url, 'kind': 'article'}
+                    if self.name == 'fed' and '/monetarypolicy/fomcminutes' in url:
+                        # The URL identifies the meeting; the calendar gives the release day.
+                        match = re.search(r'Released\s+([A-Za-z]+ \d{1,2}, \d{4})', a.xpath('normalize-space(parent::*)').get() or '')
+                        if not match:
+                            raise ValueError('FOMC 纪要链接缺少官方发布日期')
+                        doc.update(release_date=published(match[1])[0].date().isoformat(), release_date_source=response.url)
+                    docs.append(doc)
                 if a.attrib.get('rel') == 'next' or re.match(r'^(?:Next|Next page|下一页)', a.xpath('normalize-space(.)').get() or '', re.I):
                     next_url = url
             docs = list({d['url']: d for d in docs}.values())
             if not docs:
                 raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        if self.name == 'fed':
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            def in_range(doc):
+                if doc.get('release_date'):
+                    day = published(doc['release_date'])[0].date()
+                elif match := re.search(r'((?:19|20)\d{6})', doc['url']):
+                    day = datetime.strptime(match[1], '%Y%m%d').date()
+                else:
+                    return True
+                return day >= begin.date()
+            docs = [doc for doc in docs if in_range(doc)]
         if lane == 'realtime':
             # OFAC's recent-actions entry is a descending day-indexed feed.
             if self.name == 'ofac' and docs:
@@ -253,11 +271,6 @@ class USOfficialSpider(NewsSpider):
             threshold = (self.run.started_at - timedelta(hours=24)).timestamp()
             self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
             docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
-            if self.name == 'fed':
-                def recent(d):
-                    found = re.search(r'(202[5-9]\d{4})', d['url'])
-                    return not found or datetime.strptime(found[1], '%Y%m%d').replace(tzinfo=UTC) >= self.run.cutoff
-                docs = [d for d in docs if recent(d)]
         if next_url and official_url(self.name, next_url, response.url) == response.url:
             raise ValueError('官方列表翻页重复当前页面，未推进断点')
         visited = state.setdefault('visited_pages', [])
@@ -431,7 +444,9 @@ class USOfficialSpider(NewsSpider):
             if doc.get('pdf'):
                 parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
         else:
-            parsed = article(self.name, response.text, response.url)
+            parsed = article(self.name, response.text, response.url, release_date=doc.get('release_date'))
+            if doc.get('release_date'):
+                parsed['extra'].update(published_at_basis='official_calendar_release', published_at_source=doc['release_date_source'])
         if doc.get('bill'):
             context = self.cursor[lane]['bill']
             context['bodies'] += 1
@@ -446,7 +461,8 @@ class USOfficialSpider(NewsSpider):
         begin = self.run.history_target if lane == 'history' else self.run.cutoff
         if doc.get('attachment_retry') or (doc.get('bill') and self.cursor[lane]['bill'].get('retry')):
             begin = datetime(2025, 1, 1, tzinfo=UTC)
-        if parsed['published_at'] < begin:
+        outside_range = parsed['published_at'].date() < begin.date() if parsed['extra'].get('published_at_precision') == 'date' else parsed['published_at'] < begin
+        if outside_range:
             self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
             for result in self.complete(lane):
                 yield result

backend/tests/test_us_official_news.py

@@ -139,6 +139,99 @@ def test_fed_entries_match_real_archives_and_discover_documents(trading):
         assert obj.run.stats['discovered'] > 0
 
 
+def test_fed_realtime_completes_all_real_discovery_entries(trading, monkeypatch):
+    from dataclasses import replace
+    import gzip
+    from pathlib import Path
+    samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes()))
+    bodies = {s['url']: s['body'] for s in samples}
+    monkeypatch.setitem(SOURCES, 'fed', replace(SOURCES['fed'], entries=tuple(bodies)))
+    obj = spider(trading, 'fed')
+    requests = obj.first_requests()
+    visited = []
+    while requests:
+        request = requests.pop()
+        if request.meta.get('document'):
+            assert '2024' not in request.url
+            requests.extend(obj.complete('realtime'))
+        else:
+            assert request.url not in visited
+            visited.append(request.url)
+            requests.extend(obj.parse_listing(response(request.url, bodies[request.url], lane='realtime')))
+    assert visited == list(SOURCES['fed'].entries)
+    assert obj.cursor['realtime'] == {'entry': 0}
+    assert not obj.run.errors
+
+
+@pytest.mark.parametrize('lane', ['history', 'realtime'])
+def test_fed_discovery_respects_old_and_boundary_dates(trading, lane):
+    obj = spider(trading, 'fed')
+    obj.run.cutoff = datetime(2026, 11, 1, 15, tzinfo=UTC)
+    body = '<h1>FOMC</h1>' + ''.join(
+        f'<a href="/newsevents/pressreleases/monetary{day}a1.htm">Note</a>'
+        for day in ('20240131', '20251031', '20261101')
+    )
+    list(obj.parse_listing(response('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', body, lane=lane)))
+    pending = obj.cursor[lane]['pending']
+    assert all('20240131' not in doc['url'] for doc in pending)
+    assert any('20261101' in doc['url'] for doc in pending)
+    assert any('20251031' in doc['url'] for doc in pending) is (lane == 'history')
+
+
+@pytest.mark.parametrize('lane', ['history', 'realtime'])
+def test_fed_minutes_use_official_release_day_instead_of_meeting_day(trading, monkeypatch, lane):
+    import gzip
+    from pathlib import Path
+    samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_archives.json.gz').read_bytes()))
+    sample = next(s for s in samples if 'fomccalendars' in s['url'])
+    obj = spider(trading, 'fed')
+    obj.run.cutoff = obj.run.history_target = datetime(2026, 2, 18, 15, tzinfo=UTC)
+    list(obj.parse_listing(response(sample['url'], sample['body'], lane=lane)))
+    doc = next(d for d in obj.cursor[lane]['pending'] if 'fomcminutes20260128.htm' in d['url'])
+    assert doc['release_date'] == '2026-02-18'
+    assert doc['release_date_source'] == sample['url']
+    bodies = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_materials.json.gz').read_bytes()))
+    body = next(s['body'] for s in bodies if s['url'] == doc['url'])
+    from alphapulse.crawler.spiders import us_official
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def scalar(self, query): return None
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    monkeypatch.setattr(obj, 'attach_or_finish', lambda parsed, lane: iter([parsed]))
+    async def collect():
+        return [p async for p in obj.parse_document(response(doc['url'], body, lane=lane, document=doc))]
+    parsed, = asyncio.run(collect())
+    assert parsed['published_at'] == datetime(2026, 2, 18, tzinfo=UTC)
+    assert parsed['extra']['published_at_basis'] == 'official_calendar_release'
+    assert parsed['extra']['published_at_source'] == sample['url']
+
+
+def test_fed_minutes_missing_release_date_fail_explicitly(trading):
+    obj = spider(trading, 'fed')
+    body = '<h1>FOMC</h1><a href="/monetarypolicy/fomcminutes20260128.htm">HTML</a>'
+    with pytest.raises(ValueError, match='官方发布日期'):
+        list(obj.parse_listing(response(SOURCES['fed'].entries[-2], body, lane='history')))
+
+
+def test_fed_real_policy_bodies_have_titles_and_publication_dates():
+    import gzip
+    from pathlib import Path
+    samples = json.loads(gzip.decompress((Path(__file__).parent / 'fixtures/us_news/fed_materials.json.gz').read_bytes()))
+    for sample in samples:
+        is_minutes = '/fomcminutes' in sample['url']
+        if is_minutes:
+            with pytest.raises(ValueError, match='官方日历发布日期'):
+                article('fed', sample['body'], sample['url'])
+            with pytest.raises(ValueError, match='可识别的发布时间'):
+                article('fed', sample['body'], sample['url'], release_date='invalid')
+        parsed = article('fed', sample['body'], sample['url'], release_date=sample['published_date'] if is_minutes else None)
+        assert parsed['title'] == sample['title']
+        assert parsed['published_at'].date().isoformat() == sample['published_date']
+        assert parsed['extra']['published_at_precision'] == 'date'
+        assert len(parsed['content_html']) > 1000
+
+
 def test_fifty_real_public_documents_parse_without_io():
     import gzip
     from pathlib import Path