Fix FinCEN PDF lists and exclude CFTC commissioner indexes

backend/src/alphapulse/core/us_news.py

@@ -21,7 +21,7 @@ class OfficialSource:
 
 SOURCES = {
     'sec': OfficialSource('SEC·证券监管', ('https://www.sec.gov/newsroom/press-releases', 'https://www.sec.gov/newsroom/speeches-statements', 'https://www.sec.gov/rules-regulations/rulemaking-activity', 'https://www.sec.gov/rules-regulations/self-regulatory-organization-rulemaking'), r'/(?:newsroom/(?:press-releases|speeches-statements)/[^/]+|rules-regulations/.*\d.*)', '.node-details-layout__main-region__content .field--name-body', ('www.sec.gov',)),
-    'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+)|csl/)', 'article .field--name-body', ('www.cftc.gov',)),
+    'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+$)|csl/)', 'article .field--name-body', ('www.cftc.gov',)),
     'treasury': OfficialSource('美国财政部', ('https://home.treasury.gov/news/press-releases',), r'/news/press-releases/[a-z]+\d+/?$', '.field--name-field-news-body', ('home.treasury.gov', 'www.treasury.gov', 'ofac.treasury.gov')),
     'ofac': OfficialSource('OFAC·金融制裁', ('https://ofac.treasury.gov/recent-actions',), r'/recent-actions/\d{8}[^/]*$', 'article.node--type-ofac-recent-action .field--name-field-body', ('ofac.treasury.gov', 'home.treasury.gov')),
     'fincen': OfficialSource('FinCEN·反洗钱', ('https://www.fincen.gov/news/press-releases', 'https://www.fincen.gov/news/enforcement-actions', 'https://www.fincen.gov/resources/advisoriesbulletinsfact-sheets'), r'/(?:news/news-releases/[^/]+|resources/advisories/[^/]+|news/enforcement-actions/[^/]+)', 'article .field--name-body', ('www.fincen.gov',)),

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -116,6 +116,15 @@ class USOfficialSpider(NewsSpider):
         if error:
             self.run.errors.append(error)
             return []
+        if self.name == 'cftc':
+            # Earlier discovery treated commissioner indexes as article gaps.
+            retries = self.cursor.get('document_retries', {})
+            indexes = {url for url in retries if re.fullmatch(r'/PressRoom/SpeechesTestimony/[^/]+/index\.htm', urlsplit(url).path)}
+            for url in indexes:
+                retries.pop(url)
+            for lane in ('realtime', 'history'):
+                if 'pending' in self.cursor.get(lane, {}):
+                    self.cursor[lane]['pending'] = [d for d in self.cursor[lane]['pending'] if d['url'] not in indexes]
         state = self.cursor.setdefault('realtime', {'entry': 0})
         retries = self.cursor.get('document_retries', {})
         due_documents = [{**value['document'], 'document_retry': True} for value in retries.values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
@@ -262,9 +271,23 @@ class USOfficialSpider(NewsSpider):
                     url = official_url(self.name, a.attrib['href'], response.url)
                 except ValueError:
                     continue
-                if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
+                fincen_row = a.xpath('ancestor::tr[ancestor::main][1]') if self.name == 'fincen' and response.url.split('?')[0] in SOURCES['fincen'].entries[1:] else []
+                fincen_pdf = bool(fincen_row and urlsplit(url).path.lower().endswith('.pdf'))
+                if fincen_pdf or (re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf')):
                     doc = {'url': url, 'kind': 'article'}
-                    if self.name in ('treasury', 'cftc', 'fincen', 'whitehouse'):
+                    if fincen_row:
+                        # Table times use noon as a date marker, not a release instant.
+                        date = fincen_row.xpath('normalize-space(td[2])').get()
+                        begin = self.run.history_target if lane == 'history' else self.run.cutoff
+                        if re.fullmatch(r'(?:0?[1-9]|1[0-2])/\d{4}', date) and int(date[-4:]) < begin.year:
+                            continue
+                        doc['listing_date'] = published(date)[0].date().isoformat()
+                        if fincen_pdf:
+                            title = fincen_row.xpath('normalize-space(td[3])').get() if response.url.split('?')[0] == SOURCES['fincen'].entries[2] else a.xpath('normalize-space(.)').get()
+                            if not title:
+                                raise ValueError('FinCEN PDF 列表缺少标题')
+                            doc.update(format='pdf', title=title, listing_url=response.url)
+                    elif self.name in ('treasury', 'cftc', 'fincen', 'whitehouse'):
                         container = {
                             'treasury': 'ancestor::div[h3][1]',
                             'cftc': 'ancestor::tr[1]',
@@ -522,16 +545,20 @@ class USOfficialSpider(NewsSpider):
                     parsed['source_updated_at'] = published(times[0])[0]
             parsed['extra']['official_uuid'] = record['uuid']
             doc = {**doc, 'canonical': canonical}
-        elif doc['kind'] == 'text':
-            at, precision = published(doc['at'])
+        elif doc['kind'] == 'text' or doc.get('format') == 'pdf':
+            at, precision = published(doc['listing_date'] if doc.get('format') == 'pdf' else doc['at'])
             if response.body.startswith(b'%PDF'):
                 content = '<pre>' + escape(pdf_text(response.body)) + '</pre>'
             else:
                 content = response.text
                 validate_body(content)
+                if doc.get('format') == 'pdf':
+                    raise ValueError('官方 PDF 返回非 PDF 正文')
                 if '<' not in content:
                     content = '<pre>' + escape(content) + '</pre>'
             parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}}
+            if doc.get('format') == 'pdf':
+                parsed['extra'].update(published_at_basis='official_listing', published_at_source=doc['listing_url'])
             if doc.get('pdf'):
                 parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
         else:

backend/tests/test_us_official_news.py

@@ -81,6 +81,93 @@ def test_treasury_real_realtime_stops_at_date_boundary_before_broken_pagination(
         list(obj.parse_listing(response(sample['url'], sample['body'], lane='history')))
 
 
+@pytest.mark.parametrize('suffix', ['/news/enforcement-actions', '/resources/advisoriesbulletinsfact-sheets'])
+def test_fincen_real_pdf_tables_finish_outside_realtime_window(trading, suffix):
+    obj = spider(trading, 'fincen')
+    sample = collection_sample('fincen', suffix)
+    obj.cursor['realtime']['entry'] = SOURCES['fincen'].entries.index(sample['url'])
+    requests = list(obj.parse_listing(response(sample['url'], sample['body'], lane='realtime')))
+    assert not obj.cursor['realtime'].get('pending')
+    assert all(not r.meta.get('document') for r in requests)
+    assert not obj.run.errors
+
+
+@pytest.mark.parametrize('suffix,title', [('/news/enforcement-actions', 'Bank of America'), ('/resources/advisoriesbulletinsfact-sheets', 'Foreign-Agent-Directed')])
+def test_fincen_real_pdf_tables_preserve_title_and_date_for_history(trading, suffix, title):
+    obj = spider(trading, 'fincen')
+    sample = collection_sample('fincen', suffix)
+    request, = obj.parse_listing(response(sample['url'], sample['body'], lane='history'))
+    doc = request.meta['document']
+    assert doc['kind'] == 'article' and doc['format'] == 'pdf'
+    assert title in doc['title'] and len(doc['listing_date']) == 10
+    assert doc['listing_url'] == sample['url']
+    assert request.url.endswith('.pdf')
+
+
+def test_fincen_pdf_keeps_listing_date_raw_evidence_and_selection(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    obj = spider(trading, 'fincen')
+    doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]}
+    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries))
+    monkeypatch.setattr(us_official, 'pdf_text', lambda body: 'This enforcement action concerns a virtual currency exchange financial institution.')
+    request = Request(doc['url'], meta={'lane': 'realtime', 'document': doc, 'raw_response_id': 12})
+    async def collect():
+        return [item async for item in obj.parse_document(Response(doc['url'], body=b'%PDF-test', request=request))]
+    item, = asyncio.run(collect())
+    assert item['published_at'] == datetime(2026, 11, 3, tzinfo=UTC)
+    assert item['raw_response_id'] == 12
+    assert item['extra']['published_at_precision'] == 'date'
+    assert item['extra']['published_at_basis'] == 'official_listing'
+    assert item['extra']['selection_reasons']
+    assert doc['url'] in obj.cursor['checked']
+
+
+def test_fincen_invalid_pdf_remains_a_document_gap(trading):
+    obj = spider(trading, 'fincen')
+    doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]}
+    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries))
+    async def collect():
+        return [item async for item in obj.parse_document(response(doc['url'], '<h1>Document</h1><p>The server returned an HTML document instead of the expected official PDF.</p>', lane='realtime', document=doc))]
+    assert not asyncio.run(collect())
+    assert doc['url'] not in obj.cursor.get('checked', {})
+    assert obj.cursor['document_retries'][doc['url']]['attempts'] == 1
+    assert '非 PDF' in obj.run.errors[0]
+
+
+def test_fincen_pdf_table_missing_date_fails_explicitly(trading):
+    obj = spider(trading, 'fincen')
+    body = '<main><h1>Actions</h1><table><tr><td><a href="/system/files/order.pdf">Consent order</a></td><td></td></tr></table></main>'
+    with pytest.raises(ValueError, match='发布时间'):
+        list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+
+
+def test_fincen_pdf_table_does_not_invent_day_for_current_month(trading):
+    obj = spider(trading, 'fincen')
+    body = '<main><h1>Actions</h1><table><tr><td><a href="/system/files/order.pdf">Consent order</a></td><td>11/2026</td></tr></table></main>'
+    with pytest.raises(ValueError, match='发布时间'):
+        list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+
+
+def test_cftc_commissioner_index_is_not_a_speech(trading):
+    obj = spider(trading, 'cftc')
+    body = '<main><h1>Speeches</h1><a href="/PressRoom/SpeechesTestimony/MichaelSelig/index.htm">Commissioner</a><a href="/PressRoom/SpeechesTestimony/opaselig1">Speech</a></main>'
+    request, = obj.parse_listing(response(SOURCES['cftc'].entries[1], body, lane='realtime'))
+    assert request.url.endswith('/opaselig1')
+    assert len(obj.cursor['realtime']['pending']) == 1
+
+
+def test_cftc_drops_only_confirmed_profile_gaps_from_old_cursor(trading):
+    obj = spider(trading, 'cftc')
+    profile = {'url': 'https://www.cftc.gov/PressRoom/SpeechesTestimony/MichaelSelig/index.htm', 'kind': 'article'}
+    speech = {'url': 'https://www.cftc.gov/PressRoom/SpeechesTestimony/opaselig1', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [profile, speech]
+    obj.cursor['document_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': obj.run.started_at.timestamp() + 3600} for doc in (profile, speech)}
+    list(obj.first_requests())
+    assert profile['url'] not in obj.cursor['document_retries']
+    assert speech['url'] in obj.cursor['document_retries']
+    assert all(d['url'] != profile['url'] for d in obj.cursor['realtime'].get('pending', []))
+
+
 def test_undated_or_unordered_lists_do_not_stop_early(trading):
     obj = spider(trading, 'treasury')
     url = SOURCES[obj.name].entries[0]