diff --git a/backend/src/alphapulse/core/us_news.py b/backend/src/alphapulse/core/us_news.py index 3be7383..8a38d25 100644 --- a/backend/src/alphapulse/core/us_news.py +++ b/backend/src/alphapulse/core/us_news.py @@ -21,7 +21,7 @@ class OfficialSource: SOURCES = { 'sec': OfficialSource('SEC·证券监管', ('https://www.sec.gov/newsroom/press-releases', 'https://www.sec.gov/newsroom/speeches-statements', 'https://www.sec.gov/rules-regulations/rulemaking-activity', 'https://www.sec.gov/rules-regulations/self-regulatory-organization-rulemaking'), r'/(?:newsroom/(?:press-releases|speeches-statements)/[^/]+|rules-regulations/.*\d.*)', '.node-details-layout__main-region__content .field--name-body', ('www.sec.gov',)), - 'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+)|csl/)', 'article .field--name-body', ('www.cftc.gov',)), + 'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+$)|csl/)', 'article .field--name-body', ('www.cftc.gov',)), 'treasury': OfficialSource('美国财政部', ('https://home.treasury.gov/news/press-releases',), r'/news/press-releases/[a-z]+\d+/?$', '.field--name-field-news-body', ('home.treasury.gov', 'www.treasury.gov', 'ofac.treasury.gov')), 'ofac': OfficialSource('OFAC·金融制裁', ('https://ofac.treasury.gov/recent-actions',), r'/recent-actions/\d{8}[^/]*$', 'article.node--type-ofac-recent-action .field--name-field-body', ('ofac.treasury.gov', 'home.treasury.gov')), 'fincen': OfficialSource('FinCEN·反洗钱', ('https://www.fincen.gov/news/press-releases', 'https://www.fincen.gov/news/enforcement-actions', 'https://www.fincen.gov/resources/advisoriesbulletinsfact-sheets'), r'/(?:news/news-releases/[^/]+|resources/advisories/[^/]+|news/enforcement-actions/[^/]+)', 'article .field--name-body', ('www.fincen.gov',)), diff --git a/backend/src/alphapulse/crawler/spiders/us_official.py b/backend/src/alphapulse/crawler/spiders/us_official.py index 1ec3d5b..a5aaaab 100644 --- a/backend/src/alphapulse/crawler/spiders/us_official.py +++ b/backend/src/alphapulse/crawler/spiders/us_official.py @@ -116,6 +116,15 @@ class USOfficialSpider(NewsSpider): if error: self.run.errors.append(error) return [] + if self.name == 'cftc': + # Earlier discovery treated commissioner indexes as article gaps. + retries = self.cursor.get('document_retries', {}) + indexes = {url for url in retries if re.fullmatch(r'/PressRoom/SpeechesTestimony/[^/]+/index\.htm', urlsplit(url).path)} + for url in indexes: + retries.pop(url) + for lane in ('realtime', 'history'): + if 'pending' in self.cursor.get(lane, {}): + self.cursor[lane]['pending'] = [d for d in self.cursor[lane]['pending'] if d['url'] not in indexes] state = self.cursor.setdefault('realtime', {'entry': 0}) retries = self.cursor.get('document_retries', {}) due_documents = [{**value['document'], 'document_retry': True} for value in retries.values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()] @@ -262,9 +271,23 @@ class USOfficialSpider(NewsSpider): url = official_url(self.name, a.attrib['href'], response.url) except ValueError: continue - if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'): + fincen_row = a.xpath('ancestor::tr[ancestor::main][1]') if self.name == 'fincen' and response.url.split('?')[0] in SOURCES['fincen'].entries[1:] else [] + fincen_pdf = bool(fincen_row and urlsplit(url).path.lower().endswith('.pdf')) + if fincen_pdf or (re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf')): doc = {'url': url, 'kind': 'article'} - if self.name in ('treasury', 'cftc', 'fincen', 'whitehouse'): + if fincen_row: + # Table times use noon as a date marker, not a release instant. + date = fincen_row.xpath('normalize-space(td[2])').get() + begin = self.run.history_target if lane == 'history' else self.run.cutoff + if re.fullmatch(r'(?:0?[1-9]|1[0-2])/\d{4}', date) and int(date[-4:]) < begin.year: + continue + doc['listing_date'] = published(date)[0].date().isoformat() + if fincen_pdf: + title = fincen_row.xpath('normalize-space(td[3])').get() if response.url.split('?')[0] == SOURCES['fincen'].entries[2] else a.xpath('normalize-space(.)').get() + if not title: + raise ValueError('FinCEN PDF 列表缺少标题') + doc.update(format='pdf', title=title, listing_url=response.url) + elif self.name in ('treasury', 'cftc', 'fincen', 'whitehouse'): container = { 'treasury': 'ancestor::div[h3][1]', 'cftc': 'ancestor::tr[1]', @@ -522,16 +545,20 @@ class USOfficialSpider(NewsSpider): parsed['source_updated_at'] = published(times[0])[0] parsed['extra']['official_uuid'] = record['uuid'] doc = {**doc, 'canonical': canonical} - elif doc['kind'] == 'text': - at, precision = published(doc['at']) + elif doc['kind'] == 'text' or doc.get('format') == 'pdf': + at, precision = published(doc['listing_date'] if doc.get('format') == 'pdf' else doc['at']) if response.body.startswith(b'%PDF'): content = '
' + escape(pdf_text(response.body)) + '
' else: content = response.text validate_body(content) + if doc.get('format') == 'pdf': + raise ValueError('官方 PDF 返回非 PDF 正文') if '<' not in content: content = '
' + escape(content) + '
' parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}} + if doc.get('format') == 'pdf': + parsed['extra'].update(published_at_basis='official_listing', published_at_source=doc['listing_url']) if doc.get('pdf'): parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'}) else: diff --git a/backend/tests/fixtures/us_news/collection_failures.json.gz b/backend/tests/fixtures/us_news/collection_failures.json.gz index fec6ab5..ce3197e 100644 Binary files a/backend/tests/fixtures/us_news/collection_failures.json.gz and b/backend/tests/fixtures/us_news/collection_failures.json.gz differ diff --git a/backend/tests/test_us_official_news.py b/backend/tests/test_us_official_news.py index b5ab880..41130b3 100644 --- a/backend/tests/test_us_official_news.py +++ b/backend/tests/test_us_official_news.py @@ -81,6 +81,93 @@ def test_treasury_real_realtime_stops_at_date_boundary_before_broken_pagination( list(obj.parse_listing(response(sample['url'], sample['body'], lane='history'))) +@pytest.mark.parametrize('suffix', ['/news/enforcement-actions', '/resources/advisoriesbulletinsfact-sheets']) +def test_fincen_real_pdf_tables_finish_outside_realtime_window(trading, suffix): + obj = spider(trading, 'fincen') + sample = collection_sample('fincen', suffix) + obj.cursor['realtime']['entry'] = SOURCES['fincen'].entries.index(sample['url']) + requests = list(obj.parse_listing(response(sample['url'], sample['body'], lane='realtime'))) + assert not obj.cursor['realtime'].get('pending') + assert all(not r.meta.get('document') for r in requests) + assert not obj.run.errors + + +@pytest.mark.parametrize('suffix,title', [('/news/enforcement-actions', 'Bank of America'), ('/resources/advisoriesbulletinsfact-sheets', 'Foreign-Agent-Directed')]) +def test_fincen_real_pdf_tables_preserve_title_and_date_for_history(trading, suffix, title): + obj = spider(trading, 'fincen') + sample = collection_sample('fincen', suffix) + request, = obj.parse_listing(response(sample['url'], sample['body'], lane='history')) + doc = request.meta['document'] + assert doc['kind'] == 'article' and doc['format'] == 'pdf' + assert title in doc['title'] and len(doc['listing_date']) == 10 + assert doc['listing_url'] == sample['url'] + assert request.url.endswith('.pdf') + + +def test_fincen_pdf_keeps_listing_date_raw_evidence_and_selection(trading, monkeypatch): + from alphapulse.crawler.spiders import us_official + obj = spider(trading, 'fincen') + doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]} + obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries)) + monkeypatch.setattr(us_official, 'pdf_text', lambda body: 'This enforcement action concerns a virtual currency exchange financial institution.') + request = Request(doc['url'], meta={'lane': 'realtime', 'document': doc, 'raw_response_id': 12}) + async def collect(): + return [item async for item in obj.parse_document(Response(doc['url'], body=b'%PDF-test', request=request))] + item, = asyncio.run(collect()) + assert item['published_at'] == datetime(2026, 11, 3, tzinfo=UTC) + assert item['raw_response_id'] == 12 + assert item['extra']['published_at_precision'] == 'date' + assert item['extra']['published_at_basis'] == 'official_listing' + assert item['extra']['selection_reasons'] + assert doc['url'] in obj.cursor['checked'] + + +def test_fincen_invalid_pdf_remains_a_document_gap(trading): + obj = spider(trading, 'fincen') + doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]} + obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries)) + async def collect(): + return [item async for item in obj.parse_document(response(doc['url'], '

Document

The server returned an HTML document instead of the expected official PDF.

', lane='realtime', document=doc))] + assert not asyncio.run(collect()) + assert doc['url'] not in obj.cursor.get('checked', {}) + assert obj.cursor['document_retries'][doc['url']]['attempts'] == 1 + assert '非 PDF' in obj.run.errors[0] + + +def test_fincen_pdf_table_missing_date_fails_explicitly(trading): + obj = spider(trading, 'fincen') + body = '

Actions

Consent order
' + with pytest.raises(ValueError, match='发布时间'): + list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime'))) + + +def test_fincen_pdf_table_does_not_invent_day_for_current_month(trading): + obj = spider(trading, 'fincen') + body = '

Actions

Consent order11/2026
' + with pytest.raises(ValueError, match='发布时间'): + list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime'))) + + +def test_cftc_commissioner_index_is_not_a_speech(trading): + obj = spider(trading, 'cftc') + body = '

Speeches

CommissionerSpeech
' + request, = obj.parse_listing(response(SOURCES['cftc'].entries[1], body, lane='realtime')) + assert request.url.endswith('/opaselig1') + assert len(obj.cursor['realtime']['pending']) == 1 + + +def test_cftc_drops_only_confirmed_profile_gaps_from_old_cursor(trading): + obj = spider(trading, 'cftc') + profile = {'url': 'https://www.cftc.gov/PressRoom/SpeechesTestimony/MichaelSelig/index.htm', 'kind': 'article'} + speech = {'url': 'https://www.cftc.gov/PressRoom/SpeechesTestimony/opaselig1', 'kind': 'article'} + obj.cursor['realtime']['pending'] = [profile, speech] + obj.cursor['document_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': obj.run.started_at.timestamp() + 3600} for doc in (profile, speech)} + list(obj.first_requests()) + assert profile['url'] not in obj.cursor['document_retries'] + assert speech['url'] in obj.cursor['document_retries'] + assert all(d['url'] != profile['url'] for d in obj.cursor['realtime'].get('pending', [])) + + def test_undated_or_unordered_lists_do_not_stop_early(trading): obj = spider(trading, 'treasury') url = SOURCES[obj.name].entries[0]