From dcad5b76a85c62bb909ea9ee727bd3eb48211922 Mon Sep 17 00:00:00 2001 From: eezd Date: Thu, 5 Nov 2026 11:58:41 +0800 Subject: [PATCH] Reuse Federal Register discovery checkpoints across rounds --- .../alphapulse/crawler/spiders/us_official.py | 15 +++++++-- backend/tests/test_us_official_news.py | 32 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/backend/src/alphapulse/crawler/spiders/us_official.py b/backend/src/alphapulse/crawler/spiders/us_official.py index a5aaaab..5b79ad4 100644 --- a/backend/src/alphapulse/crawler/spiders/us_official.py +++ b/backend/src/alphapulse/crawler/spiders/us_official.py @@ -116,6 +116,14 @@ class USOfficialSpider(NewsSpider): if error: self.run.errors.append(error) return [] + if self.name == 'federal_register': + # Existing checkpoints identify completed bodies rather than discovery metadata. + checked = self.cursor.setdefault('checked', {}) + for url, at in list(checked.items()): + match = re.fullmatch(r'https://www\.federalregister\.gov/documents/full_text/html/\d{4}/\d{2}/\d{2}/([A-Za-z0-9-]+)\.html', url) + if match: + metadata = f'https://www.federalregister.gov/api/v1/documents/{match[1]}.json' + checked[metadata] = max(checked.get(metadata, 0), at) if self.name == 'cftc': # Earlier discovery treated commissioner indexes as article gaps. retries = self.cursor.get('document_retries', {}) @@ -504,7 +512,7 @@ class USOfficialSpider(NewsSpider): url = data.get('body_html_url') or data.get('raw_text_url') if not url: raise ValueError('规则文档缺少正文地址') - self.cursor[lane]['pending'][0] = {'url': official_url(self.name, url), 'kind': 'text', 'title': data['title'], 'at': data['publication_date'], 'canonical': data['html_url'], 'document_number': data['document_number'], 'effective_on': data.get('effective_on'), 'pdf': data.get('pdf_url')} + self.cursor[lane]['pending'][0] = {'url': official_url(self.name, url), 'kind': 'text', 'title': data['title'], 'at': data['publication_date'], 'canonical': data['html_url'], 'document_number': data['document_number'], 'effective_on': data.get('effective_on'), 'pdf': data.get('pdf_url'), 'metadata_url': response.url} for result in self.advance(lane): yield result return @@ -666,7 +674,10 @@ class USOfficialSpider(NewsSpider): doc = self.cursor[lane]['pending'].pop(0) self.cursor.get('document_retries', {}).pop(doc['url'], None) if lane == 'realtime': - self.cursor.setdefault('checked', {})[doc['url']] = self.run.started_at.timestamp() + checked = self.cursor.setdefault('checked', {}) + checked[doc['url']] = self.run.started_at.timestamp() + if doc.get('metadata_url'): + checked[doc['metadata_url']] = self.run.started_at.timestamp() return self.advance(lane) diff --git a/backend/tests/test_us_official_news.py b/backend/tests/test_us_official_news.py index 6b55ede..e504ce1 100644 --- a/backend/tests/test_us_official_news.py +++ b/backend/tests/test_us_official_news.py @@ -49,6 +49,38 @@ def test_real_empty_register_day_continues_to_previous_day(trading): assert not obj.run.errors +@pytest.mark.parametrize('recent', [False, True]) +def test_register_completed_body_checkpoint_matches_discovery_after_restart(trading, recent): + obj = spider(trading, 'federal_register') + body = 'https://www.federalregister.gov/documents/full_text/html/2026/11/03/2026-22248.html' + metadata = 'https://www.federalregister.gov/api/v1/documents/2026-22248.json' + obj.cursor['checked'] = {body: (obj.run.started_at - timedelta(hours=1 if recent else 25)).timestamp()} + obj.first_requests() + listing = json.dumps({'count': 1, 'results': [{'document_number': '2026-22248'}]}) + requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], listing, lane='realtime'))) + assert any(r.url == metadata for r in requests) is not recent + + +def test_register_completes_metadata_only_after_body_and_pdf(trading, monkeypatch): + from alphapulse.crawler.spiders import us_official + obj = spider(trading, 'federal_register') + metadata = 'https://www.federalregister.gov/api/v1/documents/2026-22248.json' + body = 'https://www.federalregister.gov/documents/full_text/html/2026/11/03/2026-22248.html' + pdf = 'https://www.govinfo.gov/content/pkg/FR-2026-11-03/pdf/2026-22248.pdf' + doc = {'url': metadata, 'kind': 'register'} + obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES[obj.name].entries)) + data = {'body_html_url': body, 'title': 'Digital asset exchange rule', 'publication_date': '2026-11-03', 'html_url': 'https://www.federalregister.gov/documents/2026/11/03/2026-22248/rule', 'document_number': '2026-22248', 'pdf_url': pdf} + async def parse(res): + return [item async for item in obj.parse_document(res)] + body_request, = asyncio.run(parse(response(metadata, json.dumps(data), lane='realtime', document=doc))) + assert metadata not in obj.cursor.get('checked', {}) + pdf_request, = asyncio.run(parse(response(body, '

This official rule regulates digital asset exchanges and financial institutions.

', **body_request.meta))) + assert metadata not in obj.cursor.get('checked', {}) + monkeypatch.setattr(us_official, 'pdf_text', lambda value: 'The official digital asset exchange rule and supporting material.') + list(obj.parse_attachment(Response(pdf, body=b'%PDF-test', request=pdf_request.replace(meta={**pdf_request.meta, 'raw_response_id': 13})))) + assert metadata in obj.cursor['checked'] and body in obj.cursor['checked'] + + @pytest.mark.parametrize('data', [{'count': 2}, {'count': '0', 'description': 'x'}, {'count': 0}, {'count': 0, 'description': 'x', 'results': None}]) def test_register_does_not_hide_malformed_responses(trading, data): obj = spider(trading, 'federal_register')