From 6035ed8332489cd36705100e61404ae25599bb65 Mon Sep 17 00:00:00 2001 From: eezd Date: Thu, 5 Nov 2026 13:48:23 +0800 Subject: [PATCH] Count attachment transport failures and preserve updated article bodies --- .../alphapulse/crawler/spiders/us_official.py | 16 +++++++--- backend/tests/test_us_official_news.py | 31 +++++++++++++++++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/backend/src/alphapulse/crawler/spiders/us_official.py b/backend/src/alphapulse/crawler/spiders/us_official.py index 02cf08d..7ebb794 100644 --- a/backend/src/alphapulse/crawler/spiders/us_official.py +++ b/backend/src/alphapulse/crawler/spiders/us_official.py @@ -400,6 +400,10 @@ class USOfficialSpider(NewsSpider): self.block_until(until) except (ValueError, TypeError, OverflowError): self.logger.warning('忽略无效 Retry-After 响应头') + if self.name in ('sec', 'doj', 'congress') and (parsed := failure.request.meta.get('parsed')): + attachment = next(a for a in parsed['extra']['attachments'] if a['status'] == 'pending') + attachment.update(status='unavailable', error=f'请求失败：{status or repr(failure.value)}') + return list(self.attach_or_finish(parsed, failure.request.meta['lane'])) if status not in (403, 429) or self.name in ('sec', 'doj', 'congress'): requests = self.document_failed(failure.request, f'请求失败 {failure.request.url}：{status or repr(failure.value)}') if requests is not None: @@ -650,9 +654,11 @@ class USOfficialSpider(NewsSpider): self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp() def attach_or_finish(self, parsed, lane): - if self.cursor.get('attachment_retries', {}).get(parsed['url'], {}).get('retired_at'): - yield from self.complete(lane) - return + retired = self.cursor.get('attachment_retries', {}).get(parsed['url'], {}) + if retired.get('retired_at'): + for attachment in parsed['extra']['attachments']: + if attachment['status'] == 'pending': + attachment.update(status='unavailable', error=retired['error']) pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None) if pending: request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed) @@ -665,10 +671,10 @@ class USOfficialSpider(NewsSpider): incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments']) retry_doc = parsed.pop('retry_document', None) retries = self.cursor.setdefault('attachment_retries', {}) - if incomplete and retry_doc: + if incomplete and retry_doc and not retired.get('retired_at'): previous = retries.get(parsed['url'], {}) error = '；'.join(f"{a['url']}：{a.get('error') or '附件尚未完成文字提取'}" for a in parsed['extra']['attachments'] if a['status'] != 'complete') - retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error} + retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error, 'failed_urls': [a['url'] for a in parsed['extra']['attachments'] if a['status'] != 'complete']} elif not incomplete: retries.pop(parsed['url'], None) self.retire_exhausted() diff --git a/backend/tests/test_us_official_news.py b/backend/tests/test_us_official_news.py index b967415..736c098 100644 --- a/backend/tests/test_us_official_news.py +++ b/backend/tests/test_us_official_news.py @@ -1348,11 +1348,18 @@ def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_ def test_terminal_news_gap_survives_rediscovery(trading): obj = spider(trading, 'doj') url = 'https://www.justice.gov/opa/pr/example' - obj.cursor['attachment_retries'] = {url: {'attempts': 3, 'error': 'unsupported image'}} + doc = {'url': url, 'kind': 'article'} + obj.cursor['history']['pending'] = [doc] + obj.run.history_deadline = time.monotonic() - 1 + obj.cursor['attachment_retries'] = {url: {'attempts': 3, 'error': 'unsupported image', 'failed_urls': [url + '.png']}} obj.retire_exhausted() + parsed = {'url': url, 'title': 'Crypto exchange regulation', 'external_id': 'x', 'content_html': '

Updated cryptocurrency exchange regulations.

', + 'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc, + 'extra': {'attachments': [{'url': url + '.png', 'title': 'Document', 'status': 'pending'}]}} + item, = obj.attach_or_finish(parsed, 'history') + assert 'Updated' in item['content_html'] + assert obj.cursor['attachment_retries'][url]['attempts'] == 3 assert obj.cursor['attachment_retries'][url]['retired_at'] - obj.complete = lambda lane: [] - assert list(obj.attach_or_finish({'url': url}, 'realtime')) == [] def test_selected_news_requests_have_one_retry_owner(trading): @@ -1386,3 +1393,21 @@ def test_sec_uses_official_dateline_not_last_reviewed_date(): item = article('sec', body, 'https://www.sec.gov/newsroom/press-releases/example') assert item['published_at'] == datetime(2026, 9, 30, tzinfo=UTC) assert item['extra']['published_at_precision'] == 'date' + + +def test_attachment_http_failure_spends_durable_retry_budget(trading): + obj = spider(trading, 'doj') + obj.run.history_deadline = time.monotonic() - 1 + url = 'https://www.justice.gov/opa/pr/example' + doc = {'url': url, 'kind': 'doj'} + obj.cursor['history']['pending'] = [doc] + parsed = {'url': url, 'title': 'Crypto exchange regulation', 'external_id': 'x', 'content_html': '

Cryptocurrency exchange regulations.

', + 'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc, + 'extra': {'attachments': [{'url': 'https://www.justice.gov/media/1/dl', 'title': 'Document', 'status': 'pending'}]}} + request = Request('https://www.justice.gov/media/1/dl', meta={'lane': 'history', 'parsed': parsed}) + failure = Failure(HttpError(Response(request.url, status=503, request=request))) + failure.request = request + item, = obj.on_request_error(failure) + assert item['extra']['content_scope'] == 'partial' + assert obj.cursor['attachment_retries'][url]['attempts'] == 1 + assert '503' in obj.cursor['attachment_retries'][url]['error']