Count attachment transport failures and preserve updated article bodies

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -400,6 +400,10 @@ class USOfficialSpider(NewsSpider):
                         self.block_until(until)
                 except (ValueError, TypeError, OverflowError):
                     self.logger.warning('忽略无效 Retry-After 响应头')
+        if self.name in ('sec', 'doj', 'congress') and (parsed := failure.request.meta.get('parsed')):
+            attachment = next(a for a in parsed['extra']['attachments'] if a['status'] == 'pending')
+            attachment.update(status='unavailable', error=f'请求失败：{status or repr(failure.value)}')
+            return list(self.attach_or_finish(parsed, failure.request.meta['lane']))
         if status not in (403, 429) or self.name in ('sec', 'doj', 'congress'):
             requests = self.document_failed(failure.request, f'请求失败 {failure.request.url}：{status or repr(failure.value)}')
             if requests is not None:
@@ -650,9 +654,11 @@ class USOfficialSpider(NewsSpider):
             self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
 
     def attach_or_finish(self, parsed, lane):
-        if self.cursor.get('attachment_retries', {}).get(parsed['url'], {}).get('retired_at'):
-            yield from self.complete(lane)
-            return
+        retired = self.cursor.get('attachment_retries', {}).get(parsed['url'], {})
+        if retired.get('retired_at'):
+            for attachment in parsed['extra']['attachments']:
+                if attachment['status'] == 'pending':
+                    attachment.update(status='unavailable', error=retired['error'])
         pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
         if pending:
             request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
@@ -665,10 +671,10 @@ class USOfficialSpider(NewsSpider):
         incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
         retry_doc = parsed.pop('retry_document', None)
         retries = self.cursor.setdefault('attachment_retries', {})
-        if incomplete and retry_doc:
+        if incomplete and retry_doc and not retired.get('retired_at'):
             previous = retries.get(parsed['url'], {})
             error = '；'.join(f"{a['url']}：{a.get('error') or '附件尚未完成文字提取'}" for a in parsed['extra']['attachments'] if a['status'] != 'complete')
-            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error}
+            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error, 'failed_urls': [a['url'] for a in parsed['extra']['attachments'] if a['status'] != 'complete']}
         elif not incomplete:
             retries.pop(parsed['url'], None)
         self.retire_exhausted()

backend/tests/test_us_official_news.py

@@ -1348,11 +1348,18 @@ def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_
 def test_terminal_news_gap_survives_rediscovery(trading):
     obj = spider(trading, 'doj')
     url = 'https://www.justice.gov/opa/pr/example'
-    obj.cursor['attachment_retries'] = {url: {'attempts': 3, 'error': 'unsupported image'}}
+    doc = {'url': url, 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.run.history_deadline = time.monotonic() - 1
+    obj.cursor['attachment_retries'] = {url: {'attempts': 3, 'error': 'unsupported image', 'failed_urls': [url + '.png']}}
     obj.retire_exhausted()
+    parsed = {'url': url, 'title': 'Crypto exchange regulation', 'external_id': 'x', 'content_html': '<p>Updated cryptocurrency exchange regulations.</p>',
+              'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
+              'extra': {'attachments': [{'url': url + '.png', 'title': 'Document', 'status': 'pending'}]}}
+    item, = obj.attach_or_finish(parsed, 'history')
+    assert 'Updated' in item['content_html']
+    assert obj.cursor['attachment_retries'][url]['attempts'] == 3
     assert obj.cursor['attachment_retries'][url]['retired_at']
-    obj.complete = lambda lane: []
-    assert list(obj.attach_or_finish({'url': url}, 'realtime')) == []
 
 
 def test_selected_news_requests_have_one_retry_owner(trading):
@@ -1386,3 +1393,21 @@ def test_sec_uses_official_dateline_not_last_reviewed_date():
     item = article('sec', body, 'https://www.sec.gov/newsroom/press-releases/example')
     assert item['published_at'] == datetime(2026, 9, 30, tzinfo=UTC)
     assert item['extra']['published_at_precision'] == 'date'
+
+
+def test_attachment_http_failure_spends_durable_retry_budget(trading):
+    obj = spider(trading, 'doj')
+    obj.run.history_deadline = time.monotonic() - 1
+    url = 'https://www.justice.gov/opa/pr/example'
+    doc = {'url': url, 'kind': 'doj'}
+    obj.cursor['history']['pending'] = [doc]
+    parsed = {'url': url, 'title': 'Crypto exchange regulation', 'external_id': 'x', 'content_html': '<p>Cryptocurrency exchange regulations.</p>',
+              'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
+              'extra': {'attachments': [{'url': 'https://www.justice.gov/media/1/dl', 'title': 'Document', 'status': 'pending'}]}}
+    request = Request('https://www.justice.gov/media/1/dl', meta={'lane': 'history', 'parsed': parsed})
+    failure = Failure(HttpError(Response(request.url, status=503, request=request)))
+    failure.request = request
+    item, = obj.on_request_error(failure)
+    assert item['extra']['content_scope'] == 'partial'
+    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
+    assert '503' in obj.cursor['attachment_retries'][url]['error']