Repair official collection and retire exhausted material tasks

backend/src/alphapulse/crawler/repair_collection.py

@@ -0,0 +1,51 @@
+"""本次采集修复初始化：默认只读，--execute 幂等建立一次新预算。运行前停止 crawler。"""
+import argparse
+import asyncio
+import copy
+import json
+from datetime import UTC, datetime
+
+from sqlalchemy import select, text
+
+from alphapulse.core.db import SessionLocal, engine
+from alphapulse.models import Source
+
+REPAIR = 'collection-repair-20261105-v1'
+CODES = ('sec', 'doj', 'congress')
+
+
+async def run(execute=False):
+    output = []
+    async with SessionLocal() as session:
+        if not execute:
+            await session.execute(text('SET TRANSACTION READ ONLY'))
+        query = select(Source).where(Source.code.in_(CODES)).order_by(Source.code)
+        if execute:
+            query = query.with_for_update()
+        for source in await session.scalars(query):
+            cursor = copy.deepcopy(source.history_cursor or {})
+            if cursor.get('collection_repair') == REPAIR:
+                output.append({'source': source.code, 'already_initialized': True})
+                continue
+            affected = []
+            for key in ('document_retries', 'attachment_retries', 'bill_retries'):
+                for url, entry in cursor.get(key, {}).items():
+                    affected.append({'kind': key, 'url': url, 'previous_attempts': entry['attempts']})
+                    entry['repair_previous'] = {k: v for k, v in entry.items() if k != 'document'}
+                    entry['attempts'] = 0
+                    entry['next_at'] = datetime.now(UTC).timestamp()
+                    entry.pop('retired_at', None)
+            cursor['collection_repair'] = REPAIR
+            if execute:
+                source.history_cursor = cursor
+            output.append({'source': source.code, 'items': affected})
+        if execute:
+            await session.commit()
+    await engine.dispose()
+    return {'mode': 'execute' if execute else 'dry-run', 'sources': output}
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--execute', action='store_true')
+    print(json.dumps(asyncio.run(run(parser.parse_args().execute)), ensure_ascii=False))

backend/src/alphapulse/crawler/spiders/base.py

@@ -253,16 +253,22 @@ class NewsSpider(RedisSpider):
             if history_saved:
                 source.history_position = run.history_position
                 source.history_done = run.history_done
-            if failed:
+            bounded = self.name in ('sec', 'doj', 'congress')
+            source_failed = failed and (not bounded or not run.stats.get('source_readable'))
+            if source_failed:
                 source.consecutive_failures += 1
                 delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
                 if run.cooldown_until:
                     delay = max(delay, int((run.cooldown_until - now).total_seconds()) + 1)
+                if bounded:
+                    delay = max(delay, 3600)
+                    if source.consecutive_failures >= 3:
+                        source.enabled = False
                 source.paused_until = now + timedelta(seconds=delay)
                 source.last_error = run.errors[0][:1000]
             else:
                 source.consecutive_failures = 0
-                source.paused_until = None
+                source.paused_until = run.cooldown_until if bounded else None
                 source.last_error = None
                 source.last_success_at = now
             await session.commit()
@@ -270,6 +276,7 @@ class NewsSpider(RedisSpider):
         elapsed = (now - run.started_at).total_seconds()
         summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
         if failed:
+            delay = max(0, int((source.paused_until - now).total_seconds())) if source.paused_until else 0
             self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
         else:
             self.logger.info("本轮完成：%s", summary)

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -6,6 +6,7 @@ import time
 import uuid
 from datetime import UTC, datetime, timedelta
 from html import escape, unescape
+from email.utils import parsedate_to_datetime
 
 from lxml import etree
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -155,6 +156,8 @@ class USOfficialSpider(NewsSpider):
     def request_for(self, url, callback, lane, **meta):
         url = official_url(self.name, url)
         request = self.request(url, callback, lane=lane, official_source=self.name, **meta)
+        if self.name in ('sec', 'doj', 'congress'):
+            request.meta['dont_retry'] = True
         request.priority = 10 if lane == 'realtime' else -10
         # Credentials never travel to document hosts or into persisted URLs.
         if urlsplit(url).hostname == 'api.congress.gov':
@@ -317,6 +320,7 @@ class USOfficialSpider(NewsSpider):
             docs = list({d['url']: d for d in docs}.values())
             if not docs:
                 raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        self.run.stats['source_readable'] = 1
         if self.name == 'fed':
             begin = self.run.history_target if lane == 'history' else self.run.cutoff
             def in_range(doc):
@@ -380,11 +384,23 @@ class USOfficialSpider(NewsSpider):
         self.run.errors.append(error)
         self.logger.error('文档缺口：%s；%s', doc['url'], error)
         state['pending'].pop(0)
+        self.retire_exhausted()
         return self.advance(request.meta['lane'])
 
     def on_request_error(self, failure):
         status = failure.value.response.status if failure.check(HttpError) else None
-        if status not in (403, 429):
+        if self.name in ('sec', 'doj', 'congress') and status in (403, 429, 503):
+            if status in (403, 429):
+                self.block_until(datetime.now(UTC) + timedelta(hours=1))
+            value = failure.value.response.headers.get('Retry-After', b'').decode('ascii', errors='ignore')
+            if value:
+                try:
+                    until = datetime.now(UTC) + timedelta(seconds=int(value)) if value.isdigit() else parsedate_to_datetime(value)
+                    if until.tzinfo is not None:
+                        self.block_until(until)
+                except (ValueError, TypeError, OverflowError):
+                    self.logger.warning('忽略无效 Retry-After 响应头')
+        if status not in (403, 429) or self.name in ('sec', 'doj', 'congress'):
             requests = self.document_failed(failure.request, f'请求失败 {failure.request.url}：{status or repr(failure.value)}')
             if requests is not None:
                 return requests
@@ -539,6 +555,10 @@ class USOfficialSpider(NewsSpider):
                     file_id = attachment['file']['id']
                     if not str(file_id).isdigit():
                         raise ValueError('DOJ API 附件编号无效')
+                    if (f'/media/{file_id}' in (record.get('image') or '') and
+                            re.match(r'Photo[-\s]', attachment.get('alt', ''), re.I) and
+                            'left to right' in attachment.get('alt', '').lower()):
+                        continue  # Explicitly captioned illustration, not a text document.
                     api_attachments.append({'url': f'https://www.justice.gov/media/{file_id}/dl', 'title': attachment.get('description') or 'Official attachment', 'status': 'pending'})
                 attachment_html = ''
             elif not isinstance(attachment_html, str):
@@ -623,18 +643,23 @@ class USOfficialSpider(NewsSpider):
             retries.pop(context['url'], None)
             if not context['reasons']:
                 self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        self.retire_exhausted()
         self.run.stats['unresolved_bills'] = len(retries)
         self.cursor[lane]['pending'].pop(0)
         if lane == 'realtime':
             self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
 
     def attach_or_finish(self, parsed, lane):
+        if self.cursor.get('attachment_retries', {}).get(parsed['url'], {}).get('retired_at'):
+            yield from self.complete(lane)
+            return
         pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
         if pending:
             request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
             request.meta['handle_httpstatus_list'] = [404, 410]
             yield request
             return
+        self.retire_exhausted()
         text = html_to_text(parsed['content_html'])
         reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
         incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
@@ -646,6 +671,7 @@ class USOfficialSpider(NewsSpider):
             retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error}
         elif not incomplete:
             retries.pop(parsed['url'], None)
+        self.retire_exhausted()
         if reasons:
             parsed['extra']['selection_reasons'] = reasons
             parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
@@ -676,6 +702,14 @@ class USOfficialSpider(NewsSpider):
             self.logger.warning('附件缺口：%s', response.url)
         yield from self.attach_or_finish(parsed, response.meta['lane'])
 
+    def retire_exhausted(self):
+        if self.name not in ('sec', 'doj', 'congress'):
+            return
+        for key in ('document_retries', 'attachment_retries', 'bill_retries'):
+            for entry in self.cursor.get(key, {}).values():
+                if entry['attempts'] >= 3:
+                    entry.setdefault('retired_at', datetime.now(UTC).isoformat())
+
     def complete(self, lane):
         doc = self.cursor[lane]['pending'].pop(0)
         self.cursor.get('document_retries', {}).pop(doc['url'], None)