Repair official collection and retire exhausted material tasks

backend/src/alphapulse/api/crawler.py

@@ -65,6 +65,8 @@ class CalendarSourceOut(BaseModel):
     code: str
     name: str
     region_code: str
+    enabled: bool
+    retired_count: int
     worker_alive: bool
     heartbeat_at: datetime | None
     schedule_checked_at: datetime | None
@@ -124,13 +126,14 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
     now = datetime.now(UTC)
     raw = await redis.get(f"{KEY_PREFIX}calendar:status")
     heartbeat = json.loads(raw) if raw else None
-    pending = EconomicRelease.first_received_at.is_(None) & (EconomicRelease.schedule_status == "scheduled")
+    pending = EconomicRelease.first_received_at.is_(None) & (EconomicRelease.schedule_status == "scheduled") & EconomicRelease.collection_retired_at.is_(None)
     published = EconomicRelease.first_received_at.is_not(None)
     rows = (await session.execute(select(
         EconomicRelease.source,
         func.max(EconomicRelease.schedule_checked_at).label("schedule_success_at"),
         func.max(EconomicRelease.first_received_at).label("latest_result_at"),
         func.min(EconomicRelease.scheduled_at).filter(pending & (EconomicRelease.scheduled_at > now)).label("next_event_at"),
+        func.count().filter(EconomicRelease.collection_retired_at.is_not(None)).label("retired_count"),
         func.count().filter(pending & EconomicRelease.scheduled_at.between(now - timedelta(days=45), now)).label("overdue_count"),
         func.min(EconomicRelease.scheduled_at).filter(published).label("history_start"),
         func.max(EconomicRelease.scheduled_at).filter(published).label("history_end"),
@@ -138,7 +141,7 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
     stats = {row["source"]: dict(row) for row in rows}
     result_errors = {}
     for source, error in await session.execute(select(EconomicRelease.source, EconomicRelease.error).where(
-        EconomicRelease.error.is_not(None), EconomicRelease.schedule_status == "scheduled",
+        EconomicRelease.error.is_not(None), EconomicRelease.collection_retired_at.is_(None), EconomicRelease.schedule_status == "scheduled",
         EconomicRelease.scheduled_at >= now - timedelta(days=45),
     ).order_by(EconomicRelease.last_checked_at.desc())):
         result_errors.setdefault(source, error)
@@ -158,9 +161,12 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
         errors = [s["error"] for s in checks if s["error"]]
         if source.code in result_errors:
             errors.append(result_errors[source.code])
+        if not source.enabled and source.last_error:
+            errors.append(source.last_error)
         data = stats.get(source.code, {})
         output.append(CalendarSourceOut(
             code=source.code, name=source.name, region_code=REGIONS[source.code], worker_alive=source_alive,
+            enabled=source.enabled, retired_count=data.get("retired_count", 0),
             heartbeat_at=current_heartbeat["at"] if current_heartbeat else None,
             schedule_checked_at=max((s["checked_at"] for s in checks), default=None),
             **{key: data.get(key) for key in ("schedule_success_at", "latest_result_at", "next_event_at", "history_start", "history_end")},
@@ -197,6 +203,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
         bills = (source.history_cursor or {}).get('bill_retries', {})
         bill_pending = sum(value['attempts'] < 3 for value in bills.values())
         attachment_stats[source.code] = {
+            'retired_count': sum(bool(v.get('retired_at')) for group in (retries, documents, bills) for v in group.values()),
+            'active_gaps': sum(not v.get('retired_at') for group in (retries, documents, bills) for v in group.values()),
             'attachment_gaps': len(retries),
             'attachment_retry_pending': pending,
             'attachment_retry_exhausted': len(retries) - pending,

backend/src/alphapulse/calendar/eia.py

@@ -77,7 +77,7 @@ def archive_links(body: str, start: date, end: date) -> list[str]:
 
 def csv_link(body: str, base: str) -> str:
     links = [urljoin(base, link) for link in Selector(text=body).css("a[href]::attr(href)").getall()
-             if link == "download/table1.csv"]
+             if link in ("download/table1.csv", "csv/table1.csv")]
     if len(set(links)) != 1:
         raise ValueError("EIA 归档未唯一列示 Table 1 CSV")
     return links[0]

backend/src/alphapulse/calendar/service.py

@@ -109,6 +109,8 @@ def save_result(session: AsyncSession, row: EconomicRelease, values: dict, url:
     if ((period is not None and row.period is not None and period != row.period)
             or (stage is not None and row.stage is not None and stage != row.stage)):
         raise ValueError("公告统计期或发布阶段与日历不符")
+    row.collection_failures = 0
+    row.collection_retired_at = None
     if row.values != values or row.release_url != url or (period is not None and row.period != period) or (stage is not None and row.stage != stage):
         row.values = values
         row.release_url = url
@@ -177,6 +179,8 @@ def release_rows(row: EconomicRelease, now: datetime) -> list[dict]:
         state = "withdrawn"
     elif row.first_received_at is not None:
         state = "released"
+    elif getattr(row, "collection_retired_at", None) is not None:
+        state = "retired"
     elif now - row.scheduled_at > timedelta(days=45):
         state = "historical"
     elif row.error:
@@ -214,7 +218,7 @@ def release_rows(row: EconomicRelease, now: datetime) -> list[dict]:
                        "impact_basis": "forecast" if forecast is not None else "previous" if previous is not None else None,
                        "impact": direction, "explanation": explanation, "release_url": row.release_url,
                        "schedule_url": row.schedule_url, "first_received_at": row.first_received_at,
-                       "last_checked_at": row.last_checked_at, "error": row.error, "revision_no": row.revision_no})
+                       "last_checked_at": row.last_checked_at, "collection_retired_at": getattr(row, "collection_retired_at", None), "error": row.error, "revision_no": row.revision_no})
     return result
 
 

backend/src/alphapulse/calendar/worker.py

@@ -11,20 +11,21 @@ import httpx
 from parsel import Selector
 from sqlalchemy import select
 
-from alphapulse.calendar.catalog import SCHEDULE_URLS, UMICH_CSV, UMICH_HISTORY, UMICH_HOME, UMICH_INFO
+from alphapulse.calendar.catalog import SCHEDULE_URLS, UMICH_CSV, UMICH_HISTORY, UMICH_HOME, UMICH_INFO, UMICH_REPORTS
 from alphapulse.calendar import adp, dol, eia, europe
-from alphapulse.calendar.archives import ArchiveMissing
+from alphapulse.calendar.archives import ArchiveMissing, parse_umich_report
 from alphapulse.calendar.parsers import EASTERN, MONTHS, bea_release_link, parse_fed_calendar, parse_ics, parse_values, release_date, bls_periods, release_metadata
 from alphapulse.calendar.service import apply_bls_periods, record_revision, next_poll, save_raw, save_result, save_schedule, save_series
-from alphapulse.calendar.umich import current_result, document_links, monthly_series, parse_schedule, pdf_text
+from alphapulse.calendar.umich import current_result, document_links, monthly_series, parse_schedule, pdf_text, report_links
 from alphapulse.core.db import SessionLocal
 from alphapulse.core.official_sources import FEEDS, HOSTS, feed_entries
 from alphapulse.core.redis import KEY_PREFIX, redis
-from alphapulse.models import EconomicRelease
+from alphapulse.models import EconomicRelease, Source
 
 logger = logging.getLogger("alphapulse.calendar")
 STATUS_KEY = f"{KEY_PREFIX}calendar:status"
 HISTORY_DAYS = 45
+BOUNDED_SOURCES = frozenset({"dol", "eia", "umich", "adp"})
 
 
 def retry_after_at(value: str | None, now: datetime) -> datetime | None:
@@ -95,12 +96,13 @@ class CalendarWorker:
         self.index_cache[source, url] = (datetime.now(UTC) + timedelta(hours=1), page)
         return page
 
-    async def adp_links(self, start, end):
+    async def adp_links(self, start, end, first_page=None):
         url, visited = adp.INDEX, set()
         for _ in range(8):
             if url in visited: raise ValueError('ADP 官方分页形成循环')
             visited.add(url)
-            response, _ = await self.index('adp', url)
+            response, _ = first_page if first_page is not None else await self.index('adp', url)
+            first_page = None
             links, following = adp.release_links(response.text, url, start, end)
             if links or not following: return links
             url = following
@@ -112,7 +114,13 @@ class CalendarWorker:
         module = dol if source == 'dol' else eia
         index_url = dol.SCHEDULE if source == 'dol' else eia.ARCHIVE
         response, _ = await self.index(source, index_url)
-        links = module.archive_links(response.text, day, day)
+        try:
+            links = module.archive_links(response.text, day, day)
+        except ArchiveMissing:
+            if source != 'dol':
+                raise
+            # The index can lag behind published PDFs; validate the PDF date below.
+            links = [f'https://oui.doleta.gov/press/{day:%Y}/{day:%m%d%y}.pdf']
         if len(links) != 1: raise ValueError(f'{source} 当期官方归档不唯一')
         url = links[0]
         if source == 'eia':
@@ -140,7 +148,7 @@ class CalendarWorker:
             if delay > 0: await asyncio.sleep(delay)
             async with self.fetch_slots:
                 client = self.bls if source == "bls" else self.http
-                response = await client.post(url, json=query) if query is not None else await client.get(url)
+                response = await client.post(url, json=query) if query is not None else await client.get(url, **({"timeout": httpx.Timeout(20, connect=8)} if source == "adp" else {}))
             now = datetime.now(UTC)
             self.host_ready[host] = now + timedelta(seconds=1)
             if response.status_code in (403, 429, 503):
@@ -169,6 +177,10 @@ class CalendarWorker:
         urls += [("dol", dol.SCHEDULE)]
         urls += [("eia", eia.SCHEDULE)]
         urls += [("adp", adp.INDEX)]
+        async with SessionLocal() as session:
+            disabled = set(await session.scalars(select(Source.code).where(Source.code.in_(BOUNDED_SOURCES), Source.enabled.is_(False))))
+        urls = [(s, u) for s, u in urls if s not in disabled]
+        checked = {}
         # 启动时明确各来源尚未就绪；月份滚动后移除不再检查的旧月状态。
         self.sources = {url: self.sources.get(url, {"source": source, "checked_at": now.isoformat(), "error": "尚未同步"})
                         for source, url in urls}
@@ -184,7 +196,7 @@ class CalendarWorker:
                     response, raw_id = await self.fetch(source, fetch_url, query=europe.INSEE_QUERY if source == "insee" else None)
                 response.raise_for_status()
                 if source == "adp":
-                    links = await self.adp_links(datetime.min.date(), now.astimezone(EASTERN).date())
+                    links = await self.adp_links(datetime.min.date(), now.astimezone(EASTERN).date(), (response, raw_id))
                     if not links:
                         raise ValueError("ADP 官方新闻索引缺少近期月报")
                     report, evidence = await self.fetch(source, links[0])
@@ -223,8 +235,11 @@ class CalendarWorker:
                     async with SessionLocal() as session:
                         await save_schedule(session, source, url, events, raw_id, datetime.now(UTC))
                         await session.commit()
+                checked.setdefault(source, []).append(None)
                 self.sources[url] = {"source": source, "checked_at": datetime.now(UTC).isoformat(), "error": None}
             except (httpx.HTTPError, ValueError, KeyError, TypeError, IndexError, StopIteration) as exc:
+                if not isinstance(exc, SourceCooling):
+                    checked.setdefault(source, []).append(str(exc))
                 state = {"source": source, "checked_at": datetime.now(UTC).isoformat(), "error": str(exc)}
                 retry_at = getattr(exc, 'retry_at', None)
                 if isinstance(exc, httpx.HTTPStatusError):
@@ -237,6 +252,24 @@ class CalendarWorker:
                 self.sources[url] = state
                 logger.exception("日程采集失败：%s", url)
 
+        async with SessionLocal() as session:
+            for code, outcomes in checked.items():
+                if code not in BOUNDED_SOURCES:
+                    continue
+                source = await session.get(Source, code, with_for_update=True)
+                if source is None:
+                    continue
+                if None in outcomes:
+                    source.consecutive_failures = 0
+                    source.last_success_at = datetime.now(UTC)
+                    source.last_error = None
+                else:
+                    source.consecutive_failures += 1
+                    source.last_error = outcomes[0]
+                    if source.consecutive_failures >= 3:
+                        source.enabled = False
+            await session.commit()
+
     async def umich_schedule(self, info: str, now: datetime) -> None:
         history, _ = await self.fetch("umich", UMICH_HISTORY)
         history.raise_for_status()
@@ -351,7 +384,17 @@ class CalendarWorker:
                 return None
             if row.first_received_at is not None:
                 return None  # 最新页已换期；保留已取得的当期值，不误报历史缺口。
-            raise ValueError("密歇根当前页已晚于所选历史期次，未取得当期公告")
+            index, _ = await self.index('umich', f'{UMICH_REPORTS}?year={row.period[:4]}')
+            url = report_links(index.text, int(row.period[:4])).get((row.period, row.stage))
+            if url is None:
+                raise ArchiveMissing("密歇根官方年度归档尚未列出所选期次")
+            response, raw_id = await self.fetch('umich', url)
+            response.raise_for_status()
+            announcement = parse_umich_report(await asyncio.to_thread(pdf_text, response.content))
+            if (announcement.at, announcement.period, announcement.stage) != (row.scheduled_at, row.period, row.stage):
+                raise ValueError("密歇根归档日期、月份或阶段与所选期次不符")
+            response.extensions['calendar_announcement'] = announcement
+            return response, raw_id
         if row.source == "bls":
             code = "empsit" if row.family == "jobs" else row.family
             # 本机 BLS archive 路径返回 403，最新公告页可访问；下面必须核对正式发布日期。
@@ -439,6 +482,8 @@ class CalendarWorker:
     async def poll_one(self, release_id: int) -> None:
         async with SessionLocal() as session:
             row = await session.scalar(select(EconomicRelease).where(EconomicRelease.id == release_id).with_for_update())
+            if row is None or row.collection_retired_at is not None:
+                return
             now = datetime.now(UTC)
             retry_at = None
             try:
@@ -458,18 +503,29 @@ class CalendarWorker:
                         announcement = europe.parse_announcement(row.family, response.text)
                         period, stage, values = announcement.period, announcement.stage, announcement.values
                     elif row.family == "umich":
-                        period, stage, values = current_result(response.text)
+                        if announcement := response.extensions.get('calendar_announcement'):
+                            period, stage, values = announcement.period, announcement.stage, announcement.values
+                        else:
+                            period, stage, values = current_result(response.text)
                     else:
                         values = parse_values(row.family, response.text)
                         text = " ".join(Selector(text=response.text).css("h1,#bodytext").xpath("string(.)").getall())
                         period, stage = release_metadata(row.family, text)
                     save_result(session, row, values, response.extensions.get("calendar_url", str(response.url)), raw_id, datetime.now(UTC), period=period, stage=stage)
+                if page is not None:
+                    row.collection_failures = 0
                 row.error = None
             except (httpx.HTTPError, ValueError, KeyError, TypeError, IndexError) as exc:
                 row.error = f"{type(exc).__name__}: {exc}"
-                retry_at = getattr(exc, 'retry_at', None)
+                if row.source in BOUNDED_SOURCES and not isinstance(exc, SourceCooling):
+                    row.collection_failures += 1
+                    if row.collection_failures >= 3:
+                        row.collection_retired_at = datetime.now(UTC)
+                if row.source in BOUNDED_SOURCES:
+                    retry_at = datetime.now(UTC) + timedelta(hours=1)
+                retry_at = max(filter(None, (retry_at, getattr(exc, 'retry_at', None))), default=None)
                 if isinstance(exc, httpx.HTTPStatusError):
-                    retry_at = self.cooldowns.get(exc.request.url.host, retry_at)
+                    retry_at = max(filter(None, (retry_at, self.cooldowns.get(exc.request.url.host))), default=None)
                 if retry_at: row.error += f"；下次尝试 {retry_at.isoformat()}"
                 if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (429, 503):
                     retry_at = max(retry_at or datetime.now(UTC), retry_after_at(exc.response.headers.get("Retry-After"), datetime.now(UTC)) or datetime.now(UTC))
@@ -483,7 +539,8 @@ class CalendarWorker:
     async def results(self) -> None:
         now = datetime.now(UTC)
         async with SessionLocal() as session:
-            ids = list(await session.scalars(select(EconomicRelease.id).where(
+            ids = list(await session.scalars(select(EconomicRelease.id).join(Source, Source.code == EconomicRelease.source).where(
+                Source.enabled.is_(True), EconomicRelease.collection_retired_at.is_(None),
                 EconomicRelease.source != "jin10",
                 EconomicRelease.schedule_status == "scheduled", EconomicRelease.scheduled_at <= now,
                 EconomicRelease.scheduled_at >= now - timedelta(days=HISTORY_DAYS), EconomicRelease.next_check_at <= now,

backend/src/alphapulse/crawler/repair_collection.py

@@ -0,0 +1,51 @@
+"""本次采集修复初始化：默认只读，--execute 幂等建立一次新预算。运行前停止 crawler。"""
+import argparse
+import asyncio
+import copy
+import json
+from datetime import UTC, datetime
+
+from sqlalchemy import select, text
+
+from alphapulse.core.db import SessionLocal, engine
+from alphapulse.models import Source
+
+REPAIR = 'collection-repair-20261105-v1'
+CODES = ('sec', 'doj', 'congress')
+
+
+async def run(execute=False):
+    output = []
+    async with SessionLocal() as session:
+        if not execute:
+            await session.execute(text('SET TRANSACTION READ ONLY'))
+        query = select(Source).where(Source.code.in_(CODES)).order_by(Source.code)
+        if execute:
+            query = query.with_for_update()
+        for source in await session.scalars(query):
+            cursor = copy.deepcopy(source.history_cursor or {})
+            if cursor.get('collection_repair') == REPAIR:
+                output.append({'source': source.code, 'already_initialized': True})
+                continue
+            affected = []
+            for key in ('document_retries', 'attachment_retries', 'bill_retries'):
+                for url, entry in cursor.get(key, {}).items():
+                    affected.append({'kind': key, 'url': url, 'previous_attempts': entry['attempts']})
+                    entry['repair_previous'] = {k: v for k, v in entry.items() if k != 'document'}
+                    entry['attempts'] = 0
+                    entry['next_at'] = datetime.now(UTC).timestamp()
+                    entry.pop('retired_at', None)
+            cursor['collection_repair'] = REPAIR
+            if execute:
+                source.history_cursor = cursor
+            output.append({'source': source.code, 'items': affected})
+        if execute:
+            await session.commit()
+    await engine.dispose()
+    return {'mode': 'execute' if execute else 'dry-run', 'sources': output}
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--execute', action='store_true')
+    print(json.dumps(asyncio.run(run(parser.parse_args().execute)), ensure_ascii=False))

backend/src/alphapulse/crawler/spiders/base.py

@@ -253,16 +253,22 @@ class NewsSpider(RedisSpider):
             if history_saved:
                 source.history_position = run.history_position
                 source.history_done = run.history_done
-            if failed:
+            bounded = self.name in ('sec', 'doj', 'congress')
+            source_failed = failed and (not bounded or not run.stats.get('source_readable'))
+            if source_failed:
                 source.consecutive_failures += 1
                 delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
                 if run.cooldown_until:
                     delay = max(delay, int((run.cooldown_until - now).total_seconds()) + 1)
+                if bounded:
+                    delay = max(delay, 3600)
+                    if source.consecutive_failures >= 3:
+                        source.enabled = False
                 source.paused_until = now + timedelta(seconds=delay)
                 source.last_error = run.errors[0][:1000]
             else:
                 source.consecutive_failures = 0
-                source.paused_until = None
+                source.paused_until = run.cooldown_until if bounded else None
                 source.last_error = None
                 source.last_success_at = now
             await session.commit()
@@ -270,6 +276,7 @@ class NewsSpider(RedisSpider):
         elapsed = (now - run.started_at).total_seconds()
         summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
         if failed:
+            delay = max(0, int((source.paused_until - now).total_seconds())) if source.paused_until else 0
             self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
         else:
             self.logger.info("本轮完成：%s", summary)

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -6,6 +6,7 @@ import time
 import uuid
 from datetime import UTC, datetime, timedelta
 from html import escape, unescape
+from email.utils import parsedate_to_datetime
 
 from lxml import etree
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -155,6 +156,8 @@ class USOfficialSpider(NewsSpider):
     def request_for(self, url, callback, lane, **meta):
         url = official_url(self.name, url)
         request = self.request(url, callback, lane=lane, official_source=self.name, **meta)
+        if self.name in ('sec', 'doj', 'congress'):
+            request.meta['dont_retry'] = True
         request.priority = 10 if lane == 'realtime' else -10
         # Credentials never travel to document hosts or into persisted URLs.
         if urlsplit(url).hostname == 'api.congress.gov':
@@ -317,6 +320,7 @@ class USOfficialSpider(NewsSpider):
             docs = list({d['url']: d for d in docs}.values())
             if not docs:
                 raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        self.run.stats['source_readable'] = 1
         if self.name == 'fed':
             begin = self.run.history_target if lane == 'history' else self.run.cutoff
             def in_range(doc):
@@ -380,11 +384,23 @@ class USOfficialSpider(NewsSpider):
         self.run.errors.append(error)
         self.logger.error('文档缺口：%s；%s', doc['url'], error)
         state['pending'].pop(0)
+        self.retire_exhausted()
         return self.advance(request.meta['lane'])
 
     def on_request_error(self, failure):
         status = failure.value.response.status if failure.check(HttpError) else None
-        if status not in (403, 429):
+        if self.name in ('sec', 'doj', 'congress') and status in (403, 429, 503):
+            if status in (403, 429):
+                self.block_until(datetime.now(UTC) + timedelta(hours=1))
+            value = failure.value.response.headers.get('Retry-After', b'').decode('ascii', errors='ignore')
+            if value:
+                try:
+                    until = datetime.now(UTC) + timedelta(seconds=int(value)) if value.isdigit() else parsedate_to_datetime(value)
+                    if until.tzinfo is not None:
+                        self.block_until(until)
+                except (ValueError, TypeError, OverflowError):
+                    self.logger.warning('忽略无效 Retry-After 响应头')
+        if status not in (403, 429) or self.name in ('sec', 'doj', 'congress'):
             requests = self.document_failed(failure.request, f'请求失败 {failure.request.url}：{status or repr(failure.value)}')
             if requests is not None:
                 return requests
@@ -539,6 +555,10 @@ class USOfficialSpider(NewsSpider):
                     file_id = attachment['file']['id']
                     if not str(file_id).isdigit():
                         raise ValueError('DOJ API 附件编号无效')
+                    if (f'/media/{file_id}' in (record.get('image') or '') and
+                            re.match(r'Photo[-\s]', attachment.get('alt', ''), re.I) and
+                            'left to right' in attachment.get('alt', '').lower()):
+                        continue  # Explicitly captioned illustration, not a text document.
                     api_attachments.append({'url': f'https://www.justice.gov/media/{file_id}/dl', 'title': attachment.get('description') or 'Official attachment', 'status': 'pending'})
                 attachment_html = ''
             elif not isinstance(attachment_html, str):
@@ -623,18 +643,23 @@ class USOfficialSpider(NewsSpider):
             retries.pop(context['url'], None)
             if not context['reasons']:
                 self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        self.retire_exhausted()
         self.run.stats['unresolved_bills'] = len(retries)
         self.cursor[lane]['pending'].pop(0)
         if lane == 'realtime':
             self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
 
     def attach_or_finish(self, parsed, lane):
+        if self.cursor.get('attachment_retries', {}).get(parsed['url'], {}).get('retired_at'):
+            yield from self.complete(lane)
+            return
         pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
         if pending:
             request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
             request.meta['handle_httpstatus_list'] = [404, 410]
             yield request
             return
+        self.retire_exhausted()
         text = html_to_text(parsed['content_html'])
         reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
         incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
@@ -646,6 +671,7 @@ class USOfficialSpider(NewsSpider):
             retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error}
         elif not incomplete:
             retries.pop(parsed['url'], None)
+        self.retire_exhausted()
         if reasons:
             parsed['extra']['selection_reasons'] = reasons
             parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
@@ -676,6 +702,14 @@ class USOfficialSpider(NewsSpider):
             self.logger.warning('附件缺口：%s', response.url)
         yield from self.attach_or_finish(parsed, response.meta['lane'])
 
+    def retire_exhausted(self):
+        if self.name not in ('sec', 'doj', 'congress'):
+            return
+        for key in ('document_retries', 'attachment_retries', 'bill_retries'):
+            for entry in self.cursor.get(key, {}).values():
+                if entry['attempts'] >= 3:
+                    entry.setdefault('retired_at', datetime.now(UTC).isoformat())
+
     def complete(self, lane):
         doc = self.cursor[lane]['pending'].pop(0)
         self.cursor.get('document_retries', {}).pop(doc['url'], None)

backend/src/alphapulse/models/calendar.py

@@ -27,6 +27,8 @@ class EconomicRelease(Base):
     first_received_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     next_check_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    collection_failures: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    collection_retired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     error: Mapped[str | None] = mapped_column(Text)
     # 按指标代码保存实际值、官方报告内前值和引用；数字以十进制字符串存储。
     values: Mapped[dict] = mapped_column(JSONB, default=dict)