Add resumable US policy news collectors and provenance

Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled.

Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.

backend/alembic/versions/20261104_da1234567890_us_official_news.py

@@ -0,0 +1,25 @@
+"""Add disabled US official collectors and resumable provenance metadata."""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+revision = 'da1234567890'
+down_revision = 'c91234567890'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('sources', sa.Column('history_cursor', JSONB, nullable=False, server_default='{}'))
+    op.add_column('crawl_runs', sa.Column('stats', JSONB, nullable=False, server_default='{}'))
+    op.add_column('news_revisions', sa.Column('extra', JSONB, nullable=True))
+    table = sa.table('sources', *(sa.column(k) for k in ('code', 'name', 'list_url', 'adapter', 'collector', 'enabled', 'interval_seconds', 'backfill_hours', 'consecutive_failures')))
+    op.bulk_insert(table, [{'code': 'sec', 'name': 'SEC·证券监管', 'list_url': 'https://www.sec.gov/newsroom/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'cftc', 'name': 'CFTC·衍生品监管', 'list_url': 'https://www.cftc.gov/PressRoom/PressReleases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'treasury', 'name': '美国财政部', 'list_url': 'https://home.treasury.gov/news/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'ofac', 'name': 'OFAC·金融制裁', 'list_url': 'https://ofac.treasury.gov/recent-actions', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'fincen', 'name': 'FinCEN·反洗钱', 'list_url': 'https://www.fincen.gov/news/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'whitehouse', 'name': '白宫·政策', 'list_url': 'https://www.whitehouse.gov/presidential-actions/', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'congress', 'name': '美国国会·立法', 'list_url': 'https://api.congress.gov/v3/bill?format=json&limit=250', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 900, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'federal_register', 'name': 'Federal Register·规则', 'list_url': 'https://www.federalregister.gov/api/v1/documents.json', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 900, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'occ', 'name': 'OCC·银行与稳定币', 'list_url': 'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/news-releases/2026-news-releases.html', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'doj', 'name': 'DOJ·加密机构执法', 'list_url': 'https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}])
+
+
+def downgrade():
+    # Foreign keys intentionally reject removal of sources with collected data.
+    op.execute("DELETE FROM sources WHERE code IN ('sec','cftc','treasury','ofac','fincen','whitehouse','congress','federal_register','occ','doj')")
+    op.drop_column('news_revisions', 'extra')
+    op.drop_column('crawl_runs', 'stats')
+    op.drop_column('sources', 'history_cursor')

backend/src/alphapulse/ai/tools.py

@@ -22,6 +22,7 @@ from alphapulse.api.calendar import calendar_data
 from alphapulse.api.market import call_binance, client, tradable_symbols
 from alphapulse.api.news import news_filters
 from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import SOURCES as US_SOURCES
 from alphapulse.models import ContextVersion, NewsItem, SkillVersion
 from alphapulse.models.market import INTERVALS
 
@@ -34,6 +35,8 @@ SOURCE_NAMES = {
     "coindesk": "CoinDesk", "fed": "美联储·政策与讲话", "bls": "美国劳工统计局·CPI/PPI/就业", "bea": "美国经济分析局",
 }
 
+SOURCE_NAMES.update({code: spec.name for code, spec in US_SOURCES.items()})
+
 
 @dataclass
 class ToolOutcome:
@@ -59,7 +62,7 @@ def dump(data: Any) -> str:
 
 class SearchNews(BaseModel):
     query: Annotated[str, Field(description="关键字，按标题和正文字面匹配；比特币/Bitcoin、以太坊/Ethereum、美联储、CPI/PPI/非农/GDP/PCE 等完整主题词会同时匹配中英文；其他词请分别使用原文语言检索")]
-    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea"]] | None, Field(description="限定来源，不填为全部")] = None
+    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea", "sec", "cftc", "treasury", "ofac", "fincen", "whitehouse", "congress", "federal_register", "occ", "doj", "ecb", "boe", "eurostat", "ons", "destatis", "insee"]] | None, Field(description="限定来源，不填为全部")] = None
     start: Annotated[datetime | None, Field(description="开始时间（ISO 8601，带时区），如 2026-10-01T00:00:00+08:00")] = None
     end: Annotated[datetime | None, Field(description="结束时间（ISO 8601，带时区）")] = None
     limit: Annotated[int, Field(ge=1, le=20)] = 10
@@ -70,11 +73,11 @@ async def search_news(args: SearchNews) -> ToolOutcome:
         conditions = news_filters(args.sources, args.query, args.start, args.end)
         total = await session.scalar(select(func.count()).select_from(NewsItem).where(*conditions))
         rows = (await session.execute(
-            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200))
+            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200), NewsItem.extra)
             .where(*conditions).order_by(NewsItem.published_at.desc()).limit(args.limit)
         )).all()
     items = [
-        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": local(r[2]), "title": r[3], "brief": r[4] or r[5]}
+        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": r[6].get("published_date") if r[6].get("published_at_precision") == "date" else local(r[2]), "title": r[3], "brief": r[4] or r[5]}
         for r in rows
     ]
     return ToolOutcome(dump({"total": total, "returned": len(items), "items": items}))
@@ -90,10 +93,13 @@ async def get_news(args: GetNews) -> ToolOutcome:
     if news is None:
         return ToolOutcome(dump({"error": f"新闻 {args.id} 不存在"}))
     return ToolOutcome(dump({
-        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": local(news.published_at),
+        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": news.extra.get("published_date") if news.extra.get("published_at_precision") == "date" else local(news.published_at),
         "title": news.title, "url": news.url, "is_paywalled": news.is_paywalled,
         "content": news.content_text[:NEWS_BODY_LIMIT], "truncated": len(news.content_text) > NEWS_BODY_LIMIT,
         "content_scope": news.extra.get("content_scope", "article"),
+        "published_at_precision": news.extra.get("published_at_precision"),
+        "published_date": news.extra.get("published_date"),
+        "attachments": news.extra.get("attachments", []),
     }))
 
 

backend/src/alphapulse/api/crawler.py

@@ -19,6 +19,7 @@ from alphapulse.core.redis import KEY_PREFIX, redis
 from alphapulse.core.official_sources import REGIONS
 from alphapulse.calendar.catalog import FAMILIES
 from alphapulse.crawler.events import CONTROL_CHANNEL, HISTORY_SOURCES, LOG_CHANNEL, STATUS_KEY
+from alphapulse.crawler.spiders.us_official import configuration_error
 from alphapulse.models import CrawlLog, CrawlRun, EconomicRelease, NewsItem, Source
 
 router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
@@ -46,10 +47,13 @@ class SourceOut(BaseModel):
     last_success_at: datetime | None
     last_error: str | None
     paused_until: datetime | None
+    configuration_error: str | None = None
+    history_cursor: dict = Field(default_factory=dict)
     supports_history: bool
     history_target: datetime | None
     history_position: datetime | None
     history_done: bool
+    collection_stats: dict = Field(default_factory=dict)
     # 管理进程未运行时为 None
     process: ProcessStatus | None
     today_new: int
@@ -180,6 +184,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
     totals = dict((await session.execute(
         select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
     )).all())
+    latest_runs = (await session.scalars(select(CrawlRun).order_by(CrawlRun.source_code, CrawlRun.started_at.desc()).distinct(CrawlRun.source_code))).all()
+    run_stats = {run.source_code: run.stats for run in latest_runs}
     statuses = await redis.hgetall(STATUS_KEY)
     db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
     return success(SourcesOut(
@@ -191,6 +197,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
                 **s.__dict__,
                 "region_code": REGIONS.get(s.code),
                 "supports_history": s.code in HISTORY_SOURCES,
+                "configuration_error": configuration_error(s.code),
+                "collection_stats": run_stats.get(s.code, {}),
                 "process": json.loads(statuses[s.code]) if s.code in statuses else None,
                 "today_new": today_counts.get(s.code, (0, 0))[0],
                 "today_revised": today_counts.get(s.code, (0, 0))[1],
@@ -206,6 +214,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
     source = await session.get(Source, code)
     if source is None or source.collector != "news":
         raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    if body.enabled and (error := configuration_error(code)):
+        raise HTTPException(status.HTTP_409_CONFLICT, error)
     for field, value in body.model_dump(exclude_none=True, exclude={"history_target"}).items():
         setattr(source, field, value)
     if "history_target" in body.model_fields_set:
@@ -214,6 +224,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
         if body.history_target is not None and body.history_target >= datetime.now(LOCAL_TZ):
             raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "回补目标时间必须早于现在")
         # 保留断点：改目标或重新开始时从已回补的位置继续
+        if body.history_target is not None and (source.history_cursor or {}).get("history", {}).get("scanned"):
+            source.history_cursor = {k: v for k, v in source.history_cursor.items() if k != "history"}
         source.history_target = body.history_target
         source.history_done = False
     await session.commit()

backend/src/alphapulse/api/news.py

@@ -55,6 +55,8 @@ class NewsBrief(BaseModel):
     first_seen_at: datetime
     revision_no: int
     risk: NewsRiskBrief | None = None
+    published_at_precision: str | None = None
+    published_date: str | None = None
 
 
 class NewsPage(BaseModel):
@@ -71,6 +73,7 @@ class NewsDetail(NewsBrief):
 
 
 class Revision(BaseModel):
+    extra: dict | None = None
     revision_no: int
     title: str
     summary: str
@@ -131,7 +134,7 @@ async def list_news(
     items = []
     for news, excerpt, package in rows:
         risk = {'state': package.state} if package else None
-        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk}))
+        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk, 'published_at_precision': news.extra.get('published_at_precision'), 'published_date': news.extra.get('published_date')}))
     return success(NewsPage(items=items, total=total))
 
 
@@ -154,7 +157,7 @@ async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_se
     news = await session.get(NewsItem, news_id)
     if news is None:
         raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
-    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]}))
+    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH], "published_at_precision": news.extra.get("published_at_precision"), "published_date": news.extra.get("published_date")}))
 
 
 @router.get("/{news_id}/revisions")

backend/src/alphapulse/core/config.py

@@ -7,6 +7,9 @@ class Settings(BaseSettings):
     # 容器内由 compose 的 env_file 注入；只有一套配置，见仓库根目录 .env
     model_config = SettingsConfigDict(extra="ignore")
 
+    congress_api_key: str = ""
+    crawler_contact_email: str = ""
+
     searxng_url: str = "http://searxng:8080"
     database_url: str
     redis_url: str

backend/src/alphapulse/core/official_sources.py

@@ -98,3 +98,7 @@ def relevant_news(source: str, title: str, url: str) -> bool:
     if source == "insee":
         return any(t in title for t in ("prix à la consommation", "taux de chômage", "le pib", "le produit intérieur brut"))
     raise ValueError(f"未支持的欧洲新闻来源：{source}")
+
+# US adapters share the existing regional directory.
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+REGIONS.update({code: "US" for code in US_SOURCES})

backend/src/alphapulse/core/us_news.py

@@ -0,0 +1,171 @@
+"""Official US news catalog and deterministic collection policy."""
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+from parsel import Selector
+
+
+@dataclass(frozen=True)
+class OfficialSource:
+    name: str
+    entries: tuple[str, ...]
+    article_path: str
+    body: str
+    hosts: tuple[str, ...]
+    interval: int = 300
+
+
+SOURCES = {
+    'sec': OfficialSource('SEC·证券监管', ('https://www.sec.gov/newsroom/press-releases', 'https://www.sec.gov/newsroom/speeches-statements', 'https://www.sec.gov/rules-regulations/rulemaking-activity', 'https://www.sec.gov/rules-regulations/self-regulatory-organization-rulemaking'), r'/(?:newsroom/(?:press-releases|speeches-statements)/[^/]+|rules-regulations/.*\d.*)', '.node-details-layout__main-region__content .field--name-body', ('www.sec.gov',)),
+    'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+)|csl/)', 'article .field--name-body', ('www.cftc.gov',)),
+    'treasury': OfficialSource('美国财政部', ('https://home.treasury.gov/news/press-releases',), r'/news/press-releases/[a-z]+\d+/?$', '.field--name-field-news-body', ('home.treasury.gov', 'www.treasury.gov', 'ofac.treasury.gov')),
+    'ofac': OfficialSource('OFAC·金融制裁', ('https://ofac.treasury.gov/recent-actions',), r'/recent-actions/\d{8}[^/]*$', '.field--name-body, .body-content', ('ofac.treasury.gov', 'home.treasury.gov')),
+    'fincen': OfficialSource('FinCEN·反洗钱', ('https://www.fincen.gov/news/press-releases', 'https://www.fincen.gov/news/enforcement-actions', 'https://www.fincen.gov/resources/advisoriesbulletinsfact-sheets'), r'/(?:news/news-releases/[^/]+|resources/advisories/[^/]+|news/enforcement-actions/[^/]+)', 'article .field--name-body', ('www.fincen.gov',)),
+    'whitehouse': OfficialSource('白宫·政策', ('https://www.whitehouse.gov/presidential-actions/', 'https://www.whitehouse.gov/fact-sheets/', 'https://www.whitehouse.gov/briefings-statements/', 'https://bidenwhitehouse.archives.gov/briefing-room/'), r'/(?:presidential-actions|fact-sheets|briefings-statements|briefing-room/[^/]+)/202[5-9]/\d{2}/\d{2}/|/(?:presidential-actions|fact-sheets|briefings-statements)/202[5-9]/\d{2}/[^/]+', '.wp-block-post-content, .body-content, .entry-content', ('www.whitehouse.gov', 'bidenwhitehouse.archives.gov')),
+    'congress': OfficialSource('美国国会·立法', ('https://api.congress.gov/v3/bill?format=json&limit=250', 'https://api.congress.gov/v3/committee-meeting?format=json&limit=250'), r'/bill/', '', ('api.congress.gov', 'www.congress.gov', 'www.govinfo.gov'), 900),
+    'federal_register': OfficialSource('Federal Register·规则', ('https://www.federalregister.gov/api/v1/documents.json',), r'/documents/', '', ('www.federalregister.gov', 'public-inspection.federalregister.gov', 'www.govinfo.gov'), 900),
+    'occ': OfficialSource('OCC·银行与稳定币', tuple(f'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/{kind}/{year}-{kind}.html' for year in range(datetime.now(UTC).year, 2024, -1) for kind in ('news-releases', 'bulletins')), r'/news-issuances/(?:news-releases|bulletins|interpretations-and-precedents)/202[5-9]/[^/]+', '.occgov-issuance-content', ('www.occ.gov', 'occ.gov', 'www.occ.treas.gov', 'occ.treas.gov')),
+    'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)),
+    'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speeches', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
+}
+RULE_VERSION = 'us-policy-v1'
+CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I)
+POLICY = re.compile(r'\b(?:regulat\w*|rules?|guidance|legislation|acts?|sanctions?|licenses?|executive order|monetary policy|federal funds|interest rates?|balance sheet|liquidity|quantitative|refunding|borrowing|debt limit|tariffs?|fiscal|budget|appropriation|tax reform)\b', re.I)
+INSTITUTION = re.compile(r'\b(?:exchanges?|banks?|financial institutions?|payment processors?|custodians?|stablecoin issuers?|binance|coinbase|tether|circle|kraken|bybit|bitfinex)\b', re.I)
+MACRO = re.compile(r'\b(?:monetary policy|federal funds|interest rates?|fomc|balance sheet|liquidity|quantitative|refunding|borrowing estimates|debt (?:limit|ceiling)|tariffs?|fiscal|budget|appropriation|tax reform|financial sanctions)\b', re.I)
+
+
+def selection(source: str, title: str, text: str) -> list[str]:
+    content = f'{title}\n{text}'
+    crypto = bool(CRYPTO.search(content))
+    if crypto and (source not in ('doj', 'ofac', 'fincen') or POLICY.search(title) or INSTITUTION.search(content)):
+        return ['crypto_policy' if POLICY.search(content) else 'crypto_institution']
+    if source not in ('doj', 'cftc', 'sec') and MACRO.search(content):
+        return ['us_macro_policy']
+    if source == 'ofac' and INSTITUTION.search(content) and POLICY.search(content):
+        return ['financial_sanctions']
+    return []
+
+
+def official_url(source: str, url: str, base: str = '') -> str:
+    value = urlsplit(urljoin(base, url))
+    if value.scheme != 'https' or value.hostname not in SOURCES[source].hosts or value.username or value.password:
+        raise ValueError('链接不是该来源的官方 HTTPS 地址')
+    if re.search(r'(?:api_key|token|key)=', value.query, re.I):
+        raise ValueError('官方链接含凭据参数')
+    return urlunsplit((value.scheme, value.netloc, value.path, value.query, ''))
+
+
+def identity(url: str) -> str:
+    return hashlib.sha256(url.encode()).hexdigest()
+
+
+def published(value: str) -> tuple[datetime, str]:
+    if not isinstance(value, str):
+        raise ValueError('官方材料缺少可识别的发布时间')
+    value = value.strip().replace('\xa0', ' ')
+    if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
+        return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
+    try:
+        dt = datetime.fromisoformat(value.replace('Z', '+00:00'))
+        if dt.tzinfo:
+            return dt.astimezone(UTC), 'datetime'
+    except ValueError:
+        pass
+    match = re.search(r'([A-Z][a-z]+\.? \d{1,2}, \d{4}|\d{1,2}/\d{1,2}/\d{4})', value)
+    if match:
+        for fmt in ('%B %d, %Y', '%b %d, %Y', '%m/%d/%Y'):
+            try:
+                return datetime.strptime(match[0].replace('.', ''), fmt).replace(tzinfo=UTC), 'date'
+            except ValueError:
+                pass
+    raise ValueError('官方材料缺少可识别的发布时间')
+
+
+def validate_body(content: str) -> None:
+    """Reject error documents without requiring full-page HTML around fragments."""
+    if content.lstrip().startswith('<?xml'):
+        from lxml import etree
+        try:
+            etree.fromstring(content.encode(), etree.XMLParser(resolve_entities=False, no_network=True))
+        except etree.XMLSyntaxError as exc:
+            raise ValueError('官方 XML 正文无法解析') from exc
+    sel = Selector(content)
+    heading = ' '.join(sel.css('title, h1').xpath('normalize-space(.)').getall())
+    text = ' '.join(sel.xpath('//text()[not(ancestor::script or ancestor::style)]').getall()).strip()
+    if re.search(r'just a moment|access denied|verify (?:you are|your)|sign in|log in|captcha|page not found|service unavailable|bad gateway', heading, re.I) or sel.css('#challenge-form, #cf-challenge-running, script[src*="/cdn-cgi/challenge"]'):
+        raise ValueError('官方正文返回验证或登录页面')
+    if re.search(r'enable javascript and cookies to continue', text, re.I):
+        raise ValueError('官方正文返回验证页面')
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+
+
+def article(source: str, html: str, url: str) -> dict:
+    sel = Selector(html)
+    validate_body(html)
+    selectors = {
+        'fed': ('#article h3.title', '#article h3', '#article h2'),
+        'treasury': ('h2.uswds-page-title',),
+        'cftc': ('article h1:not(.press-release-title)',),
+    }.get(source, ('.main-content__main h1', '#main_content h1', 'main h1', 'h1'))
+    title = next((value for css in selectors if (value := sel.css(css).xpath('normalize-space(.)').get())), None)
+    if source == 'fed' and '/releases/h41/' in url:
+        title = 'Federal Reserve Balance Sheet — H.4.1'
+    if not title:
+        raise ValueError('官方正文缺少标题')
+    values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
+    for script in sel.css('script[type="application/ld+json"]::text').getall():
+        try:
+            data = json.loads(script)
+            nodes = data if isinstance(data, list) else data.get('@graph', [data])
+            values += [n['datePublished'] for n in nodes if isinstance(n, dict) and n.get('datePublished')]
+        except (ValueError, TypeError, AttributeError):
+            continue
+    values += sel.css('.node-details-layout__main-region__content, .date, .date-display-single, .release-date, .article__time, .published-date, .field--name-field-date, .field--name-field-release-date, .press-release-date').xpath('normalize-space(.)').getall()
+    values += re.findall(r'(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}', ' '.join(sel.css('main, article, #content').xpath('string(.)').getall()))[:3]
+    # OFAC action identifiers encode the publisher's day, not the fetch time.
+    if source == 'fed' and (m := re.search(r'/releases/h41/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    if source == 'ofac' and (m := re.search(r'/recent-actions/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    at = precision = None
+    for value in values:
+        try:
+            at, precision = published(value)
+            break
+        except ValueError:
+            continue
+    if at is None:
+        raise ValueError('官方材料缺少可识别的发布时间')
+    body = sel.css(SOURCES[source].body)
+    if not body:
+        raise ValueError('官方正文结构未识别')
+    for node in body.css('script, style, nav, header, footer, form'):
+        node.root.getparent().remove(node.root)
+    body = sorted(body, key=lambda node: len(node.xpath('string(.)').get()), reverse=True)
+    content = body[0].get()
+    text = ' '.join(body[0].xpath('.//text()').getall()).strip()
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+    attachments = []
+    for link in body[0].css('a[href]'):
+        href = link.attrib['href']
+        label = link.xpath('normalize-space(.)').get()
+        if re.search(r'\.pdf(?:\?|$)|/media/\d+/(?:download|dl)|/csl/[^/]+/download', href, re.I):
+            try:
+                link_url = official_url(source, href, url)
+            except ValueError:
+                candidate = urljoin(url, href)
+                if urlsplit(candidate).scheme == 'https':
+                    attachments.append({'url': candidate, 'title': label, 'status': 'unsupported_host', 'error': '附件不在该来源的官方域名范围内，未下载'})
+                continue
+            if link_url not in [a['url'] for a in attachments]:
+                attachments.append({'url': link_url, 'title': label, 'status': 'pending'})
+    return dict(title=title, content_html=content, published_at=at,
+                extra={'language': 'en', 'content_scope': 'article', 'published_at_precision': precision,
+                       'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'attachments': attachments})

backend/src/alphapulse/crawler/events.py

@@ -12,7 +12,8 @@ NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
 CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
 STATUS_KEY = f"{KEY_PREFIX}crawler:status"
 # Sources whose list endpoints support historical pagination.
-HISTORY_SOURCES = ("wscn_live", "wscn_news")
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+HISTORY_SOURCES = ("wscn_live", "wscn_news", *US_SOURCES)
 
 
 def start_urls_key(source_code: str) -> str:

backend/src/alphapulse/crawler/middlewares.py

@@ -4,6 +4,9 @@ import hashlib
 from urllib.parse import urlparse
 
 from scrapy.crawler import Crawler
+from scrapy.exceptions import IgnoreRequest
+
+from alphapulse.core.us_news import official_url
 from scrapy.http import Request, Response
 from sqlalchemy import insert
 from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -19,6 +22,13 @@ class RedisRateLimitMiddleware:
     interval_ms = 1000
 
     async def process_request(self, request: Request) -> None:
+        if source := request.meta.get("official_source"):
+            try:
+                official_url(source, request.url)
+            except ValueError as exc:
+                raise IgnoreRequest(str(exc)) from exc
+        if urlparse(request.url).hostname != "api.congress.gov":
+            request.headers.pop("X-Api-Key", None)
         key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
         while not await redis.set(key, "1", px=self.interval_ms, nx=True):
             ttl = await redis.pttl(key)

backend/src/alphapulse/crawler/pipelines.py

@@ -51,6 +51,8 @@ class NewsPipeline:
                 )
                 # 插入成功取新行；冲突说明另一处刚写入，按已存在条目走比较/修订流程
                 news = await session.scalar(locked)
+            if inserted_id is None and item["extra"].get("content_scope") == "partial" and news.extra.get("content_scope") == "article":
+                return item
             if inserted_id is not None:
                 kind = "new"
             else:
@@ -74,6 +76,7 @@ class NewsPipeline:
                     content_html=item["content_html"],
                     content_hash=digest,
                     raw_response_id=raw_response_id,
+                    extra=item["extra"],
                 ))
             news_id = news.id
             await session.commit()

backend/src/alphapulse/crawler/spiders/base.py

@@ -65,6 +65,8 @@ class RunState:
     history_position: datetime | None = None
     history_done: bool = False
     history_deadline: float = 0
+    official_cursor: dict | None = None
+    stats: dict = field(default_factory=dict)
 
     @property
     def full_scan(self) -> bool:
@@ -104,6 +106,7 @@ class NewsSpider(RedisSpider):
         self._tasks: set[asyncio.Task] = set()
         # 上一轮成功后的回补进度 (目标, 断点, 是否结束)。写库是异步的，管理进程推来的下一条触发可能还带着旧断点
         self.history_checkpoint: tuple[datetime, datetime, bool] | None = None
+        self.finish_task: asyncio.Task | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
@@ -177,7 +180,7 @@ class NewsSpider(RedisSpider):
         run, self.run = self.run, None
         if run.history_target is not None and not run.errors:
             self.history_checkpoint = (run.history_target, run.history_position, run.history_done)
-        self._spawn(self._finish_run(run))
+        self.finish_task = self._spawn(self._finish_run(run))
 
     def _on_error(self, failure: Failure, response: Response) -> None:
         if self.run:
@@ -191,10 +194,11 @@ class NewsSpider(RedisSpider):
         self.logger.info("采集进程退出")
         await self._flush_logs()
 
-    def _spawn(self, coro: Coroutine) -> None:
+    def _spawn(self, coro: Coroutine) -> asyncio.Task:
         task = asyncio.get_running_loop().create_task(coro)
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
+        return task
 
     async def _insert_run(self, run: RunState) -> None:
         async with SessionLocal() as session:
@@ -211,6 +215,7 @@ class NewsSpider(RedisSpider):
         values = {
             "finished_at": now,
             "status": "failed" if failed else "success",
+            "stats": run.stats,
             "requests": run.requests,
             "items_seen": run.items_seen,
             "items_new": run.items_new,
@@ -227,6 +232,12 @@ class NewsSpider(RedisSpider):
             source = await session.get(Source, self.name, with_for_update=True)
             # 本轮失败不推进断点，下一轮从原位置重做（写入幂等）；目标期间被修改或取消时丢弃本轮进度
             history_saved = not failed and run.history_target is not None and source.history_target == run.history_target
+            if not failed and run.official_cursor is not None:
+                cursor = dict(source.history_cursor or {})
+                cursor.update({k: v for k, v in run.official_cursor.items() if k != "history"})
+                if history_saved:
+                    cursor["history"] = run.official_cursor.get("history", {})
+                source.history_cursor = cursor
             if history_saved:
                 source.history_position = run.history_position
                 source.history_done = run.history_done

backend/src/alphapulse/crawler/spiders/rss.py

@@ -97,12 +97,11 @@ class CoinDeskSpider(RssNewsSpider):
     feeds = ("https://www.coindesk.com/arc/outboundfeeds/rss/",)
 
 
-class FedSpider(RssNewsSpider):
+from alphapulse.crawler.spiders.us_official import USOfficialSpider
+
+
+class FedSpider(USOfficialSpider):
     name = "fed"
-    feeds = (
-        "https://www.federalreserve.gov/feeds/press_monetary.xml",
-        "https://www.federalreserve.gov/feeds/speeches_and_testimony.xml",
-    )
 
 
 class BlsSpider(RssNewsSpider):

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -0,0 +1,572 @@
+"""US official collectors: serialized documents, bounded history, durable page cursors."""
+import copy
+import json
+import re
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from html import escape, unescape
+
+from lxml import etree
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from scrapy import Request
+from scrapy.http import Response
+from sqlalchemy import select
+
+from alphapulse.calendar.umich import pdf_text
+from alphapulse.core.config import get_settings
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import RULE_VERSION, SOURCES, article, identity, official_url, published, selection, validate_body
+from alphapulse.crawler.spiders.base import HISTORY_RUN_SECONDS, NewsSpider, html_to_text
+from alphapulse.models import NewsItem, Source
+
+
+def configure_url(url, **params):
+    parts = urlsplit(url)
+    query = dict(parse_qsl(parts.query))
+    query.update(params)
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def congress_bill_url(api_url):
+    parts = urlsplit(api_url).path.strip('/').split('/')
+    _, _, congress, kind, number = parts
+    labels = {'hr': 'house-bill', 's': 'senate-bill', 'hjres': 'house-joint-resolution', 'sjres': 'senate-joint-resolution', 'hconres': 'house-concurrent-resolution', 'sconres': 'senate-concurrent-resolution', 'hres': 'house-resolution', 'sres': 'senate-resolution'}
+    return f'https://www.congress.gov/bill/{congress}th-congress/{labels[kind.lower()]}/{number}'
+
+
+def configuration_error(code):
+    settings = get_settings()
+    if code == 'congress' and not settings.congress_api_key:
+        return '待配置 CONGRESS_API_KEY'
+    if code == 'sec' and not settings.crawler_contact_email:
+        return '待配置 CRAWLER_CONTACT_EMAIL'
+    return None
+
+
+class USOfficialSpider(NewsSpider):
+    """Each lane holds only one page's remaining documents and its next URL."""
+    custom_settings = {"DOWNLOAD_MAXSIZE": 25 * 1024 * 1024}
+
+    def make_request_from_data(self, data):
+        if getattr(self, '_starting', False):
+            return []
+        if self.run is not None:
+            if not self.crawler.engine.spider_is_idle():
+                return super().make_request_from_data(data)
+            self._close_run()
+        self._starting = True
+        self._spawn(self.start_round(json.loads(data)['trigger']))
+        return []
+
+    def _close_run(self):
+        self._checkpoint_run = self.run
+        self._checkpoint_attempts = 1
+        super()._close_run()
+
+    async def start_round(self, trigger):
+        try:
+            if self.finish_task is not None:
+                if self.finish_task.done() and not self.finish_task.cancelled() and self.finish_task.exception() is not None and getattr(self, '_checkpoint_run', None) is not None:
+                    if self._checkpoint_attempts >= 3:
+                        self.logger.error('断点保存连续失败3次，需重启采集进程；未启动下一轮')
+                        return
+                    self._checkpoint_attempts += 1
+                    self.finish_task = self._spawn(self._finish_run(self._checkpoint_run))
+                await self.finish_task
+            async with SessionLocal() as session:
+                source = await session.get(Source, self.name)
+                if source is None or not source.enabled:
+                    return
+                if source.paused_until and source.paused_until > datetime.now(UTC):
+                    return
+                self.cursor = copy.deepcopy(source.history_cursor or {})
+                history = (source.history_target, source.history_position) if source.history_target and not source.history_done else None
+                hours = source.backfill_hours
+            requests = super().make_request_from_data(json.dumps({'trigger': trigger, 'backfill_hours': hours}).encode())
+            self.run.official_cursor = self.cursor
+            if history:
+                target, position = history
+                self.run.history_target = target
+                self.run.history_position = position or self.run.started_at
+                self.run.history_deadline = time.monotonic() + HISTORY_RUN_SECONDS
+                state = self.cursor.setdefault('history', {'entry': 0})
+                if state.get('target') != target.isoformat():
+                    self.cursor['history'] = {'entry': 0, 'target': target.isoformat()}
+                requests += self.advance('history')
+            for request in requests:
+                self.crawler.engine.crawl(request)
+        except Exception:
+            self.logger.exception('官方采集轮次启动失败，未使用队列旧断点')
+            if self.run is not None:
+                self.run.errors.append('官方采集轮次启动失败')
+        finally:
+            self._starting = False
+
+    def first_requests(self):
+        if self.name == 'doj' and self.cursor.get('transport') != 'doj-api-v1':
+            # Retain retry evidence; HTML discovery cursors cannot use API pagination.
+            self.cursor = {k: v for k, v in self.cursor.items() if k not in ('history', 'realtime', 'checked')}
+            self.cursor['transport'] = 'doj-api-v1'
+            self.run.official_cursor = self.cursor
+        error = configuration_error(self.name)
+        if error:
+            self.run.errors.append(error)
+            return []
+        state = self.cursor.setdefault('realtime', {'entry': 0})
+        due = [value['document'] for value in self.cursor.get('attachment_retries', {}).values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
+        if due and not state.get('pending'):
+            state['pending'] = [{**doc, 'attachment_retry': True} for doc in due]
+        bill_due = [{**v['document'], 'bill_retry': True} for v in self.cursor.get('bill_retries', {}).values() if v['attempts'] < 3 and v['next_at'] <= self.run.started_at.timestamp()]
+        if bill_due:
+            state.setdefault('pending', []).extend(d for d in bill_due if d['url'] not in {p['url'] for p in state['pending']})
+        self.run.stats['unresolved_bills'] = len(self.cursor.get('bill_retries', {}))
+        return self.advance('realtime')
+
+    def request_for(self, url, callback, lane, **meta):
+        url = official_url(self.name, url)
+        request = self.request(url, callback, lane=lane, official_source=self.name, **meta)
+        request.priority = 10 if lane == 'realtime' else -10
+        # Credentials never travel to document hosts or into persisted URLs.
+        if urlsplit(url).hostname == 'api.congress.gov':
+            request.headers['X-Api-Key'] = get_settings().congress_api_key
+        request.headers['User-Agent'] = 'AlphaPulse official news collector ' + get_settings().crawler_contact_email
+        return request
+
+    def advance(self, lane):
+        state = self.cursor[lane]
+        if configuration_error(self.name):
+            return []
+        if lane == 'history' and time.monotonic() >= self.run.history_deadline:
+            return []
+        if state.get('pending'):
+            doc = state['pending'][0]
+            if doc['kind'] == 'bill_finish':
+                self.finish_bill(lane)
+                return self.advance(lane)
+            if doc['kind'] == 'actions' and not state['bill']['reasons']:
+                state['pending'].pop(0)
+                return self.advance(lane)
+            return [self.request_for(doc['url'], self.parse_document, lane, document=doc)]
+        if state.get('page_finished'):
+            if state.pop('day_complete', False):
+                state['day'] = (datetime.fromisoformat(state['day']) - timedelta(days=1)).date().isoformat()
+                state['entry'] -= 1
+            if state.get('next'):
+                state['page'] = state.pop('next')
+            else:
+                state['entry'] += 1
+                state.pop('page', None)
+                state.pop('visited_pages', None)
+            state.pop('page_finished', None)
+        spec = SOURCES[self.name]
+        if state['entry'] >= len(spec.entries):
+            if lane == 'history':
+                self.run.stats['attachment_gaps'] = len(self.cursor.get('attachment_retries', {}))
+                self.run.history_done = True
+                self.run.history_position = self.run.history_target
+                state['scanned'] = True
+            else:
+                self.cursor['realtime'] = {'entry': 0}
+            return []
+        url = state.get('page') or spec.entries[state['entry']]
+        if self.name == 'congress' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            url = configure_url(url, fromDateTime=begin.strftime('%Y-%m-%dT%H:%M:%SZ'), toDateTime=self.run.started_at.strftime('%Y-%m-%dT%H:%M:%SZ'))
+        if self.name == 'federal_register' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            # Daily partitions stay below the API's 2,000-result pagination ceiling.
+            day = state.setdefault('day', self.run.started_at.date().isoformat())
+            if day < begin.date().isoformat():
+                state['entry'] += 1
+                return self.advance(lane)
+            url = configure_url(url, **{'conditions[publication_date][is]': day, 'per_page': '1000', 'order': 'oldest'})
+        state['page'] = url
+        return [self.request_for(url, self.parse_listing, lane)]
+
+    def parse_listing(self, response):
+        lane = response.meta['lane']
+        state = self.cursor[lane]
+        docs, next_url = [], None
+        if self.name == 'doj':
+            data = response.json()
+            resultset = data['metadata']['resultset']
+            if data['metadata']['responseInfo']['status'] != 200 or not isinstance(data['results'], list):
+                raise ValueError('DOJ API 列表结构未识别')
+            dates = []
+            for record in data['results']:
+                dates.append(datetime.fromtimestamp(int(record['date']), UTC))
+                docs.append({'url': f"https://www.justice.gov/api/v1/press_releases/{uuid.UUID(record['uuid'])}.json", 'kind': 'doj'})
+            page, size, count = (int(resultset[k]) for k in ('page', 'pagesize', 'count'))
+            if size <= 0 or page < 0 or (not docs and page * size < count):
+                raise ValueError('DOJ API 分页结果不完整')
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            if (page + 1) * size < count and not (dates and all(d.date() < begin.date() for d in dates)):
+                next_url = configure_url(response.url, page=page + 1)
+        elif self.name == 'fed' and response.url.endswith('releaseDates.json'):
+            data = response.json()
+            if not isinstance(data, list) or not data:
+                raise ValueError('H.4.1 日期列表结构未识别')
+            docs = [{'url': f'https://www.federalreserve.gov/releases/h41/{day}/', 'kind': 'article'} for year in data for month in year['Months'] for day in month['Dates'] if int(year['yearValue']) >= 2025]
+            docs.sort(key=lambda d: d['url'], reverse=True)
+        elif self.name in ('congress', 'federal_register'):
+            data = response.json()
+            key = ('committeeMeetings' if '/committee-meeting' in response.url else 'bills') if self.name == 'congress' else 'results'
+            if key not in data or not isinstance(data[key], list):
+                raise ValueError('官方 API 列表结构未识别')
+            if self.name == 'federal_register' and data.get('count', 0) > 2000:
+                day = state['day']
+                url = f'https://www.govinfo.gov/content/pkg/FR-{day}/xml/FR-{day}.xml'
+                state['pending'] = [{'url': url, 'kind': 'register_issue', 'day': day, 'count': data['count']}]
+                yield from self.advance(lane)
+                return
+            for doc in data[key]:
+                if self.name == 'congress':
+                    docs.append({'url': configure_url(doc['url'], format='json'), 'kind': 'meeting' if key == 'committeeMeetings' else 'bill'})
+                else:
+                    docs.append({'url': f"https://www.federalregister.gov/api/v1/documents/{doc['document_number']}.json", 'kind': 'register'})
+            next_url = data.get('pagination', {}).get('next') if self.name == 'congress' else data.get('next_page_url')
+        else:
+            if not response.css('h1'):
+                raise ValueError('官方列表页面未识别')
+            pattern = SOURCES[self.name].article_path
+            for a in response.css('a[href]'):
+                try:
+                    url = official_url(self.name, a.attrib['href'], response.url)
+                except ValueError:
+                    continue
+                if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
+                    docs.append({'url': url, 'kind': 'article'})
+                if a.attrib.get('rel') == 'next' or re.match(r'^(?:Next|Next page|下一页)', a.xpath('normalize-space(.)').get() or '', re.I):
+                    next_url = url
+            docs = list({d['url']: d for d in docs}.values())
+            if not docs:
+                raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        if lane == 'realtime':
+            # OFAC's recent-actions entry is a descending day-indexed feed.
+            if self.name == 'ofac' and docs:
+                days = [re.search(r'/recent-actions/(\d{8})', d['url']) for d in docs]
+                if all(days) and all(m[1] < self.run.cutoff.strftime('%Y%m%d') for m in days):
+                    next_url = None
+            checked = self.cursor.setdefault('checked', {})
+            threshold = (self.run.started_at - timedelta(hours=24)).timestamp()
+            self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
+            docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
+            if self.name == 'fed':
+                def recent(d):
+                    found = re.search(r'(202[5-9]\d{4})', d['url'])
+                    return not found or datetime.strptime(found[1], '%Y%m%d').replace(tzinfo=UTC) >= self.run.cutoff
+                docs = [d for d in docs if recent(d)]
+        if next_url and official_url(self.name, next_url, response.url) == response.url:
+            raise ValueError('官方列表翻页重复当前页面，未推进断点')
+        visited = state.setdefault('visited_pages', [])
+        if response.url in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        visited.append(response.url)
+        if next_url and official_url(self.name, next_url, response.url) in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        state['pending'] = docs
+        state['next'] = official_url(self.name, next_url, response.url) if next_url else None
+        state['page_finished'] = True
+        self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(docs)
+        if self.name == 'federal_register' and not next_url:
+            state['day_complete'] = True
+        yield from self.advance(lane)
+
+    async def parse_document(self, response):
+        lane, doc = response.meta['lane'], response.meta['document']
+        if doc['kind'] == 'register_issue':
+            root = etree.fromstring(response.body, etree.XMLParser(resolve_entities=False, no_network=True))
+            if root.tag != 'FEDREG' or published(root.findtext('DATE', default=''))[0].date().isoformat() != doc['day']:
+                raise ValueError('Federal Register XML 日期或结构不符')
+            numbers = set()
+            for node in root.iter('FRDOC'):
+                match = re.search(r'FR Doc\.\s+([^\s\]]+)', ''.join(node.itertext()))
+                if not match:
+                    raise ValueError('Federal Register XML 文档编号无法解析')
+                numbers.add(match[1])
+            if len(numbers) != doc['count']:
+                raise ValueError('Federal Register XML 文档数量与 API 不符')
+            state = self.cursor[lane]
+            state['pending'] = [{'url': f'https://www.federalregister.gov/api/v1/documents/{number}.json', 'kind': 'register'} for number in sorted(numbers)]
+            state.update(page_finished=True, next=None, day_complete=True)
+            self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(numbers)
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'meeting':
+            data = response.json()['committeeMeeting']
+            at, precision = published(data['updateDate'])
+            if not selection(self.name, data['title'], json.dumps(data.get('relatedItems', {}))):
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+                for result in self.complete(lane):
+                    yield result
+                return
+            parsed = {'title': data['title'], 'content_html': '<pre>' + escape(json.dumps(data, ensure_ascii=False, indent=2)) + '</pre>', 'published_at': at, 'url': response.url, 'external_id': identity(response.url), 'raw_response_id': response.meta['raw_response_id'],
+                      'extra': {'content_scope': 'article', 'published_at_precision': precision, 'published_at_basis': 'official_update', 'published_date': at.date().isoformat(), 'official_meeting': data, 'rule_version': RULE_VERSION, 'attachments': [{'url': official_url(self.name, d['url']), 'title': d.get('name') or d.get('documentType', ''), 'status': 'pending'} for d in {d['url']: d for d in data.get('meetingDocuments', []) + data.get('witnessDocuments', []) if d.get('format') == 'PDF'}.values()]}}
+            for result in self.attach_or_finish(parsed, lane):
+                yield result
+            return
+        if doc['kind'] == 'bill':
+            data = response.json()['bill']
+            base = response.url.split('?')[0]
+            state = self.cursor[lane]
+            previous = self.cursor.setdefault('bill_retries', {}).get(base)
+            version = data.get('updateDateIncludingText') or data.get('updateDate')
+            if previous and previous.get('version') != version:
+                self.cursor['bill_retries'].pop(base)
+            if previous and previous.get('version') == version and (previous['attempts'] >= 3 or previous['next_at'] > self.run.started_at.timestamp()):
+                for result in self.complete(lane):
+                    yield result
+                return
+            reasons = selection(self.name, data['title'], json.dumps(data.get('policyArea', {})))
+            state['bill'] = {'url': base, 'document_url': doc['url'], 'title': data['title'], 'reasons': reasons, 'bodies': 0, 'version': version, 'retry': bool(doc.get('bill_retry') or doc.get('attachment_retry'))}
+            kinds = ([] if reasons else ['subjects', 'summaries']) + ['texts', 'actions', 'bill_finish']
+            state['pending'][0:1] = [{'url': base + '/' + ('text' if kind == 'texts' else kind) + '?format=json&limit=250', 'kind': kind, 'title': data['title'], 'bill': base} for kind in kinds]
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('subjects', 'summaries'):
+            data = response.json()
+            key = 'subjects' if doc['kind'] == 'subjects' else 'summaries'
+            if key not in data:
+                raise ValueError('官方法案补充资料结构未识别')
+            state = self.cursor[lane]
+            state['bill']['reasons'] = state['bill']['reasons'] or selection(self.name, doc['title'], json.dumps(data[key]))
+            children = []
+            if next_url := data.get('pagination', {}).get('next'):
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            state['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('actions', 'texts'):
+            data = response.json()
+            children = []
+            if doc['kind'] == 'actions':
+                for action in data['actions']:
+                    key = identity(doc['bill'] + json.dumps(action, sort_keys=True))
+                    at, precision = published(action['actionDate'])
+                    begin = datetime(2025, 1, 1, tzinfo=UTC) if self.cursor[lane]['bill'].get('retry') else (self.run.history_target if lane == 'history' else self.run.cutoff)
+                    if at < begin:
+                        continue
+                    item = self.item(response, external_id=key, url=congress_bill_url(doc['bill']), title=doc['title'], summary='', content_html=f"<p>{escape(action['text'])}</p>", published_at=at,
+                                     extra={'content_scope': 'official_action', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'bill': doc['bill'], 'official_action': action, 'rule_version': RULE_VERSION})
+                    yield item
+            else:
+                for version in data['textVersions']:
+                    if not version.get('date'):
+                        self.cursor[lane]['bill']['missing_material'] = True
+                        continue
+                    formats = version.get('formats', [])
+                    chosen = next((f for f in formats if f['type'] == 'Formatted Text'), None) or next((f for f in formats if f['type'] == 'Formatted XML'), None) or next((f for f in formats if f['type'] == 'PDF'), None)
+                    if not chosen:
+                        self.cursor[lane]['bill']['missing_material'] = True
+                    if chosen:
+                        children.append({'url': chosen['url'], 'kind': 'text', 'title': doc['title'] + ' — ' + version['type'], 'at': version['date'], 'bill': doc['bill']})
+            next_url = data.get('pagination', {}).get('next')
+            if next_url:
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            self.cursor[lane]['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'register':
+            data = response.json()
+            url = data.get('body_html_url') or data.get('raw_text_url')
+            if not url:
+                raise ValueError('规则文档缺少正文地址')
+            self.cursor[lane]['pending'][0] = {'url': official_url(self.name, url), 'kind': 'text', 'title': data['title'], 'at': data['publication_date'], 'canonical': data['html_url'], 'document_number': data['document_number'], 'effective_on': data.get('effective_on'), 'pdf': data.get('pdf_url')}
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'doj':
+            data = response.json()
+            records = data['results']
+            if data['metadata']['responseInfo']['status'] != 200 or len(records) != 1:
+                raise ValueError('DOJ API 正文结构未识别')
+            record = records[0]
+            if str(uuid.UUID(record['uuid'])) != doc['url'].rsplit('/', 1)[1].removesuffix('.json'):
+                raise ValueError('DOJ API 正文 UUID 不符')
+            canonical = official_url(self.name, record['url'].replace('https://justice.gov/', 'https://www.justice.gov/', 1))
+            title = unescape(record['title']).strip()
+            if not title:
+                raise ValueError('DOJ 正文缺少标题')
+            # The API release date is a calendar date, not a release instant.
+            date = datetime.fromtimestamp(int(record['date']), UTC).date().isoformat()
+            attachment_html = record.get('attachment') or ''
+            api_attachments = []
+            if isinstance(attachment_html, list):
+                for attachment in attachment_html:
+                    file_id = attachment['file']['id']
+                    if not str(file_id).isdigit():
+                        raise ValueError('DOJ API 附件编号无效')
+                    api_attachments.append({'url': f'https://www.justice.gov/media/{file_id}/dl', 'title': attachment.get('description') or 'Official attachment', 'status': 'pending'})
+                attachment_html = ''
+            elif not isinstance(attachment_html, str):
+                raise ValueError('DOJ API 附件结构未识别')
+            html = f'<h1>{escape(title)}</h1><time datetime="{date}"></time><article><div class="field--name-body">{record["body"]}{attachment_html}</div></article>'
+            validate_body(record['body'])
+            parsed = article(self.name, html, canonical)
+            known = {a['url'] for a in parsed['extra']['attachments']}
+            parsed['extra']['attachments'].extend(a for a in api_attachments if a['url'] not in known)
+            changed = etree.HTML(record.get('changed') or '')
+            if changed is not None:
+                times = changed.xpath('//time/@datetime')
+                if times:
+                    parsed['source_updated_at'] = published(times[0])[0]
+            parsed['extra']['official_uuid'] = record['uuid']
+            doc = {**doc, 'canonical': canonical}
+        elif doc['kind'] == 'text':
+            at, precision = published(doc['at'])
+            if response.body.startswith(b'%PDF'):
+                content = '<pre>' + escape(pdf_text(response.body)) + '</pre>'
+            else:
+                content = response.text
+                validate_body(content)
+                if '<' not in content:
+                    content = '<pre>' + escape(content) + '</pre>'
+            parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}}
+            if doc.get('pdf'):
+                parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
+        else:
+            parsed = article(self.name, response.text, response.url)
+        if doc.get('bill'):
+            context = self.cursor[lane]['bill']
+            context['bodies'] += 1
+            if not context['reasons']:
+                context['reasons'] = selection(self.name, context['title'], html_to_text(parsed['content_html']))
+                if context['reasons']:
+                    # A later version can establish relevance for earlier versions.
+                    self.cursor[lane]['pending'][1:1] = context.pop('unmatched_texts', [])
+                else:
+                    context.setdefault('unmatched_texts', []).append(copy.deepcopy(doc))
+            parsed['extra']['selection_reasons'] = context['reasons']
+        begin = self.run.history_target if lane == 'history' else self.run.cutoff
+        if doc.get('attachment_retry') or (doc.get('bill') and self.cursor[lane]['bill'].get('retry')):
+            begin = datetime(2025, 1, 1, tzinfo=UTC)
+        if parsed['published_at'] < begin:
+            self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
+            for result in self.complete(lane):
+                yield result
+            return
+        if lane == 'history':
+            self.run.history_position = min(self.run.history_position or self.run.started_at, parsed['published_at'])
+        parsed['url'] = doc.get('canonical') or response.url
+        parsed['external_id'] = identity(parsed['url'])
+        if self.name == 'fed':
+            # Preserve pre-existing RSS GUID identities during full-text enrichment.
+            async with SessionLocal() as session:
+                existing = await session.scalar(select(NewsItem.external_id).where(NewsItem.source_code == 'fed', NewsItem.url == parsed['url']).limit(1))
+                if existing:
+                    parsed['external_id'] = existing
+        parsed['raw_response_id'] = response.meta['raw_response_id']
+        parsed['retry_document'] = {'url': doc['bill'], 'kind': 'bill'} if doc.get('bill') else doc
+        for result in self.attach_or_finish(parsed, lane):
+            yield result
+
+    def finish_bill(self, lane):
+        context = self.cursor[lane].pop('bill')
+        retries = self.cursor.setdefault('bill_retries', {})
+        if not context['bodies'] or context.get('missing_material'):
+            previous = retries.get(context['url'], {})
+            retries[context['url']] = {'document': {'url': context['url'], 'kind': 'bill'}, 'version': context['version'], 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        else:
+            retries.pop(context['url'], None)
+            if not context['reasons']:
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        self.run.stats['unresolved_bills'] = len(retries)
+        self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
+
+    def attach_or_finish(self, parsed, lane):
+        pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
+        if pending:
+            request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
+            request.meta['handle_httpstatus_list'] = [404, 410]
+            yield request
+            return
+        text = html_to_text(parsed['content_html'])
+        reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
+        incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
+        retry_doc = parsed.pop('retry_document', None)
+        retries = self.cursor.setdefault('attachment_retries', {})
+        if incomplete and retry_doc:
+            previous = retries.get(parsed['url'], {})
+            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        elif not incomplete:
+            retries.pop(parsed['url'], None)
+        if reasons or incomplete:
+            parsed['extra']['selection_reasons'] = reasons
+            parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
+            self.run.stats['partial'] = self.run.stats.get('partial', 0) + int(incomplete)
+            raw_id = parsed.pop('raw_response_id')
+            item = self.item(Response(parsed['url'], request=Request(parsed['url'], meta={'raw_response_id': raw_id})), summary='', **parsed)
+            yield item
+        else:
+            self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        yield from self.complete(lane)
+
+    def parse_attachment(self, response):
+        parsed = response.meta['parsed']
+        attachment = next(a for a in parsed['extra']['attachments'] if a['url'] == response.url or a['status'] == 'pending')
+        attachment['raw_response_id'] = response.meta['raw_response_id']
+        try:
+            if response.status != 200:
+                raise ValueError(f'HTTP {response.status}')
+            text = pdf_text(response.body)
+            attachment['status'] = 'complete'
+            parsed['content_html'] += f"<section><h2>{escape(attachment['title'])}</h2><pre>{escape(text)}</pre></section>"
+        except (ValueError, OSError) as exc:
+            attachment.update(status='unavailable', error=str(exc))
+            self.logger.warning('附件缺口：%s', response.url)
+        yield from self.attach_or_finish(parsed, response.meta['lane'])
+
+    def complete(self, lane):
+        doc = self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[doc['url']] = self.run.started_at.timestamp()
+        return self.advance(lane)
+
+
+class SecSpider(USOfficialSpider):
+    name = 'sec'
+
+
+class CftcSpider(USOfficialSpider):
+    name = 'cftc'
+
+
+class TreasurySpider(USOfficialSpider):
+    name = 'treasury'
+
+
+class OfacSpider(USOfficialSpider):
+    name = 'ofac'
+
+
+class FincenSpider(USOfficialSpider):
+    name = 'fincen'
+
+
+class WhitehouseSpider(USOfficialSpider):
+    name = 'whitehouse'
+
+
+class CongressSpider(USOfficialSpider):
+    name = 'congress'
+
+
+class FederalRegisterSpider(USOfficialSpider):
+    name = 'federal_register'
+
+
+class OccSpider(USOfficialSpider):
+    name = 'occ'
+
+
+class DojSpider(USOfficialSpider):
+    custom_settings = {**USOfficialSpider.custom_settings, 'DOWNLOAD_DELAY': 1.0}
+    name = 'doj'

backend/src/alphapulse/models/news.py

@@ -25,6 +25,7 @@ class Source(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # 历史回补：目标时间（为空表示不回补）、已回补到的最早时间（断点）、是否已结束（到达目标或来源无更早数据）
+    history_cursor: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     history_target: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_position: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
@@ -71,6 +72,7 @@ class NewsRevision(Base):
     summary: Mapped[str] = mapped_column(Text)
     content_html: Mapped[str] = mapped_column(Text)
     content_hash: Mapped[str] = mapped_column(String(64))
+    extra: Mapped[dict | None] = mapped_column(JSONB)
     captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
 
@@ -110,6 +112,7 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # running / success / failed / interrupted
     status: Mapped[str] = mapped_column(String(16))
+    stats: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     requests: Mapped[int] = mapped_column(Integer, default=0)
     items_seen: Mapped[int] = mapped_column(Integer, default=0)
     items_new: Mapped[int] = mapped_column(Integer, default=0)

backend/tests/test_congress_capture.py

@@ -0,0 +1,93 @@
+import asyncio
+import base64
+import gzip
+import json
+from pathlib import Path
+import runpy
+
+import httpx
+import pytest
+
+
+SCRIPT = Path(__file__).resolve().parents[2] / 'scripts/capture_congress.py'
+
+
+def test_capture_redacts_and_reuses_successful_response(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    seen = []
+    def handle(request):
+        seen.append(request)
+        return httpx.Response(200, json={'request': {'apiKey': 'private-test-secret'}, 'pagination': {'next': 'https://api.congress.gov/v3/bill?page=2&api_key=private-test-secret'}})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        url = 'https://api.congress.gov/v3/bill?format=json'
+        first = capture.fetch(url)
+        assert capture.fetch(url) == first and len(seen) == 1
+        loaded = json.load(gzip.open(path, 'rt'))
+        assert 'private-test-secret' not in base64.b64decode(loaded[url]['body']).decode()
+        assert seen[0].headers['X-Api-Key'] == 'private-test-secret'
+        capture.fetch('https://www.govinfo.gov/test.pdf')
+        assert 'X-Api-Key' not in seen[-1].headers
+
+
+def test_capture_stops_after_rate_limit_and_reports_incomplete(trading, tmp_path):
+    module = runpy.run_path(str(SCRIPT))
+    calls = []
+    def handle(request):
+        calls.append(request)
+        return httpx.Response(429, headers={'Retry-After': '123'})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        report = asyncio.run(module['run'](capture))
+    assert len(calls) == 1 and not path.exists()
+    assert report['accepted'] is False
+    assert len(report['cases']) == 1
+    assert '123' in report['cases'][0]['error']
+    assert 'private-test-secret' not in json.dumps(report)
+
+
+def test_capture_keeps_progress_on_budget_exhaustion(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b'body'))) as client:
+        capture = module['Capture'](path, 'test', limit=1, client=client)
+        capture.fetch('https://www.govinfo.gov/first.pdf')
+        with pytest.raises(module['CaptureBlocked'], match='预算'):
+            capture.fetch('https://www.govinfo.gov/second.pdf')
+        resumed = module['Capture'](path, 'test', limit=1, client=client)
+        assert resumed.fetch('https://www.govinfo.gov/first.pdf') == b'body'
+        assert resumed.requests == 0
+        assert resumed.fetch('https://www.govinfo.gov/second.pdf') == b'body'
+
+
+def test_capture_replays_all_case_callbacks_without_external_io(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    def handle(request):
+        path = request.url.path
+        if request.url.host == 'www.congress.gov':
+            return httpx.Response(200, text='<p>The law establishes regulatory requirements for stablecoins and digital assets.</p>')
+        if path == '/v3/committee-meeting/119':
+            data = {'committeeMeetings': [{'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1'}]}
+        elif '/committee-meeting/' in path:
+            data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z'}}
+        elif path.endswith('/text'):
+            data = {'textVersions': [{'date': '2025-01-01', 'type': 'Introduced', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov' + path + '.html'}]}]}
+        elif path.endswith('/actions'):
+            data = {'actions': [{'actionDate': '2025-01-01', 'text': 'Passed'}]}
+        else:
+            data = {'bill': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01'}}
+        return httpx.Response(200, json=data)
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](tmp_path / 'cache.gz', 'test', client=client)
+        report = asyncio.run(module['run'](capture))
+        assert report['accepted'] is True
+        assert len(report['cases']) == 4
+        assert sum(case['items'] for case in report['cases']) == 7
+        resumed = module['Capture'](capture.path, 'test', client=client)
+        assert asyncio.run(module['run'](resumed))['accepted'] is True
+        assert resumed.requests == 0

backend/tests/test_news_sources.py

@@ -33,7 +33,7 @@ ITEM = """<item><title>Bitcoin &amp; Ethereum</title>
 <dc:creator>News desk</dc:creator><category>Markets</category></item>"""
 
 
-@pytest.mark.parametrize("source", ["CoinDeskSpider", "FedSpider", "BeaSpider"])
+@pytest.mark.parametrize("source", ["CoinDeskSpider", "BeaSpider"])
 def test_rss_preserves_original_summary_time_and_provenance(trading, source):
     from alphapulse.crawler.spiders import rss
     spider = make_spider(getattr(rss, source))
@@ -53,8 +53,8 @@ def test_rss_preserves_original_summary_time_and_provenance(trading, source):
 
 
 def test_rss_identity_survives_revisions_and_different_subfeeds(trading):
-    from alphapulse.crawler.spiders.rss import FedSpider
-    spider = make_spider(FedSpider)
+    from alphapulse.crawler.spiders.rss import CoinDeskSpider
+    spider = make_spider(CoinDeskSpider)
     original, = spider.parse_feed(feed_response(ITEM))
     revised, = spider.parse_feed(feed_response(ITEM.replace("Bitcoin &amp; Ethereum", "Revised headline"), "https://example.invalid/other.xml"))
     assert original["external_id"] == revised["external_id"]

backend/tests/test_us_official_news.py

@@ -0,0 +1,659 @@
+import asyncio
+import copy
+import json
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+
+import pytest
+from scrapy import Request
+from scrapy.http import HtmlResponse
+
+from alphapulse.core.us_news import SOURCES, article, official_url, published, selection
+
+
+def spider(trading, source='ofac'):
+    from alphapulse.crawler.spiders.base import RunState
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    obj = USOfficialSpider(name=source)
+    obj.name = source
+    now = datetime(2026, 11, 4, tzinfo=UTC)
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now - timedelta(hours=72))
+    obj.run.history_target = datetime(2025, 1, 1, tzinfo=UTC)
+    obj.run.history_deadline = time.monotonic() + 20
+    obj.cursor = {'history': {'entry': 0, 'pending': []}, 'realtime': {'entry': 0, 'pending': []}}
+    obj.lanes = {}
+    return obj
+
+
+def response(url, body, **meta):
+    return HtmlResponse(url, body=body.encode(), encoding='utf-8', request=Request(url, meta={'raw_response_id': 12, **meta}))
+
+
+@pytest.mark.parametrize('source,title,text,expected', [
+    ('doj', 'Man sentenced for bitcoin theft', 'Personal bitcoin theft worth $300', False),
+    ('doj', 'Exchange charged', 'Cryptocurrency exchange charged with money laundering', True),
+    ('fincen', 'Digital asset rule', 'FinCEN withdraws proposed digital asset rule', True),
+    ('whitehouse', 'Tariffs', 'President adjusts import tariffs', True),
+    ('treasury', 'Meeting with delegation', 'Greetings and cooperation', False),
+    ('ofac', 'New sanctions', 'Sanctions against a financial institution', True),
+    ('sec', 'Ethernet equipment', 'Method and hardware', False),
+])
+def test_selection_boundaries(source, title, text, expected):
+    assert bool(selection(source, title, text)) is expected
+
+
+@pytest.mark.parametrize('url', ['http://ofac.treasury.gov/recent-actions/20261104', 'https://evil.example/a.pdf', 'https://[email protected]/a', 'https://ofac.treasury.gov/a?api_key=secret'])
+def test_official_links_reject_external_and_credentials(url):
+    with pytest.raises(ValueError):
+        official_url('ofac', url)
+
+
+def test_dates_do_not_fabricate_publisher_time():
+    assert published('2025-01-01') == (datetime(2025, 1, 1, tzinfo=UTC), 'date')
+    assert published('2026-11-01T01:30:00-04:00')[0] == datetime(2026, 11, 1, 5, 30, tzinfo=UTC)
+    with pytest.raises(ValueError):
+        published('yesterday')
+
+
+def test_ofac_original_body_and_attachment_boundaries():
+    data = article('ofac', '''<h1>Crypto sanctions</h1><main><div class="field--name-body"><p>New sanctions against a virtual currency exchange institution.</p>
+    <a href="/media/123/download">License</a><a href="https://evil.example/a.pdf">external</a></div></main>''', 'https://ofac.treasury.gov/recent-actions/20250101')
+    assert data['published_at'] == datetime(2025, 1, 1, tzinfo=UTC)
+    assert data['extra']['published_at_precision'] == 'date'
+    assert len(data['extra']['attachments']) == 2
+    assert data['extra']['attachments'][1]['status'] == 'unsupported_host'
+
+
+def test_challenge_is_failure_not_empty_success(trading):
+    obj = spider(trading)
+    with pytest.raises(ValueError, match='未识别'):
+        list(obj.parse_listing(response(SOURCES['ofac'].entries[0], '<html>Please wait</html>', lane='history')))
+
+
+def test_history_pending_documents_survive_deadline_and_resume(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.run.history_deadline = time.monotonic() - 1
+    assert obj.advance('history') == []
+    saved = copy.deepcopy(obj.cursor)
+    resumed = spider(trading)
+    resumed.cursor = saved
+    request, = resumed.advance('history')
+    assert request.url == doc['url'] and resumed.cursor['history']['pending'] == [doc]
+    assert not resumed.run.history_done
+
+
+@pytest.mark.parametrize('count', [1999, 2000, 2001])
+def test_register_oversized_day_uses_xml(trading, count):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-01'
+    req, = obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': count, 'results': [{'document_number': '2025-1'}]}), lane='history'))
+    assert ('govinfo.gov' in req.url) is (count > 2000)
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_keys_only_sent_to_congress_api(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test-key', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    assert obj.request_for('https://api.congress.gov/v3/bill', lambda x: x, 'history').headers['X-Api-Key'] == b'test-key'
+    assert 'X-Api-Key' not in obj.request_for('https://www.congress.gov/text', lambda x: x, 'history').headers
+
+
+def test_register_body_and_pdf_are_part_of_the_same_package(trading):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/api/v1/documents/2025-1.json', 'kind': 'register'}
+    obj.cursor['history']['pending'] = [doc]
+    res = response(doc['url'], json.dumps({'title': 'Digital asset rule', 'publication_date': '2025-01-01', 'document_number': '2025-1', 'body_html_url': 'https://www.federalregister.gov/documents/full_text/html/2025/01/01/2025-1.html', 'html_url': 'https://www.federalregister.gov/documents/2025/01/01/2025-1/a', 'pdf_url': 'https://www.govinfo.gov/a.pdf'}), lane='history', document=doc)
+    async def collect():
+        return [v async for v in obj.parse_document(res)]
+    request, = asyncio.run(collect())
+    assert request.meta['document']['pdf'] == 'https://www.govinfo.gov/a.pdf'
+
+
+def test_summary_scope_tests_do_not_hide_fed_enrichment(trading):
+    from alphapulse.crawler.spiders.rss import FedSpider
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    assert issubclass(FedSpider, USOfficialSpider)
+    assert len(SOURCES['fed'].entries) >= 8
+
+
+def test_fifty_real_public_documents_parse_without_io():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    assert len(samples) == 50
+    assert len({s['source'] for s in samples}) == 8
+    for sample in samples:
+        parsed = article(sample['source'], sample['body'], sample['url'])
+        assert parsed['title'] == sample['expected_title']
+        assert len(parsed['content_html']) > 40
+        assert parsed['published_at'].year >= 2025
+        assert parsed['extra']['published_at_precision'] in ('date', 'datetime')
+
+
+def test_daily_register_partitions_resume_without_marking_done(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-02'
+    requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': 0, 'results': []}), lane='history')))
+    assert len(requests) == 1
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_h41_uses_publisher_release_dates_not_assumed_thursdays(trading):
+    obj = spider(trading, 'fed')
+    data = [{'yearValue': '2025', 'Months': [{'MonthName': 'January', 'Dates': ['20250103', '20250109']}]}]
+    req, = obj.parse_listing(response('https://www.federalreserve.gov/releases/h41/releaseDates.json', json.dumps(data), lane='history'))
+    assert req.url == 'https://www.federalreserve.gov/releases/h41/20250109/'
+    assert obj.cursor['history']['pending'][1]['url'].endswith('20250103/')
+
+
+def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
+    obj = spider(trading)
+    obj.run.history_deadline = time.monotonic() - 1
+    url = 'https://ofac.treasury.gov/recent-actions/20250101'
+    doc = {'url': url, 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    parsed = {'url': url, 'title': 'Crypto sanctions', 'external_id': 'x', 'content_html': '<p>Virtual currency exchange sanctions.</p>', 'published_at': datetime(2025, 1, 1, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
+              'extra': {'attachments': [{'url': 'https://ofac.treasury.gov/a.pdf', 'title': 'License', 'status': 'unavailable'}]}}
+    item, = obj.attach_or_finish(parsed, 'history')
+    assert item['extra']['content_scope'] == 'partial'
+    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
+    assert obj.cursor['attachment_retries'][url]['document'] == doc
+
+
+@pytest.mark.parametrize('canceled,failed', [(False, False), (True, False), (False, True)])
+def test_checkpoint_does_not_advance_failed_or_canceled_history(trading, monkeypatch, canceled, failed):
+    from alphapulse.crawler.spiders import base
+    now = datetime.now(UTC)
+    target = now - timedelta(days=300)
+    source = SimpleNamespace(history_target=None if canceled else target, history_cursor={'history': {'entry': 0}}, consecutive_failures=0)
+    class Session:
+        async def __aenter__(self):
+            return self
+        async def __aexit__(self, *args):
+            pass
+        async def execute(self, *args):
+            pass
+        async def get(self, *args, **kwargs):
+            return source
+        async def commit(self):
+            pass
+    monkeypatch.setattr(base, 'SessionLocal', Session)
+    obj = spider(trading)
+    run = base.RunState(uuid.uuid4(), 'schedule', now, target, history_target=target, history_position=now, official_cursor={'history': {'entry': 5}, 'realtime': {'entry': 2}})
+    if failed:
+        run.errors.append('write failed')
+    asyncio.run(obj._finish_run(run))
+    assert source.history_cursor['history']['entry'] == (0 if canceled or failed else 5)
+
+
+def test_congress_action_links_use_public_bill_type(trading):
+    from alphapulse.crawler.spiders.us_official import congress_bill_url
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/hr/1') == 'https://www.congress.gov/bill/119th-congress/house-bill/1'
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/s/1582') == 'https://www.congress.gov/bill/119th-congress/senate-bill/1582'
+
+
+def test_tick_during_active_round_does_not_replace_its_cursors(trading):
+    obj = spider(trading)
+    obj.crawler = SimpleNamespace(engine=SimpleNamespace(spider_is_idle=lambda: False))
+    before = copy.deepcopy(obj.cursor)
+    assert obj.make_request_from_data(json.dumps({'trigger': 'schedule', 'official_cursor': {'history': {'entry': 99}}}).encode()) == []
+    assert obj.cursor == before
+
+
+def test_realtime_keeps_next_page(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20261104">Crypto</a><a rel="next" href="?page=1">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] == url + '?page=1'
+
+
+@pytest.mark.parametrize('body', ['', '<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>'])
+def test_text_challenge_never_emits_or_advances(trading, body):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    with pytest.raises(ValueError):
+        asyncio.run(collect())
+    assert obj.cursor['history']['pending'] == [doc]
+
+
+def test_genius_bill_requires_evidence_before_exclusion(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/bill/119/s/1582', 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'bill': {'title': 'GENIUS Act', 'policyArea': {'name': 'Finance and Financial Sector'}}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert '/subjects?' in req.url
+    assert not obj.run.stats.get('excluded')
+
+
+def test_real_titles_are_article_headlines():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    for s in samples:
+        if s['source'] == 'fed' and '2026-press.htm' in s['url']:
+            continue
+        title = article(s['source'], s['body'], s['url'])['title']
+        assert title not in ('Board of Governors of the Federal Reserve System', 'U.S. Department of the Treasury')
+        assert not title.startswith('Release Number ')
+
+
+@pytest.mark.parametrize('enabled,history_enabled,save_failed', [(True, True, False), (False, True, False), (True, False, False), (True, True, True)])
+def test_round_waits_for_commit_and_reads_fresh_state(trading, monkeypatch, enabled, history_enabled, save_failed):
+    from alphapulse.crawler.spiders import us_official
+    fresh = {'history': {'entry': 0, 'target': '2025-01-01T00:00:00+00:00', 'pending': [{'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}]}}
+    source = SimpleNamespace(enabled=enabled, paused_until=None, history_cursor=fresh, history_target=datetime(2025, 1, 1, tzinfo=UTC) if history_enabled else None, history_done=False, history_position=None, backfill_hours=72)
+    reads = []
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args):
+            reads.append(True)
+            return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def run():
+        obj = spider(trading)
+        obj.run = None
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=reads.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        saved = asyncio.Event()
+        async def finish():
+            await saved.wait()
+            if save_failed: raise RuntimeError('test write failure')
+        obj.finish_task = asyncio.create_task(finish())
+        boot = asyncio.create_task(obj.start_round('schedule'))
+        await asyncio.sleep(0)
+        assert not reads
+        saved.set()
+        await boot
+        requests = [r for r in reads if isinstance(r, Request)]
+        if enabled and not save_failed:
+            assert bool([r for r in requests if r.meta['lane'] == 'history']) is history_enabled
+            if history_enabled:
+                assert requests[-1].url.endswith('20250101')
+        else:
+            assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(run())
+
+
+def test_register_realtime_finishes_all_pages_before_changing_day(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['realtime']['day'] = '2026-11-04'
+    url = SOURCES[obj.name].entries[0]
+    list(obj.parse_listing(response(url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1234'}], 'next_page_url': url + '?page=2'}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    req, = obj.advance('realtime')
+    assert req.url.endswith('?page=2')
+    list(obj.parse_listing(response(req.url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1235'}]}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    obj.advance('realtime')
+    assert obj.cursor['realtime']['day'] == '2026-11-03'
+
+
+def test_pagination_cycle_does_not_claim_success(trading):
+    obj = spider(trading)
+    url = SOURCES[obj.name].entries[0]
+    obj.cursor['history']['visited_pages'] = [url]
+    with pytest.raises(ValueError, match='循环'):
+        list(obj.parse_listing(response(url + '?page=2', f'<h1>News</h1><a href="/recent-actions/20261104">News</a><a rel="next" href="{url}">Next</a>', lane='history')))
+    assert not obj.run.history_done
+
+
+def test_fragment_is_valid_without_page_heading(trading):
+    obj = spider(trading, 'federal_register')
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>This digital asset regulation establishes requirements for stablecoin issuers.</p>', lane='history', document=doc))]
+    item, = asyncio.run(collect())
+    assert item['extra']['content_scope'] == 'article'
+
+
+def test_bill_full_text_can_supply_relevance_and_resume(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(body):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    asyncio.run(parse(json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': '2026-11-04'}})))
+    asyncio.run(parse(json.dumps({'subjects': {'legislativeSubjects': []}})))
+    asyncio.run(parse(json.dumps({'summaries': []})))
+    asyncio.run(parse(json.dumps({'textVersions': [{'type': 'Enrolled', 'date': '2026-11-04', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov/text'}]}]})))
+    obj.cursor = json.loads(json.dumps(obj.cursor))
+    output = asyncio.run(parse('<p>The bill establishes the regulatory framework for payment stablecoins and digital assets.</p>'))
+    assert output[0]['title'] == 'GENIUS Act — Enrolled'
+    assert '/actions?' in output[1].url
+    output = asyncio.run(parse(json.dumps({'actions': [{'actionDate': '2026-11-04', 'text': 'Became law'}]})))
+    assert output[0]['extra']['content_scope'] == 'official_action'
+    assert not obj.cursor.get('bill_retries')
+
+
+def test_missing_bill_material_is_a_retryable_gap(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(data):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), lane='history', document=doc))]
+    asyncio.run(parse({'bill': {'title': 'GENIUS Act'}}))
+    asyncio.run(parse({'subjects': {}}))
+    asyncio.run(parse({'summaries': []}))
+    asyncio.run(parse({'textVersions': []}))
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+    assert not obj.run.stats.get('excluded')
+
+
+@pytest.mark.parametrize('attempts,version,expected_kind', [(3, 'v1', 'listing'), (1, 'v1', 'listing'), (3, 'v2', 'subjects')])
+def test_bill_retries_are_bounded_until_official_update(trading, monkeypatch, attempts, version, expected_kind):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    url = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': url, 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.cursor['bill_retries'] = {url: {'version': 'v1', 'attempts': attempts, 'next_at': (obj.run.started_at + timedelta(minutes=30)).timestamp()}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(url, json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': version}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert ('/subjects?' in req.url) is (expected_kind == 'subjects')
+
+
+def test_fed_archive_is_not_discovered_as_article(trading):
+    obj = spider(trading, 'fed')
+    url = SOURCES['fed'].entries[0]
+    req, = obj.parse_listing(response(url, '<h1>Press releases</h1><a href="2026-press.htm#content">Archive</a><a href="monetary20261104a.htm">News</a>', lane='history'))
+    assert req.url.endswith('monetary20261104a.htm')
+
+
+def test_ofac_entire_old_page_can_end_realtime_window(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20250101">Old</a><a rel="next" href="?page=2">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] is None
+
+
+@pytest.mark.parametrize('body', ['<title>Page not found</title><p>The requested official document could not be located.</p>', '<?xml version="1.0"?><bill><section>Digital asset regulation establishes requirements for banks.'])
+def test_error_html_and_malformed_xml_are_rejected(body):
+    from alphapulse.core.us_news import validate_body
+    with pytest.raises(ValueError):
+        validate_body(body)
+
+
+def test_later_bill_version_recalls_earlier_version(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    first = {'url': 'https://www.congress.gov/first', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2026-11-04', 'bill': base}
+    second = {**first, 'url': 'https://www.congress.gov/second'}
+    obj.cursor['history'].update(bill={'url': base, 'reasons': [], 'title': 'GENIUS Act', 'bodies': 0}, pending=[first, second])
+    async def parse(doc, body):
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert len(output) == 1 and isinstance(output[0], Request)
+    output = asyncio.run(parse(second, '<p>The legislation creates a regulatory framework for stablecoins and digital assets.</p>'))
+    assert output[0]['extra']['selection_reasons']
+    assert output[1].url == first['url']
+    obj.run.history_deadline = time.monotonic() - 1
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert output[0]['title'] == 'GENIUS Act'
+
+
+def test_bill_retry_keeps_historical_text_outside_realtime_window(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': 'https://www.congress.gov/old', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2025-01-01', 'bill': base}
+    obj.cursor['realtime'].update(bill={'url': base, 'title': 'GENIUS Act', 'reasons': ['crypto_policy'], 'bodies': 0, 'retry': True}, pending=[doc])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>The law establishes requirements for stablecoin issuers and digital assets.</p>', lane='realtime', document=doc))]
+    assert asyncio.run(collect())[0]['published_at'].year == 2025
+
+
+def test_failed_checkpoint_save_recovers_on_later_trigger(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=None, history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        requests, attempts = [], []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        original = obj.run
+        async def finish(run):
+            attempts.append(run)
+            if len(attempts) == 1:
+                raise RuntimeError('temporary checkpoint failure')
+        obj._finish_run = finish
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        await obj.start_round('schedule')
+        assert attempts == [original, original]
+        assert requests and obj.run is not None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_queued_tick_obeys_newly_saved_backoff(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=datetime.now(UTC) + timedelta(minutes=5), history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        obj.run = None
+        requests = []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        await obj.start_round('schedule')
+        assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_checkpoint_retry_budget_preserves_original_round(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'SessionLocal', lambda: pytest.fail('Uncommitted checkpoint must block source reads'))
+    async def check():
+        obj = spider(trading)
+        original, attempts = obj.run, []
+        async def fail(run):
+            attempts.append(run)
+            raise RuntimeError('persistent checkpoint failure')
+        obj._finish_run = fail
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        for _ in range(5):
+            await obj.start_round('schedule')
+        assert attempts == [original] * 3
+        assert obj.run is None and not obj._starting
+    asyncio.run(check())
+
+
+@pytest.mark.parametrize('failure', ['', 'count', 'date', 'malformed'])
+def test_register_xml_validates_and_resumes(trading, failure):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.govinfo.gov/issue.xml', 'kind': 'register_issue', 'day': '2025-01-01', 'count': 2001}
+    obj.cursor['history'].update(day=doc['day'], pending=[doc])
+    xml = '<FEDREG><DATE>January 1, 2025</DATE>' + ''.join(f'<FRDOC>[FR Doc. 2025-{n} Filed 1-1-25]</FRDOC>' for n in range(2001)) + '<FRDOC>[FR Doc. 2025-0 Filed 1-1-25]</FRDOC></FEDREG>'
+    if failure == 'count': doc['count'] += 1
+    if failure == 'date': doc['day'] = '2025-01-02'
+    if failure == 'malformed': xml = xml[:-10]
+    before = copy.deepcopy(obj.cursor)
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], xml, lane='history', document=doc))]
+    if failure:
+        from lxml import etree
+        with pytest.raises((ValueError, etree.XMLSyntaxError)):
+            asyncio.run(collect())
+        assert obj.cursor == before
+    else:
+        req, = asyncio.run(collect())
+        assert len(obj.cursor['history']['pending']) == 2001
+        assert req.meta['document']['kind'] == 'register'
+        assert obj.cursor['history']['day'] == '2025-01-01'
+        obj.run.history_deadline = time.monotonic() - 1
+        assert not obj.complete('history')
+        resumed = spider(trading, 'federal_register')
+        resumed.cursor = json.loads(json.dumps(obj.cursor))
+        assert resumed.advance('history')[0].meta['document'] == obj.cursor['history']['pending'][0]
+
+
+def test_doj_api_pagination_and_old_day_stop(trading):
+    obj = spider(trading, 'doj')
+    url = SOURCES['doj'].entries[0]
+    def data(day):
+        return {'metadata': {'responseInfo': {'status': 200}, 'resultset': {'count': '100', 'page': 0, 'pagesize': 50}}, 'results': [{'uuid': str(uuid.uuid4()), 'date': str(int(day.timestamp()))}]}
+    req, = obj.parse_listing(response(url, json.dumps(data(obj.run.started_at)), lane='realtime'))
+    assert req.meta['document']['kind'] == 'doj'
+    assert 'page=1' in obj.cursor['realtime']['next']
+    obj = spider(trading, 'doj')
+    list(obj.parse_listing(response(url, json.dumps(data(datetime(2024,1,1,tzinfo=UTC))), lane='history')))
+    assert obj.cursor['history']['next'] is None
+
+
+@pytest.mark.parametrize('bad_body', [False, True])
+def test_doj_body_identity_date_and_validation(trading, bad_body):
+    obj = spider(trading, 'doj')
+    obj.run.history_deadline = time.monotonic() - 1
+    identifier = str(uuid.uuid4())
+    doc = {'url': f'https://www.justice.gov/api/v1/press_releases/{identifier}.json', 'kind': 'doj'}
+    obj.cursor['history']['pending'] = [doc]
+    body = '<h1>Just a moment...</h1>' if bad_body else '<p>The cryptocurrency exchange pleaded guilty to violations of financial regulations.</p>'
+    record = {'uuid': identifier, 'url': 'https://justice.gov/opa/pr/test', 'date': '1735732800', 'title': 'Exchange &amp; settlement', 'body': body, 'changed': '<time datetime="2025-01-02T12:00:00Z">x</time>'}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'metadata': {'responseInfo': {'status': 200}}, 'results': [record]}), lane='history', document=doc))]
+    if bad_body:
+        with pytest.raises(ValueError): asyncio.run(collect())
+        assert obj.cursor['history']['pending'] == [doc]
+    else:
+        item, = asyncio.run(collect())
+        assert item['url'] == 'https://www.justice.gov/opa/pr/test'
+        assert item['title'] == 'Exchange & settlement'
+        assert item['extra']['published_at_precision'] == 'date'
+        assert item['source_updated_at'] == datetime(2025,1,2,12,tzinfo=UTC)
+
+
+def test_doj_old_discovery_cursor_is_rebuilt(trading):
+    obj = spider(trading, 'doj')
+    obj.cursor.update(checked={'old': 1}, history={'entry': 2}, attachment_retries={})
+    obj.first_requests()
+    assert obj.cursor['transport'] == 'doj-api-v1'
+    assert 'history' not in obj.cursor and obj.cursor['realtime']['entry'] == 0
+    assert obj.run.history_target == datetime(2025,1,1,tzinfo=UTC)
+
+
+def test_bill_undated_text_is_visible_retry_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    finish = {'url': base, 'kind': 'bill_finish'}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, finish])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': None, 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+def test_public_api_and_pdf_fixtures_replay_without_io(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    items = asyncio.run(script['public_api_items']())
+    assert len(items) == 3
+    assert all(item['extra']['content_scope'] == 'article' for item in items)
+
+
+
+def test_congress_official_xml_crosscheck(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    assert len(asyncio.run(script['congress_reference_items']())) == 3
+
+
+def test_congress_meeting_includes_deduplicated_witness_pdfs(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1', 'kind': 'meeting'}
+    obj.cursor['history']['pending'] = [doc]
+    pdf = {'url': 'https://www.congress.gov/witness.pdf', 'format': 'PDF', 'documentType': 'Witness Statement'}
+    data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z', 'meetingDocuments': [pdf], 'witnessDocuments': [pdf]}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), document=doc, lane='history'))]
+    req, = asyncio.run(collect())
+    assert req.url == pdf['url']
+    assert len(req.meta['parsed']['extra']['attachments']) == 1
+    assert req.meta['parsed']['extra']['attachments'][0]['title'] == 'Witness Statement'
+
+
+def test_dated_but_unavailable_bill_text_remains_a_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, {'url': base, 'kind': 'bill_finish'}])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': '2025-01-01', 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+
+def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_path):
+    import runpy
+    import shutil
+    from pathlib import Path
+    scripts = tmp_path / 'scripts'
+    scripts.mkdir()
+    original = Path(__file__).resolve().parents[2] / 'scripts'
+    for name in ('smoke_us_news.py', 'capture_congress.py'):
+        shutil.copyfile(original / name, scripts / name)
+    namespace = runpy.run_path(str(scripts / 'smoke_us_news.py'))
+    items = asyncio.run(namespace['congress_api_items']())
+    assert len(items) == 100
+    meetings = [item for item in items if item['extra'].get('official_meeting')]
+    assert len(meetings) == 1
+    assert meetings[0]['extra']['content_scope'] == 'partial'
+    attachments = meetings[0]['extra']['attachments']
+    assert len(attachments) == 15
+    assert len({a['url'] for a in attachments}) == 15
+    assert sum(a['status'] == 'complete' for a in attachments) == 14
+    assert sum(a['status'] == 'unavailable' for a in attachments) == 1
+    assert any(item['extra']['content_scope'] == 'official_action' for item in items)