Add resumable US policy news collectors and provenance

Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled.

Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.

.gitignore

@@ -5,3 +5,4 @@ backups/
 __pycache__/
 /docs/
 /build.json
+/.env.congress-test

backend/alembic/versions/20261104_da1234567890_us_official_news.py

@@ -0,0 +1,25 @@
+"""Add disabled US official collectors and resumable provenance metadata."""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+revision = 'da1234567890'
+down_revision = 'c91234567890'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('sources', sa.Column('history_cursor', JSONB, nullable=False, server_default='{}'))
+    op.add_column('crawl_runs', sa.Column('stats', JSONB, nullable=False, server_default='{}'))
+    op.add_column('news_revisions', sa.Column('extra', JSONB, nullable=True))
+    table = sa.table('sources', *(sa.column(k) for k in ('code', 'name', 'list_url', 'adapter', 'collector', 'enabled', 'interval_seconds', 'backfill_hours', 'consecutive_failures')))
+    op.bulk_insert(table, [{'code': 'sec', 'name': 'SEC·证券监管', 'list_url': 'https://www.sec.gov/newsroom/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'cftc', 'name': 'CFTC·衍生品监管', 'list_url': 'https://www.cftc.gov/PressRoom/PressReleases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'treasury', 'name': '美国财政部', 'list_url': 'https://home.treasury.gov/news/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'ofac', 'name': 'OFAC·金融制裁', 'list_url': 'https://ofac.treasury.gov/recent-actions', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'fincen', 'name': 'FinCEN·反洗钱', 'list_url': 'https://www.fincen.gov/news/press-releases', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'whitehouse', 'name': '白宫·政策', 'list_url': 'https://www.whitehouse.gov/presidential-actions/', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'congress', 'name': '美国国会·立法', 'list_url': 'https://api.congress.gov/v3/bill?format=json&limit=250', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 900, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'federal_register', 'name': 'Federal Register·规则', 'list_url': 'https://www.federalregister.gov/api/v1/documents.json', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 900, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'occ', 'name': 'OCC·银行与稳定币', 'list_url': 'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/news-releases/2026-news-releases.html', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}, {'code': 'doj', 'name': 'DOJ·加密机构执法', 'list_url': 'https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date', 'adapter': 'builtin', 'collector': 'news', 'enabled': False, 'interval_seconds': 300, 'backfill_hours': 72, 'consecutive_failures': 0}])
+
+
+def downgrade():
+    # Foreign keys intentionally reject removal of sources with collected data.
+    op.execute("DELETE FROM sources WHERE code IN ('sec','cftc','treasury','ofac','fincen','whitehouse','congress','federal_register','occ','doj')")
+    op.drop_column('news_revisions', 'extra')
+    op.drop_column('crawl_runs', 'stats')
+    op.drop_column('sources', 'history_cursor')

backend/src/alphapulse/ai/tools.py

@@ -22,6 +22,7 @@ from alphapulse.api.calendar import calendar_data
 from alphapulse.api.market import call_binance, client, tradable_symbols
 from alphapulse.api.news import news_filters
 from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import SOURCES as US_SOURCES
 from alphapulse.models import ContextVersion, NewsItem, SkillVersion
 from alphapulse.models.market import INTERVALS
 
@@ -34,6 +35,8 @@ SOURCE_NAMES = {
     "coindesk": "CoinDesk", "fed": "美联储·政策与讲话", "bls": "美国劳工统计局·CPI/PPI/就业", "bea": "美国经济分析局",
 }
 
+SOURCE_NAMES.update({code: spec.name for code, spec in US_SOURCES.items()})
+
 
 @dataclass
 class ToolOutcome:
@@ -59,7 +62,7 @@ def dump(data: Any) -> str:
 
 class SearchNews(BaseModel):
     query: Annotated[str, Field(description="关键字，按标题和正文字面匹配；比特币/Bitcoin、以太坊/Ethereum、美联储、CPI/PPI/非农/GDP/PCE 等完整主题词会同时匹配中英文；其他词请分别使用原文语言检索")]
-    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea"]] | None, Field(description="限定来源，不填为全部")] = None
+    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea", "sec", "cftc", "treasury", "ofac", "fincen", "whitehouse", "congress", "federal_register", "occ", "doj", "ecb", "boe", "eurostat", "ons", "destatis", "insee"]] | None, Field(description="限定来源，不填为全部")] = None
     start: Annotated[datetime | None, Field(description="开始时间（ISO 8601，带时区），如 2026-10-01T00:00:00+08:00")] = None
     end: Annotated[datetime | None, Field(description="结束时间（ISO 8601，带时区）")] = None
     limit: Annotated[int, Field(ge=1, le=20)] = 10
@@ -70,11 +73,11 @@ async def search_news(args: SearchNews) -> ToolOutcome:
         conditions = news_filters(args.sources, args.query, args.start, args.end)
         total = await session.scalar(select(func.count()).select_from(NewsItem).where(*conditions))
         rows = (await session.execute(
-            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200))
+            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200), NewsItem.extra)
             .where(*conditions).order_by(NewsItem.published_at.desc()).limit(args.limit)
         )).all()
     items = [
-        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": local(r[2]), "title": r[3], "brief": r[4] or r[5]}
+        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": r[6].get("published_date") if r[6].get("published_at_precision") == "date" else local(r[2]), "title": r[3], "brief": r[4] or r[5]}
         for r in rows
     ]
     return ToolOutcome(dump({"total": total, "returned": len(items), "items": items}))
@@ -90,10 +93,13 @@ async def get_news(args: GetNews) -> ToolOutcome:
     if news is None:
         return ToolOutcome(dump({"error": f"新闻 {args.id} 不存在"}))
     return ToolOutcome(dump({
-        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": local(news.published_at),
+        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": news.extra.get("published_date") if news.extra.get("published_at_precision") == "date" else local(news.published_at),
         "title": news.title, "url": news.url, "is_paywalled": news.is_paywalled,
         "content": news.content_text[:NEWS_BODY_LIMIT], "truncated": len(news.content_text) > NEWS_BODY_LIMIT,
         "content_scope": news.extra.get("content_scope", "article"),
+        "published_at_precision": news.extra.get("published_at_precision"),
+        "published_date": news.extra.get("published_date"),
+        "attachments": news.extra.get("attachments", []),
     }))
 
 

backend/src/alphapulse/api/crawler.py

@@ -19,6 +19,7 @@ from alphapulse.core.redis import KEY_PREFIX, redis
 from alphapulse.core.official_sources import REGIONS
 from alphapulse.calendar.catalog import FAMILIES
 from alphapulse.crawler.events import CONTROL_CHANNEL, HISTORY_SOURCES, LOG_CHANNEL, STATUS_KEY
+from alphapulse.crawler.spiders.us_official import configuration_error
 from alphapulse.models import CrawlLog, CrawlRun, EconomicRelease, NewsItem, Source
 
 router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
@@ -46,10 +47,13 @@ class SourceOut(BaseModel):
     last_success_at: datetime | None
     last_error: str | None
     paused_until: datetime | None
+    configuration_error: str | None = None
+    history_cursor: dict = Field(default_factory=dict)
     supports_history: bool
     history_target: datetime | None
     history_position: datetime | None
     history_done: bool
+    collection_stats: dict = Field(default_factory=dict)
     # 管理进程未运行时为 None
     process: ProcessStatus | None
     today_new: int
@@ -180,6 +184,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
     totals = dict((await session.execute(
         select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
     )).all())
+    latest_runs = (await session.scalars(select(CrawlRun).order_by(CrawlRun.source_code, CrawlRun.started_at.desc()).distinct(CrawlRun.source_code))).all()
+    run_stats = {run.source_code: run.stats for run in latest_runs}
     statuses = await redis.hgetall(STATUS_KEY)
     db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
     return success(SourcesOut(
@@ -191,6 +197,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
                 **s.__dict__,
                 "region_code": REGIONS.get(s.code),
                 "supports_history": s.code in HISTORY_SOURCES,
+                "configuration_error": configuration_error(s.code),
+                "collection_stats": run_stats.get(s.code, {}),
                 "process": json.loads(statuses[s.code]) if s.code in statuses else None,
                 "today_new": today_counts.get(s.code, (0, 0))[0],
                 "today_revised": today_counts.get(s.code, (0, 0))[1],
@@ -206,6 +214,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
     source = await session.get(Source, code)
     if source is None or source.collector != "news":
         raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    if body.enabled and (error := configuration_error(code)):
+        raise HTTPException(status.HTTP_409_CONFLICT, error)
     for field, value in body.model_dump(exclude_none=True, exclude={"history_target"}).items():
         setattr(source, field, value)
     if "history_target" in body.model_fields_set:
@@ -214,6 +224,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
         if body.history_target is not None and body.history_target >= datetime.now(LOCAL_TZ):
             raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "回补目标时间必须早于现在")
         # 保留断点：改目标或重新开始时从已回补的位置继续
+        if body.history_target is not None and (source.history_cursor or {}).get("history", {}).get("scanned"):
+            source.history_cursor = {k: v for k, v in source.history_cursor.items() if k != "history"}
         source.history_target = body.history_target
         source.history_done = False
     await session.commit()

backend/src/alphapulse/api/news.py

@@ -55,6 +55,8 @@ class NewsBrief(BaseModel):
     first_seen_at: datetime
     revision_no: int
     risk: NewsRiskBrief | None = None
+    published_at_precision: str | None = None
+    published_date: str | None = None
 
 
 class NewsPage(BaseModel):
@@ -71,6 +73,7 @@ class NewsDetail(NewsBrief):
 
 
 class Revision(BaseModel):
+    extra: dict | None = None
     revision_no: int
     title: str
     summary: str
@@ -131,7 +134,7 @@ async def list_news(
     items = []
     for news, excerpt, package in rows:
         risk = {'state': package.state} if package else None
-        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk}))
+        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk, 'published_at_precision': news.extra.get('published_at_precision'), 'published_date': news.extra.get('published_date')}))
     return success(NewsPage(items=items, total=total))
 
 
@@ -154,7 +157,7 @@ async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_se
     news = await session.get(NewsItem, news_id)
     if news is None:
         raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
-    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]}))
+    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH], "published_at_precision": news.extra.get("published_at_precision"), "published_date": news.extra.get("published_date")}))
 
 
 @router.get("/{news_id}/revisions")

backend/src/alphapulse/core/config.py

@@ -7,6 +7,9 @@ class Settings(BaseSettings):
     # 容器内由 compose 的 env_file 注入；只有一套配置，见仓库根目录 .env
     model_config = SettingsConfigDict(extra="ignore")
 
+    congress_api_key: str = ""
+    crawler_contact_email: str = ""
+
     searxng_url: str = "http://searxng:8080"
     database_url: str
     redis_url: str

backend/src/alphapulse/core/official_sources.py

@@ -98,3 +98,7 @@ def relevant_news(source: str, title: str, url: str) -> bool:
     if source == "insee":
         return any(t in title for t in ("prix à la consommation", "taux de chômage", "le pib", "le produit intérieur brut"))
     raise ValueError(f"未支持的欧洲新闻来源：{source}")
+
+# US adapters share the existing regional directory.
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+REGIONS.update({code: "US" for code in US_SOURCES})

backend/src/alphapulse/core/us_news.py

@@ -0,0 +1,171 @@
+"""Official US news catalog and deterministic collection policy."""
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+from parsel import Selector
+
+
+@dataclass(frozen=True)
+class OfficialSource:
+    name: str
+    entries: tuple[str, ...]
+    article_path: str
+    body: str
+    hosts: tuple[str, ...]
+    interval: int = 300
+
+
+SOURCES = {
+    'sec': OfficialSource('SEC·证券监管', ('https://www.sec.gov/newsroom/press-releases', 'https://www.sec.gov/newsroom/speeches-statements', 'https://www.sec.gov/rules-regulations/rulemaking-activity', 'https://www.sec.gov/rules-regulations/self-regulatory-organization-rulemaking'), r'/(?:newsroom/(?:press-releases|speeches-statements)/[^/]+|rules-regulations/.*\d.*)', '.node-details-layout__main-region__content .field--name-body', ('www.sec.gov',)),
+    'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+)|csl/)', 'article .field--name-body', ('www.cftc.gov',)),
+    'treasury': OfficialSource('美国财政部', ('https://home.treasury.gov/news/press-releases',), r'/news/press-releases/[a-z]+\d+/?$', '.field--name-field-news-body', ('home.treasury.gov', 'www.treasury.gov', 'ofac.treasury.gov')),
+    'ofac': OfficialSource('OFAC·金融制裁', ('https://ofac.treasury.gov/recent-actions',), r'/recent-actions/\d{8}[^/]*$', '.field--name-body, .body-content', ('ofac.treasury.gov', 'home.treasury.gov')),
+    'fincen': OfficialSource('FinCEN·反洗钱', ('https://www.fincen.gov/news/press-releases', 'https://www.fincen.gov/news/enforcement-actions', 'https://www.fincen.gov/resources/advisoriesbulletinsfact-sheets'), r'/(?:news/news-releases/[^/]+|resources/advisories/[^/]+|news/enforcement-actions/[^/]+)', 'article .field--name-body', ('www.fincen.gov',)),
+    'whitehouse': OfficialSource('白宫·政策', ('https://www.whitehouse.gov/presidential-actions/', 'https://www.whitehouse.gov/fact-sheets/', 'https://www.whitehouse.gov/briefings-statements/', 'https://bidenwhitehouse.archives.gov/briefing-room/'), r'/(?:presidential-actions|fact-sheets|briefings-statements|briefing-room/[^/]+)/202[5-9]/\d{2}/\d{2}/|/(?:presidential-actions|fact-sheets|briefings-statements)/202[5-9]/\d{2}/[^/]+', '.wp-block-post-content, .body-content, .entry-content', ('www.whitehouse.gov', 'bidenwhitehouse.archives.gov')),
+    'congress': OfficialSource('美国国会·立法', ('https://api.congress.gov/v3/bill?format=json&limit=250', 'https://api.congress.gov/v3/committee-meeting?format=json&limit=250'), r'/bill/', '', ('api.congress.gov', 'www.congress.gov', 'www.govinfo.gov'), 900),
+    'federal_register': OfficialSource('Federal Register·规则', ('https://www.federalregister.gov/api/v1/documents.json',), r'/documents/', '', ('www.federalregister.gov', 'public-inspection.federalregister.gov', 'www.govinfo.gov'), 900),
+    'occ': OfficialSource('OCC·银行与稳定币', tuple(f'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/{kind}/{year}-{kind}.html' for year in range(datetime.now(UTC).year, 2024, -1) for kind in ('news-releases', 'bulletins')), r'/news-issuances/(?:news-releases|bulletins|interpretations-and-precedents)/202[5-9]/[^/]+', '.occgov-issuance-content', ('www.occ.gov', 'occ.gov', 'www.occ.treas.gov', 'occ.treas.gov')),
+    'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)),
+    'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speeches', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
+}
+RULE_VERSION = 'us-policy-v1'
+CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I)
+POLICY = re.compile(r'\b(?:regulat\w*|rules?|guidance|legislation|acts?|sanctions?|licenses?|executive order|monetary policy|federal funds|interest rates?|balance sheet|liquidity|quantitative|refunding|borrowing|debt limit|tariffs?|fiscal|budget|appropriation|tax reform)\b', re.I)
+INSTITUTION = re.compile(r'\b(?:exchanges?|banks?|financial institutions?|payment processors?|custodians?|stablecoin issuers?|binance|coinbase|tether|circle|kraken|bybit|bitfinex)\b', re.I)
+MACRO = re.compile(r'\b(?:monetary policy|federal funds|interest rates?|fomc|balance sheet|liquidity|quantitative|refunding|borrowing estimates|debt (?:limit|ceiling)|tariffs?|fiscal|budget|appropriation|tax reform|financial sanctions)\b', re.I)
+
+
+def selection(source: str, title: str, text: str) -> list[str]:
+    content = f'{title}\n{text}'
+    crypto = bool(CRYPTO.search(content))
+    if crypto and (source not in ('doj', 'ofac', 'fincen') or POLICY.search(title) or INSTITUTION.search(content)):
+        return ['crypto_policy' if POLICY.search(content) else 'crypto_institution']
+    if source not in ('doj', 'cftc', 'sec') and MACRO.search(content):
+        return ['us_macro_policy']
+    if source == 'ofac' and INSTITUTION.search(content) and POLICY.search(content):
+        return ['financial_sanctions']
+    return []
+
+
+def official_url(source: str, url: str, base: str = '') -> str:
+    value = urlsplit(urljoin(base, url))
+    if value.scheme != 'https' or value.hostname not in SOURCES[source].hosts or value.username or value.password:
+        raise ValueError('链接不是该来源的官方 HTTPS 地址')
+    if re.search(r'(?:api_key|token|key)=', value.query, re.I):
+        raise ValueError('官方链接含凭据参数')
+    return urlunsplit((value.scheme, value.netloc, value.path, value.query, ''))
+
+
+def identity(url: str) -> str:
+    return hashlib.sha256(url.encode()).hexdigest()
+
+
+def published(value: str) -> tuple[datetime, str]:
+    if not isinstance(value, str):
+        raise ValueError('官方材料缺少可识别的发布时间')
+    value = value.strip().replace('\xa0', ' ')
+    if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
+        return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
+    try:
+        dt = datetime.fromisoformat(value.replace('Z', '+00:00'))
+        if dt.tzinfo:
+            return dt.astimezone(UTC), 'datetime'
+    except ValueError:
+        pass
+    match = re.search(r'([A-Z][a-z]+\.? \d{1,2}, \d{4}|\d{1,2}/\d{1,2}/\d{4})', value)
+    if match:
+        for fmt in ('%B %d, %Y', '%b %d, %Y', '%m/%d/%Y'):
+            try:
+                return datetime.strptime(match[0].replace('.', ''), fmt).replace(tzinfo=UTC), 'date'
+            except ValueError:
+                pass
+    raise ValueError('官方材料缺少可识别的发布时间')
+
+
+def validate_body(content: str) -> None:
+    """Reject error documents without requiring full-page HTML around fragments."""
+    if content.lstrip().startswith('<?xml'):
+        from lxml import etree
+        try:
+            etree.fromstring(content.encode(), etree.XMLParser(resolve_entities=False, no_network=True))
+        except etree.XMLSyntaxError as exc:
+            raise ValueError('官方 XML 正文无法解析') from exc
+    sel = Selector(content)
+    heading = ' '.join(sel.css('title, h1').xpath('normalize-space(.)').getall())
+    text = ' '.join(sel.xpath('//text()[not(ancestor::script or ancestor::style)]').getall()).strip()
+    if re.search(r'just a moment|access denied|verify (?:you are|your)|sign in|log in|captcha|page not found|service unavailable|bad gateway', heading, re.I) or sel.css('#challenge-form, #cf-challenge-running, script[src*="/cdn-cgi/challenge"]'):
+        raise ValueError('官方正文返回验证或登录页面')
+    if re.search(r'enable javascript and cookies to continue', text, re.I):
+        raise ValueError('官方正文返回验证页面')
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+
+
+def article(source: str, html: str, url: str) -> dict:
+    sel = Selector(html)
+    validate_body(html)
+    selectors = {
+        'fed': ('#article h3.title', '#article h3', '#article h2'),
+        'treasury': ('h2.uswds-page-title',),
+        'cftc': ('article h1:not(.press-release-title)',),
+    }.get(source, ('.main-content__main h1', '#main_content h1', 'main h1', 'h1'))
+    title = next((value for css in selectors if (value := sel.css(css).xpath('normalize-space(.)').get())), None)
+    if source == 'fed' and '/releases/h41/' in url:
+        title = 'Federal Reserve Balance Sheet — H.4.1'
+    if not title:
+        raise ValueError('官方正文缺少标题')
+    values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
+    for script in sel.css('script[type="application/ld+json"]::text').getall():
+        try:
+            data = json.loads(script)
+            nodes = data if isinstance(data, list) else data.get('@graph', [data])
+            values += [n['datePublished'] for n in nodes if isinstance(n, dict) and n.get('datePublished')]
+        except (ValueError, TypeError, AttributeError):
+            continue
+    values += sel.css('.node-details-layout__main-region__content, .date, .date-display-single, .release-date, .article__time, .published-date, .field--name-field-date, .field--name-field-release-date, .press-release-date').xpath('normalize-space(.)').getall()
+    values += re.findall(r'(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}', ' '.join(sel.css('main, article, #content').xpath('string(.)').getall()))[:3]
+    # OFAC action identifiers encode the publisher's day, not the fetch time.
+    if source == 'fed' and (m := re.search(r'/releases/h41/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    if source == 'ofac' and (m := re.search(r'/recent-actions/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    at = precision = None
+    for value in values:
+        try:
+            at, precision = published(value)
+            break
+        except ValueError:
+            continue
+    if at is None:
+        raise ValueError('官方材料缺少可识别的发布时间')
+    body = sel.css(SOURCES[source].body)
+    if not body:
+        raise ValueError('官方正文结构未识别')
+    for node in body.css('script, style, nav, header, footer, form'):
+        node.root.getparent().remove(node.root)
+    body = sorted(body, key=lambda node: len(node.xpath('string(.)').get()), reverse=True)
+    content = body[0].get()
+    text = ' '.join(body[0].xpath('.//text()').getall()).strip()
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+    attachments = []
+    for link in body[0].css('a[href]'):
+        href = link.attrib['href']
+        label = link.xpath('normalize-space(.)').get()
+        if re.search(r'\.pdf(?:\?|$)|/media/\d+/(?:download|dl)|/csl/[^/]+/download', href, re.I):
+            try:
+                link_url = official_url(source, href, url)
+            except ValueError:
+                candidate = urljoin(url, href)
+                if urlsplit(candidate).scheme == 'https':
+                    attachments.append({'url': candidate, 'title': label, 'status': 'unsupported_host', 'error': '附件不在该来源的官方域名范围内，未下载'})
+                continue
+            if link_url not in [a['url'] for a in attachments]:
+                attachments.append({'url': link_url, 'title': label, 'status': 'pending'})
+    return dict(title=title, content_html=content, published_at=at,
+                extra={'language': 'en', 'content_scope': 'article', 'published_at_precision': precision,
+                       'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'attachments': attachments})

backend/src/alphapulse/crawler/events.py

@@ -12,7 +12,8 @@ NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
 CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
 STATUS_KEY = f"{KEY_PREFIX}crawler:status"
 # Sources whose list endpoints support historical pagination.
-HISTORY_SOURCES = ("wscn_live", "wscn_news")
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+HISTORY_SOURCES = ("wscn_live", "wscn_news", *US_SOURCES)
 
 
 def start_urls_key(source_code: str) -> str:

backend/src/alphapulse/crawler/middlewares.py

@@ -4,6 +4,9 @@ import hashlib
 from urllib.parse import urlparse
 
 from scrapy.crawler import Crawler
+from scrapy.exceptions import IgnoreRequest
+
+from alphapulse.core.us_news import official_url
 from scrapy.http import Request, Response
 from sqlalchemy import insert
 from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -19,6 +22,13 @@ class RedisRateLimitMiddleware:
     interval_ms = 1000
 
     async def process_request(self, request: Request) -> None:
+        if source := request.meta.get("official_source"):
+            try:
+                official_url(source, request.url)
+            except ValueError as exc:
+                raise IgnoreRequest(str(exc)) from exc
+        if urlparse(request.url).hostname != "api.congress.gov":
+            request.headers.pop("X-Api-Key", None)
         key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
         while not await redis.set(key, "1", px=self.interval_ms, nx=True):
             ttl = await redis.pttl(key)

backend/src/alphapulse/crawler/pipelines.py

@@ -51,6 +51,8 @@ class NewsPipeline:
                 )
                 # 插入成功取新行；冲突说明另一处刚写入，按已存在条目走比较/修订流程
                 news = await session.scalar(locked)
+            if inserted_id is None and item["extra"].get("content_scope") == "partial" and news.extra.get("content_scope") == "article":
+                return item
             if inserted_id is not None:
                 kind = "new"
             else:
@@ -74,6 +76,7 @@ class NewsPipeline:
                     content_html=item["content_html"],
                     content_hash=digest,
                     raw_response_id=raw_response_id,
+                    extra=item["extra"],
                 ))
             news_id = news.id
             await session.commit()

backend/src/alphapulse/crawler/spiders/base.py

@@ -65,6 +65,8 @@ class RunState:
     history_position: datetime | None = None
     history_done: bool = False
     history_deadline: float = 0
+    official_cursor: dict | None = None
+    stats: dict = field(default_factory=dict)
 
     @property
     def full_scan(self) -> bool:
@@ -104,6 +106,7 @@ class NewsSpider(RedisSpider):
         self._tasks: set[asyncio.Task] = set()
         # 上一轮成功后的回补进度 (目标, 断点, 是否结束)。写库是异步的，管理进程推来的下一条触发可能还带着旧断点
         self.history_checkpoint: tuple[datetime, datetime, bool] | None = None
+        self.finish_task: asyncio.Task | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
@@ -177,7 +180,7 @@ class NewsSpider(RedisSpider):
         run, self.run = self.run, None
         if run.history_target is not None and not run.errors:
             self.history_checkpoint = (run.history_target, run.history_position, run.history_done)
-        self._spawn(self._finish_run(run))
+        self.finish_task = self._spawn(self._finish_run(run))
 
     def _on_error(self, failure: Failure, response: Response) -> None:
         if self.run:
@@ -191,10 +194,11 @@ class NewsSpider(RedisSpider):
         self.logger.info("采集进程退出")
         await self._flush_logs()
 
-    def _spawn(self, coro: Coroutine) -> None:
+    def _spawn(self, coro: Coroutine) -> asyncio.Task:
         task = asyncio.get_running_loop().create_task(coro)
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
+        return task
 
     async def _insert_run(self, run: RunState) -> None:
         async with SessionLocal() as session:
@@ -211,6 +215,7 @@ class NewsSpider(RedisSpider):
         values = {
             "finished_at": now,
             "status": "failed" if failed else "success",
+            "stats": run.stats,
             "requests": run.requests,
             "items_seen": run.items_seen,
             "items_new": run.items_new,
@@ -227,6 +232,12 @@ class NewsSpider(RedisSpider):
             source = await session.get(Source, self.name, with_for_update=True)
             # 本轮失败不推进断点，下一轮从原位置重做（写入幂等）；目标期间被修改或取消时丢弃本轮进度
             history_saved = not failed and run.history_target is not None and source.history_target == run.history_target
+            if not failed and run.official_cursor is not None:
+                cursor = dict(source.history_cursor or {})
+                cursor.update({k: v for k, v in run.official_cursor.items() if k != "history"})
+                if history_saved:
+                    cursor["history"] = run.official_cursor.get("history", {})
+                source.history_cursor = cursor
             if history_saved:
                 source.history_position = run.history_position
                 source.history_done = run.history_done

backend/src/alphapulse/crawler/spiders/rss.py

@@ -97,12 +97,11 @@ class CoinDeskSpider(RssNewsSpider):
     feeds = ("https://www.coindesk.com/arc/outboundfeeds/rss/",)
 
 
-class FedSpider(RssNewsSpider):
+from alphapulse.crawler.spiders.us_official import USOfficialSpider
+
+
+class FedSpider(USOfficialSpider):
     name = "fed"
-    feeds = (
-        "https://www.federalreserve.gov/feeds/press_monetary.xml",
-        "https://www.federalreserve.gov/feeds/speeches_and_testimony.xml",
-    )
 
 
 class BlsSpider(RssNewsSpider):

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -0,0 +1,572 @@
+"""US official collectors: serialized documents, bounded history, durable page cursors."""
+import copy
+import json
+import re
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from html import escape, unescape
+
+from lxml import etree
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from scrapy import Request
+from scrapy.http import Response
+from sqlalchemy import select
+
+from alphapulse.calendar.umich import pdf_text
+from alphapulse.core.config import get_settings
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import RULE_VERSION, SOURCES, article, identity, official_url, published, selection, validate_body
+from alphapulse.crawler.spiders.base import HISTORY_RUN_SECONDS, NewsSpider, html_to_text
+from alphapulse.models import NewsItem, Source
+
+
+def configure_url(url, **params):
+    parts = urlsplit(url)
+    query = dict(parse_qsl(parts.query))
+    query.update(params)
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def congress_bill_url(api_url):
+    parts = urlsplit(api_url).path.strip('/').split('/')
+    _, _, congress, kind, number = parts
+    labels = {'hr': 'house-bill', 's': 'senate-bill', 'hjres': 'house-joint-resolution', 'sjres': 'senate-joint-resolution', 'hconres': 'house-concurrent-resolution', 'sconres': 'senate-concurrent-resolution', 'hres': 'house-resolution', 'sres': 'senate-resolution'}
+    return f'https://www.congress.gov/bill/{congress}th-congress/{labels[kind.lower()]}/{number}'
+
+
+def configuration_error(code):
+    settings = get_settings()
+    if code == 'congress' and not settings.congress_api_key:
+        return '待配置 CONGRESS_API_KEY'
+    if code == 'sec' and not settings.crawler_contact_email:
+        return '待配置 CRAWLER_CONTACT_EMAIL'
+    return None
+
+
+class USOfficialSpider(NewsSpider):
+    """Each lane holds only one page's remaining documents and its next URL."""
+    custom_settings = {"DOWNLOAD_MAXSIZE": 25 * 1024 * 1024}
+
+    def make_request_from_data(self, data):
+        if getattr(self, '_starting', False):
+            return []
+        if self.run is not None:
+            if not self.crawler.engine.spider_is_idle():
+                return super().make_request_from_data(data)
+            self._close_run()
+        self._starting = True
+        self._spawn(self.start_round(json.loads(data)['trigger']))
+        return []
+
+    def _close_run(self):
+        self._checkpoint_run = self.run
+        self._checkpoint_attempts = 1
+        super()._close_run()
+
+    async def start_round(self, trigger):
+        try:
+            if self.finish_task is not None:
+                if self.finish_task.done() and not self.finish_task.cancelled() and self.finish_task.exception() is not None and getattr(self, '_checkpoint_run', None) is not None:
+                    if self._checkpoint_attempts >= 3:
+                        self.logger.error('断点保存连续失败3次，需重启采集进程；未启动下一轮')
+                        return
+                    self._checkpoint_attempts += 1
+                    self.finish_task = self._spawn(self._finish_run(self._checkpoint_run))
+                await self.finish_task
+            async with SessionLocal() as session:
+                source = await session.get(Source, self.name)
+                if source is None or not source.enabled:
+                    return
+                if source.paused_until and source.paused_until > datetime.now(UTC):
+                    return
+                self.cursor = copy.deepcopy(source.history_cursor or {})
+                history = (source.history_target, source.history_position) if source.history_target and not source.history_done else None
+                hours = source.backfill_hours
+            requests = super().make_request_from_data(json.dumps({'trigger': trigger, 'backfill_hours': hours}).encode())
+            self.run.official_cursor = self.cursor
+            if history:
+                target, position = history
+                self.run.history_target = target
+                self.run.history_position = position or self.run.started_at
+                self.run.history_deadline = time.monotonic() + HISTORY_RUN_SECONDS
+                state = self.cursor.setdefault('history', {'entry': 0})
+                if state.get('target') != target.isoformat():
+                    self.cursor['history'] = {'entry': 0, 'target': target.isoformat()}
+                requests += self.advance('history')
+            for request in requests:
+                self.crawler.engine.crawl(request)
+        except Exception:
+            self.logger.exception('官方采集轮次启动失败，未使用队列旧断点')
+            if self.run is not None:
+                self.run.errors.append('官方采集轮次启动失败')
+        finally:
+            self._starting = False
+
+    def first_requests(self):
+        if self.name == 'doj' and self.cursor.get('transport') != 'doj-api-v1':
+            # Retain retry evidence; HTML discovery cursors cannot use API pagination.
+            self.cursor = {k: v for k, v in self.cursor.items() if k not in ('history', 'realtime', 'checked')}
+            self.cursor['transport'] = 'doj-api-v1'
+            self.run.official_cursor = self.cursor
+        error = configuration_error(self.name)
+        if error:
+            self.run.errors.append(error)
+            return []
+        state = self.cursor.setdefault('realtime', {'entry': 0})
+        due = [value['document'] for value in self.cursor.get('attachment_retries', {}).values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
+        if due and not state.get('pending'):
+            state['pending'] = [{**doc, 'attachment_retry': True} for doc in due]
+        bill_due = [{**v['document'], 'bill_retry': True} for v in self.cursor.get('bill_retries', {}).values() if v['attempts'] < 3 and v['next_at'] <= self.run.started_at.timestamp()]
+        if bill_due:
+            state.setdefault('pending', []).extend(d for d in bill_due if d['url'] not in {p['url'] for p in state['pending']})
+        self.run.stats['unresolved_bills'] = len(self.cursor.get('bill_retries', {}))
+        return self.advance('realtime')
+
+    def request_for(self, url, callback, lane, **meta):
+        url = official_url(self.name, url)
+        request = self.request(url, callback, lane=lane, official_source=self.name, **meta)
+        request.priority = 10 if lane == 'realtime' else -10
+        # Credentials never travel to document hosts or into persisted URLs.
+        if urlsplit(url).hostname == 'api.congress.gov':
+            request.headers['X-Api-Key'] = get_settings().congress_api_key
+        request.headers['User-Agent'] = 'AlphaPulse official news collector ' + get_settings().crawler_contact_email
+        return request
+
+    def advance(self, lane):
+        state = self.cursor[lane]
+        if configuration_error(self.name):
+            return []
+        if lane == 'history' and time.monotonic() >= self.run.history_deadline:
+            return []
+        if state.get('pending'):
+            doc = state['pending'][0]
+            if doc['kind'] == 'bill_finish':
+                self.finish_bill(lane)
+                return self.advance(lane)
+            if doc['kind'] == 'actions' and not state['bill']['reasons']:
+                state['pending'].pop(0)
+                return self.advance(lane)
+            return [self.request_for(doc['url'], self.parse_document, lane, document=doc)]
+        if state.get('page_finished'):
+            if state.pop('day_complete', False):
+                state['day'] = (datetime.fromisoformat(state['day']) - timedelta(days=1)).date().isoformat()
+                state['entry'] -= 1
+            if state.get('next'):
+                state['page'] = state.pop('next')
+            else:
+                state['entry'] += 1
+                state.pop('page', None)
+                state.pop('visited_pages', None)
+            state.pop('page_finished', None)
+        spec = SOURCES[self.name]
+        if state['entry'] >= len(spec.entries):
+            if lane == 'history':
+                self.run.stats['attachment_gaps'] = len(self.cursor.get('attachment_retries', {}))
+                self.run.history_done = True
+                self.run.history_position = self.run.history_target
+                state['scanned'] = True
+            else:
+                self.cursor['realtime'] = {'entry': 0}
+            return []
+        url = state.get('page') or spec.entries[state['entry']]
+        if self.name == 'congress' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            url = configure_url(url, fromDateTime=begin.strftime('%Y-%m-%dT%H:%M:%SZ'), toDateTime=self.run.started_at.strftime('%Y-%m-%dT%H:%M:%SZ'))
+        if self.name == 'federal_register' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            # Daily partitions stay below the API's 2,000-result pagination ceiling.
+            day = state.setdefault('day', self.run.started_at.date().isoformat())
+            if day < begin.date().isoformat():
+                state['entry'] += 1
+                return self.advance(lane)
+            url = configure_url(url, **{'conditions[publication_date][is]': day, 'per_page': '1000', 'order': 'oldest'})
+        state['page'] = url
+        return [self.request_for(url, self.parse_listing, lane)]
+
+    def parse_listing(self, response):
+        lane = response.meta['lane']
+        state = self.cursor[lane]
+        docs, next_url = [], None
+        if self.name == 'doj':
+            data = response.json()
+            resultset = data['metadata']['resultset']
+            if data['metadata']['responseInfo']['status'] != 200 or not isinstance(data['results'], list):
+                raise ValueError('DOJ API 列表结构未识别')
+            dates = []
+            for record in data['results']:
+                dates.append(datetime.fromtimestamp(int(record['date']), UTC))
+                docs.append({'url': f"https://www.justice.gov/api/v1/press_releases/{uuid.UUID(record['uuid'])}.json", 'kind': 'doj'})
+            page, size, count = (int(resultset[k]) for k in ('page', 'pagesize', 'count'))
+            if size <= 0 or page < 0 or (not docs and page * size < count):
+                raise ValueError('DOJ API 分页结果不完整')
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            if (page + 1) * size < count and not (dates and all(d.date() < begin.date() for d in dates)):
+                next_url = configure_url(response.url, page=page + 1)
+        elif self.name == 'fed' and response.url.endswith('releaseDates.json'):
+            data = response.json()
+            if not isinstance(data, list) or not data:
+                raise ValueError('H.4.1 日期列表结构未识别')
+            docs = [{'url': f'https://www.federalreserve.gov/releases/h41/{day}/', 'kind': 'article'} for year in data for month in year['Months'] for day in month['Dates'] if int(year['yearValue']) >= 2025]
+            docs.sort(key=lambda d: d['url'], reverse=True)
+        elif self.name in ('congress', 'federal_register'):
+            data = response.json()
+            key = ('committeeMeetings' if '/committee-meeting' in response.url else 'bills') if self.name == 'congress' else 'results'
+            if key not in data or not isinstance(data[key], list):
+                raise ValueError('官方 API 列表结构未识别')
+            if self.name == 'federal_register' and data.get('count', 0) > 2000:
+                day = state['day']
+                url = f'https://www.govinfo.gov/content/pkg/FR-{day}/xml/FR-{day}.xml'
+                state['pending'] = [{'url': url, 'kind': 'register_issue', 'day': day, 'count': data['count']}]
+                yield from self.advance(lane)
+                return
+            for doc in data[key]:
+                if self.name == 'congress':
+                    docs.append({'url': configure_url(doc['url'], format='json'), 'kind': 'meeting' if key == 'committeeMeetings' else 'bill'})
+                else:
+                    docs.append({'url': f"https://www.federalregister.gov/api/v1/documents/{doc['document_number']}.json", 'kind': 'register'})
+            next_url = data.get('pagination', {}).get('next') if self.name == 'congress' else data.get('next_page_url')
+        else:
+            if not response.css('h1'):
+                raise ValueError('官方列表页面未识别')
+            pattern = SOURCES[self.name].article_path
+            for a in response.css('a[href]'):
+                try:
+                    url = official_url(self.name, a.attrib['href'], response.url)
+                except ValueError:
+                    continue
+                if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
+                    docs.append({'url': url, 'kind': 'article'})
+                if a.attrib.get('rel') == 'next' or re.match(r'^(?:Next|Next page|下一页)', a.xpath('normalize-space(.)').get() or '', re.I):
+                    next_url = url
+            docs = list({d['url']: d for d in docs}.values())
+            if not docs:
+                raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        if lane == 'realtime':
+            # OFAC's recent-actions entry is a descending day-indexed feed.
+            if self.name == 'ofac' and docs:
+                days = [re.search(r'/recent-actions/(\d{8})', d['url']) for d in docs]
+                if all(days) and all(m[1] < self.run.cutoff.strftime('%Y%m%d') for m in days):
+                    next_url = None
+            checked = self.cursor.setdefault('checked', {})
+            threshold = (self.run.started_at - timedelta(hours=24)).timestamp()
+            self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
+            docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
+            if self.name == 'fed':
+                def recent(d):
+                    found = re.search(r'(202[5-9]\d{4})', d['url'])
+                    return not found or datetime.strptime(found[1], '%Y%m%d').replace(tzinfo=UTC) >= self.run.cutoff
+                docs = [d for d in docs if recent(d)]
+        if next_url and official_url(self.name, next_url, response.url) == response.url:
+            raise ValueError('官方列表翻页重复当前页面，未推进断点')
+        visited = state.setdefault('visited_pages', [])
+        if response.url in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        visited.append(response.url)
+        if next_url and official_url(self.name, next_url, response.url) in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        state['pending'] = docs
+        state['next'] = official_url(self.name, next_url, response.url) if next_url else None
+        state['page_finished'] = True
+        self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(docs)
+        if self.name == 'federal_register' and not next_url:
+            state['day_complete'] = True
+        yield from self.advance(lane)
+
+    async def parse_document(self, response):
+        lane, doc = response.meta['lane'], response.meta['document']
+        if doc['kind'] == 'register_issue':
+            root = etree.fromstring(response.body, etree.XMLParser(resolve_entities=False, no_network=True))
+            if root.tag != 'FEDREG' or published(root.findtext('DATE', default=''))[0].date().isoformat() != doc['day']:
+                raise ValueError('Federal Register XML 日期或结构不符')
+            numbers = set()
+            for node in root.iter('FRDOC'):
+                match = re.search(r'FR Doc\.\s+([^\s\]]+)', ''.join(node.itertext()))
+                if not match:
+                    raise ValueError('Federal Register XML 文档编号无法解析')
+                numbers.add(match[1])
+            if len(numbers) != doc['count']:
+                raise ValueError('Federal Register XML 文档数量与 API 不符')
+            state = self.cursor[lane]
+            state['pending'] = [{'url': f'https://www.federalregister.gov/api/v1/documents/{number}.json', 'kind': 'register'} for number in sorted(numbers)]
+            state.update(page_finished=True, next=None, day_complete=True)
+            self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(numbers)
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'meeting':
+            data = response.json()['committeeMeeting']
+            at, precision = published(data['updateDate'])
+            if not selection(self.name, data['title'], json.dumps(data.get('relatedItems', {}))):
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+                for result in self.complete(lane):
+                    yield result
+                return
+            parsed = {'title': data['title'], 'content_html': '<pre>' + escape(json.dumps(data, ensure_ascii=False, indent=2)) + '</pre>', 'published_at': at, 'url': response.url, 'external_id': identity(response.url), 'raw_response_id': response.meta['raw_response_id'],
+                      'extra': {'content_scope': 'article', 'published_at_precision': precision, 'published_at_basis': 'official_update', 'published_date': at.date().isoformat(), 'official_meeting': data, 'rule_version': RULE_VERSION, 'attachments': [{'url': official_url(self.name, d['url']), 'title': d.get('name') or d.get('documentType', ''), 'status': 'pending'} for d in {d['url']: d for d in data.get('meetingDocuments', []) + data.get('witnessDocuments', []) if d.get('format') == 'PDF'}.values()]}}
+            for result in self.attach_or_finish(parsed, lane):
+                yield result
+            return
+        if doc['kind'] == 'bill':
+            data = response.json()['bill']
+            base = response.url.split('?')[0]
+            state = self.cursor[lane]
+            previous = self.cursor.setdefault('bill_retries', {}).get(base)
+            version = data.get('updateDateIncludingText') or data.get('updateDate')
+            if previous and previous.get('version') != version:
+                self.cursor['bill_retries'].pop(base)
+            if previous and previous.get('version') == version and (previous['attempts'] >= 3 or previous['next_at'] > self.run.started_at.timestamp()):
+                for result in self.complete(lane):
+                    yield result
+                return
+            reasons = selection(self.name, data['title'], json.dumps(data.get('policyArea', {})))
+            state['bill'] = {'url': base, 'document_url': doc['url'], 'title': data['title'], 'reasons': reasons, 'bodies': 0, 'version': version, 'retry': bool(doc.get('bill_retry') or doc.get('attachment_retry'))}
+            kinds = ([] if reasons else ['subjects', 'summaries']) + ['texts', 'actions', 'bill_finish']
+            state['pending'][0:1] = [{'url': base + '/' + ('text' if kind == 'texts' else kind) + '?format=json&limit=250', 'kind': kind, 'title': data['title'], 'bill': base} for kind in kinds]
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('subjects', 'summaries'):
+            data = response.json()
+            key = 'subjects' if doc['kind'] == 'subjects' else 'summaries'
+            if key not in data:
+                raise ValueError('官方法案补充资料结构未识别')
+            state = self.cursor[lane]
+            state['bill']['reasons'] = state['bill']['reasons'] or selection(self.name, doc['title'], json.dumps(data[key]))
+            children = []
+            if next_url := data.get('pagination', {}).get('next'):
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            state['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('actions', 'texts'):
+            data = response.json()
+            children = []
+            if doc['kind'] == 'actions':
+                for action in data['actions']:
+                    key = identity(doc['bill'] + json.dumps(action, sort_keys=True))
+                    at, precision = published(action['actionDate'])
+                    begin = datetime(2025, 1, 1, tzinfo=UTC) if self.cursor[lane]['bill'].get('retry') else (self.run.history_target if lane == 'history' else self.run.cutoff)
+                    if at < begin:
+                        continue
+                    item = self.item(response, external_id=key, url=congress_bill_url(doc['bill']), title=doc['title'], summary='', content_html=f"<p>{escape(action['text'])}</p>", published_at=at,
+                                     extra={'content_scope': 'official_action', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'bill': doc['bill'], 'official_action': action, 'rule_version': RULE_VERSION})
+                    yield item
+            else:
+                for version in data['textVersions']:
+                    if not version.get('date'):
+                        self.cursor[lane]['bill']['missing_material'] = True
+                        continue
+                    formats = version.get('formats', [])
+                    chosen = next((f for f in formats if f['type'] == 'Formatted Text'), None) or next((f for f in formats if f['type'] == 'Formatted XML'), None) or next((f for f in formats if f['type'] == 'PDF'), None)
+                    if not chosen:
+                        self.cursor[lane]['bill']['missing_material'] = True
+                    if chosen:
+                        children.append({'url': chosen['url'], 'kind': 'text', 'title': doc['title'] + ' — ' + version['type'], 'at': version['date'], 'bill': doc['bill']})
+            next_url = data.get('pagination', {}).get('next')
+            if next_url:
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            self.cursor[lane]['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'register':
+            data = response.json()
+            url = data.get('body_html_url') or data.get('raw_text_url')
+            if not url:
+                raise ValueError('规则文档缺少正文地址')
+            self.cursor[lane]['pending'][0] = {'url': official_url(self.name, url), 'kind': 'text', 'title': data['title'], 'at': data['publication_date'], 'canonical': data['html_url'], 'document_number': data['document_number'], 'effective_on': data.get('effective_on'), 'pdf': data.get('pdf_url')}
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'doj':
+            data = response.json()
+            records = data['results']
+            if data['metadata']['responseInfo']['status'] != 200 or len(records) != 1:
+                raise ValueError('DOJ API 正文结构未识别')
+            record = records[0]
+            if str(uuid.UUID(record['uuid'])) != doc['url'].rsplit('/', 1)[1].removesuffix('.json'):
+                raise ValueError('DOJ API 正文 UUID 不符')
+            canonical = official_url(self.name, record['url'].replace('https://justice.gov/', 'https://www.justice.gov/', 1))
+            title = unescape(record['title']).strip()
+            if not title:
+                raise ValueError('DOJ 正文缺少标题')
+            # The API release date is a calendar date, not a release instant.
+            date = datetime.fromtimestamp(int(record['date']), UTC).date().isoformat()
+            attachment_html = record.get('attachment') or ''
+            api_attachments = []
+            if isinstance(attachment_html, list):
+                for attachment in attachment_html:
+                    file_id = attachment['file']['id']
+                    if not str(file_id).isdigit():
+                        raise ValueError('DOJ API 附件编号无效')
+                    api_attachments.append({'url': f'https://www.justice.gov/media/{file_id}/dl', 'title': attachment.get('description') or 'Official attachment', 'status': 'pending'})
+                attachment_html = ''
+            elif not isinstance(attachment_html, str):
+                raise ValueError('DOJ API 附件结构未识别')
+            html = f'<h1>{escape(title)}</h1><time datetime="{date}"></time><article><div class="field--name-body">{record["body"]}{attachment_html}</div></article>'
+            validate_body(record['body'])
+            parsed = article(self.name, html, canonical)
+            known = {a['url'] for a in parsed['extra']['attachments']}
+            parsed['extra']['attachments'].extend(a for a in api_attachments if a['url'] not in known)
+            changed = etree.HTML(record.get('changed') or '')
+            if changed is not None:
+                times = changed.xpath('//time/@datetime')
+                if times:
+                    parsed['source_updated_at'] = published(times[0])[0]
+            parsed['extra']['official_uuid'] = record['uuid']
+            doc = {**doc, 'canonical': canonical}
+        elif doc['kind'] == 'text':
+            at, precision = published(doc['at'])
+            if response.body.startswith(b'%PDF'):
+                content = '<pre>' + escape(pdf_text(response.body)) + '</pre>'
+            else:
+                content = response.text
+                validate_body(content)
+                if '<' not in content:
+                    content = '<pre>' + escape(content) + '</pre>'
+            parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}}
+            if doc.get('pdf'):
+                parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
+        else:
+            parsed = article(self.name, response.text, response.url)
+        if doc.get('bill'):
+            context = self.cursor[lane]['bill']
+            context['bodies'] += 1
+            if not context['reasons']:
+                context['reasons'] = selection(self.name, context['title'], html_to_text(parsed['content_html']))
+                if context['reasons']:
+                    # A later version can establish relevance for earlier versions.
+                    self.cursor[lane]['pending'][1:1] = context.pop('unmatched_texts', [])
+                else:
+                    context.setdefault('unmatched_texts', []).append(copy.deepcopy(doc))
+            parsed['extra']['selection_reasons'] = context['reasons']
+        begin = self.run.history_target if lane == 'history' else self.run.cutoff
+        if doc.get('attachment_retry') or (doc.get('bill') and self.cursor[lane]['bill'].get('retry')):
+            begin = datetime(2025, 1, 1, tzinfo=UTC)
+        if parsed['published_at'] < begin:
+            self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
+            for result in self.complete(lane):
+                yield result
+            return
+        if lane == 'history':
+            self.run.history_position = min(self.run.history_position or self.run.started_at, parsed['published_at'])
+        parsed['url'] = doc.get('canonical') or response.url
+        parsed['external_id'] = identity(parsed['url'])
+        if self.name == 'fed':
+            # Preserve pre-existing RSS GUID identities during full-text enrichment.
+            async with SessionLocal() as session:
+                existing = await session.scalar(select(NewsItem.external_id).where(NewsItem.source_code == 'fed', NewsItem.url == parsed['url']).limit(1))
+                if existing:
+                    parsed['external_id'] = existing
+        parsed['raw_response_id'] = response.meta['raw_response_id']
+        parsed['retry_document'] = {'url': doc['bill'], 'kind': 'bill'} if doc.get('bill') else doc
+        for result in self.attach_or_finish(parsed, lane):
+            yield result
+
+    def finish_bill(self, lane):
+        context = self.cursor[lane].pop('bill')
+        retries = self.cursor.setdefault('bill_retries', {})
+        if not context['bodies'] or context.get('missing_material'):
+            previous = retries.get(context['url'], {})
+            retries[context['url']] = {'document': {'url': context['url'], 'kind': 'bill'}, 'version': context['version'], 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        else:
+            retries.pop(context['url'], None)
+            if not context['reasons']:
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        self.run.stats['unresolved_bills'] = len(retries)
+        self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
+
+    def attach_or_finish(self, parsed, lane):
+        pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
+        if pending:
+            request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
+            request.meta['handle_httpstatus_list'] = [404, 410]
+            yield request
+            return
+        text = html_to_text(parsed['content_html'])
+        reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
+        incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
+        retry_doc = parsed.pop('retry_document', None)
+        retries = self.cursor.setdefault('attachment_retries', {})
+        if incomplete and retry_doc:
+            previous = retries.get(parsed['url'], {})
+            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        elif not incomplete:
+            retries.pop(parsed['url'], None)
+        if reasons or incomplete:
+            parsed['extra']['selection_reasons'] = reasons
+            parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
+            self.run.stats['partial'] = self.run.stats.get('partial', 0) + int(incomplete)
+            raw_id = parsed.pop('raw_response_id')
+            item = self.item(Response(parsed['url'], request=Request(parsed['url'], meta={'raw_response_id': raw_id})), summary='', **parsed)
+            yield item
+        else:
+            self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        yield from self.complete(lane)
+
+    def parse_attachment(self, response):
+        parsed = response.meta['parsed']
+        attachment = next(a for a in parsed['extra']['attachments'] if a['url'] == response.url or a['status'] == 'pending')
+        attachment['raw_response_id'] = response.meta['raw_response_id']
+        try:
+            if response.status != 200:
+                raise ValueError(f'HTTP {response.status}')
+            text = pdf_text(response.body)
+            attachment['status'] = 'complete'
+            parsed['content_html'] += f"<section><h2>{escape(attachment['title'])}</h2><pre>{escape(text)}</pre></section>"
+        except (ValueError, OSError) as exc:
+            attachment.update(status='unavailable', error=str(exc))
+            self.logger.warning('附件缺口：%s', response.url)
+        yield from self.attach_or_finish(parsed, response.meta['lane'])
+
+    def complete(self, lane):
+        doc = self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[doc['url']] = self.run.started_at.timestamp()
+        return self.advance(lane)
+
+
+class SecSpider(USOfficialSpider):
+    name = 'sec'
+
+
+class CftcSpider(USOfficialSpider):
+    name = 'cftc'
+
+
+class TreasurySpider(USOfficialSpider):
+    name = 'treasury'
+
+
+class OfacSpider(USOfficialSpider):
+    name = 'ofac'
+
+
+class FincenSpider(USOfficialSpider):
+    name = 'fincen'
+
+
+class WhitehouseSpider(USOfficialSpider):
+    name = 'whitehouse'
+
+
+class CongressSpider(USOfficialSpider):
+    name = 'congress'
+
+
+class FederalRegisterSpider(USOfficialSpider):
+    name = 'federal_register'
+
+
+class OccSpider(USOfficialSpider):
+    name = 'occ'
+
+
+class DojSpider(USOfficialSpider):
+    custom_settings = {**USOfficialSpider.custom_settings, 'DOWNLOAD_DELAY': 1.0}
+    name = 'doj'

backend/src/alphapulse/models/news.py

@@ -25,6 +25,7 @@ class Source(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # 历史回补：目标时间（为空表示不回补）、已回补到的最早时间（断点）、是否已结束（到达目标或来源无更早数据）
+    history_cursor: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     history_target: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_position: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
@@ -71,6 +72,7 @@ class NewsRevision(Base):
     summary: Mapped[str] = mapped_column(Text)
     content_html: Mapped[str] = mapped_column(Text)
     content_hash: Mapped[str] = mapped_column(String(64))
+    extra: Mapped[dict | None] = mapped_column(JSONB)
     captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
 
@@ -110,6 +112,7 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # running / success / failed / interrupted
     status: Mapped[str] = mapped_column(String(16))
+    stats: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     requests: Mapped[int] = mapped_column(Integer, default=0)
     items_seen: Mapped[int] = mapped_column(Integer, default=0)
     items_new: Mapped[int] = mapped_column(Integer, default=0)

backend/tests/test_congress_capture.py

@@ -0,0 +1,93 @@
+import asyncio
+import base64
+import gzip
+import json
+from pathlib import Path
+import runpy
+
+import httpx
+import pytest
+
+
+SCRIPT = Path(__file__).resolve().parents[2] / 'scripts/capture_congress.py'
+
+
+def test_capture_redacts_and_reuses_successful_response(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    seen = []
+    def handle(request):
+        seen.append(request)
+        return httpx.Response(200, json={'request': {'apiKey': 'private-test-secret'}, 'pagination': {'next': 'https://api.congress.gov/v3/bill?page=2&api_key=private-test-secret'}})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        url = 'https://api.congress.gov/v3/bill?format=json'
+        first = capture.fetch(url)
+        assert capture.fetch(url) == first and len(seen) == 1
+        loaded = json.load(gzip.open(path, 'rt'))
+        assert 'private-test-secret' not in base64.b64decode(loaded[url]['body']).decode()
+        assert seen[0].headers['X-Api-Key'] == 'private-test-secret'
+        capture.fetch('https://www.govinfo.gov/test.pdf')
+        assert 'X-Api-Key' not in seen[-1].headers
+
+
+def test_capture_stops_after_rate_limit_and_reports_incomplete(trading, tmp_path):
+    module = runpy.run_path(str(SCRIPT))
+    calls = []
+    def handle(request):
+        calls.append(request)
+        return httpx.Response(429, headers={'Retry-After': '123'})
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](path, 'private-test-secret', client=client)
+        report = asyncio.run(module['run'](capture))
+    assert len(calls) == 1 and not path.exists()
+    assert report['accepted'] is False
+    assert len(report['cases']) == 1
+    assert '123' in report['cases'][0]['error']
+    assert 'private-test-secret' not in json.dumps(report)
+
+
+def test_capture_keeps_progress_on_budget_exhaustion(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    path = tmp_path / 'cache.gz'
+    with httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b'body'))) as client:
+        capture = module['Capture'](path, 'test', limit=1, client=client)
+        capture.fetch('https://www.govinfo.gov/first.pdf')
+        with pytest.raises(module['CaptureBlocked'], match='预算'):
+            capture.fetch('https://www.govinfo.gov/second.pdf')
+        resumed = module['Capture'](path, 'test', limit=1, client=client)
+        assert resumed.fetch('https://www.govinfo.gov/first.pdf') == b'body'
+        assert resumed.requests == 0
+        assert resumed.fetch('https://www.govinfo.gov/second.pdf') == b'body'
+
+
+def test_capture_replays_all_case_callbacks_without_external_io(trading, tmp_path, monkeypatch):
+    module = runpy.run_path(str(SCRIPT))
+    monkeypatch.setattr(module['time'], 'sleep', lambda seconds: None)
+    def handle(request):
+        path = request.url.path
+        if request.url.host == 'www.congress.gov':
+            return httpx.Response(200, text='<p>The law establishes regulatory requirements for stablecoins and digital assets.</p>')
+        if path == '/v3/committee-meeting/119':
+            data = {'committeeMeetings': [{'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1'}]}
+        elif '/committee-meeting/' in path:
+            data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z'}}
+        elif path.endswith('/text'):
+            data = {'textVersions': [{'date': '2025-01-01', 'type': 'Introduced', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov' + path + '.html'}]}]}
+        elif path.endswith('/actions'):
+            data = {'actions': [{'actionDate': '2025-01-01', 'text': 'Passed'}]}
+        else:
+            data = {'bill': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01'}}
+        return httpx.Response(200, json=data)
+    with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+        capture = module['Capture'](tmp_path / 'cache.gz', 'test', client=client)
+        report = asyncio.run(module['run'](capture))
+        assert report['accepted'] is True
+        assert len(report['cases']) == 4
+        assert sum(case['items'] for case in report['cases']) == 7
+        resumed = module['Capture'](capture.path, 'test', client=client)
+        assert asyncio.run(module['run'](resumed))['accepted'] is True
+        assert resumed.requests == 0

backend/tests/test_news_sources.py

@@ -33,7 +33,7 @@ ITEM = """<item><title>Bitcoin &amp; Ethereum</title>
 <dc:creator>News desk</dc:creator><category>Markets</category></item>"""
 
 
-@pytest.mark.parametrize("source", ["CoinDeskSpider", "FedSpider", "BeaSpider"])
+@pytest.mark.parametrize("source", ["CoinDeskSpider", "BeaSpider"])
 def test_rss_preserves_original_summary_time_and_provenance(trading, source):
     from alphapulse.crawler.spiders import rss
     spider = make_spider(getattr(rss, source))
@@ -53,8 +53,8 @@ def test_rss_preserves_original_summary_time_and_provenance(trading, source):
 
 
 def test_rss_identity_survives_revisions_and_different_subfeeds(trading):
-    from alphapulse.crawler.spiders.rss import FedSpider
-    spider = make_spider(FedSpider)
+    from alphapulse.crawler.spiders.rss import CoinDeskSpider
+    spider = make_spider(CoinDeskSpider)
     original, = spider.parse_feed(feed_response(ITEM))
     revised, = spider.parse_feed(feed_response(ITEM.replace("Bitcoin &amp; Ethereum", "Revised headline"), "https://example.invalid/other.xml"))
     assert original["external_id"] == revised["external_id"]

backend/tests/test_us_official_news.py

@@ -0,0 +1,659 @@
+import asyncio
+import copy
+import json
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+
+import pytest
+from scrapy import Request
+from scrapy.http import HtmlResponse
+
+from alphapulse.core.us_news import SOURCES, article, official_url, published, selection
+
+
+def spider(trading, source='ofac'):
+    from alphapulse.crawler.spiders.base import RunState
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    obj = USOfficialSpider(name=source)
+    obj.name = source
+    now = datetime(2026, 11, 4, tzinfo=UTC)
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now - timedelta(hours=72))
+    obj.run.history_target = datetime(2025, 1, 1, tzinfo=UTC)
+    obj.run.history_deadline = time.monotonic() + 20
+    obj.cursor = {'history': {'entry': 0, 'pending': []}, 'realtime': {'entry': 0, 'pending': []}}
+    obj.lanes = {}
+    return obj
+
+
+def response(url, body, **meta):
+    return HtmlResponse(url, body=body.encode(), encoding='utf-8', request=Request(url, meta={'raw_response_id': 12, **meta}))
+
+
+@pytest.mark.parametrize('source,title,text,expected', [
+    ('doj', 'Man sentenced for bitcoin theft', 'Personal bitcoin theft worth $300', False),
+    ('doj', 'Exchange charged', 'Cryptocurrency exchange charged with money laundering', True),
+    ('fincen', 'Digital asset rule', 'FinCEN withdraws proposed digital asset rule', True),
+    ('whitehouse', 'Tariffs', 'President adjusts import tariffs', True),
+    ('treasury', 'Meeting with delegation', 'Greetings and cooperation', False),
+    ('ofac', 'New sanctions', 'Sanctions against a financial institution', True),
+    ('sec', 'Ethernet equipment', 'Method and hardware', False),
+])
+def test_selection_boundaries(source, title, text, expected):
+    assert bool(selection(source, title, text)) is expected
+
+
+@pytest.mark.parametrize('url', ['http://ofac.treasury.gov/recent-actions/20261104', 'https://evil.example/a.pdf', 'https://[email protected]/a', 'https://ofac.treasury.gov/a?api_key=secret'])
+def test_official_links_reject_external_and_credentials(url):
+    with pytest.raises(ValueError):
+        official_url('ofac', url)
+
+
+def test_dates_do_not_fabricate_publisher_time():
+    assert published('2025-01-01') == (datetime(2025, 1, 1, tzinfo=UTC), 'date')
+    assert published('2026-11-01T01:30:00-04:00')[0] == datetime(2026, 11, 1, 5, 30, tzinfo=UTC)
+    with pytest.raises(ValueError):
+        published('yesterday')
+
+
+def test_ofac_original_body_and_attachment_boundaries():
+    data = article('ofac', '''<h1>Crypto sanctions</h1><main><div class="field--name-body"><p>New sanctions against a virtual currency exchange institution.</p>
+    <a href="/media/123/download">License</a><a href="https://evil.example/a.pdf">external</a></div></main>''', 'https://ofac.treasury.gov/recent-actions/20250101')
+    assert data['published_at'] == datetime(2025, 1, 1, tzinfo=UTC)
+    assert data['extra']['published_at_precision'] == 'date'
+    assert len(data['extra']['attachments']) == 2
+    assert data['extra']['attachments'][1]['status'] == 'unsupported_host'
+
+
+def test_challenge_is_failure_not_empty_success(trading):
+    obj = spider(trading)
+    with pytest.raises(ValueError, match='未识别'):
+        list(obj.parse_listing(response(SOURCES['ofac'].entries[0], '<html>Please wait</html>', lane='history')))
+
+
+def test_history_pending_documents_survive_deadline_and_resume(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.run.history_deadline = time.monotonic() - 1
+    assert obj.advance('history') == []
+    saved = copy.deepcopy(obj.cursor)
+    resumed = spider(trading)
+    resumed.cursor = saved
+    request, = resumed.advance('history')
+    assert request.url == doc['url'] and resumed.cursor['history']['pending'] == [doc]
+    assert not resumed.run.history_done
+
+
+@pytest.mark.parametrize('count', [1999, 2000, 2001])
+def test_register_oversized_day_uses_xml(trading, count):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-01'
+    req, = obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': count, 'results': [{'document_number': '2025-1'}]}), lane='history'))
+    assert ('govinfo.gov' in req.url) is (count > 2000)
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_keys_only_sent_to_congress_api(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test-key', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    assert obj.request_for('https://api.congress.gov/v3/bill', lambda x: x, 'history').headers['X-Api-Key'] == b'test-key'
+    assert 'X-Api-Key' not in obj.request_for('https://www.congress.gov/text', lambda x: x, 'history').headers
+
+
+def test_register_body_and_pdf_are_part_of_the_same_package(trading):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/api/v1/documents/2025-1.json', 'kind': 'register'}
+    obj.cursor['history']['pending'] = [doc]
+    res = response(doc['url'], json.dumps({'title': 'Digital asset rule', 'publication_date': '2025-01-01', 'document_number': '2025-1', 'body_html_url': 'https://www.federalregister.gov/documents/full_text/html/2025/01/01/2025-1.html', 'html_url': 'https://www.federalregister.gov/documents/2025/01/01/2025-1/a', 'pdf_url': 'https://www.govinfo.gov/a.pdf'}), lane='history', document=doc)
+    async def collect():
+        return [v async for v in obj.parse_document(res)]
+    request, = asyncio.run(collect())
+    assert request.meta['document']['pdf'] == 'https://www.govinfo.gov/a.pdf'
+
+
+def test_summary_scope_tests_do_not_hide_fed_enrichment(trading):
+    from alphapulse.crawler.spiders.rss import FedSpider
+    from alphapulse.crawler.spiders.us_official import USOfficialSpider
+    assert issubclass(FedSpider, USOfficialSpider)
+    assert len(SOURCES['fed'].entries) >= 8
+
+
+def test_fifty_real_public_documents_parse_without_io():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    assert len(samples) == 50
+    assert len({s['source'] for s in samples}) == 8
+    for sample in samples:
+        parsed = article(sample['source'], sample['body'], sample['url'])
+        assert parsed['title'] == sample['expected_title']
+        assert len(parsed['content_html']) > 40
+        assert parsed['published_at'].year >= 2025
+        assert parsed['extra']['published_at_precision'] in ('date', 'datetime')
+
+
+def test_daily_register_partitions_resume_without_marking_done(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['history']['day'] = '2025-01-02'
+    requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps({'count': 0, 'results': []}), lane='history')))
+    assert len(requests) == 1
+    assert obj.cursor['history']['day'] == '2025-01-01'
+    assert not obj.run.history_done
+
+
+def test_h41_uses_publisher_release_dates_not_assumed_thursdays(trading):
+    obj = spider(trading, 'fed')
+    data = [{'yearValue': '2025', 'Months': [{'MonthName': 'January', 'Dates': ['20250103', '20250109']}]}]
+    req, = obj.parse_listing(response('https://www.federalreserve.gov/releases/h41/releaseDates.json', json.dumps(data), lane='history'))
+    assert req.url == 'https://www.federalreserve.gov/releases/h41/20250109/'
+    assert obj.cursor['history']['pending'][1]['url'].endswith('20250103/')
+
+
+def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
+    obj = spider(trading)
+    obj.run.history_deadline = time.monotonic() - 1
+    url = 'https://ofac.treasury.gov/recent-actions/20250101'
+    doc = {'url': url, 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    parsed = {'url': url, 'title': 'Crypto sanctions', 'external_id': 'x', 'content_html': '<p>Virtual currency exchange sanctions.</p>', 'published_at': datetime(2025, 1, 1, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
+              'extra': {'attachments': [{'url': 'https://ofac.treasury.gov/a.pdf', 'title': 'License', 'status': 'unavailable'}]}}
+    item, = obj.attach_or_finish(parsed, 'history')
+    assert item['extra']['content_scope'] == 'partial'
+    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
+    assert obj.cursor['attachment_retries'][url]['document'] == doc
+
+
+@pytest.mark.parametrize('canceled,failed', [(False, False), (True, False), (False, True)])
+def test_checkpoint_does_not_advance_failed_or_canceled_history(trading, monkeypatch, canceled, failed):
+    from alphapulse.crawler.spiders import base
+    now = datetime.now(UTC)
+    target = now - timedelta(days=300)
+    source = SimpleNamespace(history_target=None if canceled else target, history_cursor={'history': {'entry': 0}}, consecutive_failures=0)
+    class Session:
+        async def __aenter__(self):
+            return self
+        async def __aexit__(self, *args):
+            pass
+        async def execute(self, *args):
+            pass
+        async def get(self, *args, **kwargs):
+            return source
+        async def commit(self):
+            pass
+    monkeypatch.setattr(base, 'SessionLocal', Session)
+    obj = spider(trading)
+    run = base.RunState(uuid.uuid4(), 'schedule', now, target, history_target=target, history_position=now, official_cursor={'history': {'entry': 5}, 'realtime': {'entry': 2}})
+    if failed:
+        run.errors.append('write failed')
+    asyncio.run(obj._finish_run(run))
+    assert source.history_cursor['history']['entry'] == (0 if canceled or failed else 5)
+
+
+def test_congress_action_links_use_public_bill_type(trading):
+    from alphapulse.crawler.spiders.us_official import congress_bill_url
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/hr/1') == 'https://www.congress.gov/bill/119th-congress/house-bill/1'
+    assert congress_bill_url('https://api.congress.gov/v3/bill/119/s/1582') == 'https://www.congress.gov/bill/119th-congress/senate-bill/1582'
+
+
+def test_tick_during_active_round_does_not_replace_its_cursors(trading):
+    obj = spider(trading)
+    obj.crawler = SimpleNamespace(engine=SimpleNamespace(spider_is_idle=lambda: False))
+    before = copy.deepcopy(obj.cursor)
+    assert obj.make_request_from_data(json.dumps({'trigger': 'schedule', 'official_cursor': {'history': {'entry': 99}}}).encode()) == []
+    assert obj.cursor == before
+
+
+def test_realtime_keeps_next_page(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20261104">Crypto</a><a rel="next" href="?page=1">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] == url + '?page=1'
+
+
+@pytest.mark.parametrize('body', ['', '<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>'])
+def test_text_challenge_never_emits_or_advances(trading, body):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    with pytest.raises(ValueError):
+        asyncio.run(collect())
+    assert obj.cursor['history']['pending'] == [doc]
+
+
+def test_genius_bill_requires_evidence_before_exclusion(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/bill/119/s/1582', 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'bill': {'title': 'GENIUS Act', 'policyArea': {'name': 'Finance and Financial Sector'}}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert '/subjects?' in req.url
+    assert not obj.run.stats.get('excluded')
+
+
+def test_real_titles_are_article_headlines():
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/public_samples.json.gz', 'rt'))
+    for s in samples:
+        if s['source'] == 'fed' and '2026-press.htm' in s['url']:
+            continue
+        title = article(s['source'], s['body'], s['url'])['title']
+        assert title not in ('Board of Governors of the Federal Reserve System', 'U.S. Department of the Treasury')
+        assert not title.startswith('Release Number ')
+
+
+@pytest.mark.parametrize('enabled,history_enabled,save_failed', [(True, True, False), (False, True, False), (True, False, False), (True, True, True)])
+def test_round_waits_for_commit_and_reads_fresh_state(trading, monkeypatch, enabled, history_enabled, save_failed):
+    from alphapulse.crawler.spiders import us_official
+    fresh = {'history': {'entry': 0, 'target': '2025-01-01T00:00:00+00:00', 'pending': [{'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article'}]}}
+    source = SimpleNamespace(enabled=enabled, paused_until=None, history_cursor=fresh, history_target=datetime(2025, 1, 1, tzinfo=UTC) if history_enabled else None, history_done=False, history_position=None, backfill_hours=72)
+    reads = []
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args):
+            reads.append(True)
+            return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def run():
+        obj = spider(trading)
+        obj.run = None
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=reads.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        saved = asyncio.Event()
+        async def finish():
+            await saved.wait()
+            if save_failed: raise RuntimeError('test write failure')
+        obj.finish_task = asyncio.create_task(finish())
+        boot = asyncio.create_task(obj.start_round('schedule'))
+        await asyncio.sleep(0)
+        assert not reads
+        saved.set()
+        await boot
+        requests = [r for r in reads if isinstance(r, Request)]
+        if enabled and not save_failed:
+            assert bool([r for r in requests if r.meta['lane'] == 'history']) is history_enabled
+            if history_enabled:
+                assert requests[-1].url.endswith('20250101')
+        else:
+            assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(run())
+
+
+def test_register_realtime_finishes_all_pages_before_changing_day(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['realtime']['day'] = '2026-11-04'
+    url = SOURCES[obj.name].entries[0]
+    list(obj.parse_listing(response(url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1234'}], 'next_page_url': url + '?page=2'}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    req, = obj.advance('realtime')
+    assert req.url.endswith('?page=2')
+    list(obj.parse_listing(response(req.url, json.dumps({'count': 1001, 'results': [{'document_number': '2026-1235'}]}), lane='realtime')))
+    assert obj.cursor['realtime']['day'] == '2026-11-04'
+    obj.cursor['realtime']['pending'] = []
+    obj.advance('realtime')
+    assert obj.cursor['realtime']['day'] == '2026-11-03'
+
+
+def test_pagination_cycle_does_not_claim_success(trading):
+    obj = spider(trading)
+    url = SOURCES[obj.name].entries[0]
+    obj.cursor['history']['visited_pages'] = [url]
+    with pytest.raises(ValueError, match='循环'):
+        list(obj.parse_listing(response(url + '?page=2', f'<h1>News</h1><a href="/recent-actions/20261104">News</a><a rel="next" href="{url}">Next</a>', lane='history')))
+    assert not obj.run.history_done
+
+
+def test_fragment_is_valid_without_page_heading(trading):
+    obj = spider(trading, 'federal_register')
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset regulation', 'at': '2026-11-04'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>This digital asset regulation establishes requirements for stablecoin issuers.</p>', lane='history', document=doc))]
+    item, = asyncio.run(collect())
+    assert item['extra']['content_scope'] == 'article'
+
+
+def test_bill_full_text_can_supply_relevance_and_resume(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(body):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    asyncio.run(parse(json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': '2026-11-04'}})))
+    asyncio.run(parse(json.dumps({'subjects': {'legislativeSubjects': []}})))
+    asyncio.run(parse(json.dumps({'summaries': []})))
+    asyncio.run(parse(json.dumps({'textVersions': [{'type': 'Enrolled', 'date': '2026-11-04', 'formats': [{'type': 'Formatted Text', 'url': 'https://www.congress.gov/text'}]}]})))
+    obj.cursor = json.loads(json.dumps(obj.cursor))
+    output = asyncio.run(parse('<p>The bill establishes the regulatory framework for payment stablecoins and digital assets.</p>'))
+    assert output[0]['title'] == 'GENIUS Act — Enrolled'
+    assert '/actions?' in output[1].url
+    output = asyncio.run(parse(json.dumps({'actions': [{'actionDate': '2026-11-04', 'text': 'Became law'}]})))
+    assert output[0]['extra']['content_scope'] == 'official_action'
+    assert not obj.cursor.get('bill_retries')
+
+
+def test_missing_bill_material_is_a_retryable_gap(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    obj.cursor['history']['pending'] = [{'url': base, 'kind': 'bill'}]
+    async def parse(data):
+        doc = obj.cursor['history']['pending'][0]
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), lane='history', document=doc))]
+    asyncio.run(parse({'bill': {'title': 'GENIUS Act'}}))
+    asyncio.run(parse({'subjects': {}}))
+    asyncio.run(parse({'summaries': []}))
+    asyncio.run(parse({'textVersions': []}))
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+    assert not obj.run.stats.get('excluded')
+
+
+@pytest.mark.parametrize('attempts,version,expected_kind', [(3, 'v1', 'listing'), (1, 'v1', 'listing'), (3, 'v2', 'subjects')])
+def test_bill_retries_are_bounded_until_official_update(trading, monkeypatch, attempts, version, expected_kind):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    url = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': url, 'kind': 'bill'}
+    obj.cursor['history']['pending'] = [doc]
+    obj.cursor['bill_retries'] = {url: {'version': 'v1', 'attempts': attempts, 'next_at': (obj.run.started_at + timedelta(minutes=30)).timestamp()}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(url, json.dumps({'bill': {'title': 'GENIUS Act', 'updateDate': version}}), lane='history', document=doc))]
+    req, = asyncio.run(collect())
+    assert ('/subjects?' in req.url) is (expected_kind == 'subjects')
+
+
+def test_fed_archive_is_not_discovered_as_article(trading):
+    obj = spider(trading, 'fed')
+    url = SOURCES['fed'].entries[0]
+    req, = obj.parse_listing(response(url, '<h1>Press releases</h1><a href="2026-press.htm#content">Archive</a><a href="monetary20261104a.htm">News</a>', lane='history'))
+    assert req.url.endswith('monetary20261104a.htm')
+
+
+def test_ofac_entire_old_page_can_end_realtime_window(trading):
+    obj = spider(trading)
+    url = SOURCES['ofac'].entries[0]
+    list(obj.parse_listing(response(url, '<h1>Recent actions</h1><a href="/recent-actions/20250101">Old</a><a rel="next" href="?page=2">Next</a>', lane='realtime')))
+    assert obj.cursor['realtime']['next'] is None
+
+
+@pytest.mark.parametrize('body', ['<title>Page not found</title><p>The requested official document could not be located.</p>', '<?xml version="1.0"?><bill><section>Digital asset regulation establishes requirements for banks.'])
+def test_error_html_and_malformed_xml_are_rejected(body):
+    from alphapulse.core.us_news import validate_body
+    with pytest.raises(ValueError):
+        validate_body(body)
+
+
+def test_later_bill_version_recalls_earlier_version(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    first = {'url': 'https://www.congress.gov/first', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2026-11-04', 'bill': base}
+    second = {**first, 'url': 'https://www.congress.gov/second'}
+    obj.cursor['history'].update(bill={'url': base, 'reasons': [], 'title': 'GENIUS Act', 'bodies': 0}, pending=[first, second])
+    async def parse(doc, body):
+        return [v async for v in obj.parse_document(response(doc['url'], body, lane='history', document=doc))]
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert len(output) == 1 and isinstance(output[0], Request)
+    output = asyncio.run(parse(second, '<p>The legislation creates a regulatory framework for stablecoins and digital assets.</p>'))
+    assert output[0]['extra']['selection_reasons']
+    assert output[1].url == first['url']
+    obj.run.history_deadline = time.monotonic() - 1
+    output = asyncio.run(parse(first, '<p>This legislation establishes new administrative requirements for federal agencies.</p>'))
+    assert output[0]['title'] == 'GENIUS Act'
+
+
+def test_bill_retry_keeps_historical_text_outside_realtime_window(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': 'https://www.congress.gov/old', 'kind': 'text', 'title': 'GENIUS Act', 'at': '2025-01-01', 'bill': base}
+    obj.cursor['realtime'].update(bill={'url': base, 'title': 'GENIUS Act', 'reasons': ['crypto_policy'], 'bodies': 0, 'retry': True}, pending=[doc])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>The law establishes requirements for stablecoin issuers and digital assets.</p>', lane='realtime', document=doc))]
+    assert asyncio.run(collect())[0]['published_at'].year == 2025
+
+
+def test_failed_checkpoint_save_recovers_on_later_trigger(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=None, history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        requests, attempts = [], []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        original = obj.run
+        async def finish(run):
+            attempts.append(run)
+            if len(attempts) == 1:
+                raise RuntimeError('temporary checkpoint failure')
+        obj._finish_run = finish
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        await obj.start_round('schedule')
+        assert attempts == [original, original]
+        assert requests and obj.run is not None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_queued_tick_obeys_newly_saved_backoff(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    source = SimpleNamespace(enabled=True, paused_until=datetime.now(UTC) + timedelta(minutes=5), history_cursor={}, history_target=None, history_done=False, backfill_hours=72)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def get(self, *args): return source
+    monkeypatch.setattr(us_official, 'SessionLocal', Session)
+    async def check():
+        obj = spider(trading)
+        obj.run = None
+        requests = []
+        obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=requests.append))
+        obj._insert_run = lambda run: asyncio.sleep(0)
+        await obj.start_round('schedule')
+        assert not requests and obj.run is None
+        await asyncio.gather(*obj._tasks)
+    asyncio.run(check())
+
+
+def test_checkpoint_retry_budget_preserves_original_round(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'SessionLocal', lambda: pytest.fail('Uncommitted checkpoint must block source reads'))
+    async def check():
+        obj = spider(trading)
+        original, attempts = obj.run, []
+        async def fail(run):
+            attempts.append(run)
+            raise RuntimeError('persistent checkpoint failure')
+        obj._finish_run = fail
+        obj._close_run()
+        await asyncio.gather(obj.finish_task, return_exceptions=True)
+        for _ in range(5):
+            await obj.start_round('schedule')
+        assert attempts == [original] * 3
+        assert obj.run is None and not obj._starting
+    asyncio.run(check())
+
+
+@pytest.mark.parametrize('failure', ['', 'count', 'date', 'malformed'])
+def test_register_xml_validates_and_resumes(trading, failure):
+    obj = spider(trading, 'federal_register')
+    doc = {'url': 'https://www.govinfo.gov/issue.xml', 'kind': 'register_issue', 'day': '2025-01-01', 'count': 2001}
+    obj.cursor['history'].update(day=doc['day'], pending=[doc])
+    xml = '<FEDREG><DATE>January 1, 2025</DATE>' + ''.join(f'<FRDOC>[FR Doc. 2025-{n} Filed 1-1-25]</FRDOC>' for n in range(2001)) + '<FRDOC>[FR Doc. 2025-0 Filed 1-1-25]</FRDOC></FEDREG>'
+    if failure == 'count': doc['count'] += 1
+    if failure == 'date': doc['day'] = '2025-01-02'
+    if failure == 'malformed': xml = xml[:-10]
+    before = copy.deepcopy(obj.cursor)
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], xml, lane='history', document=doc))]
+    if failure:
+        from lxml import etree
+        with pytest.raises((ValueError, etree.XMLSyntaxError)):
+            asyncio.run(collect())
+        assert obj.cursor == before
+    else:
+        req, = asyncio.run(collect())
+        assert len(obj.cursor['history']['pending']) == 2001
+        assert req.meta['document']['kind'] == 'register'
+        assert obj.cursor['history']['day'] == '2025-01-01'
+        obj.run.history_deadline = time.monotonic() - 1
+        assert not obj.complete('history')
+        resumed = spider(trading, 'federal_register')
+        resumed.cursor = json.loads(json.dumps(obj.cursor))
+        assert resumed.advance('history')[0].meta['document'] == obj.cursor['history']['pending'][0]
+
+
+def test_doj_api_pagination_and_old_day_stop(trading):
+    obj = spider(trading, 'doj')
+    url = SOURCES['doj'].entries[0]
+    def data(day):
+        return {'metadata': {'responseInfo': {'status': 200}, 'resultset': {'count': '100', 'page': 0, 'pagesize': 50}}, 'results': [{'uuid': str(uuid.uuid4()), 'date': str(int(day.timestamp()))}]}
+    req, = obj.parse_listing(response(url, json.dumps(data(obj.run.started_at)), lane='realtime'))
+    assert req.meta['document']['kind'] == 'doj'
+    assert 'page=1' in obj.cursor['realtime']['next']
+    obj = spider(trading, 'doj')
+    list(obj.parse_listing(response(url, json.dumps(data(datetime(2024,1,1,tzinfo=UTC))), lane='history')))
+    assert obj.cursor['history']['next'] is None
+
+
+@pytest.mark.parametrize('bad_body', [False, True])
+def test_doj_body_identity_date_and_validation(trading, bad_body):
+    obj = spider(trading, 'doj')
+    obj.run.history_deadline = time.monotonic() - 1
+    identifier = str(uuid.uuid4())
+    doc = {'url': f'https://www.justice.gov/api/v1/press_releases/{identifier}.json', 'kind': 'doj'}
+    obj.cursor['history']['pending'] = [doc]
+    body = '<h1>Just a moment...</h1>' if bad_body else '<p>The cryptocurrency exchange pleaded guilty to violations of financial regulations.</p>'
+    record = {'uuid': identifier, 'url': 'https://justice.gov/opa/pr/test', 'date': '1735732800', 'title': 'Exchange &amp; settlement', 'body': body, 'changed': '<time datetime="2025-01-02T12:00:00Z">x</time>'}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'metadata': {'responseInfo': {'status': 200}}, 'results': [record]}), lane='history', document=doc))]
+    if bad_body:
+        with pytest.raises(ValueError): asyncio.run(collect())
+        assert obj.cursor['history']['pending'] == [doc]
+    else:
+        item, = asyncio.run(collect())
+        assert item['url'] == 'https://www.justice.gov/opa/pr/test'
+        assert item['title'] == 'Exchange & settlement'
+        assert item['extra']['published_at_precision'] == 'date'
+        assert item['source_updated_at'] == datetime(2025,1,2,12,tzinfo=UTC)
+
+
+def test_doj_old_discovery_cursor_is_rebuilt(trading):
+    obj = spider(trading, 'doj')
+    obj.cursor.update(checked={'old': 1}, history={'entry': 2}, attachment_retries={})
+    obj.first_requests()
+    assert obj.cursor['transport'] == 'doj-api-v1'
+    assert 'history' not in obj.cursor and obj.cursor['realtime']['entry'] == 0
+    assert obj.run.history_target == datetime(2025,1,1,tzinfo=UTC)
+
+
+def test_bill_undated_text_is_visible_retry_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    finish = {'url': base, 'kind': 'bill_finish'}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, finish])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': None, 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.cursor['bill_retries'][base]['attempts'] == 1
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+def test_public_api_and_pdf_fixtures_replay_without_io(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    items = asyncio.run(script['public_api_items']())
+    assert len(items) == 3
+    assert all(item['extra']['content_scope'] == 'article' for item in items)
+
+
+
+def test_congress_official_xml_crosscheck(trading):
+    import runpy
+    from pathlib import Path
+    script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
+    assert len(asyncio.run(script['congress_reference_items']())) == 3
+
+
+def test_congress_meeting_includes_deduplicated_witness_pdfs(trading, monkeypatch):
+    from alphapulse.crawler.spiders import us_official
+    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+    obj = spider(trading, 'congress')
+    doc = {'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1', 'kind': 'meeting'}
+    obj.cursor['history']['pending'] = [doc]
+    pdf = {'url': 'https://www.congress.gov/witness.pdf', 'format': 'PDF', 'documentType': 'Witness Statement'}
+    data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z', 'meetingDocuments': [pdf], 'witnessDocuments': [pdf]}}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), document=doc, lane='history'))]
+    req, = asyncio.run(collect())
+    assert req.url == pdf['url']
+    assert len(req.meta['parsed']['extra']['attachments']) == 1
+    assert req.meta['parsed']['extra']['attachments'][0]['title'] == 'Witness Statement'
+
+
+def test_dated_but_unavailable_bill_text_remains_a_gap(trading):
+    obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
+    base = 'https://api.congress.gov/v3/bill/119/s/1582'
+    doc = {'url': base + '/text', 'kind': 'texts', 'title': 'GENIUS Act', 'bill': base}
+    obj.cursor['history'].update(bill={'url': base, 'document_url': base, 'title': 'GENIUS Act', 'version': 'v1', 'reasons': ['crypto_policy'], 'bodies': 1}, pending=[doc, {'url': base, 'kind': 'bill_finish'}])
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'textVersions': [{'date': '2025-01-01', 'formats': []}]}), lane='history', document=doc))]
+    asyncio.run(collect())
+    obj.finish_bill('history')
+    assert obj.run.stats['unresolved_bills'] == 1
+
+
+
+def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_path):
+    import runpy
+    import shutil
+    from pathlib import Path
+    scripts = tmp_path / 'scripts'
+    scripts.mkdir()
+    original = Path(__file__).resolve().parents[2] / 'scripts'
+    for name in ('smoke_us_news.py', 'capture_congress.py'):
+        shutil.copyfile(original / name, scripts / name)
+    namespace = runpy.run_path(str(scripts / 'smoke_us_news.py'))
+    items = asyncio.run(namespace['congress_api_items']())
+    assert len(items) == 100
+    meetings = [item for item in items if item['extra'].get('official_meeting')]
+    assert len(meetings) == 1
+    assert meetings[0]['extra']['content_scope'] == 'partial'
+    attachments = meetings[0]['extra']['attachments']
+    assert len(attachments) == 15
+    assert len({a['url'] for a in attachments}) == 15
+    assert sum(a['status'] == 'complete' for a in attachments) == 14
+    assert sum(a['status'] == 'unavailable' for a in attachments) == 1
+    assert any(item['extra']['content_scope'] == 'official_action' for item in items)

deploy/.env.example

@@ -10,3 +10,7 @@ ADMIN_PASSWORD_HASH='<python -m alphapulse.core.auth hash-password 的输出>'
 LOGIN_CAPTCHA_ENABLED=false
 # 加密存库的第三方 API key（Fernet）；丢失后已存的 key 无法解密，需要在页面重新填写
 SECRET_ENCRYPTION_KEY=<python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())">
+
+# Official collectors: provide a private API key and a public bot contact.
+CONGRESS_API_KEY=
+CRAWLER_CONTACT_EMAIL=

deploy/README.md

@@ -209,3 +209,158 @@ With production authorization, retry `remove-cls --execute --build /path/to/buil
 using an accepted current dev build. Completed cleanup and migration stages are
 not repeated; failed stages are retried. A newer accepted build can resume if dev
 has advanced. Ordinary `recover` refuses restoration after maintenance has begun.
+
+### US official news collectors
+
+The US catalog adds `sec`, `cftc`, `treasury`, `ofac`, `fincen`, `whitehouse`,
+`congress`, `federal_register`, `occ`, and `doj`, and enriches `fed` with full
+text and H.4.1 releases. Migration `da1234567890` seeds the ten new sources
+**disabled**; it does not enable news AI or run any historical collection.
+Database migration and deployment still require the project's separate authorization.
+
+For this rollout from `c91234567890` to `da1234567890`:
+
+1. Review and authorize the feature commit/push, the one migration, and deployment
+   to the existing instance. Push `dev` through the existing CI. Its schema guard
+   must block activation until the independently authorized migration completes.
+2. Obtain the accepted build receipt and verify the candidate images using
+   `scripts/deploy.py`. Under the instance `.deploy.lock`, reject pending or stale
+   deployments and confirm the current schema is exactly `c91234567890`.
+3. Create a private full database backup and verify a complete restore into a
+   disposable database. Do not treat a dump listing as a restoration check.
+4. Still holding the lock, run only `alembic upgrade da1234567890` using the
+   verified candidate backend image and the existing private instance environment.
+   Verify three additive columns and ten new disabled sources. Do not modify
+   existing news, source enablement, history targets, or news AI settings.
+5. Release the lock and run `python3 scripts/deploy.py update --build` with that
+   same accepted receipt. Verify the nine services, API/web revisions, Congress
+   configuration presence, paused news AI, and disabled new sources. Report only
+   key presence/match, never its value or expanded container environment.
+
+The already enabled `fed` source keeps its enablement and uses the new full-body
+adapter after deployment. New collectors and historical backfill require a
+separate activation decision. SEC also requires a public contact email; absence
+must remain an explicit configuration error rather than an invented address.
+Rollback uses the existing deployment controller and retains additive columns
+and collected data; do not downgrade or stamp the database as an image rollback.
+
+Set `CONGRESS_API_KEY` and a public `CRAWLER_CONTACT_EMAIL` in the instance
+environment before enabling Congress and SEC. Keys are sent only in the Congress
+API request header and must never appear in URLs. Source access errors, including
+HTTP 200 challenge pages, fail visibly; they are not empty successful collections.
+
+After source acceptance, use the existing collection console to enable sources
+and set history to `2025-01-01`. Real-time discovery runs first; historical work
+issues new document requests for at most 20 seconds per batch and saves page and
+pending-document cursors only after successful processing. Canceling history
+preserves its cursor; restarting a completed traversal scans it again. Official
+rounds wait for the previous checkpoint commit and reload current source settings
+and cursors from the database; queued messages cannot restore canceled history
+or bypass a newly saved failure backoff. A failed checkpoint write is retried on
+subsequent triggers, with three total write attempts per round; exhausted writes
+block new collection with an explicit error until the collector is restarted. An ended
+list traversal does not prove complete bodies or attachments. Attachment failures
+remain visible on the news item, with up to three scheduled attempts one hour apart;
+exhausted gaps require explicit investigation or another traversal.
+
+Real-time discovery follows pagination, including every Federal Register page
+within a day. Only verified descending feeds stop early at a fully dated page
+outside the lookback window; unknown ordering is traversed conservatively.
+Congress screening reads subjects, CRS summaries, and available text before
+excluding a bill whose title alone is inconclusive. Missing bill evidence remains
+an unresolved gap, retried at most three times one hour apart per official update;
+exhausted gaps remain visible in the collection console. No AI is called.
+
+Original documents remain separate news packages across publishers. Body and
+attachment text are searchable and versioned. Date-only releases are displayed as
+dates, not precise release instants. Backfilled documents describe the version
+available at capture time; they are not historical point-in-time evidence.
+
+Validation without production credentials:
+
+- `cd backend && uv run pytest && uvx ruff check --select F,E9,B src tests`
+- `cd web && pnpm lint:check && pnpm typecheck && pnpm test && pnpm build`
+- `scripts/smoke_us_news.py` requires an isolated PostgreSQL database with user
+  `ci`, password `ci-only`, database `ci`, host `127.0.0.1`; it creates its own
+  news tables and must run in an empty disposable database. Redis publication is mocked.
+
+The 50 public HTML fixtures cover eight publishers. Additional API/PDF fixtures
+cover three 2025 DOJ institutional cases (OPA and USAO), their four attachments,
+one excluded personal crime case, and a Federal Register daily XML containing 106
+unique document numbers. The isolated database smoke checks idempotency and
+revisions after API-package edits. Three additional
+packages are actual GovInfo bill XML texts for GENIUS, CLARITY and STABLE;
+this cross-check is separate from the captured Congress API responses.
+The Congress API fixture contains 39 successful responses and produces 100
+packages: GENIUS 52, CLARITY 37, STABLE 10, and one crypto committee meeting.
+The smoke also replays those API packages without network access. On 2026-11-04,
+the isolated PostgreSQL run finished with 156 news packages and 158 revisions;
+replaying unchanged inputs created no extra revisions, and two deliberate edits
+each created one revision.
+
+DOJ discovery uses the public news API with 50 metadata records per page, then
+fetches individual bodies by UUID. It traverses publication dates newest first;
+old HTML discovery cursors are rebuilt without deleting news or changing the
+history target. API attachment file IDs use the verified official `/media/{id}/dl`
+endpoint. Calendar publication dates remain date-only; official update timestamps
+are stored separately.
+
+Federal Register daily results above 2,000 use the GovInfo issue XML to enumerate
+unique document numbers, then reuse the document API/body/PDF path. The issue date
+and unique count must match the daily API total; mismatch or download failure does
+not advance the checkpoint. A day advances only after its pending documents finish.
+Overflow tests use constructed 2,001-document data, not a claim of a real overflow day.
+
+An independent test key completed the three bill parser chains and a selected
+crypto committee meeting on 2026-11-04. This verifies the sampled API/text/action
+and attachment paths, not complete historical discovery or full source quality.
+The report keeps `accepted=false`: GENIUS has one undated text version, and one
+meeting PDF has no extractable text. Production credentials were not used in
+tests, and no key is hardcoded into the collector or saved in the fixtures.
+
+Three Federal Register API/body/PDF documents were previously read successfully
+on 2026-11-04; font encoding fidelity was not independently verified. Fixture replay
+and limited live reads do not prove complete 2025 coverage or authorize production
+collection activation.
+
+The GovInfo GENIUS BILLSTATUS XML captured on 2026-11-04 confirms that the
+Enrolled Bill version has an empty date. The offline cross-check preserves that
+gap and validates a dated text from each of the three bills without inventing a
+date. A dated version without a supported format also remains a bounded retry
+gap even when other versions succeed. Meeting PDFs include both meeting documents
+and witness documents, deduplicated by URL. The selected real meeting lists 16 PDF
+entries with 15 unique URLs: 14 yielded text, while a Truth in Testimony form did not. The meeting
+package remains `partial`; no OCR dependency or invented text/date hides the gap.
+
+Earlier public-key probes returned HTTP 429 and stopped immediately. The
+independent test key resolved that access blocker. Do not reuse production
+credentials for this test.
+
+
+Resumable Congress acceptance capture (no database, Redis, model, or production
+configuration access):
+
+```sh
+PYTHONPATH=backend/src backend/.venv/bin/python scripts/capture_congress.py --cache /tmp/alphapulse-congress-acceptance/responses.json.gz
+```
+
+For a separate local test key, enter `CONGRESS_TEST_API_KEY` in the ignored,
+mode-0600 `/srv/AlphaPulse/.env.congress-test`, then load it with uv:
+
+```sh
+PYTHONPATH=backend/src uv run --project backend --no-sync --env-file /srv/AlphaPulse/.env.congress-test python scripts/capture_congress.py --cache /tmp/alphapulse-congress-acceptance/responses.json.gz --meeting-url https://api.congress.gov/v3/committee-meeting/119/house/119541
+```
+
+The script uses `CONGRESS_TEST_API_KEY` when separately configured, otherwise the
+public DEMO_KEY. It atomically saves each successful, credential-redacted response,
+reuses cached responses on another invocation, and stops on HTTP 429 or its bounded
+request budget (default 60, maximum 100). It never follows redirects with API
+credentials. A separate compressed report records incomplete cases; an incomplete
+run exits nonzero. GENIUS's known undated version remains an explicit gap rather
+than a passing complete case. Meeting selection requires a crypto-relevant title
+or related metadata; `--meeting-url` accepts a known official single-meeting API
+URL to avoid an unrelated meeting scan. `parser_accepted` means the selected
+callback chain finished, including explicit partial packages; it does not mean
+full source quality passed. Exhausted candidate/request budgets remain incomplete.
+Mocked tests and real captured responses are separate evidence; neither proves
+complete 2025 coverage or authorizes production activation.

scripts/capture_congress.py

@@ -0,0 +1,179 @@
+"""Capture bounded, resumable Congress acceptance evidence without production I/O."""
+import argparse
+import base64
+import gzip
+import json
+import os
+from pathlib import Path
+import time
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import httpx
+
+CREDENTIAL_FIELDS = {'apikey', 'api_key', 'token', 'key'}
+
+
+def clean_url(url):
+    parts = urlsplit(url)
+    query = [(k, v) for k, v in parse_qsl(parts.query) if k.lower() not in CREDENTIAL_FIELDS]
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def redact(value):
+    if isinstance(value, dict):
+        return {k: redact(v) for k, v in value.items() if k.lower() not in CREDENTIAL_FIELDS}
+    if isinstance(value, list):
+        return [redact(v) for v in value]
+    if isinstance(value, str) and value.startswith('https://'):
+        return clean_url(value)
+    return value
+
+
+def save(path, data):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + '.tmp')
+    with gzip.open(temporary, 'wt') as file:
+        json.dump(data, file)
+    temporary.replace(path)
+
+
+class CaptureBlocked(RuntimeError):
+    pass
+
+
+class Capture:
+    def __init__(self, path, key, limit=60, client=None):
+        self.path, self.key, self.limit = path, key, limit
+        self.client = client or httpx.Client(timeout=25, follow_redirects=False)
+        self.records = json.load(gzip.open(path, 'rt')) if path.exists() else {}
+        self.requests = 0
+        self.used_urls = set()
+
+    def fetch(self, url):
+        url = clean_url(url)
+        from alphapulse.core.us_news import official_url
+        official_url('congress', url)
+        self.used_urls.add(url)
+        if url in self.records:
+            return base64.b64decode(self.records[url]['body'])
+        if self.requests >= self.limit:
+            raise CaptureBlocked('本轮请求预算耗尽；成功响应已保存，下次可恢复')
+        self.requests += 1
+        headers = {'X-Api-Key': self.key} if urlsplit(url).hostname == 'api.congress.gov' else {}
+        response = self.client.get(url, headers=headers)
+        if response.status_code == 429:
+            raise CaptureBlocked(f"HTTP 429；Retry-After={response.headers.get('retry-after', 'unknown')}；已停止请求")
+        if response.status_code != 200:
+            raise CaptureBlocked(f'HTTP {response.status_code}；未保存错误响应')
+        body = response.content
+        if 'json' in response.headers.get('content-type', ''):
+            body = json.dumps(redact(response.json())).encode()
+        self.records[url] = {'body': base64.b64encode(body).decode()}
+        save(self.path, self.records)
+        time.sleep(1)
+        return body
+
+
+async def run(capture, item_sink=None, meeting_url=None):
+    # Override every application I/O setting before importing the parser modules.
+    os.environ.update(DATABASE_URL='postgresql+asyncpg://test:[email protected]:1/test', REDIS_URL='redis://127.0.0.1:1/0', JWT_SECRET='test-only', ADMIN_USERNAME='test', SECRET_ENCRYPTION_KEY='MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=')
+    from datetime import UTC, datetime
+    from types import SimpleNamespace
+    from unittest.mock import patch
+    import uuid
+    from scrapy import Request
+    from scrapy.http import Response
+    from alphapulse.crawler.spiders import us_official
+    from alphapulse.crawler.spiders.base import RunState
+    from alphapulse.core.us_news import CRYPTO
+    report = {'accepted': False, 'cases': [], 'requests': 0}
+    roots = [('GENIUS', 'bill/119/s/1582'), ('CLARITY', 'bill/119/hr/3633'), ('STABLE', 'bill/119/hr/2392'), ('meeting', 'committee-meeting/119')]
+    with patch.object(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key=capture.key, crawler_contact_email='[email protected]')):
+        for label, route in roots:
+            obj = us_official.USOfficialSpider(name='congress')
+            now = datetime.now(UTC)
+            obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025,1,1,tzinfo=UTC), history_deadline=float('inf'))
+            url = f'https://api.congress.gov/v3/{route}?format=json&limit=250'
+            obj.cursor = {'history': {'entry': 0, 'pending': []}}
+            case = {'name': label, 'items': 0, 'article_items': 0, 'partial_items': 0, 'complete': False}
+            report['cases'].append(case)
+            try:
+                if label == 'meeting' and meeting_url:
+                    from alphapulse.core.us_news import official_url
+                    selected = official_url('congress', meeting_url)
+                    if urlsplit(selected).hostname != 'api.congress.gov' or '/committee-meeting/119/' not in urlsplit(selected).path:
+                        raise CaptureBlocked('会议地址必须是119届Congress API单场会议地址')
+                    obj.cursor['history']['pending'] = [{'url': us_official.configure_url(selected, format='json'), 'kind': 'meeting'}]
+                elif label == 'meeting':
+                    listing = json.loads(capture.fetch(url))
+                    obj.cursor['history']['pending'] = [{'url': us_official.configure_url(entry['url'], format='json'), 'kind': 'meeting'} for entry in listing['committeeMeetings']]
+                    if not obj.cursor['history']['pending']:
+                        raise CaptureBlocked('会议列表为空，不能认定验收完成')
+                else:
+                    obj.cursor['history']['pending'] = [{'url':url, 'kind':'bill'}]
+                request = None
+                while obj.cursor['history']['pending']:
+                    if request is None:
+                        request = obj.advance('history')[0]
+                    body = capture.fetch(request.url)
+                    if label == 'meeting' and request.callback == obj.parse_document:
+                        meeting = json.loads(body)['committeeMeeting']
+                        if not CRYPTO.search(meeting['title'] + json.dumps(meeting.get('relatedItems', {}))):
+                            obj.cursor['history']['pending'].pop(0)
+                            request = None
+                            continue
+                    request.meta['raw_response_id'] = None
+                    response = Response(request.url, body=body, request=request)
+                    if request.callback == obj.parse_attachment:
+                        output = list(obj.parse_attachment(response))
+                    else:
+                        from scrapy.http import HtmlResponse
+                        output = [v async for v in obj.parse_document(HtmlResponse(response.url, body=body, encoding='utf-8', request=request))]
+                    requests = [v for v in output if isinstance(v, Request) and v.meta.get('document')]
+                    # Attachment requests hold parsed state rather than a document.
+                    requests += [v for v in output if isinstance(v, Request) and v.meta.get('parsed')]
+                    for item in (v for v in output if not isinstance(v, Request)):
+                        if item_sink is not None:
+                            item_sink.append(dict(item))
+                        case['items'] += 1
+                        case['article_items'] += int(item['extra']['content_scope'] == 'article')
+                        case['partial_items'] += int(item['extra']['content_scope'] == 'partial')
+                    if label == 'meeting' and case['article_items']:
+                        break
+                    request = requests[0] if requests else None
+                    if request is None and not obj.cursor['history']['pending']:
+                        break
+                case['traversal_complete'] = not obj.cursor['history']['pending'] if label != 'meeting' else bool(case['article_items'] + case['partial_items'])
+                case['parser_accepted'] = case['traversal_complete'] and bool(case['article_items'] + case['partial_items'])
+                case['unresolved_bills'] = obj.run.stats.get('unresolved_bills', 0)
+                case['complete'] = bool(case['article_items']) and not case['unresolved_bills'] and not obj.cursor.get('attachment_retries')
+            except Exception as exc:
+                # Error text may contain a credential-bearing upstream URL.
+                case['error'] = str(exc) if isinstance(exc, CaptureBlocked) else type(exc).__name__
+                break
+    report['requests'] = capture.requests
+    report['accepted'] = len(report['cases']) == len(roots) and all(c['complete'] for c in report['cases'])
+    return report
+
+
+def main():
+    import asyncio
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--cache', type=Path, required=True)
+    parser.add_argument('--meeting-url', help='Official Congress API URL for a known crypto hearing')
+    parser.add_argument('--max-requests', type=int, default=60)
+    args = parser.parse_args()
+    if not 1 <= args.max_requests <= 100:
+        parser.error('--max-requests must be between 1 and 100')
+    capture = Capture(args.cache, os.environ.get('CONGRESS_TEST_API_KEY') or 'DEMO_KEY', args.max_requests)
+    try:
+        report = asyncio.run(run(capture, meeting_url=args.meeting_url))
+        save(args.cache.with_suffix('.report.gz'), report)
+        print(json.dumps(report, ensure_ascii=False))
+    finally:
+        capture.client.close()
+    raise SystemExit(0 if report['accepted'] else 1)
+
+
+if __name__ == '__main__':
+    main()

scripts/smoke_us_news.py

@@ -0,0 +1,242 @@
+"""Validate source migration and real public-document idempotency in an isolated database."""
+import asyncio
+import copy
+import gzip
+import json
+import os
+from pathlib import Path
+import runpy
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+from urllib.parse import urlsplit
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import func, select, text
+
+from alphapulse.core.db import SessionLocal, engine
+from alphapulse.core.us_news import SOURCES, article, identity
+from alphapulse.crawler.spiders.base import RunState, html_to_text
+from alphapulse.crawler.spiders.us_official import USOfficialSpider
+from alphapulse.crawler import pipelines
+from alphapulse.models import CrawlRun, NewsItem, NewsRevision, RawBody, RawResponse, Source
+
+ROOT = Path(__file__).resolve().parents[1]
+BACKEND_ROOT = Path(pipelines.__file__).resolve().parents[3]
+
+
+async def public_api_items():
+    """Replay captured public API bodies and PDFs without network or database I/O."""
+    import base64
+    import time
+    import uuid
+    from datetime import UTC, datetime
+    from scrapy import Request
+    from scrapy.http import HtmlResponse, Response
+    from alphapulse.crawler.spiders.base import RunState
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/api_samples.json.gz', 'rt'))
+    pdfs = {s['url']: base64.b64decode(s['base64']) for s in samples if s['kind'] == 'pdf'}
+    items = []
+    for sample in (s for s in samples if s['kind'] != 'pdf'):
+        obj = USOfficialSpider(name=sample['source'])
+        now = datetime.now(UTC)
+        obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025, 1, 1, tzinfo=UTC), history_deadline=time.monotonic()-1)
+        doc = {k: sample[k] for k in ('url', 'kind')}
+        if doc['kind'] == 'register_issue':
+            doc.update(day='2025-07-18', count=106)
+        obj.cursor = {'history': {'entry': 0, 'pending': [doc], 'day': '2025-07-18'}}
+        req = Request(doc['url'], meta={'lane': 'history', 'document': doc, 'raw_response_id': None})
+        output = [v async for v in obj.parse_document(HtmlResponse(req.url, body=sample['body'].encode(), encoding='utf-8', request=req))]
+        if doc['kind'] == 'register_issue':
+            assert len(obj.cursor['history']['pending']) == 106
+            continue
+        if sample.get('excluded'):
+            assert not output
+            continue
+        while output and isinstance(output[0], Request):
+            req = output[0]
+            req.meta['raw_response_id'] = None
+            output = list(obj.parse_attachment(Response(req.url, body=pdfs[req.url], request=req)))
+        assert len(output) == 1 and output[0]['extra']['content_scope'] == 'article'
+        assert output[0]['extra']['attachments']
+        items += output
+    assert len(items) == 3
+    return items
+
+
+async def congress_reference_items():
+    """Cross-check public GovInfo XML through the text adapter, not the API adapter."""
+    import time
+    import uuid
+    from datetime import UTC, datetime
+    from lxml import etree
+    from scrapy import Request
+    from scrapy.http import HtmlResponse
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/congress_billstatus.json.gz', 'rt'))
+    items = []
+    assert len(samples) == 3
+    for sample in samples:
+        root = etree.fromstring(sample['body'].encode(), etree.XMLParser(resolve_entities=False, no_network=True))
+        versions = root.findall('.//textVersions/item')
+        if sample['bill'] == 's1582':
+            assert any(v.findtext('type') == 'Enrolled Bill' and not v.findtext('date') for v in versions)
+        obj = USOfficialSpider(name='congress')
+        now = datetime.now(UTC)
+        obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025,1,1,tzinfo=UTC), history_deadline=time.monotonic()-1)
+        text = sample['text']
+        doc = {'url': text['url'], 'kind': 'text', 'title': sample['title'], 'at': text['date']}
+        obj.cursor = {'history': {'entry': 0, 'pending': [doc]}}
+        req = Request(doc['url'], meta={'document': doc, 'lane': 'history', 'raw_response_id': None})
+        output = [v async for v in obj.parse_document(HtmlResponse(req.url, body=text['body'].encode(), encoding='utf-8', request=req))]
+        assert len(output) == 1 and output[0]['extra']['selection_reasons']
+        items += output
+    return items
+
+
+async def congress_api_items():
+    """Replay captured Congress API evidence with zero network requests."""
+    namespace = runpy.run_path(str(ROOT / 'scripts/capture_congress.py'))
+    cache = BACKEND_ROOT / 'tests/fixtures/us_news/congress_api.json.gz'
+    capture = namespace['Capture'](cache, 'offline-test-only', limit=0)
+    items = []
+    try:
+        report = await namespace['run'](capture, item_sink=items, meeting_url='https://api.congress.gov/v3/committee-meeting/119/house/119541')
+    finally:
+        capture.client.close()
+    assert capture.requests == 0
+    assert len(report['cases']) == 4
+    assert all(case.get('parser_accepted') for case in report['cases'])
+    assert report['cases'][0]['unresolved_bills'] == 1
+    assert report['accepted'] is False  # Undated GENIUS text and an unreadable meeting PDF remain gaps.
+    assert len(items) == sum(case['items'] for case in report['cases'])
+    return items
+
+
+async def main():
+    url = urlsplit(os.environ['DATABASE_URL'])
+    assert url.username == 'ci' and url.password == 'ci-only' and url.path == '/ci'
+    assert url.hostname == '127.0.0.1'
+    tables = [model.__table__ for model in (Source, CrawlRun, RawBody, RawResponse, NewsItem, NewsRevision)]
+    async with engine.begin() as connection:
+        await connection.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
+        await connection.run_sync(lambda conn: Source.metadata.create_all(conn, tables=tables))
+        for table, column in [('sources', 'history_cursor'), ('crawl_runs', 'stats'), ('news_revisions', 'extra')]:
+            await connection.execute(text(f'ALTER TABLE {table} DROP COLUMN {column}'))
+        namespace = runpy.run_path(str(BACKEND_ROOT / 'alembic/versions/20261104_da1234567890_us_official_news.py'))
+        def migrate(conn):
+            with Operations.context(MigrationContext.configure(conn)):
+                namespace['upgrade']()
+        await connection.run_sync(migrate)
+    async with SessionLocal() as session:
+        sources = list(await session.scalars(select(Source)))
+        assert len(sources) == 10 and all(not s.enabled for s in sources)
+        session.add(Source(code='fed', name='Fed', list_url=SOURCES['fed'].entries[0], enabled=False, interval_seconds=60))
+        await session.commit()
+    samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/public_samples.json.gz', 'rt'))
+    assert len(samples) == len({s['url'] for s in samples}) == 50
+    spider = SimpleNamespace(run=SimpleNamespace(items_seen=0, items_new=0, items_revised=0))
+    pipeline = pipelines.NewsPipeline(SimpleNamespace(spider=spider))
+    async def ingest(sample):
+        data = article(sample['source'], sample['body'], sample['url'])
+        return await pipeline.process_item(dict(source_code=sample['source'], external_id=identity(sample['url']), url=sample['url'], title=data['title'], summary='', content_html=data['content_html'], content_text=html_to_text(data['content_html']), author=None, tags=[], is_paywalled=False, published_at=data['published_at'], source_updated_at=None, extra=data['extra'], raw_response_id=None))
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        for sample in samples:
+            await ingest(sample)
+        for sample in samples:
+            await ingest(sample)
+        assert spider.run.items_new == 50 and spider.run.items_revised == 0
+        sample = samples[0]
+        changed = {**sample, 'body': sample['body'].replace('</h1>', ' Updated</h1>', 1)}
+        await ingest(changed)
+        assert spider.run.items_revised == 1
+    async with SessionLocal() as session:
+        assert await session.scalar(select(func.count()).select_from(NewsItem)) == 50
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == 51
+        assert await session.scalar(select(func.count()).select_from(NewsRevision).where(NewsRevision.extra.is_not(None))) == 51
+    # Preserve the legacy RSS identity while enriching from the same public URL.
+    from datetime import UTC, datetime, timedelta
+    import time
+    import uuid
+    from scrapy import Request
+    from scrapy.http import HtmlResponse
+    sample = next(s for s in samples if s['source'] == 'fed')
+    async with SessionLocal() as session:
+        news = await session.scalar(select(NewsItem).where(NewsItem.url == sample['url']))
+        news.external_id = 'legacy-rss-guid'
+        baseline = (news.content_hash, news.revision_no)
+        await session.commit()
+    obj = USOfficialSpider(name='fed')
+    now = datetime.now(UTC)
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now-timedelta(hours=72), history_target=datetime(2025, 1, 1, tzinfo=UTC), history_position=now, history_deadline=time.monotonic()-1)
+    obj.cursor = {'history': {'entry': 0, 'pending': []}}
+    captured = []
+    obj.attach_or_finish = lambda parsed, lane: captured.append(parsed) or []
+    doc = {'url': sample['url'], 'kind': 'article'}
+    request = Request(sample['url'], meta={'document': doc, 'lane': 'history', 'raw_response_id': None})
+    valid = HtmlResponse(sample['url'], body=sample['body'].encode(), encoding='utf-8', request=request)
+    assert not [v async for v in obj.parse_document(valid)]
+    assert captured[0]['external_id'] == 'legacy-rss-guid'
+    captured.clear()
+    invalid = HtmlResponse(sample['url'], body=b'<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>', encoding='utf-8', request=request)
+    try:
+        async for item in obj.parse_document(invalid):
+            await pipeline.process_item(item)
+    except ValueError:
+        pass
+    else:
+        raise AssertionError('Challenge must fail before the pipeline')
+    async with SessionLocal() as session:
+        news = await session.scalar(select(NewsItem).where(NewsItem.external_id == 'legacy-rss-guid'))
+        assert (news.content_hash, news.revision_no) == baseline
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == 51
+    # A failed checkpoint commit must recover before a new round can read it.
+    async with SessionLocal() as session:
+        source = await session.get(Source, 'ofac')
+        source.enabled = True
+        await session.commit()
+    obj = USOfficialSpider(name='ofac')
+    obj.run = RunState(uuid.uuid4(), 'schedule', now, now-timedelta(hours=72), official_cursor={'checkpoint_probe': 'committed'})
+    obj.crawler = SimpleNamespace(engine=SimpleNamespace(crawl=lambda request: None))
+    finish = obj._finish_run
+    attempts = []
+    async def transient_failure(run):
+        attempts.append(run.id)
+        if len(attempts) == 1:
+            raise RuntimeError('isolated checkpoint failure')
+        await finish(run)
+    obj._finish_run = transient_failure
+    original_id = obj.run.id
+    obj._close_run()
+    await asyncio.gather(obj.finish_task, return_exceptions=True)
+    await obj.start_round('schedule')
+    await asyncio.gather(*obj._tasks)
+    assert attempts == [original_id, original_id] and obj.run is not None
+    assert obj.cursor['checkpoint_probe'] == 'committed'
+    async with SessionLocal() as session:
+        saved = await session.get(CrawlRun, original_id)
+        assert saved.status == 'success'
+        assert (await session.get(Source, 'ofac')).history_cursor['checkpoint_probe'] == 'committed'
+    # API/PDF packages use the real parser and database revision pipeline too.
+    api_items = await public_api_items() + await congress_reference_items() + await congress_api_items()
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        for item in api_items:
+            await pipeline.process_item(dict(item))
+        async with SessionLocal() as session:
+            baseline_count = await session.scalar(select(func.count()).select_from(NewsRevision))
+        for item in api_items:
+            await pipeline.process_item(dict(item))
+        async with SessionLocal() as session:
+            assert await session.scalar(select(func.count()).select_from(NewsItem)) == 50 + len(api_items)
+            assert await session.scalar(select(func.count()).select_from(NewsRevision)) == baseline_count
+    revised = copy.deepcopy(api_items[0])
+    revised['title'] += ' Updated'
+    with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
+        await pipeline.process_item(dict(revised))
+    async with SessionLocal() as session:
+        assert await session.scalar(select(func.count()).select_from(NewsRevision)) == baseline_count + 1
+    await engine.dispose()
+    print('US source migration, HTML, API/PDF, GovInfo and Congress packages, API/PDF replay and revision, negative screening, daily XML, RSS identity, challenge rejection and checkpoint recovery passed')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

web/src/common/apis/crawler/type.ts

@@ -17,6 +17,9 @@ export interface SourceInfo {
   last_success_at: string | null
   last_error: string | null
   paused_until: string | null
+  collection_stats?: Record<string, number>
+  configuration_error?: string | null
+  history_cursor?: { history?: { scanned?: boolean }, realtime?: { entry: number } }
   supports_history: boolean
   history_target: string | null
   history_position: string | null

web/src/common/apis/news/type.ts

@@ -8,6 +8,8 @@ export interface NewsBrief {
   author: string | null
   tags: string[]
   is_paywalled: boolean
+  published_at_precision?: string | null
+  published_date?: string | null
   published_at: string
   first_seen_at: string
   revision_no: number

web/src/pages/crawler/components/SourcePanel.vue

@@ -54,8 +54,8 @@ async function saveInterval() {
   emit("changed")
 }
 
-// 默认回补到 2026 年初；已设置过目标时沿用
-const DEFAULT_HISTORY_TARGET = "2026-01-01"
+// Default to 2025; retain a previously selected target.
+const DEFAULT_HISTORY_TARGET = "2025-01-01"
 const historyDate = ref(props.source.history_target ? dayjs(props.source.history_target).format("YYYY-MM-DD") : DEFAULT_HISTORY_TARGET)
 
 const history = computed(() => {
@@ -68,7 +68,7 @@ const history = computed(() => {
   const percent = Math.min(100, Math.max(0, Math.round(covered / span * 1000) / 10))
   const reached = formatTime(position, "YYYY-MM-DD HH:mm")
   if (done && position && new Date(position) <= new Date(target))
-    return { text: `已完成，已到 ${reached}`, percent: 100, status: "success" } as const
+    return { text: `列表遍历完成，已到 ${reached}；正文及附件缺口以条目状态为准`, percent: 100, status: "success" } as const
   if (done)
     return { text: `来源最早只能翻到 ${reached}，更早的数据接口不提供`, percent, status: "warning" } as const
   return { text: position ? `进行中，已到 ${reached}` : "等待下一轮开始", percent, status: undefined }
@@ -88,6 +88,8 @@ async function cancelHistory() {
 }
 
 const statusTag = computed(() => {
+  if (props.source.configuration_error)
+    return { type: "warning", text: props.source.configuration_error } as const
   const process = props.source.process
   if (props.restarting || process?.state === "restarting")
     return { type: "warning", text: "重启中" } as const
@@ -121,7 +123,7 @@ onMounted(loadLatest)
         告警
       </el-tag>
       <div class="flex-1" />
-      <el-switch :model-value="source.enabled" @change="toggleEnabled" />
+      <el-switch :disabled="Boolean(source.configuration_error)" :model-value="source.enabled" @change="toggleEnabled" />
     </div>
     <div class="flex items-center gap-2 text-xs">
       <span class="h-2 w-2 rounded-full" :style="{ background: `var(--el-color-${statusTag.type})` }" />
@@ -155,6 +157,12 @@ onMounted(loadLatest)
         </div>
       </div>
     </div>
+    <div v-if="source.collection_stats?.unresolved_bills" class="text-xs text-amber-600">
+      待核验法案 {{ source.collection_stats.unresolved_bills }}；官方资料尚不充分
+    </div>
+    <div v-if="source.collection_stats?.discovered" class="text-xs text-slate-500">
+      本轮发现 {{ source.collection_stats.discovered }}；筛除 {{ source.collection_stats.excluded ?? 0 }}；附件缺口 {{ source.collection_stats.partial ?? 0 }}
+    </div>
     <div class="flex justify-between text-xs text-slate-500">
       <span>今日新增 <b class="num text-[var(--el-color-primary)]">+{{ source.today_new }}</b></span>
       <span>今日修订 <b class="num text-slate-900">{{ source.today_revised }}</b></span>

web/src/pages/news/components/NewsDetail.vue

@@ -89,7 +89,7 @@ const diffStats = computed(() => ({
       </div>
       <div class="grid grid-cols-2 gap-x-6 gap-y-2 border border-slate-200 rounded-lg border-solid bg-slate-50 px-3 py-2 text-sm">
         <div>
-          <span class="label">发布时间</span><span class="num">{{ formatTime(detail.published_at, "YYYY-MM-DD HH:mm:ss") }}</span>
+          <span class="label">发布时间</span><span class="num">{{ detail.published_at_precision === "date" ? `${detail.published_date}（仅日期）` : formatTime(detail.published_at, "YYYY-MM-DD HH:mm:ss") }}</span>
         </div>
         <div>
           <span class="label">作者</span>{{ detail.author || "-" }}
@@ -112,6 +112,13 @@ const diffStats = computed(() => ({
         title="此条新闻仅采集公开摘要，完整内容请查看原文链接。"
         type="info" :closable="false" show-icon
       />
+      <el-alert v-if="detail.extra.content_scope === 'partial'" title="正文或关键附件存在采集缺口，请核对官方原文。" type="warning" :closable="false" />
+      <div v-if="Array.isArray(detail.extra.attachments)" class="flex flex-wrap gap-2 text-xs">
+        <span v-for="attachment in (detail.extra.attachments as { url: string, title: string, status: string }[])" :key="attachment.url">
+          <el-link :href="attachment.url" target="_blank" type="primary">{{ attachment.title || '附件' }}</el-link>
+          （{{ attachment.status === 'complete' ? '已采集' : '存在缺口' }}）
+        </span>
+      </div>
       <el-tabs v-model="activeTab" class="min-h-0 flex flex-1 flex-col">
         <el-tab-pane :label="summaryOnly ? '摘要' : '正文'" name="content" class="h-full">
           <iframe :srcdoc="srcdoc" sandbox="allow-popups allow-popups-to-escape-sandbox" class="h-full w-full border-0" />

web/src/pages/news/index.vue

@@ -68,7 +68,7 @@ onBeforeUnmount(() => media?.removeEventListener("change", updateMobile))
               <el-tag size="small">
                 {{ sourceNames[item.source_code] ?? item.source_code }}
               </el-tag>
-              <span class="num">{{ formatTime(item.published_at) }}</span>
+              <span class="num">{{ item.published_at_precision === "date" ? item.published_date : formatTime(item.published_at) }}</span>
               <div class="flex-1" />
               <el-tag v-if="item.revision_no > 1" size="small" type="warning">
                 已修订 {{ item.revision_no - 1 }} 次