Add resumable US policy news collectors and provenance

Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled.

Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.

backend/src/alphapulse/ai/tools.py

@@ -22,6 +22,7 @@ from alphapulse.api.calendar import calendar_data
 from alphapulse.api.market import call_binance, client, tradable_symbols
 from alphapulse.api.news import news_filters
 from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import SOURCES as US_SOURCES
 from alphapulse.models import ContextVersion, NewsItem, SkillVersion
 from alphapulse.models.market import INTERVALS
 
@@ -34,6 +35,8 @@ SOURCE_NAMES = {
     "coindesk": "CoinDesk", "fed": "美联储·政策与讲话", "bls": "美国劳工统计局·CPI/PPI/就业", "bea": "美国经济分析局",
 }
 
+SOURCE_NAMES.update({code: spec.name for code, spec in US_SOURCES.items()})
+
 
 @dataclass
 class ToolOutcome:
@@ -59,7 +62,7 @@ def dump(data: Any) -> str:
 
 class SearchNews(BaseModel):
     query: Annotated[str, Field(description="关键字，按标题和正文字面匹配；比特币/Bitcoin、以太坊/Ethereum、美联储、CPI/PPI/非农/GDP/PCE 等完整主题词会同时匹配中英文；其他词请分别使用原文语言检索")]
-    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea"]] | None, Field(description="限定来源，不填为全部")] = None
+    sources: Annotated[list[Literal["wscn_live", "wscn_news", "coindesk", "fed", "bls", "bea", "sec", "cftc", "treasury", "ofac", "fincen", "whitehouse", "congress", "federal_register", "occ", "doj", "ecb", "boe", "eurostat", "ons", "destatis", "insee"]] | None, Field(description="限定来源，不填为全部")] = None
     start: Annotated[datetime | None, Field(description="开始时间（ISO 8601，带时区），如 2026-10-01T00:00:00+08:00")] = None
     end: Annotated[datetime | None, Field(description="结束时间（ISO 8601，带时区）")] = None
     limit: Annotated[int, Field(ge=1, le=20)] = 10
@@ -70,11 +73,11 @@ async def search_news(args: SearchNews) -> ToolOutcome:
         conditions = news_filters(args.sources, args.query, args.start, args.end)
         total = await session.scalar(select(func.count()).select_from(NewsItem).where(*conditions))
         rows = (await session.execute(
-            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200))
+            select(NewsItem.id, NewsItem.source_code, NewsItem.published_at, NewsItem.title, NewsItem.summary, func.left(NewsItem.content_text, 200), NewsItem.extra)
             .where(*conditions).order_by(NewsItem.published_at.desc()).limit(args.limit)
         )).all()
     items = [
-        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": local(r[2]), "title": r[3], "brief": r[4] or r[5]}
+        {"id": r[0], "source": SOURCE_NAMES.get(r[1], r[1]), "published_at": r[6].get("published_date") if r[6].get("published_at_precision") == "date" else local(r[2]), "title": r[3], "brief": r[4] or r[5]}
         for r in rows
     ]
     return ToolOutcome(dump({"total": total, "returned": len(items), "items": items}))
@@ -90,10 +93,13 @@ async def get_news(args: GetNews) -> ToolOutcome:
     if news is None:
         return ToolOutcome(dump({"error": f"新闻 {args.id} 不存在"}))
     return ToolOutcome(dump({
-        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": local(news.published_at),
+        "id": news.id, "source": SOURCE_NAMES.get(news.source_code, news.source_code), "published_at": news.extra.get("published_date") if news.extra.get("published_at_precision") == "date" else local(news.published_at),
         "title": news.title, "url": news.url, "is_paywalled": news.is_paywalled,
         "content": news.content_text[:NEWS_BODY_LIMIT], "truncated": len(news.content_text) > NEWS_BODY_LIMIT,
         "content_scope": news.extra.get("content_scope", "article"),
+        "published_at_precision": news.extra.get("published_at_precision"),
+        "published_date": news.extra.get("published_date"),
+        "attachments": news.extra.get("attachments", []),
     }))
 
 

backend/src/alphapulse/api/crawler.py

@@ -19,6 +19,7 @@ from alphapulse.core.redis import KEY_PREFIX, redis
 from alphapulse.core.official_sources import REGIONS
 from alphapulse.calendar.catalog import FAMILIES
 from alphapulse.crawler.events import CONTROL_CHANNEL, HISTORY_SOURCES, LOG_CHANNEL, STATUS_KEY
+from alphapulse.crawler.spiders.us_official import configuration_error
 from alphapulse.models import CrawlLog, CrawlRun, EconomicRelease, NewsItem, Source
 
 router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
@@ -46,10 +47,13 @@ class SourceOut(BaseModel):
     last_success_at: datetime | None
     last_error: str | None
     paused_until: datetime | None
+    configuration_error: str | None = None
+    history_cursor: dict = Field(default_factory=dict)
     supports_history: bool
     history_target: datetime | None
     history_position: datetime | None
     history_done: bool
+    collection_stats: dict = Field(default_factory=dict)
     # 管理进程未运行时为 None
     process: ProcessStatus | None
     today_new: int
@@ -180,6 +184,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
     totals = dict((await session.execute(
         select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
     )).all())
+    latest_runs = (await session.scalars(select(CrawlRun).order_by(CrawlRun.source_code, CrawlRun.started_at.desc()).distinct(CrawlRun.source_code))).all()
+    run_stats = {run.source_code: run.stats for run in latest_runs}
     statuses = await redis.hgetall(STATUS_KEY)
     db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
     return success(SourcesOut(
@@ -191,6 +197,8 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
                 **s.__dict__,
                 "region_code": REGIONS.get(s.code),
                 "supports_history": s.code in HISTORY_SOURCES,
+                "configuration_error": configuration_error(s.code),
+                "collection_stats": run_stats.get(s.code, {}),
                 "process": json.loads(statuses[s.code]) if s.code in statuses else None,
                 "today_new": today_counts.get(s.code, (0, 0))[0],
                 "today_revised": today_counts.get(s.code, (0, 0))[1],
@@ -206,6 +214,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
     source = await session.get(Source, code)
     if source is None or source.collector != "news":
         raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    if body.enabled and (error := configuration_error(code)):
+        raise HTTPException(status.HTTP_409_CONFLICT, error)
     for field, value in body.model_dump(exclude_none=True, exclude={"history_target"}).items():
         setattr(source, field, value)
     if "history_target" in body.model_fields_set:
@@ -214,6 +224,8 @@ async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSes
         if body.history_target is not None and body.history_target >= datetime.now(LOCAL_TZ):
             raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "回补目标时间必须早于现在")
         # 保留断点：改目标或重新开始时从已回补的位置继续
+        if body.history_target is not None and (source.history_cursor or {}).get("history", {}).get("scanned"):
+            source.history_cursor = {k: v for k, v in source.history_cursor.items() if k != "history"}
         source.history_target = body.history_target
         source.history_done = False
     await session.commit()

backend/src/alphapulse/api/news.py

@@ -55,6 +55,8 @@ class NewsBrief(BaseModel):
     first_seen_at: datetime
     revision_no: int
     risk: NewsRiskBrief | None = None
+    published_at_precision: str | None = None
+    published_date: str | None = None
 
 
 class NewsPage(BaseModel):
@@ -71,6 +73,7 @@ class NewsDetail(NewsBrief):
 
 
 class Revision(BaseModel):
+    extra: dict | None = None
     revision_no: int
     title: str
     summary: str
@@ -131,7 +134,7 @@ async def list_news(
     items = []
     for news, excerpt, package in rows:
         risk = {'state': package.state} if package else None
-        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk}))
+        items.append(NewsBrief.model_validate({**news.__dict__, 'excerpt': excerpt, 'risk': risk, 'published_at_precision': news.extra.get('published_at_precision'), 'published_date': news.extra.get('published_date')}))
     return success(NewsPage(items=items, total=total))
 
 
@@ -154,7 +157,7 @@ async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_se
     news = await session.get(NewsItem, news_id)
     if news is None:
         raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
-    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]}))
+    return success(NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH], "published_at_precision": news.extra.get("published_at_precision"), "published_date": news.extra.get("published_date")}))
 
 
 @router.get("/{news_id}/revisions")

backend/src/alphapulse/core/config.py

@@ -7,6 +7,9 @@ class Settings(BaseSettings):
     # 容器内由 compose 的 env_file 注入；只有一套配置，见仓库根目录 .env
     model_config = SettingsConfigDict(extra="ignore")
 
+    congress_api_key: str = ""
+    crawler_contact_email: str = ""
+
     searxng_url: str = "http://searxng:8080"
     database_url: str
     redis_url: str

backend/src/alphapulse/core/official_sources.py

@@ -98,3 +98,7 @@ def relevant_news(source: str, title: str, url: str) -> bool:
     if source == "insee":
         return any(t in title for t in ("prix à la consommation", "taux de chômage", "le pib", "le produit intérieur brut"))
     raise ValueError(f"未支持的欧洲新闻来源：{source}")
+
+# US adapters share the existing regional directory.
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+REGIONS.update({code: "US" for code in US_SOURCES})

backend/src/alphapulse/core/us_news.py

@@ -0,0 +1,171 @@
+"""Official US news catalog and deterministic collection policy."""
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+from parsel import Selector
+
+
+@dataclass(frozen=True)
+class OfficialSource:
+    name: str
+    entries: tuple[str, ...]
+    article_path: str
+    body: str
+    hosts: tuple[str, ...]
+    interval: int = 300
+
+
+SOURCES = {
+    'sec': OfficialSource('SEC·证券监管', ('https://www.sec.gov/newsroom/press-releases', 'https://www.sec.gov/newsroom/speeches-statements', 'https://www.sec.gov/rules-regulations/rulemaking-activity', 'https://www.sec.gov/rules-regulations/self-regulatory-organization-rulemaking'), r'/(?:newsroom/(?:press-releases|speeches-statements)/[^/]+|rules-regulations/.*\d.*)', '.node-details-layout__main-region__content .field--name-body', ('www.sec.gov',)),
+    'cftc': OfficialSource('CFTC·衍生品监管', ('https://www.cftc.gov/PressRoom/PressReleases', 'https://www.cftc.gov/PressRoom/SpeechesTestimony', 'https://www.cftc.gov/LawRegulation/CFTCStaffLetters/letters.htm'), r'/(?:PressRoom/(?:PressReleases/\d+-\d+|SpeechesTestimony/(?!index)[^/]+)|csl/)', 'article .field--name-body', ('www.cftc.gov',)),
+    'treasury': OfficialSource('美国财政部', ('https://home.treasury.gov/news/press-releases',), r'/news/press-releases/[a-z]+\d+/?$', '.field--name-field-news-body', ('home.treasury.gov', 'www.treasury.gov', 'ofac.treasury.gov')),
+    'ofac': OfficialSource('OFAC·金融制裁', ('https://ofac.treasury.gov/recent-actions',), r'/recent-actions/\d{8}[^/]*$', '.field--name-body, .body-content', ('ofac.treasury.gov', 'home.treasury.gov')),
+    'fincen': OfficialSource('FinCEN·反洗钱', ('https://www.fincen.gov/news/press-releases', 'https://www.fincen.gov/news/enforcement-actions', 'https://www.fincen.gov/resources/advisoriesbulletinsfact-sheets'), r'/(?:news/news-releases/[^/]+|resources/advisories/[^/]+|news/enforcement-actions/[^/]+)', 'article .field--name-body', ('www.fincen.gov',)),
+    'whitehouse': OfficialSource('白宫·政策', ('https://www.whitehouse.gov/presidential-actions/', 'https://www.whitehouse.gov/fact-sheets/', 'https://www.whitehouse.gov/briefings-statements/', 'https://bidenwhitehouse.archives.gov/briefing-room/'), r'/(?:presidential-actions|fact-sheets|briefings-statements|briefing-room/[^/]+)/202[5-9]/\d{2}/\d{2}/|/(?:presidential-actions|fact-sheets|briefings-statements)/202[5-9]/\d{2}/[^/]+', '.wp-block-post-content, .body-content, .entry-content', ('www.whitehouse.gov', 'bidenwhitehouse.archives.gov')),
+    'congress': OfficialSource('美国国会·立法', ('https://api.congress.gov/v3/bill?format=json&limit=250', 'https://api.congress.gov/v3/committee-meeting?format=json&limit=250'), r'/bill/', '', ('api.congress.gov', 'www.congress.gov', 'www.govinfo.gov'), 900),
+    'federal_register': OfficialSource('Federal Register·规则', ('https://www.federalregister.gov/api/v1/documents.json',), r'/documents/', '', ('www.federalregister.gov', 'public-inspection.federalregister.gov', 'www.govinfo.gov'), 900),
+    'occ': OfficialSource('OCC·银行与稳定币', tuple(f'https://www.occ.gov/news-events/newsroom/news-issuances-by-year/{kind}/{year}-{kind}.html' for year in range(datetime.now(UTC).year, 2024, -1) for kind in ('news-releases', 'bulletins')), r'/news-issuances/(?:news-releases|bulletins|interpretations-and-precedents)/202[5-9]/[^/]+', '.occgov-issuance-content', ('www.occ.gov', 'occ.gov', 'www.occ.treas.gov', 'occ.treas.gov')),
+    'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)),
+    'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speeches', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
+}
+RULE_VERSION = 'us-policy-v1'
+CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I)
+POLICY = re.compile(r'\b(?:regulat\w*|rules?|guidance|legislation|acts?|sanctions?|licenses?|executive order|monetary policy|federal funds|interest rates?|balance sheet|liquidity|quantitative|refunding|borrowing|debt limit|tariffs?|fiscal|budget|appropriation|tax reform)\b', re.I)
+INSTITUTION = re.compile(r'\b(?:exchanges?|banks?|financial institutions?|payment processors?|custodians?|stablecoin issuers?|binance|coinbase|tether|circle|kraken|bybit|bitfinex)\b', re.I)
+MACRO = re.compile(r'\b(?:monetary policy|federal funds|interest rates?|fomc|balance sheet|liquidity|quantitative|refunding|borrowing estimates|debt (?:limit|ceiling)|tariffs?|fiscal|budget|appropriation|tax reform|financial sanctions)\b', re.I)
+
+
+def selection(source: str, title: str, text: str) -> list[str]:
+    content = f'{title}\n{text}'
+    crypto = bool(CRYPTO.search(content))
+    if crypto and (source not in ('doj', 'ofac', 'fincen') or POLICY.search(title) or INSTITUTION.search(content)):
+        return ['crypto_policy' if POLICY.search(content) else 'crypto_institution']
+    if source not in ('doj', 'cftc', 'sec') and MACRO.search(content):
+        return ['us_macro_policy']
+    if source == 'ofac' and INSTITUTION.search(content) and POLICY.search(content):
+        return ['financial_sanctions']
+    return []
+
+
+def official_url(source: str, url: str, base: str = '') -> str:
+    value = urlsplit(urljoin(base, url))
+    if value.scheme != 'https' or value.hostname not in SOURCES[source].hosts or value.username or value.password:
+        raise ValueError('链接不是该来源的官方 HTTPS 地址')
+    if re.search(r'(?:api_key|token|key)=', value.query, re.I):
+        raise ValueError('官方链接含凭据参数')
+    return urlunsplit((value.scheme, value.netloc, value.path, value.query, ''))
+
+
+def identity(url: str) -> str:
+    return hashlib.sha256(url.encode()).hexdigest()
+
+
+def published(value: str) -> tuple[datetime, str]:
+    if not isinstance(value, str):
+        raise ValueError('官方材料缺少可识别的发布时间')
+    value = value.strip().replace('\xa0', ' ')
+    if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
+        return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
+    try:
+        dt = datetime.fromisoformat(value.replace('Z', '+00:00'))
+        if dt.tzinfo:
+            return dt.astimezone(UTC), 'datetime'
+    except ValueError:
+        pass
+    match = re.search(r'([A-Z][a-z]+\.? \d{1,2}, \d{4}|\d{1,2}/\d{1,2}/\d{4})', value)
+    if match:
+        for fmt in ('%B %d, %Y', '%b %d, %Y', '%m/%d/%Y'):
+            try:
+                return datetime.strptime(match[0].replace('.', ''), fmt).replace(tzinfo=UTC), 'date'
+            except ValueError:
+                pass
+    raise ValueError('官方材料缺少可识别的发布时间')
+
+
+def validate_body(content: str) -> None:
+    """Reject error documents without requiring full-page HTML around fragments."""
+    if content.lstrip().startswith('<?xml'):
+        from lxml import etree
+        try:
+            etree.fromstring(content.encode(), etree.XMLParser(resolve_entities=False, no_network=True))
+        except etree.XMLSyntaxError as exc:
+            raise ValueError('官方 XML 正文无法解析') from exc
+    sel = Selector(content)
+    heading = ' '.join(sel.css('title, h1').xpath('normalize-space(.)').getall())
+    text = ' '.join(sel.xpath('//text()[not(ancestor::script or ancestor::style)]').getall()).strip()
+    if re.search(r'just a moment|access denied|verify (?:you are|your)|sign in|log in|captcha|page not found|service unavailable|bad gateway', heading, re.I) or sel.css('#challenge-form, #cf-challenge-running, script[src*="/cdn-cgi/challenge"]'):
+        raise ValueError('官方正文返回验证或登录页面')
+    if re.search(r'enable javascript and cookies to continue', text, re.I):
+        raise ValueError('官方正文返回验证页面')
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+
+
+def article(source: str, html: str, url: str) -> dict:
+    sel = Selector(html)
+    validate_body(html)
+    selectors = {
+        'fed': ('#article h3.title', '#article h3', '#article h2'),
+        'treasury': ('h2.uswds-page-title',),
+        'cftc': ('article h1:not(.press-release-title)',),
+    }.get(source, ('.main-content__main h1', '#main_content h1', 'main h1', 'h1'))
+    title = next((value for css in selectors if (value := sel.css(css).xpath('normalize-space(.)').get())), None)
+    if source == 'fed' and '/releases/h41/' in url:
+        title = 'Federal Reserve Balance Sheet — H.4.1'
+    if not title:
+        raise ValueError('官方正文缺少标题')
+    values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
+    for script in sel.css('script[type="application/ld+json"]::text').getall():
+        try:
+            data = json.loads(script)
+            nodes = data if isinstance(data, list) else data.get('@graph', [data])
+            values += [n['datePublished'] for n in nodes if isinstance(n, dict) and n.get('datePublished')]
+        except (ValueError, TypeError, AttributeError):
+            continue
+    values += sel.css('.node-details-layout__main-region__content, .date, .date-display-single, .release-date, .article__time, .published-date, .field--name-field-date, .field--name-field-release-date, .press-release-date').xpath('normalize-space(.)').getall()
+    values += re.findall(r'(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}', ' '.join(sel.css('main, article, #content').xpath('string(.)').getall()))[:3]
+    # OFAC action identifiers encode the publisher's day, not the fetch time.
+    if source == 'fed' and (m := re.search(r'/releases/h41/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    if source == 'ofac' and (m := re.search(r'/recent-actions/(\d{4})(\d{2})(\d{2})', url)):
+        values.insert(0, '-'.join(m.groups()))
+    at = precision = None
+    for value in values:
+        try:
+            at, precision = published(value)
+            break
+        except ValueError:
+            continue
+    if at is None:
+        raise ValueError('官方材料缺少可识别的发布时间')
+    body = sel.css(SOURCES[source].body)
+    if not body:
+        raise ValueError('官方正文结构未识别')
+    for node in body.css('script, style, nav, header, footer, form'):
+        node.root.getparent().remove(node.root)
+    body = sorted(body, key=lambda node: len(node.xpath('string(.)').get()), reverse=True)
+    content = body[0].get()
+    text = ' '.join(body[0].xpath('.//text()').getall()).strip()
+    if len(text) < 40:
+        raise ValueError('官方正文为空或过短')
+    attachments = []
+    for link in body[0].css('a[href]'):
+        href = link.attrib['href']
+        label = link.xpath('normalize-space(.)').get()
+        if re.search(r'\.pdf(?:\?|$)|/media/\d+/(?:download|dl)|/csl/[^/]+/download', href, re.I):
+            try:
+                link_url = official_url(source, href, url)
+            except ValueError:
+                candidate = urljoin(url, href)
+                if urlsplit(candidate).scheme == 'https':
+                    attachments.append({'url': candidate, 'title': label, 'status': 'unsupported_host', 'error': '附件不在该来源的官方域名范围内，未下载'})
+                continue
+            if link_url not in [a['url'] for a in attachments]:
+                attachments.append({'url': link_url, 'title': label, 'status': 'pending'})
+    return dict(title=title, content_html=content, published_at=at,
+                extra={'language': 'en', 'content_scope': 'article', 'published_at_precision': precision,
+                       'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'attachments': attachments})

backend/src/alphapulse/crawler/events.py

@@ -12,7 +12,8 @@ NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
 CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
 STATUS_KEY = f"{KEY_PREFIX}crawler:status"
 # Sources whose list endpoints support historical pagination.
-HISTORY_SOURCES = ("wscn_live", "wscn_news")
+from alphapulse.core.us_news import SOURCES as US_SOURCES
+HISTORY_SOURCES = ("wscn_live", "wscn_news", *US_SOURCES)
 
 
 def start_urls_key(source_code: str) -> str:

backend/src/alphapulse/crawler/middlewares.py

@@ -4,6 +4,9 @@ import hashlib
 from urllib.parse import urlparse
 
 from scrapy.crawler import Crawler
+from scrapy.exceptions import IgnoreRequest
+
+from alphapulse.core.us_news import official_url
 from scrapy.http import Request, Response
 from sqlalchemy import insert
 from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -19,6 +22,13 @@ class RedisRateLimitMiddleware:
     interval_ms = 1000
 
     async def process_request(self, request: Request) -> None:
+        if source := request.meta.get("official_source"):
+            try:
+                official_url(source, request.url)
+            except ValueError as exc:
+                raise IgnoreRequest(str(exc)) from exc
+        if urlparse(request.url).hostname != "api.congress.gov":
+            request.headers.pop("X-Api-Key", None)
         key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
         while not await redis.set(key, "1", px=self.interval_ms, nx=True):
             ttl = await redis.pttl(key)

backend/src/alphapulse/crawler/pipelines.py

@@ -51,6 +51,8 @@ class NewsPipeline:
                 )
                 # 插入成功取新行；冲突说明另一处刚写入，按已存在条目走比较/修订流程
                 news = await session.scalar(locked)
+            if inserted_id is None and item["extra"].get("content_scope") == "partial" and news.extra.get("content_scope") == "article":
+                return item
             if inserted_id is not None:
                 kind = "new"
             else:
@@ -74,6 +76,7 @@ class NewsPipeline:
                     content_html=item["content_html"],
                     content_hash=digest,
                     raw_response_id=raw_response_id,
+                    extra=item["extra"],
                 ))
             news_id = news.id
             await session.commit()

backend/src/alphapulse/crawler/spiders/base.py

@@ -65,6 +65,8 @@ class RunState:
     history_position: datetime | None = None
     history_done: bool = False
     history_deadline: float = 0
+    official_cursor: dict | None = None
+    stats: dict = field(default_factory=dict)
 
     @property
     def full_scan(self) -> bool:
@@ -104,6 +106,7 @@ class NewsSpider(RedisSpider):
         self._tasks: set[asyncio.Task] = set()
         # 上一轮成功后的回补进度 (目标, 断点, 是否结束)。写库是异步的，管理进程推来的下一条触发可能还带着旧断点
         self.history_checkpoint: tuple[datetime, datetime, bool] | None = None
+        self.finish_task: asyncio.Task | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
@@ -177,7 +180,7 @@ class NewsSpider(RedisSpider):
         run, self.run = self.run, None
         if run.history_target is not None and not run.errors:
             self.history_checkpoint = (run.history_target, run.history_position, run.history_done)
-        self._spawn(self._finish_run(run))
+        self.finish_task = self._spawn(self._finish_run(run))
 
     def _on_error(self, failure: Failure, response: Response) -> None:
         if self.run:
@@ -191,10 +194,11 @@ class NewsSpider(RedisSpider):
         self.logger.info("采集进程退出")
         await self._flush_logs()
 
-    def _spawn(self, coro: Coroutine) -> None:
+    def _spawn(self, coro: Coroutine) -> asyncio.Task:
         task = asyncio.get_running_loop().create_task(coro)
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
+        return task
 
     async def _insert_run(self, run: RunState) -> None:
         async with SessionLocal() as session:
@@ -211,6 +215,7 @@ class NewsSpider(RedisSpider):
         values = {
             "finished_at": now,
             "status": "failed" if failed else "success",
+            "stats": run.stats,
             "requests": run.requests,
             "items_seen": run.items_seen,
             "items_new": run.items_new,
@@ -227,6 +232,12 @@ class NewsSpider(RedisSpider):
             source = await session.get(Source, self.name, with_for_update=True)
             # 本轮失败不推进断点，下一轮从原位置重做（写入幂等）；目标期间被修改或取消时丢弃本轮进度
             history_saved = not failed and run.history_target is not None and source.history_target == run.history_target
+            if not failed and run.official_cursor is not None:
+                cursor = dict(source.history_cursor or {})
+                cursor.update({k: v for k, v in run.official_cursor.items() if k != "history"})
+                if history_saved:
+                    cursor["history"] = run.official_cursor.get("history", {})
+                source.history_cursor = cursor
             if history_saved:
                 source.history_position = run.history_position
                 source.history_done = run.history_done

backend/src/alphapulse/crawler/spiders/rss.py

@@ -97,12 +97,11 @@ class CoinDeskSpider(RssNewsSpider):
     feeds = ("https://www.coindesk.com/arc/outboundfeeds/rss/",)
 
 
-class FedSpider(RssNewsSpider):
+from alphapulse.crawler.spiders.us_official import USOfficialSpider
+
+
+class FedSpider(USOfficialSpider):
     name = "fed"
-    feeds = (
-        "https://www.federalreserve.gov/feeds/press_monetary.xml",
-        "https://www.federalreserve.gov/feeds/speeches_and_testimony.xml",
-    )
 
 
 class BlsSpider(RssNewsSpider):

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -0,0 +1,572 @@
+"""US official collectors: serialized documents, bounded history, durable page cursors."""
+import copy
+import json
+import re
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from html import escape, unescape
+
+from lxml import etree
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from scrapy import Request
+from scrapy.http import Response
+from sqlalchemy import select
+
+from alphapulse.calendar.umich import pdf_text
+from alphapulse.core.config import get_settings
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.us_news import RULE_VERSION, SOURCES, article, identity, official_url, published, selection, validate_body
+from alphapulse.crawler.spiders.base import HISTORY_RUN_SECONDS, NewsSpider, html_to_text
+from alphapulse.models import NewsItem, Source
+
+
+def configure_url(url, **params):
+    parts = urlsplit(url)
+    query = dict(parse_qsl(parts.query))
+    query.update(params)
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def congress_bill_url(api_url):
+    parts = urlsplit(api_url).path.strip('/').split('/')
+    _, _, congress, kind, number = parts
+    labels = {'hr': 'house-bill', 's': 'senate-bill', 'hjres': 'house-joint-resolution', 'sjres': 'senate-joint-resolution', 'hconres': 'house-concurrent-resolution', 'sconres': 'senate-concurrent-resolution', 'hres': 'house-resolution', 'sres': 'senate-resolution'}
+    return f'https://www.congress.gov/bill/{congress}th-congress/{labels[kind.lower()]}/{number}'
+
+
+def configuration_error(code):
+    settings = get_settings()
+    if code == 'congress' and not settings.congress_api_key:
+        return '待配置 CONGRESS_API_KEY'
+    if code == 'sec' and not settings.crawler_contact_email:
+        return '待配置 CRAWLER_CONTACT_EMAIL'
+    return None
+
+
+class USOfficialSpider(NewsSpider):
+    """Each lane holds only one page's remaining documents and its next URL."""
+    custom_settings = {"DOWNLOAD_MAXSIZE": 25 * 1024 * 1024}
+
+    def make_request_from_data(self, data):
+        if getattr(self, '_starting', False):
+            return []
+        if self.run is not None:
+            if not self.crawler.engine.spider_is_idle():
+                return super().make_request_from_data(data)
+            self._close_run()
+        self._starting = True
+        self._spawn(self.start_round(json.loads(data)['trigger']))
+        return []
+
+    def _close_run(self):
+        self._checkpoint_run = self.run
+        self._checkpoint_attempts = 1
+        super()._close_run()
+
+    async def start_round(self, trigger):
+        try:
+            if self.finish_task is not None:
+                if self.finish_task.done() and not self.finish_task.cancelled() and self.finish_task.exception() is not None and getattr(self, '_checkpoint_run', None) is not None:
+                    if self._checkpoint_attempts >= 3:
+                        self.logger.error('断点保存连续失败3次，需重启采集进程；未启动下一轮')
+                        return
+                    self._checkpoint_attempts += 1
+                    self.finish_task = self._spawn(self._finish_run(self._checkpoint_run))
+                await self.finish_task
+            async with SessionLocal() as session:
+                source = await session.get(Source, self.name)
+                if source is None or not source.enabled:
+                    return
+                if source.paused_until and source.paused_until > datetime.now(UTC):
+                    return
+                self.cursor = copy.deepcopy(source.history_cursor or {})
+                history = (source.history_target, source.history_position) if source.history_target and not source.history_done else None
+                hours = source.backfill_hours
+            requests = super().make_request_from_data(json.dumps({'trigger': trigger, 'backfill_hours': hours}).encode())
+            self.run.official_cursor = self.cursor
+            if history:
+                target, position = history
+                self.run.history_target = target
+                self.run.history_position = position or self.run.started_at
+                self.run.history_deadline = time.monotonic() + HISTORY_RUN_SECONDS
+                state = self.cursor.setdefault('history', {'entry': 0})
+                if state.get('target') != target.isoformat():
+                    self.cursor['history'] = {'entry': 0, 'target': target.isoformat()}
+                requests += self.advance('history')
+            for request in requests:
+                self.crawler.engine.crawl(request)
+        except Exception:
+            self.logger.exception('官方采集轮次启动失败，未使用队列旧断点')
+            if self.run is not None:
+                self.run.errors.append('官方采集轮次启动失败')
+        finally:
+            self._starting = False
+
+    def first_requests(self):
+        if self.name == 'doj' and self.cursor.get('transport') != 'doj-api-v1':
+            # Retain retry evidence; HTML discovery cursors cannot use API pagination.
+            self.cursor = {k: v for k, v in self.cursor.items() if k not in ('history', 'realtime', 'checked')}
+            self.cursor['transport'] = 'doj-api-v1'
+            self.run.official_cursor = self.cursor
+        error = configuration_error(self.name)
+        if error:
+            self.run.errors.append(error)
+            return []
+        state = self.cursor.setdefault('realtime', {'entry': 0})
+        due = [value['document'] for value in self.cursor.get('attachment_retries', {}).values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
+        if due and not state.get('pending'):
+            state['pending'] = [{**doc, 'attachment_retry': True} for doc in due]
+        bill_due = [{**v['document'], 'bill_retry': True} for v in self.cursor.get('bill_retries', {}).values() if v['attempts'] < 3 and v['next_at'] <= self.run.started_at.timestamp()]
+        if bill_due:
+            state.setdefault('pending', []).extend(d for d in bill_due if d['url'] not in {p['url'] for p in state['pending']})
+        self.run.stats['unresolved_bills'] = len(self.cursor.get('bill_retries', {}))
+        return self.advance('realtime')
+
+    def request_for(self, url, callback, lane, **meta):
+        url = official_url(self.name, url)
+        request = self.request(url, callback, lane=lane, official_source=self.name, **meta)
+        request.priority = 10 if lane == 'realtime' else -10
+        # Credentials never travel to document hosts or into persisted URLs.
+        if urlsplit(url).hostname == 'api.congress.gov':
+            request.headers['X-Api-Key'] = get_settings().congress_api_key
+        request.headers['User-Agent'] = 'AlphaPulse official news collector ' + get_settings().crawler_contact_email
+        return request
+
+    def advance(self, lane):
+        state = self.cursor[lane]
+        if configuration_error(self.name):
+            return []
+        if lane == 'history' and time.monotonic() >= self.run.history_deadline:
+            return []
+        if state.get('pending'):
+            doc = state['pending'][0]
+            if doc['kind'] == 'bill_finish':
+                self.finish_bill(lane)
+                return self.advance(lane)
+            if doc['kind'] == 'actions' and not state['bill']['reasons']:
+                state['pending'].pop(0)
+                return self.advance(lane)
+            return [self.request_for(doc['url'], self.parse_document, lane, document=doc)]
+        if state.get('page_finished'):
+            if state.pop('day_complete', False):
+                state['day'] = (datetime.fromisoformat(state['day']) - timedelta(days=1)).date().isoformat()
+                state['entry'] -= 1
+            if state.get('next'):
+                state['page'] = state.pop('next')
+            else:
+                state['entry'] += 1
+                state.pop('page', None)
+                state.pop('visited_pages', None)
+            state.pop('page_finished', None)
+        spec = SOURCES[self.name]
+        if state['entry'] >= len(spec.entries):
+            if lane == 'history':
+                self.run.stats['attachment_gaps'] = len(self.cursor.get('attachment_retries', {}))
+                self.run.history_done = True
+                self.run.history_position = self.run.history_target
+                state['scanned'] = True
+            else:
+                self.cursor['realtime'] = {'entry': 0}
+            return []
+        url = state.get('page') or spec.entries[state['entry']]
+        if self.name == 'congress' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            url = configure_url(url, fromDateTime=begin.strftime('%Y-%m-%dT%H:%M:%SZ'), toDateTime=self.run.started_at.strftime('%Y-%m-%dT%H:%M:%SZ'))
+        if self.name == 'federal_register' and not state.get('page'):
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            # Daily partitions stay below the API's 2,000-result pagination ceiling.
+            day = state.setdefault('day', self.run.started_at.date().isoformat())
+            if day < begin.date().isoformat():
+                state['entry'] += 1
+                return self.advance(lane)
+            url = configure_url(url, **{'conditions[publication_date][is]': day, 'per_page': '1000', 'order': 'oldest'})
+        state['page'] = url
+        return [self.request_for(url, self.parse_listing, lane)]
+
+    def parse_listing(self, response):
+        lane = response.meta['lane']
+        state = self.cursor[lane]
+        docs, next_url = [], None
+        if self.name == 'doj':
+            data = response.json()
+            resultset = data['metadata']['resultset']
+            if data['metadata']['responseInfo']['status'] != 200 or not isinstance(data['results'], list):
+                raise ValueError('DOJ API 列表结构未识别')
+            dates = []
+            for record in data['results']:
+                dates.append(datetime.fromtimestamp(int(record['date']), UTC))
+                docs.append({'url': f"https://www.justice.gov/api/v1/press_releases/{uuid.UUID(record['uuid'])}.json", 'kind': 'doj'})
+            page, size, count = (int(resultset[k]) for k in ('page', 'pagesize', 'count'))
+            if size <= 0 or page < 0 or (not docs and page * size < count):
+                raise ValueError('DOJ API 分页结果不完整')
+            begin = self.run.history_target if lane == 'history' else self.run.cutoff
+            if (page + 1) * size < count and not (dates and all(d.date() < begin.date() for d in dates)):
+                next_url = configure_url(response.url, page=page + 1)
+        elif self.name == 'fed' and response.url.endswith('releaseDates.json'):
+            data = response.json()
+            if not isinstance(data, list) or not data:
+                raise ValueError('H.4.1 日期列表结构未识别')
+            docs = [{'url': f'https://www.federalreserve.gov/releases/h41/{day}/', 'kind': 'article'} for year in data for month in year['Months'] for day in month['Dates'] if int(year['yearValue']) >= 2025]
+            docs.sort(key=lambda d: d['url'], reverse=True)
+        elif self.name in ('congress', 'federal_register'):
+            data = response.json()
+            key = ('committeeMeetings' if '/committee-meeting' in response.url else 'bills') if self.name == 'congress' else 'results'
+            if key not in data or not isinstance(data[key], list):
+                raise ValueError('官方 API 列表结构未识别')
+            if self.name == 'federal_register' and data.get('count', 0) > 2000:
+                day = state['day']
+                url = f'https://www.govinfo.gov/content/pkg/FR-{day}/xml/FR-{day}.xml'
+                state['pending'] = [{'url': url, 'kind': 'register_issue', 'day': day, 'count': data['count']}]
+                yield from self.advance(lane)
+                return
+            for doc in data[key]:
+                if self.name == 'congress':
+                    docs.append({'url': configure_url(doc['url'], format='json'), 'kind': 'meeting' if key == 'committeeMeetings' else 'bill'})
+                else:
+                    docs.append({'url': f"https://www.federalregister.gov/api/v1/documents/{doc['document_number']}.json", 'kind': 'register'})
+            next_url = data.get('pagination', {}).get('next') if self.name == 'congress' else data.get('next_page_url')
+        else:
+            if not response.css('h1'):
+                raise ValueError('官方列表页面未识别')
+            pattern = SOURCES[self.name].article_path
+            for a in response.css('a[href]'):
+                try:
+                    url = official_url(self.name, a.attrib['href'], response.url)
+                except ValueError:
+                    continue
+                if re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf'):
+                    docs.append({'url': url, 'kind': 'article'})
+                if a.attrib.get('rel') == 'next' or re.match(r'^(?:Next|Next page|下一页)', a.xpath('normalize-space(.)').get() or '', re.I):
+                    next_url = url
+            docs = list({d['url']: d for d in docs}.values())
+            if not docs:
+                raise ValueError('官方列表未发现文档，不能认定历史遍历完成')
+        if lane == 'realtime':
+            # OFAC's recent-actions entry is a descending day-indexed feed.
+            if self.name == 'ofac' and docs:
+                days = [re.search(r'/recent-actions/(\d{8})', d['url']) for d in docs]
+                if all(days) and all(m[1] < self.run.cutoff.strftime('%Y%m%d') for m in days):
+                    next_url = None
+            checked = self.cursor.setdefault('checked', {})
+            threshold = (self.run.started_at - timedelta(hours=24)).timestamp()
+            self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
+            docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
+            if self.name == 'fed':
+                def recent(d):
+                    found = re.search(r'(202[5-9]\d{4})', d['url'])
+                    return not found or datetime.strptime(found[1], '%Y%m%d').replace(tzinfo=UTC) >= self.run.cutoff
+                docs = [d for d in docs if recent(d)]
+        if next_url and official_url(self.name, next_url, response.url) == response.url:
+            raise ValueError('官方列表翻页重复当前页面，未推进断点')
+        visited = state.setdefault('visited_pages', [])
+        if response.url in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        visited.append(response.url)
+        if next_url and official_url(self.name, next_url, response.url) in visited:
+            raise ValueError('官方列表分页出现循环，未推进断点')
+        state['pending'] = docs
+        state['next'] = official_url(self.name, next_url, response.url) if next_url else None
+        state['page_finished'] = True
+        self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(docs)
+        if self.name == 'federal_register' and not next_url:
+            state['day_complete'] = True
+        yield from self.advance(lane)
+
+    async def parse_document(self, response):
+        lane, doc = response.meta['lane'], response.meta['document']
+        if doc['kind'] == 'register_issue':
+            root = etree.fromstring(response.body, etree.XMLParser(resolve_entities=False, no_network=True))
+            if root.tag != 'FEDREG' or published(root.findtext('DATE', default=''))[0].date().isoformat() != doc['day']:
+                raise ValueError('Federal Register XML 日期或结构不符')
+            numbers = set()
+            for node in root.iter('FRDOC'):
+                match = re.search(r'FR Doc\.\s+([^\s\]]+)', ''.join(node.itertext()))
+                if not match:
+                    raise ValueError('Federal Register XML 文档编号无法解析')
+                numbers.add(match[1])
+            if len(numbers) != doc['count']:
+                raise ValueError('Federal Register XML 文档数量与 API 不符')
+            state = self.cursor[lane]
+            state['pending'] = [{'url': f'https://www.federalregister.gov/api/v1/documents/{number}.json', 'kind': 'register'} for number in sorted(numbers)]
+            state.update(page_finished=True, next=None, day_complete=True)
+            self.run.stats['discovered'] = self.run.stats.get('discovered', 0) + len(numbers)
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'meeting':
+            data = response.json()['committeeMeeting']
+            at, precision = published(data['updateDate'])
+            if not selection(self.name, data['title'], json.dumps(data.get('relatedItems', {}))):
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+                for result in self.complete(lane):
+                    yield result
+                return
+            parsed = {'title': data['title'], 'content_html': '<pre>' + escape(json.dumps(data, ensure_ascii=False, indent=2)) + '</pre>', 'published_at': at, 'url': response.url, 'external_id': identity(response.url), 'raw_response_id': response.meta['raw_response_id'],
+                      'extra': {'content_scope': 'article', 'published_at_precision': precision, 'published_at_basis': 'official_update', 'published_date': at.date().isoformat(), 'official_meeting': data, 'rule_version': RULE_VERSION, 'attachments': [{'url': official_url(self.name, d['url']), 'title': d.get('name') or d.get('documentType', ''), 'status': 'pending'} for d in {d['url']: d for d in data.get('meetingDocuments', []) + data.get('witnessDocuments', []) if d.get('format') == 'PDF'}.values()]}}
+            for result in self.attach_or_finish(parsed, lane):
+                yield result
+            return
+        if doc['kind'] == 'bill':
+            data = response.json()['bill']
+            base = response.url.split('?')[0]
+            state = self.cursor[lane]
+            previous = self.cursor.setdefault('bill_retries', {}).get(base)
+            version = data.get('updateDateIncludingText') or data.get('updateDate')
+            if previous and previous.get('version') != version:
+                self.cursor['bill_retries'].pop(base)
+            if previous and previous.get('version') == version and (previous['attempts'] >= 3 or previous['next_at'] > self.run.started_at.timestamp()):
+                for result in self.complete(lane):
+                    yield result
+                return
+            reasons = selection(self.name, data['title'], json.dumps(data.get('policyArea', {})))
+            state['bill'] = {'url': base, 'document_url': doc['url'], 'title': data['title'], 'reasons': reasons, 'bodies': 0, 'version': version, 'retry': bool(doc.get('bill_retry') or doc.get('attachment_retry'))}
+            kinds = ([] if reasons else ['subjects', 'summaries']) + ['texts', 'actions', 'bill_finish']
+            state['pending'][0:1] = [{'url': base + '/' + ('text' if kind == 'texts' else kind) + '?format=json&limit=250', 'kind': kind, 'title': data['title'], 'bill': base} for kind in kinds]
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('subjects', 'summaries'):
+            data = response.json()
+            key = 'subjects' if doc['kind'] == 'subjects' else 'summaries'
+            if key not in data:
+                raise ValueError('官方法案补充资料结构未识别')
+            state = self.cursor[lane]
+            state['bill']['reasons'] = state['bill']['reasons'] or selection(self.name, doc['title'], json.dumps(data[key]))
+            children = []
+            if next_url := data.get('pagination', {}).get('next'):
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            state['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] in ('actions', 'texts'):
+            data = response.json()
+            children = []
+            if doc['kind'] == 'actions':
+                for action in data['actions']:
+                    key = identity(doc['bill'] + json.dumps(action, sort_keys=True))
+                    at, precision = published(action['actionDate'])
+                    begin = datetime(2025, 1, 1, tzinfo=UTC) if self.cursor[lane]['bill'].get('retry') else (self.run.history_target if lane == 'history' else self.run.cutoff)
+                    if at < begin:
+                        continue
+                    item = self.item(response, external_id=key, url=congress_bill_url(doc['bill']), title=doc['title'], summary='', content_html=f"<p>{escape(action['text'])}</p>", published_at=at,
+                                     extra={'content_scope': 'official_action', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'bill': doc['bill'], 'official_action': action, 'rule_version': RULE_VERSION})
+                    yield item
+            else:
+                for version in data['textVersions']:
+                    if not version.get('date'):
+                        self.cursor[lane]['bill']['missing_material'] = True
+                        continue
+                    formats = version.get('formats', [])
+                    chosen = next((f for f in formats if f['type'] == 'Formatted Text'), None) or next((f for f in formats if f['type'] == 'Formatted XML'), None) or next((f for f in formats if f['type'] == 'PDF'), None)
+                    if not chosen:
+                        self.cursor[lane]['bill']['missing_material'] = True
+                    if chosen:
+                        children.append({'url': chosen['url'], 'kind': 'text', 'title': doc['title'] + ' — ' + version['type'], 'at': version['date'], 'bill': doc['bill']})
+            next_url = data.get('pagination', {}).get('next')
+            if next_url:
+                children.append({**doc, 'url': official_url(self.name, next_url)})
+            self.cursor[lane]['pending'][0:1] = children
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'register':
+            data = response.json()
+            url = data.get('body_html_url') or data.get('raw_text_url')
+            if not url:
+                raise ValueError('规则文档缺少正文地址')
+            self.cursor[lane]['pending'][0] = {'url': official_url(self.name, url), 'kind': 'text', 'title': data['title'], 'at': data['publication_date'], 'canonical': data['html_url'], 'document_number': data['document_number'], 'effective_on': data.get('effective_on'), 'pdf': data.get('pdf_url')}
+            for result in self.advance(lane):
+                yield result
+            return
+        if doc['kind'] == 'doj':
+            data = response.json()
+            records = data['results']
+            if data['metadata']['responseInfo']['status'] != 200 or len(records) != 1:
+                raise ValueError('DOJ API 正文结构未识别')
+            record = records[0]
+            if str(uuid.UUID(record['uuid'])) != doc['url'].rsplit('/', 1)[1].removesuffix('.json'):
+                raise ValueError('DOJ API 正文 UUID 不符')
+            canonical = official_url(self.name, record['url'].replace('https://justice.gov/', 'https://www.justice.gov/', 1))
+            title = unescape(record['title']).strip()
+            if not title:
+                raise ValueError('DOJ 正文缺少标题')
+            # The API release date is a calendar date, not a release instant.
+            date = datetime.fromtimestamp(int(record['date']), UTC).date().isoformat()
+            attachment_html = record.get('attachment') or ''
+            api_attachments = []
+            if isinstance(attachment_html, list):
+                for attachment in attachment_html:
+                    file_id = attachment['file']['id']
+                    if not str(file_id).isdigit():
+                        raise ValueError('DOJ API 附件编号无效')
+                    api_attachments.append({'url': f'https://www.justice.gov/media/{file_id}/dl', 'title': attachment.get('description') or 'Official attachment', 'status': 'pending'})
+                attachment_html = ''
+            elif not isinstance(attachment_html, str):
+                raise ValueError('DOJ API 附件结构未识别')
+            html = f'<h1>{escape(title)}</h1><time datetime="{date}"></time><article><div class="field--name-body">{record["body"]}{attachment_html}</div></article>'
+            validate_body(record['body'])
+            parsed = article(self.name, html, canonical)
+            known = {a['url'] for a in parsed['extra']['attachments']}
+            parsed['extra']['attachments'].extend(a for a in api_attachments if a['url'] not in known)
+            changed = etree.HTML(record.get('changed') or '')
+            if changed is not None:
+                times = changed.xpath('//time/@datetime')
+                if times:
+                    parsed['source_updated_at'] = published(times[0])[0]
+            parsed['extra']['official_uuid'] = record['uuid']
+            doc = {**doc, 'canonical': canonical}
+        elif doc['kind'] == 'text':
+            at, precision = published(doc['at'])
+            if response.body.startswith(b'%PDF'):
+                content = '<pre>' + escape(pdf_text(response.body)) + '</pre>'
+            else:
+                content = response.text
+                validate_body(content)
+                if '<' not in content:
+                    content = '<pre>' + escape(content) + '</pre>'
+            parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}}
+            if doc.get('pdf'):
+                parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
+        else:
+            parsed = article(self.name, response.text, response.url)
+        if doc.get('bill'):
+            context = self.cursor[lane]['bill']
+            context['bodies'] += 1
+            if not context['reasons']:
+                context['reasons'] = selection(self.name, context['title'], html_to_text(parsed['content_html']))
+                if context['reasons']:
+                    # A later version can establish relevance for earlier versions.
+                    self.cursor[lane]['pending'][1:1] = context.pop('unmatched_texts', [])
+                else:
+                    context.setdefault('unmatched_texts', []).append(copy.deepcopy(doc))
+            parsed['extra']['selection_reasons'] = context['reasons']
+        begin = self.run.history_target if lane == 'history' else self.run.cutoff
+        if doc.get('attachment_retry') or (doc.get('bill') and self.cursor[lane]['bill'].get('retry')):
+            begin = datetime(2025, 1, 1, tzinfo=UTC)
+        if parsed['published_at'] < begin:
+            self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
+            for result in self.complete(lane):
+                yield result
+            return
+        if lane == 'history':
+            self.run.history_position = min(self.run.history_position or self.run.started_at, parsed['published_at'])
+        parsed['url'] = doc.get('canonical') or response.url
+        parsed['external_id'] = identity(parsed['url'])
+        if self.name == 'fed':
+            # Preserve pre-existing RSS GUID identities during full-text enrichment.
+            async with SessionLocal() as session:
+                existing = await session.scalar(select(NewsItem.external_id).where(NewsItem.source_code == 'fed', NewsItem.url == parsed['url']).limit(1))
+                if existing:
+                    parsed['external_id'] = existing
+        parsed['raw_response_id'] = response.meta['raw_response_id']
+        parsed['retry_document'] = {'url': doc['bill'], 'kind': 'bill'} if doc.get('bill') else doc
+        for result in self.attach_or_finish(parsed, lane):
+            yield result
+
+    def finish_bill(self, lane):
+        context = self.cursor[lane].pop('bill')
+        retries = self.cursor.setdefault('bill_retries', {})
+        if not context['bodies'] or context.get('missing_material'):
+            previous = retries.get(context['url'], {})
+            retries[context['url']] = {'document': {'url': context['url'], 'kind': 'bill'}, 'version': context['version'], 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        else:
+            retries.pop(context['url'], None)
+            if not context['reasons']:
+                self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        self.run.stats['unresolved_bills'] = len(retries)
+        self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
+
+    def attach_or_finish(self, parsed, lane):
+        pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
+        if pending:
+            request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
+            request.meta['handle_httpstatus_list'] = [404, 410]
+            yield request
+            return
+        text = html_to_text(parsed['content_html'])
+        reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
+        incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
+        retry_doc = parsed.pop('retry_document', None)
+        retries = self.cursor.setdefault('attachment_retries', {})
+        if incomplete and retry_doc:
+            previous = retries.get(parsed['url'], {})
+            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (self.run.started_at + timedelta(hours=1)).timestamp()}
+        elif not incomplete:
+            retries.pop(parsed['url'], None)
+        if reasons or incomplete:
+            parsed['extra']['selection_reasons'] = reasons
+            parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
+            self.run.stats['partial'] = self.run.stats.get('partial', 0) + int(incomplete)
+            raw_id = parsed.pop('raw_response_id')
+            item = self.item(Response(parsed['url'], request=Request(parsed['url'], meta={'raw_response_id': raw_id})), summary='', **parsed)
+            yield item
+        else:
+            self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
+        yield from self.complete(lane)
+
+    def parse_attachment(self, response):
+        parsed = response.meta['parsed']
+        attachment = next(a for a in parsed['extra']['attachments'] if a['url'] == response.url or a['status'] == 'pending')
+        attachment['raw_response_id'] = response.meta['raw_response_id']
+        try:
+            if response.status != 200:
+                raise ValueError(f'HTTP {response.status}')
+            text = pdf_text(response.body)
+            attachment['status'] = 'complete'
+            parsed['content_html'] += f"<section><h2>{escape(attachment['title'])}</h2><pre>{escape(text)}</pre></section>"
+        except (ValueError, OSError) as exc:
+            attachment.update(status='unavailable', error=str(exc))
+            self.logger.warning('附件缺口：%s', response.url)
+        yield from self.attach_or_finish(parsed, response.meta['lane'])
+
+    def complete(self, lane):
+        doc = self.cursor[lane]['pending'].pop(0)
+        if lane == 'realtime':
+            self.cursor.setdefault('checked', {})[doc['url']] = self.run.started_at.timestamp()
+        return self.advance(lane)
+
+
+class SecSpider(USOfficialSpider):
+    name = 'sec'
+
+
+class CftcSpider(USOfficialSpider):
+    name = 'cftc'
+
+
+class TreasurySpider(USOfficialSpider):
+    name = 'treasury'
+
+
+class OfacSpider(USOfficialSpider):
+    name = 'ofac'
+
+
+class FincenSpider(USOfficialSpider):
+    name = 'fincen'
+
+
+class WhitehouseSpider(USOfficialSpider):
+    name = 'whitehouse'
+
+
+class CongressSpider(USOfficialSpider):
+    name = 'congress'
+
+
+class FederalRegisterSpider(USOfficialSpider):
+    name = 'federal_register'
+
+
+class OccSpider(USOfficialSpider):
+    name = 'occ'
+
+
+class DojSpider(USOfficialSpider):
+    custom_settings = {**USOfficialSpider.custom_settings, 'DOWNLOAD_DELAY': 1.0}
+    name = 'doj'

backend/src/alphapulse/models/news.py

@@ -25,6 +25,7 @@ class Source(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # 历史回补：目标时间（为空表示不回补）、已回补到的最早时间（断点）、是否已结束（到达目标或来源无更早数据）
+    history_cursor: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     history_target: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_position: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     history_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
@@ -71,6 +72,7 @@ class NewsRevision(Base):
     summary: Mapped[str] = mapped_column(Text)
     content_html: Mapped[str] = mapped_column(Text)
     content_hash: Mapped[str] = mapped_column(String(64))
+    extra: Mapped[dict | None] = mapped_column(JSONB)
     captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
 
@@ -110,6 +112,7 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     # running / success / failed / interrupted
     status: Mapped[str] = mapped_column(String(16))
+    stats: Mapped[dict] = mapped_column(JSONB, default=dict, server_default="{}")
     requests: Mapped[int] = mapped_column(Integer, default=0)
     items_seen: Mapped[int] = mapped_column(Integer, default=0)
     items_new: Mapped[int] = mapped_column(Integer, default=0)