Collect news text and image references without file attachments

Skip unsupported document formats and retire attachment work from active gap reporting. Preserve HTTP failures, normalize dynamic page content, and keep legacy bill queues resumable.

backend/src/alphapulse/api/crawler.py

@@ -18,6 +18,7 @@ from alphapulse.core.db import get_session
 from alphapulse.core.redis import KEY_PREFIX, redis
 from alphapulse.core.official_sources import REGIONS
 from alphapulse.calendar.catalog import FAMILIES
+from alphapulse.crawler.content_policy import ignored_retry
 from alphapulse.crawler.events import CONTROL_CHANNEL, HISTORY_SOURCES, LOG_CHANNEL, STATUS_KEY
 from alphapulse.crawler.spiders.us_official import configuration_error
 from alphapulse.models import CrawlLog, CrawlRun, EconomicRelease, NewsItem, Source
@@ -196,13 +197,16 @@ async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -
     db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
     attachment_stats = {}
     for source in sources:
-        retries = (source.history_cursor or {}).get('attachment_retries', {})
+        groups = {key: {url: value for url, value in (source.history_cursor or {}).get(key, {}).items() if not ignored_retry(key, value)} for key in ('attachment_retries', 'document_retries', 'bill_retries')}
+        ignored = sum(ignored_retry(key, value) for key in groups for value in (source.history_cursor or {}).get(key, {}).values())
+        retries = groups['attachment_retries']
         pending = sum(value['attempts'] < 3 for value in retries.values())
-        documents = (source.history_cursor or {}).get('document_retries', {})
+        documents = groups['document_retries']
         document_pending = sum(value['attempts'] < 3 for value in documents.values())
-        bills = (source.history_cursor or {}).get('bill_retries', {})
+        bills = groups['bill_retries']
         bill_pending = sum(value['attempts'] < 3 for value in bills.values())
         attachment_stats[source.code] = {
+            'policy_ignored': ignored,
             'retired_count': sum(bool(v.get('retired_at')) for group in (retries, documents, bills) for v in group.values()),
             'active_gaps': sum(not v.get('retired_at') for group in (retries, documents, bills) for v in group.values()),
             'attachment_gaps': len(retries),

backend/src/alphapulse/core/public_http.py

@@ -19,6 +19,9 @@ async def public_address(url):
 async def fetch_public_response(url):
     async with httpx.AsyncClient(timeout=20, trust_env=False) as client:
         for _ in range(4):
+            from alphapulse.crawler.content_policy import file_url
+            if file_url(url):
+                raise ValueError('订阅地址不支持 PDF 或附件')
             parsed, address = await public_address(url)
             # Pin the checked address while retaining TLS hostname verification.
             target = httpx.URL(url).copy_with(host=address)
@@ -27,11 +30,18 @@ async def fetch_public_response(url):
                     location = response.headers.get('location')
                     if not location: raise ValueError('重定向缺少地址')
                     url = str(httpx.URL(url).join(location)); continue
-                chunks, size = [], 0
+                from scrapy.http import Headers
+                from alphapulse.crawler.content_policy import binary_headers
+                if binary_headers(Headers(response.headers.multi_items())):
+                    raise ValueError('订阅返回不支持的文件类型')
+                chunks, size, prefix = [], 0, b''
                 async for chunk in response.aiter_bytes():
                     size += len(chunk)
                     if size > 2*1024*1024: raise ValueError('响应超过2MiB限制')
                     chunks.append(chunk)
+                    prefix = (prefix + chunk)[:8]
+                    if prefix.lstrip().startswith(b'%PDF'):
+                        raise ValueError('订阅返回 PDF 文件')
                 body = b''.join(chunks)
                 # Bytes are already decompressed; Scrapy must not decode them again.
                 headers = [(k, v) for k, v in response.headers.multi_items()

backend/src/alphapulse/core/us_news.py

@@ -32,7 +32,7 @@ SOURCES = {
     'doj': OfficialSource('DOJ·加密机构执法', ('https://www.justice.gov/api/v1/press_releases.json?pagesize=50&sort=date&direction=DESC&fields=uuid,date',), r'/(?:opa|usao[^/]*|archives/opa)/(?:pr|speech)/[^/]+', '.field--name-body, .field--name-field-pr-body', ('www.justice.gov',)),
     'fed': OfficialSource('美联储·政策与讲话', tuple(f'https://www.federalreserve.gov/newsevents/{kind}/{year}-{suffix}.htm' for year in range(datetime.now(UTC).year, 2024, -1) for kind, suffix in (('pressreleases', 'press'), ('speech', 'speeches'), ('testimony', 'testimony'))) + ('https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm', 'https://www.federalreserve.gov/releases/h41/releaseDates.json'), r'/(?:newsevents/(?:pressreleases|speech|testimony)/[^/]+\.htm|monetarypolicy/(?:fomcminutes|fomcprojtabl)\d+\.htm|releases/h41/\d+/)', '#article, .col-xs-12.col-sm-8.col-md-8, #content', ('www.federalreserve.gov',), 60),
 }
-RULE_VERSION = 'us-policy-v2'
+RULE_VERSION = 'us-policy-v3-text-images'
 CRYPTO = re.compile(r'\b(?:crypto(?:currency|currencies|[ -]assets?)?|digital[ -]assets?|virtual[ -]currenc(?:y|ies)|stablecoins?|bitcoin|ethereum|blockchain|tokeniz(?:ed|ation))\b', re.I)
 POLICY = re.compile(r'\b(?:regulat\w*|rules?|guidance|legislation|acts?|sanctions?|licenses?|executive order|monetary policy|federal funds|interest rates?|balance sheet|liquidity|quantitative|refunding|borrowing|debt limit|tariffs?|fiscal|budget|appropriation|tax reform)\b', re.I)
 INSTITUTION = re.compile(r'\b(?:exchanges?|banks?|financial institutions?|payment processors?|custodians?|stablecoin issuers?|binance|coinbase|tether|circle|kraken|bybit|bitfinex)\b', re.I)
@@ -88,7 +88,7 @@ def published(value: str) -> tuple[datetime, str]:
     raise ValueError('官方材料缺少可识别的发布时间')
 
 
-def validate_body(content: str) -> None:
+def validate_body(content: str, *, allow_images: bool = False) -> None:
     """Reject error documents without requiring full-page HTML around fragments."""
     if content.lstrip().startswith('<?xml'):
         from lxml import etree
@@ -103,13 +103,21 @@ def validate_body(content: str) -> None:
         raise ValueError('官方正文返回验证或登录页面')
     if re.search(r'enable javascript and cookies to continue', text, re.I):
         raise ValueError('官方正文返回验证页面')
-    if len(text) < 40:
+    if len(text) < 40 and not (allow_images and sel.css('img[src], img[data-src]')):
         raise ValueError('官方正文为空或过短')
 
 
+class UnsupportedDocument(ValueError):
+    """官方页面只提供文件，没有支持的网页正文。"""
+
+
 def article(source: str, html: str, url: str, *, release_date: str | None = None) -> dict:
     sel = Selector(html)
-    validate_body(html)
+    from alphapulse.crawler.content_policy import SKIP_REASON
+    file_entry = (source == 'sec' and sel.css('article.node--type-regulation a[href*=".pdf"]')) or (source == 'fincen' and sel.css('main .field--name-field-advisory-file, main a[href*="/system/files/"]'))
+    if file_entry and not sel.css(SOURCES[source].body):
+        raise UnsupportedDocument(SKIP_REASON)
+    validate_body(html, allow_images=True)
     selectors = {
         'fed': ('#article h3.title', '#article h3', '#article h2', '#content .heading h3', '#content .col-xs-12.col-sm-8.col-md-8 h3'),
         'treasury': ('h2.uswds-page-title',),
@@ -158,9 +166,10 @@ def article(source: str, html: str, url: str, *, release_date: str | None = None
         if parent is not None:
             parent.remove(node.root)
     body = sorted(body, key=lambda node: len(node.xpath('string(.)').get()), reverse=True)
-    content = body[0].get()
+    from alphapulse.crawler.content_policy import clean_html
+    content = clean_html(body[0].get(), url)
     text = ' '.join(body[0].xpath('.//text()').getall()).strip()
-    if len(text) < 40:
+    if len(text) < 40 and not body[0].css('img[src], img[data-src]'):
         raise ValueError('官方正文为空或过短')
     attachments = []
     for link in body[0].css('a[href]'):

backend/src/alphapulse/crawler/content_policy.py

@@ -0,0 +1,64 @@
+"""新闻只采集网页文字与图片引用；经济日历不使用此策略。"""
+import re
+from urllib.parse import urljoin, urlsplit
+
+from parsel import Selector
+
+POLICY = 'web_text_images_v1'
+SKIP_REASON = '按采集规则不下载 PDF 和附件'
+FILE_PATH = re.compile(r'\.(?:pdf|docx?|xlsx?|pptx?|zip|gz|rar|7z|rtf|odt|ods|odp|exe|bin)(?:$|/)', re.I)
+
+
+def file_url(url):
+    parts = urlsplit(url)
+    return bool(FILE_PATH.search(parts.path) or re.search(r'\.pdf(?:[?&#]|$)', parts.query, re.I))
+
+
+def binary_headers(headers):
+    kind = headers.get(b'Content-Type', b'').split(b';')[0].strip().lower()
+    if not kind:
+        return False  # 无类型时由首段 PDF 签名和后续正文解析继续校验。
+    return not (kind.startswith(b'text/') or kind in (
+        b'application/json', b'application/xml', b'application/rss+xml',
+        b'application/atom+xml', b'application/xhtml+xml',
+    ) or kind.endswith((b'+json', b'+xml')))
+
+
+
+def clean_html(html, base=''):
+    """剔除执行/嵌入节点和已确认的动态分享编码，保留正文及图片。"""
+    sel = Selector(html.replace('\x00', ''))
+    for node in sel.css('script, style, iframe, object, embed, video, audio, .shareDL'):
+        parent = node.root.getparent()
+        if parent is not None:
+            previous, tail = node.root.getprevious(), node.root.tail
+            parent.remove(node.root)
+            if tail:
+                if previous is None:
+                    parent.text = (parent.text or '') + tail
+                else:
+                    previous.tail = (previous.tail or '') + tail
+    for node in sel.css('[href*="/cdn-cgi/l/email-protection"]'):
+        node.root.attrib.pop('href', None)
+    for node in sel.css('[data-cfemail]'):
+        node.root.attrib.pop('data-cfemail', None)
+    for node in sel.css('img, picture source'):
+        if node.root.tag == 'img':
+            src = node.attrib.get('data-src') or node.attrib.get('src')
+            if src:
+                node.root.set('src', urljoin(base, src))
+        for attr in ('srcset', 'data-srcset'):
+            if value := node.attrib.get(attr):
+                candidates = []
+                for part in value.split(','):
+                    fields = part.strip().split()
+                    if fields:
+                        candidates.append(' '.join([urljoin(base, fields[0]), *fields[1:]]))
+                node.root.set('srcset', ', '.join(candidates))
+    # Selector adds an html/body wrapper; keep the original fragment contract.
+    body = sel.css('body')
+    return ''.join(body[0].xpath('node()').getall()) if body else sel.get()
+
+
+def ignored_retry(group, value):
+    return group == 'attachment_retries' or bool(value.get('skipped_policy')) or file_url(value.get('document', {}).get('url', ''))

backend/src/alphapulse/crawler/middlewares.py

@@ -46,6 +46,12 @@ class RawResponseMiddleware:
         return cls(crawler)
 
     async def process_response(self, request: Request, response: Response) -> Response:
+        if response.body.lstrip().startswith(b'%PDF'):
+            if not 200 <= response.status < 300:
+                from scrapy.spidermiddlewares.httperror import HttpError
+                raise HttpError(response)
+            request.meta['policy_skipped'] = True
+            raise IgnoreRequest('按采集规则不保存 PDF 响应')
         sha256 = hashlib.sha256(response.body).hexdigest()
         headers = {k.decode(): b", ".join(v).decode("latin-1") for k, v in response.headers.items()}
         async with SessionLocal() as session:
@@ -66,3 +72,38 @@ class RawResponseMiddleware:
             await session.commit()
         request.meta["raw_response_id"] = raw_id
         return response
+
+
+class NewsContentPolicyMiddleware:
+    """在下载前及响应头阶段阻止文件；此下载器仅由新闻采集使用。"""
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        from scrapy import signals
+        obj = cls()
+        crawler.signals.connect(obj.headers_received, signal=signals.headers_received)
+        crawler.signals.connect(obj.bytes_received, signal=signals.bytes_received)
+        return obj
+
+    def process_request(self, request):
+        from alphapulse.crawler.content_policy import file_url, SKIP_REASON
+        if file_url(request.url) or request.meta.get('parsed'):
+            request.meta['policy_skipped'] = True
+            raise IgnoreRequest(SKIP_REASON)
+
+    def headers_received(self, headers, request, **kwargs):
+        from scrapy.exceptions import StopDownload
+        from alphapulse.crawler.content_policy import binary_headers
+        if binary_headers(headers):
+            request.meta['policy_skipped'] = True
+            raise StopDownload()
+
+    def bytes_received(self, data, request, **kwargs):
+        from scrapy.exceptions import StopDownload
+        prefix = request.meta.get('_content_prefix', b'')
+        if len(prefix) < 8:
+            prefix = (prefix + data)[:8]
+            request.meta['_content_prefix'] = prefix
+            if prefix.lstrip().startswith(b'%PDF'):
+                request.meta['policy_skipped'] = True
+                raise StopDownload()

backend/src/alphapulse/crawler/pipelines.py

@@ -6,7 +6,9 @@ from scrapy.crawler import Crawler
 from sqlalchemy import func, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
+from alphapulse.crawler.content_policy import clean_html
 from alphapulse.core.db import SessionLocal
+from alphapulse.crawler.spiders.base import html_to_text
 from alphapulse.core.redis import redis
 from alphapulse.crawler.events import NEWS_CHANNEL
 from alphapulse.models import NewsItem, NewsRevision
@@ -31,6 +33,11 @@ class NewsPipeline:
 
     async def process_item(self, item: dict[str, Any]) -> dict[str, Any]:
         spider = self.crawler.spider
+        for field in ('title', 'summary', 'author'):
+            if isinstance(item.get(field), str):
+                item[field] = item[field].replace('\x00', '')
+        item['content_html'] = clean_html(item['content_html'], item['url'])
+        item['content_text'] = html_to_text(item['content_html'])
         digest = content_hash(item)
         raw_response_id = item.pop("raw_response_id")
         locked = (
@@ -56,11 +63,14 @@ class NewsPipeline:
             if inserted_id is not None:
                 kind = "new"
             else:
+                same_content = (news.title.replace('\x00', '') == item['title']
+                    and news.summary.replace('\x00', '') == item['summary']
+                    and clean_html(news.content_html, news.url) == item['content_html'])
                 # 主记录保存最新元数据；正文修订仍只由内容哈希变化决定。
                 for field in UPDATABLE_FIELDS:
                     setattr(news, field, item[field])
                 news.last_seen_at = func.now()
-                if news.content_hash != digest:
+                if not same_content and news.content_hash != digest:
                     kind = "revised"
                     news.content_hash = digest
                     news.revision_no += 1

backend/src/alphapulse/crawler/settings.py

@@ -25,6 +25,7 @@ RETRY_TIMES = 2
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
 
 DOWNLOADER_MIDDLEWARES = {
+    "alphapulse.crawler.middlewares.NewsContentPolicyMiddleware": 90,
     "alphapulse.crawler.middlewares.RedisRateLimitMiddleware": 100,
     # process_response 按优先级从大到小调用：排在 HttpCompression(590) 之后拿到解压后的响应体，
     # 排在 Retry(550) 之前，被重试的每次响应也都会记录

backend/src/alphapulse/crawler/spiders/rss.py

@@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo
 
 import httpx
 from scrapy.crawler import Crawler
-from scrapy.http import Request, Response
+from scrapy.http import Request, Response, Headers
 from scrapy.settings.default_settings import RETRY_EXCEPTIONS
 
 from alphapulse.crawler.spiders.base import NewsSpider, html_to_text
@@ -164,7 +164,16 @@ class BlsDownloadHandler:
         async with httpx.AsyncClient(verify=self.ssl_context, timeout=request.meta.get("download_timeout", self.timeout)) as client:
             async with client.stream("GET", request.url, headers=headers) as response:
                 # 保留原始压缩体，避免 httpx 与 Scrapy 的 HttpCompressionMiddleware 重复解压。
-                body = b"".join([chunk async for chunk in response.aiter_raw()])
+                from alphapulse.crawler.content_policy import binary_headers
+                if binary_headers(Headers(response.headers.multi_items())):
+                    raise ValueError('订阅返回不支持的文件类型')
+                chunks, prefix = [], b''
+                async for chunk in response.aiter_raw():
+                    chunks.append(chunk)
+                    prefix = (prefix + chunk)[:8]
+                    if prefix.lstrip().startswith(b'%PDF'):
+                        raise ValueError('订阅返回 PDF 文件')
+                body = b''.join(chunks)
                 return Response(
                     str(response.url), status=response.status_code, headers=response.headers.multi_items(),
                     body=body, request=request,

backend/src/alphapulse/crawler/spiders/us_official.py

@@ -4,24 +4,24 @@ import json
 import re
 import time
 import uuid
-from contextlib import nullcontext
 from datetime import UTC, datetime, timedelta
 from html import escape, unescape
 from email.utils import parsedate_to_datetime
 
 from lxml import etree
-from pypdf import apply_configuration
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 from scrapy import Request
+from scrapy.exceptions import StopDownload
 from scrapy.http import Response
 from scrapy.spidermiddlewares.httperror import HttpError
 from sqlalchemy import select
+from twisted.python.failure import Failure
 
-from alphapulse.calendar.umich import pdf_text
+from alphapulse.crawler.content_policy import POLICY, SKIP_REASON, file_url
 from alphapulse.core.config import get_settings
 from alphapulse.core.db import SessionLocal
-from alphapulse.core.us_news import RULE_VERSION, SOURCES, article, identity, official_url, published, selection, validate_body
+from alphapulse.core.us_news import RULE_VERSION, SOURCES, UnsupportedDocument, article, identity, official_url, published, selection, validate_body
 from alphapulse.crawler.spiders.base import HISTORY_RUN_SECONDS, NewsSpider, html_to_text
 from alphapulse.models import NewsItem, Source
 
@@ -143,17 +143,11 @@ class USOfficialSpider(NewsSpider):
                     self.cursor[lane]['pending'] = [d for d in self.cursor[lane]['pending'] if d['url'] not in indexes]
         state = self.cursor.setdefault('realtime', {'entry': 0})
         retries = self.cursor.get('document_retries', {})
-        due_documents = [{**value['document'], 'document_retry': True} for value in retries.values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
+        rechecks = [{**v['document'], 'document_retry': True, 'policy_recheck': True} for v in retries.values() if v['attempts'] >= 3 and not v.get('policy_checked') and not v.get('skipped_policy') and not file_url(v['document']['url'])][:3]
+        state.setdefault('pending', []).extend(d for d in rechecks if d['url'] not in {p['url'] for p in state['pending']})
+        due_documents = [{**value['document'], 'document_retry': True} for value in retries.values() if not value.get('skipped_policy') and value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
         if due_documents:
             state.setdefault('pending', []).extend(copy.deepcopy(d) for d in due_documents if d['url'] not in {p['url'] for p in state['pending']})
-        due = [value['document'] for value in self.cursor.get('attachment_retries', {}).values() if value['attempts'] < 3 and value['next_at'] <= self.run.started_at.timestamp()]
-        for doc in due:
-            pending = state.setdefault('pending', [])
-            existing = next((value for value in pending if value['url'] == doc['url']), None)
-            if existing is None:
-                pending.append({**copy.deepcopy(doc), 'attachment_retry': True})
-            else:
-                existing['attachment_retry'] = True
         bill_due = [{**v['document'], 'bill_retry': True} for v in self.cursor.get('bill_retries', {}).values() if v['attempts'] < 3 and v['next_at'] <= self.run.started_at.timestamp()]
         if bill_due:
             state.setdefault('pending', []).extend(d for d in bill_due if d['url'] not in {p['url'] for p in state['pending']})
@@ -182,10 +176,15 @@ class USOfficialSpider(NewsSpider):
             return []
         if state.get('pending'):
             doc = state['pending'][0]
+            if file_url(doc['url']) or doc.get('format') == 'pdf':
+                return self.skip_document(lane, doc)
             gap = self.cursor.get('document_retries', {}).get(doc['url'])
-            if doc['url'] in self.run.document_retries_started or (gap and (gap['attempts'] >= 3 or gap['next_at'] > self.run.started_at.timestamp())):
+            recheck = doc.get('policy_recheck') and gap and not gap.get('policy_checked')
+            if doc['url'] in self.run.document_retries_started or (not recheck and gap and (gap.get('skipped_policy') or gap['attempts'] >= 3 or gap['next_at'] > self.run.started_at.timestamp())):
                 state['pending'].pop(0)
                 return self.advance(lane)
+            if recheck:
+                gap['policy_checked'] = POLICY
             if gap:
                 doc['document_retry'] = True
                 self.run.document_retries_started.add(doc['url'])
@@ -344,6 +343,9 @@ class USOfficialSpider(NewsSpider):
                     continue
                 fincen_row = a.xpath('ancestor::tr[ancestor::main][1]') if self.name == 'fincen' and response.url.split('?')[0] in SOURCES['fincen'].entries[1:] else []
                 fincen_pdf = bool(fincen_row and urlsplit(url).path.lower().endswith('.pdf'))
+                if file_url(url) and re.search(pattern, urlsplit(url).path) and not fincen_pdf:
+                    docs.append({'url': url, 'kind': 'article', 'format': 'pdf'})
+                    continue
                 if fincen_pdf or (re.search(pattern, urlsplit(url).path) and url not in SOURCES[self.name].entries and not re.search(r'/\d{4}-(?:press|speeches|testimony)\.htm$', urlsplit(url).path) and not url.lower().endswith('.pdf')):
                     doc = {'url': url, 'kind': 'article'}
                     if fincen_row:
@@ -352,11 +354,13 @@ class USOfficialSpider(NewsSpider):
                         begin = self.run.history_target if lane == 'history' else self.run.cutoff
                         if re.fullmatch(r'(?:0?[1-9]|1[0-2])/\d{4}', date) and int(date[-4:]) < begin.year:
                             continue
-                        doc['listing_date'] = published(date)[0].date().isoformat()
+                        try:
+                            doc['listing_date'] = published(date)[0].date().isoformat()
+                        except ValueError:
+                            if not fincen_pdf:
+                                raise
                         if fincen_pdf:
                             title = fincen_row.xpath('normalize-space(td[3])').get() if response.url.split('?')[0] == SOURCES['fincen'].entries[2] else a.xpath('normalize-space(.)').get()
-                            if not title:
-                                raise ValueError('FinCEN PDF 列表缺少标题')
                             doc.update(format='pdf', title=title, listing_url=response.url)
                     elif self.name in ('sec', 'treasury', 'cftc', 'fincen', 'whitehouse'):
                         container = {
@@ -410,7 +414,7 @@ class USOfficialSpider(NewsSpider):
             self.cursor['checked'] = {k: v for k, v in checked.items() if v >= threshold}
             docs = [d for d in docs if self.cursor['checked'].get(d['url'], 0) < threshold]
         retries = self.cursor.get('document_retries', {})
-        docs = [d for d in docs if d['url'] not in retries or (retries[d['url']]['attempts'] < 3 and retries[d['url']]['next_at'] <= self.run.started_at.timestamp())]
+        docs = [d for d in docs if d['url'] not in retries or (not retries[d['url']].get('skipped_policy') and retries[d['url']]['attempts'] < 3 and retries[d['url']]['next_at'] <= self.run.started_at.timestamp())]
         docs = [{**d, 'document_retry': True} if d['url'] in retries else d for d in docs]
         if next_url and official_url(self.name, next_url, response.url) == response.url:
             raise ValueError('官方列表翻页重复当前页面，未推进断点')
@@ -440,7 +444,7 @@ class USOfficialSpider(NewsSpider):
         now = datetime.now(UTC)
         if not previous or previous['next_at'] <= now.timestamp():
             retries[doc['url']] = {'document': copy.deepcopy(doc), 'attempts': previous.get('attempts', 0) + 1,
-                                  'next_at': (now + timedelta(hours=1)).timestamp(), 'error': error}
+                                  'next_at': (now + timedelta(hours=1)).timestamp(), 'error': error, **({'policy_checked': POLICY} if previous.get('policy_checked') else {})}
         self.run.document_retries_started.add(doc['url'])
         self.run.errors.append(error)
         self.logger.error('文档缺口：%s；%s', doc['url'], error)
@@ -449,7 +453,15 @@ class USOfficialSpider(NewsSpider):
         return self.advance(request.meta['lane'])
 
     def on_request_error(self, failure):
+        # 文件响应可在响应头阶段被截断，但 HTTP 故障仍须保留退避和重试语义。
+        if failure.check(StopDownload) and not 200 <= failure.value.response.status < 300:
+            response, request = failure.value.response, failure.request
+            request.meta.pop('policy_skipped', None)
+            failure = Failure(HttpError(response))
+            failure.request = request
         status = failure.value.response.status if failure.check(HttpError) else None
+        if failure.request.meta.get('parsed'):
+            return list(self.attach_or_finish(failure.request.meta['parsed'], failure.request.meta['lane']))
         if self.name in ('sec', 'doj', 'congress') and status in (403, 429, 503):
             if status in (403, 429):
                 self.block_until(datetime.now(UTC) + timedelta(hours=1))
@@ -461,10 +473,10 @@ class USOfficialSpider(NewsSpider):
                         self.block_until(until)
                 except (ValueError, TypeError, OverflowError):
                     self.logger.warning('忽略无效 Retry-After 响应头')
-        if self.name in ('sec', 'doj', 'congress') and (parsed := failure.request.meta.get('parsed')):
-            attachment = next(a for a in parsed['extra']['attachments'] if a['status'] == 'pending')
-            attachment.update(status='unavailable', error=f'请求失败：{status or repr(failure.value)}')
-            return list(self.attach_or_finish(parsed, failure.request.meta['lane']))
+        if failure.request.meta.get('policy_skipped'):
+            if doc := failure.request.meta.get('document'):
+                return self.skip_document(failure.request.meta['lane'], doc)
+            return super().on_request_error(failure)
         if status not in (403, 429) or self.name in ('sec', 'doj', 'congress'):
             requests = self.document_failed(failure.request, f'请求失败 {failure.request.url}：{status or repr(failure.value)}')
             if requests is not None:
@@ -475,6 +487,9 @@ class USOfficialSpider(NewsSpider):
         try:
             async for result in self._parse_document(response):
                 yield result
+        except UnsupportedDocument:
+            for request in self.skip_document(response.meta['lane'], response.meta['document']):
+                yield request
         except (ValueError, KeyError, TypeError) as exc:
             if '验证' in str(exc) or '登录页面' in str(exc):
                 self.block_until(datetime.now(UTC) + timedelta(minutes=15))
@@ -487,6 +502,10 @@ class USOfficialSpider(NewsSpider):
     async def _parse_document(self, response):
         lane, doc = response.meta['lane'], response.meta['document']
         begin = self.run.history_target if lane == 'history' else self.run.cutoff
+        if file_url(doc['url']) or doc.get('format') == 'pdf':
+            for request in self.skip_document(lane, doc):
+                yield request
+            return
         if not (doc.get('document_retry') or doc.get('attachment_retry')) and doc.get('listing_date') and published(doc['listing_date'])[0].date() < begin.date():
             self.run.stats['outside_range'] = self.run.stats.get('outside_range', 0) + 1
             for result in self.complete(lane):
@@ -572,12 +591,16 @@ class USOfficialSpider(NewsSpider):
                     yield item
             else:
                 for version in data['textVersions']:
+                    if version.get('formats') and all(f['type'] == 'PDF' for f in version['formats']):
+                        self.cursor[lane]['bill']['policy_skipped'] = True
+                        self.run.stats['policy_skipped'] = self.run.stats.get('policy_skipped', 0) + 1
+                        continue
                     if not version.get('date'):
                         self.cursor[lane]['bill']['missing_material'] = True
                         self.cursor[lane]['bill'].setdefault('material_errors', []).append(f"{version.get('type', '正文版本')} 缺少官方日期")
                         continue
                     formats = version.get('formats', [])
-                    chosen = next((f for f in formats if f['type'] == 'Formatted Text'), None) or next((f for f in formats if f['type'] == 'Formatted XML'), None) or next((f for f in formats if f['type'] == 'PDF'), None)
+                    chosen = next((f for f in formats if f['type'] == 'Formatted Text'), None) or next((f for f in formats if f['type'] == 'Formatted XML'), None) or None
                     if not chosen:
                         self.cursor[lane]['bill']['missing_material'] = True
                         self.cursor[lane]['bill'].setdefault('material_errors', []).append(f"{version.get('type', '正文版本')} 缺少支持的正文格式")
@@ -628,7 +651,7 @@ class USOfficialSpider(NewsSpider):
                 attachment_html = ''
             elif not isinstance(attachment_html, str):
                 raise ValueError('DOJ API 附件结构未识别')
-            html = f'<h1>{escape(title)}</h1><time datetime="{date}"></time><article><div class="field--name-body">{record["body"]}{attachment_html}</div></article>'
+            html = f'<h1>{escape(title)}</h1><time datetime="{date}"></time><article><div class="field--name-body">{record["body"]}{record.get("image") or ""}{attachment_html}</div></article>'
             validate_body(record['body'])
             parsed = article(self.name, html, canonical)
             known = {a['url'] for a in parsed['extra']['attachments']}
@@ -640,20 +663,13 @@ class USOfficialSpider(NewsSpider):
                     parsed['source_updated_at'] = published(times[0])[0]
             parsed['extra']['official_uuid'] = record['uuid']
             doc = {**doc, 'canonical': canonical}
-        elif doc['kind'] == 'text' or doc.get('format') == 'pdf':
-            at, precision = published(doc['listing_date'] if doc.get('format') == 'pdf' else doc['at'])
-            if response.body.startswith(b'%PDF'):
-                content = '<pre>' + escape(self.pdf_text(response.body)) + '</pre>'
-            else:
-                content = response.text
-                validate_body(content)
-                if doc.get('format') == 'pdf':
-                    raise ValueError('官方 PDF 返回非 PDF 正文')
-                if '<' not in content:
-                    content = '<pre>' + escape(content) + '</pre>'
+        elif doc['kind'] == 'text':
+            at, precision = published(doc['at'])
+            content = response.text
+            validate_body(content)
+            if '<' not in content:
+                content = '<pre>' + escape(content) + '</pre>'
             parsed = {'title': doc['title'], 'content_html': content, 'published_at': at, 'extra': {'content_scope': 'article', 'language': 'en', 'published_at_precision': precision, 'published_date': at.date().isoformat(), 'rule_version': RULE_VERSION, 'official_document': doc, 'attachments': []}}
-            if doc.get('format') == 'pdf':
-                parsed['extra'].update(published_at_basis='official_listing', published_at_source=doc['listing_url'])
             if doc.get('pdf'):
                 parsed['extra']['attachments'].append({'url': official_url(self.name, doc['pdf']), 'title': 'Official PDF', 'status': 'pending'})
         else:
@@ -698,7 +714,7 @@ class USOfficialSpider(NewsSpider):
     def finish_bill(self, lane):
         context = self.cursor[lane].pop('bill')
         retries = self.cursor.setdefault('bill_retries', {})
-        if not context['bodies'] or context.get('missing_material'):
+        if (not context['bodies'] and not context.get('policy_skipped')) or context.get('missing_material'):
             previous = retries.get(context['url'], {})
             errors = list(context.get('material_errors', []))
             if not context['bodies']:
@@ -714,66 +730,38 @@ class USOfficialSpider(NewsSpider):
         if lane == 'realtime':
             self.cursor.setdefault('checked', {})[context['document_url']] = self.run.started_at.timestamp()
 
+    def skip_document(self, lane, doc):
+        if doc.get('bill'):
+            self.cursor[lane]['bill']['policy_skipped'] = True
+        previous = self.cursor.get('document_retries', {}).get(doc['url'], {})
+        self.cursor.setdefault('document_retries', {})[doc['url']] = {
+            **previous, 'document': doc, 'attempts': previous.get('attempts', 0),
+            'next_at': 0, 'skipped_policy': POLICY, 'error': SKIP_REASON,
+            'previous_error': previous.get('previous_error') or previous.get('error'),
+        }
+        self.run.stats['policy_skipped'] = self.run.stats.get('policy_skipped', 0) + 1
+        self.cursor[lane]['pending'].pop(0)
+        return self.advance(lane)
+
     def attach_or_finish(self, parsed, lane):
-        retired = self.cursor.get('attachment_retries', {}).get(parsed['url'], {})
-        if retired.get('retired_at'):
-            for attachment in parsed['extra']['attachments']:
-                if attachment['status'] == 'pending':
-                    attachment.update(status='unavailable', error=retired['error'])
-        pending = next((a for a in parsed['extra']['attachments'] if a['status'] == 'pending'), None)
-        if pending:
-            request = self.request_for(pending['url'], self.parse_attachment, lane, parsed=parsed)
-            request.meta['handle_httpstatus_list'] = [404, 410]
-            yield request
-            return
-        self.retire_exhausted()
+        for attachment in parsed['extra'].get('attachments', []):
+            attachment.update(status='skipped_policy', error=SKIP_REASON)
+        parsed['extra']['collection_policy'] = POLICY
+        parsed['extra'].setdefault('content_scope', 'article')
+        parsed.pop('retry_document', None)
         text = html_to_text(parsed['content_html'])
         reasons = selection(self.name, parsed['title'], text) or parsed['extra'].get('selection_reasons', [])
-        incomplete = any(a['status'] != 'complete' for a in parsed['extra']['attachments'])
-        retry_doc = parsed.pop('retry_document', None)
-        retries = self.cursor.setdefault('attachment_retries', {})
-        if incomplete and retry_doc and not retired.get('retired_at'):
-            previous = retries.get(parsed['url'], {})
-            error = '；'.join(f"{a['url']}：{a.get('error') or '附件尚未完成文字提取'}" for a in parsed['extra']['attachments'] if a['status'] != 'complete')
-            retries[parsed['url']] = {'document': retry_doc, 'attempts': previous.get('attempts', 0) + 1, 'next_at': (datetime.now(UTC) + timedelta(hours=1)).timestamp(), 'error': error, 'failed_urls': [a['url'] for a in parsed['extra']['attachments'] if a['status'] != 'complete']}
-        elif not incomplete:
-            retries.pop(parsed['url'], None)
-        self.retire_exhausted()
         if reasons:
             parsed['extra']['selection_reasons'] = reasons
-            parsed['extra']['content_scope'] = 'partial' if incomplete else 'article'
-            self.run.stats['partial'] = self.run.stats.get('partial', 0) + int(incomplete)
             raw_id = parsed.pop('raw_response_id')
-            item = self.item(Response(parsed['url'], request=Request(parsed['url'], meta={'raw_response_id': raw_id})), summary='', **parsed)
-            yield item
+            yield self.item(Response(parsed['url'], request=Request(parsed['url'], meta={'raw_response_id': raw_id})), summary='', **parsed)
         else:
             self.run.stats['excluded'] = self.run.stats.get('excluded', 0) + 1
-            if incomplete:
-                self.run.stats['unclassified_attachment_gaps'] = self.run.stats.get('unclassified_attachment_gaps', 0) + 1
         yield from self.complete(lane)
 
-    def pdf_text(self, body):
-        # Bound the known complex FinCEN report without changing other collectors' limits.
-        context = apply_configuration(stream_parsing_work_maximum_count=4_000_000) if self.name == 'fincen' else nullcontext()
-        with context:
-            return pdf_text(body)
-
     def parse_attachment(self, response):
-        parsed = response.meta['parsed']
-        attachment = next(a for a in parsed['extra']['attachments'] if a['url'] == response.url or a['status'] == 'pending')
-        attachment['raw_response_id'] = response.meta['raw_response_id']
-        try:
-            if response.status != 200:
-                raise ValueError(f'HTTP {response.status}')
-            if response.headers.get('Content-Type', b'').lower().startswith(b'image/') and response.body.startswith((b'\x89PNG\r\n\x1a\n', b'\xff\xd8\xff', b'GIF87a', b'GIF89a')):
-                raise ValueError('官方图片附件尚未提取文字，请核对原文')
-            text = self.pdf_text(response.body)
-            attachment['status'] = 'complete'
-            parsed['content_html'] += f"<section><h2>{escape(attachment['title'])}</h2><pre>{escape(text)}</pre></section>"
-        except (ValueError, OSError) as exc:
-            attachment.update(status='unavailable', error=str(exc))
-            self.logger.warning('附件缺口：%s', response.url)
-        yield from self.attach_or_finish(parsed, response.meta['lane'])
+        """只供 Redis 遗留回调反序列化；新流程不创建此请求。"""
+        yield from self.attach_or_finish(response.meta['parsed'], response.meta['lane'])
 
     def retire_exhausted(self):
         if self.name not in ('sec', 'doj', 'congress'):

backend/tests/test_news_content_policy.py

@@ -0,0 +1,123 @@
+"""新闻格式限制、内容稳定性及历史哈希兼容。"""
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from scrapy import Request
+from scrapy.exceptions import IgnoreRequest, StopDownload
+from scrapy.http import Headers
+
+from alphapulse.crawler.content_policy import clean_html, ignored_retry
+
+
+@pytest.fixture
+def guard(trading):
+    from alphapulse.crawler.middlewares import NewsContentPolicyMiddleware
+    return NewsContentPolicyMiddleware()
+
+
+def test_body_images_tables_and_real_changes_survive_normalization():
+    html = '<p>政策正文\x00</p><script>noise</script><iframe src="a.pdf"></iframe><table><tr><td>3%</td></tr></table><img data-src="/chart.png" srcset="/a.png 1x, /b.png 2x"><figcaption>图表说明</figcaption>'
+    value = clean_html(html, 'https://official.example/news/1')
+    assert '政策正文' in value and '3%' in value and '图表说明' in value
+    assert 'https://official.example/chart.png' in value
+    assert 'https://official.example/a.png 1x' in value
+    assert 'iframe' not in value and 'script' not in value and '\x00' not in value
+    assert clean_html(value, 'https://official.example/news/1') == value
+    assert clean_html(html.replace('3%', '4%')) != clean_html(html)
+
+
+def test_email_share_encoding_is_not_news_content():
+    html = '<p>Monetary policy.</p><a href="/cdn-cgi/l/email-protection#abc" data-cfemail="abc">Email</a>'
+    assert clean_html(html) == clean_html(html.replace('abc', 'def'))
+
+
+def test_picture_sources_and_lazy_candidates_resolve_to_publisher():
+    html = ('<picture><source type="image/webp" srcset="/wide.webp 1x, ../large.webp 2x">'
+        '<source media="(min-width: 800px)" data-srcset="//cdn.example/chart.webp 800w">'
+        '<img src="/fallback.png" alt="政策图表"></picture>')
+    base = 'https://official.example/news/article'
+    value = clean_html(html, base)
+    from parsel import Selector
+    sel = Selector(value)
+    assert sel.css('picture source::attr(srcset)').getall() == [
+        'https://official.example/wide.webp 1x, https://official.example/large.webp 2x',
+        'https://cdn.example/chart.webp 800w',
+    ]
+    assert sel.css('img::attr(src)').get() == 'https://official.example/fallback.png'
+    assert sel.css('img::attr(alt)').get() == '政策图表'
+    assert clean_html(value, base) == value
+
+
+@pytest.mark.parametrize('url', ['https://official.example/a.PDF?download=1', 'https://official.example/a.docx', 'https://official.example/exit?https://other.example/a.pdf'])
+def test_file_urls_are_rejected_before_download(url, guard):
+    request = Request(url)
+    with pytest.raises(IgnoreRequest):
+        guard.process_request(request)
+    assert request.meta['policy_skipped']
+
+
+def test_unknown_file_and_split_magic_stop_without_a_complete_body(guard):
+    req = Request('https://official.example/download')
+    with pytest.raises(StopDownload):
+        guard.headers_received(Headers({'Content-Type': 'application/pdf'}), req)
+    req = Request('https://official.example/mislabeled')
+    guard.bytes_received(b'%P', req)
+    with pytest.raises(StopDownload):
+        guard.bytes_received(b'DF-1.7', req)
+    assert req.meta['policy_skipped']
+
+
+def test_legacy_attachment_request_is_rejected_even_without_file_extension(guard):
+    request = Request('https://official.example/media/1/download', meta={'parsed': {'title': 'old'}})
+    with pytest.raises(IgnoreRequest):
+        guard.process_request(request)
+    assert ignored_retry('attachment_retries', {'attempts': 3})
+    assert not ignored_retry('document_retries', {'document': {'url': 'https://official.example/news/1'}})
+
+
+def test_normalization_preserves_existing_hash_and_sends_only_real_updates(trading, monkeypatch):
+    from alphapulse.crawler import pipelines
+    html = '<p>Monetary policy.</p><a href="/cdn-cgi/l/email-protection#abc">Email</a>'
+    fields = dict(url='https://official.example/news/1', title='Policy', summary='', content_html=html,
+        content_text='Monetary policy.Email', author=None, tags=[], is_paywalled=False,
+        published_at=None, source_updated_at=None, extra={})
+    news = SimpleNamespace(id=1, revision_no=5, content_hash='legacy-hash', **fields)
+    session = AsyncMock()
+    session.scalar.return_value = news
+    session.add = lambda row: None
+    context = AsyncMock()
+    context.__aenter__.return_value = session
+    monkeypatch.setattr(pipelines, 'SessionLocal', lambda: context)
+    publish = AsyncMock()
+    monkeypatch.setattr(pipelines, 'redis', SimpleNamespace(publish=publish))
+    run = SimpleNamespace(items_seen=0, items_revised=0)
+    pipeline = pipelines.NewsPipeline(SimpleNamespace(spider=SimpleNamespace(run=run)))
+    def item(content):
+        return {**fields, 'source_code': 'fed', 'external_id': 'id', 'content_html': content, 'raw_response_id': None}
+    asyncio.run(pipeline.process_item(item(html.replace('abc', 'def'))))
+    assert news.revision_no == 5 and news.content_hash == 'legacy-hash'
+    publish.assert_not_awaited()
+    asyncio.run(pipeline.process_item(item(html.replace('Monetary policy.', 'New interest rate.'))))
+    assert news.revision_no == 6 and run.items_revised == 1
+    publish.assert_awaited_once()
+
+
+def test_html_cleanup_keeps_text_after_removed_nodes_and_image_only_articles():
+    from alphapulse.core.us_news import article
+    assert '保留文字' in clean_html('<script>noise</script>保留文字')
+    html = '<time datetime="2026-11-06"></time><article><h1>Policy chart</h1><div class="field--name-body"><img src="/chart.png"><figcaption>利率图</figcaption></div></article>'
+    parsed = article('cftc', html, 'https://www.cftc.gov/PressRoom/PressReleases/1-26')
+    assert 'https://www.cftc.gov/chart.png' in parsed['content_html']
+
+
+@pytest.mark.parametrize('source,body', [
+    ('sec', '<article class="node--type-regulation"><a href="/files/rules/report.pdf">Final Rule</a></article>'),
+    ('fincen', '<main><a href="/system/files/advisory.pdf">Official advisory</a></main>'),
+])
+def test_file_entry_pages_are_not_mistaken_for_news(source, body):
+    from alphapulse.core.us_news import UnsupportedDocument, article
+    html = '<h1>Official policy document</h1><time datetime="2026-11-06"></time>' + body
+    with pytest.raises(UnsupportedDocument):
+        article(source, html, 'https://official.example/document')

backend/tests/test_us_official_news.py

@@ -12,6 +12,7 @@ from scrapy.http import HtmlResponse, Response
 from scrapy.spidermiddlewares.httperror import HttpError
 from twisted.python.failure import Failure
 
+from alphapulse.crawler.content_policy import file_url
 from alphapulse.core.us_news import SOURCES, article, official_url, published, selection
 
 
@@ -40,6 +41,72 @@ def collection_sample(source, suffix=''):
     return next(s for s in samples if s['source'] == source and s['url'].endswith(suffix))
 
 
+@pytest.mark.parametrize('source', ['ofac', 'sec'])
+@pytest.mark.parametrize('status', [200, 403, 429, 503])
+def test_stopped_binary_response_preserves_http_errors(trading, source, status):
+    from scrapy.exceptions import StopDownload
+    from scrapy.http import Headers
+    from alphapulse.crawler.middlewares import NewsContentPolicyMiddleware
+    obj = spider(trading, source)
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': SOURCES[source].entries[0] + '/document', 'kind': 'article'}
+    obj.cursor['history']['pending'] = [doc]
+    request = Request(doc['url'], meta={'document': doc, 'lane': 'history'})
+    headers = Headers({'Content-Type': 'application/octet-stream', 'Retry-After': '120'})
+    with pytest.raises(StopDownload) as stopped:
+        NewsContentPolicyMiddleware().headers_received(headers, request)
+    stopped.value.response = Response(request.url, status=status, headers=headers, request=request)
+    failure = Failure(stopped.value)
+    failure.request = request
+    obj.on_request_error(failure)
+    gap = obj.cursor.get('document_retries', {}).get(doc['url'], {})
+    if status == 200:
+        assert gap['skipped_policy']
+        assert obj.run.stats['policy_skipped'] == 1
+        assert not obj.run.errors
+    else:
+        assert not request.meta.get('policy_skipped')
+        assert not gap.get('skipped_policy')
+        assert not obj.run.stats.get('policy_skipped')
+        assert str(status) in obj.run.errors[0]
+        if gap:
+            assert gap['attempts'] == 1 and gap['next_at'] > obj.run.started_at.timestamp()
+        if source == 'sec' or status in (403, 429):
+            assert obj.run.blocked and obj.run.cooldown_until is not None
+
+
+@pytest.mark.parametrize('missing_material', [False, True])
+def test_legacy_bill_pdf_skip_does_not_hide_real_text_gaps(trading, monkeypatch, missing_material):
+    monkeypatch.setattr('alphapulse.crawler.spiders.us_official.configuration_error', lambda _: None)
+    obj = spider(trading, 'congress')
+    base = 'https://api.congress.gov/v3/bill/119/s/1'
+    pdf = {'url': 'https://www.congress.gov/old.pdf', 'kind': 'text', 'bill': base}
+    obj.cursor['history'].update(entry=len(SOURCES['congress'].entries), pending=[pdf, {'url': base, 'kind': 'bill_finish'}],
+        bill={'url': base, 'document_url': base, 'title': 'Crypto policy', 'version': 'v1', 'bodies': 0,
+              'reasons': ['crypto_policy'], 'missing_material': missing_material,
+              'material_errors': ['网页正文缺少官方日期'] if missing_material else []})
+    assert obj.advance('history') == []
+    assert obj.cursor['document_retries'][pdf['url']]['skipped_policy']
+    assert not obj.cursor['history']['pending']
+    if missing_material:
+        gap = obj.cursor['bill_retries'][base]
+        assert '网页正文缺少官方日期' in gap['error'] and gap['attempts'] == 1
+    else:
+        assert not obj.cursor['bill_retries']
+        assert obj.run.stats['unresolved_bills'] == 0
+
+
+def test_decoded_pdf_error_keeps_http_failure_before_raw_storage(trading):
+    from alphapulse.crawler.middlewares import RawResponseMiddleware
+    obj = spider(trading)
+    request = Request('https://ofac.treasury.gov/recent-actions/20261104')
+    response = Response(request.url, status=503, body=b'%PDF error body', request=request)
+    with pytest.raises(HttpError) as error:
+        asyncio.run(RawResponseMiddleware(SimpleNamespace(spider=obj)).process_response(request, response))
+    assert error.value.response.status == 503
+    assert not request.meta.get('policy_skipped')
+
+
 def test_real_empty_register_day_continues_to_previous_day(trading):
     obj = spider(trading, 'federal_register')
     obj.cursor['realtime']['day'] = '2026-11-05'
@@ -61,24 +128,18 @@ def test_register_completed_body_checkpoint_matches_discovery_after_restart(trad
     assert any(r.url == metadata for r in requests) is not recent
 
 
-def test_register_completes_metadata_only_after_body_and_pdf(trading, monkeypatch):
-    from alphapulse.crawler.spiders import us_official
+def test_register_completes_after_text_without_pdf(trading, monkeypatch):
     obj = spider(trading, 'federal_register')
-    metadata = 'https://www.federalregister.gov/api/v1/documents/2026-22248.json'
-    body = 'https://www.federalregister.gov/documents/full_text/html/2026/11/03/2026-22248.html'
-    pdf = 'https://www.govinfo.gov/content/pkg/FR-2026-11-03/pdf/2026-22248.pdf'
-    doc = {'url': metadata, 'kind': 'register'}
-    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES[obj.name].entries))
-    data = {'body_html_url': body, 'title': 'Digital asset exchange rule', 'publication_date': '2026-11-03', 'html_url': 'https://www.federalregister.gov/documents/2026/11/03/2026-22248/rule', 'document_number': '2026-22248', 'pdf_url': pdf}
-    async def parse(res):
-        return [item async for item in obj.parse_document(res)]
-    body_request, = asyncio.run(parse(response(metadata, json.dumps(data), lane='realtime', document=doc)))
-    assert metadata not in obj.cursor.get('checked', {})
-    pdf_request, = asyncio.run(parse(response(body, '<p>This official rule regulates digital asset exchanges and financial institutions.</p>', **body_request.meta)))
-    assert metadata not in obj.cursor.get('checked', {})
-    monkeypatch.setattr(us_official, 'pdf_text', lambda value: 'The official digital asset exchange rule and supporting material.')
-    list(obj.parse_attachment(Response(pdf, body=b'%PDF-test', request=pdf_request.replace(meta={**pdf_request.meta, 'raw_response_id': 13}))))
-    assert metadata in obj.cursor['checked'] and body in obj.cursor['checked']
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.federalregister.gov/body', 'kind': 'text', 'title': 'Digital asset rule', 'at': '2026-11-03', 'pdf': 'https://www.govinfo.gov/a.pdf', 'metadata_url': 'https://www.federalregister.gov/metadata'}
+    obj.cursor['history']['pending'] = [doc]
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], '<p>Rules for digital asset exchanges and financial institutions.</p>', lane='history', document=doc))]
+    item, = asyncio.run(collect())
+    assert item['extra']['attachments'][0]['status'] == 'skipped_policy'
+    assert not obj.run.errors
+    assert not obj.cursor.get('attachment_retries')
+
 
 
 @pytest.mark.parametrize('data', [{'count': 2}, {'count': '0', 'description': 'x'}, {'count': 0}, {'count': 0, 'description': 'x', 'results': None}])
@@ -125,59 +186,55 @@ def test_fincen_real_pdf_tables_finish_outside_realtime_window(trading, suffix):
 
 
 @pytest.mark.parametrize('suffix,title', [('/news/enforcement-actions', 'Bank of America'), ('/resources/advisoriesbulletinsfact-sheets', 'Foreign-Agent-Directed')])
-def test_fincen_real_pdf_tables_preserve_title_and_date_for_history(trading, suffix, title):
+def test_fincen_pdf_tables_skip_files_and_continue_history(trading, suffix, title):
     obj = spider(trading, 'fincen')
     sample = collection_sample('fincen', suffix)
-    request, = obj.parse_listing(response(sample['url'], sample['body'], lane='history'))
-    doc = request.meta['document']
-    assert doc['kind'] == 'article' and doc['format'] == 'pdf'
-    assert title in doc['title'] and len(doc['listing_date']) == 10
-    assert doc['listing_url'] == sample['url']
-    assert request.url.endswith('.pdf')
+    output = list(obj.parse_listing(response(sample['url'], sample['body'], lane='history')))
+    assert all(not file_url(r.url) for r in output if isinstance(r, Request))
+    assert obj.run.stats.get('policy_skipped', 0) > 0
 
 
-def test_fincen_pdf_keeps_listing_date_raw_evidence_and_selection(trading, monkeypatch):
-    from alphapulse.crawler.spiders import us_official
+
+def test_fincen_direct_pdf_is_skipped(trading, monkeypatch):
     obj = spider(trading, 'fincen')
-    doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]}
-    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries))
-    monkeypatch.setattr(us_official, 'pdf_text', lambda body: 'This enforcement action concerns a virtual currency exchange financial institution.')
-    request = Request(doc['url'], meta={'lane': 'realtime', 'document': doc, 'raw_response_id': 12})
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.fincen.gov/system/files/report.pdf', 'kind': 'article', 'format': 'pdf'}
+    obj.cursor['history']['pending'] = [doc]
     async def collect():
-        return [item async for item in obj.parse_document(Response(doc['url'], body=b'%PDF-test', request=request))]
-    item, = asyncio.run(collect())
-    assert item['published_at'] == datetime(2026, 11, 3, tzinfo=UTC)
-    assert item['raw_response_id'] == 12
-    assert item['extra']['published_at_precision'] == 'date'
-    assert item['extra']['published_at_basis'] == 'official_listing'
-    assert item['extra']['selection_reasons']
-    assert doc['url'] in obj.cursor['checked']
+        return [v async for v in obj.parse_document(response(doc['url'], '<h1>PDF or error document</h1>', lane='history', document=doc))]
+    assert asyncio.run(collect()) == []
+    assert obj.cursor['document_retries'][doc['url']]['skipped_policy']
+    assert not obj.run.errors and not obj.cursor['history']['pending']
 
 
-def test_fincen_invalid_pdf_remains_a_document_gap(trading):
+
+def test_fincen_pdf_error_response_is_still_policy_skipped(trading):
     obj = spider(trading, 'fincen')
-    doc = {'url': 'https://www.fincen.gov/system/files/crypto.pdf', 'kind': 'article', 'format': 'pdf', 'title': 'Crypto exchange consent order', 'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1]}
-    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries))
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.fincen.gov/system/files/report.pdf', 'kind': 'article', 'format': 'pdf'}
+    obj.cursor['history']['pending'] = [doc]
     async def collect():
-        return [item async for item in obj.parse_document(response(doc['url'], '<h1>Document</h1><p>The server returned an HTML document instead of the expected official PDF.</p>', lane='realtime', document=doc))]
-    assert not asyncio.run(collect())
-    assert doc['url'] not in obj.cursor.get('checked', {})
-    assert obj.cursor['document_retries'][doc['url']]['attempts'] == 1
-    assert '非 PDF' in obj.run.errors[0]
+        return [v async for v in obj.parse_document(response(doc['url'], '<h1>PDF or error document</h1>', lane='history', document=doc))]
+    assert asyncio.run(collect()) == []
+    assert obj.cursor['document_retries'][doc['url']]['skipped_policy']
+    assert not obj.run.errors and not obj.cursor['history']['pending']
+
 
 
 def test_fincen_pdf_table_missing_date_fails_explicitly(trading):
     obj = spider(trading, 'fincen')
     body = '<main><h1>Actions</h1><table><tr><td><a href="/system/files/order.pdf">Consent order</a></td><td></td></tr></table></main>'
-    with pytest.raises(ValueError, match='发布时间'):
-        list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+    output = list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+    assert all(not file_url(r.url) for r in output)
+    assert not obj.run.errors
 
 
 def test_fincen_pdf_table_does_not_invent_day_for_current_month(trading):
     obj = spider(trading, 'fincen')
     body = '<main><h1>Actions</h1><table><tr><td><a href="/system/files/order.pdf">Consent order</a></td><td>11/2026</td></tr></table></main>'
-    with pytest.raises(ValueError, match='发布时间'):
-        list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+    output = list(obj.parse_listing(response(SOURCES['fincen'].entries[1], body, lane='realtime')))
+    assert all(not file_url(r.url) for r in output)
+    assert not obj.run.errors
 
 
 def test_cftc_commissioner_index_is_not_a_speech(trading):
@@ -262,41 +319,29 @@ def test_due_retry_in_old_pending_cursor_is_not_discarded_by_moving_window(tradi
 
 
 @pytest.mark.parametrize('restored_pending', [False, True])
-def test_attachment_retry_outside_realtime_window_fetches_and_completes_pdf(trading, monkeypatch, restored_pending):
-    from alphapulse.crawler.spiders import us_official
+def test_legacy_attachment_retry_is_not_scheduled(trading, monkeypatch, restored_pending):
     obj = spider(trading)
     doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article', 'listing_date': '2025-01-01'}
     obj.cursor['attachment_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0}}
     if restored_pending:
         obj.cursor['realtime']['pending'] = [copy.deepcopy(doc)]
-    request, = obj.first_requests()
-    body = '<h1>Crypto sanctions</h1><main><article class="node--type-ofac-recent-action"><div class="field--name-field-body"><p>New sanctions against a virtual currency exchange institution.</p><a href="/media/123/download">Order</a></div></article></main>'
-    async def collect():
-        return [item async for item in obj.parse_document(response(request.url, body, **request.meta))]
-    attachment, = asyncio.run(collect())
-    assert isinstance(attachment, Request) and attachment.url.endswith('/media/123/download')
+    requests = obj.first_requests()
+    assert all(not r.meta.get('parsed') and not r.meta.get('document', {}).get('attachment_retry') for r in requests)
     assert obj.cursor['attachment_retries'][doc['url']]['attempts'] == 1
-    assert doc['url'] not in obj.cursor.get('checked', {})
-    monkeypatch.setattr(us_official, 'pdf_text', lambda body: 'The official sanctions order applies to a virtual currency exchange.')
-    items = list(obj.parse_attachment(Response(attachment.url, body=b'%PDF-test', request=attachment.replace(meta={**attachment.meta, 'raw_response_id': 13}))))
-    item = next(item for item in items if isinstance(item, dict))
-    assert item['extra']['content_scope'] == 'article'
-    assert item['extra']['attachments'][0]['status'] == 'complete'
-    assert not obj.cursor['attachment_retries']
-    assert doc['url'] in obj.cursor['checked']
 
 
-def test_due_attachment_is_queued_alongside_document_retry_without_duplicates(trading):
+
+def test_only_document_retry_is_scheduled(trading):
     obj = spider(trading)
     documents = [{'url': f'https://ofac.treasury.gov/recent-actions/2026110{day}', 'kind': 'article'} for day in (3, 4)]
     obj.cursor['document_retries'] = {documents[0]['url']: {'document': documents[0], 'attempts': 1, 'next_at': 0, 'error': 'HTTP503'}}
     obj.cursor['attachment_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0} for doc in documents}
     request, = obj.first_requests()
     pending = obj.cursor['realtime']['pending']
-    assert [doc['url'] for doc in pending] == [doc['url'] for doc in documents]
-    assert request.meta['document']['document_retry'] and request.meta['document']['attachment_retry']
+    assert [doc['url'] for doc in pending] == [documents[0]['url']]
+    assert request.meta['document']['document_retry'] and not request.meta['document'].get('attachment_retry')
     requests = obj.complete('realtime')
-    assert requests[0].url == documents[1]['url'] and requests[0].meta['document']['attachment_retry']
+    assert all(r.url != documents[1]['url'] for r in requests)
 
 
 @pytest.mark.parametrize('first_lane', ['realtime', 'history'])
@@ -529,14 +574,13 @@ def test_unrelated_failed_attachment_is_not_news(trading):
     parsed = {'url': doc['url'], 'title': 'Federal prosecutor receives national award', 'external_id': 'award', 'content_html': '<p>The department recognized superior performance in Indian Country.</p>', 'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
               'extra': {'attachments': [{'url': 'https://www.justice.gov/media/1/dl', 'status': 'unavailable'}]}}
     assert list(obj.attach_or_finish(parsed, 'history')) == []
-    assert obj.run.stats['unclassified_attachment_gaps'] == 1
-    assert obj.cursor['attachment_retries'][doc['url']]['attempts'] == 1
+    assert not obj.run.stats.get('unclassified_attachment_gaps')
+    assert not obj.cursor.get('attachment_retries')
 
 
 @pytest.mark.parametrize('content_type,body', [('image/png', b'\x89PNG\r\n\x1a\n'), ('image/jpeg', b'\xff\xd8\xff'), ('image/gif', b'GIF89a')])
 @pytest.mark.parametrize('relevant', [True, False])
-def test_image_attachment_preserves_unknown_text_gap(trading, monkeypatch, content_type, body, relevant):
-    from alphapulse.crawler.spiders import us_official
+def test_legacy_image_attachment_does_not_create_a_gap(trading, monkeypatch, content_type, body, relevant):
     obj = spider(trading, 'doj')
     obj.run.history_deadline = time.monotonic() - 1
     url = 'https://www.justice.gov/media/1/dl'
@@ -544,25 +588,21 @@ def test_image_attachment_preserves_unknown_text_gap(trading, monkeypatch, conte
     obj.cursor['history']['pending'] = [doc]
     parsed = {'url': doc['url'], 'title': 'Cryptocurrency exchange charged' if relevant else 'Department announces an enforcement action', 'external_id': 'case', 'content_html': '<p>The cryptocurrency exchange was charged with money laundering.</p>' if relevant else '<p>See the attached consent order for the findings and penalty.</p>', 'published_at': datetime(2026, 11, 4, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
               'extra': {'attachments': [{'url': url, 'title': 'Consent Order', 'status': 'pending'}]}}
-    monkeypatch.setattr(us_official, 'pdf_text', lambda body: pytest.fail('Images must not reach the PDF parser'))
     res = Response(url, body=body, headers={'Content-Type': content_type}, request=Request(url, meta={'lane': 'history', 'parsed': parsed, 'raw_response_id': 13}))
     items = list(obj.parse_attachment(res))
     if relevant:
         item, = items
-        assert item['extra']['content_scope'] == 'partial'
+        assert item['extra']['content_scope'] == 'article'
     else:
         assert items == []
-        assert obj.run.stats['unclassified_attachment_gaps'] == 1
+        assert not obj.run.stats.get('unclassified_attachment_gaps')
     attachment = parsed['extra']['attachments'][0]
-    assert attachment['status'] == 'unavailable'
-    assert '图片' in attachment['error']
-    assert attachment['error'] in obj.cursor['attachment_retries'][doc['url']]['error']
-    assert url in obj.cursor['attachment_retries'][doc['url']]['error']
-    assert attachment['raw_response_id'] == 13
-    assert obj.cursor['attachment_retries'][doc['url']]['attempts'] == 1
+    assert attachment['status'] == 'skipped_policy'
+    assert '采集规则' in attachment['error']
+    assert not obj.cursor.get('attachment_retries')
 
 
-def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeypatch):
+def test_source_api_separates_ignored_attachments_from_real_gaps(trading, monkeypatch):
     from unittest.mock import AsyncMock
     from alphapulse.api import crawler
     from alphapulse.models import Source
@@ -586,9 +626,10 @@ def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeyp
     for stats in ({'discovered': 2, 'attachment_gaps': 999, 'unresolved_bills': 999}, {}):
         latest.stats = stats
         result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
-        assert result.collection_stats['attachment_gaps'] == 3
-        assert result.collection_stats['attachment_retry_pending'] == 2
-        assert result.collection_stats['attachment_retry_exhausted'] == 1
+        assert result.collection_stats['attachment_gaps'] == 0
+        assert result.collection_stats['policy_ignored'] == 3
+        assert result.collection_stats['attachment_retry_pending'] == 0
+        assert result.collection_stats['attachment_retry_exhausted'] == 0
         assert result.collection_stats['document_gaps'] == 2
         assert result.collection_stats['document_retry_pending'] == 1
         assert result.collection_stats['document_retry_exhausted'] == 1
@@ -603,8 +644,9 @@ def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeyp
         assert result.collection_stats['round_running'] == 0
     source.history_cursor['attachment_retries'].pop('a')
     result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
-    assert result.collection_stats['attachment_gaps'] == 2
-    assert result.collection_stats['attachment_retry_pending'] == 1
+    assert result.collection_stats['attachment_gaps'] == 0
+    assert result.collection_stats['policy_ignored'] == 2
+    assert result.collection_stats['attachment_retry_pending'] == 0
     source.history_cursor = {}
     latest.stats = {'attachment_gaps': 999}
     result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
@@ -816,7 +858,7 @@ def test_h41_uses_publisher_release_dates_not_assumed_thursdays(trading):
     assert obj.cursor['history']['pending'][1]['url'].endswith('20250103/')
 
 
-def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
+def test_attachment_reference_is_skipped_without_retry(trading):
     obj = spider(trading)
     obj.run.history_deadline = time.monotonic() - 1
     url = 'https://ofac.treasury.gov/recent-actions/20250101'
@@ -824,13 +866,9 @@ def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
     obj.cursor['history']['pending'] = [doc]
     parsed = {'url': url, 'title': 'Crypto sanctions', 'external_id': 'x', 'content_html': '<p>Virtual currency exchange sanctions.</p>', 'published_at': datetime(2025, 1, 1, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
               'extra': {'attachments': [{'url': 'https://ofac.treasury.gov/a.pdf', 'title': 'License', 'status': 'unavailable'}]}}
-    before = datetime.now(UTC)
     item, = obj.attach_or_finish(parsed, 'history')
-    after = datetime.now(UTC)
-    assert item['extra']['content_scope'] == 'partial'
-    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
-    assert obj.cursor['attachment_retries'][url]['document'] == doc
-    assert (before + timedelta(hours=1)).timestamp() <= obj.cursor['attachment_retries'][url]['next_at'] <= (after + timedelta(hours=1)).timestamp()
+    assert item['extra']['attachments'][0]['status'] == 'skipped_policy'
+    assert not obj.cursor.get('attachment_retries')
 
 
 @pytest.mark.parametrize('canceled,failed', [(False, False), (True, False), (False, True)])
@@ -1269,7 +1307,7 @@ def test_bill_undated_text_is_visible_retry_gap(trading):
     assert '缺少官方日期' in obj.cursor['bill_retries'][base]['error']
 
 
-def test_public_api_and_pdf_fixtures_replay_without_io(trading):
+def test_public_api_text_fixtures_replay_without_io(trading):
     import runpy
     from pathlib import Path
     script = runpy.run_path(str(Path(__file__).resolve().parents[2] / 'scripts/smoke_us_news.py'))
@@ -1286,25 +1324,20 @@ def test_congress_official_xml_crosscheck(trading):
     assert len(asyncio.run(script['congress_reference_items']())) == 3
 
 
-def test_congress_meeting_includes_deduplicated_witness_pdfs(trading, monkeypatch):
-    from alphapulse.crawler.spiders import us_official
-    monkeypatch.setattr(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key='test', crawler_contact_email='[email protected]'))
+def test_congress_meeting_keeps_deduplicated_pdf_references(trading, monkeypatch):
     obj = spider(trading, 'congress')
+    obj.run.history_deadline = time.monotonic() - 1
     doc = {'url': 'https://api.congress.gov/v3/committee-meeting/119/house/1', 'kind': 'meeting'}
     obj.cursor['history']['pending'] = [doc]
     pdf = {'url': 'https://www.congress.gov/witness.pdf', 'format': 'PDF', 'documentType': 'Witness Statement'}
     data = {'committeeMeeting': {'title': 'Digital asset regulation', 'updateDate': '2025-01-01T12:00:00Z', 'meetingDocuments': [pdf], 'witnessDocuments': [pdf]}}
     async def collect():
         return [v async for v in obj.parse_document(response(doc['url'], json.dumps(data), document=doc, lane='history'))]
-    req, = asyncio.run(collect())
-    assert req.url == pdf['url']
-    assert len(req.meta['parsed']['extra']['attachments']) == 1
-    assert req.meta['parsed']['extra']['attachments'][0]['title'] == 'Witness Statement'
-    obj.run.history_deadline = time.monotonic() - 1
-    req.meta['raw_response_id'] = 13
-    item, = obj.parse_attachment(Response(req.url, status=404, request=req))
-    assert item['extra']['content_scope'] == 'partial'
-    assert obj.cursor['attachment_retries'][doc['url']]['document'] == doc
+    item, = asyncio.run(collect())
+    assert len(item['extra']['attachments']) == 1
+    assert item['extra']['attachments'][0]['status'] == 'skipped_policy'
+    assert not obj.cursor.get('attachment_retries')
+
 
 
 def test_dated_but_unavailable_bill_text_remains_a_gap(trading):
@@ -1336,12 +1369,12 @@ def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_
     assert len(items) == 100
     meetings = [item for item in items if item['extra'].get('official_meeting')]
     assert len(meetings) == 1
-    assert meetings[0]['extra']['content_scope'] == 'partial'
+    assert meetings[0]['extra']['content_scope'] == 'article'
     attachments = meetings[0]['extra']['attachments']
     assert len(attachments) == 15
     assert len({a['url'] for a in attachments}) == 15
-    assert sum(a['status'] == 'complete' for a in attachments) == 14
-    assert sum(a['status'] == 'unavailable' for a in attachments) == 1
+    assert all(a['status'] == 'skipped_policy' for a in attachments)
+    assert not any(a['status'] == 'unavailable' for a in attachments)
     assert any(item['extra']['content_scope'] == 'official_action' for item in items)
 
 
@@ -1395,7 +1428,7 @@ def test_sec_uses_official_dateline_not_last_reviewed_date():
     assert item['extra']['published_at_precision'] == 'date'
 
 
-def test_attachment_http_failure_spends_durable_retry_budget(trading):
+def test_legacy_attachment_failure_does_not_spend_retry_budget(trading):
     obj = spider(trading, 'doj')
     obj.run.history_deadline = time.monotonic() - 1
     url = 'https://www.justice.gov/opa/pr/example'
@@ -1408,9 +1441,8 @@ def test_attachment_http_failure_spends_durable_retry_budget(trading):
     failure = Failure(HttpError(Response(request.url, status=503, request=request)))
     failure.request = request
     item, = obj.on_request_error(failure)
-    assert item['extra']['content_scope'] == 'partial'
-    assert obj.cursor['attachment_retries'][url]['attempts'] == 1
-    assert '503' in obj.cursor['attachment_retries'][url]['error']
+    assert item['extra']['content_scope'] == 'article'
+    assert not obj.cursor.get('attachment_retries')
 
 
 @pytest.mark.parametrize('dated', [True, False])
@@ -1447,7 +1479,7 @@ def treasury_manifest():
 
 def test_treasury_json_replaces_only_discovery_cursors(trading):
     obj = spider(trading, 'treasury')
-    obj.cursor.update(checked={'completed': 1}, document_retries={'gap': {'document': {}, 'attempts': 3}},
+    obj.cursor.update(checked={'completed': 1}, document_retries={'gap': {'document': {'url': 'https://home.treasury.gov/news/press-releases/sb0001'}, 'attempts': 3, 'next_at': 0, 'policy_checked': 'web_text_images_v1'}},
                       attachment_retries={'attachment': {'attempts': 3}}, transport='old')
     obj.cursor['history']['page'] = SOURCES['treasury'].entries[0] + '?page=2'
     requests = obj.first_requests()
@@ -1496,34 +1528,29 @@ def test_treasury_json_incomplete_manifest_cannot_complete_history(trading, fail
 
 
 @pytest.mark.parametrize('source, expected', [('fincen', 4_000_000), ('sec', 2_000_000)])
-def test_pdf_budget_is_bounded_scoped_and_restored(trading, monkeypatch, source, expected):
-    from pypdf import get_configuration
-    from alphapulse.crawler.spiders import us_official
-    def extract(body):
-        assert get_configuration().stream_parsing_work_maximum_count == expected
-        raise ValueError('known PDF error')
-    monkeypatch.setattr(us_official, 'pdf_text', extract)
-    with pytest.raises(ValueError, match='known PDF error'):
-        spider(trading, source).pdf_text(b'%PDF-test')
-    assert get_configuration().stream_parsing_work_maximum_count == 2_000_000
+def test_news_spider_has_no_pdf_parser(trading, monkeypatch, source, expected):
+    from alphapulse.crawler.middlewares import NewsContentPolicyMiddleware
+    from scrapy.exceptions import IgnoreRequest
+    obj = spider(trading, source)
+    assert not hasattr(obj, 'pdf_text')
+    request = Request(SOURCES[source].entries[0] + '/report.pdf')
+    with pytest.raises(IgnoreRequest):
+        NewsContentPolicyMiddleware().process_request(request)
+    assert request.meta['policy_skipped']
 
 
-def test_pdf_resource_failure_retains_reason_as_a_gap(trading, monkeypatch):
-    from pypdf.errors import LimitReachedError
-    from alphapulse.calendar import umich
-    def reader(_):
-        raise LimitReachedError('stream work budget exhausted')
-    monkeypatch.setattr(umich, 'PdfReader', reader)
+
+def test_legacy_pdf_resource_is_a_policy_skip(trading, monkeypatch):
     obj = spider(trading, 'fincen')
-    doc = {'url': 'https://www.fincen.gov/system/files/report.pdf', 'kind': 'article', 'format': 'pdf',
-           'listing_date': '2026-11-03', 'listing_url': SOURCES['fincen'].entries[1], 'title': 'Official report'}
-    obj.cursor['realtime'].update(pending=[doc], entry=len(SOURCES['fincen'].entries))
+    obj.run.history_deadline = time.monotonic() - 1
+    doc = {'url': 'https://www.fincen.gov/system/files/report.pdf', 'kind': 'article', 'format': 'pdf'}
+    obj.cursor['history']['pending'] = [doc]
     async def collect():
-        return [item async for item in obj.parse_document(Response(doc['url'], body=b'%PDF-test',
-            request=Request(doc['url'], meta={'lane': 'realtime', 'document': doc, 'raw_response_id': 12})))]
+        return [v async for v in obj.parse_document(response(doc['url'], '<h1>PDF or error document</h1>', lane='history', document=doc))]
     assert asyncio.run(collect()) == []
-    assert 'stream work budget exhausted' in obj.cursor['document_retries'][doc['url']]['error']
-    assert doc['url'] not in obj.cursor.get('checked', {})
+    assert obj.cursor['document_retries'][doc['url']]['skipped_policy']
+    assert not obj.run.errors and not obj.cursor['history']['pending']
+
 
 
 @pytest.mark.parametrize('failure', ['count', 'date', 'host', 'path'])

scripts/smoke_us_news.py

@@ -26,16 +26,14 @@ BACKEND_ROOT = Path(pipelines.__file__).resolve().parents[3]
 
 
 async def public_api_items():
-    """Replay captured public API bodies and PDFs without network or database I/O."""
-    import base64
+    """Replay captured public API text bodies without network or database I/O."""
     import time
     import uuid
     from datetime import UTC, datetime
     from scrapy import Request
-    from scrapy.http import HtmlResponse, Response
+    from scrapy.http import HtmlResponse
     from alphapulse.crawler.spiders.base import RunState
     samples = json.load(gzip.open(BACKEND_ROOT / 'tests/fixtures/us_news/api_samples.json.gz', 'rt'))
-    pdfs = {s['url']: base64.b64decode(s['base64']) for s in samples if s['kind'] == 'pdf'}
     items = []
     for sample in (s for s in samples if s['kind'] != 'pdf'):
         obj = USOfficialSpider(name=sample['source'])
@@ -53,12 +51,8 @@ async def public_api_items():
         if sample.get('excluded'):
             assert not output
             continue
-        while output and isinstance(output[0], Request):
-            req = output[0]
-            req.meta['raw_response_id'] = None
-            output = list(obj.parse_attachment(Response(req.url, body=pdfs[req.url], request=req)))
         assert len(output) == 1 and output[0]['extra']['content_scope'] == 'article'
-        assert output[0]['extra']['attachments']
+        assert all(a['status'] == 'skipped_policy' for a in output[0]['extra']['attachments'])
         items += output
     assert len(items) == 3
     return items
@@ -107,7 +101,7 @@ async def congress_api_items():
     assert len(report['cases']) == 4
     assert all(case.get('parser_accepted') for case in report['cases'])
     assert report['cases'][0]['unresolved_bills'] == 1
-    assert report['accepted'] is False  # Undated GENIUS text and an unreadable meeting PDF remain gaps.
+    assert report['accepted'] is False  # Undated GENIUS text remains a genuine gap.
     assert len(items) == sum(case['items'] for case in report['cases'])
     return items
 
@@ -239,7 +233,7 @@ async def main():
     await obj._finish_run(unsafe_run)
     async with SessionLocal() as session:
         assert (await session.get(Source, 'ofac')).history_cursor['checked'] == completed
-    # API/PDF packages use the real parser and database revision pipeline too.
+    # API text packages use the real parser and database revision pipeline too.
     api_items = await public_api_items() + await congress_reference_items() + await congress_api_items()
     with patch.object(pipelines, 'redis', SimpleNamespace(publish=AsyncMock())):
         for item in api_items:
@@ -258,7 +252,7 @@ async def main():
     async with SessionLocal() as session:
         assert await session.scalar(select(func.count()).select_from(NewsRevision)) == baseline_count + 1
     await engine.dispose()
-    print('US source migration, HTML, API/PDF, GovInfo and Congress packages, API/PDF replay and revision, negative screening, daily XML, RSS identity, challenge rejection, checkpoint recovery and failed-document persistence passed')
+    print('US source migration, HTML, API text, GovInfo and Congress packages, API text replay and revision, negative screening, daily XML, RSS identity, challenge rejection, checkpoint recovery and failed-document persistence passed')
 
 
 if __name__ == '__main__':

web/src/common/apis/crawler/type.ts

@@ -38,6 +38,8 @@ export interface SourceInfo {
 
 export interface CollectionRetry {
   retired_at?: string
+  skipped_policy?: string
+  previous_error?: string
   document: { url: string, canonical?: string }
   attempts: number
   next_at: number

web/src/pages/crawler/components/SourcePanel.vue

@@ -69,7 +69,7 @@ const history = computed(() => {
   const percent = Math.min(100, Math.max(0, Math.round(covered / span * 1000) / 10))
   const reached = formatTime(position, "YYYY-MM-DD HH:mm")
   if (done && position && new Date(position) <= new Date(target))
-    return { text: `列表遍历完成，已到 ${reached}；正文及附件缺口以条目状态为准`, percent: 100, status: "success" } as const
+    return { text: `列表遍历完成，已到 ${reached}；仅遍历支持的文字与图片；正文缺口及格式跳过见详情`, percent: 100, status: "success" } as const
   if (done)
     return { text: `来源最早只能翻到 ${reached}，更早的数据接口不提供`, percent, status: "warning" } as const
   return { text: position ? `进行中，已到 ${reached}` : "等待下一轮开始", percent, status: undefined }
@@ -153,6 +153,9 @@ onMounted(loadLatest)
     <div v-if="source.collection_stats?.active_gaps !== 0 && source.collection_stats?.document_gaps" class="text-xs text-amber-600">
       正文待核验 {{ source.collection_stats.document_gaps }}；待重试 {{ source.collection_stats.document_retry_pending }}；重试已耗尽 {{ source.collection_stats.document_retry_exhausted }}，请核对来源日志
     </div>
+    <p v-if="source.collection_stats?.policy_ignored" class="text-xs text-slate-500">
+      按文字与图片规则不再采集 {{ source.collection_stats.policy_ignored }} 项历史文件记录；原始原因保留。
+    </p>
     <p v-if="source.collection_stats?.retired_count" class="text-xs text-slate-500">
       已停止采集 {{ source.collection_stats.retired_count }} 条，原因及原文保留在详情中。
     </p>
@@ -161,7 +164,7 @@ onMounted(loadLatest)
         缺口详情（{{ gaps.length }}）
       </summary>
       <div v-for="gap in gaps" :key="gap.key" class="mt-2 border border-amber-200 rounded border-solid p-2">
-        <div>{{ gap.label }}；已尝试 {{ gap.attempts }} 次；{{ gap.retired ? "已停止采集" : gap.exhausted ? "重试已耗尽" : `下次重试不早于 ${formatTime(gap.nextAt)}` }}</div>
+        <div>{{ gap.label }}；已尝试 {{ gap.attempts }} 次；{{ gap.ignored ? "按规则不再采集" : gap.retired ? "已停止采集" : gap.exhausted ? "重试已耗尽" : `下次重试不早于 ${formatTime(gap.nextAt)}` }}</div>
         <div class="my-1 whitespace-pre-wrap break-words">
           {{ gap.error }}
         </div>
@@ -169,7 +172,7 @@ onMounted(loadLatest)
       </div>
     </details>
     <div v-if="source.collection_stats?.discovered" class="text-xs text-slate-500">
-      本轮发现 {{ source.collection_stats.discovered }}；筛除 {{ source.collection_stats.excluded ?? 0 }}；不完整包 {{ source.collection_stats.partial ?? 0 }}
+      本轮格式跳过 {{ source.collection_stats.policy_skipped ?? 0 }}；发现 {{ source.collection_stats.discovered }}；筛除 {{ source.collection_stats.excluded ?? 0 }}；不完整包 {{ source.collection_stats.partial ?? 0 }}
     </div>
     <div class="flex justify-between text-xs text-slate-500">
       <span>今日新增 <b class="num text-[var(--el-color-primary)]">+{{ source.today_new }}</b></span>

web/src/pages/crawler/sourceStatus.ts

@@ -47,8 +47,9 @@ export function collectionGaps(source: Pick<SourceInfo, "history_cursor">) {
     key: `${key}:${url}`,
     label,
     url: originalUrl(retry.document.canonical ?? retry.document.url),
-    error: retry.error || "历史记录未保存具体原因，请核对原文",
+    error: retry.previous_error || retry.error || "历史记录未保存具体原因，请核对原文",
     attempts: retry.attempts,
+    ignored: key === "attachment_retries" || !!retry.skipped_policy || /\.(?:pdf|docx?|xlsx?|pptx?|zip|gz|rar|7z|rtf|odt|ods|odp|exe|bin)(?:$|[?#/])/i.test(retry.document.url),
     retired: !!retry.retired_at,
     exhausted: retry.attempts >= 3,
     nextAt: new Date(retry.next_at * 1000).toISOString()

web/src/pages/news/components/NewsDetail.vue

@@ -112,11 +112,12 @@ const diffStats = computed(() => ({
         title="此条新闻仅采集公开摘要，完整内容请查看原文链接。"
         type="info" :closable="false" show-icon
       />
+      <el-alert v-if="detail.extra.collection_policy === 'web_text_images_v1'" title="仅采集网页文字与图片引用，未读取链接中的 PDF 或附件。" type="info" :closable="false" />
       <el-alert v-if="detail.extra.content_scope === 'partial'" title="正文或关键附件存在采集缺口，请核对官方原文。" type="warning" :closable="false" />
       <div v-if="Array.isArray(detail.extra.attachments)" class="flex flex-wrap gap-2 text-xs">
         <span v-for="attachment in (detail.extra.attachments as { url: string, title: string, status: string }[])" :key="attachment.url">
           <el-link :href="attachment.url" target="_blank" type="primary">{{ attachment.title || '附件' }}</el-link>
-          （{{ attachment.status === 'complete' ? '已采集' : '存在缺口' }}）
+          （{{ attachment.status === 'complete' ? '已采集' : attachment.status === 'skipped_policy' ? '按规则未采集' : '历史采集缺口' }}）
         </span>
       </div>
       <el-tabs v-model="activeTab" class="min-h-0 flex flex-1 flex-col">

web/tests/crawler.test.ts

@@ -62,6 +62,8 @@ test("gap details retain reasons, exhausted state and legacy records while colle
   assert.equal(gaps[0].nextAt, new Date(1793853098 * 1000).toISOString())
   assert.equal(gaps[1].url, "https://www.justice.gov/release")
   assert.match(gaps[1].error, /图片/)
+  assert.equal(gaps[1].ignored, true)
+  assert.equal(gaps[0].ignored, false)
   assert.equal(gaps[1].exhausted, false)
   assert.equal(gaps[2].exhausted, true)
   assert.equal(gaps[2].url, "https://www.congress.gov/bill/119th-congress/house-bill/1")