eezd
89c729092c
Collect official crypto and US policy documents with deterministic screening, full bodies and attachments, bounded retries, and durable history cursors. Preserve publication precision and source identity while keeping new collectors disabled. Add real Congress, GovInfo, DOJ and Federal Register fixtures, offline capture replay, isolated database validation, and rollout instructions. Validate 982 backend tests, frontend checks, and 57 deployment control tests.
180 lines
9.3 KiB
Python
180 lines
9.3 KiB
Python
"""Capture bounded, resumable Congress acceptance evidence without production I/O."""
|
||
import argparse
|
||
import base64
|
||
import gzip
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import time
|
||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||
|
||
import httpx
|
||
|
||
CREDENTIAL_FIELDS = {'apikey', 'api_key', 'token', 'key'}
|
||
|
||
|
||
def clean_url(url):
|
||
parts = urlsplit(url)
|
||
query = [(k, v) for k, v in parse_qsl(parts.query) if k.lower() not in CREDENTIAL_FIELDS]
|
||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||
|
||
|
||
def redact(value):
|
||
if isinstance(value, dict):
|
||
return {k: redact(v) for k, v in value.items() if k.lower() not in CREDENTIAL_FIELDS}
|
||
if isinstance(value, list):
|
||
return [redact(v) for v in value]
|
||
if isinstance(value, str) and value.startswith('https://'):
|
||
return clean_url(value)
|
||
return value
|
||
|
||
|
||
def save(path, data):
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
temporary = path.with_suffix(path.suffix + '.tmp')
|
||
with gzip.open(temporary, 'wt') as file:
|
||
json.dump(data, file)
|
||
temporary.replace(path)
|
||
|
||
|
||
class CaptureBlocked(RuntimeError):
|
||
pass
|
||
|
||
|
||
class Capture:
|
||
def __init__(self, path, key, limit=60, client=None):
|
||
self.path, self.key, self.limit = path, key, limit
|
||
self.client = client or httpx.Client(timeout=25, follow_redirects=False)
|
||
self.records = json.load(gzip.open(path, 'rt')) if path.exists() else {}
|
||
self.requests = 0
|
||
self.used_urls = set()
|
||
|
||
def fetch(self, url):
|
||
url = clean_url(url)
|
||
from alphapulse.core.us_news import official_url
|
||
official_url('congress', url)
|
||
self.used_urls.add(url)
|
||
if url in self.records:
|
||
return base64.b64decode(self.records[url]['body'])
|
||
if self.requests >= self.limit:
|
||
raise CaptureBlocked('本轮请求预算耗尽；成功响应已保存，下次可恢复')
|
||
self.requests += 1
|
||
headers = {'X-Api-Key': self.key} if urlsplit(url).hostname == 'api.congress.gov' else {}
|
||
response = self.client.get(url, headers=headers)
|
||
if response.status_code == 429:
|
||
raise CaptureBlocked(f"HTTP 429；Retry-After={response.headers.get('retry-after', 'unknown')}；已停止请求")
|
||
if response.status_code != 200:
|
||
raise CaptureBlocked(f'HTTP {response.status_code}；未保存错误响应')
|
||
body = response.content
|
||
if 'json' in response.headers.get('content-type', ''):
|
||
body = json.dumps(redact(response.json())).encode()
|
||
self.records[url] = {'body': base64.b64encode(body).decode()}
|
||
save(self.path, self.records)
|
||
time.sleep(1)
|
||
return body
|
||
|
||
|
||
async def run(capture, item_sink=None, meeting_url=None):
|
||
# Override every application I/O setting before importing the parser modules.
|
||
os.environ.update(DATABASE_URL='postgresql+asyncpg://test:[email protected]:1/test', REDIS_URL='redis://127.0.0.1:1/0', JWT_SECRET='test-only', ADMIN_USERNAME='test', SECRET_ENCRYPTION_KEY='MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=')
|
||
from datetime import UTC, datetime
|
||
from types import SimpleNamespace
|
||
from unittest.mock import patch
|
||
import uuid
|
||
from scrapy import Request
|
||
from scrapy.http import Response
|
||
from alphapulse.crawler.spiders import us_official
|
||
from alphapulse.crawler.spiders.base import RunState
|
||
from alphapulse.core.us_news import CRYPTO
|
||
report = {'accepted': False, 'cases': [], 'requests': 0}
|
||
roots = [('GENIUS', 'bill/119/s/1582'), ('CLARITY', 'bill/119/hr/3633'), ('STABLE', 'bill/119/hr/2392'), ('meeting', 'committee-meeting/119')]
|
||
with patch.object(us_official, 'get_settings', lambda: SimpleNamespace(congress_api_key=capture.key, crawler_contact_email='[email protected]')):
|
||
for label, route in roots:
|
||
obj = us_official.USOfficialSpider(name='congress')
|
||
now = datetime.now(UTC)
|
||
obj.run = RunState(uuid.uuid4(), 'schedule', now, now, history_target=datetime(2025,1,1,tzinfo=UTC), history_deadline=float('inf'))
|
||
url = f'https://api.congress.gov/v3/{route}?format=json&limit=250'
|
||
obj.cursor = {'history': {'entry': 0, 'pending': []}}
|
||
case = {'name': label, 'items': 0, 'article_items': 0, 'partial_items': 0, 'complete': False}
|
||
report['cases'].append(case)
|
||
try:
|
||
if label == 'meeting' and meeting_url:
|
||
from alphapulse.core.us_news import official_url
|
||
selected = official_url('congress', meeting_url)
|
||
if urlsplit(selected).hostname != 'api.congress.gov' or '/committee-meeting/119/' not in urlsplit(selected).path:
|
||
raise CaptureBlocked('会议地址必须是119届Congress API单场会议地址')
|
||
obj.cursor['history']['pending'] = [{'url': us_official.configure_url(selected, format='json'), 'kind': 'meeting'}]
|
||
elif label == 'meeting':
|
||
listing = json.loads(capture.fetch(url))
|
||
obj.cursor['history']['pending'] = [{'url': us_official.configure_url(entry['url'], format='json'), 'kind': 'meeting'} for entry in listing['committeeMeetings']]
|
||
if not obj.cursor['history']['pending']:
|
||
raise CaptureBlocked('会议列表为空，不能认定验收完成')
|
||
else:
|
||
obj.cursor['history']['pending'] = [{'url':url, 'kind':'bill'}]
|
||
request = None
|
||
while obj.cursor['history']['pending']:
|
||
if request is None:
|
||
request = obj.advance('history')[0]
|
||
body = capture.fetch(request.url)
|
||
if label == 'meeting' and request.callback == obj.parse_document:
|
||
meeting = json.loads(body)['committeeMeeting']
|
||
if not CRYPTO.search(meeting['title'] + json.dumps(meeting.get('relatedItems', {}))):
|
||
obj.cursor['history']['pending'].pop(0)
|
||
request = None
|
||
continue
|
||
request.meta['raw_response_id'] = None
|
||
response = Response(request.url, body=body, request=request)
|
||
if request.callback == obj.parse_attachment:
|
||
output = list(obj.parse_attachment(response))
|
||
else:
|
||
from scrapy.http import HtmlResponse
|
||
output = [v async for v in obj.parse_document(HtmlResponse(response.url, body=body, encoding='utf-8', request=request))]
|
||
requests = [v for v in output if isinstance(v, Request) and v.meta.get('document')]
|
||
# Attachment requests hold parsed state rather than a document.
|
||
requests += [v for v in output if isinstance(v, Request) and v.meta.get('parsed')]
|
||
for item in (v for v in output if not isinstance(v, Request)):
|
||
if item_sink is not None:
|
||
item_sink.append(dict(item))
|
||
case['items'] += 1
|
||
case['article_items'] += int(item['extra']['content_scope'] == 'article')
|
||
case['partial_items'] += int(item['extra']['content_scope'] == 'partial')
|
||
if label == 'meeting' and case['article_items']:
|
||
break
|
||
request = requests[0] if requests else None
|
||
if request is None and not obj.cursor['history']['pending']:
|
||
break
|
||
case['traversal_complete'] = not obj.cursor['history']['pending'] if label != 'meeting' else bool(case['article_items'] + case['partial_items'])
|
||
case['parser_accepted'] = case['traversal_complete'] and bool(case['article_items'] + case['partial_items'])
|
||
case['unresolved_bills'] = obj.run.stats.get('unresolved_bills', 0)
|
||
case['complete'] = bool(case['article_items']) and not case['unresolved_bills'] and not obj.cursor.get('attachment_retries')
|
||
except Exception as exc:
|
||
# Error text may contain a credential-bearing upstream URL.
|
||
case['error'] = str(exc) if isinstance(exc, CaptureBlocked) else type(exc).__name__
|
||
break
|
||
report['requests'] = capture.requests
|
||
report['accepted'] = len(report['cases']) == len(roots) and all(c['complete'] for c in report['cases'])
|
||
return report
|
||
|
||
|
||
def main():
|
||
import asyncio
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--cache', type=Path, required=True)
|
||
parser.add_argument('--meeting-url', help='Official Congress API URL for a known crypto hearing')
|
||
parser.add_argument('--max-requests', type=int, default=60)
|
||
args = parser.parse_args()
|
||
if not 1 <= args.max_requests <= 100:
|
||
parser.error('--max-requests must be between 1 and 100')
|
||
capture = Capture(args.cache, os.environ.get('CONGRESS_TEST_API_KEY') or 'DEMO_KEY', args.max_requests)
|
||
try:
|
||
report = asyncio.run(run(capture, meeting_url=args.meeting_url))
|
||
save(args.cache.with_suffix('.report.gz'), report)
|
||
print(json.dumps(report, ensure_ascii=False))
|
||
finally:
|
||
capture.client.close()
|
||
raise SystemExit(0 if report['accepted'] else 1)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|