Add resumable official news and calendar backfill to January 2025
This commit is contained in:
eezd committed
1 parent
6035ed8332
commit
93afe4358b
7 files changed
+454
-9
No files matched your search
@@ -0,0 +1,200 @@
|
||||
"""可恢复历史摘要导出；输出真实响应供既有 import_history 导入，不写业务表。"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from parsel import Selector
|
||||
from sqlalchemy import select
|
||||
|
||||
from alphapulse.calendar.backfill import fetch_documents, family_for
|
||||
from alphapulse.calendar.archives import parse_announcements, publication_time
|
||||
from alphapulse.crawler.europe_history import capture as european_capture
|
||||
from alphapulse.crawler.import_history import ArchivedArticle, Capture
|
||||
from alphapulse.core.db import SessionLocal, engine
|
||||
from alphapulse.core.official_sources import FEEDS, relevant_news
|
||||
from alphapulse.models import RawBody, RawResponse
|
||||
|
||||
SOURCES = ('coindesk', 'bls', 'bea', *FEEDS)
|
||||
|
||||
|
||||
def timeline_articles(data):
|
||||
if not isinstance(data.get('articles'), list) or not isinstance(data.get('hasMore'), bool):
|
||||
raise ValueError('CoinDesk 时间线结构未识别')
|
||||
return [ArchivedArticle(guid=a['_id'], url=urljoin('https://www.coindesk.com', a['pathname']),
|
||||
title=a['title'], summary=a.get('description') or '',
|
||||
published_at=a['articleDates']['displayDate']) for a in data['articles']]
|
||||
|
||||
|
||||
def ecb_articles(data):
|
||||
if not isinstance(data, list) or len(data) % 13:
|
||||
raise ValueError('ECB 官方索引记录宽度已变化')
|
||||
articles = []
|
||||
for i in range(0, len(data), 13):
|
||||
row = data[i:i + 13]
|
||||
at = datetime.fromtimestamp(row[1], UTC)
|
||||
for path in row[9]:
|
||||
url = urljoin('https://www.ecb.europa.eu', path)
|
||||
title = row[10]['Title']
|
||||
if relevant_news('ecb', title, url) and not url.endswith('.pdf'):
|
||||
articles.append(ArchivedArticle(guid=url, url=url, title=title, summary='', published_at=at))
|
||||
return articles
|
||||
|
||||
|
||||
def package(source, url, body, at, headers, articles):
|
||||
return Capture(source=source, url=url, status=200, headers=headers,
|
||||
body=base64.b64encode(body).decode(), body_encoding='base64',
|
||||
fetched_at=at, articles=articles)
|
||||
|
||||
|
||||
async def export(source, begin, end, output):
|
||||
errors, captures = [], []
|
||||
existing = []
|
||||
async with SessionLocal() as session:
|
||||
rows = (await session.execute(select(RawResponse, RawBody.body).join(
|
||||
RawBody, RawBody.sha256 == RawResponse.body_sha256).where(
|
||||
RawResponse.source_code == source, RawResponse.status == 200,
|
||||
RawResponse.url.contains('/chunk_') if source == 'ecb' else
|
||||
RawResponse.url.contains('/api/v1/articles/timeline'))
|
||||
.distinct(RawResponse.url, RawResponse.body_sha256)
|
||||
.order_by(RawResponse.url, RawResponse.body_sha256, RawResponse.id))).all() if source in ('ecb', 'coindesk') else []
|
||||
for raw, body in rows:
|
||||
body = gzip.decompress(body)
|
||||
try:
|
||||
articles = ecb_articles(json.loads(body)) if source == 'ecb' else timeline_articles(json.loads(body))
|
||||
captures.append(package(source, raw.url, body, raw.fetched_at, raw.headers, articles))
|
||||
existing.append(raw.url)
|
||||
except (ValueError, KeyError, TypeError, IndexError) as exc:
|
||||
errors.append({'url': raw.url, 'error': str(exc)})
|
||||
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as client:
|
||||
async def get(url, host):
|
||||
await asyncio.sleep(1)
|
||||
for _ in range(6):
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme != 'https' or parsed.hostname != host:
|
||||
raise ValueError('归档链接或重定向不属于预期官方域名')
|
||||
r = await client.get(url, headers={'User-Agent': 'AlphaPulse historical news collector'})
|
||||
if not r.is_redirect:
|
||||
break
|
||||
url = urljoin(url, r.headers['location'])
|
||||
else:
|
||||
raise ValueError('归档重定向超过五次')
|
||||
r.raise_for_status()
|
||||
return r
|
||||
try:
|
||||
if source == 'coindesk':
|
||||
url, visited = 'https://www.coindesk.com/api/v1/articles/timeline?size=16&lang=en', set()
|
||||
while url:
|
||||
if url in visited:
|
||||
raise ValueError('CoinDesk 历史分页形成循环')
|
||||
visited.add(url)
|
||||
r = await get(url, 'www.coindesk.com'); data = r.json()
|
||||
articles = timeline_articles(data)
|
||||
captures.append(package(source, str(r.url), r.content, datetime.now(UTC), dict(r.headers), articles))
|
||||
if not articles:
|
||||
if data['hasMore']:
|
||||
raise ValueError('CoinDesk 空页仍声明有后续数据')
|
||||
break
|
||||
dates = [a.published_at for a in articles]
|
||||
if dates != sorted(dates, reverse=True):
|
||||
raise ValueError('CoinDesk 时间线日期非倒序')
|
||||
if min(dates) < begin or not data['hasMore']:
|
||||
break
|
||||
a = data['articles'][-1]
|
||||
url = str(httpx.URL('https://www.coindesk.com/api/v1/articles/timeline', params={
|
||||
'size': 16, 'lang': 'en', 'lastDisplayDate': a['articleDates']['displayDate'], 'lastId': a['_id']}))
|
||||
elif source == 'bea':
|
||||
url, visited = 'https://www.bea.gov/news/archive?field_related_product_target_id=All&created_1=All&title=', set()
|
||||
while url:
|
||||
if url in visited:
|
||||
raise ValueError('BEA 归档分页形成循环')
|
||||
visited.add(url)
|
||||
r = await get(url, 'www.bea.gov'); sel = Selector(text=r.text)
|
||||
links = list(dict.fromkeys(urljoin(str(r.url), u) for u in sel.css('a[href]::attr(href)').getall()
|
||||
if any('/news/' + str(y) + '/' in u for y in range(begin.year, end.year + 1))))
|
||||
for link in links:
|
||||
page = await get(link, 'www.bea.gov'); article = Selector(text=page.text)
|
||||
stamp = article.css('.field--name-field-release-date').xpath('string(.)').get()
|
||||
if not stamp:
|
||||
raise ValueError('BEA 公告缺少正式发布日期')
|
||||
a = ArchivedArticle(guid=link, url=link, title=article.css('h1').xpath('string(.)').get(),
|
||||
summary=article.css('meta[name="description"]::attr(content)').get() or '', published_at=publication_time(stamp))
|
||||
if begin <= a.published_at < end:
|
||||
captures.append(package(source, link, page.content, datetime.now(UTC), dict(page.headers), [a]))
|
||||
following = sel.css('.pager__item--next a::attr(href)').get()
|
||||
years = re.findall(r'/news/(20\d\d)/', r.text)
|
||||
url = urljoin(str(r.url), following) if following and (not years or max(map(int, years)) >= begin.year) else None
|
||||
elif source == 'ecb':
|
||||
if not existing:
|
||||
raise ValueError('ECB 缺少已验证的官方索引入口')
|
||||
for url in dict.fromkeys(existing):
|
||||
r = await get(url, 'www.ecb.europa.eu')
|
||||
captures.append(package(source, str(r.url), r.content, datetime.now(UTC), dict(r.headers), ecb_articles(r.json())))
|
||||
errors.append({'error': 'ECB 已存索引已重验；动态索引版本至任务终点的完整性仍未证实'})
|
||||
elif source == 'boe':
|
||||
r = await get('https://www.bankofengland.co.uk/sitemap/speeches', 'www.bankofengland.co.uk')
|
||||
links = list(dict.fromkeys(urljoin(str(r.url), p) for p in Selector(text=r.text).css('a[href]::attr(href)').getall() if '/speech/' in p))
|
||||
if not links:
|
||||
raise ValueError('BoE 讲话索引没有可识别的链接')
|
||||
for url in links:
|
||||
page = await get(url, 'www.bankofengland.co.uk');sel = Selector(text=page.text)
|
||||
dates = sel.css('time::attr(datetime), meta[property="article:published_time"]::attr(content)').getall()
|
||||
if not dates:
|
||||
raise ValueError('BoE 讲话缺少可识别的正式日期')
|
||||
a = ArchivedArticle(guid=url, url=url, title=sel.css('h1').xpath('string(.)').get(),
|
||||
summary=sel.css('.page-content p').xpath('string(.)').get() or '', published_at=dates[0])
|
||||
if begin <= a.published_at < end:
|
||||
captures.append(package(source, url, page.content, datetime.now(UTC), dict(page.headers), [a]))
|
||||
except (httpx.HTTPError, ValueError, KeyError, TypeError, IndexError) as exc:
|
||||
item = {'error': f'{type(exc).__name__}: {exc}'}
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
item['retry_after'] = exc.response.headers.get('Retry-After')
|
||||
errors.append(item)
|
||||
if source != 'coindesk' and not (source in ('boe', 'ecb') and any('HTTPStatusError' in e.get('error', '') for e in errors)):
|
||||
documents = await fetch_documents(begin.date(), end.date(), set(), errors, sources={source})
|
||||
for doc in documents:
|
||||
try:
|
||||
if source in FEEDS:
|
||||
captures.append(european_capture(doc))
|
||||
else:
|
||||
family = family_for(doc)
|
||||
if family:
|
||||
announcements = parse_announcements(family, doc.body, pdf=doc.body.startswith(b'%PDF'))
|
||||
title = Selector(text=doc.body.decode(errors='replace')).css('h1').xpath('string(.)').get() or family
|
||||
articles = [ArchivedArticle(guid=doc.url, url=doc.url, title=title, summary='', published_at=a.at) for a in announcements]
|
||||
captures.append(package(source, doc.url, doc.body, doc.captured_at, doc.headers, articles))
|
||||
except (ValueError, KeyError, TypeError, IndexError) as exc:
|
||||
errors.append({'url': doc.url, 'error': str(exc)})
|
||||
with gzip.open(output, 'xt', encoding='utf-8') as stream:
|
||||
for capture in captures:
|
||||
capture.articles = [a for a in capture.articles if begin <= a.published_at < end]
|
||||
stream.write(capture.model_dump_json() + '\n')
|
||||
return {'source': source, 'responses': len(captures), 'articles': sum(len(c.articles) for c in captures),
|
||||
'errors': errors, 'output': str(output)}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('source', choices=SOURCES)
|
||||
parser.add_argument('--start', required=True, type=datetime.fromisoformat)
|
||||
parser.add_argument('--end', required=True, type=datetime.fromisoformat)
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.start.tzinfo is None or args.end.tzinfo is None or args.start >= args.end:
|
||||
parser.error('范围须带时区且开始早于结束')
|
||||
async def run():
|
||||
try:
|
||||
return await export(args.source, args.start, args.end, args.output)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
print(json.dumps(asyncio.run(run()), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user