592 lines
35 KiB
Python
592 lines
35 KiB
Python
"""独立日历采集进程：日程每小时刷新，公布窗口按 5/15/60 秒检查官方结果。"""
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import ssl
|
|
from datetime import UTC, datetime, timedelta
|
|
from email.utils import parsedate_to_datetime
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import httpx
|
|
from parsel import Selector
|
|
from sqlalchemy import select
|
|
|
|
from alphapulse.calendar.catalog import SCHEDULE_URLS, UMICH_CSV, UMICH_HISTORY, UMICH_HOME, UMICH_INFO, UMICH_REPORTS
|
|
from alphapulse.calendar import adp, dol, eia, europe
|
|
from alphapulse.calendar.archives import ArchiveMissing, parse_umich_report
|
|
from alphapulse.calendar.parsers import EASTERN, MONTHS, bea_release_link, parse_fed_calendar, parse_ics, parse_values, release_date, bls_periods, release_metadata
|
|
from alphapulse.calendar.service import apply_bls_periods, record_revision, next_poll, save_raw, save_result, save_schedule, save_series
|
|
from alphapulse.calendar.umich import current_result, document_links, monthly_series, parse_schedule, pdf_text, report_links
|
|
from alphapulse.core.db import SessionLocal
|
|
from alphapulse.core.official_sources import FEEDS, HOSTS, feed_entries
|
|
from alphapulse.core.redis import KEY_PREFIX, redis
|
|
from alphapulse.models import EconomicRelease, Source
|
|
|
|
logger = logging.getLogger("alphapulse.calendar")
|
|
STATUS_KEY = f"{KEY_PREFIX}calendar:status"
|
|
HISTORY_DAYS = 45
|
|
BOUNDED_SOURCES = frozenset({"dol", "eia", "umich", "adp"})
|
|
|
|
|
|
def retry_after_at(value: str | None, now: datetime) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
value = value.strip()
|
|
if value.isdecimal():
|
|
return now + timedelta(seconds=int(value))
|
|
at = parsedate_to_datetime(value)
|
|
if at.tzinfo is None:
|
|
raise ValueError("Retry-After 日期缺少时区")
|
|
return at.astimezone(UTC)
|
|
except (ValueError, OverflowError, TypeError) as exc:
|
|
logger.warning("官方 Retry-After 无法解析，沿用既有轮询间隔：%s", exc)
|
|
return None
|
|
|
|
|
|
class SourceCooling(ValueError):
|
|
def __init__(self, host, retry_at):
|
|
self.retry_at = retry_at
|
|
super().__init__(f'{host} 请求冷却，下次尝试 {retry_at.isoformat()}')
|
|
|
|
|
|
def fed_month_urls(now: datetime) -> list[str]:
|
|
month = now.astimezone(EASTERN).replace(day=1)
|
|
following = (month + timedelta(days=32)).replace(day=1)
|
|
names = {v: k.lower() for k, v in MONTHS.items()}
|
|
return [f"https://www.federalreserve.gov/newsevents/{d.year}-{names[d.month]}.htm" for d in (month, following)]
|
|
|
|
|
|
class CalendarWorker:
|
|
def __init__(self) -> None:
|
|
# 沿用已验证的 BLS TLS 设置；其余来源保持默认配置。
|
|
ctx = ssl.create_default_context()
|
|
ctx.set_ciphers("DEFAULT")
|
|
self.http = httpx.AsyncClient(timeout=8, follow_redirects=True, headers={"User-Agent": "AlphaPulse economic calendar"})
|
|
self.bls = httpx.AsyncClient(timeout=8, follow_redirects=True, verify=ctx, headers={"User-Agent": "AlphaPulse economic calendar"})
|
|
self.sources: dict[str, dict] = {}
|
|
self.eia_schedule_body: str | None = None
|
|
self.fetch_slots = asyncio.Semaphore(2)
|
|
self.host_locks = {}
|
|
self.host_ready = {}
|
|
self.cooldowns = {}
|
|
self.failures = {}
|
|
self.index_cache = {}
|
|
self.poll_cache = None
|
|
|
|
async def fetch(self, source: str, url: str, *, query: dict | None = None) -> tuple[httpx.Response, int]:
|
|
key = (source, url, json.dumps(query, sort_keys=True))
|
|
if self.poll_cache is None:
|
|
return await self._fetch(source, url, query=query)
|
|
if key not in self.poll_cache:
|
|
self.poll_cache[key] = asyncio.create_task(self._fetch(source, url, query=query))
|
|
return await self.poll_cache[key]
|
|
|
|
async def index(self, source, url):
|
|
# Live ADP discovery must follow result polling rather than archive cache TTL.
|
|
if source == 'adp':
|
|
page = await self.fetch(source, url)
|
|
page[0].raise_for_status()
|
|
return page
|
|
cached = self.index_cache.get((source, url))
|
|
if cached and cached[0] > datetime.now(UTC):
|
|
return cached[1]
|
|
page = await self.fetch(source, url)
|
|
page[0].raise_for_status()
|
|
self.index_cache[source, url] = (datetime.now(UTC) + timedelta(hours=1), page)
|
|
return page
|
|
|
|
async def adp_links(self, start, end, first_page=None):
|
|
url, visited = adp.INDEX, set()
|
|
for _ in range(8):
|
|
if url in visited: raise ValueError('ADP 官方分页形成循环')
|
|
visited.add(url)
|
|
response, _ = first_page if first_page is not None else await self.index('adp', url)
|
|
first_page = None
|
|
links, following = adp.release_links(response.text, url, start, end)
|
|
if links or not following: return links
|
|
url = following
|
|
raise ValueError('ADP 在线索引已扫描8页，目标月报尚未取得')
|
|
|
|
async def archive_page(self, row):
|
|
day = row.scheduled_at.astimezone(EASTERN).date()
|
|
source = row.source
|
|
module = dol if source == 'dol' else eia
|
|
index_url = dol.SCHEDULE if source == 'dol' else eia.ARCHIVE
|
|
response, _ = await self.index(source, index_url)
|
|
try:
|
|
links = module.archive_links(response.text, day, day)
|
|
except ArchiveMissing:
|
|
if source != 'dol':
|
|
raise
|
|
# The index can lag behind published PDFs; validate the PDF date below.
|
|
links = [f'https://oui.doleta.gov/press/{day:%Y}/{day:%m%d%y}.pdf']
|
|
if len(links) != 1: raise ValueError(f'{source} 当期官方归档不唯一')
|
|
url = links[0]
|
|
if source == 'eia':
|
|
page, _ = await self.fetch(source, url)
|
|
page.raise_for_status()
|
|
url = eia.csv_link(page.text, url)
|
|
response, raw_id = await self.fetch(source, url)
|
|
response.raise_for_status()
|
|
response.extensions['calendar_url'] = url
|
|
return response, raw_id
|
|
|
|
async def _fetch(self, source: str, url: str, *, query: dict | None = None) -> tuple[httpx.Response, int]:
|
|
# 链接来自公开响应，限制到对应官方域名，避免外部内容扩大访问范围。
|
|
allowed = {"bls": {"www.bls.gov"}, "bea": {"www.bea.gov", "apps.bea.gov"}, "fed": {"www.federalreserve.gov"},
|
|
"umich": {"www.sca.isr.umich.edu", "data.sca.isr.umich.edu"}, "dol": {"www.dol.gov", "oui.doleta.gov"},
|
|
"eia": {"www.eia.gov", "ir.eia.gov"}, "adp": {"mediacenter.adp.com"}, **HOSTS}
|
|
if urlparse(url).scheme != "https" or urlparse(url).hostname not in allowed[source]:
|
|
raise ValueError("公告链接不是对应机构的官方 HTTPS 地址")
|
|
host = urlparse(url).hostname
|
|
async with self.host_locks.setdefault(host, asyncio.Lock()):
|
|
now = datetime.now(UTC)
|
|
if self.cooldowns.get(host, now) > now:
|
|
raise SourceCooling(host, self.cooldowns[host])
|
|
delay = (self.host_ready.get(host, now) - now).total_seconds()
|
|
if delay > 0: await asyncio.sleep(delay)
|
|
async with self.fetch_slots:
|
|
client = self.bls if source == "bls" else self.http
|
|
response = await client.post(url, json=query) if query is not None else await client.get(url, **({"timeout": httpx.Timeout(20, connect=8)} if source == "adp" else {}))
|
|
now = datetime.now(UTC)
|
|
self.host_ready[host] = now + timedelta(seconds=1)
|
|
if response.status_code in (403, 429, 503):
|
|
self.failures[host] = min(self.failures.get(host, 0) + 1, 7)
|
|
retry = retry_after_at(response.headers.get('Retry-After'), now)
|
|
seconds = 3600 if response.status_code == 403 else min(3600, 60 * 2 ** (self.failures[host] - 1))
|
|
self.cooldowns[host] = max(retry or now, now + timedelta(seconds=seconds))
|
|
elif response.is_success:
|
|
self.failures.pop(host, None)
|
|
if urlparse(str(response.url)).hostname not in allowed[source] or response.url.scheme != "https":
|
|
raise ValueError("官方响应重定向到了不受支持的地址")
|
|
now = datetime.now(UTC)
|
|
async with SessionLocal() as session:
|
|
raw_id = await save_raw(session, source, str(response.url), response.status_code,
|
|
{k: v for k, v in response.headers.items() if k in ("content-type", "date", "last-modified", "etag", "retry-after")},
|
|
response.content, now, method=response.request.method)
|
|
await session.commit()
|
|
return response, raw_id
|
|
|
|
async def schedules(self) -> None:
|
|
now = datetime.now(UTC)
|
|
urls = [(s, u) for s, u in SCHEDULE_URLS.items() if s != "fed"] + [("fed", u) for u in fed_month_urls(now)]
|
|
urls += [("bls", f"https://www.bls.gov/schedule/news_release/{code}.htm") for code in ("cpi", "ppi", "empsit")]
|
|
urls += [("umich", UMICH_INFO), ("umich", UMICH_CSV)]
|
|
urls += list(europe.SCHEDULES.items())
|
|
urls += [("dol", dol.SCHEDULE)]
|
|
urls += [("eia", eia.SCHEDULE)]
|
|
urls += [("adp", adp.INDEX)]
|
|
async with SessionLocal() as session:
|
|
controls = list(await session.scalars(select(Source).where(Source.code.in_(BOUNDED_SOURCES))))
|
|
disabled = {source.code for source in controls if not source.enabled}
|
|
paused = {source.code for source in controls if source.paused_until and source.paused_until > now}
|
|
urls = [(s, u) for s, u in urls if s not in disabled]
|
|
checked = {}
|
|
# 启动时明确各来源尚未就绪；月份滚动后移除不再检查的旧月状态。
|
|
self.sources = {url: self.sources.get(url, {"source": source, "checked_at": now.isoformat(), "error": "尚未同步"})
|
|
for source, url in urls}
|
|
for source, url in urls:
|
|
if source in paused:
|
|
continue
|
|
retry_at = self.sources[url].get("retry_at")
|
|
if retry_at and datetime.fromisoformat(retry_at) > datetime.now(UTC):
|
|
continue
|
|
try:
|
|
fetch_url = europe.eurostat_calendar_url(now) if source == "eurostat" else url
|
|
if source == "adp":
|
|
response, raw_id = await self.index(source, fetch_url)
|
|
else:
|
|
response, raw_id = await self.fetch(source, fetch_url, query=europe.INSEE_QUERY if source == "insee" else None)
|
|
response.raise_for_status()
|
|
if source == "adp":
|
|
links = await self.adp_links(datetime.min.date(), now.astimezone(EASTERN).date(), (response, raw_id))
|
|
if not links:
|
|
raise ValueError("ADP 官方新闻索引缺少近期月报")
|
|
report, evidence = await self.fetch(source, links[0])
|
|
report.raise_for_status()
|
|
events = adp.schedule(report.text, str(report.url))
|
|
async with SessionLocal() as session:
|
|
await save_schedule(session, source, url, events, evidence, datetime.now(UTC), withdraw_missing=False)
|
|
await session.commit()
|
|
elif source in ("dol", "eia"):
|
|
events = (dol if source == "dol" else eia).schedule(response.text, now)
|
|
async with SessionLocal() as session:
|
|
await save_schedule(session, source, url, events, raw_id, datetime.now(UTC), withdraw_missing=False)
|
|
await session.commit()
|
|
if source == "eia":
|
|
self.eia_schedule_body = response.text
|
|
elif source in europe.SCHEDULES:
|
|
events = europe.parse_schedule(source, response.text)
|
|
async with SessionLocal() as session:
|
|
# 欧洲公开日程是滚动/分页窗口，缺失不等于撤回。
|
|
await save_schedule(session, source, url, events, raw_id, datetime.now(UTC), withdraw_missing=False)
|
|
await session.commit()
|
|
elif url == UMICH_INFO:
|
|
await self.umich_schedule(response.text, now)
|
|
elif url == UMICH_CSV:
|
|
points = [(p, v) for p, v in monthly_series(response.text) if p >= "2024-12" and p <= now.strftime("%Y-%m")]
|
|
async with SessionLocal() as session:
|
|
await save_series(session, "umich", "umich_sentiment", points, "final", raw_id, datetime.now(UTC))
|
|
await session.commit()
|
|
elif source == "bls" and url.endswith(".htm"):
|
|
code = url.rsplit("/", 1)[1].split(".")[0]
|
|
async with SessionLocal() as session:
|
|
await apply_bls_periods(session, "jobs" if code == "empsit" else code, bls_periods(response.text), raw_id)
|
|
await session.commit()
|
|
else:
|
|
events = parse_fed_calendar(response.text) if source == "fed" else parse_ics(response.text, source)
|
|
async with SessionLocal() as session:
|
|
await save_schedule(session, source, url, events, raw_id, datetime.now(UTC))
|
|
await session.commit()
|
|
checked.setdefault(source, []).append(None)
|
|
self.sources[url] = {"source": source, "checked_at": datetime.now(UTC).isoformat(), "error": None}
|
|
except (httpx.HTTPError, ValueError, KeyError, TypeError, IndexError, StopIteration) as exc:
|
|
if not isinstance(exc, SourceCooling):
|
|
checked.setdefault(source, []).append(str(exc))
|
|
state = {"source": source, "checked_at": datetime.now(UTC).isoformat(), "error": str(exc)}
|
|
retry_at = getattr(exc, 'retry_at', None)
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
retry_at = self.cooldowns.get(exc.request.url.host)
|
|
if retry_at is None and exc.response.status_code in (429, 503):
|
|
retry_at = retry_after_at(exc.response.headers.get('Retry-After'), datetime.now(UTC))
|
|
if retry_at is not None:
|
|
state['retry_at'] = retry_at.isoformat()
|
|
state['error'] += f"；下次尝试 {retry_at.isoformat()}"
|
|
self.sources[url] = state
|
|
logger.exception("日程采集失败：%s", url)
|
|
|
|
async with SessionLocal() as session:
|
|
for code, outcomes in checked.items():
|
|
if code not in BOUNDED_SOURCES:
|
|
continue
|
|
source = await session.get(Source, code, with_for_update=True)
|
|
if source is None:
|
|
continue
|
|
if None in outcomes:
|
|
source.consecutive_failures = 0
|
|
source.last_success_at = datetime.now(UTC)
|
|
source.last_error = None
|
|
source.paused_until = None
|
|
else:
|
|
source.consecutive_failures += 1
|
|
source.last_error = outcomes[0]
|
|
deadlines = [datetime.fromisoformat(state['retry_at']) for state in self.sources.values()
|
|
if state['source'] == code and state.get('retry_at')]
|
|
source.paused_until = max([datetime.now(UTC) + timedelta(hours=1), *deadlines])
|
|
if source.consecutive_failures >= 3:
|
|
source.enabled = False
|
|
await session.commit()
|
|
|
|
async def umich_schedule(self, info: str, now: datetime) -> None:
|
|
history, _ = await self.fetch("umich", UMICH_HISTORY)
|
|
history.raise_for_status()
|
|
links = document_links(history.text, UMICH_HISTORY, {"Historical Prelim/Final Release Dates"})
|
|
if "Historical Prelim/Final Release Dates" not in links:
|
|
raise ValueError("密歇根官方资料缺少历史发布日期 PDF")
|
|
documents = list(links.values())
|
|
yearly = document_links(info, UMICH_INFO, {f"{year} Release Dates" for year in (now.year, now.year + 1)})
|
|
if f"{now.year} Release Dates" not in yearly:
|
|
raise ValueError("密歇根官方资料缺少当年日程 PDF")
|
|
documents.extend(yearly.values())
|
|
events = {}
|
|
raw_id = None
|
|
for url in documents:
|
|
response, raw_id = await self.fetch("umich", url)
|
|
response.raise_for_status()
|
|
parsed = await asyncio.to_thread(parse_schedule, await asyncio.to_thread(pdf_text, response.content))
|
|
# 年度日程在历史表之后处理，以官方最新年度安排为准；同一稳定身份不创建重复事件。
|
|
for event in parsed:
|
|
events[event.external_id] = (event, raw_id)
|
|
async with SessionLocal() as session:
|
|
# 不按单份 PDF 撤回其他年度事件；合并日程一次同步，修订引用对应 PDF。
|
|
existing = {r.external_id: r for r in await session.scalars(select(EconomicRelease).where(
|
|
EconomicRelease.source == "umich"))}
|
|
batches = {}
|
|
for event, evidence in events.values():
|
|
if event.at < datetime(2024, 12, 31, 16, tzinfo=UTC):
|
|
continue
|
|
# 每份 PDF 是部分日程，不能让 save_schedule 的撤回逻辑作用于其他事件。
|
|
batches.setdefault(evidence, []).append(event)
|
|
for evidence, batch in batches.items():
|
|
await save_schedule(session, "umich", UMICH_INFO, batch, evidence, now, withdraw_missing=False)
|
|
if raw_id is not None:
|
|
for row in existing.values():
|
|
if row.external_id not in events and row.scheduled_at > now and row.first_received_at is None and row.schedule_status != "withdrawn":
|
|
row.schedule_status = "withdrawn"
|
|
row.revision_no += 1
|
|
record_revision(session, row, raw_id)
|
|
await session.commit()
|
|
|
|
async def result_page(self, row: EconomicRelease) -> tuple[httpx.Response, int] | None:
|
|
if row.source == "adp":
|
|
if row.release_url:
|
|
url = row.release_url
|
|
else:
|
|
links = await self.adp_links(row.scheduled_at.astimezone(EASTERN).date(), row.scheduled_at.astimezone(EASTERN).date())
|
|
if not links:
|
|
return None
|
|
if len(links) != 1:
|
|
raise ValueError("ADP 同日月度公告不唯一")
|
|
url = links[0]
|
|
response, raw_id = await self.fetch("adp", url)
|
|
response.raise_for_status()
|
|
announcement = adp.report(response.text)
|
|
if announcement.at != row.scheduled_at or announcement.period != row.period:
|
|
raise ValueError("ADP 公告时间或统计月与所选日程不符")
|
|
return response, raw_id
|
|
if row.source == "eia":
|
|
if self.eia_schedule_body is None:
|
|
schedule, _ = await self.fetch("eia", eia.SCHEDULE)
|
|
schedule.raise_for_status()
|
|
eia.exceptions(schedule.text)
|
|
self.eia_schedule_body = schedule.text
|
|
fixed_url = row.release_url if row.release_url and "/archive/" in urlparse(row.release_url).path else None
|
|
if row.scheduled_at.astimezone(EASTERN).date() < datetime.now(EASTERN).date() and not fixed_url:
|
|
try:
|
|
response, raw_id = await self.archive_page(row)
|
|
except ArchiveMissing:
|
|
response, raw_id = await self.fetch("eia", eia.LATEST)
|
|
response.extensions["calendar_url"] = eia.LATEST
|
|
else:
|
|
response, raw_id = await self.fetch("eia", fixed_url or eia.LATEST)
|
|
response.extensions["calendar_url"] = fixed_url or eia.LATEST
|
|
response.raise_for_status()
|
|
announcement = eia.report(response.text, self.eia_schedule_body)
|
|
if announcement.at < row.scheduled_at or (announcement.at > row.scheduled_at and row.first_received_at is not None):
|
|
return None
|
|
if announcement.at != row.scheduled_at or announcement.period != row.period:
|
|
raise ValueError("EIA 公告统计周或时间与所选日程不符，未填入其他期次的数据")
|
|
return response, raw_id
|
|
if row.source == "dol":
|
|
if row.scheduled_at.astimezone(EASTERN).date() < datetime.now(EASTERN).date() and (not row.release_url or "/press/" not in urlparse(row.release_url).path):
|
|
try:
|
|
response, raw_id = await self.archive_page(row)
|
|
except ArchiveMissing:
|
|
response, raw_id = await self.fetch("dol", dol.LATEST)
|
|
else:
|
|
try:
|
|
response, raw_id = await self.fetch("dol", row.release_url or dol.LATEST)
|
|
response.raise_for_status()
|
|
except (SourceCooling, httpx.HTTPStatusError) as error:
|
|
if isinstance(error, httpx.HTTPStatusError) and error.response.status_code != 403: raise
|
|
response, raw_id = await self.archive_page(row)
|
|
response.raise_for_status()
|
|
announcements = await asyncio.to_thread(dol.report, await asyncio.to_thread(pdf_text, response.content))
|
|
announcement = announcements[row.family]
|
|
if announcement.at < row.scheduled_at or (announcement.at > row.scheduled_at and row.first_received_at is not None):
|
|
return None
|
|
if announcement.at != row.scheduled_at:
|
|
raise ValueError("DOL 公告时间与所选日程不符，未填入其他期次的数据")
|
|
return response, raw_id
|
|
if row.source in europe.SCHEDULES:
|
|
return await self.european_result(row)
|
|
date = row.scheduled_at.astimezone(EASTERN).date()
|
|
if row.source == "umich":
|
|
response, raw_id = await self.fetch("umich", UMICH_HOME)
|
|
response.raise_for_status()
|
|
period, stage, _ = current_result(response.text)
|
|
if period == row.period and stage == row.stage:
|
|
return response, raw_id
|
|
if period < row.period or (period == row.period and stage == "preliminary" and row.stage == "final"):
|
|
return None
|
|
if row.first_received_at is not None:
|
|
return None # 最新页已换期；保留已取得的当期值，不误报历史缺口。
|
|
index, _ = await self.index('umich', f'{UMICH_REPORTS}?year={row.period[:4]}')
|
|
url = report_links(index.text, int(row.period[:4])).get((row.period, row.stage))
|
|
if url is None:
|
|
raise ArchiveMissing("密歇根官方年度归档尚未列出所选期次")
|
|
response, raw_id = await self.fetch('umich', url)
|
|
response.raise_for_status()
|
|
announcement = parse_umich_report(await asyncio.to_thread(pdf_text, response.content))
|
|
if (announcement.at, announcement.period, announcement.stage) != (row.scheduled_at, row.period, row.stage):
|
|
raise ValueError("密歇根归档日期、月份或阶段与所选期次不符")
|
|
response.extensions['calendar_announcement'] = announcement
|
|
return response, raw_id
|
|
if row.source == "bls":
|
|
code = "empsit" if row.family == "jobs" else row.family
|
|
# 本机 BLS archive 路径返回 403，最新公告页可访问；下面必须核对正式发布日期。
|
|
url = f"https://www.bls.gov/news.release/{code}.nr0.htm"
|
|
elif row.source == "fed":
|
|
url = f"https://www.federalreserve.gov/newsevents/pressreleases/monetary{date:%Y%m%d}a.htm"
|
|
else:
|
|
# 现场核验 BEA RSS 漏了已发布的 PCE；使用官方日程的公告链接，不依赖 RSS 推送时效。
|
|
url = row.release_url
|
|
if url is None:
|
|
schedule, _ = await self.fetch("bea", "https://www.bea.gov/news/schedule/full")
|
|
schedule.raise_for_status()
|
|
url = bea_release_link(schedule.text, date, row.family)
|
|
if url is None:
|
|
return None
|
|
response, raw_id = await self.fetch(row.source, url)
|
|
if response.status_code == 404:
|
|
return None # 预约地址尚未发布；保留原始响应，不宣称采集成功。
|
|
response.raise_for_status()
|
|
published = release_date(response.text).date()
|
|
if row.source == "bls" and published < date:
|
|
return None # 到点后官网仍缓存上一期，等待当期公告，不填旧值。
|
|
if row.source == "bls" and published > date and row.first_received_at is not None:
|
|
return None # 滚动最新页不再包含本期，不能据此否认已取得的结果。
|
|
if published != date:
|
|
raise ValueError("官方公告日期与日历不符，历史公布值尚未取得")
|
|
return response, raw_id
|
|
|
|
async def european_result(self, row: EconomicRelease) -> tuple[httpx.Response, int] | None:
|
|
day = row.scheduled_at.astimezone(europe.ZoneInfo(europe.ZONES[row.source])).date()
|
|
url = row.release_url
|
|
if url is None and row.source == "ecb":
|
|
index, _ = await self.fetch("ecb", f"https://www.ecb.europa.eu/press/govcdec/mopo/{day.year}/html/index_include.en.html")
|
|
index.raise_for_status()
|
|
sel = Selector(text=index.text)
|
|
urls = [urljoin(str(index.url), u) for u in sel.css('a[href]::attr(href)').getall()
|
|
if f"ecb.mp{day:%y%m%d}~" in u and u.endswith(".en.html")]
|
|
url = next(iter(dict.fromkeys(urls)), None)
|
|
elif url is None and row.source == "destatis":
|
|
# 官方主题页在 RSS 滚出后仍保留最近公告，且提供英文公告的真实链接。
|
|
paths = {"de_cpi": "Economy/Prices/Consumer-Price-Index", "de_gdp": "Economy/National-Accounts-Domestic-Product", "de_jobs": "Labour/Labour-Market/Employment"}
|
|
index, _ = await self.fetch("destatis", f"https://www.destatis.de/EN/Themes/{paths[row.family]}/_node.html")
|
|
index.raise_for_status()
|
|
suffix = {"de_cpi": "_611.html", "de_gdp": "_811.html", "de_jobs": "_132.html"}[row.family]
|
|
candidates = [urljoin("https://www.destatis.de/", u) for u in Selector(text=index.text).css('a[href]::attr(href)').getall()
|
|
if "/Press/" in u and "/PE" in u and u.endswith(suffix)]
|
|
for candidate in dict.fromkeys(candidates):
|
|
response, raw_id = await self.fetch(row.source, candidate)
|
|
response.raise_for_status()
|
|
announcement = europe.parse_announcement(row.family, response.text)
|
|
if announcement.at.date() == row.scheduled_at.date() and announcement.period == row.period and announcement.stage == row.stage:
|
|
return response, raw_id
|
|
return None
|
|
elif url is None:
|
|
entries = []
|
|
for feed in FEEDS[row.source]:
|
|
response, _ = await self.fetch(row.source, feed)
|
|
response.raise_for_status()
|
|
entries.extend(feed_entries(response.content, row.source))
|
|
for entry in entries:
|
|
target = europe.family_for(row.source, entry.title)
|
|
if target and target[0] == row.family and entry.at.astimezone(europe.ZoneInfo(europe.ZONES[row.source])).date() == day:
|
|
# 只有当期同日同系列才取原文；最终还要核对公告的统计期和阶段。
|
|
url = entry.url
|
|
break
|
|
if url is None:
|
|
return None
|
|
response, raw_id = await self.fetch(row.source, url)
|
|
if response.status_code == 404:
|
|
return None
|
|
response.raise_for_status()
|
|
if row.source == "ons" and "/releases/" in str(response.url):
|
|
patterns = {"uk_cpi": "/bulletins/consumerpriceinflation/", "uk_gdp": "/bulletins/gdp", "uk_jobs": "/bulletins/uklabourmarket/"}
|
|
matches = [urljoin(str(response.url), u) for u in Selector(text=response.text).css('a[href]::attr(href)').getall() if patterns[row.family] in u]
|
|
unique = list(dict.fromkeys(matches))
|
|
if len(unique) != 1:
|
|
raise ValueError("ONS 发布页未唯一指向目标统计公报")
|
|
response, raw_id = await self.fetch(row.source, unique[0])
|
|
response.raise_for_status()
|
|
announcement = europe.parse_announcement(row.family, response.text)
|
|
if announcement.at.astimezone(europe.ZoneInfo(europe.ZONES[row.source])).date() != day:
|
|
raise ValueError("欧洲官方公告日期与所选日程不符，未填入其他期次的数据")
|
|
return response, raw_id
|
|
|
|
async def poll_one(self, release_id: int) -> None:
|
|
async with SessionLocal() as session:
|
|
row = await session.scalar(select(EconomicRelease).where(EconomicRelease.id == release_id).with_for_update())
|
|
if row is None or row.collection_retired_at is not None:
|
|
return
|
|
now = datetime.now(UTC)
|
|
retry_at = None
|
|
try:
|
|
page = await self.result_page(row)
|
|
if page is not None:
|
|
response, raw_id = page
|
|
if row.source == "adp":
|
|
announcement = adp.report(response.text)
|
|
period, stage, values = announcement.period, announcement.stage, announcement.values
|
|
elif row.source == "eia":
|
|
announcement = eia.report(response.text, self.eia_schedule_body)
|
|
period, stage, values = announcement.period, announcement.stage, announcement.values
|
|
elif row.source == "dol":
|
|
announcement = (await asyncio.to_thread(dol.report, await asyncio.to_thread(pdf_text, response.content)))[row.family]
|
|
period, stage, values = announcement.period, announcement.stage, announcement.values
|
|
elif row.source in europe.SCHEDULES:
|
|
announcement = europe.parse_announcement(row.family, response.text)
|
|
period, stage, values = announcement.period, announcement.stage, announcement.values
|
|
elif row.family == "umich":
|
|
if announcement := response.extensions.get('calendar_announcement'):
|
|
period, stage, values = announcement.period, announcement.stage, announcement.values
|
|
else:
|
|
period, stage, values = current_result(response.text)
|
|
else:
|
|
values = parse_values(row.family, response.text)
|
|
text = " ".join(Selector(text=response.text).css("h1,#bodytext").xpath("string(.)").getall())
|
|
period, stage = release_metadata(row.family, text)
|
|
save_result(session, row, values, response.extensions.get("calendar_url", str(response.url)), raw_id, datetime.now(UTC), period=period, stage=stage)
|
|
if page is not None:
|
|
row.collection_failures = 0
|
|
row.error = None
|
|
except (httpx.HTTPError, ValueError, KeyError, TypeError, IndexError) as exc:
|
|
row.error = f"{type(exc).__name__}: {exc}"
|
|
if row.source in BOUNDED_SOURCES and not isinstance(exc, SourceCooling):
|
|
row.collection_failures += 1
|
|
if row.collection_failures >= 3:
|
|
row.collection_retired_at = datetime.now(UTC)
|
|
if row.source in BOUNDED_SOURCES:
|
|
retry_at = datetime.now(UTC) + timedelta(hours=1)
|
|
retry_at = max(filter(None, (retry_at, getattr(exc, 'retry_at', None))), default=None)
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
retry_at = max(filter(None, (retry_at, self.cooldowns.get(exc.request.url.host))), default=None)
|
|
if retry_at: row.error += f"；下次尝试 {retry_at.isoformat()}"
|
|
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (429, 503):
|
|
retry_at = max(retry_at or datetime.now(UTC), retry_after_at(exc.response.headers.get("Retry-After"), datetime.now(UTC)) or datetime.now(UTC))
|
|
logger.exception("公布值采集失败：%s %s", row.family, row.scheduled_at)
|
|
row.last_checked_at = now
|
|
row.next_check_at = next_poll(row.scheduled_at, datetime.now(UTC), row.first_received_at is not None, bool(row.error))
|
|
if retry_at is not None:
|
|
row.next_check_at = max(row.next_check_at, retry_at)
|
|
await session.commit()
|
|
|
|
async def results(self) -> None:
|
|
now = datetime.now(UTC)
|
|
async with SessionLocal() as session:
|
|
ids = list(await session.scalars(select(EconomicRelease.id).join(Source, Source.code == EconomicRelease.source).where(
|
|
Source.enabled.is_(True), EconomicRelease.collection_retired_at.is_(None),
|
|
EconomicRelease.source != "jin10",
|
|
EconomicRelease.schedule_status == "scheduled", EconomicRelease.scheduled_at <= now,
|
|
EconomicRelease.scheduled_at >= now - timedelta(days=HISTORY_DAYS), EconomicRelease.next_check_at <= now,
|
|
).order_by(EconomicRelease.scheduled_at.desc()).limit(8)))
|
|
# 少量同时发布的不同报告独立获取，历史修订不阻塞当期公布。
|
|
self.poll_cache = {}
|
|
try:
|
|
await asyncio.gather(*(self.poll_one(id_) for id_ in ids))
|
|
finally:
|
|
self.poll_cache = None
|
|
|
|
async def run(self) -> None:
|
|
async def schedules_loop() -> None:
|
|
while True:
|
|
await self.schedules()
|
|
await asyncio.sleep(3600)
|
|
|
|
async def results_loop() -> None:
|
|
while True:
|
|
await self.results()
|
|
await asyncio.sleep(1)
|
|
|
|
async def heartbeat_loop() -> None:
|
|
while True:
|
|
await redis.set(STATUS_KEY, json.dumps({"at": datetime.now(UTC).isoformat(), "sources": self.sources}), ex=30)
|
|
await asyncio.sleep(5)
|
|
|
|
try:
|
|
async with asyncio.TaskGroup() as tasks:
|
|
for loop in (schedules_loop, results_loop, heartbeat_loop):
|
|
tasks.create_task(loop())
|
|
finally:
|
|
await self.http.aclose()
|
|
await self.bls.aclose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
asyncio.run(CalendarWorker().run())
|