diff --git a/backend/alembic/versions/20261030_e6f9ab243cd1_us_calendar_sources.py b/backend/alembic/versions/20261030_e6f9ab243cd1_us_calendar_sources.py new file mode 100644 index 0000000..d647048 --- /dev/null +++ b/backend/alembic/versions/20261030_e6f9ab243cd1_us_calendar_sources.py @@ -0,0 +1,28 @@ +"""Register DOL and EIA calendar sources without enabling news crawlers.""" +from alembic import op +import sqlalchemy as sa + +revision = "e6f9ab243cd1" +down_revision = "d5e8fa132bc0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + source = sa.table("sources", *[sa.column(key, kind) for key, kind in [ + ("code", sa.String), ("name", sa.String), ("list_url", sa.String), ("collector", sa.String), + ("enabled", sa.Boolean), ("interval_seconds", sa.Integer), ("backfill_hours", sa.Integer), + ("consecutive_failures", sa.Integer), + ]]) + op.bulk_insert(source, [{"code": code, "name": name, "list_url": url, + "collector": "calendar", "enabled": True, "interval_seconds": 3600, + "backfill_hours": 72, "consecutive_failures": 0} + for code, name, url in [ + ("dol", "美国劳工部·失业金周报", "https://oui.doleta.gov/unemploy/archive.asp"), + ("eia", "美国能源信息署·石油库存周报", "https://www.eia.gov/petroleum/supply/weekly/schedule.php"), + ]]) + + +def downgrade() -> None: + # Existing raw responses prevent source deletion through their foreign key. + op.execute("DELETE FROM sources WHERE code IN ('dol', 'eia')") diff --git a/backend/src/alphapulse/calendar/backfill.py b/backend/src/alphapulse/calendar/backfill.py index cacf700..765d580 100644 --- a/backend/src/alphapulse/calendar/backfill.py +++ b/backend/src/alphapulse/calendar/backfill.py @@ -24,7 +24,7 @@ from parsel import Selector from sqlalchemy import select, text from alphapulse.calendar.archives import Announcement, parse_announcements, parse_umich_report -from alphapulse.calendar import europe +from alphapulse.calendar import dol, eia, europe from alphapulse.calendar.catalog import UMICH_CSV, UMICH_HISTORY, UMICH_HOME, UMICH_INFO, UMICH_REPORTS from alphapulse.calendar.parsers import Scheduled, bea_family, compact from alphapulse.calendar.service import record_revision, save_raw, save_result, save_schedule, save_series @@ -37,6 +37,8 @@ LOCAL = ZoneInfo("Asia/Shanghai") HOSTS = {"bls": {"www.bls.gov", "www.dol.gov"}, "bea": {"www.bea.gov"}, "fed": {"www.federalreserve.gov"}, "umich": {"www.sca.isr.umich.edu", "data.sca.isr.umich.edu"}} US_SOURCES = frozenset(HOSTS) +HOSTS["dol"] = {"www.dol.gov", "oui.doleta.gov"} +HOSTS["eia"] = {"www.eia.gov", "ir.eia.gov"} HOSTS.update(EUROPE_HOSTS) @@ -77,6 +79,8 @@ def family_for(document: Document) -> str | None: def archive_links(source: str, body: str, base: str, start: date, end: date) -> tuple[list[str], str | None]: """只跟随官方索引的实际链接，不按月份合成公告地址。""" + if source == "dol": + return dol.archive_links(body, start, end), None sel = Selector(text=body, type="html") links = [] for node in sel.css("a[href]"): @@ -157,6 +161,7 @@ async def fetch_documents(start: date, end: date, known: set[str], errors: list[ indexes = [("bls", f"https://www.bls.gov/bls/news-release/{name}.htm") for name in ("cpi", "ppi", "empsit")] indexes += [("bea", "https://www.bea.gov/news/archive?field_related_product_target_id=All&created_1=All&title="), ("fed", "https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm")] + indexes += [("dol", dol.SCHEDULE)] for source, initial in indexes: if source not in selected or source in stopped_sources: continue @@ -172,6 +177,12 @@ async def fetch_documents(start: date, end: date, known: set[str], errors: list[ links, following = archive_links(source, index.body.decode("utf-8"), url, start, end) if not links and url == initial: raise ValueError("官方归档未识别到目标公告，覆盖范围尚未核实") + if source == "dol": + years = {int(urlparse(link).path.split("/")[2]) for link in links} + absent = set(range(start.year, end.year + 1)) - years + if absent: + errors.append({"source": source, "url": url, + "error": f"DOL 归档索引未列出年份 {sorted(absent)}，历史覆盖不完整"}) for link in links: if link in known: continue @@ -186,6 +197,23 @@ async def fetch_documents(start: date, end: date, known: set[str], errors: list[ except (httpx.HTTPError, ValueError) as exc: report_error(source, str(exc.request.url) if isinstance(exc, httpx.HTTPError) else url, exc) break + if "dol" in selected and "dol" not in stopped_sources and dol.LATEST not in known: + try: + documents.append(await fetch("dol", dol.LATEST)) + except (httpx.HTTPError, ValueError) as exc: + report_error("dol", dol.LATEST, exc) + if "eia" in selected: + try: + documents.append(await fetch("eia", eia.SCHEDULE)) + index = await fetch("eia", eia.ARCHIVE) + for url in eia.archive_links(index.body.decode(), start, end): + page = await fetch("eia", url) + link = eia.csv_link(page.body.decode(), url) + if link not in known: + documents.append(await fetch("eia", link)) + known.add(link) + except (httpx.HTTPError, ValueError) as exc: + report_error("eia", str(exc.request.url) if isinstance(exc, httpx.HTTPError) else eia.ARCHIVE, exc) from alphapulse.calendar.europe_archives import collect for source in sorted(selected & EUROPE_HOSTS.keys()): try: @@ -282,7 +310,7 @@ async def run(start: date, end: date, *, commit: bool = False, stored_only: bool RawResponse.source_code.in_(selected), RawResponse.status == 200, (RawResponse.url.contains("/news.release/") | RawResponse.url.contains("/newsroom/economicdata/") | RawResponse.url.contains("/news/202") | RawResponse.url.contains("/pressreleases/monetary") | - (RawResponse.source_code == "umich") | RawResponse.source_code.in_(EUROPE_HOSTS)), + RawResponse.source_code.in_(("umich", "dol", "eia")) | RawResponse.source_code.in_(EUROPE_HOSTS)), ).distinct(RawResponse.source_code, RawResponse.url, RawResponse.body_sha256) .order_by(RawResponse.source_code, RawResponse.url, RawResponse.body_sha256, RawResponse.id))).all() documents = [Document(r.source_code, r.url, gzip.decompress(body), r.fetched_at, r.id, r.headers) for r, body in rows] @@ -292,6 +320,8 @@ async def run(start: date, end: date, *, commit: bool = False, stored_only: bool documents += await fetch_documents(start, end, {d.url for d in documents}, errors, **({"sources": sources} if sources is not None else {})) announcements = [] series_points = [] + eia_schedules = [doc for doc in documents if doc.source == "eia" and doc.url == eia.SCHEDULE] + eia_schedule = max(eia_schedules, key=lambda doc: doc.captured_at) if eia_schedules else None # 先处理历史表，再处理年度更正；重跑的稳定身份不依赖日程 PDF 的文件名。 parsed_schedules = [] reports = [] @@ -320,6 +350,25 @@ async def run(start: date, end: date, *, commit: bool = False, stored_only: bool announcements.append((document, "umich", announcement, schedules[identity][0].title)) for document in sorted(documents, key=lambda d: d.captured_at): try: + if document.source == "eia": + if urlparse(document.url).path.endswith("/table1.csv"): + if eia_schedule is None: + raise ValueError("EIA 缺少已采集的公布规则和节假日例外表") + announcement = eia.report(document.body.decode(), eia_schedule.body.decode()) + path = urlparse(document.url).path + date_match = re.search(r"/archive/20\d\d/(\d{4}_\d{2}_\d{2})/", path) + if date_match and datetime.strptime(date_match[1], "%Y_%m_%d").date() != announcement.at.astimezone(eia.EASTERN).date(): + raise ValueError("EIA 报告统计周与归档公布日期不符") + if begin <= announcement.at < finish: + announcements.append((document, "eia_stocks", announcement, "Weekly Petroleum Status Report")) + continue + if document.source == "dol": + if document.body.startswith(b"%PDF"): + parsed = await asyncio.to_thread(dol.report, await asyncio.to_thread(pdf_text, document.body)) + for family, announcement in parsed.items(): + if begin <= announcement.at < finish: + announcements.append((document, family, announcement, "Unemployment Insurance Weekly Claims")) + continue if document.source == "umich": if document.url == UMICH_CSV: series_points.append((document, [(p, v) for p, v in monthly_series(document.body.decode()) @@ -348,6 +397,10 @@ async def run(start: date, end: date, *, commit: bool = False, stored_only: bool changed = 0 if commit: await ensure_quiet_window() + if eia_schedule is not None and eia_schedule.raw_id is None: + async with SessionLocal() as session: + await save_raw(session, "eia", eia_schedule.url, 200, eia_schedule.headers, eia_schedule.body, eia_schedule.captured_at) + await session.commit() batches = {} for event, document in schedules.values(): _, batch = batches.setdefault((document.url, document.captured_at), (document, [])) diff --git a/backend/src/alphapulse/calendar/catalog.py b/backend/src/alphapulse/calendar/catalog.py index c8185ad..cf13da9 100644 --- a/backend/src/alphapulse/calendar/catalog.py +++ b/backend/src/alphapulse/calendar/catalog.py @@ -37,6 +37,11 @@ INDICATORS = { "gdp": Indicator("实际 GDP 年化季率", 4), "fed_rate": Indicator("联邦基金利率目标上限", 5, rate_channel="inverse"), "umich_sentiment": Indicator("密歇根大学消费者信心指数", 3, "点"), + "initial_claims": Indicator("初请失业金人数（季调）", 4, "万人"), + "continued_claims": Indicator("续请失业金人数（季调）", 3, "万人"), + "eia_crude": Indicator("EIA 商业原油库存变动（不含战略储备）", 4, "万桶"), + "eia_gasoline": Indicator("EIA 汽油库存变动", 3, "万桶"), + "eia_distillate": Indicator("EIA 精炼油库存变动", 3, "万桶"), } FAMILIES = { "cpi": Family("US", "bls", ("cpi_mom", "cpi_yoy", "core_cpi_mom", "core_cpi_yoy")), @@ -46,6 +51,9 @@ FAMILIES = { "gdp": Family("US", "bea", ("gdp",)), "fomc": Family("US", "fed", ("fed_rate",)), "umich": Family("US", "umich", ("umich_sentiment",)), + "initial_claims": Family("US", "dol", ("initial_claims",)), + "continued_claims": Family("US", "dol", ("continued_claims",)), + "eia_stocks": Family("US", "eia", ("eia_crude", "eia_gasoline", "eia_distillate")), } STAGES = {"preliminary": "初值", "final": "终值", "initial": "首发值", "updated": "更新估计", "advance": "初值", "second": "二次估计", "third": "终值"} STAGES["flash"] = "快报值" diff --git a/backend/src/alphapulse/calendar/dol.py b/backend/src/alphapulse/calendar/dol.py new file mode 100644 index 0000000..0f96cae --- /dev/null +++ b/backend/src/alphapulse/calendar/dol.py @@ -0,0 +1,100 @@ +"""DOL weekly claims: use published schedules and the seasonally adjusted release.""" +import re +from datetime import UTC, date, datetime, time, timedelta +from decimal import Decimal +from urllib.parse import urljoin, urlparse + +from parsel import Selector + +from alphapulse.calendar.archives import Announcement, publication_time +from alphapulse.calendar.parsers import EASTERN, MONTHS, Scheduled, compact + +SCHEDULE = "https://oui.doleta.gov/unemploy/archive.asp" +LATEST = "https://www.dol.gov/ui/data.pdf" +FAMILIES = ("initial_claims", "continued_claims") +MONTH_PATTERN = "(?:" + "|".join(MONTHS) + ")" + + +def schedule(body: str, now: datetime) -> list[Scheduled]: + sel = Selector(text=body) + text = compact(sel.xpath("string(//body)").get() or "") + if not re.search(r"Publication Schedule:.*Thursday morning at 8:30am EST", text): + raise ValueError("DOL 缺少已验证的每周公布规则") + exceptions = {} + tables = [table for table in sel.css("table") if {compact(n.xpath("string(.)").get()) for n in table.css("th")} == {"Release Date", "Release Time"}] + if len(tables) != 1: + raise ValueError("DOL 缺少唯一的节假日例外表") + for row in tables[0].css("tbody tr"): + cells = [compact(n.xpath("string(.)").get()) for n in row.css("td")] + if len(cells) != 2 or not re.fullmatch(r"\w+, " + MONTH_PATTERN + r" \d{1,2}, 20\d\d", cells[0]): + raise ValueError("DOL 例外公布日期格式已变化") + day = datetime.strptime(cells[0], "%A, %B %d, %Y").date() + if cells[1] != "8:30 AM EST": + raise ValueError("DOL 例外公布时刻已变化") + thursday = day + timedelta(days=3 - day.weekday()) + if thursday in exceptions: + raise ValueError("DOL 同周公布例外不唯一") + exceptions[thursday] = day + start = now.astimezone(EASTERN).date() - timedelta(days=45) + end = now.astimezone(EASTERN).date() + timedelta(days=90) + days = {start + timedelta(days=i) for i in range((end - start).days + 1)} + result = [] + for thursday in sorted(day for day in days if day.weekday() == 3): + day = exceptions.get(thursday, thursday) + at = datetime.combine(day, time(8, 30), EASTERN).astimezone(UTC) + for family in FAMILIES: + result.append(Scheduled(f"dol-{family}-{day}", family, "Unemployment Insurance Weekly Claims", at)) + return result + + +def archive_links(body: str, start: date, end: date) -> list[str]: + links = [] + for link in Selector(text=body).css("a[href]::attr(href)").getall(): + url = urljoin(SCHEDULE, link) + parsed = urlparse(url) + match = re.fullmatch(r"/press/(20\d\d)/(\d{6})\.pdf", parsed.path) + if parsed.scheme != "https" or parsed.hostname != "oui.doleta.gov" or match is None: + continue + day = datetime.strptime(match[2], "%m%d%y").date() + if day.year != int(match[1]): + raise ValueError("DOL 归档路径年份与日期不符") + if start <= day <= end: + links.append(url) + if not links: + raise ValueError("DOL 归档未列出所选范围的报告，历史覆盖尚未确认") + return list(dict.fromkeys(links)) + + +def report(text: str) -> dict[str, Announcement]: + text = compact(text) + if "UNEMPLOYMENT INSURANCE WEEKLY CLAIMS" not in text: + raise ValueError("DOL 响应不是失业金周报") + at = publication_time(text) + match = re.search(r"SEASONALLY ADJUSTED DATA (.*?) UNADJUSTED DATA", text) + if match is None: + raise ValueError("DOL 缺少独立的季调数据正文") + adjusted = match[1] + initial = re.search(r"In the week ending (" + MONTH_PATTERN + r" \d{1,2}), the advance figure for seasonally adjusted initial claims was ([\d,]+)", adjusted) + continued = re.search(r"advance number for seasonally adjusted insured unemployment during the week ending (" + MONTH_PATTERN + r" \d{1,2}) was ([\d,]+)", adjusted) + if initial is None or continued is None: + raise ValueError("DOL 缺少初请或续请季调读数") + result = {} + for family, reading in zip(FAMILIES, (initial, continued), strict=True): + # Resolve year rollover from the official publication date, never from collection time. + period = datetime.strptime(f"{reading[1]}, {at.astimezone(EASTERN).year}", "%B %d, %Y").date() + published = at.astimezone(EASTERN).date() + if period > published: + period = period.replace(year=period.year - 1) + if not 0 <= (published - period).days <= 21: + raise ValueError("DOL 统计周与公布日期不符") + following = adjusted[reading.end():].split("The 4-week moving average", 1)[0] + previous = re.search(r"previous week's level was (?:revised (?:up|down) by [\d,]+ from [\d,]+ to|unrevised at) ([\d,]+)", following) + if previous is None: + previous = re.search(r"previous week's unrevised level of ([\d,]+)", following) + if previous is None: + raise ValueError("DOL 缺少本报告列示的可比前值") + value = {"actual": format(Decimal(reading[2].replace(",", "")) / 10000, "f"), + "previous": format(Decimal(previous[1].replace(",", "")) / 10000, "f"), + "previous_basis": "DOL 本次季调报告列示的上周值（可能已修订）"} + result[family] = Announcement(at, period.isoformat(), None, {family: value}) + return result diff --git a/backend/src/alphapulse/calendar/eia.py b/backend/src/alphapulse/calendar/eia.py new file mode 100644 index 0000000..4256d04 --- /dev/null +++ b/backend/src/alphapulse/calendar/eia.py @@ -0,0 +1,111 @@ +"""EIA commercial inventory changes from the official, unrounded Table 1 CSV.""" +import csv +from datetime import UTC, date, datetime, time, timedelta +from decimal import Decimal +from io import StringIO +import re +from urllib.parse import urljoin, urlparse + +from parsel import Selector + +from alphapulse.calendar.archives import Announcement +from alphapulse.calendar.parsers import EASTERN, Scheduled, compact + +SCHEDULE = "https://www.eia.gov/petroleum/supply/weekly/schedule.php" +ARCHIVE = "https://www.eia.gov/petroleum/supply/weekly/archive/" +LATEST = "https://ir.eia.gov/wpsr/table1.csv" +ROWS = {"Commercial (Excluding SPR)": "eia_crude", "Total Motor Gasoline": "eia_gasoline", "Distillate Fuel Oil": "eia_distillate"} + + +def exceptions(body: str) -> dict[date, datetime]: + sel = Selector(text=body) + text = compact(sel.xpath("string(//body)").get() or "") + if "10:30 a.m. eastern time on Wednesday" not in text or "All times are eastern" not in text: + raise ValueError("EIA 缺少已验证的每周公布规则") + tables = sel.css("table.schedule") + if len(tables) != 1 or "Data for the week ending" not in tables[0].xpath("string(.)").get(): + raise ValueError("EIA 缺少唯一的节假日例外表") + result = {} + for row in tables[0].css("tbody tr"): + cells = [compact(n.xpath("string(.)").get()) for n in row.css("th,td")] + if len(cells) != 5: + raise ValueError("EIA 例外日程列格式已变化") + period = datetime.strptime(cells[0], "%B %d, %Y").date() + day = datetime.strptime(cells[1], "%B %d, %Y").date() + clock = datetime.strptime(cells[3].replace("a.m.", "AM").replace("p.m.", "PM"), "%I:%M %p").time() + if cells[2] != day.strftime("%A") or period in result: + raise ValueError("EIA 例外日程星期或统计周不符") + result[period] = datetime.combine(day, clock, EASTERN).astimezone(UTC) + return result + + +def release_at(period: date, changes: dict[date, datetime]) -> datetime: + if period.weekday() != 4: + raise ValueError("EIA 统计期不是周五") + return changes.get(period, datetime.combine(period + timedelta(days=5), time(10, 30), EASTERN).astimezone(UTC)) + + +def schedule(body: str, now: datetime) -> list[Scheduled]: + changes = exceptions(body) + start = now.astimezone(EASTERN).date() - timedelta(days=60) + result = [] + for i in range(160): + period = start + timedelta(days=i) + if period.weekday() == 4: + result.append(Scheduled(f"eia-stocks-{period}", "eia_stocks", "Weekly Petroleum Status Report", + release_at(period, changes), period=period.isoformat())) + return result + + +def archive_links(body: str, start: date, end: date) -> list[str]: + result = [] + for link in Selector(text=body).css("a[href]::attr(href)").getall(): + url = urljoin(ARCHIVE, link) + parsed = urlparse(url) + match = re.fullmatch(r"/petroleum/supply/weekly/archive/(20\d\d)/(\d{4}_\d{2}_\d{2})/wpsr_\2\.php", parsed.path) + if parsed.scheme != "https" or parsed.hostname != "www.eia.gov" or match is None: + continue + day = datetime.strptime(match[2], "%Y_%m_%d").date() + if day.year != int(match[1]): + raise ValueError("EIA 归档年份与日期不符") + if start <= day <= end: + result.append(url) + if not result: + raise ValueError("EIA 归档没有所选范围的已公布报告") + return list(dict.fromkeys(result)) + + +def csv_link(body: str, base: str) -> str: + links = [urljoin(base, link) for link in Selector(text=body).css("a[href]::attr(href)").getall() + if link == "download/table1.csv"] + if len(set(links)) != 1: + raise ValueError("EIA 归档未唯一列示 Table 1 CSV") + return links[0] + + +def report(body: str, schedule_body: str) -> Announcement: + rows = csv.reader(StringIO(body)) + header = next(rows, []) + if len(header) != 8 or header[0] != "STUB_1" or header[3] != "Difference": + raise ValueError("EIA Table 1 CSV 表头已变化") + period, previous = (datetime.strptime(value, "%m/%d/%y").date() for value in header[1:3]) + if period - previous != timedelta(days=7): + raise ValueError("EIA Table 1 当前周与上周不连续") + values = {} + for row in rows: + if row and row[0] == "STUB_1": + break # Following tables use daily supply rates, not inventory levels. + if not row or row[0] not in ROWS: + continue + if len(row) != 8 or ROWS[row[0]] in values: + raise ValueError("EIA 库存数据行重复或缺列") + number = row[3].replace(",", "") + if not re.fullmatch(r"[+-]?\d+(?:\.\d+)?", number): + raise ValueError("EIA 库存变动不是有限数值") + change = Decimal(number) + # Table 1 inventories are million barrels; the calendar displays ten thousand barrels. + # Previous inventory levels are not the previous week's inventory CHANGE. + values[ROWS[row[0]]] = {"actual": format(change * 100, "f"), "previous": None} + if values.keys() != set(ROWS.values()): + raise ValueError("EIA 缺少商业原油、汽油或精炼油库存变动") + return Announcement(release_at(period, exceptions(schedule_body)), period.isoformat(), None, values) diff --git a/backend/src/alphapulse/calendar/service.py b/backend/src/alphapulse/calendar/service.py index 170a40f..2fb731b 100644 --- a/backend/src/alphapulse/calendar/service.py +++ b/backend/src/alphapulse/calendar/service.py @@ -187,7 +187,10 @@ def release_rows(row: EconomicRelease, now: datetime) -> list[dict]: country_name = COUNTRIES[family.country_code] period_name = "" if row.period is not None: - period_name = f"第{row.period[-1]}季度" if "-Q" in row.period else f"{int(row.period[5:7])}月" + if len(row.period) == 10: + period_name = f"截至{row.period}当周" + else: + period_name = f"第{row.period[-1]}季度" if "-Q" in row.period else f"{int(row.period[5:7])}月" for key in family.indicators: indicator = INDICATORS[key] value = row.values.get(key, {}) diff --git a/backend/src/alphapulse/calendar/worker.py b/backend/src/alphapulse/calendar/worker.py index c21abff..b8f0f79 100644 --- a/backend/src/alphapulse/calendar/worker.py +++ b/backend/src/alphapulse/calendar/worker.py @@ -12,7 +12,7 @@ from parsel import Selector from sqlalchemy import select from alphapulse.calendar.catalog import SCHEDULE_URLS, UMICH_CSV, UMICH_HISTORY, UMICH_HOME, UMICH_INFO -from alphapulse.calendar import europe +from alphapulse.calendar import dol, eia, europe from alphapulse.calendar.parsers import EASTERN, MONTHS, bea_release_link, parse_fed_calendar, parse_ics, parse_values, release_date, bls_periods, release_metadata from alphapulse.calendar.service import apply_bls_periods, record_revision, next_poll, save_raw, save_result, save_schedule, save_series from alphapulse.calendar.umich import current_result, document_links, monthly_series, parse_schedule, pdf_text @@ -57,12 +57,14 @@ class CalendarWorker: self.http = httpx.AsyncClient(timeout=8, follow_redirects=True, headers={"User-Agent": "AlphaPulse economic calendar"}) self.bls = httpx.AsyncClient(timeout=8, follow_redirects=True, verify=ctx, headers={"User-Agent": "AlphaPulse economic calendar"}) self.sources: dict[str, dict] = {} + self.eia_schedule_body: str | None = None self.fetch_slots = asyncio.Semaphore(2) async def fetch(self, source: str, url: str, *, query: dict | None = None) -> tuple[httpx.Response, int]: # 链接来自公开响应，限制到对应官方域名，避免外部内容扩大访问范围。 allowed = {"bls": {"www.bls.gov"}, "bea": {"www.bea.gov", "apps.bea.gov"}, "fed": {"www.federalreserve.gov"}, - "umich": {"www.sca.isr.umich.edu", "data.sca.isr.umich.edu"}, **HOSTS} + "umich": {"www.sca.isr.umich.edu", "data.sca.isr.umich.edu"}, "dol": {"www.dol.gov", "oui.doleta.gov"}, + "eia": {"www.eia.gov", "ir.eia.gov"}, **HOSTS} if urlparse(url).scheme != "https" or urlparse(url).hostname not in allowed[source]: raise ValueError("公告链接不是对应机构的官方 HTTPS 地址") async with self.fetch_slots: @@ -84,6 +86,8 @@ class CalendarWorker: urls += [("bls", f"https://www.bls.gov/schedule/news_release/{code}.htm") for code in ("cpi", "ppi", "empsit")] urls += [("umich", UMICH_INFO), ("umich", UMICH_CSV)] urls += list(europe.SCHEDULES.items()) + urls += [("dol", dol.SCHEDULE)] + urls += [("eia", eia.SCHEDULE)] # 启动时明确各来源尚未就绪；月份滚动后移除不再检查的旧月状态。 self.sources = {url: self.sources.get(url, {"source": source, "checked_at": now.isoformat(), "error": "尚未同步"}) for source, url in urls} @@ -95,7 +99,14 @@ class CalendarWorker: fetch_url = europe.eurostat_calendar_url(now) if source == "eurostat" else url response, raw_id = await self.fetch(source, fetch_url, query=europe.INSEE_QUERY if source == "insee" else None) response.raise_for_status() - if source in europe.SCHEDULES: + if source in ("dol", "eia"): + events = (dol if source == "dol" else eia).schedule(response.text, now) + async with SessionLocal() as session: + await save_schedule(session, source, url, events, raw_id, datetime.now(UTC), withdraw_missing=False) + await session.commit() + if source == "eia": + self.eia_schedule_body = response.text + elif source in europe.SCHEDULES: events = europe.parse_schedule(source, response.text) async with SessionLocal() as session: # 欧洲公开日程是滚动/分页窗口，缺失不等于撤回。 @@ -168,6 +179,30 @@ class CalendarWorker: await session.commit() async def result_page(self, row: EconomicRelease) -> tuple[httpx.Response, int] | None: + if row.source == "eia": + if self.eia_schedule_body is None: + schedule, _ = await self.fetch("eia", eia.SCHEDULE) + schedule.raise_for_status() + eia.exceptions(schedule.text) + self.eia_schedule_body = schedule.text + response, raw_id = await self.fetch("eia", row.release_url or eia.LATEST) + response.raise_for_status() + announcement = eia.report(response.text, self.eia_schedule_body) + if announcement.at < row.scheduled_at or (announcement.at > row.scheduled_at and row.first_received_at is not None): + return None + if announcement.at != row.scheduled_at or announcement.period != row.period: + raise ValueError("EIA 公告统计周或时间与所选日程不符，未填入其他期次的数据") + return response, raw_id + if row.source == "dol": + response, raw_id = await self.fetch("dol", row.release_url or dol.LATEST) + response.raise_for_status() + announcements = await asyncio.to_thread(dol.report, await asyncio.to_thread(pdf_text, response.content)) + announcement = announcements[row.family] + if announcement.at < row.scheduled_at or (announcement.at > row.scheduled_at and row.first_received_at is not None): + return None + if announcement.at != row.scheduled_at: + raise ValueError("DOL 公告时间与所选日程不符，未填入其他期次的数据") + return response, raw_id if row.source in europe.SCHEDULES: return await self.european_result(row) date = row.scheduled_at.astimezone(EASTERN).date() @@ -275,7 +310,13 @@ class CalendarWorker: page = await self.result_page(row) if page is not None: response, raw_id = page - if row.source in europe.SCHEDULES: + if row.source == "eia": + announcement = eia.report(response.text, self.eia_schedule_body) + period, stage, values = announcement.period, announcement.stage, announcement.values + elif row.source == "dol": + announcement = (await asyncio.to_thread(dol.report, await asyncio.to_thread(pdf_text, response.content)))[row.family] + period, stage, values = announcement.period, announcement.stage, announcement.values + elif row.source in europe.SCHEDULES: announcement = europe.parse_announcement(row.family, response.text) period, stage, values = announcement.period, announcement.stage, announcement.values elif row.family == "umich": diff --git a/backend/src/alphapulse/core/official_sources.py b/backend/src/alphapulse/core/official_sources.py index 3f51073..ac9b2bc 100644 --- a/backend/src/alphapulse/core/official_sources.py +++ b/backend/src/alphapulse/core/official_sources.py @@ -19,7 +19,7 @@ FEEDS = { "insee": ("https://www.insee.fr/fr/flux/3",), } REGIONS = {"wscn_news": "CN", "wscn_live": "CN", "cls_depth": "CN", "cls_telegraph": "CN", "coindesk": "GLOBAL", - "bls": "US", "bea": "US", "fed": "US", "umich": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR"} + "bls": "US", "bea": "US", "fed": "US", "umich": "US", "dol": "US", "eia": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR"} HOSTS = {"ecb": {"www.ecb.europa.eu"}, "eurostat": {"ec.europa.eu"}, "boe": {"www.bankofengland.co.uk"}, "ons": {"www.ons.gov.uk"}, "destatis": {"www.destatis.de"}, "insee": {"www.insee.fr"}} LANGUAGES = {"destatis": "de", "insee": "fr"} diff --git a/backend/tests/fixtures/calendar/dol-2025-01-08.txt b/backend/tests/fixtures/calendar/dol-2025-01-08.txt new file mode 100644 index 0000000..5480649 --- /dev/null +++ b/backend/tests/fixtures/calendar/dol-2025-01-08.txt @@ -0,0 +1,46 @@ +1 + +News Release +Connect with DOL at +https://blog.dol.gov + + + +TRANSMISSION OF MATERIALS IN THIS RELEASE IS EMBARGOED UNTIL +8:30 A.M. (Eastern) Wednesday, January 8, 2025 + +UNEMPLOYMENT INSURANCE WEEKLY CLAIMS +SEASONALLY ADJUSTED DATA +In the week ending January 4, the advance figure for seasonally adjusted initial claims was 201,000, a decrease of +10,000 from the previous week's unrevised level of 211,000. The 4-week moving average was 213,000, a decrease of +10,250 from the previous week's unrevised average of 223,250. +The advance seasonally adjusted insured unemployment rate was 1.2 percent for the week ending December 28, +unchanged from the previous week's unrevised rate. The advance number for seasonally adjusted insured +unemployment during the week ending December 28 was 1,867,000, an increase of 33,000 from the previous week's +revised level. The previous week's level was revised down by 10,000 from 1,844,000 to 1,834,000. The 4-week moving +average was 1,865,500, a decrease of 3,000 from the previous week's revised average. The previous week's average was +revised down by 2,250 from 1,870,750 to 1,868,500. + + + +2 + + +UNADJUSTED DATA +The advance number of actual initial claims under state programs, unadjusted, totaled 304,741 in the week ending +January 4, an increase of 21,253 (or 7.5 percent) from the previous week. The seasonal factors had expected an increase +of 35,647 (or 12.6 percent) from the previous week. There were 318,906 initial claims in the comparable week in 2024. +The advance unadjusted insured unemployment rate was 1.4 percent during the week ending December 28, an increase of +0.2 percentage point from the prior week. The advance unadjusted level of insured unemployment in state programs +totaled 2,175,478, an increase of 316,122 (or 17.0 percent) from the preceding week. The seasonal factors had expected +an increase of 276,886 (or 14.9 percent) from the previous week. A year earlier the rate was 1.4 percent and the volume +was 2,104,272. + + +The total number of continued weeks claimed for benefits in all programs for the week ending December 21 was +1,886,283, a decrease of 88,519 from the previous week. There were 1,927,770 weekly claims filed for benefits in all +programs in the comparable week in 2023. +No state was triggered "on" the Extended Benefits program during the week ending December 21. +Initial claims for UI benefits filed by former Federal civilian employees totaled 340 in the week ending December 28, a +decrease of 166 from the prior week. There were 201 initial claims filed by newly discharged veterans, a decrease of 113 +from the preceding week. diff --git a/backend/tests/fixtures/calendar/dol-current.txt b/backend/tests/fixtures/calendar/dol-current.txt new file mode 100644 index 0000000..a32d1b4 --- /dev/null +++ b/backend/tests/fixtures/calendar/dol-current.txt @@ -0,0 +1,49 @@ +1 + +News Release +Connect with DOL at +https://blog.dol.gov + + + +TRANSMISSION OF MATERIALS IN THIS RELEASE IS EMBARGOED UNTIL +8:30 A.M. (Eastern) Thursday, October 29, 2026 + +UNEMPLOYMENT INSURANCE WEEKLY CLAIMS +SEASONALLY ADJUSTED DATA +In the week ending October 24, the advance figure for seasonally adjusted initial claims was 194,000, a decrease of +1,000 from the previous week's revised level. The previous week's level was revised up by 1,000 from 194,000 to +195,000. The 4-week moving average was 196,750, a decrease of 1,250 from the previous week's revised average. The +previous week's average was revised up by 250 from 197,750 to 198,000. +The advance seasonally adjusted insured unemployment rate was 1.1 percent for the week ending October 17, +unchanged from the previous week's unrevised rate. The advance number for seasonally adjusted insured +unemployment during the week ending October 17 was 1,701,000, a decrease of 21,000 from the previous week's +revised level. The previous week's level was revised down by 6,000 from 1,728,000 to 1,722,000. The 4-week moving +average was 1,710,000, an increase of 500 from the previous week's revised average. The previous week's average was +revised down by 1,500 from 1,711,000 to 1,709,500. + + + +2 + + +UNADJUSTED DATA +The advance number of actual initial claims under state programs, unadjusted, totaled 178,851 in the week ending +October 24, an increase of 5,648 (or 3.3 percent) from the previous week. The seasonal factors had expected an increase +of 6,040 (or 3.5 percent) from the previous week. There were 203,012 initial claims in the comparable week in 2025. +The advance unadjusted insured unemployment rate was 1.0 percent during the week ending October 17, unchanged +from the prior week. The advance unadjusted level of insured unemployment in state programs totaled 1,490,071, an +increase of 14,007 (or 0.9 percent) from the preceding week. The seasonal factors had expected an increase of 32,725 (or +2.2 percent) from the previous week. A year earlier the rate was 1.1 percent and the volume was 1,708,691. + + +The total number of continued weeks claimed for benefits in all programs for the week ending October 10 was +1,500,078, an increase of 14,467 from the previous week. There were 1,712,123 weekly claims filed for benefits in all +programs in the comparable week in 2025. +No state was triggered "on" the Extended Benefits program during the week ending October 10. +Initial claims for UI benefits filed by former Federal civilian employees totaled 442 in the week ending October 17, a +decrease of 113 from the prior week. There were 387 initial claims filed by newly discharged veterans, a decrease of 40 +from the preceding week. +There were 4,934 continued weeks claimed filed by former Federal civilian employees the week ending October 10, a +decrease of 100 from the previous week. Newly discharged veterans claiming benefits totaled 5,875, an increase of 1,057 +from the prior week. diff --git a/backend/tests/fixtures/calendar/eia-2026-10-23.csv b/backend/tests/fixtures/calendar/eia-2026-10-23.csv new file mode 100644 index 0000000..aeb4e3f --- /dev/null +++ b/backend/tests/fixtures/calendar/eia-2026-10-23.csv @@ -0,0 +1,59 @@ +"STUB_1","10/23/26","10/16/26","Difference","Percent Change","10/24/25","Difference","Percent Change" +"Crude Oil","693.208","695.200","-1.992","-0.300","825.063","-131.855","-16.000" +"Commercial (Excluding SPR)","410.625","412.517","-1.892","-0.500","415.966","-5.341","-1.300" +"Strategic Petroleum Reserve (SPR)","282.583","282.683","-0.100","0.000","409.097","-126.514","-30.900" +"Total Motor Gasoline","209.180","205.310","3.870","1.900","210.738","-1.558","-0.700" +"Reformulated","0.022","0.025","-0.003","-12.000","0.028","-0.006","-21.400" +"Conventional","14.103","13.710","0.393","2.900","15.235","-1.131","-7.400" +"Blending Components","195.055","191.575","3.480","1.800","195.476","-0.421","-0.200" +"Fuel Ethanol","24.523","24.558","-0.036","-0.100","22.367","2.156","9.600" +"Kerosene-Type Jet Fuel","42.217","42.606","-0.389","-0.900","41.420","0.797","1.900" +"Distillate Fuel Oil","102.824","104.286","-1.462","-1.400","112.189","-9.365","-8.300" +"15 ppm sulfur and Under","91.071","92.880","-1.809","-1.900","101.624","-10.553","-10.400" +"> 15 ppm to 500 ppm sulfur","4.423","4.239","0.183","4.300","3.515","0.907","25.800" +"> 500 ppm sulfur","7.331","7.167","0.164","2.300","7.050","0.281","4.000" +"Residual Fuel Oil","23.399","23.733","-0.334","-1.400","21.802","1.597","7.300" +"Propane/Propylene","105.735","105.846","-0.111","-0.100","105.681","0.054","0.100" +"Other Oils","294.053","296.705","-2.652","-0.900","338.582","-44.529","-13.200" +"Unfinished Oils","76.191","76.051","0.140","0.200","85.483","-9.293","-10.900" +"Total Stocks (Including SPR)","1,495.139","1,498.245","-3.106","-0.200","1,677.842","-182.703","-10.900" +"Total Stocks (Excluding SPR)","1,212.556","1,215.562","-3.006","-0.200","1,268.745","-56.189","-4.400" +"STUB_1","STUB_2","10/23/26","10/16/26","Difference","10/24/25","Difference","10/23/26","10/24/25","Percent Change","10/23/26","10/24/25","Percent Change" +"Crude Oil Supply ","(1) Domestic Production","13,992","13,269","723","13,644","348","13,730","13,635","0.7","13,740","13,459","2.1" +"Crude Oil Supply ","(2) Alaska","482","512","-30","432","50","487","429","13.6","432","419","3.2" +"Crude Oil Supply ","(3) Lower 48","13,510","12,757","753","13,212","298","13,243","13,206","0.3","13,308","13,040","2.1" +"Crude Oil Supply ","(4) Transfers to Crude Oil Supply","540","537","3","463","77","517","411","25.6","539","524","2.8" +"Crude Oil Supply ","(5) Alaskan NGL production","47","46","1","47","0","45","46","-2.3","44","47","-6.7" +"Crude Oil Supply ","(6) Natural Gasoline, Condensate, and Unfinished Oils","493","491","2","416","78","472","366","29.1","495","477","3.7" +"Crude Oil Supply ","(7) Net Imports (Including SPR)","1,520","2,316","-796","690","830","1,847","1,569","17.7","1,910","2,211","-13.6" +"Crude Oil Supply ","(8) Imports","6,374","5,758","616","5,051","1,323","6,255","5,724","9.3","6,165","6,101","1.0" +"Crude Oil Supply ","(9) Commercial Crude Oil","6,374","5,758","616","5,051","1,323","6,255","5,724","9.3","6,165","6,101","1.0" +"Crude Oil Supply ","(10) Imports by SPR","0","0","0","0","0","0","0","0.0","0","0","0.0" +"Crude Oil Supply ","(11) Imports into SPR by Others","0","0","0","0","0","0","0","0.0","0","0","0.0" +"Crude Oil Supply ","(12) Exports","4,854","3,442","1,412","4,361","493","4,408","4,155","6.1","4,254","3,891","9.3" +"Crude Oil Supply ","(13) Stock Change (+/build; -/draw)","-285","-480","196","-904","619","-639","65","� �","-474","55","� �" +"Crude Oil Supply ","(14) Commercial Stock Change","-270","-437","167","-980","709","-596","-21","� �","-30","4","� �" +"Crude Oil Supply ","(15) SPR Stock Change","-14","-43","29","76","-90","-42","86","� �","-444","52","� �" +"Crude Oil Supply ","(16) Adjustment","189","47","142","-481","670","-165","44","� �","-5","109","� �" +"Crude Oil Supply ","(17) Crude Oil Input to Refineries","16,526","16,649","-123","15,219","1,307","16,567","15,594","6.2","16,658","16,247","2.5" +"Other Supply ","(18) Production","10,189","10,144","45","10,053","137","10,188","10,026","1.6","10,072","9,551","5.5" +"Other Supply ","(19) Natural Gas Plant Liquids","7,724","7,697","27","7,739","-14","7,755","7,693","0.8","7,677","7,208","6.5" +"Other Supply ","(20) Renewable Fuels/Oxygenate Plant","1,508","1,484","25","1,396","112","1,473","1,393","5.7","1,410","1,367","3.1" +"Other Supply ","(21) Fuel Ethanol","1,118","1,094","25","1,091","27","1,083","1,087","-0.4","1,090","1,070","1.9" +"Other Supply ","(22) Other","390","390","0","305","85","390","306","27.7","319","297","7.5" +"Other Supply ","(23) Refinery Processing Gain","956","964","-7","918","39","960","940","2.2","986","977","0.9" +"Other Supply ","(24) Net Imports","-6,359","-6,030","-329","-5,026","-1,334","-6,213","-5,344","� �","-6,012","-5,012","� �" +"Other Supply ","(25) Imports","1,315","1,377","-62","1,640","-326","1,419","1,614","-12.1","1,548","1,731","-10.5" +"Other Supply ","(26) Exports","7,674","7,407","267","6,666","1,008","7,632","6,957","9.7","7,560","6,743","12.1" +"Other Supply ","(27) Stock Change (+/build; -/draw)","-159","-770","611","-1,292","1,133","-538","-681","� �","-241","116","� �" +"Other Supply ","(28) Adjustment","234","234","0","206","28","235","207","� �","224","212","� �" +"Other Supply ","(29) Transfers to Crude Oil Supply","-540","-537","-3","-463","-77","-517","-411","25.6","-539","-524","�" +"Products Supplied ","(30) Total","20,209","21,231","-1,022","21,281","-1,072","20,799","20,753","0.2","20,645","20,358","1.4" +"Products Supplied ","(31) Finished Motor Gasoline","8,358","8,982","-625","8,924","-566","8,629","8,688","-0.7","8,766","8,782","-0.2" +"Products Supplied ","(32) Kerosene-Type Jet Fuel","1,679","1,813","-133","1,948","-269","1,703","1,764","-3.5","1,725","1,716","0.5" +"Products Supplied ","(33) Distillate Fuel Oil","4,207","3,732","475","3,580","627","3,978","4,002","-0.6","3,846","3,809","1.0" +"Products Supplied ","(34) Residual Fuel Oil","372","431","-60","376","-4","323","225","43.6","308","281","9.8" +"Products Supplied ","(35) Propane/Propylene","760","1,020","-259","964","-204","941","962","-2.2","969","974","-0.5" +"Products Supplied ","(36) Other Oils","4,834","5,254","-420","5,489","-655","5,226","5,113","2.2","5,030","4,797","4.9" +"Net Imports of Crude and Petroleum Products ","(33) Total","-4,839","-3,714","-1,125","-4,336","-503","-4,365","-3,774","15.7","-4,101","-2,801","46.4" + diff --git a/backend/tests/fixtures/calendar/eia-schedule.html b/backend/tests/fixtures/calendar/eia-schedule.html new file mode 100644 index 0000000..eeff54f --- /dev/null +++ b/backend/tests/fixtures/calendar/eia-schedule.html @@ -0,0 +1,143 @@ +The standard release time and day of the week will be at 10:30 a.m. eastern time on Wednesday with the following exceptions. All times are eastern. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data for the week endingAlternate release dateRelease dayRelease timeHoliday
December 27, 2024January 2, 2025Thursday11:00 a.m.New Year's Day
January 17, 2025January 23, 2025Thursday12:00 p.m.Martin Luther King Jr. Day / Inauguration
February 14, 2025February 20, 2025Thursday12:00 p.m.President's Day
May 23, 2025May 29, 2025Thursday12:00 p.m.Memorial Day
August 29, 2025September 4, 2025Thursday12:00 p.m.Labor Day
October 10, 2025October 16, 2025Thursday12:00 p.m.Columbus Day
November 7, 2025November 13, 2025Thursday12:00 p.m.Veterans Day
December 19, 2025December 29, 2025Monday5:00 p.m.Christmas
January 16, 2026January 22, 2026Thursday12:00 p.m.Martin Luther King Jr. Day
February 13, 2026February 19, 2026Thursday12:00 p.m.President's Day
May 22, 2026May 28, 2026Thursday12:00 p.m.Memorial Day
September 4, 2026September 10, 2026Thursday12:00 p.m.Labor Day
October 9, 2026October 15, 2026Thursday12:00 p.m.Columbus Day
November 6, 2026November 12, 2026Thursday12:00 p.m.Veterans Day
diff --git a/backend/tests/test_calendar.py b/backend/tests/test_calendar.py index b16edc5..1157e25 100644 --- a/backend/tests/test_calendar.py +++ b/backend/tests/test_calendar.py @@ -282,7 +282,7 @@ def test_worker_initializes_all_sources_and_drops_expired_month_status(trading): worker = CalendarWorker() worker.sources["https://www.federalreserve.gov/old-month.htm"] = {"error": "old"} async def unavailable(source, url, *, query=None): - assert len(worker.sources) == 15 + assert len(worker.sources) == 17 assert all(s["error"] for s in worker.sources.values()) raise httpx.ConnectError("source unavailable") worker.fetch = unavailable diff --git a/backend/tests/test_dol.py b/backend/tests/test_dol.py new file mode 100644 index 0000000..d4ac0b9 --- /dev/null +++ b/backend/tests/test_dol.py @@ -0,0 +1,180 @@ +"""DOL regression coverage uses a public release extract and isolated I/O.""" +import asyncio +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from alphapulse.calendar import dol + +REPORT = (Path(__file__).parent / "fixtures/calendar/dol-current.txt").read_text() +SCHEDULE = """ +Publication Schedule: The UI Weekly Claims News Release is published each week on Thursday morning at 8:30am EST. + + +
Release DateRelease Time
Wednesday, November 25, 20268:30 AM EST
""" + + +def test_report_separates_statistical_weeks_and_revised_previous(): + initial, continued = (dol.report(REPORT)[family] for family in dol.FAMILIES) + assert initial.at == continued.at == datetime(2026, 10, 29, 12, 30, tzinfo=UTC) + assert initial.period == "2026-10-24" and continued.period == "2026-10-17" + assert initial.values["initial_claims"]["actual"] == "19.4" + assert initial.values["initial_claims"]["previous"] == "19.5" + assert continued.values["continued_claims"]["actual"] == "170.1" + assert continued.values["continued_claims"]["previous"] == "172.2" + assert "修订" in continued.values["continued_claims"]["previous_basis"] + + +def test_report_resolves_new_year_from_publication_date(): + text = REPORT.replace("October 29, 2026", "January 2, 2027").replace("October 24", "December 26").replace("October 17", "December 19") + result = dol.report(text) + assert result["initial_claims"].period == "2026-12-26" + assert result["continued_claims"].period == "2026-12-19" + assert result["initial_claims"].at == datetime(2027, 1, 2, 13, 30, tzinfo=UTC) + + +def test_historical_report_keeps_explicit_unrevised_previous(): + # Source: https://oui.doleta.gov/press/2025/010825.pdf (first two pages). + text = (Path(__file__).parent / "fixtures/calendar/dol-2025-01-08.txt").read_text() + result = dol.report(text) + assert result["initial_claims"].at == datetime(2025, 1, 8, 13, 30, tzinfo=UTC) + assert result["initial_claims"].values["initial_claims"]["actual"] == "20.1" + assert result["initial_claims"].values["initial_claims"]["previous"] == "21.1" + assert result["continued_claims"].period == "2024-12-28" + assert result["continued_claims"].values["continued_claims"]["previous"] == "183.4" + + +@pytest.mark.parametrize("text", [ + "Access denied", + REPORT.replace("SEASONALLY ADJUSTED DATA", "CHANGED HEADER"), + REPORT.replace("initial claims was 194,000", "initial claims pending"), + REPORT.replace("October 24", "September 1"), + REPORT.replace("195,000.", "unavailable."), +]) +def test_unrecognized_report_does_not_fabricate_values(text): + with pytest.raises(ValueError): + dol.report(text) + + +def test_schedule_respects_dst_and_thanksgiving_exception(): + events = dol.schedule(SCHEDULE, datetime(2026, 10, 30, tzinfo=UTC)) + initial = {item.at.date(): item for item in events if item.family == "initial_claims"} + assert initial[date(2026, 10, 29)].at.hour == 12 + assert initial[date(2026, 11, 5)].at.hour == 13 + assert initial[date(2026, 11, 25)].at.hour == 13 + assert date(2026, 11, 26) not in initial + assert all(item.period is None for item in events) + assert len({item.external_id for item in events}) == len(events) + + +@pytest.mark.parametrize("body", [ + SCHEDULE.replace("Thursday morning at 8:30am EST", "Schedule changed"), + SCHEDULE.replace("8:30 AM EST", "9:00 AM EST"), + SCHEDULE.replace("November 25, 2026", "11/25/2026"), + SCHEDULE.replace("Release Date", "Changed heading"), +]) +def test_changed_schedule_or_exception_format_fails_explicitly(body): + with pytest.raises(ValueError): + dol.schedule(body, datetime(2026, 10, 30, tzinfo=UTC)) + + +def test_archives_only_follow_official_links_and_requested_range(): + body = '''8Duplicate + Other host + Old''' + assert dol.archive_links(body, date(2025, 1, 1), date(2025, 12, 31)) == ["https://oui.doleta.gov/press/2025/010825.pdf"] + with pytest.raises(ValueError, match="历史覆盖尚未确认"): + dol.archive_links(body, date(2026, 1, 1), date(2026, 12, 31)) + with pytest.raises(ValueError, match="年份"): + dol.archive_links('Bad year', date(2025, 1, 1), date(2026, 12, 31)) + + +@pytest.mark.parametrize("offset,received,expected", [(0, False, "page"), (7, False, "pending"), (-7, True, "pending"), (-7, False, "error")]) +def test_live_result_never_applies_another_week(trading, monkeypatch, offset, received, expected): + from alphapulse.calendar import worker as module + + async def run(): + worker = module.CalendarWorker() + try: + response = httpx.Response(200, content=b"synthetic PDF", request=httpx.Request("GET", dol.LATEST)) + worker.fetch = AsyncMock(return_value=(response, 7)) + monkeypatch.setattr(module, "pdf_text", lambda body: REPORT) + at = datetime(2026, 10, 29, 12, 30, tzinfo=UTC) + timedelta(days=offset) + row = SimpleNamespace(source="dol", family="initial_claims", release_url=None, scheduled_at=at, + first_received_at=at if received else None) + if expected == "error": + with pytest.raises(ValueError, match="公告时间"): + await worker.result_page(row) + else: + page = await worker.result_page(row) + assert (page is not None) == (expected == "page") + finally: + await worker.http.aclose() + await worker.bls.aclose() + asyncio.run(run()) + + +def test_weekly_rows_show_their_own_week_and_uncertain_crypto_impact(trading): + from alphapulse.calendar.service import release_rows + announcement = dol.report(REPORT)["continued_claims"] + row = SimpleNamespace(id=1, family="continued_claims", source="dol", title="Weekly claims", + period=announcement.period, stage=None, scheduled_at=announcement.at, schedule_status="scheduled", + first_received_at=announcement.at, last_checked_at=announcement.at, values=announcement.values, + error=None, release_url=dol.LATEST, schedule_url=dol.SCHEDULE, revision_no=2) + [item] = release_rows(row, announcement.at) + assert item["actual"] == "170.1" and item["unit"] == "万人" + assert "2026-10-17" in item["event_name"] + assert item["impact"] == "uncertain" and item["forecast"] is None + + +def test_history_preview_includes_both_series_without_writing(trading, monkeypatch): + from alphapulse.calendar import backfill + + class Session: + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + async def execute(self, _): + return SimpleNamespace(all=lambda: []) + + async def scalars(self, _): + return [] + + doc = backfill.Document("dol", dol.LATEST, b"%PDF synthetic report", datetime(2026, 10, 30, tzinfo=UTC)) + monkeypatch.setattr(backfill, "SessionLocal", Session) + monkeypatch.setattr(backfill, "fetch_documents", AsyncMock(return_value=[doc])) + monkeypatch.setattr(backfill, "pdf_text", lambda body: REPORT) + save = AsyncMock() + monkeypatch.setattr(backfill, "save_announcement", save) + result = asyncio.run(backfill.run(date(2026, 10, 29), date(2026, 10, 29), sources={"dol"})) + assert result["mode"] == "dry-run" + assert result["announcements"] == {"initial_claims": 1, "continued_claims": 1} + assert not result["errors"] and not result["missing_announcements"] + save.assert_not_awaited() + + +def test_archive_year_gaps_are_explicit_and_current_report_is_still_collected(trading, monkeypatch): + from alphapulse.calendar import backfill + requests = [] + + def respond(request): + requests.append(str(request.url)) + if request.url.path.endswith("archive.asp"): + return httpx.Response(200, text='8') + return httpx.Response(200, content=b"%PDF synthetic report") + + client = httpx.AsyncClient + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: client(**kwargs, transport=httpx.MockTransport(respond))) + monkeypatch.setattr(backfill, "ensure_quiet_window", AsyncMock()) + monkeypatch.setattr(backfill.asyncio, "sleep", AsyncMock()) + errors = [] + documents = asyncio.run(backfill.fetch_documents(date(2025, 1, 1), date(2026, 10, 30), set(), errors, sources={"dol"})) + assert len(documents) == 2 and dol.LATEST in requests + assert len(errors) == 1 and "2026" in errors[0]["error"] and "不完整" in errors[0]["error"] diff --git a/backend/tests/test_eia.py b/backend/tests/test_eia.py new file mode 100644 index 0000000..1043895 --- /dev/null +++ b/backend/tests/test_eia.py @@ -0,0 +1,125 @@ +"""Verify EIA units, calendar exceptions, archives and reporting boundaries.""" +import asyncio +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from urllib.parse import urlparse + +import httpx +import pytest + +from alphapulse.calendar import eia + +FIXTURES = Path(__file__).parent / "fixtures/calendar" +CSV = (FIXTURES / "eia-2026-10-23.csv").read_text() +SCHEDULE = (FIXTURES / "eia-schedule.html").read_text() + + +def test_official_csv_preserves_precision_units_and_excludes_spr(): + report = eia.report(CSV, SCHEDULE) + assert report.at == datetime(2026, 10, 28, 14, 30, tzinfo=UTC) + assert report.period == "2026-10-23" + assert report.values["eia_crude"]["actual"] == "-189.200" + assert report.values["eia_gasoline"]["actual"] == "387.000" + assert report.values["eia_distillate"]["actual"] == "-146.200" + assert all(v["previous"] is None for v in report.values.values()) + + +def test_schedule_uses_holiday_times_and_eastern_dst(): + events = eia.schedule(SCHEDULE, datetime(2026, 10, 30, tzinfo=UTC)) + by_period = {item.period: item.at for item in events} + assert by_period["2026-10-23"] == datetime(2026, 10, 28, 14, 30, tzinfo=UTC) + assert by_period["2026-10-30"] == datetime(2026, 11, 4, 15, 30, tzinfo=UTC) + assert by_period["2026-11-06"] == datetime(2026, 11, 12, 17, tzinfo=UTC) + assert eia.release_at(date(2025, 12, 19), eia.exceptions(SCHEDULE)) == datetime(2025, 12, 29, 22, tzinfo=UTC) + + +@pytest.mark.parametrize("body", [ + CSV.replace('"Difference"', '"Changed heading"', 1), + CSV.replace('"10/16/26"', '"10/15/26"', 1), + CSV.replace('"Commercial (Excluding SPR)"', '"Other stock"'), + CSV.replace('"-1.892"', '"NaN"', 1), +]) +def test_unrecognized_csv_fails_instead_of_filling_other_data(body): + with pytest.raises(ValueError): + eia.report(body, SCHEDULE) + + +def test_following_supply_table_does_not_override_inventory_data(): + body = CSV + '\n"Commercial (Excluding SPR)",1,2,3,4,5,6,7' + assert eia.report(body, SCHEDULE).values == eia.report(CSV, SCHEDULE).values + + +@pytest.mark.parametrize("body", ["Access denied", SCHEDULE.replace("10:30 a.m. eastern time on Wednesday", "Changed schedule"), + SCHEDULE.replace("12:00 p.m.", "Time pending", 1)]) +def test_unrecognized_schedule_fails(body): + with pytest.raises(ValueError): + eia.exceptions(body) + + +def test_archive_discovery_follows_published_report_and_csv_links(): + url = "https://www.eia.gov/petroleum/supply/weekly/archive/2026/2026_10_28/wpsr_2026_10_28.php" + body = f'28Other host' + assert eia.archive_links(body, date(2026, 10, 1), date(2026, 10, 30)) == [url] + assert eia.csv_link('CSV', url) == url.rsplit("/", 1)[0] + "/download/table1.csv" + with pytest.raises(ValueError): + eia.archive_links(body, date(2025, 1, 1), date(2025, 12, 31)) + with pytest.raises(ValueError): + eia.csv_link('Other file', url) + + +@pytest.mark.parametrize("offset,received,expected", [(0, False, "page"), (7, False, "pending"), (-7, True, "pending"), (-7, False, "error")]) +def test_worker_does_not_apply_latest_values_to_another_week(trading, offset, received, expected): + from alphapulse.calendar.worker import CalendarWorker + + async def run(): + worker = CalendarWorker() + try: + worker.eia_schedule_body = SCHEDULE + response = httpx.Response(200, text=CSV, request=httpx.Request("GET", eia.LATEST)) + worker.fetch = AsyncMock(return_value=(response, 1)) + at = datetime(2026, 10, 28, 14, 30, tzinfo=UTC) + timedelta(days=offset) + row = SimpleNamespace(source="eia", family="eia_stocks", period="2026-10-23", release_url=None, + scheduled_at=at, first_received_at=at if received else None) + if expected == "error": + with pytest.raises(ValueError, match="所选日程不符"): + await worker.result_page(row) + else: + result = await worker.result_page(row) + assert (result is not None) == (expected == "page") + finally: + await worker.http.aclose() + await worker.bls.aclose() + asyncio.run(run()) + + +@pytest.mark.parametrize("archive_date,valid", [("2026_10_28", True), ("2026_10_21", False)]) +def test_history_preview_requires_matching_archive_date_without_writes(trading, monkeypatch, archive_date, valid): + from alphapulse.calendar import backfill + + class Session: + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + async def execute(self, _): + return SimpleNamespace(all=lambda: []) + + async def scalars(self, _): + return [] + + now = datetime(2026, 10, 30, tzinfo=UTC) + docs = [backfill.Document("eia", eia.SCHEDULE, SCHEDULE.encode(), now), + backfill.Document("eia", f"https://www.eia.gov/petroleum/supply/weekly/archive/2026/{archive_date}/download/table1.csv", CSV.encode(), now)] + monkeypatch.setattr(backfill, "SessionLocal", Session) + monkeypatch.setattr(backfill, "fetch_documents", AsyncMock(return_value=docs)) + save = AsyncMock() + monkeypatch.setattr(backfill, "save_announcement", save) + result = asyncio.run(backfill.run(date(2026, 10, 28), date(2026, 10, 28), sources={"eia"})) + assert result["mode"] == "dry-run" + assert result["announcements"].get("eia_stocks", 0) == int(valid) + assert bool(result["errors"]) != valid + save.assert_not_awaited() diff --git a/web/src/pages/calendar/index.vue b/web/src/pages/calendar/index.vue index 2ee7812..42669ba 100644 --- a/web/src/pages/calendar/index.vue +++ b/web/src/pages/calendar/index.vue @@ -23,7 +23,7 @@ const country = ref("") const impactLabels = { pending: "待公布", bullish: "利多", bearish: "利空", neutral: "影响较小", uncertain: "待判断" } const countries = computed(() => data.value?.countries ?? []) const labels = { scheduled: "待公布", waiting: "获取中", delayed: "取数延迟", released: "已公布", withdrawn: "已撤回", error: "采集异常", historical: "仅有日程", unavailable: "官方未公布" } -const sources: Record = { bls: "BLS", bea: "BEA", fed: "美联储", umich: "密歇根大学", ecb: "欧洲央行", eurostat: "Eurostat", boe: "英国央行", ons: "ONS", destatis: "Destatis", insee: "INSEE" } +const sources: Record = { bls: "BLS", bea: "BEA", fed: "美联储", umich: "密歇根大学", ecb: "欧洲央行", eurostat: "Eurostat", boe: "英国央行", ons: "ONS", destatis: "Destatis", insee: "INSEE", dol: "美国劳工部 DOL", eia: "美国能源信息署 EIA" } const formatted = (time: string) => dayjs(time).tz(zone).format("MM-DD HH:mm:ss") const value = (v: string | null, unit: string) => v === null ? "—" : `${v}${unit}` const groups = computed(() => {