Add authenticated global economic calendar with Jin10 fallback

Collect macroeconomic releases, events and holidays through the official browser SDK with resumable history from 2025. Preserve evidence, revisions and observation times, and merge verified series with official fields taking priority.

Add category filters, field provenance, coverage and connection status. Verify calendar persistence and deployment recovery in isolated acceptance checks.
This commit is contained in:
eezd committed 2026-10-31 12:57:14 +08:00
1 parent 57424fd661
commit e840557d22
35 files changed
+6828 -94

No files matched your search

@@ -0,0 +1,28 @@
"""Register the authenticated global calendar and resumable daily coverage."""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = "a8b1cd465ef3"
down_revision = "f7a0bc354de2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table("jin10_sync_days", sa.Column("day", sa.Date(), primary_key=True),
sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("counts", postgresql.JSONB(), nullable=False))
source = sa.table("sources", *[sa.column(key, kind) for key, kind in [
("code", sa.String), ("name", sa.String), ("list_url", sa.String), ("collector", sa.String),
("enabled", sa.Boolean), ("interval_seconds", sa.Integer), ("backfill_hours", sa.Integer),
("consecutive_failures", sa.Integer),
]])
op.bulk_insert(source, [{"code": "jin10", "name": "金十·全球宏观日历", "list_url": "https://rili.jin10.com/",
"collector": "calendar", "enabled": True, "interval_seconds": 10,
"backfill_hours": 72, "consecutive_failures": 0}])
def downgrade() -> None:
op.execute("DELETE FROM sources WHERE code = 'jin10'")
op.drop_table("jin10_sync_days")
+1
View File
@@ -16,6 +16,7 @@ dependencies = [
"httpx>=0.28.1",
"openai>=3.30.0",
"pillow>=13.0.0",
"playwright>=1.64.1,<2",
"pydantic-settings>=2.15.0",
"pyjwt>=2.15.1",
"pypdf>=6.21.2",
+35 -11
View File
@@ -9,11 +9,14 @@ from sqlalchemy.ext.asyncio import AsyncSession
from alphapulse.api.response import ApiResponse, success
from alphapulse.calendar.catalog import COUNTRIES, FAMILIES
from alphapulse.calendar.service import indicator_history, release_rows
from alphapulse.calendar.service import release_rows
from alphapulse.calendar.jin10_history import calendar_history
from alphapulse.calendar.jin10_view import merge_items
from alphapulse.core.auth import require_admin
from alphapulse.core.db import get_session
from alphapulse.core.redis import KEY_PREFIX, redis
from alphapulse.models import EconomicRelease, EconomicRevision, EconomicSeriesValue, RawResponse
from alphapulse.models.jin10 import Jin10Connection, Jin10SyncDay
router = APIRouter(prefix="/api/calendar", tags=["calendar"], dependencies=[Depends(require_admin)])
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
@@ -25,12 +28,32 @@ async def calendar_data(session: AsyncSession, start: datetime, end: datetime) -
).order_by(EconomicRelease.scheduled_at, EconomicRelease.id)))
raw = await redis.get(f"{KEY_PREFIX}calendar:status")
status = json.loads(raw) if raw else None
raw_jin10 = await redis.get(f"{KEY_PREFIX}calendar:jin10")
jin10_status = json.loads(raw_jin10) if raw_jin10 else None
if jin10_status:
connection = await session.get(Jin10Connection, 1)
if not connection or not connection.token_encrypted or jin10_status.get("connected_at") != connection.connected_at.isoformat():
jin10_status = {**jin10_status, "state": "disconnected", "error": None}
coverage = list(await session.scalars(select(Jin10SyncDay).where(
Jin10SyncDay.day >= start.astimezone(LOCAL_TZ).date(), Jin10SyncDay.day < end.astimezone(LOCAL_TZ).date(),
).order_by(Jin10SyncDay.day)))
now = datetime.now(UTC)
return {"items": [item for row in rows for item in release_rows(row, now)], "worker": status,
items = merge_items([(row, item) for row in rows if row.source != "jin10" for item in release_rows(row, now)],
[row for row in rows if row.source == "jin10"], now)
countries = {**COUNTRIES, **{item["country_code"]: item["country_name"] for item in items}}
return {"items": items, "worker": status,
"jin10": jin10_status,
"coverage": [{"date": day.day, "captured_at": day.captured_at, "counts": day.counts} for day in coverage],
"server_time": now, "timezone": "Asia/Shanghai",
"countries": [{"code": code, "name": name} for code, name in COUNTRIES.items()],
"forecast_available": any(row.values.get(key, {}).get("forecast") is not None
for row in rows for key in FAMILIES[row.family].indicators)}
"countries": [{"code": code, "name": name} for code, name in countries.items()],
"forecast_available": any(item["forecast"] is not None for item in items)}
def indicators(row: EconomicRelease) -> tuple[str, ...]:
if row.source == "jin10":
meta = row.values["_jin10"]
return (meta["indicator"],) if meta["kind"] == "data" else ()
return FAMILIES[row.family].indicators
@router.get("")
@@ -42,11 +65,12 @@ async def list_calendar(session: Annotated[AsyncSession, Depends(get_session)],
@router.get("/{release_id}/revisions")
async def revisions(release_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> ApiResponse[list[dict]]:
if await session.get(EconomicRelease, release_id) is None:
row = await session.get(EconomicRelease, release_id)
if row is None:
raise HTTPException(404, "日历事件不存在")
rows = await session.scalars(select(EconomicRevision).where(EconomicRevision.release_id == release_id)
.order_by(EconomicRevision.revision_no.desc()))
return success([{"revision_no": r.revision_no, "captured_at": r.captured_at,
return success([{"source": row.source, "revision_no": r.revision_no, "captured_at": r.captured_at,
"payload": r.payload, "raw_response_id": r.raw_response_id} for r in rows])
@@ -56,9 +80,9 @@ async def history(release_id: int, indicator: str, session: Annotated[AsyncSessi
row = await session.get(EconomicRelease, release_id)
if row is None:
raise HTTPException(404, "日历事件不存在")
if indicator not in FAMILIES[row.family].indicators:
if indicator not in indicators(row):
raise HTTPException(400, "指标不属于该事件")
return success(await indicator_history(session, row, indicator, limit))
return success(await calendar_history(session, row, indicator, limit))
@router.get("/{release_id}/series")
@@ -67,9 +91,9 @@ async def series(release_id: int, indicator: str, session: Annotated[AsyncSessio
row = await session.get(EconomicRelease, release_id)
if row is None:
raise HTTPException(404, "日历事件不存在")
if indicator not in FAMILIES[row.family].indicators:
if indicator not in indicators(row):
raise HTTPException(400, "指标不属于该事件")
if row.period is None:
if row.period is None or row.source == "jin10":
return success([])
# 此接口是当前官方序列,采集时间可以晚于所选事件;不得用作历史时点回测输入。
rows = (await session.execute(select(EconomicSeriesValue, RawResponse.url).join(
+12 -5
View File
@@ -119,7 +119,6 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
now = datetime.now(UTC)
raw = await redis.get(f"{KEY_PREFIX}calendar:status")
heartbeat = json.loads(raw) if raw else None
alive = bool(heartbeat and now - datetime.fromisoformat(heartbeat["at"]) < timedelta(seconds=30))
pending = EconomicRelease.first_received_at.is_(None) & (EconomicRelease.schedule_status == "scheduled")
published = EconomicRelease.first_received_at.is_not(None)
rows = (await session.execute(select(
@@ -138,18 +137,26 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
EconomicRelease.scheduled_at >= now - timedelta(days=45),
).order_by(EconomicRelease.last_checked_at.desc())):
result_errors.setdefault(source, error)
codes = {family.source for family in FAMILIES.values()}
codes = {family.source for family in FAMILIES.values()} | {"jin10"}
raw_jin10 = await redis.get(f"{KEY_PREFIX}calendar:jin10")
jin10_heartbeat = json.loads(raw_jin10) if raw_jin10 else None
sources = list(await session.scalars(select(Source).where(Source.code.in_(codes)).order_by(Source.code)))
output = []
for source in sources:
checks = [s for s in heartbeat["sources"].values() if s["source"] == source.code] if heartbeat else []
current_heartbeat = jin10_heartbeat if source.code == "jin10" else heartbeat
source_alive = bool(current_heartbeat and now - datetime.fromisoformat(current_heartbeat["at"]) < timedelta(seconds=30))
if source.code == "jin10":
checks = [{"source": "jin10", "checked_at": jin10_heartbeat["at"],
"error": jin10_heartbeat.get("error") or (None if jin10_heartbeat.get("state") in ("ready", "syncing") else "金十同步尚未就绪")}] if jin10_heartbeat else []
else:
checks = [s for s in heartbeat["sources"].values() if s["source"] == source.code] if heartbeat else []
errors = [s["error"] for s in checks if s["error"]]
if source.code in result_errors:
errors.append(result_errors[source.code])
data = stats.get(source.code, {})
output.append(CalendarSourceOut(
code=source.code, name=source.name, region_code=REGIONS[source.code], worker_alive=alive,
heartbeat_at=heartbeat["at"] if heartbeat else None,
code=source.code, name=source.name, region_code=REGIONS[source.code], worker_alive=source_alive,
heartbeat_at=current_heartbeat["at"] if current_heartbeat else None,
schedule_checked_at=max((s["checked_at"] for s in checks), default=None),
**{key: data.get(key) for key in ("schedule_success_at", "latest_result_at", "next_event_at", "history_start", "history_end")},
overdue_count=data.get("overdue_count", 0), errors=list(dict.fromkeys(errors)),
+12 -3
View File
@@ -1,5 +1,6 @@
"""Administrator-only Jin10 connection; credentials never leave the backend."""
from datetime import UTC, datetime, timedelta
import json
import secrets
from typing import Annotated
@@ -12,6 +13,7 @@ from alphapulse.calendar import jin10_auth
from alphapulse.core.auth import AuthSession, check_session, require_admin
from alphapulse.core.crypto import decrypt, encrypt
from alphapulse.core.db import get_session
from alphapulse.core.redis import KEY_PREFIX, redis
from alphapulse.models.jin10 import Jin10Connection
router = APIRouter(prefix="/api/settings/jin10", tags=["settings"], dependencies=[Depends(require_admin)])
@@ -33,14 +35,21 @@ def clear_task(row: Jin10Connection) -> None:
row.task_id = row.owner = row.qr_encrypted = row.expires_at = row.next_poll_at = None
def public_status(row: Jin10Connection) -> dict:
def public_status(row: Jin10Connection, sync: dict | None = None) -> dict:
current = bool(row.token_encrypted and row.connected_at and sync and sync.get("connected_at") == row.connected_at.isoformat()
and datetime.now(UTC) - datetime.fromisoformat(sync["at"]) < timedelta(seconds=30))
return {"connected": row.token_encrypted is not None, "connected_at": row.connected_at,
"error": row.error, "sync_state": "awaiting_validation" if row.token_encrypted else "disconnected"}
"error": row.error or (sync.get("error") if current else None),
"sync_state": sync["state"] if current else "awaiting_validation" if row.token_encrypted else "disconnected",
"last_success_at": sync.get("last_success_at") if current else None,
"history_next": sync.get("history_next") if current else None}
@router.get("")
async def status(session: Session) -> ApiResponse[dict]:
return success(public_status(await connection(session)))
row = await connection(session)
raw = await redis.get(f"{KEY_PREFIX}calendar:jin10")
return success(public_status(row, json.loads(raw) if raw else None))
@router.post("/qr", status_code=201)
+152
View File
@@ -0,0 +1,152 @@
"""Normalize authenticated Jin10 calendar responses without interpreting gold/USD signals."""
import base64
import binascii
import json
import re
import zlib
from dataclasses import dataclass
from datetime import UTC, date, datetime, time, timedelta
from zoneinfo import ZoneInfo
LOCAL = ZoneInfo("Asia/Shanghai")
START = date(2025, 1, 1)
KINDS = ("data", "event", "holiday")
COUNTRY_CODES = {
"美国": "US", "欧元区": "EA", "英国": "GB", "德国": "DE", "法国": "FR",
"中国": "CN", "中国香港": "HK", "中国台湾": "TW", "日本": "JP", "韩国": "KR",
"加拿大": "CA", "澳大利亚": "AU", "新西兰": "NZ", "瑞士": "CH", "瑞典": "SE",
"挪威": "NO", "意大利": "IT", "西班牙": "ES", "印度": "IN", "俄罗斯": "RU",
"巴西": "BR", "南非": "ZA", "新加坡": "SG", "乌克兰": "UA", "墨西哥": "MX",
"土耳其": "TR", "波兰": "PL", "印度尼西亚": "ID", "马来西亚": "MY", "泰国": "TH",
}
@dataclass(frozen=True)
class Entry:
identity: str
at: datetime
title: str
values: dict
def day_url(day: date) -> str:
return f"https://rili.jin10.com/day/{day.isoformat()}/"
def records(body: dict, kind: str, day: date) -> list[dict]:
if kind not in KINDS or not isinstance(body, dict) or body.get("date") != day.isoformat():
raise ValueError("金十响应日期或类别与请求不符")
rows = body.get("list")
if kind == "holiday":
# The public SDK's legacy daily holiday method returns zlib/base64 text.
if not isinstance(rows, str) or len(rows) > 2_000_000:
raise ValueError("金十假期响应格式已变化")
try:
decoder = zlib.decompressobj()
decoded = decoder.decompress(base64.b64decode(rows, validate=True), 4_000_000)
if not decoder.eof or decoder.unused_data or decoder.unconsumed_tail:
raise ValueError("金十假期压缩内容不完整或过大")
rows = json.loads(decoded)
except (ValueError, zlib.error, binascii.Error) as exc:
raise ValueError("金十假期内容无法解析") from exc
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise ValueError("金十日历列表格式已变化")
return rows
def text(row: dict, key: str, *, nullable: bool = False) -> str | None:
value = row.get(key)
if nullable and value is None:
return None
if not isinstance(value, str) or not value.strip():
raise ValueError(f"金十字段 {key} 缺失或类型错误")
return value.strip()
def identifier(row: dict, key: str) -> int:
value = row.get(key)
if type(value) is not int or value <= 0:
raise ValueError(f"金十字段 {key} 不是有效标识")
return value
def normalize(body: dict, kind: str, day: date) -> list[Entry]:
if day < START:
raise ValueError("金十历史范围不得早于2025-01-01北京时间")
result, identities = [], set()
for raw in records(body, kind, day):
identity = f"{kind}:{identifier(raw, 'data_id' if kind == 'data' else 'id')}"
if identity in identities:
raise ValueError("金十同一响应存在重复事件标识")
identities.add(identity)
country = text(raw, "country", nullable=True)
precision = "day" if kind == "holiday" else "minute"
if kind == "holiday":
if raw.get("date") != day.isoformat():
raise ValueError("金十假期记录日期与请求不符")
at = datetime.combine(day, time.min, LOCAL)
title = text(raw, "name")
description = " · ".join(v for v in [text(raw, "exchange_name"), text(raw, "rest_note", nullable=True),
text(raw, "remark", nullable=True)] if v)
else:
timestamp = text(raw, "pub_time" if kind == "data" else "event_time")
at = datetime.strptime(timestamp, "%Y-%m-%d %H:%M").replace(tzinfo=LOCAL)
if at.date() != day:
raise ValueError("金十记录公布日期与请求不符")
unix = raw.get("pub_time_unix" if kind == "data" else "event_time_unix")
if type(unix) is not int or not int(at.timestamp()) <= unix < int(at.timestamp()) + 60:
raise ValueError("金十公布时间与时间戳不符")
at = datetime.fromtimestamp(unix, LOCAL)
if raw.get("determine") not in (0, 1):
raise ValueError("金十时间确定性字段已变化")
if raw["determine"] == 1 or raw.get("time_status"):
precision = "tentative"
title = text(raw, "indicator_name" if kind == "data" else "event_content")
description = text(raw, "note", nullable=True) if kind == "event" else None
if kind != "data" and raw.get("category") != "cj":
raise ValueError("金十记录不属于宏观日历")
importance = raw.get("star") if kind != "holiday" else 0
if type(importance) is not int or not 0 <= importance <= 5:
raise ValueError("金十重要性字段已变化")
key = f"jin10_{identifier(raw, 'indicator_id')}" if kind == "data" else f"jin10_{kind}"
meta = {"kind": kind, "date": day.isoformat(), "precision": precision, "country_name": country or "未标注",
"country_code": COUNTRY_CODES.get(country, country or "UNKNOWN"),
"name": title, "description": description, "importance": importance,
"period_label": text(raw, "time_period", nullable=True) if kind == "data" else None,
"indicator": key, "unit": ("" if raw.get("unit") == "" else text(raw, "unit", nullable=True)) if kind == "data" else None,
"exchange": text(raw, "exchange_name") if kind == "holiday" else None}
values = {"_jin10": meta}
if kind == "data":
# Preserve nonnumeric values (including ranges); never coerce a missing value to zero.
values[key] = {"actual": text(raw, "actual", nullable=True), "previous": text(raw, "previous", nullable=True),
"forecast": text(raw, "consensus", nullable=True),
"previous_revised": text(raw, "revised", nullable=True)}
result.append(Entry(identity, at.astimezone(UTC), title, values))
return result
def period(label: str | None, at: datetime) -> str | None:
"""Resolve only verified monthly/weekly labels close to their publication date."""
local = at.astimezone(LOCAL).date()
month = re.fullmatch(r"(\d{1,2})月", label or "")
week = re.fullmatch(r"至(\d{1,2})月(\d{1,2})日", label or "")
if month:
number = int(month[1])
if not 1 <= number <= 12:
return None
year = local.year - (number > local.month)
return f"{year}-{number:02}"
if week:
try:
day = date(local.year, int(week[1]), int(week[2]))
if day > local:
day = day.replace(year=day.year - 1)
return day.isoformat() if local - day < timedelta(days=45) else None
except ValueError:
return None
quarter = re.fullmatch(r"第([一二三四])季度", label or "")
if quarter:
number = "一二三四".index(quarter[1]) + 1
year = local.year - (number > (local.month - 1) // 3 + 1)
return f"{year}-Q{number}"
return None
@@ -0,0 +1,78 @@
"""Use Jin10's own browser SDK rather than reproducing its encrypted transport."""
import asyncio
from datetime import date
from playwright.async_api import Error as BrowserError, async_playwright
from alphapulse.calendar.jin10 import KINDS, day_url, normalize
from alphapulse.calendar.jin10_auth import Jin10Error, uc_request
METHODS = {"data": "getCalendarData", "event": "getCalendarEvent", "holiday": "getCalendarHoliday"}
VALIDATION_DAY = date(2025, 1, 8)
def validate_access(responses: dict) -> None:
# This archived day has verified economic releases and events. An authenticated
# account alone does not prove that the SDK can read the economic calendar.
normalized = {kind: normalize(responses[kind], kind, VALIDATION_DAY) for kind in KINDS}
if not normalized["data"] or not normalized["event"]:
raise Jin10Error("金十历史日历读取权限未通过验证,请重新连接后检查账号权限")
class Jin10Browser:
def __init__(self) -> None:
self.playwright = self.browser = self.context = self.page = None
self.credential = None
self.lock = asyncio.Lock()
async def close(self) -> None:
if self.browser:
await self.browser.close()
if self.playwright:
await self.playwright.stop()
self.playwright = self.browser = self.context = self.page = None
self.credential = None
async def connect(self, credential: str) -> None:
await self.close()
info = await uc_request("/userinfo", token=credential)
if not isinstance(info.get("data"), dict) or not info["data"]:
raise Jin10Error("金十登录态已失效,请重新扫码")
try:
self.playwright = await async_playwright().start()
self.browser = await self.playwright.chromium.launch(headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"])
self.context = await self.browser.new_context()
await self.context.add_cookies([{"name": "x-token", "value": credential, "domain": ".jin10.com",
"path": "/", "secure": True, "sameSite": "Lax"}])
self.page = await self.context.new_page()
# Keep the host page outside the collection window so its initial requests do not compete.
await self.page.goto(day_url(date(2024, 1, 1)), wait_until="domcontentloaded", timeout=30000)
await self.page.wait_for_function(
"window.Jin10FlashInstance && window.Jin10FlashInstance.socket && "
"window.Jin10FlashInstance.socket.readyState === 1", timeout=30000)
await self.page.wait_for_timeout(2000)
self.credential = credential
validate_access(await self.fetch_day(VALIDATION_DAY))
except BrowserError:
await self.close()
raise Jin10Error("金十浏览器连接失败,请检查网络与浏览器运行环境") from None
async def fetch_day(self, day: date) -> dict[str, dict]:
async with self.lock:
if self.page is None:
raise Jin10Error("金十浏览器尚未连接")
try:
result = {}
for kind in KINDS:
result[kind] = await self.page.evaluate(
"async ({method, day}) => {"
"if (!window.Jin10FlashInstance || window.Jin10FlashInstance.socket.readyState !== 1) throw Error('disconnected');"
"return await window.Jin10FlashInstance[method](day, {timeout: 15000});}",
{"method": METHODS[kind], "day": day.isoformat()})
info = await uc_request("/userinfo", token=self.credential)
if not isinstance(info.get("data"), dict) or not info["data"]:
raise Jin10Error("金十登录态已失效,未将空列表记为完整日历")
return result
except BrowserError:
# Browser exceptions may include URLs or script contents; never forward them to logs.
raise Jin10Error("金十日历请求失败或超时,未将本次响应记为成功") from None
@@ -0,0 +1,73 @@
"""Read the same official/provider merge for history without rewriting either source."""
from datetime import UTC, datetime
from decimal import Decimal, InvalidOperation
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from alphapulse.calendar.catalog import FAMILIES
from alphapulse.calendar.jin10_view import MAPPINGS, definition, merge_items
from alphapulse.calendar.service import indicator_history, release_rows
from alphapulse.models import EconomicRelease, EconomicRevision
def changed(initial: str | None, current: str) -> bool:
if initial is None:
return False
try:
return Decimal(initial) != Decimal(current)
except InvalidOperation:
return initial != current
async def calendar_history(session: AsyncSession, selected: EconomicRelease, indicator: str, limit: int) -> list[dict]:
if selected.source == "jin10":
match = definition(selected.values["_jin10"])
if match is None:
return await indicator_history(session, selected, indicator, limit)
family, key, stage = match
else:
family, key, stage = selected.family, indicator, selected.stage
# IDs and exact definitions are verified by the normal read merge as well.
provider_keys = [f"jin10_{id_}" for mapping in MAPPINGS.values() for (id_, _), value in mapping.items()
if value == (family, key, stage)]
if not provider_keys:
return await indicator_history(session, selected, indicator, limit)
candidates = list(await session.scalars(select(EconomicRelease).where(
EconomicRelease.scheduled_at <= selected.scheduled_at,
EconomicRelease.schedule_status != "withdrawn",
or_((EconomicRelease.source == FAMILIES[family].source) & (EconomicRelease.family == family)
& EconomicRelease.stage.is_not_distinct_from(stage),
(EconomicRelease.source == "jin10") & EconomicRelease.values["_jin10"]["indicator"].as_string().in_(provider_keys)),
).order_by(EconomicRelease.scheduled_at.desc(), EconomicRelease.id.desc()).limit(limit * 2)))
now = datetime.now(UTC)
official = [(row, item) for row in candidates if row.source != "jin10" for item in release_rows(row, now) if item["indicator"] == key]
extras = [row for row in candidates if row.source == "jin10" and definition(row.values["_jin10"]) == (family, key, stage)]
items = [item for item in merge_items(official, extras, now) if item["actual"] is not None]
items.sort(key=lambda r: (r["scheduled_at"], r["release_id"]), reverse=True)
items = items[:limit]
by_id = {row.id: row for row in candidates}
revisions = list(await session.scalars(select(EconomicRevision).where(
EconomicRevision.release_id.in_(by_id),
).order_by(EconomicRevision.release_id, EconomicRevision.revision_no)))
first = {}
for revision in revisions:
owner = by_id[revision.release_id]
value_key = owner.values["_jin10"]["indicator"] if owner.source == "jin10" else key
actual = revision.payload["values"].get(value_key, {}).get("actual")
if actual is not None:
first.setdefault(owner.id, actual)
output = []
for item in items:
owner = by_id[item["release_id"]]
if item["field_sources"]["actual"]["source"] == "jin10" and owner.source != "jin10":
owner = by_id[item["related_release_id"]]
initial = first.get(owner.id)
output.append({"release_id": item["release_id"], "scheduled_at": item["scheduled_at"], "title": item["title"],
"period": item["period"], "period_label": item["period_label"], "stage": item["stage"],
"actual": item["actual"], "previous": item["previous"], "forecast": item["forecast"],
"forecast_before_release": item["forecast_before_release"], "forecast_observed_at": item["forecast_observed_at"],
"first_recorded_actual": initial, "first_received_at": owner.first_received_at,
"release_url": item["field_sources"]["actual"]["url"], "field_sources": item["field_sources"],
"revised": changed(initial, item["actual"])})
return output
@@ -0,0 +1,72 @@
"""Keep provider identities, raw evidence, field observation times and revisions together."""
from datetime import date, datetime
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from alphapulse.calendar.jin10 import Entry, day_url
from alphapulse.calendar.service import record_revision
from alphapulse.models import EconomicRelease
from alphapulse.models.jin10 import Jin10SyncDay
async def save_entry(session: AsyncSession, entry: Entry, raw_id: int, now: datetime) -> None:
row = await session.scalar(select(EconomicRelease).where(
EconomicRelease.source == "jin10", EconomicRelease.external_id == entry.identity,
).with_for_update())
values = {key: dict(value) for key, value in entry.values.items()}
meta = values["_jin10"]
key = meta["indicator"]
if meta["kind"] == "data":
current = values[key]
old = row.values.get(key, {}) if row else {}
observed = {}
for field in ("actual", "previous", "forecast", "previous_revised"):
if current[field] is not None:
observed[field] = old.get("observed_at", {}).get(field, now.isoformat()) if current[field] == old.get(field) else now.isoformat()
current["observed_at"] = observed
forecast_at = datetime.fromisoformat(observed["forecast"]) if "forecast" in observed else None
if forecast_at == now:
before = current["actual"] is None
else:
before = old.get("forecast_before_release", False)
current["forecast_before_release"] = bool(before and forecast_at and forecast_at < entry.at and meta["precision"] == "minute")
received = meta["kind"] != "data" or values[key]["actual"] is not None
if row is None:
row = EconomicRelease(source="jin10", external_id=entry.identity, family="jin10", title=entry.title,
scheduled_at=entry.at, schedule_status="scheduled", schedule_url=day_url(date.fromisoformat(meta["date"])),
release_url=None, period=None, stage=None, schedule_checked_at=now, last_checked_at=now,
first_received_at=now if received else None, next_check_at=now, error=None,
values=values, revision_no=1)
session.add(row)
await session.flush()
record_revision(session, row, raw_id)
return
changed = row.values != values or row.scheduled_at != entry.at or row.title != entry.title
row.schedule_checked_at = row.last_checked_at = now
row.error = None
if row.first_received_at is None and received:
row.first_received_at = now
changed = True
if changed:
row.title, row.scheduled_at, row.values = entry.title, entry.at, values
row.schedule_url = day_url(date.fromisoformat(meta["date"]))
row.revision_no += 1
record_revision(session, row, raw_id)
async def mark_day(session: AsyncSession, day: date, counts: dict, now: datetime) -> None:
stmt = insert(Jin10SyncDay).values(day=day, captured_at=now, counts=counts)
await session.execute(stmt.on_conflict_do_update(index_elements=["day"], set_={"captured_at": now, "counts": counts}))
async def next_history_day(session: AsyncSession, start: date, end: date) -> date | None:
from datetime import timedelta
covered = set(await session.scalars(select(Jin10SyncDay.day).where(Jin10SyncDay.day.between(start, end))))
day = start
while day <= end:
if day not in covered:
return day
day += timedelta(days=1)
return None
@@ -0,0 +1,206 @@
"""Join only verified series definitions; official values win independently for each field."""
from copy import deepcopy
from datetime import datetime
from decimal import Decimal, InvalidOperation
from alphapulse.calendar.catalog import FAMILIES, INDICATORS
from alphapulse.calendar.jin10 import LOCAL, period
from alphapulse.calendar.service import impact
from alphapulse.models import EconomicRelease
# (Jin10 indicator, exact name) -> (official family, indicator, release stage).
# CPI annual rates are NSA, monthly rates SA; other adjustments must not be substituted.
US = {
(234, "未季调CPI年率"): ("cpi", "cpi_yoy", None),
(230, "季调后CPI月率"): ("cpi", "cpi_mom", None),
(232, "未季调核心CPI年率"): ("cpi", "core_cpi_yoy", None),
(233, "季调后核心CPI月率"): ("cpi", "core_cpi_mom", None),
(178, "PPI年率"): ("ppi", "ppi_yoy", None),
(177, "PPI月率"): ("ppi", "ppi_mom", None),
(75, "季调后非农就业人口"): ("jobs", "payrolls", None),
(76, "失业率"): ("jobs", "unemployment", None),
(212, "PCE物价指数年率"): ("pce", "pce_yoy", None),
(213, "PCE物价指数月率"): ("pce", "pce_mom", None),
(211, "核心PCE物价指数年率"): ("pce", "core_pce_yoy", None),
(214, "核心PCE物价指数月率"): ("pce", "core_pce_mom", None),
(144, "实际GDP年化季率修正值"): ("gdp", "gdp", "second"),
(872, "实际GDP年化季率终值"): ("gdp", "gdp", "third"),
(198, "密歇根大学消费者信心指数初值"): ("umich", "umich_sentiment", "preliminary"),
(208, "密歇根大学消费者信心指数终值"): ("umich", "umich_sentiment", "final"),
(216, "实际GDP年化季率初值"): ("gdp", "gdp", "advance"),
(77, "美联储利率决定(上限)"): ("fomc", "fed_rate", None),
(78, "ADP就业人数"): ("adp_jobs", "adp_jobs", None),
(79, "当周初请失业金人数"): ("initial_claims", "initial_claims", None),
(96, "当周续请失业金人数"): ("continued_claims", "continued_claims", None),
(111, "当周EIA原油库存"): ("eia_stocks", "eia_crude", None),
(114, "当周EIA汽油库存"): ("eia_stocks", "eia_gasoline", None),
(113, "当周EIA精炼油库存"): ("eia_stocks", "eia_distillate", None),
}
EA = {
(763, "GDP季率初值"): ("ea_gdp", "gdp_qoq", "preliminary"),
(762, "GDP年率初值"): ("ea_gdp", "gdp_yoy", "preliminary"),
(930, "CPI年率初值"): ("ea_hicp", "hicp_yoy", "flash"),
(413, "CPI月率初值"): ("ea_hicp", "hicp_mom", "flash"),
(759, "核心CPI年率初值"): ("ea_hicp", "core_hicp_yoy", "flash"),
(764, "CPI年率终值"): ("ea_hicp", "hicp_yoy", "final"),
(8125, "CPI月率终值"): ("ea_hicp", "hicp_mom", "final"),
(622, "核心CPI年率终值"): ("ea_hicp", "core_hicp_yoy", "final"),
(396, "失业率"): ("ea_jobs", "ilo_unemployment", None),
(395, "欧洲央行存款机制利率"): ("ecb_rate", "ecb_deposit", None),
(753, "欧洲央行主要再融资利率"): ("ecb_rate", "ecb_main", None),
(419, "欧洲央行边际贷款利率"): ("ecb_rate", "ecb_marginal", None),
}
FR = {
(831, "GDP年率初值"): ("fr_gdp", "gdp_yoy", "initial"),
(829, "GDP年率终值"): ("fr_gdp", "gdp_yoy", "final"),
(830, "GDP季率终值"): ("fr_gdp", "gdp_qoq", "final"),
(8111, "CPI年率初值"): ("fr_cpi", "eu_cpi_yoy", "preliminary"),
(8112, "CPI月率初值"): ("fr_cpi", "eu_cpi_mom", "preliminary"),
(8113, "调和CPI年率初值"): ("fr_cpi", "hicp_yoy", "preliminary"),
(8114, "调和CPI月率初值"): ("fr_cpi", "hicp_mom", "preliminary"),
(567, "CPI年率终值"): ("fr_cpi", "eu_cpi_yoy", "final"),
(568, "CPI月率终值"): ("fr_cpi", "eu_cpi_mom", "final"),
(569, "调和CPI年率终值"): ("fr_cpi", "hicp_yoy", "final"),
(570, "调和CPI月率终值"): ("fr_cpi", "hicp_mom", "final"),
(832, "GDP季率初值"): ("fr_gdp", "gdp_qoq", "initial"),
}
DE = {
(270, "季调后GDP季率终值"): ("de_gdp", "gdp_qoq", "final"),
(842, "工作日调整后GDP年率终值"): ("de_gdp", "gdp_yoy", "final"),
(846, "CPI年率初值"): ("de_cpi", "eu_cpi_yoy", "preliminary"),
(847, "CPI月率初值"): ("de_cpi", "eu_cpi_mom", "preliminary"),
(839, "调和CPI年率初值"): ("de_cpi", "hicp_yoy", "preliminary"),
(840, "调和CPI月率初值"): ("de_cpi", "hicp_mom", "preliminary"),
(844, "CPI年率终值"): ("de_cpi", "eu_cpi_yoy", "final"),
(845, "CPI月率终值"): ("de_cpi", "eu_cpi_mom", "final"),
(837, "调和CPI年率终值"): ("de_cpi", "hicp_yoy", "final"),
(838, "调和CPI月率终值"): ("de_cpi", "hicp_mom", "final"),
(245, "季调后GDP季率初值"): ("de_gdp", "gdp_qoq", "initial"),
(843, "工作日调整后GDP年率初值"): ("de_gdp", "gdp_yoy", "initial"),
}
GB = {
(649, "三个月ILO失业率"): ("uk_jobs", "uk_unemployment", None),
(390, "GDP年率初值"): ("uk_gdp", "gdp_yoy", "initial"),
(391, "GDP季率初值"): ("uk_gdp", "gdp_qoq", "initial"),
(385, "GDP年率终值"): ("uk_gdp", "gdp_yoy", "final"),
(658, "GDP季率终值"): ("uk_gdp", "gdp_qoq", "final"),
(344, "CPI年率"): ("uk_cpi", "eu_cpi_yoy", None),
(348, "CPI月率"): ("uk_cpi", "eu_cpi_mom", None),
(349, "核心CPI年率"): ("uk_cpi", "eu_core_cpi_yoy", None),
(638, "央行利率决定"): ("boe_rate", "bank_rate", None),
}
MAPPINGS = {"US": US, "EA": EA, "FR": FR, "DE": DE, "GB": GB}
def definition(meta: dict) -> tuple[str, str, str | None] | None:
if meta["kind"] != "data" or meta["precision"] != "minute":
return None
mapping = MAPPINGS.get(meta["country_code"], {})
match = mapping.get((int(meta["indicator"].removeprefix("jin10_")), meta["name"]))
# Jin10 omits units for the verified Michigan index, whose official series is an index in points.
unit = "点" if meta["unit"] is None and meta["country_code"] == "US" and meta["indicator"] in ("jin10_198", "jin10_208") else meta["unit"]
return match if match and unit == INDICATORS[match[1]].unit else None
def provenance(source: str, at: str | datetime | None, url: str) -> dict:
return {"source": source, "observed_at": at, "url": url}
def jin10_item(row: EconomicRelease, now: datetime) -> dict:
meta = row.values["_jin10"]
key = meta["indicator"]
value = row.values.get(key, {})
actual, previous, forecast = value.get("actual"), value.get("previous"), value.get("forecast")
revised = value.get("previous_revised")
match = definition(meta)
eligible = value.get("forecast_before_release", False)
if meta["kind"] != "data":
status = "recorded"
direction, explanation = "uncertain", "日程记录不代表事件已经发生;需结合实际消息判断 BTC/ETH 影响。"
else:
status = "released" if actual is not None else "historical" if (now - row.scheduled_at).days > 45 else "scheduled" if now < row.scheduled_at else "delayed"
direction, explanation = direction_for(match[1] if match else None, actual, revised if revised is not None else previous, forecast if eligible else None)
observed = value.get("observed_at", {})
fields = {field: provenance("jin10", observed.get(field), row.schedule_url)
for field in ("actual", "previous", "forecast", "previous_revised") if value.get(field) is not None}
fields["scheduled_at"] = provenance("jin10", row.schedule_checked_at, row.schedule_url)
if revised is not None:
if "previous" in fields:
fields["previous_original"] = fields["previous"]
fields["previous"] = fields["previous_revised"]
return {"release_id": row.id, "indicator": key, "name": meta["name"], "unit": meta["unit"],
"country_code": meta["country_code"], "country_name": meta["country_name"],
"event_name": f"{meta['country_name']}{meta['period_label'] or ''}{meta['name']}",
"period": None, "period_label": meta["period_label"], "stage": None, "stage_name": None,
"title": meta["description"] or meta["name"], "source": "jin10", "scheduled_at": row.scheduled_at,
"calendar_date": meta["date"], "time_precision": meta["precision"], "kind": meta["kind"],
"importance": meta["importance"], "importance_source": "jin10", "status": status,
"actual": actual, "previous": revised if revised is not None else previous,
"previous_original": previous, "previous_revised": revised, "forecast": forecast,
"previous_basis": "金十列示的修订前值(原前值单独保留)" if revised is not None else "金十本次记录列示前值" if previous is not None else None,
"forecast_before_release": eligible, "forecast_observed_at": observed.get("forecast"),
"impact": direction, "explanation": explanation, "impact_basis": "forecast" if forecast is not None and eligible else "previous" if previous is not None or revised is not None else None,
"release_url": None, "schedule_url": row.schedule_url, "first_received_at": row.first_received_at,
"last_checked_at": row.last_checked_at, "error": row.error, "revision_no": row.revision_no,
"field_sources": fields, "related_release_id": None, "related_revision_no": None}
def direction_for(key: str | None, actual: str | None, previous: str | None, forecast: str | None) -> tuple[str, str]:
if key is None:
return "uncertain", "尚无经核实的 BTC/ETH 传导判断;金十的黄金/美元影响标签未用于加密资产。"
try:
if any(not Decimal(v).is_finite() for v in (actual, previous, forecast) if v is not None):
raise InvalidOperation()
return impact(key, actual, previous, forecast)
except InvalidOperation:
return "uncertain", "公布值或比较基准不是单一数值,暂不能判断 BTC/ETH 方向。"
def merge_items(official: list[tuple[EconomicRelease, dict]], third_party: list[EconomicRelease], now: datetime) -> list[dict]:
output = []
used = set()
for row, original in official:
item = deepcopy(original)
item.update(kind="data", time_precision="minute", calendar_date=row.scheduled_at.astimezone(LOCAL).date().isoformat(),
importance_source="project", field_sources={field: provenance(row.source, row.first_received_at, row.release_url or row.schedule_url)
for field in ("actual", "previous", "forecast") if item[field] is not None},
related_release_id=None, related_revision_no=None, period_label=None, previous_original=None, previous_revised=None,
forecast_before_release=False, forecast_observed_at=None)
item["field_sources"]["scheduled_at"] = provenance(row.source, row.schedule_checked_at, row.schedule_url)
matches = []
for extra in third_party:
meta = extra.values["_jin10"]
match = definition(meta)
if not match or match != (row.family, item["indicator"], row.stage):
continue
if FAMILIES[row.family].source != row.source or extra.scheduled_at != row.scheduled_at:
continue
# Rate decisions have no statistical reference period. All data series must agree on it.
if row.family not in ("fomc", "ecb_rate", "boe_rate") and (row.period is None or period(meta["period_label"], extra.scheduled_at) != row.period):
continue
matches.append(extra)
if len(matches) == 1:
extra = matches[0]
fallback = jin10_item(extra, now)
used.add(extra.id)
item.update(related_release_id=extra.id, related_revision_no=extra.revision_no,
previous_original=fallback["previous_original"], previous_revised=fallback["previous_revised"],
forecast_before_release=fallback["forecast_before_release"], forecast_observed_at=fallback["forecast_observed_at"])
for field in ("actual", "previous", "forecast"):
if item[field] is None and fallback[field] is not None:
item[field] = fallback[field]
provider_field = "previous_revised" if field == "previous" and fallback["previous_revised"] is not None else field
item["field_sources"][field] = fallback["field_sources"][provider_field]
if field == "previous":
item["previous_basis"] = fallback["previous_basis"]
if field == "actual":
item["first_received_at"] = fallback["first_received_at"]
if item["actual"] is not None and item["status"] != "withdrawn":
item["status"] = "released"
forecast = item["forecast"] if item.get("forecast_before_release") else None
item["impact"], item["explanation"] = direction_for(item["indicator"], item["actual"], item["previous"], forecast)
item["impact_basis"] = "forecast" if forecast is not None else "previous" if item["previous"] is not None else None
output.append(item)
output.extend(jin10_item(row, now) for row in third_party if row.id not in used)
return sorted(output, key=lambda r: (r["calendar_date"], r["scheduled_at"], r["release_id"], r["indicator"]))
@@ -0,0 +1,140 @@
"""Independent browser collector: live dates take priority over resumable daily history."""
import asyncio
import json
import logging
import time
from datetime import UTC, date, datetime, timedelta
from sqlalchemy import select
from alphapulse.calendar.jin10 import KINDS, LOCAL, START, day_url, normalize
from alphapulse.calendar.jin10_auth import Jin10Error
from alphapulse.calendar.jin10_browser import Jin10Browser
from alphapulse.calendar.jin10_store import mark_day, next_history_day, save_entry
from alphapulse.calendar.service import save_raw
from alphapulse.core.crypto import decrypt
from alphapulse.core.db import SessionLocal
from alphapulse.core.redis import KEY_PREFIX, redis
from alphapulse.models import Source
from alphapulse.models.jin10 import Jin10Connection
logger = logging.getLogger("alphapulse.calendar.jin10")
STATUS_KEY = f"{KEY_PREFIX}calendar:jin10"
class ConnectionChanged(Exception):
pass
def same_connection(row: Jin10Connection | None, encrypted: bytes, connected_at: datetime) -> bool:
return bool(row and row.token_encrypted == encrypted and row.connected_at == connected_at)
async def persist_day(day: date, responses: dict, encrypted: bytes, connected_at: datetime, now: datetime) -> dict:
raw_ids = {}
async with SessionLocal() as session:
# Disconnect/reconnect takes this same lock. An obsolete fetch must never commit data or progress.
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
if not same_connection(row, encrypted, connected_at):
raise ConnectionChanged()
for kind in KINDS:
raw = json.dumps(responses[kind], ensure_ascii=False, separators=(",", ":")).encode()
raw_ids[kind] = await save_raw(session, "jin10", day_url(day) + f"#{kind}", 200,
{"content-type": "application/json", "transport": "jin10-official-browser-sdk"}, raw, now)
await session.commit()
# Retain failed contract responses as evidence, but never advance coverage for them.
entries = {kind: normalize(responses[kind], kind, day) for kind in KINDS}
async with SessionLocal() as session:
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
if not same_connection(row, encrypted, connected_at):
raise ConnectionChanged()
for kind in KINDS:
for entry in entries[kind]:
await save_entry(session, entry, raw_ids[kind], now)
counts = {kind: len(entries[kind]) for kind in KINDS}
await mark_day(session, day, counts, now)
await session.commit()
return counts
class Jin10Worker:
def __init__(self) -> None:
self.browser = Jin10Browser()
self.identity = None
self.live_due: dict[date, float] = {}
self.state = {"state": "disconnected", "error": None, "last_success_at": None}
async def heartbeat(self) -> None:
while True:
await redis.set(STATUS_KEY, json.dumps({**self.state, "at": datetime.now(UTC).isoformat()}), ex=60)
await asyncio.sleep(5)
async def step(self) -> None:
async with SessionLocal() as session:
row = await session.get(Jin10Connection, 1)
source = await session.get(Source, "jin10")
identity = (row.token_encrypted, row.connected_at) if row and row.token_encrypted and source and source.enabled else None
if identity is None:
await self.browser.close()
self.identity = None
self.state.update(state="disconnected", error=None, connected_at=None)
await asyncio.sleep(5)
return
if identity != self.identity:
self.state = {"state": "connecting", "error": None, "connected_at": identity[1].isoformat(), "last_success_at": None}
await self.browser.connect(decrypt(identity[0]))
self.identity = identity
self.live_due.clear()
today = datetime.now(LOCAL).date()
dates = [today + timedelta(days=i) for i in range(-1, 8)]
self.live_due = {d: self.live_due.get(d, 0) for d in dates if d >= START}
due = sorted((deadline, d) for d, deadline in self.live_due.items() if deadline <= time.monotonic())
if due:
# Refresh today first when a process starts, then adjacent days in deadline order.
day = today if self.live_due.get(today, float("inf")) == 0 else due[0][1]
history = False
else:
async with SessionLocal() as session:
day = await next_history_day(session, START, today - timedelta(days=1))
history = True
if day is None:
self.state.update(state="ready", history_next=None)
await asyncio.sleep(2)
return
self.state.update(state="syncing", current_day=day.isoformat(), history=history, error=None)
responses = await self.browser.fetch_day(day)
counts = await persist_day(day, responses, *identity, datetime.now(UTC))
if not history:
self.live_due[day] = time.monotonic() + (10 if day == today else 300)
self.state.update(state="ready", last_success_at=datetime.now(UTC).isoformat(), counts=counts)
if history:
self.state["history_next"] = (day + timedelta(days=1)).isoformat()
# A single collector owns all retries and limits upstream traffic, including backfill.
await asyncio.sleep(5)
async def collect(self) -> None:
while True:
try:
await self.step()
except ConnectionChanged:
self.identity = None
self.state.update(state="disconnected", error=None)
except (Jin10Error, ValueError, KeyError) as exc:
self.state.update(state="error", error=str(exc))
logger.warning("金十日历同步失败:%s", exc)
self.identity = None
await self.browser.close()
await asyncio.sleep(30)
async def run(self) -> None:
try:
await asyncio.gather(self.heartbeat(), self.collect())
finally:
await self.browser.close()
await redis.aclose()
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
logging.getLogger("httpx").setLevel(logging.WARNING)
asyncio.run(Jin10Worker().run())
+23 -7
View File
@@ -170,6 +170,9 @@ def impact(key: str, actual: str | None, previous: str | None, forecast: str | N
def release_rows(row: EconomicRelease, now: datetime) -> list[dict]:
if row.source == "jin10":
from alphapulse.calendar.jin10_view import jin10_item
return [jin10_item(row, now)]
if row.schedule_status == "withdrawn":
state = "withdrawn"
elif row.first_received_at is not None:
@@ -221,17 +224,30 @@ async def indicator_history(session: AsyncSession, row: EconomicRelease, key: st
.where(EconomicRevision.release_id == EconomicRelease.id,
EconomicRevision.payload["values"][key]["actual"].as_string().is_not(None))
.order_by(EconomicRevision.revision_no).limit(1).correlate(EconomicRelease).scalar_subquery())
filters = []
if row.source == "jin10":
meta = row.values["_jin10"]
filters = [EconomicRelease.values["_jin10"][field].as_string().is_not_distinct_from(meta[field])
for field in ("country_code", "name", "unit")]
rows = (await session.execute(select(EconomicRelease, first_actual).where(
EconomicRelease.source == row.source, EconomicRelease.family == row.family,
EconomicRelease.stage.is_not_distinct_from(row.stage),
EconomicRelease.scheduled_at <= row.scheduled_at,
EconomicRelease.schedule_status != "withdrawn", EconomicRelease.first_received_at.is_not(None),
EconomicRelease.values[key]["actual"].as_string().is_not(None),
*filters,
).order_by(EconomicRelease.scheduled_at.desc(), EconomicRelease.id.desc()).limit(limit))).all()
return [{"release_id": item.id, "scheduled_at": item.scheduled_at, "title": item.title,
"period": item.period, "stage": item.stage,
"actual": item.values[key]["actual"], "previous": item.values[key].get("previous"),
"forecast": item.values[key].get("forecast"), "first_recorded_actual": initial,
"first_received_at": item.first_received_at, "release_url": item.release_url,
"revised": initial is not None and Decimal(initial) != Decimal(item.values[key]["actual"])}
for item, initial in rows]
output = []
for item, initial in rows:
value = item.values[key]
view = release_rows(item, datetime.now(item.scheduled_at.tzinfo))[0] if item.source == "jin10" else None
output.append({"release_id": item.id, "scheduled_at": item.scheduled_at, "title": item.title,
"period": item.period, "stage": item.stage,
"actual": value["actual"], "previous": view["previous"] if view else value.get("previous"),
"forecast": value.get("forecast"), "first_recorded_actual": initial,
"first_received_at": item.first_received_at, "release_url": item.schedule_url if view else item.release_url,
"period_label": item.values["_jin10"]["period_label"] if view else None,
"forecast_before_release": value.get("forecast_before_release", False),
"forecast_observed_at": value.get("observed_at", {}).get("forecast"),
"revised": initial is not None and (initial != value["actual"] if view else Decimal(initial) != Decimal(value["actual"]))})
return output
@@ -374,6 +374,7 @@ class CalendarWorker:
now = datetime.now(UTC)
async with SessionLocal() as session:
ids = list(await session.scalars(select(EconomicRelease.id).where(
EconomicRelease.source != "jin10",
EconomicRelease.schedule_status == "scheduled", EconomicRelease.scheduled_at <= now,
EconomicRelease.scheduled_at >= now - timedelta(days=HISTORY_DAYS), EconomicRelease.next_check_at <= now,
).order_by(EconomicRelease.scheduled_at.desc()).limit(8)))
@@ -19,7 +19,7 @@ FEEDS = {
"insee": ("https://www.insee.fr/fr/flux/3",),
}
REGIONS = {"wscn_news": "CN", "wscn_live": "CN", "cls_depth": "CN", "cls_telegraph": "CN", "coindesk": "GLOBAL",
"bls": "US", "bea": "US", "fed": "US", "umich": "US", "dol": "US", "eia": "US", "adp": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR"}
"bls": "US", "bea": "US", "fed": "US", "umich": "US", "dol": "US", "eia": "US", "adp": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR", "jin10": "GLOBAL"}
HOSTS = {"ecb": {"www.ecb.europa.eu"}, "eurostat": {"ec.europa.eu"}, "boe": {"www.bankofengland.co.uk"},
"ons": {"www.ons.gov.uk"}, "destatis": {"www.destatis.de"}, "insee": {"www.insee.fr"}}
LANGUAGES = {"destatis": "de", "insee": "fr"}
+2 -1
View File
@@ -1,4 +1,4 @@
from alphapulse.models.jin10 import Jin10Connection
from alphapulse.models.jin10 import Jin10Connection, Jin10SyncDay
from alphapulse.models.risk import RiskSettings, RiskEvent, RiskAnalysis, RiskSnapshot
from alphapulse.core.db import Base
from alphapulse.models.ai import AiProvider, ChatMessage, ChatSession, ContextVersion, SkillNode, SkillVersion
@@ -9,6 +9,7 @@ from alphapulse.models.trading import NewsSignal, TradingAccount, TradingAnalysi
__all__ = [
"Jin10Connection",
"Jin10SyncDay",
"RiskSettings", "RiskEvent", "RiskAnalysis", "RiskSnapshot",
"AiProvider",
"Base",
+12 -2
View File
@@ -1,7 +1,8 @@
"""Encrypted connection and short-lived, session-owned QR login state."""
from datetime import datetime
from datetime import date, datetime
from sqlalchemy import DateTime, LargeBinary, String, Text
from sqlalchemy import Date, DateTime, LargeBinary, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from alphapulse.core.db import Base
@@ -19,3 +20,12 @@ class Jin10Connection(Base):
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
next_poll_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
error: Mapped[str | None] = mapped_column(Text)
class Jin10SyncDay(Base):
"""A day is covered only after all three authenticated responses were validated."""
__tablename__ = "jin10_sync_days"
day: Mapped[date] = mapped_column(Date, primary_key=True)
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
counts: Mapped[dict] = mapped_column(JSONB)
File diff suppressed because it is too large. Load diff
+3 -3
View File
@@ -260,15 +260,15 @@ def test_calendar_api_requires_auth_validates_range_and_exposes_unavailable_work
assert (await client.get("/api/calendar/999/revisions")).status_code == 404
assert (await client.get("/api/calendar/999/history?indicator=cpi_mom")).status_code == 404
assert (await client.get("/api/calendar/999/series?indicator=cpi_mom")).status_code == 404
session.get.return_value = SimpleNamespace(family="cpi")
session.get.return_value = SimpleNamespace(family="cpi", source="bls")
assert (await client.get("/api/calendar/1/history?indicator=ppi_mom")).status_code == 400
assert (await client.get("/api/calendar/1/history?indicator=cpi_mom&limit=121")).status_code == 422
assert (await client.get("/api/calendar/1/series?indicator=ppi_mom")).status_code == 400
assert (await client.get("/api/calendar/1/series?indicator=cpi_mom&limit=121")).status_code == 422
session.get.return_value = SimpleNamespace(family="cpi", period=None)
session.get.return_value = SimpleNamespace(family="cpi", source="bls", period=None)
assert (await client.get("/api/calendar/1/series?indicator=cpi_mom")).json()["data"] == []
handler = AsyncMock(return_value=[{"actual": "0.3", "first_recorded_actual": "0.4", "revised": True}])
monkeypatch.setattr(calendar, "indicator_history", handler)
monkeypatch.setattr(calendar, "calendar_history", handler)
history = await client.get("/api/calendar/1/history?indicator=cpi_mom")
assert history.status_code == 200 and history.json()["data"][0]["revised"] is True
assert handler.await_args.args[2:] == ("cpi_mom", 36)
+1
View File
@@ -25,6 +25,7 @@ def qr_env(trading, monkeypatch):
app.include_router(jin10.router)
app.dependency_overrides[get_session] = lambda: session
monkeypatch.setattr(jin10, "check_session", AsyncMock())
monkeypatch.setattr(jin10.redis, "get", AsyncMock(return_value=None))
monkeypatch.setattr(jin10_auth, "generate_qr", AsyncMock(return_value="private-qr-token"))
monkeypatch.setattr(jin10_auth, "poll_qr", AsyncMock(return_value=("pending", None)))
def authorize():
+207
View File
@@ -0,0 +1,207 @@
"""Calendar tests use public fixtures and isolate all account, database and upstream I/O."""
import asyncio
from copy import deepcopy
from datetime import UTC, date, datetime, timedelta
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from alphapulse.calendar.jin10 import normalize, period
FIXTURE = json.loads((Path(__file__).parent / "fixtures/calendar/jin10.json").read_text())
def entries(day):
return normalize(deepcopy(FIXTURE[day]), "data", date.fromisoformat(day))
@pytest.mark.parametrize("day", [key for key in FIXTURE if ":" not in key])
def test_real_values_and_timestamp_seconds(day):
result = entries(day)
assert len(result) == len(FIXTURE[day]["list"])
for entry, raw in zip(result, FIXTURE[day]["list"], strict=True):
assert entry.at.timestamp() == raw["pub_time_unix"]
value = entry.values[f"jin10_{raw['indicator_id']}"]
assert value["actual"] == raw["actual"]
assert value["previous"] == raw["previous"]
assert value["previous_revised"] == raw["revised"]
assert "affect" not in value
def test_events_distinguish_midnight_from_undetermined_time():
result = normalize(FIXTURE["event:2025-01-08"], "event", date(2025, 1, 8))
ces = next(r for r in result if "CES" in r.title)
trump = next(r for r in result if "特朗普" in r.title)
assert ces.values["_jin10"]["precision"] == "tentative"
assert trump.values["_jin10"]["precision"] == "minute"
assert ces.at.hour == trump.at.hour == 16
def test_daily_holiday_and_empty_day():
entry, = normalize(FIXTURE["holiday:2026-11-03"], "holiday", date(2026, 11, 3))
assert entry.title == "文化日"
assert entry.values["_jin10"]["exchange"] == "东京证券交易所"
assert entry.values["_jin10"]["precision"] == "day"
assert normalize(FIXTURE["holiday:2026-11-04"], "holiday", date(2026, 11, 4)) == []
@pytest.mark.parametrize("change", ["date", "duplicate", "timestamp", "number", "country"])
def test_invalid_contract_does_not_become_empty_success(change):
body = deepcopy(FIXTURE["2026-11-04"])
if change == "date": body["date"] = "2026-11-05"
if change == "duplicate": body["list"] *= 2
if change == "timestamp": body["list"][0]["pub_time_unix"] += 3600
if change == "number": body["list"][0]["actual"] = 0
if change == "country": body["list"][0]["country"] = 7
with pytest.raises(ValueError):
normalize(body, "data", date(2026, 11, 4))
@pytest.mark.parametrize("payload", [None, [], "not base64", "eJwDAAAAAAE=", "eJyLjo0FAAEVALk=garbage"])
def test_broken_holiday_payload_is_not_accepted(payload):
with pytest.raises(ValueError):
normalize({"date": "2026-11-03", "list": payload}, "holiday", date(2026, 11, 3))
def test_non_numeric_text_and_unknown_units_are_preserved():
body = deepcopy(FIXTURE["2026-11-04"])
body["list"] = body["list"][:1]
body["list"][0].update(actual="4.00-4.25", unit=None, consensus=None, revised=None)
entry, = normalize(body, "data", date(2026, 11, 4))
assert entry.values["jin10_78"]["actual"] == "4.00-4.25"
assert entry.values["_jin10"]["unit"] is None
assert entry.values["jin10_78"]["forecast"] is None
def test_periods_cross_year_without_inventing_unrecognized_periods():
at = datetime(2025, 1, 8, 15, tzinfo=UTC)
assert period("12月", at) == "2024-12"
assert period("至12月30日", at) == "2024-12-30"
assert period("第四季度", at) == "2024-Q4"
assert period("1-12月", at) is None
assert period("至1月9日", at) is None
def stored(entry, id=11, now=None):
return SimpleNamespace(id=id, source="jin10", family="jin10", title=entry.title, scheduled_at=entry.at,
schedule_url="https://rili.jin10.com/day/2026-11-04/", release_url=None,
values=deepcopy(entry.values), first_received_at=now, last_checked_at=now,
schedule_checked_at=now, error=None, revision_no=1)
def test_late_forecast_is_displayed_but_not_a_historical_surprise(trading):
from alphapulse.calendar.jin10_view import jin10_item
row = stored(entries("2026-11-04")[0])
row.values["jin10_78"].update(actual="9", forecast_before_release=False, observed_at={"forecast": "2026-11-05T00:00:00+00:00"})
item = jin10_item(row, datetime(2026, 11, 5, tzinfo=UTC))
assert item["forecast"] == "7.5"
assert not item["forecast_before_release"]
assert item["impact_basis"] == "previous" and item["impact"] == "neutral"
def test_exact_merge_prefers_official_fields_and_preserves_previous_revisions(trading):
from alphapulse.calendar.jin10_view import merge_items
entry = next(e for e in entries("2026-10-29") if "jin10_79" in e.values)
extra = stored(entry)
extra.values["jin10_79"].update(observed_at={"forecast": "2026-10-28T00:00:00+00:00"}, forecast_before_release=True)
official = SimpleNamespace(source="dol", family="initial_claims", stage=None, period="2026-10-24", scheduled_at=entry.at,
first_received_at=entry.at, schedule_checked_at=entry.at, release_url="https://www.dol.gov/claims.pdf",
schedule_url="https://oui.doleta.gov/unemploy/archive.asp")
original = {"release_id": 1, "indicator": "initial_claims", "actual": "19.40", "previous": "19.5", "forecast": None,
"status": "released", "impact": "uncertain", "explanation": "official", "revision_no": 1, "scheduled_at": entry.at}
item, = merge_items([(official, original)], [extra], entry.at)
assert item["release_id"] == 1 and item["actual"] == "19.40"
assert item["forecast"] == "19.9" and item["related_release_id"] == extra.id
assert item["previous_original"] == "19.4" and item["previous_revised"] == "19.5"
assert item["field_sources"]["actual"]["source"] == "dol"
assert item["field_sources"]["forecast"]["source"] == "jin10"
assert original["forecast"] is None
official.period = "2026-10-17"
assert len(merge_items([(official, original)], [extra], entry.at)) == 2
official.period = "2026-10-24"
extra.values["_jin10"]["unit"] = "人"
assert len(merge_items([(official, original)], [extra], entry.at)) == 2
def test_cushing_is_not_total_commercial_crude(trading):
from alphapulse.calendar.jin10_view import definition
crude = next(e for e in entries("2025-01-08") if "jin10_111" in e.values)
cushing = next(e for e in entries("2025-01-08") if "jin10_953" in e.values)
assert definition(crude.values["_jin10"]) == ("eia_stocks", "eia_crude", None)
assert definition(cushing.values["_jin10"]) is None
def test_repeated_storage_preserves_observation_and_first_received_time(trading):
from alphapulse.calendar.jin10_store import save_entry
now = datetime(2026, 11, 3, tzinfo=UTC)
entry = entries("2026-11-04")[0]
row = stored(entry)
row.values["jin10_78"].update(observed_at={"previous": now.isoformat(), "forecast": now.isoformat()}, forecast_before_release=True)
old = deepcopy(row.values)
session = SimpleNamespace(scalar=AsyncMock(return_value=row), add=lambda r: pytest.fail("Unexpected revision"))
asyncio.run(save_entry(session, entry, 1, now + timedelta(hours=1)))
assert row.values == old and row.first_received_at is None
def test_reconnect_rejects_obsolete_inflight_capture(trading, monkeypatch):
from alphapulse.calendar import jin10_worker as worker
now = datetime.now(UTC)
class Session:
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
scalar = AsyncMock(return_value=SimpleNamespace(token_encrypted=b"new", connected_at=now))
commit = AsyncMock()
session = Session()
monkeypatch.setattr(worker, "SessionLocal", lambda: session)
monkeypatch.setattr(worker, "save_raw", AsyncMock())
response = {"data": FIXTURE["2026-11-04"], "event": FIXTURE["event:2026-11-04"], "holiday": FIXTURE["holiday:2026-11-04"]}
with pytest.raises(worker.ConnectionChanged):
asyncio.run(worker.persist_day(date(2026, 11, 4), response, b"old", now, now))
worker.save_raw.assert_not_awaited()
session.commit.assert_not_awaited()
def test_every_verified_mapping_has_a_public_fixture_with_the_exact_definition(trading):
from alphapulse.calendar.jin10_view import MAPPINGS, definition
seen = set()
for day in (key for key in FIXTURE if ":" not in key):
for entry in entries(day):
meta = entry.values["_jin10"]
if definition(meta):
seen.add((meta["country_code"], int(meta["indicator"].removeprefix("jin10_")), meta["name"]))
expected = {(country, id_, name) for country, mapping in MAPPINGS.items() for id_, name in mapping}
assert expected <= seen
def test_holidays_preserve_distinct_exchanges_within_the_same_country():
rows = normalize(FIXTURE["holiday:2025-01-01"], "holiday", date(2025, 1, 1))
assert len(rows) == 17
us = [row for row in rows if row.values["_jin10"]["country_code"] == "US"]
assert len(us) > 1 and len({row.identity for row in us}) == len(us)
def test_connection_status_never_accepts_a_previous_connection_heartbeat(trading):
from alphapulse.api.jin10 import public_status
now = datetime.now(UTC)
row = SimpleNamespace(token_encrypted=b"synthetic", connected_at=now, error=None)
sync = {"at": now.isoformat(), "state": "ready", "connected_at": now.isoformat(), "last_success_at": now.isoformat()}
assert public_status(row, sync)["sync_state"] == "ready"
sync["connected_at"] = (now - timedelta(seconds=1)).isoformat()
assert public_status(row, sync)["sync_state"] == "awaiting_validation"
row.token_encrypted = None
sync["connected_at"] = now.isoformat()
assert public_status(row, sync)["sync_state"] == "disconnected"
def test_account_login_does_not_prove_calendar_access():
from alphapulse.calendar.jin10_browser import validate_access
from alphapulse.calendar.jin10_auth import Jin10Error
response = {"data": FIXTURE["2025-01-08"], "event": FIXTURE["event:2025-01-08"],
"holiday": {"date": "2025-01-08", "list": FIXTURE["holiday:2026-11-04"]["list"]}}
validate_access(response)
response["data"] = {"date": "2025-01-08", "list": []}
with pytest.raises(Jin10Error, match="读取权限"):
validate_access(response)
+33
View File
@@ -178,6 +178,7 @@ dependencies = [
{ name = "httpx" },
{ name = "openai" },
{ name = "pillow" },
{ name = "playwright" },
{ name = "pydantic-settings" },
{ name = "pyjwt" },
{ name = "pypdf" },
@@ -205,6 +206,7 @@ requires-dist = [
{ name = "httpx", specifier = ">=0.28.1" },
{ name = "openai", specifier = ">=3.30.0" },
{ name = "pillow", specifier = ">=13.0.0" },
{ name = "playwright", specifier = ">=1.64.1,<2" },
{ name = "pydantic-settings", specifier = ">=2.15.0" },
{ name = "pyjwt", specifier = ">=2.15.1" },
{ name = "pypdf", specifier = ">=6.21.2" },
@@ -1686,6 +1688,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8d/15/1633010b26e88e872c93b67c0b6c5e174fb74cb6fb5c1472b4d51d4a8f22/platformdirs-4.13.0-py3-none-any.whl", hash = "sha256:3dbcf4cd708f21cf876c4eaa90e58412bc4f033d87143f41b1493ff77c25b7e1", size = 32724, upload-time = "2026-10-11T02:05:22.776Z" },
]
[[package]]
name = "playwright"
version = "1.64.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "greenlet" },
{ name = "pyee" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/8d/c8/cc677856d89a0c077673b40ab915ed5597bf1347b205cabe70121c431ca9/playwright-1.64.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:29453ff76e25c17d0b45658cacd6b92e9eec46e8db21e134fb3b7143df50f013", size = 44508971, upload-time = "2026-10-22T16:29:58.12Z" },
{ url = "https://files.pythonhosted.org/packages/72/4f/572bbbe60a5787ca61a84bad934c3ce19c08c2bc9c0e9781bf0d99cad1a6/playwright-1.64.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:59f7b041284ce8d1b72098ae50ca48a2d83692eec25154ab5ba79d4b6b4fe8a9", size = 43155232, upload-time = "2026-10-22T16:30:02.415Z" },
{ url = "https://files.pythonhosted.org/packages/44/fa/468f313a56d33a6420705147c8d47bef558f087856edf38c5b13eef8aa6c/playwright-1.64.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:6a6f99adad1ae40e6ee62efe89785db861ce7ddd57651521cb31894183cc4ae3", size = 44508977, upload-time = "2026-10-22T16:30:07.632Z" },
{ url = "https://files.pythonhosted.org/packages/15/b9/e481a34572933b5cf179d5637c98da488c732ee339dff5b73a0624d2e42b/playwright-1.64.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b8d88778c245cfbd145ae17cf53044f084b1b46c55c69081d4069d56057e2", size = 48471931, upload-time = "2026-10-22T16:30:12.831Z" },
{ url = "https://files.pythonhosted.org/packages/0c/11/545f25e47ec54e1e8a2892ef9994fc2b298f909ab37b88578e3a1f87ac2e/playwright-1.64.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a29fda44725e11b0b625c4bff6458a1f6dbee97b88c9788d12e18a8344458c2d", size = 48150994, upload-time = "2026-10-22T16:30:18.297Z" },
{ url = "https://files.pythonhosted.org/packages/33/87/8ff7af99387ee72ae9b46a66ece7a0c0aa63f73320ecf6dd3dd087fa840a/playwright-1.64.1-py3-none-win32.whl", hash = "sha256:d3c48b31da426c2f3562d0919cb9ea4e5415f6dbd29d7211fa4d66cca24fbd2f", size = 38801828, upload-time = "2026-10-22T16:30:23.154Z" },
{ url = "https://files.pythonhosted.org/packages/96/0a/64db6851aa1852a4f76be28eea71d4e2bf9f3ce493ee2e66939426729046/playwright-1.64.1-py3-none-win_amd64.whl", hash = "sha256:5e240a79464434c9260957b9ef59a5808e02e96e3fcabdc717fd7485260b9730", size = 38801836, upload-time = "2026-10-22T16:30:28.092Z" },
{ url = "https://files.pythonhosted.org/packages/c5/4b/d7ddde9467a5fcfe6a0d3c954732332e98a4f4a9025fd42c8d091c16e5f9/playwright-1.64.1-py3-none-win_arm64.whl", hash = "sha256:983ccd085453ea5a7588193657fc349a6ffbf7bf7ba663fd58dc81e4aa3af9d9", size = 34693875, upload-time = "2026-10-22T16:30:32.724Z" },
]
[[package]]
name = "pluggy"
version = "1.6.0"
@@ -1932,6 +1953,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/66/0e/9ee7bc0b48ec45d93b302fa2d787830dca4dc454d31a237faa5815995988/PyDispatcher-2.0.7-py3-none-any.whl", hash = "sha256:96543bea04115ffde08f851e1d45cacbfd1ee866ac42127d9b476dc5aefa7de0", size = 12040, upload-time = "2023-02-17T20:11:11.991Z" },
]
[[package]]
name = "pyee"
version = "14.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/81/12/5347938b1f9a6453f0dbdfcc3e2388a1320ef9b9ec17fbefbc4ab647ea98/pyee-14.0.0-py3-none-any.whl", hash = "sha256:3ac2d3229a9677f7de2c33d7f52fe25b638a46b19c413fea2edc8c6d0a644e4d", size = 15553, upload-time = "2026-08-13T04:26:09.916Z" },
]
[[package]]
name = "pygments"
version = "2.21.0"