Add authenticated global economic calendar with Jin10 fallback
Collect macroeconomic releases, events and holidays through the official browser SDK with resumable history from 2025. Preserve evidence, revisions and observation times, and merge verified series with official fields taking priority. Add category filters, field provenance, coverage and connection status. Verify calendar persistence and deployment recovery in isolated acceptance checks.
This commit is contained in:
1 parent
57424fd661
commit
e840557d22
35 files changed
+6828
-94
No files matched your search
@@ -9,11 +9,14 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.api.response import ApiResponse, success
|
||||
from alphapulse.calendar.catalog import COUNTRIES, FAMILIES
|
||||
from alphapulse.calendar.service import indicator_history, release_rows
|
||||
from alphapulse.calendar.service import release_rows
|
||||
from alphapulse.calendar.jin10_history import calendar_history
|
||||
from alphapulse.calendar.jin10_view import merge_items
|
||||
from alphapulse.core.auth import require_admin
|
||||
from alphapulse.core.db import get_session
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.models import EconomicRelease, EconomicRevision, EconomicSeriesValue, RawResponse
|
||||
from alphapulse.models.jin10 import Jin10Connection, Jin10SyncDay
|
||||
|
||||
router = APIRouter(prefix="/api/calendar", tags=["calendar"], dependencies=[Depends(require_admin)])
|
||||
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
|
||||
@@ -25,12 +28,32 @@ async def calendar_data(session: AsyncSession, start: datetime, end: datetime) -
|
||||
).order_by(EconomicRelease.scheduled_at, EconomicRelease.id)))
|
||||
raw = await redis.get(f"{KEY_PREFIX}calendar:status")
|
||||
status = json.loads(raw) if raw else None
|
||||
raw_jin10 = await redis.get(f"{KEY_PREFIX}calendar:jin10")
|
||||
jin10_status = json.loads(raw_jin10) if raw_jin10 else None
|
||||
if jin10_status:
|
||||
connection = await session.get(Jin10Connection, 1)
|
||||
if not connection or not connection.token_encrypted or jin10_status.get("connected_at") != connection.connected_at.isoformat():
|
||||
jin10_status = {**jin10_status, "state": "disconnected", "error": None}
|
||||
coverage = list(await session.scalars(select(Jin10SyncDay).where(
|
||||
Jin10SyncDay.day >= start.astimezone(LOCAL_TZ).date(), Jin10SyncDay.day < end.astimezone(LOCAL_TZ).date(),
|
||||
).order_by(Jin10SyncDay.day)))
|
||||
now = datetime.now(UTC)
|
||||
return {"items": [item for row in rows for item in release_rows(row, now)], "worker": status,
|
||||
items = merge_items([(row, item) for row in rows if row.source != "jin10" for item in release_rows(row, now)],
|
||||
[row for row in rows if row.source == "jin10"], now)
|
||||
countries = {**COUNTRIES, **{item["country_code"]: item["country_name"] for item in items}}
|
||||
return {"items": items, "worker": status,
|
||||
"jin10": jin10_status,
|
||||
"coverage": [{"date": day.day, "captured_at": day.captured_at, "counts": day.counts} for day in coverage],
|
||||
"server_time": now, "timezone": "Asia/Shanghai",
|
||||
"countries": [{"code": code, "name": name} for code, name in COUNTRIES.items()],
|
||||
"forecast_available": any(row.values.get(key, {}).get("forecast") is not None
|
||||
for row in rows for key in FAMILIES[row.family].indicators)}
|
||||
"countries": [{"code": code, "name": name} for code, name in countries.items()],
|
||||
"forecast_available": any(item["forecast"] is not None for item in items)}
|
||||
|
||||
|
||||
def indicators(row: EconomicRelease) -> tuple[str, ...]:
|
||||
if row.source == "jin10":
|
||||
meta = row.values["_jin10"]
|
||||
return (meta["indicator"],) if meta["kind"] == "data" else ()
|
||||
return FAMILIES[row.family].indicators
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -42,11 +65,12 @@ async def list_calendar(session: Annotated[AsyncSession, Depends(get_session)],
|
||||
|
||||
@router.get("/{release_id}/revisions")
|
||||
async def revisions(release_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> ApiResponse[list[dict]]:
|
||||
if await session.get(EconomicRelease, release_id) is None:
|
||||
row = await session.get(EconomicRelease, release_id)
|
||||
if row is None:
|
||||
raise HTTPException(404, "日历事件不存在")
|
||||
rows = await session.scalars(select(EconomicRevision).where(EconomicRevision.release_id == release_id)
|
||||
.order_by(EconomicRevision.revision_no.desc()))
|
||||
return success([{"revision_no": r.revision_no, "captured_at": r.captured_at,
|
||||
return success([{"source": row.source, "revision_no": r.revision_no, "captured_at": r.captured_at,
|
||||
"payload": r.payload, "raw_response_id": r.raw_response_id} for r in rows])
|
||||
|
||||
|
||||
@@ -56,9 +80,9 @@ async def history(release_id: int, indicator: str, session: Annotated[AsyncSessi
|
||||
row = await session.get(EconomicRelease, release_id)
|
||||
if row is None:
|
||||
raise HTTPException(404, "日历事件不存在")
|
||||
if indicator not in FAMILIES[row.family].indicators:
|
||||
if indicator not in indicators(row):
|
||||
raise HTTPException(400, "指标不属于该事件")
|
||||
return success(await indicator_history(session, row, indicator, limit))
|
||||
return success(await calendar_history(session, row, indicator, limit))
|
||||
|
||||
|
||||
@router.get("/{release_id}/series")
|
||||
@@ -67,9 +91,9 @@ async def series(release_id: int, indicator: str, session: Annotated[AsyncSessio
|
||||
row = await session.get(EconomicRelease, release_id)
|
||||
if row is None:
|
||||
raise HTTPException(404, "日历事件不存在")
|
||||
if indicator not in FAMILIES[row.family].indicators:
|
||||
if indicator not in indicators(row):
|
||||
raise HTTPException(400, "指标不属于该事件")
|
||||
if row.period is None:
|
||||
if row.period is None or row.source == "jin10":
|
||||
return success([])
|
||||
# 此接口是当前官方序列,采集时间可以晚于所选事件;不得用作历史时点回测输入。
|
||||
rows = (await session.execute(select(EconomicSeriesValue, RawResponse.url).join(
|
||||
|
||||
@@ -119,7 +119,6 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
|
||||
now = datetime.now(UTC)
|
||||
raw = await redis.get(f"{KEY_PREFIX}calendar:status")
|
||||
heartbeat = json.loads(raw) if raw else None
|
||||
alive = bool(heartbeat and now - datetime.fromisoformat(heartbeat["at"]) < timedelta(seconds=30))
|
||||
pending = EconomicRelease.first_received_at.is_(None) & (EconomicRelease.schedule_status == "scheduled")
|
||||
published = EconomicRelease.first_received_at.is_not(None)
|
||||
rows = (await session.execute(select(
|
||||
@@ -138,18 +137,26 @@ async def calendar_sources(session: AsyncSession) -> list[CalendarSourceOut]:
|
||||
EconomicRelease.scheduled_at >= now - timedelta(days=45),
|
||||
).order_by(EconomicRelease.last_checked_at.desc())):
|
||||
result_errors.setdefault(source, error)
|
||||
codes = {family.source for family in FAMILIES.values()}
|
||||
codes = {family.source for family in FAMILIES.values()} | {"jin10"}
|
||||
raw_jin10 = await redis.get(f"{KEY_PREFIX}calendar:jin10")
|
||||
jin10_heartbeat = json.loads(raw_jin10) if raw_jin10 else None
|
||||
sources = list(await session.scalars(select(Source).where(Source.code.in_(codes)).order_by(Source.code)))
|
||||
output = []
|
||||
for source in sources:
|
||||
checks = [s for s in heartbeat["sources"].values() if s["source"] == source.code] if heartbeat else []
|
||||
current_heartbeat = jin10_heartbeat if source.code == "jin10" else heartbeat
|
||||
source_alive = bool(current_heartbeat and now - datetime.fromisoformat(current_heartbeat["at"]) < timedelta(seconds=30))
|
||||
if source.code == "jin10":
|
||||
checks = [{"source": "jin10", "checked_at": jin10_heartbeat["at"],
|
||||
"error": jin10_heartbeat.get("error") or (None if jin10_heartbeat.get("state") in ("ready", "syncing") else "金十同步尚未就绪")}] if jin10_heartbeat else []
|
||||
else:
|
||||
checks = [s for s in heartbeat["sources"].values() if s["source"] == source.code] if heartbeat else []
|
||||
errors = [s["error"] for s in checks if s["error"]]
|
||||
if source.code in result_errors:
|
||||
errors.append(result_errors[source.code])
|
||||
data = stats.get(source.code, {})
|
||||
output.append(CalendarSourceOut(
|
||||
code=source.code, name=source.name, region_code=REGIONS[source.code], worker_alive=alive,
|
||||
heartbeat_at=heartbeat["at"] if heartbeat else None,
|
||||
code=source.code, name=source.name, region_code=REGIONS[source.code], worker_alive=source_alive,
|
||||
heartbeat_at=current_heartbeat["at"] if current_heartbeat else None,
|
||||
schedule_checked_at=max((s["checked_at"] for s in checks), default=None),
|
||||
**{key: data.get(key) for key in ("schedule_success_at", "latest_result_at", "next_event_at", "history_start", "history_end")},
|
||||
overdue_count=data.get("overdue_count", 0), errors=list(dict.fromkeys(errors)),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Administrator-only Jin10 connection; credentials never leave the backend."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import json
|
||||
import secrets
|
||||
from typing import Annotated
|
||||
|
||||
@@ -12,6 +13,7 @@ from alphapulse.calendar import jin10_auth
|
||||
from alphapulse.core.auth import AuthSession, check_session, require_admin
|
||||
from alphapulse.core.crypto import decrypt, encrypt
|
||||
from alphapulse.core.db import get_session
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.models.jin10 import Jin10Connection
|
||||
|
||||
router = APIRouter(prefix="/api/settings/jin10", tags=["settings"], dependencies=[Depends(require_admin)])
|
||||
@@ -33,14 +35,21 @@ def clear_task(row: Jin10Connection) -> None:
|
||||
row.task_id = row.owner = row.qr_encrypted = row.expires_at = row.next_poll_at = None
|
||||
|
||||
|
||||
def public_status(row: Jin10Connection) -> dict:
|
||||
def public_status(row: Jin10Connection, sync: dict | None = None) -> dict:
|
||||
current = bool(row.token_encrypted and row.connected_at and sync and sync.get("connected_at") == row.connected_at.isoformat()
|
||||
and datetime.now(UTC) - datetime.fromisoformat(sync["at"]) < timedelta(seconds=30))
|
||||
return {"connected": row.token_encrypted is not None, "connected_at": row.connected_at,
|
||||
"error": row.error, "sync_state": "awaiting_validation" if row.token_encrypted else "disconnected"}
|
||||
"error": row.error or (sync.get("error") if current else None),
|
||||
"sync_state": sync["state"] if current else "awaiting_validation" if row.token_encrypted else "disconnected",
|
||||
"last_success_at": sync.get("last_success_at") if current else None,
|
||||
"history_next": sync.get("history_next") if current else None}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def status(session: Session) -> ApiResponse[dict]:
|
||||
return success(public_status(await connection(session)))
|
||||
row = await connection(session)
|
||||
raw = await redis.get(f"{KEY_PREFIX}calendar:jin10")
|
||||
return success(public_status(row, json.loads(raw) if raw else None))
|
||||
|
||||
|
||||
@router.post("/qr", status_code=201)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Normalize authenticated Jin10 calendar responses without interpreting gold/USD signals."""
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, date, datetime, time, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
LOCAL = ZoneInfo("Asia/Shanghai")
|
||||
START = date(2025, 1, 1)
|
||||
KINDS = ("data", "event", "holiday")
|
||||
COUNTRY_CODES = {
|
||||
"美国": "US", "欧元区": "EA", "英国": "GB", "德国": "DE", "法国": "FR",
|
||||
"中国": "CN", "中国香港": "HK", "中国台湾": "TW", "日本": "JP", "韩国": "KR",
|
||||
"加拿大": "CA", "澳大利亚": "AU", "新西兰": "NZ", "瑞士": "CH", "瑞典": "SE",
|
||||
"挪威": "NO", "意大利": "IT", "西班牙": "ES", "印度": "IN", "俄罗斯": "RU",
|
||||
"巴西": "BR", "南非": "ZA", "新加坡": "SG", "乌克兰": "UA", "墨西哥": "MX",
|
||||
"土耳其": "TR", "波兰": "PL", "印度尼西亚": "ID", "马来西亚": "MY", "泰国": "TH",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Entry:
|
||||
identity: str
|
||||
at: datetime
|
||||
title: str
|
||||
values: dict
|
||||
|
||||
|
||||
def day_url(day: date) -> str:
|
||||
return f"https://rili.jin10.com/day/{day.isoformat()}/"
|
||||
|
||||
|
||||
def records(body: dict, kind: str, day: date) -> list[dict]:
|
||||
if kind not in KINDS or not isinstance(body, dict) or body.get("date") != day.isoformat():
|
||||
raise ValueError("金十响应日期或类别与请求不符")
|
||||
rows = body.get("list")
|
||||
if kind == "holiday":
|
||||
# The public SDK's legacy daily holiday method returns zlib/base64 text.
|
||||
if not isinstance(rows, str) or len(rows) > 2_000_000:
|
||||
raise ValueError("金十假期响应格式已变化")
|
||||
try:
|
||||
decoder = zlib.decompressobj()
|
||||
decoded = decoder.decompress(base64.b64decode(rows, validate=True), 4_000_000)
|
||||
if not decoder.eof or decoder.unused_data or decoder.unconsumed_tail:
|
||||
raise ValueError("金十假期压缩内容不完整或过大")
|
||||
rows = json.loads(decoded)
|
||||
except (ValueError, zlib.error, binascii.Error) as exc:
|
||||
raise ValueError("金十假期内容无法解析") from exc
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
raise ValueError("金十日历列表格式已变化")
|
||||
return rows
|
||||
|
||||
|
||||
def text(row: dict, key: str, *, nullable: bool = False) -> str | None:
|
||||
value = row.get(key)
|
||||
if nullable and value is None:
|
||||
return None
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError(f"金十字段 {key} 缺失或类型错误")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def identifier(row: dict, key: str) -> int:
|
||||
value = row.get(key)
|
||||
if type(value) is not int or value <= 0:
|
||||
raise ValueError(f"金十字段 {key} 不是有效标识")
|
||||
return value
|
||||
|
||||
|
||||
def normalize(body: dict, kind: str, day: date) -> list[Entry]:
|
||||
if day < START:
|
||||
raise ValueError("金十历史范围不得早于2025-01-01北京时间")
|
||||
result, identities = [], set()
|
||||
for raw in records(body, kind, day):
|
||||
identity = f"{kind}:{identifier(raw, 'data_id' if kind == 'data' else 'id')}"
|
||||
if identity in identities:
|
||||
raise ValueError("金十同一响应存在重复事件标识")
|
||||
identities.add(identity)
|
||||
country = text(raw, "country", nullable=True)
|
||||
precision = "day" if kind == "holiday" else "minute"
|
||||
if kind == "holiday":
|
||||
if raw.get("date") != day.isoformat():
|
||||
raise ValueError("金十假期记录日期与请求不符")
|
||||
at = datetime.combine(day, time.min, LOCAL)
|
||||
title = text(raw, "name")
|
||||
description = " · ".join(v for v in [text(raw, "exchange_name"), text(raw, "rest_note", nullable=True),
|
||||
text(raw, "remark", nullable=True)] if v)
|
||||
else:
|
||||
timestamp = text(raw, "pub_time" if kind == "data" else "event_time")
|
||||
at = datetime.strptime(timestamp, "%Y-%m-%d %H:%M").replace(tzinfo=LOCAL)
|
||||
if at.date() != day:
|
||||
raise ValueError("金十记录公布日期与请求不符")
|
||||
unix = raw.get("pub_time_unix" if kind == "data" else "event_time_unix")
|
||||
if type(unix) is not int or not int(at.timestamp()) <= unix < int(at.timestamp()) + 60:
|
||||
raise ValueError("金十公布时间与时间戳不符")
|
||||
at = datetime.fromtimestamp(unix, LOCAL)
|
||||
if raw.get("determine") not in (0, 1):
|
||||
raise ValueError("金十时间确定性字段已变化")
|
||||
if raw["determine"] == 1 or raw.get("time_status"):
|
||||
precision = "tentative"
|
||||
title = text(raw, "indicator_name" if kind == "data" else "event_content")
|
||||
description = text(raw, "note", nullable=True) if kind == "event" else None
|
||||
if kind != "data" and raw.get("category") != "cj":
|
||||
raise ValueError("金十记录不属于宏观日历")
|
||||
importance = raw.get("star") if kind != "holiday" else 0
|
||||
if type(importance) is not int or not 0 <= importance <= 5:
|
||||
raise ValueError("金十重要性字段已变化")
|
||||
key = f"jin10_{identifier(raw, 'indicator_id')}" if kind == "data" else f"jin10_{kind}"
|
||||
meta = {"kind": kind, "date": day.isoformat(), "precision": precision, "country_name": country or "未标注",
|
||||
"country_code": COUNTRY_CODES.get(country, country or "UNKNOWN"),
|
||||
"name": title, "description": description, "importance": importance,
|
||||
"period_label": text(raw, "time_period", nullable=True) if kind == "data" else None,
|
||||
"indicator": key, "unit": ("" if raw.get("unit") == "" else text(raw, "unit", nullable=True)) if kind == "data" else None,
|
||||
"exchange": text(raw, "exchange_name") if kind == "holiday" else None}
|
||||
values = {"_jin10": meta}
|
||||
if kind == "data":
|
||||
# Preserve nonnumeric values (including ranges); never coerce a missing value to zero.
|
||||
values[key] = {"actual": text(raw, "actual", nullable=True), "previous": text(raw, "previous", nullable=True),
|
||||
"forecast": text(raw, "consensus", nullable=True),
|
||||
"previous_revised": text(raw, "revised", nullable=True)}
|
||||
result.append(Entry(identity, at.astimezone(UTC), title, values))
|
||||
return result
|
||||
|
||||
|
||||
def period(label: str | None, at: datetime) -> str | None:
|
||||
"""Resolve only verified monthly/weekly labels close to their publication date."""
|
||||
local = at.astimezone(LOCAL).date()
|
||||
month = re.fullmatch(r"(\d{1,2})月", label or "")
|
||||
week = re.fullmatch(r"至(\d{1,2})月(\d{1,2})日", label or "")
|
||||
if month:
|
||||
number = int(month[1])
|
||||
if not 1 <= number <= 12:
|
||||
return None
|
||||
year = local.year - (number > local.month)
|
||||
return f"{year}-{number:02}"
|
||||
if week:
|
||||
try:
|
||||
day = date(local.year, int(week[1]), int(week[2]))
|
||||
if day > local:
|
||||
day = day.replace(year=day.year - 1)
|
||||
return day.isoformat() if local - day < timedelta(days=45) else None
|
||||
except ValueError:
|
||||
return None
|
||||
quarter = re.fullmatch(r"第([一二三四])季度", label or "")
|
||||
if quarter:
|
||||
number = "一二三四".index(quarter[1]) + 1
|
||||
year = local.year - (number > (local.month - 1) // 3 + 1)
|
||||
return f"{year}-Q{number}"
|
||||
return None
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Use Jin10's own browser SDK rather than reproducing its encrypted transport."""
|
||||
import asyncio
|
||||
from datetime import date
|
||||
|
||||
from playwright.async_api import Error as BrowserError, async_playwright
|
||||
|
||||
from alphapulse.calendar.jin10 import KINDS, day_url, normalize
|
||||
from alphapulse.calendar.jin10_auth import Jin10Error, uc_request
|
||||
|
||||
METHODS = {"data": "getCalendarData", "event": "getCalendarEvent", "holiday": "getCalendarHoliday"}
|
||||
VALIDATION_DAY = date(2025, 1, 8)
|
||||
|
||||
|
||||
def validate_access(responses: dict) -> None:
|
||||
# This archived day has verified economic releases and events. An authenticated
|
||||
# account alone does not prove that the SDK can read the economic calendar.
|
||||
normalized = {kind: normalize(responses[kind], kind, VALIDATION_DAY) for kind in KINDS}
|
||||
if not normalized["data"] or not normalized["event"]:
|
||||
raise Jin10Error("金十历史日历读取权限未通过验证,请重新连接后检查账号权限")
|
||||
|
||||
|
||||
class Jin10Browser:
|
||||
def __init__(self) -> None:
|
||||
self.playwright = self.browser = self.context = self.page = None
|
||||
self.credential = None
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.browser:
|
||||
await self.browser.close()
|
||||
if self.playwright:
|
||||
await self.playwright.stop()
|
||||
self.playwright = self.browser = self.context = self.page = None
|
||||
self.credential = None
|
||||
|
||||
async def connect(self, credential: str) -> None:
|
||||
await self.close()
|
||||
info = await uc_request("/userinfo", token=credential)
|
||||
if not isinstance(info.get("data"), dict) or not info["data"]:
|
||||
raise Jin10Error("金十登录态已失效,请重新扫码")
|
||||
try:
|
||||
self.playwright = await async_playwright().start()
|
||||
self.browser = await self.playwright.chromium.launch(headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"])
|
||||
self.context = await self.browser.new_context()
|
||||
await self.context.add_cookies([{"name": "x-token", "value": credential, "domain": ".jin10.com",
|
||||
"path": "/", "secure": True, "sameSite": "Lax"}])
|
||||
self.page = await self.context.new_page()
|
||||
# Keep the host page outside the collection window so its initial requests do not compete.
|
||||
await self.page.goto(day_url(date(2024, 1, 1)), wait_until="domcontentloaded", timeout=30000)
|
||||
await self.page.wait_for_function(
|
||||
"window.Jin10FlashInstance && window.Jin10FlashInstance.socket && "
|
||||
"window.Jin10FlashInstance.socket.readyState === 1", timeout=30000)
|
||||
await self.page.wait_for_timeout(2000)
|
||||
self.credential = credential
|
||||
validate_access(await self.fetch_day(VALIDATION_DAY))
|
||||
except BrowserError:
|
||||
await self.close()
|
||||
raise Jin10Error("金十浏览器连接失败,请检查网络与浏览器运行环境") from None
|
||||
|
||||
async def fetch_day(self, day: date) -> dict[str, dict]:
|
||||
async with self.lock:
|
||||
if self.page is None:
|
||||
raise Jin10Error("金十浏览器尚未连接")
|
||||
try:
|
||||
result = {}
|
||||
for kind in KINDS:
|
||||
result[kind] = await self.page.evaluate(
|
||||
"async ({method, day}) => {"
|
||||
"if (!window.Jin10FlashInstance || window.Jin10FlashInstance.socket.readyState !== 1) throw Error('disconnected');"
|
||||
"return await window.Jin10FlashInstance[method](day, {timeout: 15000});}",
|
||||
{"method": METHODS[kind], "day": day.isoformat()})
|
||||
info = await uc_request("/userinfo", token=self.credential)
|
||||
if not isinstance(info.get("data"), dict) or not info["data"]:
|
||||
raise Jin10Error("金十登录态已失效,未将空列表记为完整日历")
|
||||
return result
|
||||
except BrowserError:
|
||||
# Browser exceptions may include URLs or script contents; never forward them to logs.
|
||||
raise Jin10Error("金十日历请求失败或超时,未将本次响应记为成功") from None
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Read the same official/provider merge for history without rewriting either source."""
|
||||
from datetime import UTC, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.calendar.catalog import FAMILIES
|
||||
from alphapulse.calendar.jin10_view import MAPPINGS, definition, merge_items
|
||||
from alphapulse.calendar.service import indicator_history, release_rows
|
||||
from alphapulse.models import EconomicRelease, EconomicRevision
|
||||
|
||||
|
||||
def changed(initial: str | None, current: str) -> bool:
|
||||
if initial is None:
|
||||
return False
|
||||
try:
|
||||
return Decimal(initial) != Decimal(current)
|
||||
except InvalidOperation:
|
||||
return initial != current
|
||||
|
||||
|
||||
async def calendar_history(session: AsyncSession, selected: EconomicRelease, indicator: str, limit: int) -> list[dict]:
|
||||
if selected.source == "jin10":
|
||||
match = definition(selected.values["_jin10"])
|
||||
if match is None:
|
||||
return await indicator_history(session, selected, indicator, limit)
|
||||
family, key, stage = match
|
||||
else:
|
||||
family, key, stage = selected.family, indicator, selected.stage
|
||||
# IDs and exact definitions are verified by the normal read merge as well.
|
||||
provider_keys = [f"jin10_{id_}" for mapping in MAPPINGS.values() for (id_, _), value in mapping.items()
|
||||
if value == (family, key, stage)]
|
||||
if not provider_keys:
|
||||
return await indicator_history(session, selected, indicator, limit)
|
||||
candidates = list(await session.scalars(select(EconomicRelease).where(
|
||||
EconomicRelease.scheduled_at <= selected.scheduled_at,
|
||||
EconomicRelease.schedule_status != "withdrawn",
|
||||
or_((EconomicRelease.source == FAMILIES[family].source) & (EconomicRelease.family == family)
|
||||
& EconomicRelease.stage.is_not_distinct_from(stage),
|
||||
(EconomicRelease.source == "jin10") & EconomicRelease.values["_jin10"]["indicator"].as_string().in_(provider_keys)),
|
||||
).order_by(EconomicRelease.scheduled_at.desc(), EconomicRelease.id.desc()).limit(limit * 2)))
|
||||
now = datetime.now(UTC)
|
||||
official = [(row, item) for row in candidates if row.source != "jin10" for item in release_rows(row, now) if item["indicator"] == key]
|
||||
extras = [row for row in candidates if row.source == "jin10" and definition(row.values["_jin10"]) == (family, key, stage)]
|
||||
items = [item for item in merge_items(official, extras, now) if item["actual"] is not None]
|
||||
items.sort(key=lambda r: (r["scheduled_at"], r["release_id"]), reverse=True)
|
||||
items = items[:limit]
|
||||
by_id = {row.id: row for row in candidates}
|
||||
revisions = list(await session.scalars(select(EconomicRevision).where(
|
||||
EconomicRevision.release_id.in_(by_id),
|
||||
).order_by(EconomicRevision.release_id, EconomicRevision.revision_no)))
|
||||
first = {}
|
||||
for revision in revisions:
|
||||
owner = by_id[revision.release_id]
|
||||
value_key = owner.values["_jin10"]["indicator"] if owner.source == "jin10" else key
|
||||
actual = revision.payload["values"].get(value_key, {}).get("actual")
|
||||
if actual is not None:
|
||||
first.setdefault(owner.id, actual)
|
||||
output = []
|
||||
for item in items:
|
||||
owner = by_id[item["release_id"]]
|
||||
if item["field_sources"]["actual"]["source"] == "jin10" and owner.source != "jin10":
|
||||
owner = by_id[item["related_release_id"]]
|
||||
initial = first.get(owner.id)
|
||||
output.append({"release_id": item["release_id"], "scheduled_at": item["scheduled_at"], "title": item["title"],
|
||||
"period": item["period"], "period_label": item["period_label"], "stage": item["stage"],
|
||||
"actual": item["actual"], "previous": item["previous"], "forecast": item["forecast"],
|
||||
"forecast_before_release": item["forecast_before_release"], "forecast_observed_at": item["forecast_observed_at"],
|
||||
"first_recorded_actual": initial, "first_received_at": owner.first_received_at,
|
||||
"release_url": item["field_sources"]["actual"]["url"], "field_sources": item["field_sources"],
|
||||
"revised": changed(initial, item["actual"])})
|
||||
return output
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Keep provider identities, raw evidence, field observation times and revisions together."""
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.calendar.jin10 import Entry, day_url
|
||||
from alphapulse.calendar.service import record_revision
|
||||
from alphapulse.models import EconomicRelease
|
||||
from alphapulse.models.jin10 import Jin10SyncDay
|
||||
|
||||
|
||||
async def save_entry(session: AsyncSession, entry: Entry, raw_id: int, now: datetime) -> None:
|
||||
row = await session.scalar(select(EconomicRelease).where(
|
||||
EconomicRelease.source == "jin10", EconomicRelease.external_id == entry.identity,
|
||||
).with_for_update())
|
||||
values = {key: dict(value) for key, value in entry.values.items()}
|
||||
meta = values["_jin10"]
|
||||
key = meta["indicator"]
|
||||
if meta["kind"] == "data":
|
||||
current = values[key]
|
||||
old = row.values.get(key, {}) if row else {}
|
||||
observed = {}
|
||||
for field in ("actual", "previous", "forecast", "previous_revised"):
|
||||
if current[field] is not None:
|
||||
observed[field] = old.get("observed_at", {}).get(field, now.isoformat()) if current[field] == old.get(field) else now.isoformat()
|
||||
current["observed_at"] = observed
|
||||
forecast_at = datetime.fromisoformat(observed["forecast"]) if "forecast" in observed else None
|
||||
if forecast_at == now:
|
||||
before = current["actual"] is None
|
||||
else:
|
||||
before = old.get("forecast_before_release", False)
|
||||
current["forecast_before_release"] = bool(before and forecast_at and forecast_at < entry.at and meta["precision"] == "minute")
|
||||
received = meta["kind"] != "data" or values[key]["actual"] is not None
|
||||
if row is None:
|
||||
row = EconomicRelease(source="jin10", external_id=entry.identity, family="jin10", title=entry.title,
|
||||
scheduled_at=entry.at, schedule_status="scheduled", schedule_url=day_url(date.fromisoformat(meta["date"])),
|
||||
release_url=None, period=None, stage=None, schedule_checked_at=now, last_checked_at=now,
|
||||
first_received_at=now if received else None, next_check_at=now, error=None,
|
||||
values=values, revision_no=1)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
record_revision(session, row, raw_id)
|
||||
return
|
||||
changed = row.values != values or row.scheduled_at != entry.at or row.title != entry.title
|
||||
row.schedule_checked_at = row.last_checked_at = now
|
||||
row.error = None
|
||||
if row.first_received_at is None and received:
|
||||
row.first_received_at = now
|
||||
changed = True
|
||||
if changed:
|
||||
row.title, row.scheduled_at, row.values = entry.title, entry.at, values
|
||||
row.schedule_url = day_url(date.fromisoformat(meta["date"]))
|
||||
row.revision_no += 1
|
||||
record_revision(session, row, raw_id)
|
||||
|
||||
|
||||
async def mark_day(session: AsyncSession, day: date, counts: dict, now: datetime) -> None:
|
||||
stmt = insert(Jin10SyncDay).values(day=day, captured_at=now, counts=counts)
|
||||
await session.execute(stmt.on_conflict_do_update(index_elements=["day"], set_={"captured_at": now, "counts": counts}))
|
||||
|
||||
|
||||
async def next_history_day(session: AsyncSession, start: date, end: date) -> date | None:
|
||||
from datetime import timedelta
|
||||
covered = set(await session.scalars(select(Jin10SyncDay.day).where(Jin10SyncDay.day.between(start, end))))
|
||||
day = start
|
||||
while day <= end:
|
||||
if day not in covered:
|
||||
return day
|
||||
day += timedelta(days=1)
|
||||
return None
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Join only verified series definitions; official values win independently for each field."""
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
from alphapulse.calendar.catalog import FAMILIES, INDICATORS
|
||||
from alphapulse.calendar.jin10 import LOCAL, period
|
||||
from alphapulse.calendar.service import impact
|
||||
from alphapulse.models import EconomicRelease
|
||||
|
||||
# (Jin10 indicator, exact name) -> (official family, indicator, release stage).
|
||||
# CPI annual rates are NSA, monthly rates SA; other adjustments must not be substituted.
|
||||
US = {
|
||||
(234, "未季调CPI年率"): ("cpi", "cpi_yoy", None),
|
||||
(230, "季调后CPI月率"): ("cpi", "cpi_mom", None),
|
||||
(232, "未季调核心CPI年率"): ("cpi", "core_cpi_yoy", None),
|
||||
(233, "季调后核心CPI月率"): ("cpi", "core_cpi_mom", None),
|
||||
(178, "PPI年率"): ("ppi", "ppi_yoy", None),
|
||||
(177, "PPI月率"): ("ppi", "ppi_mom", None),
|
||||
(75, "季调后非农就业人口"): ("jobs", "payrolls", None),
|
||||
(76, "失业率"): ("jobs", "unemployment", None),
|
||||
(212, "PCE物价指数年率"): ("pce", "pce_yoy", None),
|
||||
(213, "PCE物价指数月率"): ("pce", "pce_mom", None),
|
||||
(211, "核心PCE物价指数年率"): ("pce", "core_pce_yoy", None),
|
||||
(214, "核心PCE物价指数月率"): ("pce", "core_pce_mom", None),
|
||||
(144, "实际GDP年化季率修正值"): ("gdp", "gdp", "second"),
|
||||
(872, "实际GDP年化季率终值"): ("gdp", "gdp", "third"),
|
||||
(198, "密歇根大学消费者信心指数初值"): ("umich", "umich_sentiment", "preliminary"),
|
||||
(208, "密歇根大学消费者信心指数终值"): ("umich", "umich_sentiment", "final"),
|
||||
(216, "实际GDP年化季率初值"): ("gdp", "gdp", "advance"),
|
||||
(77, "美联储利率决定(上限)"): ("fomc", "fed_rate", None),
|
||||
(78, "ADP就业人数"): ("adp_jobs", "adp_jobs", None),
|
||||
(79, "当周初请失业金人数"): ("initial_claims", "initial_claims", None),
|
||||
(96, "当周续请失业金人数"): ("continued_claims", "continued_claims", None),
|
||||
(111, "当周EIA原油库存"): ("eia_stocks", "eia_crude", None),
|
||||
(114, "当周EIA汽油库存"): ("eia_stocks", "eia_gasoline", None),
|
||||
(113, "当周EIA精炼油库存"): ("eia_stocks", "eia_distillate", None),
|
||||
}
|
||||
EA = {
|
||||
(763, "GDP季率初值"): ("ea_gdp", "gdp_qoq", "preliminary"),
|
||||
(762, "GDP年率初值"): ("ea_gdp", "gdp_yoy", "preliminary"),
|
||||
(930, "CPI年率初值"): ("ea_hicp", "hicp_yoy", "flash"),
|
||||
(413, "CPI月率初值"): ("ea_hicp", "hicp_mom", "flash"),
|
||||
(759, "核心CPI年率初值"): ("ea_hicp", "core_hicp_yoy", "flash"),
|
||||
(764, "CPI年率终值"): ("ea_hicp", "hicp_yoy", "final"),
|
||||
(8125, "CPI月率终值"): ("ea_hicp", "hicp_mom", "final"),
|
||||
(622, "核心CPI年率终值"): ("ea_hicp", "core_hicp_yoy", "final"),
|
||||
(396, "失业率"): ("ea_jobs", "ilo_unemployment", None),
|
||||
(395, "欧洲央行存款机制利率"): ("ecb_rate", "ecb_deposit", None),
|
||||
(753, "欧洲央行主要再融资利率"): ("ecb_rate", "ecb_main", None),
|
||||
(419, "欧洲央行边际贷款利率"): ("ecb_rate", "ecb_marginal", None),
|
||||
}
|
||||
|
||||
FR = {
|
||||
(831, "GDP年率初值"): ("fr_gdp", "gdp_yoy", "initial"),
|
||||
(829, "GDP年率终值"): ("fr_gdp", "gdp_yoy", "final"),
|
||||
(830, "GDP季率终值"): ("fr_gdp", "gdp_qoq", "final"),
|
||||
(8111, "CPI年率初值"): ("fr_cpi", "eu_cpi_yoy", "preliminary"),
|
||||
(8112, "CPI月率初值"): ("fr_cpi", "eu_cpi_mom", "preliminary"),
|
||||
(8113, "调和CPI年率初值"): ("fr_cpi", "hicp_yoy", "preliminary"),
|
||||
(8114, "调和CPI月率初值"): ("fr_cpi", "hicp_mom", "preliminary"),
|
||||
(567, "CPI年率终值"): ("fr_cpi", "eu_cpi_yoy", "final"),
|
||||
(568, "CPI月率终值"): ("fr_cpi", "eu_cpi_mom", "final"),
|
||||
(569, "调和CPI年率终值"): ("fr_cpi", "hicp_yoy", "final"),
|
||||
(570, "调和CPI月率终值"): ("fr_cpi", "hicp_mom", "final"),
|
||||
(832, "GDP季率初值"): ("fr_gdp", "gdp_qoq", "initial"),
|
||||
}
|
||||
DE = {
|
||||
(270, "季调后GDP季率终值"): ("de_gdp", "gdp_qoq", "final"),
|
||||
(842, "工作日调整后GDP年率终值"): ("de_gdp", "gdp_yoy", "final"),
|
||||
(846, "CPI年率初值"): ("de_cpi", "eu_cpi_yoy", "preliminary"),
|
||||
(847, "CPI月率初值"): ("de_cpi", "eu_cpi_mom", "preliminary"),
|
||||
(839, "调和CPI年率初值"): ("de_cpi", "hicp_yoy", "preliminary"),
|
||||
(840, "调和CPI月率初值"): ("de_cpi", "hicp_mom", "preliminary"),
|
||||
(844, "CPI年率终值"): ("de_cpi", "eu_cpi_yoy", "final"),
|
||||
(845, "CPI月率终值"): ("de_cpi", "eu_cpi_mom", "final"),
|
||||
(837, "调和CPI年率终值"): ("de_cpi", "hicp_yoy", "final"),
|
||||
(838, "调和CPI月率终值"): ("de_cpi", "hicp_mom", "final"),
|
||||
(245, "季调后GDP季率初值"): ("de_gdp", "gdp_qoq", "initial"),
|
||||
(843, "工作日调整后GDP年率初值"): ("de_gdp", "gdp_yoy", "initial"),
|
||||
}
|
||||
GB = {
|
||||
(649, "三个月ILO失业率"): ("uk_jobs", "uk_unemployment", None),
|
||||
(390, "GDP年率初值"): ("uk_gdp", "gdp_yoy", "initial"),
|
||||
(391, "GDP季率初值"): ("uk_gdp", "gdp_qoq", "initial"),
|
||||
(385, "GDP年率终值"): ("uk_gdp", "gdp_yoy", "final"),
|
||||
(658, "GDP季率终值"): ("uk_gdp", "gdp_qoq", "final"),
|
||||
(344, "CPI年率"): ("uk_cpi", "eu_cpi_yoy", None),
|
||||
(348, "CPI月率"): ("uk_cpi", "eu_cpi_mom", None),
|
||||
(349, "核心CPI年率"): ("uk_cpi", "eu_core_cpi_yoy", None),
|
||||
(638, "央行利率决定"): ("boe_rate", "bank_rate", None),
|
||||
}
|
||||
MAPPINGS = {"US": US, "EA": EA, "FR": FR, "DE": DE, "GB": GB}
|
||||
|
||||
|
||||
def definition(meta: dict) -> tuple[str, str, str | None] | None:
|
||||
if meta["kind"] != "data" or meta["precision"] != "minute":
|
||||
return None
|
||||
mapping = MAPPINGS.get(meta["country_code"], {})
|
||||
match = mapping.get((int(meta["indicator"].removeprefix("jin10_")), meta["name"]))
|
||||
# Jin10 omits units for the verified Michigan index, whose official series is an index in points.
|
||||
unit = "点" if meta["unit"] is None and meta["country_code"] == "US" and meta["indicator"] in ("jin10_198", "jin10_208") else meta["unit"]
|
||||
return match if match and unit == INDICATORS[match[1]].unit else None
|
||||
|
||||
|
||||
def provenance(source: str, at: str | datetime | None, url: str) -> dict:
|
||||
return {"source": source, "observed_at": at, "url": url}
|
||||
|
||||
|
||||
def jin10_item(row: EconomicRelease, now: datetime) -> dict:
|
||||
meta = row.values["_jin10"]
|
||||
key = meta["indicator"]
|
||||
value = row.values.get(key, {})
|
||||
actual, previous, forecast = value.get("actual"), value.get("previous"), value.get("forecast")
|
||||
revised = value.get("previous_revised")
|
||||
match = definition(meta)
|
||||
eligible = value.get("forecast_before_release", False)
|
||||
if meta["kind"] != "data":
|
||||
status = "recorded"
|
||||
direction, explanation = "uncertain", "日程记录不代表事件已经发生;需结合实际消息判断 BTC/ETH 影响。"
|
||||
else:
|
||||
status = "released" if actual is not None else "historical" if (now - row.scheduled_at).days > 45 else "scheduled" if now < row.scheduled_at else "delayed"
|
||||
direction, explanation = direction_for(match[1] if match else None, actual, revised if revised is not None else previous, forecast if eligible else None)
|
||||
observed = value.get("observed_at", {})
|
||||
fields = {field: provenance("jin10", observed.get(field), row.schedule_url)
|
||||
for field in ("actual", "previous", "forecast", "previous_revised") if value.get(field) is not None}
|
||||
fields["scheduled_at"] = provenance("jin10", row.schedule_checked_at, row.schedule_url)
|
||||
if revised is not None:
|
||||
if "previous" in fields:
|
||||
fields["previous_original"] = fields["previous"]
|
||||
fields["previous"] = fields["previous_revised"]
|
||||
return {"release_id": row.id, "indicator": key, "name": meta["name"], "unit": meta["unit"],
|
||||
"country_code": meta["country_code"], "country_name": meta["country_name"],
|
||||
"event_name": f"{meta['country_name']}{meta['period_label'] or ''}{meta['name']}",
|
||||
"period": None, "period_label": meta["period_label"], "stage": None, "stage_name": None,
|
||||
"title": meta["description"] or meta["name"], "source": "jin10", "scheduled_at": row.scheduled_at,
|
||||
"calendar_date": meta["date"], "time_precision": meta["precision"], "kind": meta["kind"],
|
||||
"importance": meta["importance"], "importance_source": "jin10", "status": status,
|
||||
"actual": actual, "previous": revised if revised is not None else previous,
|
||||
"previous_original": previous, "previous_revised": revised, "forecast": forecast,
|
||||
"previous_basis": "金十列示的修订前值(原前值单独保留)" if revised is not None else "金十本次记录列示前值" if previous is not None else None,
|
||||
"forecast_before_release": eligible, "forecast_observed_at": observed.get("forecast"),
|
||||
"impact": direction, "explanation": explanation, "impact_basis": "forecast" if forecast is not None and eligible else "previous" if previous is not None or revised is not None else None,
|
||||
"release_url": None, "schedule_url": row.schedule_url, "first_received_at": row.first_received_at,
|
||||
"last_checked_at": row.last_checked_at, "error": row.error, "revision_no": row.revision_no,
|
||||
"field_sources": fields, "related_release_id": None, "related_revision_no": None}
|
||||
|
||||
|
||||
def direction_for(key: str | None, actual: str | None, previous: str | None, forecast: str | None) -> tuple[str, str]:
|
||||
if key is None:
|
||||
return "uncertain", "尚无经核实的 BTC/ETH 传导判断;金十的黄金/美元影响标签未用于加密资产。"
|
||||
try:
|
||||
if any(not Decimal(v).is_finite() for v in (actual, previous, forecast) if v is not None):
|
||||
raise InvalidOperation()
|
||||
return impact(key, actual, previous, forecast)
|
||||
except InvalidOperation:
|
||||
return "uncertain", "公布值或比较基准不是单一数值,暂不能判断 BTC/ETH 方向。"
|
||||
|
||||
|
||||
def merge_items(official: list[tuple[EconomicRelease, dict]], third_party: list[EconomicRelease], now: datetime) -> list[dict]:
|
||||
output = []
|
||||
used = set()
|
||||
for row, original in official:
|
||||
item = deepcopy(original)
|
||||
item.update(kind="data", time_precision="minute", calendar_date=row.scheduled_at.astimezone(LOCAL).date().isoformat(),
|
||||
importance_source="project", field_sources={field: provenance(row.source, row.first_received_at, row.release_url or row.schedule_url)
|
||||
for field in ("actual", "previous", "forecast") if item[field] is not None},
|
||||
related_release_id=None, related_revision_no=None, period_label=None, previous_original=None, previous_revised=None,
|
||||
forecast_before_release=False, forecast_observed_at=None)
|
||||
item["field_sources"]["scheduled_at"] = provenance(row.source, row.schedule_checked_at, row.schedule_url)
|
||||
matches = []
|
||||
for extra in third_party:
|
||||
meta = extra.values["_jin10"]
|
||||
match = definition(meta)
|
||||
if not match or match != (row.family, item["indicator"], row.stage):
|
||||
continue
|
||||
if FAMILIES[row.family].source != row.source or extra.scheduled_at != row.scheduled_at:
|
||||
continue
|
||||
# Rate decisions have no statistical reference period. All data series must agree on it.
|
||||
if row.family not in ("fomc", "ecb_rate", "boe_rate") and (row.period is None or period(meta["period_label"], extra.scheduled_at) != row.period):
|
||||
continue
|
||||
matches.append(extra)
|
||||
if len(matches) == 1:
|
||||
extra = matches[0]
|
||||
fallback = jin10_item(extra, now)
|
||||
used.add(extra.id)
|
||||
item.update(related_release_id=extra.id, related_revision_no=extra.revision_no,
|
||||
previous_original=fallback["previous_original"], previous_revised=fallback["previous_revised"],
|
||||
forecast_before_release=fallback["forecast_before_release"], forecast_observed_at=fallback["forecast_observed_at"])
|
||||
for field in ("actual", "previous", "forecast"):
|
||||
if item[field] is None and fallback[field] is not None:
|
||||
item[field] = fallback[field]
|
||||
provider_field = "previous_revised" if field == "previous" and fallback["previous_revised"] is not None else field
|
||||
item["field_sources"][field] = fallback["field_sources"][provider_field]
|
||||
if field == "previous":
|
||||
item["previous_basis"] = fallback["previous_basis"]
|
||||
if field == "actual":
|
||||
item["first_received_at"] = fallback["first_received_at"]
|
||||
if item["actual"] is not None and item["status"] != "withdrawn":
|
||||
item["status"] = "released"
|
||||
forecast = item["forecast"] if item.get("forecast_before_release") else None
|
||||
item["impact"], item["explanation"] = direction_for(item["indicator"], item["actual"], item["previous"], forecast)
|
||||
item["impact_basis"] = "forecast" if forecast is not None else "previous" if item["previous"] is not None else None
|
||||
output.append(item)
|
||||
output.extend(jin10_item(row, now) for row in third_party if row.id not in used)
|
||||
return sorted(output, key=lambda r: (r["calendar_date"], r["scheduled_at"], r["release_id"], r["indicator"]))
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Independent browser collector: live dates take priority over resumable daily history."""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from alphapulse.calendar.jin10 import KINDS, LOCAL, START, day_url, normalize
|
||||
from alphapulse.calendar.jin10_auth import Jin10Error
|
||||
from alphapulse.calendar.jin10_browser import Jin10Browser
|
||||
from alphapulse.calendar.jin10_store import mark_day, next_history_day, save_entry
|
||||
from alphapulse.calendar.service import save_raw
|
||||
from alphapulse.core.crypto import decrypt
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.models import Source
|
||||
from alphapulse.models.jin10 import Jin10Connection
|
||||
|
||||
logger = logging.getLogger("alphapulse.calendar.jin10")
|
||||
STATUS_KEY = f"{KEY_PREFIX}calendar:jin10"
|
||||
|
||||
|
||||
class ConnectionChanged(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def same_connection(row: Jin10Connection | None, encrypted: bytes, connected_at: datetime) -> bool:
|
||||
return bool(row and row.token_encrypted == encrypted and row.connected_at == connected_at)
|
||||
|
||||
|
||||
async def persist_day(day: date, responses: dict, encrypted: bytes, connected_at: datetime, now: datetime) -> dict:
|
||||
raw_ids = {}
|
||||
async with SessionLocal() as session:
|
||||
# Disconnect/reconnect takes this same lock. An obsolete fetch must never commit data or progress.
|
||||
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
|
||||
if not same_connection(row, encrypted, connected_at):
|
||||
raise ConnectionChanged()
|
||||
for kind in KINDS:
|
||||
raw = json.dumps(responses[kind], ensure_ascii=False, separators=(",", ":")).encode()
|
||||
raw_ids[kind] = await save_raw(session, "jin10", day_url(day) + f"#{kind}", 200,
|
||||
{"content-type": "application/json", "transport": "jin10-official-browser-sdk"}, raw, now)
|
||||
await session.commit()
|
||||
# Retain failed contract responses as evidence, but never advance coverage for them.
|
||||
entries = {kind: normalize(responses[kind], kind, day) for kind in KINDS}
|
||||
async with SessionLocal() as session:
|
||||
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
|
||||
if not same_connection(row, encrypted, connected_at):
|
||||
raise ConnectionChanged()
|
||||
for kind in KINDS:
|
||||
for entry in entries[kind]:
|
||||
await save_entry(session, entry, raw_ids[kind], now)
|
||||
counts = {kind: len(entries[kind]) for kind in KINDS}
|
||||
await mark_day(session, day, counts, now)
|
||||
await session.commit()
|
||||
return counts
|
||||
|
||||
|
||||
class Jin10Worker:
|
||||
def __init__(self) -> None:
|
||||
self.browser = Jin10Browser()
|
||||
self.identity = None
|
||||
self.live_due: dict[date, float] = {}
|
||||
self.state = {"state": "disconnected", "error": None, "last_success_at": None}
|
||||
|
||||
async def heartbeat(self) -> None:
|
||||
while True:
|
||||
await redis.set(STATUS_KEY, json.dumps({**self.state, "at": datetime.now(UTC).isoformat()}), ex=60)
|
||||
await asyncio.sleep(5)
|
||||
|
||||
async def step(self) -> None:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Jin10Connection, 1)
|
||||
source = await session.get(Source, "jin10")
|
||||
identity = (row.token_encrypted, row.connected_at) if row and row.token_encrypted and source and source.enabled else None
|
||||
if identity is None:
|
||||
await self.browser.close()
|
||||
self.identity = None
|
||||
self.state.update(state="disconnected", error=None, connected_at=None)
|
||||
await asyncio.sleep(5)
|
||||
return
|
||||
if identity != self.identity:
|
||||
self.state = {"state": "connecting", "error": None, "connected_at": identity[1].isoformat(), "last_success_at": None}
|
||||
await self.browser.connect(decrypt(identity[0]))
|
||||
self.identity = identity
|
||||
self.live_due.clear()
|
||||
today = datetime.now(LOCAL).date()
|
||||
dates = [today + timedelta(days=i) for i in range(-1, 8)]
|
||||
self.live_due = {d: self.live_due.get(d, 0) for d in dates if d >= START}
|
||||
due = sorted((deadline, d) for d, deadline in self.live_due.items() if deadline <= time.monotonic())
|
||||
if due:
|
||||
# Refresh today first when a process starts, then adjacent days in deadline order.
|
||||
day = today if self.live_due.get(today, float("inf")) == 0 else due[0][1]
|
||||
history = False
|
||||
else:
|
||||
async with SessionLocal() as session:
|
||||
day = await next_history_day(session, START, today - timedelta(days=1))
|
||||
history = True
|
||||
if day is None:
|
||||
self.state.update(state="ready", history_next=None)
|
||||
await asyncio.sleep(2)
|
||||
return
|
||||
self.state.update(state="syncing", current_day=day.isoformat(), history=history, error=None)
|
||||
responses = await self.browser.fetch_day(day)
|
||||
counts = await persist_day(day, responses, *identity, datetime.now(UTC))
|
||||
if not history:
|
||||
self.live_due[day] = time.monotonic() + (10 if day == today else 300)
|
||||
self.state.update(state="ready", last_success_at=datetime.now(UTC).isoformat(), counts=counts)
|
||||
if history:
|
||||
self.state["history_next"] = (day + timedelta(days=1)).isoformat()
|
||||
# A single collector owns all retries and limits upstream traffic, including backfill.
|
||||
await asyncio.sleep(5)
|
||||
|
||||
async def collect(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self.step()
|
||||
except ConnectionChanged:
|
||||
self.identity = None
|
||||
self.state.update(state="disconnected", error=None)
|
||||
except (Jin10Error, ValueError, KeyError) as exc:
|
||||
self.state.update(state="error", error=str(exc))
|
||||
logger.warning("金十日历同步失败:%s", exc)
|
||||
self.identity = None
|
||||
await self.browser.close()
|
||||
await asyncio.sleep(30)
|
||||
|
||||
async def run(self) -> None:
|
||||
try:
|
||||
await asyncio.gather(self.heartbeat(), self.collect())
|
||||
finally:
|
||||
await self.browser.close()
|
||||
await redis.aclose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
asyncio.run(Jin10Worker().run())
|
||||
@@ -170,6 +170,9 @@ def impact(key: str, actual: str | None, previous: str | None, forecast: str | N
|
||||
|
||||
|
||||
def release_rows(row: EconomicRelease, now: datetime) -> list[dict]:
|
||||
if row.source == "jin10":
|
||||
from alphapulse.calendar.jin10_view import jin10_item
|
||||
return [jin10_item(row, now)]
|
||||
if row.schedule_status == "withdrawn":
|
||||
state = "withdrawn"
|
||||
elif row.first_received_at is not None:
|
||||
@@ -221,17 +224,30 @@ async def indicator_history(session: AsyncSession, row: EconomicRelease, key: st
|
||||
.where(EconomicRevision.release_id == EconomicRelease.id,
|
||||
EconomicRevision.payload["values"][key]["actual"].as_string().is_not(None))
|
||||
.order_by(EconomicRevision.revision_no).limit(1).correlate(EconomicRelease).scalar_subquery())
|
||||
filters = []
|
||||
if row.source == "jin10":
|
||||
meta = row.values["_jin10"]
|
||||
filters = [EconomicRelease.values["_jin10"][field].as_string().is_not_distinct_from(meta[field])
|
||||
for field in ("country_code", "name", "unit")]
|
||||
rows = (await session.execute(select(EconomicRelease, first_actual).where(
|
||||
EconomicRelease.source == row.source, EconomicRelease.family == row.family,
|
||||
EconomicRelease.stage.is_not_distinct_from(row.stage),
|
||||
EconomicRelease.scheduled_at <= row.scheduled_at,
|
||||
EconomicRelease.schedule_status != "withdrawn", EconomicRelease.first_received_at.is_not(None),
|
||||
EconomicRelease.values[key]["actual"].as_string().is_not(None),
|
||||
*filters,
|
||||
).order_by(EconomicRelease.scheduled_at.desc(), EconomicRelease.id.desc()).limit(limit))).all()
|
||||
return [{"release_id": item.id, "scheduled_at": item.scheduled_at, "title": item.title,
|
||||
"period": item.period, "stage": item.stage,
|
||||
"actual": item.values[key]["actual"], "previous": item.values[key].get("previous"),
|
||||
"forecast": item.values[key].get("forecast"), "first_recorded_actual": initial,
|
||||
"first_received_at": item.first_received_at, "release_url": item.release_url,
|
||||
"revised": initial is not None and Decimal(initial) != Decimal(item.values[key]["actual"])}
|
||||
for item, initial in rows]
|
||||
output = []
|
||||
for item, initial in rows:
|
||||
value = item.values[key]
|
||||
view = release_rows(item, datetime.now(item.scheduled_at.tzinfo))[0] if item.source == "jin10" else None
|
||||
output.append({"release_id": item.id, "scheduled_at": item.scheduled_at, "title": item.title,
|
||||
"period": item.period, "stage": item.stage,
|
||||
"actual": value["actual"], "previous": view["previous"] if view else value.get("previous"),
|
||||
"forecast": value.get("forecast"), "first_recorded_actual": initial,
|
||||
"first_received_at": item.first_received_at, "release_url": item.schedule_url if view else item.release_url,
|
||||
"period_label": item.values["_jin10"]["period_label"] if view else None,
|
||||
"forecast_before_release": value.get("forecast_before_release", False),
|
||||
"forecast_observed_at": value.get("observed_at", {}).get("forecast"),
|
||||
"revised": initial is not None and (initial != value["actual"] if view else Decimal(initial) != Decimal(value["actual"]))})
|
||||
return output
|
||||
@@ -374,6 +374,7 @@ class CalendarWorker:
|
||||
now = datetime.now(UTC)
|
||||
async with SessionLocal() as session:
|
||||
ids = list(await session.scalars(select(EconomicRelease.id).where(
|
||||
EconomicRelease.source != "jin10",
|
||||
EconomicRelease.schedule_status == "scheduled", EconomicRelease.scheduled_at <= now,
|
||||
EconomicRelease.scheduled_at >= now - timedelta(days=HISTORY_DAYS), EconomicRelease.next_check_at <= now,
|
||||
).order_by(EconomicRelease.scheduled_at.desc()).limit(8)))
|
||||
|
||||
@@ -19,7 +19,7 @@ FEEDS = {
|
||||
"insee": ("https://www.insee.fr/fr/flux/3",),
|
||||
}
|
||||
REGIONS = {"wscn_news": "CN", "wscn_live": "CN", "cls_depth": "CN", "cls_telegraph": "CN", "coindesk": "GLOBAL",
|
||||
"bls": "US", "bea": "US", "fed": "US", "umich": "US", "dol": "US", "eia": "US", "adp": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR"}
|
||||
"bls": "US", "bea": "US", "fed": "US", "umich": "US", "dol": "US", "eia": "US", "adp": "US", "ecb": "EA", "eurostat": "EA", "boe": "GB", "ons": "GB", "destatis": "DE", "insee": "FR", "jin10": "GLOBAL"}
|
||||
HOSTS = {"ecb": {"www.ecb.europa.eu"}, "eurostat": {"ec.europa.eu"}, "boe": {"www.bankofengland.co.uk"},
|
||||
"ons": {"www.ons.gov.uk"}, "destatis": {"www.destatis.de"}, "insee": {"www.insee.fr"}}
|
||||
LANGUAGES = {"destatis": "de", "insee": "fr"}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from alphapulse.models.jin10 import Jin10Connection
|
||||
from alphapulse.models.jin10 import Jin10Connection, Jin10SyncDay
|
||||
from alphapulse.models.risk import RiskSettings, RiskEvent, RiskAnalysis, RiskSnapshot
|
||||
from alphapulse.core.db import Base
|
||||
from alphapulse.models.ai import AiProvider, ChatMessage, ChatSession, ContextVersion, SkillNode, SkillVersion
|
||||
@@ -9,6 +9,7 @@ from alphapulse.models.trading import NewsSignal, TradingAccount, TradingAnalysi
|
||||
|
||||
__all__ = [
|
||||
"Jin10Connection",
|
||||
"Jin10SyncDay",
|
||||
"RiskSettings", "RiskEvent", "RiskAnalysis", "RiskSnapshot",
|
||||
"AiProvider",
|
||||
"Base",
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Encrypted connection and short-lived, session-owned QR login state."""
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import DateTime, LargeBinary, String, Text
|
||||
from sqlalchemy import Date, DateTime, LargeBinary, String, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from alphapulse.core.db import Base
|
||||
@@ -19,3 +20,12 @@ class Jin10Connection(Base):
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
next_poll_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class Jin10SyncDay(Base):
|
||||
"""A day is covered only after all three authenticated responses were validated."""
|
||||
__tablename__ = "jin10_sync_days"
|
||||
|
||||
day: Mapped[date] = mapped_column(Date, primary_key=True)
|
||||
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
counts: Mapped[dict] = mapped_column(JSONB)
|
||||
Reference in new issue
Block a user