eezd
c79522ee7c
Refresh historical previews after the local day ends and isolate retries by date without resetting browser scheduling progress. Preserve withdrawal and restoration revisions, exclude withdrawn fallback values, and cover the behavior with isolated regression checks.
159 lines
7.3 KiB
Python
159 lines
7.3 KiB
Python
"""Independent browser collector: live dates take priority over resumable daily history."""
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import time
|
|
from datetime import UTC, date, datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from alphapulse.calendar.jin10 import KINDS, LOCAL, START, day_url, normalize
|
|
from alphapulse.calendar.jin10_auth import Jin10Error
|
|
from alphapulse.calendar.jin10_browser import Jin10Browser
|
|
from alphapulse.calendar.jin10_store import mark_day, next_history_day, save_entry, withdraw_missing
|
|
from alphapulse.calendar.service import save_raw
|
|
from alphapulse.core.crypto import decrypt
|
|
from alphapulse.core.db import SessionLocal
|
|
from alphapulse.core.redis import KEY_PREFIX, redis
|
|
from alphapulse.models import Source
|
|
from alphapulse.models.jin10 import Jin10Connection
|
|
|
|
logger = logging.getLogger("alphapulse.calendar.jin10")
|
|
STATUS_KEY = f"{KEY_PREFIX}calendar:jin10"
|
|
|
|
|
|
class ConnectionChanged(Exception):
|
|
pass
|
|
|
|
|
|
def same_connection(row: Jin10Connection | None, encrypted: bytes, connected_at: datetime) -> bool:
|
|
return bool(row and row.token_encrypted == encrypted and row.connected_at == connected_at)
|
|
|
|
|
|
async def persist_day(day: date, responses: dict, encrypted: bytes, connected_at: datetime, now: datetime) -> dict:
|
|
raw_ids = {}
|
|
async with SessionLocal() as session:
|
|
# Disconnect/reconnect takes this same lock. An obsolete fetch must never commit data or progress.
|
|
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
|
|
if not same_connection(row, encrypted, connected_at):
|
|
raise ConnectionChanged()
|
|
for kind in KINDS:
|
|
raw = json.dumps(responses[kind], ensure_ascii=False, separators=(",", ":")).encode()
|
|
raw_ids[kind] = await save_raw(session, "jin10", day_url(day) + f"#{kind}", 200,
|
|
{"content-type": "application/json", "transport": "jin10-official-browser-sdk"}, raw, now)
|
|
await session.commit()
|
|
# Retain failed contract responses as evidence, but never advance coverage for them.
|
|
entries = {kind: normalize(responses[kind], kind, day) for kind in KINDS}
|
|
async with SessionLocal() as session:
|
|
row = await session.scalar(select(Jin10Connection).where(Jin10Connection.id == 1).with_for_update())
|
|
if not same_connection(row, encrypted, connected_at):
|
|
raise ConnectionChanged()
|
|
for kind in KINDS:
|
|
for entry in entries[kind]:
|
|
await save_entry(session, entry, raw_ids[kind], now)
|
|
await withdraw_missing(session, day, kind, entries[kind], raw_ids[kind], now)
|
|
counts = {kind: len(entries[kind]) for kind in KINDS}
|
|
await mark_day(session, day, counts, now)
|
|
await session.commit()
|
|
return counts
|
|
|
|
|
|
class Jin10Worker:
|
|
def __init__(self) -> None:
|
|
self.browser = Jin10Browser()
|
|
self.identity = None
|
|
self.live_due: dict[date, float] = {}
|
|
self.retry_due: dict[date, float] = {}
|
|
self.state = {"state": "disconnected", "error": None, "last_success_at": None}
|
|
|
|
async def heartbeat(self) -> None:
|
|
while True:
|
|
await redis.set(STATUS_KEY, json.dumps({**self.state, "at": datetime.now(UTC).isoformat()}), ex=60)
|
|
await asyncio.sleep(5)
|
|
|
|
async def step(self) -> None:
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Jin10Connection, 1)
|
|
source = await session.get(Source, "jin10")
|
|
identity = (row.token_encrypted, row.connected_at) if row and row.token_encrypted and source and source.enabled else None
|
|
if identity is None:
|
|
await self.browser.close()
|
|
self.identity = None
|
|
self.live_due.clear()
|
|
self.retry_due.clear()
|
|
self.state.update(state="disconnected", error=None, connected_at=None)
|
|
await asyncio.sleep(5)
|
|
return
|
|
if identity != self.identity:
|
|
self.state = {"state": "connecting", "error": None, "connected_at": identity[1].isoformat(), "last_success_at": None}
|
|
await self.browser.connect(decrypt(identity[0]))
|
|
self.identity = identity
|
|
today = datetime.now(LOCAL).date()
|
|
self.retry_due = {d: deadline for d, deadline in self.retry_due.items() if deadline > time.monotonic()}
|
|
dates = [today + timedelta(days=i) for i in range(-1, 8)]
|
|
self.live_due = {d: self.live_due.get(d, 0) for d in dates if d >= START}
|
|
due = sorted((deadline, d) for d, deadline in self.live_due.items() if deadline <= time.monotonic())
|
|
if due:
|
|
# Refresh today first when a process starts, then adjacent days in deadline order.
|
|
day = today if self.live_due.get(today, float("inf")) == 0 else due[0][1]
|
|
history = False
|
|
else:
|
|
async with SessionLocal() as session:
|
|
day = await next_history_day(session, START, today - timedelta(days=1), excluded=set(self.retry_due))
|
|
history = True
|
|
if day is None:
|
|
self.state.update(state="ready", history_next=None)
|
|
await asyncio.sleep(2)
|
|
return
|
|
self.state.update(state="syncing", current_day=day.isoformat(), history=history, error=None)
|
|
try:
|
|
responses = await self.browser.fetch_day(day)
|
|
counts = await persist_day(day, responses, *identity, datetime.now(UTC))
|
|
except (Jin10Error, ValueError, KeyError) as exc:
|
|
# A bad day must not reset progress or starve the remaining live/history dates.
|
|
self.retry_due[day] = time.monotonic() + 300
|
|
if not history:
|
|
self.live_due[day] = self.retry_due[day]
|
|
self.state.update(state="error", error=str(exc))
|
|
logger.warning("金十日历同步失败（%s）：%s", day, exc)
|
|
if isinstance(exc, Jin10Error):
|
|
self.identity = None
|
|
await self.browser.close()
|
|
await asyncio.sleep(5)
|
|
return
|
|
self.retry_due.pop(day, None)
|
|
if not history:
|
|
self.live_due[day] = time.monotonic() + (10 if day == today else 300)
|
|
self.state.update(state="ready", last_success_at=datetime.now(UTC).isoformat(), counts=counts)
|
|
if history:
|
|
self.state["history_next"] = (day + timedelta(days=1)).isoformat()
|
|
# A single collector owns all retries and limits upstream traffic, including backfill.
|
|
await asyncio.sleep(5)
|
|
|
|
async def collect(self) -> None:
|
|
while True:
|
|
try:
|
|
await self.step()
|
|
except ConnectionChanged:
|
|
self.identity = None
|
|
self.state.update(state="disconnected", error=None)
|
|
except (Jin10Error, ValueError, KeyError) as exc:
|
|
self.state.update(state="error", error=str(exc))
|
|
logger.warning("金十日历同步失败：%s", exc)
|
|
self.identity = None
|
|
await self.browser.close()
|
|
await asyncio.sleep(30)
|
|
|
|
async def run(self) -> None:
|
|
try:
|
|
await asyncio.gather(self.heartbeat(), self.collect())
|
|
finally:
|
|
await self.browser.close()
|
|
await redis.aclose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
asyncio.run(Jin10Worker().run())
|