Fix Jin10 calendar backfill, retries and withdrawn schedules

Refresh historical previews after the local day ends and isolate retries by date without resetting browser scheduling progress.

Preserve withdrawal and restoration revisions, exclude withdrawn fallback values, and cover the behavior with isolated regression checks.

backend/src/alphapulse/calendar/jin10_store.py

@@ -5,7 +5,7 @@ from sqlalchemy import select
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from alphapulse.calendar.jin10 import Entry, day_url
+from alphapulse.calendar.jin10 import LOCAL, Entry, day_url
 from alphapulse.calendar.service import record_revision
 from alphapulse.models import EconomicRelease
 from alphapulse.models.jin10 import Jin10SyncDay
@@ -43,7 +43,7 @@ async def save_entry(session: AsyncSession, entry: Entry, raw_id: int, now: date
         await session.flush()
         record_revision(session, row, raw_id)
         return
-    changed = row.values != values or row.scheduled_at != entry.at or row.title != entry.title
+    changed = row.values != values or row.scheduled_at != entry.at or row.title != entry.title or row.schedule_status != "scheduled"
     row.schedule_checked_at = row.last_checked_at = now
     row.error = None
     if row.first_received_at is None and received:
@@ -51,22 +51,42 @@ async def save_entry(session: AsyncSession, entry: Entry, raw_id: int, now: date
         changed = True
     if changed:
         row.title, row.scheduled_at, row.values = entry.title, entry.at, values
+        row.schedule_status = "scheduled"
         row.schedule_url = day_url(date.fromisoformat(meta["date"]))
         row.revision_no += 1
         record_revision(session, row, raw_id)
 
 
+async def withdraw_missing(session: AsyncSession, day: date, kind: str, entries: list[Entry], raw_id: int, now: datetime) -> None:
+    # Only a fully validated response may withdraw records from its own day/category.
+    seen = {entry.identity for entry in entries}
+    rows = await session.scalars(select(EconomicRelease).where(
+        EconomicRelease.source == "jin10",
+        EconomicRelease.values["_jin10"]["date"].as_string() == day.isoformat(),
+        EconomicRelease.values["_jin10"]["kind"].as_string() == kind,
+        EconomicRelease.schedule_status != "withdrawn",
+    ).with_for_update())
+    for row in rows:
+        if row.external_id not in seen:
+            row.schedule_status = "withdrawn"
+            row.schedule_checked_at = row.last_checked_at = now
+            row.revision_no += 1
+            record_revision(session, row, raw_id)
+
+
 async def mark_day(session: AsyncSession, day: date, counts: dict, now: datetime) -> None:
     stmt = insert(Jin10SyncDay).values(day=day, captured_at=now, counts=counts)
     await session.execute(stmt.on_conflict_do_update(index_elements=["day"], set_={"captured_at": now, "counts": counts}))
 
 
-async def next_history_day(session: AsyncSession, start: date, end: date) -> date | None:
+async def next_history_day(session: AsyncSession, start: date, end: date, *, excluded: set[date] | None = None) -> date | None:
     from datetime import timedelta
-    covered = set(await session.scalars(select(Jin10SyncDay.day).where(Jin10SyncDay.day.between(start, end))))
+    snapshots = await session.scalars(select(Jin10SyncDay).where(Jin10SyncDay.day.between(start, end)))
+    # A preview captured before the local day ended is not a completed historical snapshot.
+    covered = {row.day for row in snapshots if row.captured_at.astimezone(LOCAL).date() > row.day}
     day = start
     while day <= end:
-        if day not in covered:
+        if day not in covered and day not in (excluded or set()):
             return day
         day += timedelta(days=1)
     return None

backend/src/alphapulse/calendar/jin10_view.py

@@ -115,7 +115,10 @@ def jin10_item(row: EconomicRelease, now: datetime) -> dict:
     revised = value.get("previous_revised")
     match = definition(meta)
     eligible = value.get("forecast_before_release", False)
-    if meta["kind"] != "data":
+    if row.schedule_status == "withdrawn":
+        status = "withdrawn"
+        direction, explanation = "uncertain", "金十已移除此日程；已保存值仅供追溯，不作为当前有效日程或方向判断。"
+    elif meta["kind"] != "data":
         status = "recorded"
         direction, explanation = "uncertain", "日程记录不代表事件已经发生；需结合实际消息判断 BTC/ETH 影响。"
     else:
@@ -170,6 +173,8 @@ def merge_items(official: list[tuple[EconomicRelease, dict]], third_party: list[
         item["field_sources"]["scheduled_at"] = provenance(row.source, row.schedule_checked_at, row.schedule_url)
         matches = []
         for extra in third_party:
+            if extra.schedule_status == "withdrawn":
+                continue
             meta = extra.values["_jin10"]
             match = definition(meta)
             if not match or match != (row.family, item["indicator"], row.stage):

backend/src/alphapulse/calendar/jin10_worker.py

@@ -10,7 +10,7 @@ from sqlalchemy import select
 from alphapulse.calendar.jin10 import KINDS, LOCAL, START, day_url, normalize
 from alphapulse.calendar.jin10_auth import Jin10Error
 from alphapulse.calendar.jin10_browser import Jin10Browser
-from alphapulse.calendar.jin10_store import mark_day, next_history_day, save_entry
+from alphapulse.calendar.jin10_store import mark_day, next_history_day, save_entry, withdraw_missing
 from alphapulse.calendar.service import save_raw
 from alphapulse.core.crypto import decrypt
 from alphapulse.core.db import SessionLocal
@@ -51,6 +51,7 @@ async def persist_day(day: date, responses: dict, encrypted: bytes, connected_at
         for kind in KINDS:
             for entry in entries[kind]:
                 await save_entry(session, entry, raw_ids[kind], now)
+            await withdraw_missing(session, day, kind, entries[kind], raw_ids[kind], now)
         counts = {kind: len(entries[kind]) for kind in KINDS}
         await mark_day(session, day, counts, now)
         await session.commit()
@@ -62,6 +63,7 @@ class Jin10Worker:
         self.browser = Jin10Browser()
         self.identity = None
         self.live_due: dict[date, float] = {}
+        self.retry_due: dict[date, float] = {}
         self.state = {"state": "disconnected", "error": None, "last_success_at": None}
 
     async def heartbeat(self) -> None:
@@ -77,6 +79,8 @@ class Jin10Worker:
         if identity is None:
             await self.browser.close()
             self.identity = None
+            self.live_due.clear()
+            self.retry_due.clear()
             self.state.update(state="disconnected", error=None, connected_at=None)
             await asyncio.sleep(5)
             return
@@ -84,8 +88,8 @@ class Jin10Worker:
             self.state = {"state": "connecting", "error": None, "connected_at": identity[1].isoformat(), "last_success_at": None}
             await self.browser.connect(decrypt(identity[0]))
             self.identity = identity
-            self.live_due.clear()
         today = datetime.now(LOCAL).date()
+        self.retry_due = {d: deadline for d, deadline in self.retry_due.items() if deadline > time.monotonic()}
         dates = [today + timedelta(days=i) for i in range(-1, 8)]
         self.live_due = {d: self.live_due.get(d, 0) for d in dates if d >= START}
         due = sorted((deadline, d) for d, deadline in self.live_due.items() if deadline <= time.monotonic())
@@ -95,15 +99,29 @@ class Jin10Worker:
             history = False
         else:
             async with SessionLocal() as session:
-                day = await next_history_day(session, START, today - timedelta(days=1))
+                day = await next_history_day(session, START, today - timedelta(days=1), excluded=set(self.retry_due))
             history = True
         if day is None:
             self.state.update(state="ready", history_next=None)
             await asyncio.sleep(2)
             return
         self.state.update(state="syncing", current_day=day.isoformat(), history=history, error=None)
-        responses = await self.browser.fetch_day(day)
-        counts = await persist_day(day, responses, *identity, datetime.now(UTC))
+        try:
+            responses = await self.browser.fetch_day(day)
+            counts = await persist_day(day, responses, *identity, datetime.now(UTC))
+        except (Jin10Error, ValueError, KeyError) as exc:
+            # A bad day must not reset progress or starve the remaining live/history dates.
+            self.retry_due[day] = time.monotonic() + 300
+            if not history:
+                self.live_due[day] = self.retry_due[day]
+            self.state.update(state="error", error=str(exc))
+            logger.warning("金十日历同步失败（%s）：%s", day, exc)
+            if isinstance(exc, Jin10Error):
+                self.identity = None
+                await self.browser.close()
+            await asyncio.sleep(5)
+            return
+        self.retry_due.pop(day, None)
         if not history:
             self.live_due[day] = time.monotonic() + (10 if day == today else 300)
         self.state.update(state="ready", last_success_at=datetime.now(UTC).isoformat(), counts=counts)