105 lines
5.5 KiB
Python
105 lines
5.5 KiB
Python
"""Keep provider identities, raw evidence, field observation times and revisions together."""
|
|
from datetime import date, datetime
|
|
|
|
from sqlalchemy import select, or_
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from alphapulse.calendar.jin10 import LOCAL, Entry, day_url
|
|
from alphapulse.calendar.service import record_revision
|
|
from alphapulse.models import EconomicRelease
|
|
from alphapulse.models.jin10 import Jin10SyncDay
|
|
|
|
|
|
async def save_entry(session: AsyncSession, entry: Entry, raw_id: int, now: datetime) -> None:
|
|
row = await session.scalar(select(EconomicRelease).where(
|
|
EconomicRelease.source == "jin10", EconomicRelease.external_id == entry.identity,
|
|
).with_for_update())
|
|
values = {key: dict(value) for key, value in entry.values.items()}
|
|
meta = values["_jin10"]
|
|
old_meta = row.values["_jin10"] if row and row.schedule_status != "withdrawn" else {}
|
|
source_dates = sorted(set(old_meta.get("source_dates", [old_meta["date"]] if old_meta else [])) | {meta["date"]})
|
|
if "source_dates" in old_meta or len(source_dates) > 1:
|
|
meta["source_dates"] = source_dates
|
|
key = meta["indicator"]
|
|
if meta["kind"] == "data":
|
|
current = values[key]
|
|
old = row.values.get(key, {}) if row else {}
|
|
observed = {}
|
|
for field in ("actual", "previous", "forecast", "previous_revised"):
|
|
if current[field] is not None:
|
|
observed[field] = old.get("observed_at", {}).get(field, now.isoformat()) if current[field] == old.get(field) else now.isoformat()
|
|
current["observed_at"] = observed
|
|
forecast_at = datetime.fromisoformat(observed["forecast"]) if "forecast" in observed else None
|
|
if forecast_at == now:
|
|
before = current["actual"] is None
|
|
else:
|
|
before = old.get("forecast_before_release", False)
|
|
current["forecast_before_release"] = bool(before and forecast_at and forecast_at < entry.at and meta["precision"] == "minute")
|
|
received = meta["kind"] != "data" or values[key]["actual"] is not None
|
|
if row is None:
|
|
row = EconomicRelease(source="jin10", external_id=entry.identity, family="jin10", title=entry.title,
|
|
scheduled_at=entry.at, schedule_status="scheduled", schedule_url=day_url(date.fromisoformat(meta["date"])),
|
|
release_url=None, period=None, stage=None, schedule_checked_at=now, last_checked_at=now,
|
|
first_received_at=now if received else None, next_check_at=now, error=None,
|
|
values=values, revision_no=1)
|
|
session.add(row)
|
|
await session.flush()
|
|
record_revision(session, row, raw_id)
|
|
return
|
|
changed = row.values != values or row.scheduled_at != entry.at or row.title != entry.title or row.schedule_status != "scheduled"
|
|
row.schedule_checked_at = row.last_checked_at = now
|
|
row.error = None
|
|
if row.first_received_at is None and received:
|
|
row.first_received_at = now
|
|
changed = True
|
|
if changed:
|
|
row.title, row.scheduled_at, row.values = entry.title, entry.at, values
|
|
row.schedule_status = "scheduled"
|
|
row.schedule_url = day_url(date.fromisoformat(meta["date"]))
|
|
row.revision_no += 1
|
|
record_revision(session, row, raw_id)
|
|
|
|
|
|
async def withdraw_missing(session: AsyncSession, day: date, kind: str, entries: list[Entry], raw_id: int, now: datetime) -> None:
|
|
# Only a fully validated response may withdraw records from its own day/category.
|
|
seen = {entry.identity for entry in entries}
|
|
rows = await session.scalars(select(EconomicRelease).where(
|
|
EconomicRelease.source == "jin10",
|
|
or_(EconomicRelease.values["_jin10"]["source_dates"].contains([day.isoformat()]),
|
|
(~EconomicRelease.values["_jin10"].has_key("source_dates")) &
|
|
(EconomicRelease.values["_jin10"]["date"].as_string() == day.isoformat())),
|
|
EconomicRelease.values["_jin10"]["kind"].as_string() == kind,
|
|
EconomicRelease.schedule_status != "withdrawn",
|
|
).with_for_update())
|
|
for row in rows:
|
|
if row.external_id not in seen:
|
|
meta = dict(row.values["_jin10"])
|
|
remaining = [d for d in meta.get("source_dates", [meta["date"]]) if d != day.isoformat()]
|
|
if "source_dates" in meta:
|
|
meta["source_dates"] = remaining
|
|
row.values = {**row.values, "_jin10": meta}
|
|
if not remaining:
|
|
row.schedule_status = "withdrawn"
|
|
row.schedule_checked_at = row.last_checked_at = now
|
|
row.revision_no += 1
|
|
record_revision(session, row, raw_id)
|
|
|
|
|
|
async def mark_day(session: AsyncSession, day: date, counts: dict, now: datetime) -> None:
|
|
stmt = insert(Jin10SyncDay).values(day=day, captured_at=now, counts=counts)
|
|
await session.execute(stmt.on_conflict_do_update(index_elements=["day"], set_={"captured_at": now, "counts": counts}))
|
|
|
|
|
|
async def next_history_day(session: AsyncSession, start: date, end: date, *, excluded: set[date] | None = None) -> date | None:
|
|
from datetime import timedelta
|
|
snapshots = await session.scalars(select(Jin10SyncDay).where(Jin10SyncDay.day.between(start, end)))
|
|
# A preview captured before the local day ended is not a completed historical snapshot.
|
|
covered = {row.day for row in snapshots if row.captured_at.astimezone(LOCAL).date() > row.day}
|
|
day = start
|
|
while day <= end:
|
|
if day not in covered and day not in (excluded or set()):
|
|
return day
|
|
day += timedelta(days=1)
|
|
return None
|