Add resumable news history backfill

Let the crawler console set a backfill target per source. Each scheduled
run pages backwards from a checkpoint for at most 20s so real-time
crawling is not blocked. The checkpoint is saved only after a successful
run and is also kept in process memory, because the next trigger may be
queued before the database write lands. cls_depth is excluded since its
list API cannot page into history.
This commit is contained in:
eezd committed 2026-10-13 14:58:48 +08:00
1 parent 0831971e83
commit 5c8d773ce1
10 files changed
+229 -17

No files matched your search

+6 -1
View File
@@ -159,7 +159,12 @@ class Manager:
continue
# 子进程还没取走上一条触发时不再堆积
if await redis.llen(start_urls_key(s.code)) == 0:
await redis.lpush(start_urls_key(s.code), json.dumps({"trigger": w.trigger, "backfill_hours": s.backfill_hours}))
tick = {"trigger": w.trigger, "backfill_hours": s.backfill_hours}
if s.history_target and not s.history_done:
# 回补分段夹在常规轮次里执行,进度由子进程在本轮成功后写回
position = s.history_position.isoformat() if s.history_position else None
tick["history"] = {"target": s.history_target.isoformat(), "position": position}
await redis.lpush(start_urls_key(s.code), json.dumps(tick))
w.trigger = "schedule"
w.next_run_at = now + timedelta(seconds=s.interval_seconds)