Add resumable news history backfill
Let the crawler console set a backfill target per source. Each scheduled run pages backwards from a checkpoint for at most 20s so real-time crawling is not blocked. The checkpoint is saved only after a successful run and is also kept in process memory, because the next trigger may be queued before the database write lands. cls_depth is excluded since its list API cannot page into history.
This commit is contained in:
1 parent
0831971e83
commit
5c8d773ce1
10 files changed
+229
-17
No files matched your search
@@ -159,7 +159,12 @@ class Manager:
|
||||
continue
|
||||
# 子进程还没取走上一条触发时不再堆积
|
||||
if await redis.llen(start_urls_key(s.code)) == 0:
|
||||
await redis.lpush(start_urls_key(s.code), json.dumps({"trigger": w.trigger, "backfill_hours": s.backfill_hours}))
|
||||
tick = {"trigger": w.trigger, "backfill_hours": s.backfill_hours}
|
||||
if s.history_target and not s.history_done:
|
||||
# 回补分段夹在常规轮次里执行,进度由子进程在本轮成功后写回
|
||||
position = s.history_position.isoformat() if s.history_position else None
|
||||
tick["history"] = {"target": s.history_target.isoformat(), "position": position}
|
||||
await redis.lpush(start_urls_key(s.code), json.dumps(tick))
|
||||
w.trigger = "schedule"
|
||||
w.next_run_at = now + timedelta(seconds=s.interval_seconds)
|
||||
|
||||
|
||||
Reference in new issue
Block a user