Remove CLS collection and support a full news AI reset
This commit is contained in:
1 parent
dd50cb7df8
commit
fcd1cfc344
15 files changed
+340
-146
No files matched your search
@@ -11,8 +11,8 @@ LOG_CHANNEL = f"{KEY_PREFIX}pub:logs"
|
||||
NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
|
||||
CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
|
||||
STATUS_KEY = f"{KEY_PREFIX}crawler:status"
|
||||
# 列表接口支持按时间点往前翻的来源;cls_depth 列表只能翻约 3 页,无法回补历史
|
||||
HISTORY_SOURCES = ("cls_telegraph", "wscn_live", "wscn_news")
|
||||
# Sources whose list endpoints support historical pagination.
|
||||
HISTORY_SOURCES = ("wscn_live", "wscn_news")
|
||||
|
||||
|
||||
def start_urls_key(source_code: str) -> str:
|
||||
|
||||
Reference in new issue
Block a user