Remove CLS collection and support a full news AI reset

2026-11-04 16:47:29 +08:00
1 parent dd50cb7df8
commit fcd1cfc344
@@ -1,113 +0,0 @@
-import hashlib
-import html
-import json
-import re
-from datetime import datetime
-from urllib.parse import urlencode
-
-from scrapy import Request
-from scrapy.http import Response
-
-from alphapulse.crawler.spiders.base import NewsSpider, ts
-
-
-def signed_url(path: str, params: dict[str, str]) -> str:
-    # 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
-    query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
-    query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
-    return f"https://www.cls.cn{path}?{urlencode(query)}"
-
-
-class ClsTelegraphSpider(NewsSpider):
-    """财联社·电报：列表即全文；翻页需 refresh_type=1"""
-
-    name = "cls_telegraph"
-    request_headers = {"Referer": "https://www.cls.cn/telegraph"}
-
-    def first_requests(self) -> list[Request]:
-        return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
-
-    def list_request(self, last_time: int, page: int, **meta) -> Request:
-        params = {"category": "", "last_time": str(last_time), "refresh_type": "1", "rn": "20"}
-        return self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page, **meta)
-
-    def history_request(self, before: datetime) -> Request:
-        return self.list_request(int(before.timestamp()), 1, history=True)
-
-    async def parse_list(self, response: Response):
-        body = response.json()
-        if body["errno"] != 0:
-            raise ValueError(f"接口返回错误：{body}")
-        rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
-        # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
-        ids = [str(x["id"]) for x in rows]
-        known = await self.known_items(ids)
-        for x in rows:
-            yield self.item(
-                response,
-                external_id=str(x["id"]),
-                url=f"https://www.cls.cn/detail/{x['id']}",
-                title=x["title"],
-                summary=x["brief"],
-                content_html=f"<p>{html.escape(x['content'])}</p>",
-                published_at=ts(x["ctime"]),
-                source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
-                author=x.get("author"),
-                tags=[s["subject_name"] for s in x.get("subjects") or []],
-                extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
-            )
-        page = response.meta["page"]
-        last_time = min(x["ctime"] for x in rows) if rows else None
-        oldest = ts(last_time) if rows else None
-        if response.meta.get("history"):
-            if self.history_step(oldest):
-                yield self.list_request(last_time, page + 1, history=True)
-        elif rows and self.should_continue(ids, known, oldest, page):
-            yield self.list_request(last_time, page + 1)
-
-
-class ClsDepthSpider(NewsSpider):
-    """财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
-
-    name = "cls_depth"
-    request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
-
-    def first_requests(self) -> list[Request]:
-        # 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
-        return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
-
-    async def parse_list(self, response: Response):
-        body = response.json()
-        if body["errno"] != 0:
-            raise ValueError(f"接口返回错误：{body}")
-        rows = [x for x in body["data"] if not x["is_ad"]]
-        ids = [str(x["id"]) for x in rows]
-        known = await self.known_items(ids)
-        for x in rows:
-            if known.get(str(x["id"])) != (x["title"], x["brief"]):
-                # 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
-                yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
-        page = response.meta["page"]
-        if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
-            params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
-            yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
-
-    async def parse_detail(self, response: Response):
-        match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
-        if match is None:
-            raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
-        a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
-        yield self.item(
-            response,
-            external_id=str(a["id"]),
-            url=f"https://www.cls.cn/detail/{a['id']}",
-            title=a["title"],
-            summary=response.meta["summary"],
-            content_html=a["content"],
-            published_at=ts(a["ctime"]),
-            author=(a.get("author") or {}).get("name"),
-            tags=[s["name"] for s in a.get("subject") or []],
-            # 付费内容只保存公开部分
-            is_paywalled=not a["isFree"],
-            extra={"level": a.get("level")},
-        )