eezd
aabc81ba5b
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
103 lines
4.8 KiB
Python
103 lines
4.8 KiB
Python
import hashlib
|
|
import html
|
|
import json
|
|
import re
|
|
from urllib.parse import urlencode
|
|
|
|
from scrapy import Request
|
|
from scrapy.http import Response
|
|
|
|
from alphapulse.crawler.spiders.base import NewsSpider, ts
|
|
|
|
|
|
def signed_url(path: str, params: dict[str, str]) -> str:
|
|
# 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
|
|
query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
|
|
query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
|
|
return f"https://www.cls.cn{path}?{urlencode(query)}"
|
|
|
|
|
|
class ClsTelegraphSpider(NewsSpider):
|
|
"""财联社·电报：列表即全文；翻页需 refresh_type=1"""
|
|
|
|
name = "cls_telegraph"
|
|
request_headers = {"Referer": "https://www.cls.cn/telegraph"}
|
|
|
|
def first_requests(self) -> list[Request]:
|
|
return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
|
|
|
|
async def parse_list(self, response: Response):
|
|
body = response.json()
|
|
if body["errno"] != 0:
|
|
raise ValueError(f"接口返回错误：{body}")
|
|
rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
|
|
# 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
|
|
ids = [str(x["id"]) for x in rows]
|
|
known = await self.known_items(ids)
|
|
for x in rows:
|
|
yield self.item(
|
|
response,
|
|
external_id=str(x["id"]),
|
|
url=f"https://www.cls.cn/detail/{x['id']}",
|
|
title=x["title"],
|
|
summary=x["brief"],
|
|
content_html=f"<p>{html.escape(x['content'])}</p>",
|
|
published_at=ts(x["ctime"]),
|
|
source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
|
|
author=x.get("author"),
|
|
tags=[s["subject_name"] for s in x.get("subjects") or []],
|
|
extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
|
|
)
|
|
page = response.meta["page"]
|
|
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
|
last_time = str(min(x["ctime"] for x in rows))
|
|
params = {"category": "", "last_time": last_time, "refresh_type": "1", "rn": "20"}
|
|
yield self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page + 1)
|
|
|
|
|
|
class ClsDepthSpider(NewsSpider):
|
|
"""财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
|
|
|
|
name = "cls_depth"
|
|
request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
|
|
|
|
def first_requests(self) -> list[Request]:
|
|
# 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
|
|
return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
|
|
|
|
async def parse_list(self, response: Response):
|
|
body = response.json()
|
|
if body["errno"] != 0:
|
|
raise ValueError(f"接口返回错误：{body}")
|
|
rows = [x for x in body["data"] if not x["is_ad"]]
|
|
ids = [str(x["id"]) for x in rows]
|
|
known = await self.known_items(ids)
|
|
for x in rows:
|
|
if known.get(str(x["id"])) != (x["title"], x["brief"]):
|
|
# 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
|
|
yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
|
|
page = response.meta["page"]
|
|
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
|
params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
|
|
yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
|
|
|
|
async def parse_detail(self, response: Response):
|
|
match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
|
|
if match is None:
|
|
raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
|
|
a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
|
|
yield self.item(
|
|
response,
|
|
external_id=str(a["id"]),
|
|
url=f"https://www.cls.cn/detail/{a['id']}",
|
|
title=a["title"],
|
|
summary=response.meta["summary"],
|
|
content_html=a["content"],
|
|
published_at=ts(a["ctime"]),
|
|
author=(a.get("author") or {}).get("name"),
|
|
tags=[s["name"] for s in a.get("subject") or []],
|
|
# 付费内容只保存公开部分
|
|
is_paywalled=not a["isFree"],
|
|
extra={"level": a.get("level")},
|
|
)
|