Initial AlphaPulse implementation: news crawler and Binance klines
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed
commit
aabc81ba5b
88 files changed
+12603
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,287 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Coroutine, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from scrapy import Request, signals
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.spidermiddlewares.httperror import HttpError
|
||||
from scrapy_redis.spiders import RedisSpider
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from twisted.python.failure import Failure
|
||||
from w3lib.html import remove_tags, replace_entities
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.crawler.events import write_logs
|
||||
from alphapulse.models import CrawlRun, NewsItem, Source
|
||||
|
||||
# 日志落库失败时只输出到 stdout，避免被自己的处理器再次捕获
|
||||
sink_logger = logging.getLogger("alphapulse.logsink")
|
||||
|
||||
MAX_PAGES = 200
|
||||
BACKOFF_BASE_SECONDS = 30
|
||||
BACKOFF_MAX_SECONDS = 15 * 60
|
||||
|
||||
|
||||
def ts(epoch: int) -> datetime:
|
||||
return datetime.fromtimestamp(epoch, UTC)
|
||||
|
||||
|
||||
def html_to_text(html: str) -> str:
|
||||
text = re.sub(r"(?i)<br\s*/?>|</p>|</div>|</h\d>|</li>", "\n", html)
|
||||
text = replace_entities(remove_tags(text))
|
||||
return re.sub(r"\n\s*\n+", "\n", text).strip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunState:
|
||||
id: uuid.UUID
|
||||
trigger: str
|
||||
started_at: datetime
|
||||
# 早于该时间的内容不再翻页回溯
|
||||
cutoff: datetime
|
||||
requests: int = 0
|
||||
items_seen: int = 0
|
||||
items_new: int = 0
|
||||
items_revised: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
# 遇到 403/429 后本轮不再发新请求
|
||||
blocked: bool = False
|
||||
|
||||
@property
|
||||
def full_scan(self) -> bool:
|
||||
# 进程启动后的第一轮一直翻到回溯时间，补上次被中断的回溯；常规轮次遇到整页已知即停
|
||||
return self.trigger in ("startup", "restart")
|
||||
|
||||
|
||||
class DbLogHandler(logging.Handler):
|
||||
"""把本来源的日志和全局 WARNING 以上日志收集起来，由刷新任务批量落库并发布"""
|
||||
|
||||
def __init__(self, spider: "NewsSpider") -> None:
|
||||
super().__init__(logging.INFO)
|
||||
self.spider = spider
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if record.name == sink_logger.name:
|
||||
return
|
||||
if record.name != self.spider.name and record.levelno < logging.WARNING:
|
||||
return
|
||||
message = record.getMessage()
|
||||
# scrapy-redis 每次取到起始任务都会打这一行，没有信息量
|
||||
if message.startswith("start req url"):
|
||||
return
|
||||
if record.exc_info:
|
||||
message += "\n" + logging.Formatter().formatException(record.exc_info)
|
||||
run_id = self.spider.run.id if self.spider.run else None
|
||||
self.spider.log_buffer.append((self.spider.name, run_id, record.levelname, message))
|
||||
|
||||
|
||||
class NewsSpider(RedisSpider):
|
||||
"""一个来源一个子进程。管理进程往 Redis 推一条触发消息即开始一轮采集，引擎空闲即本轮结束。"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.run: RunState | None = None
|
||||
self.log_buffer: list[tuple[str, uuid.UUID | None, str, str]] = []
|
||||
self._tasks: set[asyncio.Task] = set()
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider._on_opened, signal=signals.spider_opened)
|
||||
crawler.signals.connect(spider._on_idle, signal=signals.spider_idle)
|
||||
crawler.signals.connect(spider._on_error, signal=signals.spider_error)
|
||||
crawler.signals.connect(spider._on_closed, signal=signals.spider_closed)
|
||||
return spider
|
||||
|
||||
# ---------- 子类实现 ----------
|
||||
|
||||
def first_requests(self) -> Iterable[Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---------- 运行生命周期 ----------
|
||||
|
||||
async def start(self):
|
||||
for request in self.next_requests():
|
||||
yield request
|
||||
|
||||
def make_request_from_data(self, data: bytes) -> list[Request]:
|
||||
tick = json.loads(data)
|
||||
if self.run is not None:
|
||||
# scrapy-redis 的 idle 处理先于 _on_idle 执行：引擎已空闲说明上一轮其实已结束
|
||||
if not self.crawler.engine.spider_is_idle():
|
||||
self.logger.warning("上一轮尚未结束，跳过本次触发（%s）", tick["trigger"])
|
||||
return []
|
||||
self._close_run()
|
||||
now = datetime.now(UTC)
|
||||
self.run = RunState(uuid.uuid4(), tick["trigger"], now, now - timedelta(hours=tick["backfill_hours"]))
|
||||
self._spawn(self._insert_run(self.run))
|
||||
self.logger.info("开始采集（触发方式：%s）", tick["trigger"])
|
||||
return list(self.first_requests())
|
||||
|
||||
def _on_opened(self) -> None:
|
||||
logging.getLogger().addHandler(DbLogHandler(self))
|
||||
self._spawn(self._flush_logs_forever())
|
||||
self.logger.info("采集进程已启动")
|
||||
|
||||
def _on_idle(self) -> None:
|
||||
# scrapy-redis 的 idle 处理先执行，可能刚取到新触发并已开始新一轮；引擎仍有待处理请求时不能结束
|
||||
if self.run is None or not self.crawler.engine.spider_is_idle():
|
||||
return
|
||||
self._close_run()
|
||||
|
||||
def _close_run(self) -> None:
|
||||
run, self.run = self.run, None
|
||||
self._spawn(self._finish_run(run))
|
||||
|
||||
def _on_error(self, failure: Failure, response: Response) -> None:
|
||||
if self.run:
|
||||
self.run.errors.append(f"解析异常 {response.url}：{failure.value!r}")
|
||||
|
||||
async def _on_closed(self) -> None:
|
||||
self.logger.info("采集进程退出")
|
||||
await self._flush_logs()
|
||||
|
||||
def _spawn(self, coro: Coroutine) -> None:
|
||||
task = asyncio.get_running_loop().create_task(coro)
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
|
||||
async def _insert_run(self, run: RunState) -> None:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
pg_insert(CrawlRun).values(
|
||||
id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, status="running"
|
||||
).on_conflict_do_nothing()
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def _finish_run(self, run: RunState) -> None:
|
||||
now = datetime.now(UTC)
|
||||
failed = bool(run.errors)
|
||||
values = {
|
||||
"finished_at": now,
|
||||
"status": "failed" if failed else "success",
|
||||
"requests": run.requests,
|
||||
"items_seen": run.items_seen,
|
||||
"items_new": run.items_new,
|
||||
"items_revised": run.items_revised,
|
||||
"error": "\n".join(run.errors)[:4000] if failed else None,
|
||||
}
|
||||
async with SessionLocal() as session:
|
||||
# 与 _insert_run 并发时谁先执行都正确
|
||||
await session.execute(
|
||||
pg_insert(CrawlRun)
|
||||
.values(id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, **values)
|
||||
.on_conflict_do_update(index_elements=[CrawlRun.id], set_=values)
|
||||
)
|
||||
source = await session.get(Source, self.name, with_for_update=True)
|
||||
if failed:
|
||||
source.consecutive_failures += 1
|
||||
delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
|
||||
source.paused_until = now + timedelta(seconds=delay)
|
||||
source.last_error = run.errors[0][:1000]
|
||||
else:
|
||||
source.consecutive_failures = 0
|
||||
source.paused_until = None
|
||||
source.last_error = None
|
||||
source.last_success_at = now
|
||||
await session.commit()
|
||||
|
||||
elapsed = (now - run.started_at).total_seconds()
|
||||
summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
|
||||
if failed:
|
||||
self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
|
||||
else:
|
||||
self.logger.info("本轮完成：%s", summary)
|
||||
|
||||
async def _flush_logs(self) -> None:
|
||||
rows, self.log_buffer = self.log_buffer, []
|
||||
try:
|
||||
await write_logs(rows)
|
||||
except Exception:
|
||||
sink_logger.exception("日志落库失败，丢弃 %d 条", len(rows))
|
||||
|
||||
async def _flush_logs_forever(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
await self._flush_logs()
|
||||
|
||||
# ---------- 子类使用的工具 ----------
|
||||
|
||||
def request(self, url: str, callback: Callable, **meta: Any) -> Request:
|
||||
self.run.requests += 1
|
||||
return Request(
|
||||
url,
|
||||
callback=callback,
|
||||
errback=self.on_request_error,
|
||||
dont_filter=True,
|
||||
meta={"run_id": self.run.id, **meta},
|
||||
headers=getattr(self, "request_headers", None),
|
||||
)
|
||||
|
||||
def on_request_error(self, failure: Failure) -> None:
|
||||
status = failure.value.response.status if failure.check(HttpError) else None
|
||||
if status in (403, 429):
|
||||
self.run.blocked = True
|
||||
message = f"请求失败 {failure.request.url}：{status or repr(failure.value)}"
|
||||
self.run.errors.append(message)
|
||||
self.logger.error(message)
|
||||
|
||||
async def known_items(self, external_ids: list[str]) -> dict[str, tuple[str, str]]:
|
||||
"""已入库条目的 (title, summary)，用于判断是否新增、是否需要重抓详情、是否继续翻页"""
|
||||
async with SessionLocal() as session:
|
||||
rows = await session.execute(
|
||||
select(NewsItem.external_id, NewsItem.title, NewsItem.summary)
|
||||
.where(NewsItem.source_code == self.name, NewsItem.external_id.in_(external_ids))
|
||||
)
|
||||
return {external_id: (title, summary) for external_id, title, summary in rows}
|
||||
|
||||
def should_continue(self, external_ids: list[str], known: dict, oldest: datetime, page: int) -> bool:
|
||||
"""未越过回溯时间、未被限流，且本页有新条目（或本轮为全量扫描）时才翻下一页"""
|
||||
return (
|
||||
not self.run.blocked
|
||||
and page < MAX_PAGES
|
||||
and oldest > self.run.cutoff
|
||||
and (self.run.full_scan or any(external_id not in known for external_id in external_ids))
|
||||
)
|
||||
|
||||
def item(
|
||||
self,
|
||||
response: Response,
|
||||
*,
|
||||
external_id: str,
|
||||
url: str,
|
||||
title: str,
|
||||
summary: str,
|
||||
content_html: str,
|
||||
published_at: datetime,
|
||||
author: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
is_paywalled: bool = False,
|
||||
source_updated_at: datetime | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"source_code": self.name,
|
||||
"external_id": external_id,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"content_html": content_html,
|
||||
"content_text": html_to_text(content_html),
|
||||
"author": author or None,
|
||||
"tags": tags or [],
|
||||
"is_paywalled": is_paywalled,
|
||||
"published_at": published_at,
|
||||
"source_updated_at": source_updated_at,
|
||||
"extra": extra or {},
|
||||
"raw_response_id": response.meta["raw_response_id"],
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
from alphapulse.crawler.spiders.base import NewsSpider, ts
|
||||
|
||||
|
||||
def signed_url(path: str, params: dict[str, str]) -> str:
|
||||
# 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
|
||||
query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
|
||||
query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
|
||||
return f"https://www.cls.cn{path}?{urlencode(query)}"
|
||||
|
||||
|
||||
class ClsTelegraphSpider(NewsSpider):
|
||||
"""财联社·电报：列表即全文；翻页需 refresh_type=1"""
|
||||
|
||||
name = "cls_telegraph"
|
||||
request_headers = {"Referer": "https://www.cls.cn/telegraph"}
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
body = response.json()
|
||||
if body["errno"] != 0:
|
||||
raise ValueError(f"接口返回错误：{body}")
|
||||
rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
|
||||
# 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(x["id"]),
|
||||
url=f"https://www.cls.cn/detail/{x['id']}",
|
||||
title=x["title"],
|
||||
summary=x["brief"],
|
||||
content_html=f"<p>{html.escape(x['content'])}</p>",
|
||||
published_at=ts(x["ctime"]),
|
||||
source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
|
||||
author=x.get("author"),
|
||||
tags=[s["subject_name"] for s in x.get("subjects") or []],
|
||||
extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
|
||||
)
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
||||
last_time = str(min(x["ctime"] for x in rows))
|
||||
params = {"category": "", "last_time": last_time, "refresh_type": "1", "rn": "20"}
|
||||
yield self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page + 1)
|
||||
|
||||
|
||||
class ClsDepthSpider(NewsSpider):
|
||||
"""财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
|
||||
|
||||
name = "cls_depth"
|
||||
request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
# 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
|
||||
return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
body = response.json()
|
||||
if body["errno"] != 0:
|
||||
raise ValueError(f"接口返回错误：{body}")
|
||||
rows = [x for x in body["data"] if not x["is_ad"]]
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
if known.get(str(x["id"])) != (x["title"], x["brief"]):
|
||||
# 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
|
||||
yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
||||
params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
|
||||
yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
|
||||
|
||||
async def parse_detail(self, response: Response):
|
||||
match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
|
||||
if match is None:
|
||||
raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
|
||||
a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(a["id"]),
|
||||
url=f"https://www.cls.cn/detail/{a['id']}",
|
||||
title=a["title"],
|
||||
summary=response.meta["summary"],
|
||||
content_html=a["content"],
|
||||
published_at=ts(a["ctime"]),
|
||||
author=(a.get("author") or {}).get("name"),
|
||||
tags=[s["name"] for s in a.get("subject") or []],
|
||||
# 付费内容只保存公开部分
|
||||
is_paywalled=not a["isFree"],
|
||||
extra={"level": a.get("level")},
|
||||
)
|
||||
@@ -0,0 +1,81 @@
|
||||
from scrapy import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
from alphapulse.crawler.spiders.base import NewsSpider, ts
|
||||
|
||||
API = "https://api-one-wscn.awtmt.com/apiv1/content"
|
||||
|
||||
|
||||
class WscnNewsSpider(NewsSpider):
|
||||
"""华尔街见闻·文章：列表只有摘要，新条目或标题/摘要变化时再拉详情"""
|
||||
|
||||
name = "wscn_news"
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.list_request("", 1)]
|
||||
|
||||
def list_request(self, cursor: str, page: int) -> Request:
|
||||
return self.request(f"{API}/information-flow?channel=global&accept=article&limit=20&cursor={cursor}", self.parse_list, page=page)
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
data = response.json()["data"]
|
||||
# 信息流里混有快讯，快讯由 wscn_live 负责
|
||||
rows = [x["resource"] for x in data["items"] if x["resource_type"] == "article"]
|
||||
ids = [str(r["id"]) for r in rows]
|
||||
known = await self.known_items(ids)
|
||||
for r in rows:
|
||||
if known.get(str(r["id"])) != (r["title"], r["content_short"]):
|
||||
yield self.request(f"{API}/articles/{r['id']}?extract=0", self.parse_detail, summary=r["content_short"])
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(r["display_time"] for r in rows)), page):
|
||||
yield self.list_request(data["next_cursor"], page + 1)
|
||||
|
||||
async def parse_detail(self, response: Response):
|
||||
a = response.json()["data"]
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(a["id"]),
|
||||
url=f"https://wallstreetcn.com/articles/{a['id']}",
|
||||
title=a["title"],
|
||||
summary=response.meta["summary"],
|
||||
content_html=a["content"],
|
||||
published_at=ts(a["display_time"]),
|
||||
author=(a.get("author") or {}).get("display_name"),
|
||||
is_paywalled=bool(a.get("is_need_pay")),
|
||||
extra={"source_name": a.get("source_name"), "is_priced": a.get("is_priced")},
|
||||
)
|
||||
|
||||
|
||||
class WscnLiveSpider(NewsSpider):
|
||||
"""华尔街见闻·快讯：列表即全文"""
|
||||
|
||||
name = "wscn_live"
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.list_request("", 1)]
|
||||
|
||||
def list_request(self, cursor: str, page: int) -> Request:
|
||||
return self.request(f"{API}/lives?channel=global-channel&limit=50&cursor={cursor}", self.parse_list, page=page)
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
data = response.json()["data"]
|
||||
rows = data["items"]
|
||||
# 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(x["id"]),
|
||||
url=x["uri"],
|
||||
title=x["title"],
|
||||
summary="",
|
||||
content_html=x["content"] + (x.get("content_more") or ""),
|
||||
published_at=ts(x["display_time"]),
|
||||
author=(x.get("author") or {}).get("display_name"),
|
||||
tags=x.get("channels"),
|
||||
extra={"score": x.get("score"), "symbols": x.get("symbols")},
|
||||
)
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["display_time"] for x in rows)), page):
|
||||
yield self.list_request(data["next_cursor"], page + 1)
|
||||
Reference in new issue
Block a user