Initial AlphaPulse implementation: news crawler and Binance klines

- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth,
  news/crawler/market APIs with SSE and WebSocket streaming
- Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth,
  cls_telegraph spiders, idempotent upserts with revisions, raw response
  archiving, per-source subprocess supervision, restart and backoff
- Market worker: full-history Binance spot/USD-M kline backfill,
  realtime WS stream, hourly and on-reconnect catch-up
- Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed,
  crawler console with live logs, Binance-style chart page
- Deploy: docker compose, nginx site, migrate and daily backup scripts

backend/src/alphapulse/crawler/events.py

@@ -0,0 +1,41 @@
+import json
+import uuid
+
+from sqlalchemy import insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.models import CrawlLog
+
+LOG_CHANNEL = f"{KEY_PREFIX}pub:logs"
+NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
+CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
+STATUS_KEY = f"{KEY_PREFIX}crawler:status"
+
+
+def start_urls_key(source_code: str) -> str:
+    return f"{KEY_PREFIX}{source_code}:start_urls"
+
+
+async def write_logs(rows: list[tuple[str, uuid.UUID | None, str, str]]) -> None:
+    """rows: (source_code, run_id, level, message)；落库后再发布，前端拿到的日志都带 id"""
+    if not rows:
+        return
+    async with SessionLocal() as session:
+        result = await session.execute(
+            insert(CrawlLog).returning(CrawlLog.id, CrawlLog.ts),
+            [{"source_code": s, "run_id": r, "level": lv, "message": m} for s, r, lv, m in rows],
+        )
+        saved = result.all()
+        await session.commit()
+    async with redis.pipeline(transaction=False) as pipe:
+        for (source_code, run_id, level, message), (log_id, ts) in zip(rows, saved, strict=True):
+            pipe.publish(LOG_CHANNEL, json.dumps({
+                "id": log_id,
+                "source_code": source_code,
+                "run_id": str(run_id) if run_id else None,
+                "ts": ts.isoformat(),
+                "level": level,
+                "message": message,
+            }, ensure_ascii=False))
+        await pipe.execute()

backend/src/alphapulse/crawler/manager.py

@@ -0,0 +1,186 @@
+"""采集管理进程：按间隔触发采集、守护每个来源的 scrapy 子进程、处理重启指令。
+
+用法（工作目录需包含 scrapy.cfg）：python -m alphapulse.crawler.manager
+"""
+
+import asyncio
+import json
+import logging
+import signal
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import select, update
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.crawler.events import CONTROL_CHANNEL, STATUS_KEY, start_urls_key, write_logs
+from alphapulse.models import CrawlRun, Source
+
+logger = logging.getLogger("alphapulse.crawler.manager")
+
+RESTART_DELAY_MIN = 5
+RESTART_DELAY_MAX = 300
+# 子进程稳定运行超过该时长后，异常退出的重启延迟重新从最小值开始
+STABLE_SECONDS = 60
+STOP_TIMEOUT = 15
+
+
+@dataclass
+class Worker:
+    code: str
+    proc: asyncio.subprocess.Process | None = None
+    started_at: datetime | None = None
+    next_run_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    # 下一次触发的类型：schedule / startup / restart
+    trigger: str = "schedule"
+    restarting: bool = False
+    restart_delay: int = RESTART_DELAY_MIN
+    wake: asyncio.Event = field(default_factory=asyncio.Event)
+
+
+async def log(code: str, level: str, message: str) -> None:
+    logger.log(logging.getLevelName(level), "[%s] %s", code, message)
+    await write_logs([(code, None, level, message)])
+
+
+class Manager:
+    def __init__(self) -> None:
+        self.workers: dict[str, Worker] = {}
+        self.stopping = asyncio.Event()
+
+    async def run(self) -> None:
+        async with SessionLocal() as session:
+            codes = (await session.scalars(select(Source.code).order_by(Source.code))).all()
+        self.workers = {code: Worker(code) for code in codes}
+
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            loop.add_signal_handler(sig, self.stopping.set)
+
+        tasks = [asyncio.create_task(self.supervise(w)) for w in self.workers.values()]
+        tasks += [asyncio.create_task(c) for c in (self.schedule_loop(), self.control_loop(), self.status_loop())]
+        await self.stopping.wait()
+        logger.info("收到退出信号，停止全部子进程")
+        await asyncio.gather(*(self.stop_process(w) for w in self.workers.values()))
+        for task in tasks:
+            task.cancel()
+        await redis.delete(STATUS_KEY)
+
+    async def supervise(self, w: Worker) -> None:
+        while not self.stopping.is_set():
+            # 上次进程遗留的请求和未收尾的轮次都已失效，清理后从干净状态开始
+            await redis.delete(f"{KEY_PREFIX}{w.code}:requests", start_urls_key(w.code))
+            async with SessionLocal() as session:
+                await session.execute(
+                    update(CrawlRun)
+                    .where(CrawlRun.source_code == w.code, CrawlRun.status == "running")
+                    .values(status="interrupted", finished_at=datetime.now(UTC), error="子进程退出时本轮未完成")
+                )
+                await session.commit()
+            w.proc = await asyncio.create_subprocess_exec("scrapy", "crawl", w.code)
+            w.started_at = datetime.now(UTC)
+            # 新进程的第一轮立即执行，并做全量回溯扫描
+            if w.trigger == "schedule":
+                w.trigger = "startup"
+            w.next_run_at = w.started_at
+            await log(w.code, "INFO", f"子进程已启动 pid={w.proc.pid}")
+            returncode = await w.proc.wait()
+            ran_seconds = (datetime.now(UTC) - w.started_at).total_seconds()
+            w.proc = None
+            if self.stopping.is_set():
+                return
+            if w.restarting:
+                w.restarting = False
+                w.restart_delay = RESTART_DELAY_MIN
+                continue
+            if ran_seconds >= STABLE_SECONDS:
+                w.restart_delay = RESTART_DELAY_MIN
+            await log(w.code, "ERROR", f"子进程异常退出 returncode={returncode}，{w.restart_delay}s 后重启")
+            w.wake.clear()
+            try:
+                await asyncio.wait_for(w.wake.wait(), timeout=w.restart_delay)
+            except TimeoutError:
+                pass
+            w.restart_delay = min(w.restart_delay * 2, RESTART_DELAY_MAX)
+
+    async def stop_process(self, w: Worker) -> None:
+        if w.proc is None:
+            return
+        w.proc.terminate()
+        try:
+            await asyncio.wait_for(w.proc.wait(), timeout=STOP_TIMEOUT)
+        except TimeoutError:
+            await log(w.code, "WARNING", f"子进程 {STOP_TIMEOUT}s 内未退出，强制结束")
+            w.proc.kill()
+
+    async def schedule_loop(self) -> None:
+        while True:
+            async with SessionLocal() as session:
+                sources = (await session.scalars(select(Source))).all()
+            now = datetime.now(UTC)
+            for s in sources:
+                w = self.workers[s.code]
+                if not s.enabled or w.proc is None or w.restarting:
+                    continue
+                if s.paused_until and s.paused_until > now:
+                    w.next_run_at = max(w.next_run_at, s.paused_until)
+                    continue
+                if now < w.next_run_at:
+                    continue
+                # 子进程还没取走上一条触发时不再堆积
+                if await redis.llen(start_urls_key(s.code)) == 0:
+                    await redis.lpush(start_urls_key(s.code), json.dumps({"trigger": w.trigger, "backfill_hours": s.backfill_hours}))
+                    w.trigger = "schedule"
+                w.next_run_at = now + timedelta(seconds=s.interval_seconds)
+            await asyncio.sleep(1)
+
+    async def control_loop(self) -> None:
+        async with redis.pubsub() as pubsub:
+            await pubsub.subscribe(CONTROL_CHANNEL)
+            async for message in pubsub.listen():
+                if message["type"] != "message":
+                    continue
+                command = json.loads(message["data"])
+                if command["action"] != "restart":
+                    logger.warning("未知指令：%s", command)
+                    continue
+                codes = list(self.workers) if command["source"] == "all" else [command["source"]]
+                await asyncio.gather(*(self.restart(self.workers[code]) for code in codes))
+
+    async def restart(self, w: Worker) -> None:
+        await log(w.code, "INFO", "收到重启指令")
+        async with SessionLocal() as session:
+            await session.execute(
+                update(Source).where(Source.code == w.code).values(consecutive_failures=0, paused_until=None)
+            )
+            await session.commit()
+        w.trigger = "restart"
+        w.next_run_at = datetime.now(UTC)
+        if w.proc is None:
+            # 处于异常退出后的等待期：立即重启
+            w.wake.set()
+            return
+        w.restarting = True
+        await self.stop_process(w)
+
+    async def status_loop(self) -> None:
+        while True:
+            mapping = {
+                w.code: json.dumps({
+                    "state": "restarting" if w.restarting else ("running" if w.proc else "stopped"),
+                    "pid": w.proc.pid if w.proc else None,
+                    "started_at": w.started_at.isoformat() if w.started_at else None,
+                    "next_run_at": w.next_run_at.isoformat(),
+                })
+                for w in self.workers.values()
+            }
+            async with redis.pipeline() as pipe:
+                # 管理进程挂掉后状态自动过期，前端据此显示"管理进程未运行"
+                await pipe.hset(STATUS_KEY, mapping=mapping).expire(STATUS_KEY, 10).execute()
+            await asyncio.sleep(2)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    asyncio.run(Manager().run())

backend/src/alphapulse/crawler/middlewares.py

@@ -0,0 +1,58 @@
+import asyncio
+import gzip
+import hashlib
+from urllib.parse import urlparse
+
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
+from sqlalchemy import insert
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.models import RawBody, RawResponse
+
+
+class RedisRateLimitMiddleware:
+    """按域名的跨进程限速：同一域名两次请求至少间隔 interval_ms"""
+
+    interval_ms = 1000
+
+    async def process_request(self, request: Request) -> None:
+        key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
+        while not await redis.set(key, "1", px=self.interval_ms, nx=True):
+            ttl = await redis.pttl(key)
+            await asyncio.sleep(max(ttl, 50) / 1000)
+
+
+class RawResponseMiddleware:
+    """每次响应都写 raw_responses；响应体按 sha256 去重存 raw_bodies"""
+
+    def __init__(self, crawler: Crawler) -> None:
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> "RawResponseMiddleware":
+        return cls(crawler)
+
+    async def process_response(self, request: Request, response: Response) -> Response:
+        sha256 = hashlib.sha256(response.body).hexdigest()
+        headers = {k.decode(): b", ".join(v).decode("latin-1") for k, v in response.headers.items()}
+        async with SessionLocal() as session:
+            await session.execute(
+                pg_insert(RawBody).values(sha256=sha256, body=gzip.compress(response.body)).on_conflict_do_nothing()
+            )
+            raw_id = await session.scalar(
+                insert(RawResponse).values(
+                    run_id=request.meta.get("run_id"),
+                    source_code=self.crawler.spider.name,
+                    url=response.url,
+                    method=request.method,
+                    status=response.status,
+                    headers=headers,
+                    body_sha256=sha256,
+                ).returning(RawResponse.id)
+            )
+            await session.commit()
+        request.meta["raw_response_id"] = raw_id
+        return response

backend/src/alphapulse/crawler/pipelines.py

@@ -0,0 +1,74 @@
+import hashlib
+import json
+from typing import Any
+
+from scrapy.crawler import Crawler
+from sqlalchemy import func, select
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import NEWS_CHANNEL
+from alphapulse.models import NewsItem, NewsRevision
+
+UPDATABLE_FIELDS = ("url", "title", "summary", "content_html", "content_text", "author", "tags",
+                    "is_paywalled", "published_at", "source_updated_at", "extra")
+
+
+def content_hash(item: dict[str, Any]) -> str:
+    return hashlib.sha256("\0".join((item["title"], item["summary"], item["content_html"])).encode()).hexdigest()
+
+
+class NewsPipeline:
+    """按 (source_code, external_id) 幂等落库；内容变化写一条修订"""
+
+    def __init__(self, crawler: Crawler) -> None:
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> "NewsPipeline":
+        return cls(crawler)
+
+    async def process_item(self, item: dict[str, Any]) -> dict[str, Any]:
+        spider = self.crawler.spider
+        digest = content_hash(item)
+        raw_response_id = item.pop("raw_response_id")
+        async with SessionLocal() as session:
+            news = await session.scalar(
+                select(NewsItem)
+                .where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
+                .with_for_update()
+            )
+            if news is None:
+                kind = "new"
+                news = NewsItem(**item, content_hash=digest, revision_no=1)
+                session.add(news)
+                await session.flush()
+            elif news.content_hash != digest:
+                kind = "revised"
+                for field in UPDATABLE_FIELDS:
+                    setattr(news, field, item[field])
+                news.content_hash = digest
+                news.revision_no += 1
+                news.last_seen_at = func.now()
+            else:
+                kind = None
+                news.last_seen_at = func.now()
+
+            if kind:
+                session.add(NewsRevision(
+                    news_id=news.id,
+                    revision_no=news.revision_no,
+                    title=item["title"],
+                    summary=item["summary"],
+                    content_html=item["content_html"],
+                    content_hash=digest,
+                    raw_response_id=raw_response_id,
+                ))
+            news_id = news.id
+            await session.commit()
+
+        spider.run.items_seen += 1
+        if kind:
+            setattr(spider.run, f"items_{kind}", getattr(spider.run, f"items_{kind}") + 1)
+            await redis.publish(NEWS_CHANNEL, json.dumps({"id": news_id, "source_code": item["source_code"], "kind": kind}))
+        return item

backend/src/alphapulse/crawler/settings.py

@@ -0,0 +1,45 @@
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+from alphapulse.core.config import get_settings
+from alphapulse.core.redis import KEY_PREFIX
+
+BOT_NAME = "alphapulse"
+SPIDER_MODULES = ["alphapulse.crawler.spiders"]
+
+# 采集的是无需登录的公开 JSON 接口；限速由 RedisRateLimitMiddleware 统一控制
+ROBOTSTXT_OBEY = False
+USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
+DEFAULT_REQUEST_HEADERS = {"Accept": "application/json, text/html;q=0.9, */*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9"}
+COOKIES_ENABLED = False
+TELNETCONSOLE_ENABLED = False
+# 2.19 默认开启的远程控制 HTTP 服务用不上，关掉避免开放端口
+REMOTE_CONTROL_ENABLED = False
+LOG_LEVEL = "INFO"
+
+CONCURRENT_REQUESTS = 4
+DOWNLOAD_TIMEOUT = 15
+RETRY_TIMES = 2
+# 429/403 不重试，交给来源退避
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+
+DOWNLOADER_MIDDLEWARES = {
+    "alphapulse.crawler.middlewares.RedisRateLimitMiddleware": 100,
+    # process_response 按优先级从大到小调用：排在 HttpCompression(590) 之后拿到解压后的响应体，
+    # 排在 Retry(550) 之前，被重试的每次响应也都会记录
+    "alphapulse.crawler.middlewares.RawResponseMiddleware": 580,
+}
+ITEM_PIPELINES = {"alphapulse.crawler.pipelines.NewsPipeline": 300}
+
+# Scrapy-Redis：请求队列与起始任务放 Redis（DB 14，统一 ap: 前缀）
+SCHEDULER = "scrapy_redis.scheduler.Scheduler"
+DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
+SCHEDULER_PERSIST = True
+SCHEDULER_QUEUE_KEY = f"{KEY_PREFIX}%(spider)s:requests"
+SCHEDULER_DUPEFILTER_KEY = f"{KEY_PREFIX}%(spider)s:dupefilter"
+REDIS_START_URLS_KEY = f"{KEY_PREFIX}%(name)s:start_urls"
+REDIS_URL = get_settings().redis_url
+
+# scrapy-redis 0.9.1 仍按旧签名给 stats 传 spider 参数，每个请求都告警，会刷屏实时日志
+warnings.filterwarnings("ignore", category=ScrapyDeprecationWarning, module="scrapy_redis")

backend/src/alphapulse/crawler/spiders/base.py

@@ -0,0 +1,287 @@
+import asyncio
+import json
+import logging
+import re
+import uuid
+from collections.abc import Callable, Coroutine, Iterable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from scrapy import Request, signals
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from scrapy.spidermiddlewares.httperror import HttpError
+from scrapy_redis.spiders import RedisSpider
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from twisted.python.failure import Failure
+from w3lib.html import remove_tags, replace_entities
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.crawler.events import write_logs
+from alphapulse.models import CrawlRun, NewsItem, Source
+
+# 日志落库失败时只输出到 stdout，避免被自己的处理器再次捕获
+sink_logger = logging.getLogger("alphapulse.logsink")
+
+MAX_PAGES = 200
+BACKOFF_BASE_SECONDS = 30
+BACKOFF_MAX_SECONDS = 15 * 60
+
+
+def ts(epoch: int) -> datetime:
+    return datetime.fromtimestamp(epoch, UTC)
+
+
+def html_to_text(html: str) -> str:
+    text = re.sub(r"(?i)<br\s*/?>|</p>|</div>|</h\d>|</li>", "\n", html)
+    text = replace_entities(remove_tags(text))
+    return re.sub(r"\n\s*\n+", "\n", text).strip()
+
+
+@dataclass
+class RunState:
+    id: uuid.UUID
+    trigger: str
+    started_at: datetime
+    # 早于该时间的内容不再翻页回溯
+    cutoff: datetime
+    requests: int = 0
+    items_seen: int = 0
+    items_new: int = 0
+    items_revised: int = 0
+    errors: list[str] = field(default_factory=list)
+    # 遇到 403/429 后本轮不再发新请求
+    blocked: bool = False
+
+    @property
+    def full_scan(self) -> bool:
+        # 进程启动后的第一轮一直翻到回溯时间，补上次被中断的回溯；常规轮次遇到整页已知即停
+        return self.trigger in ("startup", "restart")
+
+
+class DbLogHandler(logging.Handler):
+    """把本来源的日志和全局 WARNING 以上日志收集起来，由刷新任务批量落库并发布"""
+
+    def __init__(self, spider: "NewsSpider") -> None:
+        super().__init__(logging.INFO)
+        self.spider = spider
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.name == sink_logger.name:
+            return
+        if record.name != self.spider.name and record.levelno < logging.WARNING:
+            return
+        message = record.getMessage()
+        # scrapy-redis 每次取到起始任务都会打这一行，没有信息量
+        if message.startswith("start req url"):
+            return
+        if record.exc_info:
+            message += "\n" + logging.Formatter().formatException(record.exc_info)
+        run_id = self.spider.run.id if self.spider.run else None
+        self.spider.log_buffer.append((self.spider.name, run_id, record.levelname, message))
+
+
+class NewsSpider(RedisSpider):
+    """一个来源一个子进程。管理进程往 Redis 推一条触发消息即开始一轮采集，引擎空闲即本轮结束。"""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.run: RunState | None = None
+        self.log_buffer: list[tuple[str, uuid.UUID | None, str, str]] = []
+        self._tasks: set[asyncio.Task] = set()
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider._on_opened, signal=signals.spider_opened)
+        crawler.signals.connect(spider._on_idle, signal=signals.spider_idle)
+        crawler.signals.connect(spider._on_error, signal=signals.spider_error)
+        crawler.signals.connect(spider._on_closed, signal=signals.spider_closed)
+        return spider
+
+    # ---------- 子类实现 ----------
+
+    def first_requests(self) -> Iterable[Request]:
+        raise NotImplementedError
+
+    # ---------- 运行生命周期 ----------
+
+    async def start(self):
+        for request in self.next_requests():
+            yield request
+
+    def make_request_from_data(self, data: bytes) -> list[Request]:
+        tick = json.loads(data)
+        if self.run is not None:
+            # scrapy-redis 的 idle 处理先于 _on_idle 执行：引擎已空闲说明上一轮其实已结束
+            if not self.crawler.engine.spider_is_idle():
+                self.logger.warning("上一轮尚未结束，跳过本次触发（%s）", tick["trigger"])
+                return []
+            self._close_run()
+        now = datetime.now(UTC)
+        self.run = RunState(uuid.uuid4(), tick["trigger"], now, now - timedelta(hours=tick["backfill_hours"]))
+        self._spawn(self._insert_run(self.run))
+        self.logger.info("开始采集（触发方式：%s）", tick["trigger"])
+        return list(self.first_requests())
+
+    def _on_opened(self) -> None:
+        logging.getLogger().addHandler(DbLogHandler(self))
+        self._spawn(self._flush_logs_forever())
+        self.logger.info("采集进程已启动")
+
+    def _on_idle(self) -> None:
+        # scrapy-redis 的 idle 处理先执行，可能刚取到新触发并已开始新一轮；引擎仍有待处理请求时不能结束
+        if self.run is None or not self.crawler.engine.spider_is_idle():
+            return
+        self._close_run()
+
+    def _close_run(self) -> None:
+        run, self.run = self.run, None
+        self._spawn(self._finish_run(run))
+
+    def _on_error(self, failure: Failure, response: Response) -> None:
+        if self.run:
+            self.run.errors.append(f"解析异常 {response.url}：{failure.value!r}")
+
+    async def _on_closed(self) -> None:
+        self.logger.info("采集进程退出")
+        await self._flush_logs()
+
+    def _spawn(self, coro: Coroutine) -> None:
+        task = asyncio.get_running_loop().create_task(coro)
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _insert_run(self, run: RunState) -> None:
+        async with SessionLocal() as session:
+            await session.execute(
+                pg_insert(CrawlRun).values(
+                    id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, status="running"
+                ).on_conflict_do_nothing()
+            )
+            await session.commit()
+
+    async def _finish_run(self, run: RunState) -> None:
+        now = datetime.now(UTC)
+        failed = bool(run.errors)
+        values = {
+            "finished_at": now,
+            "status": "failed" if failed else "success",
+            "requests": run.requests,
+            "items_seen": run.items_seen,
+            "items_new": run.items_new,
+            "items_revised": run.items_revised,
+            "error": "\n".join(run.errors)[:4000] if failed else None,
+        }
+        async with SessionLocal() as session:
+            # 与 _insert_run 并发时谁先执行都正确
+            await session.execute(
+                pg_insert(CrawlRun)
+                .values(id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, **values)
+                .on_conflict_do_update(index_elements=[CrawlRun.id], set_=values)
+            )
+            source = await session.get(Source, self.name, with_for_update=True)
+            if failed:
+                source.consecutive_failures += 1
+                delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
+                source.paused_until = now + timedelta(seconds=delay)
+                source.last_error = run.errors[0][:1000]
+            else:
+                source.consecutive_failures = 0
+                source.paused_until = None
+                source.last_error = None
+                source.last_success_at = now
+            await session.commit()
+
+        elapsed = (now - run.started_at).total_seconds()
+        summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
+        if failed:
+            self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
+        else:
+            self.logger.info("本轮完成：%s", summary)
+
+    async def _flush_logs(self) -> None:
+        rows, self.log_buffer = self.log_buffer, []
+        try:
+            await write_logs(rows)
+        except Exception:
+            sink_logger.exception("日志落库失败，丢弃 %d 条", len(rows))
+
+    async def _flush_logs_forever(self) -> None:
+        while True:
+            await asyncio.sleep(1)
+            await self._flush_logs()
+
+    # ---------- 子类使用的工具 ----------
+
+    def request(self, url: str, callback: Callable, **meta: Any) -> Request:
+        self.run.requests += 1
+        return Request(
+            url,
+            callback=callback,
+            errback=self.on_request_error,
+            dont_filter=True,
+            meta={"run_id": self.run.id, **meta},
+            headers=getattr(self, "request_headers", None),
+        )
+
+    def on_request_error(self, failure: Failure) -> None:
+        status = failure.value.response.status if failure.check(HttpError) else None
+        if status in (403, 429):
+            self.run.blocked = True
+        message = f"请求失败 {failure.request.url}：{status or repr(failure.value)}"
+        self.run.errors.append(message)
+        self.logger.error(message)
+
+    async def known_items(self, external_ids: list[str]) -> dict[str, tuple[str, str]]:
+        """已入库条目的 (title, summary)，用于判断是否新增、是否需要重抓详情、是否继续翻页"""
+        async with SessionLocal() as session:
+            rows = await session.execute(
+                select(NewsItem.external_id, NewsItem.title, NewsItem.summary)
+                .where(NewsItem.source_code == self.name, NewsItem.external_id.in_(external_ids))
+            )
+        return {external_id: (title, summary) for external_id, title, summary in rows}
+
+    def should_continue(self, external_ids: list[str], known: dict, oldest: datetime, page: int) -> bool:
+        """未越过回溯时间、未被限流，且本页有新条目（或本轮为全量扫描）时才翻下一页"""
+        return (
+            not self.run.blocked
+            and page < MAX_PAGES
+            and oldest > self.run.cutoff
+            and (self.run.full_scan or any(external_id not in known for external_id in external_ids))
+        )
+
+    def item(
+        self,
+        response: Response,
+        *,
+        external_id: str,
+        url: str,
+        title: str,
+        summary: str,
+        content_html: str,
+        published_at: datetime,
+        author: str | None = None,
+        tags: list[str] | None = None,
+        is_paywalled: bool = False,
+        source_updated_at: datetime | None = None,
+        extra: dict | None = None,
+    ) -> dict[str, Any]:
+        return {
+            "source_code": self.name,
+            "external_id": external_id,
+            "url": url,
+            "title": title,
+            "summary": summary,
+            "content_html": content_html,
+            "content_text": html_to_text(content_html),
+            "author": author or None,
+            "tags": tags or [],
+            "is_paywalled": is_paywalled,
+            "published_at": published_at,
+            "source_updated_at": source_updated_at,
+            "extra": extra or {},
+            "raw_response_id": response.meta["raw_response_id"],
+        }

backend/src/alphapulse/crawler/spiders/cls.py

@@ -0,0 +1,102 @@
+import hashlib
+import html
+import json
+import re
+from urllib.parse import urlencode
+
+from scrapy import Request
+from scrapy.http import Response
+
+from alphapulse.crawler.spiders.base import NewsSpider, ts
+
+
+def signed_url(path: str, params: dict[str, str]) -> str:
+    # 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
+    query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
+    query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
+    return f"https://www.cls.cn{path}?{urlencode(query)}"
+
+
+class ClsTelegraphSpider(NewsSpider):
+    """财联社·电报：列表即全文；翻页需 refresh_type=1"""
+
+    name = "cls_telegraph"
+    request_headers = {"Referer": "https://www.cls.cn/telegraph"}
+
+    def first_requests(self) -> list[Request]:
+        return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
+
+    async def parse_list(self, response: Response):
+        body = response.json()
+        if body["errno"] != 0:
+            raise ValueError(f"接口返回错误：{body}")
+        rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
+        # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            yield self.item(
+                response,
+                external_id=str(x["id"]),
+                url=f"https://www.cls.cn/detail/{x['id']}",
+                title=x["title"],
+                summary=x["brief"],
+                content_html=f"<p>{html.escape(x['content'])}</p>",
+                published_at=ts(x["ctime"]),
+                source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
+                author=x.get("author"),
+                tags=[s["subject_name"] for s in x.get("subjects") or []],
+                extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
+            )
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
+            last_time = str(min(x["ctime"] for x in rows))
+            params = {"category": "", "last_time": last_time, "refresh_type": "1", "rn": "20"}
+            yield self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page + 1)
+
+
+class ClsDepthSpider(NewsSpider):
+    """财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
+
+    name = "cls_depth"
+    request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
+
+    def first_requests(self) -> list[Request]:
+        # 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
+        return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
+
+    async def parse_list(self, response: Response):
+        body = response.json()
+        if body["errno"] != 0:
+            raise ValueError(f"接口返回错误：{body}")
+        rows = [x for x in body["data"] if not x["is_ad"]]
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            if known.get(str(x["id"])) != (x["title"], x["brief"]):
+                # 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
+                yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
+            params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
+            yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
+
+    async def parse_detail(self, response: Response):
+        match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
+        if match is None:
+            raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
+        a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
+        yield self.item(
+            response,
+            external_id=str(a["id"]),
+            url=f"https://www.cls.cn/detail/{a['id']}",
+            title=a["title"],
+            summary=response.meta["summary"],
+            content_html=a["content"],
+            published_at=ts(a["ctime"]),
+            author=(a.get("author") or {}).get("name"),
+            tags=[s["name"] for s in a.get("subject") or []],
+            # 付费内容只保存公开部分
+            is_paywalled=not a["isFree"],
+            extra={"level": a.get("level")},
+        )

backend/src/alphapulse/crawler/spiders/wscn.py

@@ -0,0 +1,81 @@
+from scrapy import Request
+from scrapy.http import Response
+
+from alphapulse.crawler.spiders.base import NewsSpider, ts
+
+API = "https://api-one-wscn.awtmt.com/apiv1/content"
+
+
+class WscnNewsSpider(NewsSpider):
+    """华尔街见闻·文章：列表只有摘要，新条目或标题/摘要变化时再拉详情"""
+
+    name = "wscn_news"
+
+    def first_requests(self) -> list[Request]:
+        return [self.list_request("", 1)]
+
+    def list_request(self, cursor: str, page: int) -> Request:
+        return self.request(f"{API}/information-flow?channel=global&accept=article&limit=20&cursor={cursor}", self.parse_list, page=page)
+
+    async def parse_list(self, response: Response):
+        data = response.json()["data"]
+        # 信息流里混有快讯，快讯由 wscn_live 负责
+        rows = [x["resource"] for x in data["items"] if x["resource_type"] == "article"]
+        ids = [str(r["id"]) for r in rows]
+        known = await self.known_items(ids)
+        for r in rows:
+            if known.get(str(r["id"])) != (r["title"], r["content_short"]):
+                yield self.request(f"{API}/articles/{r['id']}?extract=0", self.parse_detail, summary=r["content_short"])
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(r["display_time"] for r in rows)), page):
+            yield self.list_request(data["next_cursor"], page + 1)
+
+    async def parse_detail(self, response: Response):
+        a = response.json()["data"]
+        yield self.item(
+            response,
+            external_id=str(a["id"]),
+            url=f"https://wallstreetcn.com/articles/{a['id']}",
+            title=a["title"],
+            summary=response.meta["summary"],
+            content_html=a["content"],
+            published_at=ts(a["display_time"]),
+            author=(a.get("author") or {}).get("display_name"),
+            is_paywalled=bool(a.get("is_need_pay")),
+            extra={"source_name": a.get("source_name"), "is_priced": a.get("is_priced")},
+        )
+
+
+class WscnLiveSpider(NewsSpider):
+    """华尔街见闻·快讯：列表即全文"""
+
+    name = "wscn_live"
+
+    def first_requests(self) -> list[Request]:
+        return [self.list_request("", 1)]
+
+    def list_request(self, cursor: str, page: int) -> Request:
+        return self.request(f"{API}/lives?channel=global-channel&limit=50&cursor={cursor}", self.parse_list, page=page)
+
+    async def parse_list(self, response: Response):
+        data = response.json()["data"]
+        rows = data["items"]
+        # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            yield self.item(
+                response,
+                external_id=str(x["id"]),
+                url=x["uri"],
+                title=x["title"],
+                summary="",
+                content_html=x["content"] + (x.get("content_more") or ""),
+                published_at=ts(x["display_time"]),
+                author=(x.get("author") or {}).get("display_name"),
+                tags=x.get("channels"),
+                extra={"score": x.get("score"), "symbols": x.get("symbols")},
+            )
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["display_time"] for x in rows)), page):
+            yield self.list_request(data["next_cursor"], page + 1)