Initial AlphaPulse implementation: news crawler and Binance klines

- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth,
  news/crawler/market APIs with SSE and WebSocket streaming
- Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth,
  cls_telegraph spiders, idempotent upserts with revisions, raw response
  archiving, per-source subprocess supervision, restart and backoff
- Market worker: full-history Binance spot/USD-M kline backfill,
  realtime WS stream, hourly and on-reconnect catch-up
- Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed,
  crawler console with live logs, Binance-style chart page
- Deploy: docker compose, nginx site, migrate and daily backup scripts

backend/src/alphapulse/api/auth.py

@@ -0,0 +1,58 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from pydantic import BaseModel
+
+from alphapulse.core.auth import COOKIE_NAME, create_token, require_admin, verify_password
+from alphapulse.core.config import get_settings
+from alphapulse.core.redis import KEY_PREFIX, redis
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+MAX_FAILURES = 5
+LOCK_SECONDS = 15 * 60
+
+
+class LoginIn(BaseModel):
+    username: str
+    password: str
+
+
+class MeOut(BaseModel):
+    username: str
+
+
+@router.post("/login")
+async def login(body: LoginIn, request: Request, response: Response) -> MeOut:
+    settings = get_settings()
+    if not settings.admin_password_hash:
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "管理员密码未配置")
+    fail_key = f"{KEY_PREFIX}login:fail:{request.client.host}"
+    if int(await redis.get(fail_key) or 0) >= MAX_FAILURES:
+        raise HTTPException(status.HTTP_429_TOO_MANY_REQUESTS, "失败次数过多，请 15 分钟后再试")
+
+    if body.username != settings.admin_username or not verify_password(body.password, settings.admin_password_hash):
+        async with redis.pipeline() as pipe:
+            await pipe.incr(fail_key).expire(fail_key, LOCK_SECONDS).execute()
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "用户名或密码错误")
+
+    await redis.delete(fail_key)
+    response.set_cookie(
+        COOKIE_NAME,
+        create_token(body.username),
+        max_age=settings.jwt_expire_days * 86400,
+        httponly=True,
+        secure=True,
+        samesite="strict",
+    )
+    return MeOut(username=body.username)
+
+
+@router.post("/logout")
+async def logout(response: Response) -> None:
+    response.delete_cookie(COOKIE_NAME, httponly=True, secure=True, samesite="strict")
+
+
+@router.get("/me")
+async def me(username: Annotated[str, Depends(require_admin)]) -> MeOut:
+    return MeOut(username=username)

backend/src/alphapulse/api/crawler.py

@@ -0,0 +1,180 @@
+import json
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime
+from typing import Annotated, Literal
+from zoneinfo import ZoneInfo
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.sse import EventSourceResponse, ServerSentEvent
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import CONTROL_CHANNEL, LOG_CHANNEL, STATUS_KEY
+from alphapulse.models import CrawlLog, CrawlRun, NewsItem, Source
+
+router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
+
+# “今日”按北京时间计算
+LOCAL_TZ = ZoneInfo("Asia/Shanghai")
+
+
+class ProcessStatus(BaseModel):
+    state: Literal["running", "stopped", "restarting"]
+    pid: int | None
+    started_at: datetime | None
+    next_run_at: datetime
+
+
+class SourceOut(BaseModel):
+    code: str
+    name: str
+    list_url: str
+    enabled: bool
+    interval_seconds: int
+    backfill_hours: int
+    consecutive_failures: int
+    last_success_at: datetime | None
+    last_error: str | None
+    paused_until: datetime | None
+    # 管理进程未运行时为 None
+    process: ProcessStatus | None
+    today_new: int
+    today_revised: int
+    total_items: int
+
+
+class SourcesOut(BaseModel):
+    manager_alive: bool
+    db_size_bytes: int
+    sources: list[SourceOut]
+
+
+class SourcePatch(BaseModel):
+    enabled: bool | None = None
+    interval_seconds: Annotated[int | None, Field(ge=5, le=86400)] = None
+
+
+class RunOut(BaseModel):
+    id: uuid.UUID
+    source_code: str
+    trigger: str
+    started_at: datetime
+    finished_at: datetime | None
+    status: str
+    requests: int
+    items_seen: int
+    items_new: int
+    items_revised: int
+    error: str | None
+
+
+class LogOut(BaseModel):
+    id: int
+    source_code: str
+    run_id: uuid.UUID | None
+    ts: datetime
+    level: str
+    message: str
+
+
+class RestartIn(BaseModel):
+    # 来源 code 或 "all"
+    source: str
+
+
+@router.get("/sources")
+async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -> SourcesOut:
+    sources = (await session.scalars(select(Source).order_by(Source.code))).all()
+    today = datetime.now(LOCAL_TZ).replace(hour=0, minute=0, second=0, microsecond=0)
+    today_counts = {
+        code: (new or 0, revised or 0)
+        for code, new, revised in await session.execute(
+            select(CrawlRun.source_code, func.sum(CrawlRun.items_new), func.sum(CrawlRun.items_revised))
+            .where(CrawlRun.started_at >= today)
+            .group_by(CrawlRun.source_code)
+        )
+    }
+    totals = dict((await session.execute(
+        select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
+    )).all())
+    statuses = await redis.hgetall(STATUS_KEY)
+    db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
+    return SourcesOut(
+        manager_alive=bool(statuses),
+        db_size_bytes=db_size,
+        sources=[
+            SourceOut.model_validate({
+                **s.__dict__,
+                "process": json.loads(statuses[s.code]) if s.code in statuses else None,
+                "today_new": today_counts.get(s.code, (0, 0))[0],
+                "today_revised": today_counts.get(s.code, (0, 0))[1],
+                "total_items": totals.get(s.code, 0),
+            })
+            for s in sources
+        ],
+    )
+
+
+@router.patch("/sources/{code}")
+async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    source = await session.get(Source, code)
+    if source is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    for field, value in body.model_dump(exclude_none=True).items():
+        setattr(source, field, value)
+    await session.commit()
+
+
+@router.get("/runs")
+async def list_runs(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: str | None = None,
+    limit: Annotated[int, Query(ge=1, le=200)] = 50,
+) -> list[RunOut]:
+    stmt = select(CrawlRun).order_by(CrawlRun.started_at.desc()).limit(limit)
+    if source:
+        stmt = stmt.where(CrawlRun.source_code == source)
+    return [RunOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
+
+
+@router.get("/logs")
+async def list_logs(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: str | None = None,
+    before_id: int | None = None,
+    limit: Annotated[int, Query(ge=1, le=1000)] = 200,
+) -> list[LogOut]:
+    """按时间正序返回 before_id 之前最近的 limit 条"""
+    stmt = select(CrawlLog).order_by(CrawlLog.id.desc()).limit(limit)
+    if source:
+        stmt = stmt.where(CrawlLog.source_code == source)
+    if before_id:
+        stmt = stmt.where(CrawlLog.id < before_id)
+    rows = [LogOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
+    return rows[::-1]
+
+
+@router.get("/logs/stream", response_class=EventSourceResponse)
+async def stream_logs(source: str | None = None) -> AsyncIterator[ServerSentEvent]:
+    async with redis.pubsub() as pubsub:
+        await pubsub.subscribe(LOG_CHANNEL)
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            if source and json.loads(message["data"])["source_code"] != source:
+                continue
+            yield ServerSentEvent(raw_data=message["data"], event="log")
+
+
+@router.post("/restart")
+async def restart(body: RestartIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    if body.source != "all" and await session.get(Source, body.source) is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    receivers = await redis.publish(CONTROL_CHANNEL, json.dumps({"action": "restart", "source": body.source}))
+    if receivers == 0:
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "采集管理进程未运行")

backend/src/alphapulse/api/main.py

@@ -0,0 +1,29 @@
+from fastapi import Depends, FastAPI
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import HTMLResponse
+
+from alphapulse.api import auth, crawler, market, news
+from alphapulse.core.auth import require_admin
+
+# 关闭默认文档路由，改为登录后才能访问
+app = FastAPI(title="AlphaPulse", docs_url=None, redoc_url=None, openapi_url=None)
+app.include_router(auth.router)
+app.include_router(news.router)
+app.include_router(crawler.router)
+app.include_router(market.router)
+app.include_router(market.ws_router)
+
+
+@app.get("/api/health", include_in_schema=False)
+async def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/api/openapi.json", include_in_schema=False, dependencies=[Depends(require_admin)])
+async def openapi() -> dict:
+    return app.openapi()
+
+
+@app.get("/api/docs", include_in_schema=False, dependencies=[Depends(require_admin)])
+async def docs() -> HTMLResponse:
+    return get_swagger_ui_html(openapi_url="/api/openapi.json", title="AlphaPulse API")

backend/src/alphapulse/api/market.py

@@ -0,0 +1,202 @@
+import asyncio
+import json
+import time
+from datetime import datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect, status
+from pydantic import BaseModel
+from sqlalchemy import delete, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin, websocket_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.market.binance import BinanceClient
+from alphapulse.market.channels import kline_channel, ticker_channel
+from alphapulse.models import Kline, KlineSyncState, MarketSymbol
+from alphapulse.models.market import INTERVALS
+
+router = APIRouter(prefix="/api/market", tags=["market"], dependencies=[Depends(require_admin)])
+# WebSocket 握手不能用基于 Request 的依赖，单独鉴权
+ws_router = APIRouter(prefix="/api/market", tags=["market"])
+
+Market = Literal["spot", "um"]
+SYMBOLS_CACHE_SECONDS = 3600
+MAX_KLINES = 1500
+
+client = BinanceClient()
+
+
+class IntervalProgress(BaseModel):
+    done: bool
+    percent: float
+
+
+class SymbolOut(BaseModel):
+    id: int
+    market: str
+    symbol: str
+    base: str
+    quote: str
+    price_precision: int
+    volume_precision: int
+    status: str
+    is_default: bool
+    added_at: datetime
+    last_error: str | None
+    progress: dict[str, IntervalProgress]
+
+
+class SymbolIn(BaseModel):
+    market: Market
+    symbol: str
+
+
+class TradableSymbol(BaseModel):
+    symbol: str
+    base: str
+    quote: str
+    price_precision: int
+    volume_precision: int
+
+
+async def tradable_symbols(market: str) -> list[dict]:
+    key = f"{KEY_PREFIX}market:symbols:{market}"
+    if cached := await redis.get(key):
+        return json.loads(cached)
+    symbols = await client.tradable_symbols(market)
+    await redis.set(key, json.dumps(symbols), ex=SYMBOLS_CACHE_SECONDS)
+    return symbols
+
+
+@router.get("/symbols")
+async def list_symbols(session: Annotated[AsyncSession, Depends(get_session)]) -> list[SymbolOut]:
+    symbols = (await session.scalars(select(MarketSymbol).order_by(MarketSymbol.is_default.desc(), MarketSymbol.added_at))).all()
+    states = {(s.market, s.symbol, s.interval): s for s in await session.scalars(select(KlineSyncState))}
+    now = int(time.time() * 1000)
+
+    def progress(s: MarketSymbol, interval: str) -> IntervalProgress:
+        state = states.get((s.market, s.symbol, interval))
+        if state is None or state.earliest_open_time is None:
+            return IntervalProgress(done=False, percent=0)
+        if state.backfill_done:
+            return IntervalProgress(done=True, percent=100)
+        span = max(now - state.earliest_open_time, 1)
+        return IntervalProgress(done=False, percent=round((state.latest_open_time - state.earliest_open_time) / span * 100, 1))
+
+    return [
+        SymbolOut.model_validate({**s.__dict__, "progress": {i: progress(s, i) for i in INTERVALS}})
+        for s in symbols
+    ]
+
+
+@router.post("/symbols", status_code=status.HTTP_201_CREATED)
+async def add_symbol(body: SymbolIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    symbol = body.symbol.strip().upper()
+    match = next((s for s in await tradable_symbols(body.market) if s["symbol"] == symbol), None)
+    if match is None:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"{body.market} 市场没有可交易的 {symbol}")
+    session.add(MarketSymbol(market=body.market, **match))
+    try:
+        await session.commit()
+    except IntegrityError:
+        raise HTTPException(status.HTTP_409_CONFLICT, f"{symbol} 已存在") from None
+
+
+@router.delete("/symbols/{symbol_id}")
+async def delete_symbol(symbol_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    """删除交易对及其 K 线（K 线可随时从币安重新补齐）"""
+    s = await session.get(MarketSymbol, symbol_id)
+    if s is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "交易对不存在")
+    await session.execute(delete(Kline).where(Kline.market == s.market, Kline.symbol == s.symbol))
+    await session.execute(delete(KlineSyncState).where(KlineSyncState.market == s.market, KlineSyncState.symbol == s.symbol))
+    await session.delete(s)
+    await session.commit()
+
+
+@router.get("/exchange-info")
+async def exchange_info(market: Market) -> list[TradableSymbol]:
+    return [TradableSymbol.model_validate(s) for s in await tradable_symbols(market)]
+
+
+@router.get("/klines")
+async def klines(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    market: Market,
+    symbol: str,
+    interval: str,
+    end: int | None = None,
+    limit: Annotated[int, Query(ge=1, le=MAX_KLINES)] = 500,
+) -> list[list[float]]:
+    """返回 open_time < end 的最近 limit 根，按时间正序：[open_time, open, high, low, close, volume, quote_volume]"""
+    if interval == "1s":
+        # 1s 不落库，直接取币安最近数据
+        if market != "spot":
+            raise HTTPException(status.HTTP_400_BAD_REQUEST, "合约没有 1s 周期")
+        params = {"symbol": symbol, "interval": "1s", "limit": limit, **({"endTime": end - 1} if end else {})}
+        rows = await client.get("spot", "klines", params)
+        return [[r[0], float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5]), float(r[7])] for r in rows]
+    if interval not in INTERVALS:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"不支持的周期：{interval}")
+    stmt = (
+        select(Kline.open_time, Kline.open, Kline.high, Kline.low, Kline.close, Kline.volume, Kline.quote_volume)
+        .where(Kline.market == market, Kline.symbol == symbol, Kline.interval == interval)
+        .order_by(Kline.open_time.desc())
+        .limit(limit)
+    )
+    if end:
+        stmt = stmt.where(Kline.open_time < end)
+    rows = (await session.execute(stmt)).all()
+    return [[r[0], *(float(v) for v in r[1:])] for r in reversed(rows)]
+
+
+@router.get("/ticker")
+async def ticker(market: Market, symbol: str) -> dict:
+    return await client.ticker(market, symbol)
+
+
+@ws_router.websocket("/ws")
+async def market_ws(websocket: WebSocket) -> None:
+    """客户端发送 {"market","symbol","interval"} 切换订阅；服务端推送 {"type":"kline"|"ticker", market, symbol, interval?, data}"""
+    if websocket_admin(websocket) is None:
+        await websocket.close(code=4401)
+        return
+    await websocket.accept()
+    pubsub = redis.pubsub()
+    subscribed = asyncio.Event()
+
+    async def forward() -> None:
+        await subscribed.wait()
+        while True:
+            message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
+            if message is None:
+                continue
+            # 频道格式：ap:pub:kline:<market>:<symbol>:<interval> 或 ap:pub:ticker:<market>:<symbol>
+            parts = message["channel"].split(":")
+            await websocket.send_text(json.dumps({
+                "type": parts[2],
+                "market": parts[3],
+                "symbol": parts[4],
+                "interval": parts[5] if parts[2] == "kline" else None,
+                "data": json.loads(message["data"]),
+            }))
+
+    task = asyncio.create_task(forward())
+    try:
+        while True:
+            sub = await websocket.receive_json()
+            if subscribed.is_set():
+                await pubsub.unsubscribe()
+            await pubsub.subscribe(
+                kline_channel(sub["market"], sub["symbol"], sub["interval"]),
+                ticker_channel(sub["market"], sub["symbol"]),
+            )
+            subscribed.set()
+    except WebSocketDisconnect:
+        pass
+    finally:
+        task.cancel()
+        await pubsub.aclose()

backend/src/alphapulse/api/news.py

@@ -0,0 +1,110 @@
+from collections.abc import AsyncIterator
+from datetime import datetime
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.sse import EventSourceResponse, ServerSentEvent
+from pydantic import BaseModel
+from sqlalchemy import func, or_, select, tuple_
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import NEWS_CHANNEL
+from alphapulse.models import NewsItem, NewsRevision
+
+router = APIRouter(prefix="/api/news", tags=["news"], dependencies=[Depends(require_admin)])
+
+EXCERPT_LENGTH = 300
+
+
+class NewsBrief(BaseModel):
+    id: int
+    source_code: str
+    url: str
+    title: str
+    summary: str
+    excerpt: str
+    author: str | None
+    tags: list
+    is_paywalled: bool
+    published_at: datetime
+    first_seen_at: datetime
+    revision_no: int
+
+
+class NewsPage(BaseModel):
+    items: list[NewsBrief]
+    # 形如 "<published_at ISO>|<id>"，原样传回即可取下一页
+    next_cursor: str | None
+
+
+class NewsDetail(NewsBrief):
+    content_html: str
+    source_updated_at: datetime | None
+    last_seen_at: datetime
+    extra: dict
+
+
+class Revision(BaseModel):
+    revision_no: int
+    title: str
+    summary: str
+    content_html: str
+    captured_at: datetime
+
+
+@router.get("")
+async def list_news(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: Annotated[list[str] | None, Query()] = None,
+    q: str | None = None,
+    start: datetime | None = None,
+    end: datetime | None = None,
+    cursor: str | None = None,
+    limit: Annotated[int, Query(ge=1, le=100)] = 30,
+) -> NewsPage:
+    stmt = select(NewsItem, func.left(NewsItem.content_text, EXCERPT_LENGTH).label("excerpt"))
+    if source:
+        stmt = stmt.where(NewsItem.source_code.in_(source))
+    if q:
+        pattern = f"%{q}%"
+        stmt = stmt.where(or_(NewsItem.title.ilike(pattern), NewsItem.content_text.ilike(pattern)))
+    if start:
+        stmt = stmt.where(NewsItem.published_at >= start)
+    if end:
+        stmt = stmt.where(NewsItem.published_at < end)
+    if cursor:
+        published_at, news_id = cursor.rsplit("|", 1)
+        stmt = stmt.where(tuple_(NewsItem.published_at, NewsItem.id) < (datetime.fromisoformat(published_at), int(news_id)))
+    rows = (await session.execute(stmt.order_by(NewsItem.published_at.desc(), NewsItem.id.desc()).limit(limit))).all()
+
+    items = [NewsBrief.model_validate({**news.__dict__, "excerpt": excerpt}) for news, excerpt in rows]
+    next_cursor = f"{items[-1].published_at.isoformat()}|{items[-1].id}" if len(items) == limit else None
+    return NewsPage(items=items, next_cursor=next_cursor)
+
+
+@router.get("/stream", response_class=EventSourceResponse)
+async def stream_news() -> AsyncIterator[ServerSentEvent]:
+    async with redis.pubsub() as pubsub:
+        await pubsub.subscribe(NEWS_CHANNEL)
+        async for message in pubsub.listen():
+            if message["type"] == "message":
+                yield ServerSentEvent(raw_data=message["data"], event="news")
+
+
+@router.get("/{news_id}")
+async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> NewsDetail:
+    news = await session.get(NewsItem, news_id)
+    if news is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
+    return NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]})
+
+
+@router.get("/{news_id}/revisions")
+async def list_revisions(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> list[Revision]:
+    rows = await session.scalars(
+        select(NewsRevision).where(NewsRevision.news_id == news_id).order_by(NewsRevision.revision_no.desc())
+    )
+    return [Revision.model_validate(r, from_attributes=True) for r in rows]

backend/src/alphapulse/core/auth.py

@@ -0,0 +1,59 @@
+import getpass
+import sys
+from datetime import UTC, datetime, timedelta
+
+import jwt
+from argon2 import PasswordHasher
+from argon2.exceptions import VerifyMismatchError
+from fastapi import HTTPException, Request, WebSocket, status
+
+from alphapulse.core.config import get_settings
+
+COOKIE_NAME = "ap_token"
+_hasher = PasswordHasher()
+
+
+def verify_password(password: str, password_hash: str) -> bool:
+    try:
+        return _hasher.verify(password_hash, password)
+    except VerifyMismatchError:
+        return False
+
+
+def create_token(username: str) -> str:
+    settings = get_settings()
+    expire = datetime.now(UTC) + timedelta(days=settings.jwt_expire_days)
+    return jwt.encode({"sub": username, "exp": expire}, settings.jwt_secret, algorithm="HS256")
+
+
+def _username_from_token(token: str | None) -> str | None:
+    if not token:
+        return None
+    try:
+        payload = jwt.decode(token, get_settings().jwt_secret, algorithms=["HS256"])
+    except jwt.InvalidTokenError:
+        return None
+    return payload["sub"]
+
+
+def require_admin(request: Request) -> str:
+    username = _username_from_token(request.cookies.get(COOKIE_NAME))
+    if username != get_settings().admin_username:
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "未登录或登录已过期")
+    return username
+
+
+def websocket_admin(websocket: WebSocket) -> str | None:
+    # WebSocket 握手阶段不能用 HTTPException，由调用方自行关闭连接
+    username = _username_from_token(websocket.cookies.get(COOKIE_NAME))
+    return username if username == get_settings().admin_username else None
+
+
+if __name__ == "__main__":
+    # 用法：python -m alphapulse.core.auth hash-password，输出写入 .env 的 ADMIN_PASSWORD_HASH
+    if sys.argv[1:] != ["hash-password"]:
+        sys.exit("用法：python -m alphapulse.core.auth hash-password")
+    password = getpass.getpass("新密码：")
+    if password != getpass.getpass("再次输入："):
+        sys.exit("两次输入不一致")
+    print(_hasher.hash(password))

backend/src/alphapulse/core/config.py

@@ -0,0 +1,20 @@
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    # 容器内由 compose 的 env_file 注入；只有一套配置，见仓库根目录 .env
+    model_config = SettingsConfigDict(extra="ignore")
+
+    database_url: str
+    redis_url: str
+    jwt_secret: str
+    admin_username: str
+    admin_password_hash: str = ""
+    jwt_expire_days: int = 7
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

backend/src/alphapulse/core/db.py

@@ -0,0 +1,19 @@
+from collections.abc import AsyncIterator
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.orm import DeclarativeBase
+
+from alphapulse.core.config import get_settings
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+engine = create_async_engine(get_settings().database_url, pool_pre_ping=True)
+SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+
+
+async def get_session() -> AsyncIterator[AsyncSession]:
+    async with SessionLocal() as session:
+        yield session

backend/src/alphapulse/core/redis.py

@@ -0,0 +1,8 @@
+from redis.asyncio import Redis
+
+from alphapulse.core.config import get_settings
+
+# DB 14 与 AFFiNE 共用：所有 key 必须带此前缀，禁止 FLUSHDB
+KEY_PREFIX = "ap:"
+
+redis = Redis.from_url(get_settings().redis_url, decode_responses=True)

backend/src/alphapulse/crawler/events.py

@@ -0,0 +1,41 @@
+import json
+import uuid
+
+from sqlalchemy import insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.models import CrawlLog
+
+LOG_CHANNEL = f"{KEY_PREFIX}pub:logs"
+NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
+CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
+STATUS_KEY = f"{KEY_PREFIX}crawler:status"
+
+
+def start_urls_key(source_code: str) -> str:
+    return f"{KEY_PREFIX}{source_code}:start_urls"
+
+
+async def write_logs(rows: list[tuple[str, uuid.UUID | None, str, str]]) -> None:
+    """rows: (source_code, run_id, level, message)；落库后再发布，前端拿到的日志都带 id"""
+    if not rows:
+        return
+    async with SessionLocal() as session:
+        result = await session.execute(
+            insert(CrawlLog).returning(CrawlLog.id, CrawlLog.ts),
+            [{"source_code": s, "run_id": r, "level": lv, "message": m} for s, r, lv, m in rows],
+        )
+        saved = result.all()
+        await session.commit()
+    async with redis.pipeline(transaction=False) as pipe:
+        for (source_code, run_id, level, message), (log_id, ts) in zip(rows, saved, strict=True):
+            pipe.publish(LOG_CHANNEL, json.dumps({
+                "id": log_id,
+                "source_code": source_code,
+                "run_id": str(run_id) if run_id else None,
+                "ts": ts.isoformat(),
+                "level": level,
+                "message": message,
+            }, ensure_ascii=False))
+        await pipe.execute()

backend/src/alphapulse/crawler/manager.py

@@ -0,0 +1,186 @@
+"""采集管理进程：按间隔触发采集、守护每个来源的 scrapy 子进程、处理重启指令。
+
+用法（工作目录需包含 scrapy.cfg）：python -m alphapulse.crawler.manager
+"""
+
+import asyncio
+import json
+import logging
+import signal
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import select, update
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.crawler.events import CONTROL_CHANNEL, STATUS_KEY, start_urls_key, write_logs
+from alphapulse.models import CrawlRun, Source
+
+logger = logging.getLogger("alphapulse.crawler.manager")
+
+RESTART_DELAY_MIN = 5
+RESTART_DELAY_MAX = 300
+# 子进程稳定运行超过该时长后，异常退出的重启延迟重新从最小值开始
+STABLE_SECONDS = 60
+STOP_TIMEOUT = 15
+
+
+@dataclass
+class Worker:
+    code: str
+    proc: asyncio.subprocess.Process | None = None
+    started_at: datetime | None = None
+    next_run_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    # 下一次触发的类型：schedule / startup / restart
+    trigger: str = "schedule"
+    restarting: bool = False
+    restart_delay: int = RESTART_DELAY_MIN
+    wake: asyncio.Event = field(default_factory=asyncio.Event)
+
+
+async def log(code: str, level: str, message: str) -> None:
+    logger.log(logging.getLevelName(level), "[%s] %s", code, message)
+    await write_logs([(code, None, level, message)])
+
+
+class Manager:
+    def __init__(self) -> None:
+        self.workers: dict[str, Worker] = {}
+        self.stopping = asyncio.Event()
+
+    async def run(self) -> None:
+        async with SessionLocal() as session:
+            codes = (await session.scalars(select(Source.code).order_by(Source.code))).all()
+        self.workers = {code: Worker(code) for code in codes}
+
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            loop.add_signal_handler(sig, self.stopping.set)
+
+        tasks = [asyncio.create_task(self.supervise(w)) for w in self.workers.values()]
+        tasks += [asyncio.create_task(c) for c in (self.schedule_loop(), self.control_loop(), self.status_loop())]
+        await self.stopping.wait()
+        logger.info("收到退出信号，停止全部子进程")
+        await asyncio.gather(*(self.stop_process(w) for w in self.workers.values()))
+        for task in tasks:
+            task.cancel()
+        await redis.delete(STATUS_KEY)
+
+    async def supervise(self, w: Worker) -> None:
+        while not self.stopping.is_set():
+            # 上次进程遗留的请求和未收尾的轮次都已失效，清理后从干净状态开始
+            await redis.delete(f"{KEY_PREFIX}{w.code}:requests", start_urls_key(w.code))
+            async with SessionLocal() as session:
+                await session.execute(
+                    update(CrawlRun)
+                    .where(CrawlRun.source_code == w.code, CrawlRun.status == "running")
+                    .values(status="interrupted", finished_at=datetime.now(UTC), error="子进程退出时本轮未完成")
+                )
+                await session.commit()
+            w.proc = await asyncio.create_subprocess_exec("scrapy", "crawl", w.code)
+            w.started_at = datetime.now(UTC)
+            # 新进程的第一轮立即执行，并做全量回溯扫描
+            if w.trigger == "schedule":
+                w.trigger = "startup"
+            w.next_run_at = w.started_at
+            await log(w.code, "INFO", f"子进程已启动 pid={w.proc.pid}")
+            returncode = await w.proc.wait()
+            ran_seconds = (datetime.now(UTC) - w.started_at).total_seconds()
+            w.proc = None
+            if self.stopping.is_set():
+                return
+            if w.restarting:
+                w.restarting = False
+                w.restart_delay = RESTART_DELAY_MIN
+                continue
+            if ran_seconds >= STABLE_SECONDS:
+                w.restart_delay = RESTART_DELAY_MIN
+            await log(w.code, "ERROR", f"子进程异常退出 returncode={returncode}，{w.restart_delay}s 后重启")
+            w.wake.clear()
+            try:
+                await asyncio.wait_for(w.wake.wait(), timeout=w.restart_delay)
+            except TimeoutError:
+                pass
+            w.restart_delay = min(w.restart_delay * 2, RESTART_DELAY_MAX)
+
+    async def stop_process(self, w: Worker) -> None:
+        if w.proc is None:
+            return
+        w.proc.terminate()
+        try:
+            await asyncio.wait_for(w.proc.wait(), timeout=STOP_TIMEOUT)
+        except TimeoutError:
+            await log(w.code, "WARNING", f"子进程 {STOP_TIMEOUT}s 内未退出，强制结束")
+            w.proc.kill()
+
+    async def schedule_loop(self) -> None:
+        while True:
+            async with SessionLocal() as session:
+                sources = (await session.scalars(select(Source))).all()
+            now = datetime.now(UTC)
+            for s in sources:
+                w = self.workers[s.code]
+                if not s.enabled or w.proc is None or w.restarting:
+                    continue
+                if s.paused_until and s.paused_until > now:
+                    w.next_run_at = max(w.next_run_at, s.paused_until)
+                    continue
+                if now < w.next_run_at:
+                    continue
+                # 子进程还没取走上一条触发时不再堆积
+                if await redis.llen(start_urls_key(s.code)) == 0:
+                    await redis.lpush(start_urls_key(s.code), json.dumps({"trigger": w.trigger, "backfill_hours": s.backfill_hours}))
+                    w.trigger = "schedule"
+                w.next_run_at = now + timedelta(seconds=s.interval_seconds)
+            await asyncio.sleep(1)
+
+    async def control_loop(self) -> None:
+        async with redis.pubsub() as pubsub:
+            await pubsub.subscribe(CONTROL_CHANNEL)
+            async for message in pubsub.listen():
+                if message["type"] != "message":
+                    continue
+                command = json.loads(message["data"])
+                if command["action"] != "restart":
+                    logger.warning("未知指令：%s", command)
+                    continue
+                codes = list(self.workers) if command["source"] == "all" else [command["source"]]
+                await asyncio.gather(*(self.restart(self.workers[code]) for code in codes))
+
+    async def restart(self, w: Worker) -> None:
+        await log(w.code, "INFO", "收到重启指令")
+        async with SessionLocal() as session:
+            await session.execute(
+                update(Source).where(Source.code == w.code).values(consecutive_failures=0, paused_until=None)
+            )
+            await session.commit()
+        w.trigger = "restart"
+        w.next_run_at = datetime.now(UTC)
+        if w.proc is None:
+            # 处于异常退出后的等待期：立即重启
+            w.wake.set()
+            return
+        w.restarting = True
+        await self.stop_process(w)
+
+    async def status_loop(self) -> None:
+        while True:
+            mapping = {
+                w.code: json.dumps({
+                    "state": "restarting" if w.restarting else ("running" if w.proc else "stopped"),
+                    "pid": w.proc.pid if w.proc else None,
+                    "started_at": w.started_at.isoformat() if w.started_at else None,
+                    "next_run_at": w.next_run_at.isoformat(),
+                })
+                for w in self.workers.values()
+            }
+            async with redis.pipeline() as pipe:
+                # 管理进程挂掉后状态自动过期，前端据此显示"管理进程未运行"
+                await pipe.hset(STATUS_KEY, mapping=mapping).expire(STATUS_KEY, 10).execute()
+            await asyncio.sleep(2)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    asyncio.run(Manager().run())

backend/src/alphapulse/crawler/middlewares.py

@@ -0,0 +1,58 @@
+import asyncio
+import gzip
+import hashlib
+from urllib.parse import urlparse
+
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
+from sqlalchemy import insert
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.models import RawBody, RawResponse
+
+
+class RedisRateLimitMiddleware:
+    """按域名的跨进程限速：同一域名两次请求至少间隔 interval_ms"""
+
+    interval_ms = 1000
+
+    async def process_request(self, request: Request) -> None:
+        key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
+        while not await redis.set(key, "1", px=self.interval_ms, nx=True):
+            ttl = await redis.pttl(key)
+            await asyncio.sleep(max(ttl, 50) / 1000)
+
+
+class RawResponseMiddleware:
+    """每次响应都写 raw_responses；响应体按 sha256 去重存 raw_bodies"""
+
+    def __init__(self, crawler: Crawler) -> None:
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> "RawResponseMiddleware":
+        return cls(crawler)
+
+    async def process_response(self, request: Request, response: Response) -> Response:
+        sha256 = hashlib.sha256(response.body).hexdigest()
+        headers = {k.decode(): b", ".join(v).decode("latin-1") for k, v in response.headers.items()}
+        async with SessionLocal() as session:
+            await session.execute(
+                pg_insert(RawBody).values(sha256=sha256, body=gzip.compress(response.body)).on_conflict_do_nothing()
+            )
+            raw_id = await session.scalar(
+                insert(RawResponse).values(
+                    run_id=request.meta.get("run_id"),
+                    source_code=self.crawler.spider.name,
+                    url=response.url,
+                    method=request.method,
+                    status=response.status,
+                    headers=headers,
+                    body_sha256=sha256,
+                ).returning(RawResponse.id)
+            )
+            await session.commit()
+        request.meta["raw_response_id"] = raw_id
+        return response

backend/src/alphapulse/crawler/pipelines.py

@@ -0,0 +1,74 @@
+import hashlib
+import json
+from typing import Any
+
+from scrapy.crawler import Crawler
+from sqlalchemy import func, select
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import NEWS_CHANNEL
+from alphapulse.models import NewsItem, NewsRevision
+
+UPDATABLE_FIELDS = ("url", "title", "summary", "content_html", "content_text", "author", "tags",
+                    "is_paywalled", "published_at", "source_updated_at", "extra")
+
+
+def content_hash(item: dict[str, Any]) -> str:
+    return hashlib.sha256("\0".join((item["title"], item["summary"], item["content_html"])).encode()).hexdigest()
+
+
+class NewsPipeline:
+    """按 (source_code, external_id) 幂等落库；内容变化写一条修订"""
+
+    def __init__(self, crawler: Crawler) -> None:
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> "NewsPipeline":
+        return cls(crawler)
+
+    async def process_item(self, item: dict[str, Any]) -> dict[str, Any]:
+        spider = self.crawler.spider
+        digest = content_hash(item)
+        raw_response_id = item.pop("raw_response_id")
+        async with SessionLocal() as session:
+            news = await session.scalar(
+                select(NewsItem)
+                .where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
+                .with_for_update()
+            )
+            if news is None:
+                kind = "new"
+                news = NewsItem(**item, content_hash=digest, revision_no=1)
+                session.add(news)
+                await session.flush()
+            elif news.content_hash != digest:
+                kind = "revised"
+                for field in UPDATABLE_FIELDS:
+                    setattr(news, field, item[field])
+                news.content_hash = digest
+                news.revision_no += 1
+                news.last_seen_at = func.now()
+            else:
+                kind = None
+                news.last_seen_at = func.now()
+
+            if kind:
+                session.add(NewsRevision(
+                    news_id=news.id,
+                    revision_no=news.revision_no,
+                    title=item["title"],
+                    summary=item["summary"],
+                    content_html=item["content_html"],
+                    content_hash=digest,
+                    raw_response_id=raw_response_id,
+                ))
+            news_id = news.id
+            await session.commit()
+
+        spider.run.items_seen += 1
+        if kind:
+            setattr(spider.run, f"items_{kind}", getattr(spider.run, f"items_{kind}") + 1)
+            await redis.publish(NEWS_CHANNEL, json.dumps({"id": news_id, "source_code": item["source_code"], "kind": kind}))
+        return item

backend/src/alphapulse/crawler/settings.py

@@ -0,0 +1,45 @@
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+from alphapulse.core.config import get_settings
+from alphapulse.core.redis import KEY_PREFIX
+
+BOT_NAME = "alphapulse"
+SPIDER_MODULES = ["alphapulse.crawler.spiders"]
+
+# 采集的是无需登录的公开 JSON 接口；限速由 RedisRateLimitMiddleware 统一控制
+ROBOTSTXT_OBEY = False
+USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
+DEFAULT_REQUEST_HEADERS = {"Accept": "application/json, text/html;q=0.9, */*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9"}
+COOKIES_ENABLED = False
+TELNETCONSOLE_ENABLED = False
+# 2.19 默认开启的远程控制 HTTP 服务用不上，关掉避免开放端口
+REMOTE_CONTROL_ENABLED = False
+LOG_LEVEL = "INFO"
+
+CONCURRENT_REQUESTS = 4
+DOWNLOAD_TIMEOUT = 15
+RETRY_TIMES = 2
+# 429/403 不重试，交给来源退避
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+
+DOWNLOADER_MIDDLEWARES = {
+    "alphapulse.crawler.middlewares.RedisRateLimitMiddleware": 100,
+    # process_response 按优先级从大到小调用：排在 HttpCompression(590) 之后拿到解压后的响应体，
+    # 排在 Retry(550) 之前，被重试的每次响应也都会记录
+    "alphapulse.crawler.middlewares.RawResponseMiddleware": 580,
+}
+ITEM_PIPELINES = {"alphapulse.crawler.pipelines.NewsPipeline": 300}
+
+# Scrapy-Redis：请求队列与起始任务放 Redis（DB 14，统一 ap: 前缀）
+SCHEDULER = "scrapy_redis.scheduler.Scheduler"
+DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
+SCHEDULER_PERSIST = True
+SCHEDULER_QUEUE_KEY = f"{KEY_PREFIX}%(spider)s:requests"
+SCHEDULER_DUPEFILTER_KEY = f"{KEY_PREFIX}%(spider)s:dupefilter"
+REDIS_START_URLS_KEY = f"{KEY_PREFIX}%(name)s:start_urls"
+REDIS_URL = get_settings().redis_url
+
+# scrapy-redis 0.9.1 仍按旧签名给 stats 传 spider 参数，每个请求都告警，会刷屏实时日志
+warnings.filterwarnings("ignore", category=ScrapyDeprecationWarning, module="scrapy_redis")

backend/src/alphapulse/crawler/spiders/base.py

@@ -0,0 +1,287 @@
+import asyncio
+import json
+import logging
+import re
+import uuid
+from collections.abc import Callable, Coroutine, Iterable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from scrapy import Request, signals
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from scrapy.spidermiddlewares.httperror import HttpError
+from scrapy_redis.spiders import RedisSpider
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from twisted.python.failure import Failure
+from w3lib.html import remove_tags, replace_entities
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.crawler.events import write_logs
+from alphapulse.models import CrawlRun, NewsItem, Source
+
+# 日志落库失败时只输出到 stdout，避免被自己的处理器再次捕获
+sink_logger = logging.getLogger("alphapulse.logsink")
+
+MAX_PAGES = 200
+BACKOFF_BASE_SECONDS = 30
+BACKOFF_MAX_SECONDS = 15 * 60
+
+
+def ts(epoch: int) -> datetime:
+    return datetime.fromtimestamp(epoch, UTC)
+
+
+def html_to_text(html: str) -> str:
+    text = re.sub(r"(?i)<br\s*/?>|</p>|</div>|</h\d>|</li>", "\n", html)
+    text = replace_entities(remove_tags(text))
+    return re.sub(r"\n\s*\n+", "\n", text).strip()
+
+
+@dataclass
+class RunState:
+    id: uuid.UUID
+    trigger: str
+    started_at: datetime
+    # 早于该时间的内容不再翻页回溯
+    cutoff: datetime
+    requests: int = 0
+    items_seen: int = 0
+    items_new: int = 0
+    items_revised: int = 0
+    errors: list[str] = field(default_factory=list)
+    # 遇到 403/429 后本轮不再发新请求
+    blocked: bool = False
+
+    @property
+    def full_scan(self) -> bool:
+        # 进程启动后的第一轮一直翻到回溯时间，补上次被中断的回溯；常规轮次遇到整页已知即停
+        return self.trigger in ("startup", "restart")
+
+
+class DbLogHandler(logging.Handler):
+    """把本来源的日志和全局 WARNING 以上日志收集起来，由刷新任务批量落库并发布"""
+
+    def __init__(self, spider: "NewsSpider") -> None:
+        super().__init__(logging.INFO)
+        self.spider = spider
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.name == sink_logger.name:
+            return
+        if record.name != self.spider.name and record.levelno < logging.WARNING:
+            return
+        message = record.getMessage()
+        # scrapy-redis 每次取到起始任务都会打这一行，没有信息量
+        if message.startswith("start req url"):
+            return
+        if record.exc_info:
+            message += "\n" + logging.Formatter().formatException(record.exc_info)
+        run_id = self.spider.run.id if self.spider.run else None
+        self.spider.log_buffer.append((self.spider.name, run_id, record.levelname, message))
+
+
+class NewsSpider(RedisSpider):
+    """一个来源一个子进程。管理进程往 Redis 推一条触发消息即开始一轮采集，引擎空闲即本轮结束。"""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.run: RunState | None = None
+        self.log_buffer: list[tuple[str, uuid.UUID | None, str, str]] = []
+        self._tasks: set[asyncio.Task] = set()
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider._on_opened, signal=signals.spider_opened)
+        crawler.signals.connect(spider._on_idle, signal=signals.spider_idle)
+        crawler.signals.connect(spider._on_error, signal=signals.spider_error)
+        crawler.signals.connect(spider._on_closed, signal=signals.spider_closed)
+        return spider
+
+    # ---------- 子类实现 ----------
+
+    def first_requests(self) -> Iterable[Request]:
+        raise NotImplementedError
+
+    # ---------- 运行生命周期 ----------
+
+    async def start(self):
+        for request in self.next_requests():
+            yield request
+
+    def make_request_from_data(self, data: bytes) -> list[Request]:
+        tick = json.loads(data)
+        if self.run is not None:
+            # scrapy-redis 的 idle 处理先于 _on_idle 执行：引擎已空闲说明上一轮其实已结束
+            if not self.crawler.engine.spider_is_idle():
+                self.logger.warning("上一轮尚未结束，跳过本次触发（%s）", tick["trigger"])
+                return []
+            self._close_run()
+        now = datetime.now(UTC)
+        self.run = RunState(uuid.uuid4(), tick["trigger"], now, now - timedelta(hours=tick["backfill_hours"]))
+        self._spawn(self._insert_run(self.run))
+        self.logger.info("开始采集（触发方式：%s）", tick["trigger"])
+        return list(self.first_requests())
+
+    def _on_opened(self) -> None:
+        logging.getLogger().addHandler(DbLogHandler(self))
+        self._spawn(self._flush_logs_forever())
+        self.logger.info("采集进程已启动")
+
+    def _on_idle(self) -> None:
+        # scrapy-redis 的 idle 处理先执行，可能刚取到新触发并已开始新一轮；引擎仍有待处理请求时不能结束
+        if self.run is None or not self.crawler.engine.spider_is_idle():
+            return
+        self._close_run()
+
+    def _close_run(self) -> None:
+        run, self.run = self.run, None
+        self._spawn(self._finish_run(run))
+
+    def _on_error(self, failure: Failure, response: Response) -> None:
+        if self.run:
+            self.run.errors.append(f"解析异常 {response.url}：{failure.value!r}")
+
+    async def _on_closed(self) -> None:
+        self.logger.info("采集进程退出")
+        await self._flush_logs()
+
+    def _spawn(self, coro: Coroutine) -> None:
+        task = asyncio.get_running_loop().create_task(coro)
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _insert_run(self, run: RunState) -> None:
+        async with SessionLocal() as session:
+            await session.execute(
+                pg_insert(CrawlRun).values(
+                    id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, status="running"
+                ).on_conflict_do_nothing()
+            )
+            await session.commit()
+
+    async def _finish_run(self, run: RunState) -> None:
+        now = datetime.now(UTC)
+        failed = bool(run.errors)
+        values = {
+            "finished_at": now,
+            "status": "failed" if failed else "success",
+            "requests": run.requests,
+            "items_seen": run.items_seen,
+            "items_new": run.items_new,
+            "items_revised": run.items_revised,
+            "error": "\n".join(run.errors)[:4000] if failed else None,
+        }
+        async with SessionLocal() as session:
+            # 与 _insert_run 并发时谁先执行都正确
+            await session.execute(
+                pg_insert(CrawlRun)
+                .values(id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, **values)
+                .on_conflict_do_update(index_elements=[CrawlRun.id], set_=values)
+            )
+            source = await session.get(Source, self.name, with_for_update=True)
+            if failed:
+                source.consecutive_failures += 1
+                delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
+                source.paused_until = now + timedelta(seconds=delay)
+                source.last_error = run.errors[0][:1000]
+            else:
+                source.consecutive_failures = 0
+                source.paused_until = None
+                source.last_error = None
+                source.last_success_at = now
+            await session.commit()
+
+        elapsed = (now - run.started_at).total_seconds()
+        summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
+        if failed:
+            self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
+        else:
+            self.logger.info("本轮完成：%s", summary)
+
+    async def _flush_logs(self) -> None:
+        rows, self.log_buffer = self.log_buffer, []
+        try:
+            await write_logs(rows)
+        except Exception:
+            sink_logger.exception("日志落库失败，丢弃 %d 条", len(rows))
+
+    async def _flush_logs_forever(self) -> None:
+        while True:
+            await asyncio.sleep(1)
+            await self._flush_logs()
+
+    # ---------- 子类使用的工具 ----------
+
+    def request(self, url: str, callback: Callable, **meta: Any) -> Request:
+        self.run.requests += 1
+        return Request(
+            url,
+            callback=callback,
+            errback=self.on_request_error,
+            dont_filter=True,
+            meta={"run_id": self.run.id, **meta},
+            headers=getattr(self, "request_headers", None),
+        )
+
+    def on_request_error(self, failure: Failure) -> None:
+        status = failure.value.response.status if failure.check(HttpError) else None
+        if status in (403, 429):
+            self.run.blocked = True
+        message = f"请求失败 {failure.request.url}：{status or repr(failure.value)}"
+        self.run.errors.append(message)
+        self.logger.error(message)
+
+    async def known_items(self, external_ids: list[str]) -> dict[str, tuple[str, str]]:
+        """已入库条目的 (title, summary)，用于判断是否新增、是否需要重抓详情、是否继续翻页"""
+        async with SessionLocal() as session:
+            rows = await session.execute(
+                select(NewsItem.external_id, NewsItem.title, NewsItem.summary)
+                .where(NewsItem.source_code == self.name, NewsItem.external_id.in_(external_ids))
+            )
+        return {external_id: (title, summary) for external_id, title, summary in rows}
+
+    def should_continue(self, external_ids: list[str], known: dict, oldest: datetime, page: int) -> bool:
+        """未越过回溯时间、未被限流，且本页有新条目（或本轮为全量扫描）时才翻下一页"""
+        return (
+            not self.run.blocked
+            and page < MAX_PAGES
+            and oldest > self.run.cutoff
+            and (self.run.full_scan or any(external_id not in known for external_id in external_ids))
+        )
+
+    def item(
+        self,
+        response: Response,
+        *,
+        external_id: str,
+        url: str,
+        title: str,
+        summary: str,
+        content_html: str,
+        published_at: datetime,
+        author: str | None = None,
+        tags: list[str] | None = None,
+        is_paywalled: bool = False,
+        source_updated_at: datetime | None = None,
+        extra: dict | None = None,
+    ) -> dict[str, Any]:
+        return {
+            "source_code": self.name,
+            "external_id": external_id,
+            "url": url,
+            "title": title,
+            "summary": summary,
+            "content_html": content_html,
+            "content_text": html_to_text(content_html),
+            "author": author or None,
+            "tags": tags or [],
+            "is_paywalled": is_paywalled,
+            "published_at": published_at,
+            "source_updated_at": source_updated_at,
+            "extra": extra or {},
+            "raw_response_id": response.meta["raw_response_id"],
+        }

backend/src/alphapulse/crawler/spiders/cls.py

@@ -0,0 +1,102 @@
+import hashlib
+import html
+import json
+import re
+from urllib.parse import urlencode
+
+from scrapy import Request
+from scrapy.http import Response
+
+from alphapulse.crawler.spiders.base import NewsSpider, ts
+
+
+def signed_url(path: str, params: dict[str, str]) -> str:
+    # 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
+    query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
+    query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
+    return f"https://www.cls.cn{path}?{urlencode(query)}"
+
+
+class ClsTelegraphSpider(NewsSpider):
+    """财联社·电报：列表即全文；翻页需 refresh_type=1"""
+
+    name = "cls_telegraph"
+    request_headers = {"Referer": "https://www.cls.cn/telegraph"}
+
+    def first_requests(self) -> list[Request]:
+        return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
+
+    async def parse_list(self, response: Response):
+        body = response.json()
+        if body["errno"] != 0:
+            raise ValueError(f"接口返回错误：{body}")
+        rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
+        # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            yield self.item(
+                response,
+                external_id=str(x["id"]),
+                url=f"https://www.cls.cn/detail/{x['id']}",
+                title=x["title"],
+                summary=x["brief"],
+                content_html=f"<p>{html.escape(x['content'])}</p>",
+                published_at=ts(x["ctime"]),
+                source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
+                author=x.get("author"),
+                tags=[s["subject_name"] for s in x.get("subjects") or []],
+                extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
+            )
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
+            last_time = str(min(x["ctime"] for x in rows))
+            params = {"category": "", "last_time": last_time, "refresh_type": "1", "rn": "20"}
+            yield self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page + 1)
+
+
+class ClsDepthSpider(NewsSpider):
+    """财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
+
+    name = "cls_depth"
+    request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
+
+    def first_requests(self) -> list[Request]:
+        # 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
+        return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
+
+    async def parse_list(self, response: Response):
+        body = response.json()
+        if body["errno"] != 0:
+            raise ValueError(f"接口返回错误：{body}")
+        rows = [x for x in body["data"] if not x["is_ad"]]
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            if known.get(str(x["id"])) != (x["title"], x["brief"]):
+                # 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
+                yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
+            params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
+            yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
+
+    async def parse_detail(self, response: Response):
+        match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
+        if match is None:
+            raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
+        a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
+        yield self.item(
+            response,
+            external_id=str(a["id"]),
+            url=f"https://www.cls.cn/detail/{a['id']}",
+            title=a["title"],
+            summary=response.meta["summary"],
+            content_html=a["content"],
+            published_at=ts(a["ctime"]),
+            author=(a.get("author") or {}).get("name"),
+            tags=[s["name"] for s in a.get("subject") or []],
+            # 付费内容只保存公开部分
+            is_paywalled=not a["isFree"],
+            extra={"level": a.get("level")},
+        )

backend/src/alphapulse/crawler/spiders/wscn.py

@@ -0,0 +1,81 @@
+from scrapy import Request
+from scrapy.http import Response
+
+from alphapulse.crawler.spiders.base import NewsSpider, ts
+
+API = "https://api-one-wscn.awtmt.com/apiv1/content"
+
+
+class WscnNewsSpider(NewsSpider):
+    """华尔街见闻·文章：列表只有摘要，新条目或标题/摘要变化时再拉详情"""
+
+    name = "wscn_news"
+
+    def first_requests(self) -> list[Request]:
+        return [self.list_request("", 1)]
+
+    def list_request(self, cursor: str, page: int) -> Request:
+        return self.request(f"{API}/information-flow?channel=global&accept=article&limit=20&cursor={cursor}", self.parse_list, page=page)
+
+    async def parse_list(self, response: Response):
+        data = response.json()["data"]
+        # 信息流里混有快讯，快讯由 wscn_live 负责
+        rows = [x["resource"] for x in data["items"] if x["resource_type"] == "article"]
+        ids = [str(r["id"]) for r in rows]
+        known = await self.known_items(ids)
+        for r in rows:
+            if known.get(str(r["id"])) != (r["title"], r["content_short"]):
+                yield self.request(f"{API}/articles/{r['id']}?extract=0", self.parse_detail, summary=r["content_short"])
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(r["display_time"] for r in rows)), page):
+            yield self.list_request(data["next_cursor"], page + 1)
+
+    async def parse_detail(self, response: Response):
+        a = response.json()["data"]
+        yield self.item(
+            response,
+            external_id=str(a["id"]),
+            url=f"https://wallstreetcn.com/articles/{a['id']}",
+            title=a["title"],
+            summary=response.meta["summary"],
+            content_html=a["content"],
+            published_at=ts(a["display_time"]),
+            author=(a.get("author") or {}).get("display_name"),
+            is_paywalled=bool(a.get("is_need_pay")),
+            extra={"source_name": a.get("source_name"), "is_priced": a.get("is_priced")},
+        )
+
+
+class WscnLiveSpider(NewsSpider):
+    """华尔街见闻·快讯：列表即全文"""
+
+    name = "wscn_live"
+
+    def first_requests(self) -> list[Request]:
+        return [self.list_request("", 1)]
+
+    def list_request(self, cursor: str, page: int) -> Request:
+        return self.request(f"{API}/lives?channel=global-channel&limit=50&cursor={cursor}", self.parse_list, page=page)
+
+    async def parse_list(self, response: Response):
+        data = response.json()["data"]
+        rows = data["items"]
+        # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
+        ids = [str(x["id"]) for x in rows]
+        known = await self.known_items(ids)
+        for x in rows:
+            yield self.item(
+                response,
+                external_id=str(x["id"]),
+                url=x["uri"],
+                title=x["title"],
+                summary="",
+                content_html=x["content"] + (x.get("content_more") or ""),
+                published_at=ts(x["display_time"]),
+                author=(x.get("author") or {}).get("display_name"),
+                tags=x.get("channels"),
+                extra={"score": x.get("score"), "symbols": x.get("symbols")},
+            )
+        page = response.meta["page"]
+        if rows and self.should_continue(ids, known, ts(min(x["display_time"] for x in rows)), page):
+            yield self.list_request(data["next_cursor"], page + 1)

backend/src/alphapulse/market/binance.py

@@ -0,0 +1,87 @@
+"""币安公开行情接口（无需 API Key）"""
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger("alphapulse.market.binance")
+
+REST = {
+    "spot": {"base": "https://api.binance.com", "klines": "/api/v3/klines", "exchange_info": "/api/v3/exchangeInfo",
+             "ticker": "/api/v3/ticker/24hr", "weight_limit": 6000, "kline_limit": 1000},
+    "um": {"base": "https://fapi.binance.com", "klines": "/fapi/v1/klines", "exchange_info": "/fapi/v1/exchangeInfo",
+           "ticker": "/fapi/v1/ticker/24hr", "weight_limit": 2400, "kline_limit": 1500},
+}
+WS = {"spot": "wss://stream.binance.com:9443/stream", "um": "wss://fstream.binance.com/stream"}
+
+# 权重用量超过上限的这个比例就等到下一分钟
+WEIGHT_RATIO = 0.5
+MAX_RETRIES = 3
+
+
+def decimals(step: str) -> int:
+    """"0.01000000" -> 2，"1.0" -> 0"""
+    return len(step.rstrip("0").split(".")[1]) if "." in step.rstrip("0") else 0
+
+
+class BinanceClient:
+    def __init__(self) -> None:
+        self.http = httpx.AsyncClient(timeout=20)
+
+    async def get(self, market: str, path_key: str, params: dict[str, Any] | None = None) -> Any:
+        conf = REST[market]
+        for attempt in range(1, MAX_RETRIES + 1):
+            try:
+                resp = await self.http.get(conf["base"] + conf[path_key], params=params)
+            except httpx.TransportError as e:
+                if attempt == MAX_RETRIES:
+                    raise
+                logger.warning("币安请求网络错误（第 %d 次）：%r", attempt, e)
+                await asyncio.sleep(2 * attempt)
+                continue
+            if resp.status_code in (418, 429):
+                # 418 表示 IP 已被临时封禁，必须按 Retry-After 等待
+                wait = int(resp.headers.get("Retry-After", "60"))
+                logger.warning("币安限流 %d，暂停 %ds", resp.status_code, wait)
+                await asyncio.sleep(wait)
+                continue
+            if resp.status_code >= 500 and attempt < MAX_RETRIES:
+                await asyncio.sleep(2 * attempt)
+                continue
+            resp.raise_for_status()
+            used = int(resp.headers.get("X-MBX-USED-WEIGHT-1M", "0"))
+            if used > conf["weight_limit"] * WEIGHT_RATIO:
+                wait = 61 - time.time() % 60
+                logger.info("%s 权重已用 %d，等待 %.0fs 到下一分钟", market, used, wait)
+                await asyncio.sleep(wait)
+            return resp.json()
+        raise RuntimeError(f"币安请求重试 {MAX_RETRIES} 次仍被限流：{path_key} {params}")
+
+    async def klines(self, market: str, symbol: str, interval: str, start_time: int, end_time: int | None = None) -> list[list]:
+        params: dict[str, Any] = {"symbol": symbol, "interval": interval, "startTime": start_time, "limit": REST[market]["kline_limit"]}
+        if end_time is not None:
+            params["endTime"] = end_time
+        return await self.get(market, "klines", params)
+
+    async def tradable_symbols(self, market: str) -> list[dict[str, Any]]:
+        """可交易的交易对；合约只保留永续（含 TRADIFI_PERPETUAL 等美股永续）"""
+        info = await self.get(market, "exchange_info")
+        result = []
+        for s in info["symbols"]:
+            if s["status"] != "TRADING" or (market == "um" and not s["contractType"].endswith("PERPETUAL")):
+                continue
+            filters = {f["filterType"]: f for f in s["filters"]}
+            result.append({
+                "symbol": s["symbol"],
+                "base": s["baseAsset"],
+                "quote": s["quoteAsset"],
+                "price_precision": decimals(filters["PRICE_FILTER"]["tickSize"]),
+                "volume_precision": decimals(filters["LOT_SIZE"]["stepSize"]),
+            })
+        return result
+
+    async def ticker(self, market: str, symbol: str) -> dict[str, Any]:
+        return await self.get(market, "ticker", {"symbol": symbol})

backend/src/alphapulse/market/channels.py

@@ -0,0 +1,9 @@
+from alphapulse.core.redis import KEY_PREFIX
+
+
+def kline_channel(market: str, symbol: str, interval: str) -> str:
+    return f"{KEY_PREFIX}pub:kline:{market}:{symbol}:{interval}"
+
+
+def ticker_channel(market: str, symbol: str) -> str:
+    return f"{KEY_PREFIX}pub:ticker:{market}:{symbol}"

backend/src/alphapulse/market/worker.py

@@ -0,0 +1,224 @@
+"""行情进程：全历史补齐、断线/定时追平、币安 WS 实时推送。
+
+用法：python -m alphapulse.market.worker
+"""
+
+import asyncio
+import json
+import logging
+import time
+
+import httpx
+import websockets
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import redis
+from alphapulse.market.binance import REST, WS, BinanceClient
+from alphapulse.market.channels import kline_channel, ticker_channel
+from alphapulse.models import Kline, KlineSyncState, MarketSymbol
+from alphapulse.models.market import INTERVALS, MARKETS
+
+logger = logging.getLogger("alphapulse.market.worker")
+
+# 每个补齐任务连续拉取的页数，之后重新挑选优先级最高的任务（新加的交易对能插队补大周期）
+PAGES_PER_TURN = 20
+CATCHUP_SECONDS = 3600
+SYMBOL_CHECK_SECONDS = 10
+ACTIVE_STATUSES = ("pending", "backfilling", "ready")
+
+client = BinanceClient()
+# WS（重新）连上后触发一次追平，补断线期间的缺口
+catchup_needed = asyncio.Event()
+
+
+def row_to_values(market: str, symbol: str, interval: str, row: list) -> dict:
+    return {
+        "market": market, "symbol": symbol, "interval": interval,
+        "open_time": row[0], "open": row[1], "high": row[2], "low": row[3], "close": row[4], "volume": row[5],
+        "close_time": row[6], "quote_volume": row[7], "trades": row[8], "taker_buy_base": row[9], "taker_buy_quote": row[10],
+    }
+
+
+async def upsert_klines(values: list[dict]) -> None:
+    stmt = pg_insert(Kline).values(values)
+    updatable = ("open", "high", "low", "close", "volume", "close_time", "quote_volume", "trades", "taker_buy_base", "taker_buy_quote")
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[Kline.market, Kline.symbol, Kline.interval, Kline.open_time],
+        set_={c: stmt.excluded[c] for c in updatable},
+    )
+    async with SessionLocal() as session:
+        await session.execute(stmt)
+        await session.commit()
+
+
+async def sync_page(market: str, symbol: str, interval: str) -> bool:
+    """从上次同步位置拉一页并落库，返回是否已追平到最新"""
+    async with SessionLocal() as session:
+        state = await session.get(KlineSyncState, (market, symbol, interval))
+        if state is None:
+            state = KlineSyncState(market=market, symbol=symbol, interval=interval)
+            session.add(state)
+        # 从最后一根（可能未收盘）重新拉起；首次从 0 开始，币安会返回上市后的第一根
+        rows = await client.klines(market, symbol, interval, state.latest_open_time or 0)
+        if rows:
+            await upsert_klines([row_to_values(market, symbol, interval, r) for r in rows])
+            state.earliest_open_time = state.earliest_open_time or rows[0][0]
+            state.latest_open_time = rows[-1][0]
+        caught_up = len(rows) < REST[market]["kline_limit"]
+        if caught_up:
+            state.backfill_done = True
+        await session.commit()
+    return caught_up
+
+
+async def next_backfill_task(market: str) -> tuple[MarketSymbol, str] | None:
+    async with SessionLocal() as session:
+        symbols = (await session.scalars(
+            select(MarketSymbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(("pending", "backfilling")))
+            .order_by(MarketSymbol.added_at)
+        )).all()
+        done = set((await session.execute(
+            select(KlineSyncState.symbol, KlineSyncState.interval)
+            .where(KlineSyncState.market == market, KlineSyncState.backfill_done)
+        )).all())
+    unfinished = []
+    for s in symbols:
+        if all((s.symbol, interval) in done for interval in INTERVALS):
+            await set_symbol_status(s.id, "ready")
+            logger.info("%s %s 全部周期补齐完成", market, s.symbol)
+        else:
+            unfinished.append(s)
+    for interval in INTERVALS:
+        for s in unfinished:
+            if (s.symbol, interval) not in done:
+                return s, interval
+    return None
+
+
+async def set_symbol_status(symbol_id: int, status: str, error: str | None = None) -> None:
+    async with SessionLocal() as session:
+        s = await session.get(MarketSymbol, symbol_id)
+        s.status = status
+        s.last_error = error
+        await session.commit()
+
+
+async def backfill_loop(market: str) -> None:
+    while True:
+        task = await next_backfill_task(market)
+        if task is None:
+            await asyncio.sleep(5)
+            continue
+        s, interval = task
+        if s.status == "pending":
+            await set_symbol_status(s.id, "backfilling")
+            logger.info("%s %s 开始补齐", market, s.symbol)
+        try:
+            for _ in range(PAGES_PER_TURN):
+                if await sync_page(market, s.symbol, interval):
+                    logger.info("%s %s %s 补齐完成", market, s.symbol, interval)
+                    break
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 400:
+                # 交易对不存在或已下架：标记错误，不再重试
+                await set_symbol_status(s.id, "error", e.response.text[:500])
+                logger.error("%s %s 补齐失败：%s", market, s.symbol, e.response.text[:200])
+            else:
+                logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
+                await asyncio.sleep(10)
+        except Exception:
+            logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
+            await asyncio.sleep(10)
+
+
+async def catchup_loop() -> None:
+    """已补齐的周期从上次 REST 同步位置追平到最新：每小时一次，WS 重连后立即一次"""
+    while True:
+        try:
+            await asyncio.wait_for(catchup_needed.wait(), timeout=CATCHUP_SECONDS)
+        except TimeoutError:
+            pass
+        catchup_needed.clear()
+        async with SessionLocal() as session:
+            states = (await session.execute(
+                select(KlineSyncState.market, KlineSyncState.symbol, KlineSyncState.interval)
+                .join(MarketSymbol, (MarketSymbol.market == KlineSyncState.market) & (MarketSymbol.symbol == KlineSyncState.symbol))
+                .where(KlineSyncState.backfill_done, MarketSymbol.status.in_(ACTIVE_STATUSES))
+            )).all()
+        started = time.monotonic()
+        for market, symbol, interval in states:
+            try:
+                while not await sync_page(market, symbol, interval):
+                    pass
+            except Exception:
+                logger.exception("%s %s %s 追平出错", market, symbol, interval)
+        logger.info("追平完成：%d 个周期，用时 %.1fs", len(states), time.monotonic() - started)
+
+
+async def active_symbols(market: str) -> list[str]:
+    async with SessionLocal() as session:
+        return list((await session.scalars(
+            select(MarketSymbol.symbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(ACTIVE_STATUSES))
+            .order_by(MarketSymbol.symbol)
+        )).all())
+
+
+async def handle_message(market: str, raw: str | bytes) -> None:
+    data = json.loads(raw)["data"]
+    symbol = data["s"]
+    if data["e"] == "kline":
+        k = data["k"]
+        payload = {"t": k["t"], "o": k["o"], "h": k["h"], "l": k["l"], "c": k["c"], "v": k["v"], "q": k["q"], "x": k["x"]}
+        await redis.publish(kline_channel(market, symbol, k["i"]), json.dumps(payload))
+        # 收盘 K 线入库；1s 只推送不落库
+        if k["x"] and k["i"] != "1s":
+            await upsert_klines([{
+                "market": market, "symbol": symbol, "interval": k["i"],
+                "open_time": k["t"], "open": k["o"], "high": k["h"], "low": k["l"], "close": k["c"], "volume": k["v"],
+                "close_time": k["T"], "quote_volume": k["q"], "trades": k["n"], "taker_buy_base": k["V"], "taker_buy_quote": k["Q"],
+            }])
+    elif data["e"] == "24hrMiniTicker":
+        payload = {"c": data["c"], "o": data["o"], "h": data["h"], "l": data["l"], "v": data["v"], "q": data["q"], "E": data["E"]}
+        await redis.publish(ticker_channel(market, symbol), json.dumps(payload))
+
+
+async def stream_loop(market: str) -> None:
+    while True:
+        symbols = await active_symbols(market)
+        if not symbols:
+            await asyncio.sleep(SYMBOL_CHECK_SECONDS)
+            continue
+        intervals = INTERVALS + (["1s"] if market == "spot" else [])
+        streams = [f"{s.lower()}@kline_{i}" for s in symbols for i in intervals] + [f"{s.lower()}@miniTicker" for s in symbols]
+        try:
+            async with websockets.connect(f"{WS[market]}?streams={'/'.join(streams)}", max_size=2**22) as ws:
+                logger.info("%s WS 已连接：%d 个交易对，%d 个流", market, len(symbols), len(streams))
+                catchup_needed.set()
+                next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
+                async for raw in ws:
+                    await handle_message(market, raw)
+                    if time.monotonic() >= next_check:
+                        next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
+                        if await active_symbols(market) != symbols:
+                            logger.info("%s 交易对列表变化，重新订阅", market)
+                            break
+        except Exception:
+            logger.exception("%s WS 断开，5s 后重连", market)
+            await asyncio.sleep(5)
+
+
+async def main() -> None:
+    await asyncio.gather(
+        *(backfill_loop(m) for m in MARKETS),
+        *(stream_loop(m) for m in MARKETS),
+        catchup_loop(),
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    # httpx 每个请求一行 INFO，补齐时刷屏
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    asyncio.run(main())

backend/src/alphapulse/models/__init__.py

@@ -0,0 +1,17 @@
+from alphapulse.core.db import Base
+from alphapulse.models.market import Kline, KlineSyncState, MarketSymbol
+from alphapulse.models.news import CrawlLog, CrawlRun, NewsItem, NewsRevision, RawBody, RawResponse, Source
+
+__all__ = [
+    "Base",
+    "CrawlLog",
+    "CrawlRun",
+    "Kline",
+    "KlineSyncState",
+    "MarketSymbol",
+    "NewsItem",
+    "NewsRevision",
+    "RawBody",
+    "RawResponse",
+    "Source",
+]

backend/src/alphapulse/models/market.py

@@ -0,0 +1,72 @@
+from datetime import datetime
+from decimal import Decimal
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Integer, Numeric, PrimaryKeyConstraint, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from alphapulse.core.db import Base
+
+# 落库的周期，顺序即补齐优先级（大周期先补，图表最先可看）；1s 只做实时推送不落库
+INTERVALS = ["1M", "1w", "3d", "1d", "12h", "8h", "6h", "4h", "2h", "1h", "30m", "15m", "5m", "3m", "1m"]
+MARKETS = ("spot", "um")
+
+
+class MarketSymbol(Base):
+    __tablename__ = "market_symbols"
+    __table_args__ = (UniqueConstraint("market", "symbol"),)
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # spot 现货 / um U 本位永续
+    market: Mapped[str] = mapped_column(String(8))
+    symbol: Mapped[str] = mapped_column(String(32))
+    base: Mapped[str] = mapped_column(String(16))
+    quote: Mapped[str] = mapped_column(String(16))
+    # 由币安 tickSize / stepSize 推出的小数位，用于图表显示
+    price_precision: Mapped[int] = mapped_column(Integer)
+    volume_precision: Mapped[int] = mapped_column(Integer)
+    # pending / backfilling / ready / error / delisted
+    status: Mapped[str] = mapped_column(String(16), default="pending")
+    is_default: Mapped[bool] = mapped_column(Boolean, default=False)
+    added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    last_error: Mapped[str | None] = mapped_column(Text)
+
+
+class Kline(Base):
+    """时间字段沿用币安的毫秒时间戳；表按 interval 做 LIST 分区（分区在迁移中创建）"""
+
+    __tablename__ = "klines"
+    __table_args__ = (
+        PrimaryKeyConstraint("market", "symbol", "interval", "open_time"),
+        {"postgresql_partition_by": "LIST (interval)"},
+    )
+
+    market: Mapped[str] = mapped_column(String(8))
+    symbol: Mapped[str] = mapped_column(String(32))
+    interval: Mapped[str] = mapped_column(String(4))
+    open_time: Mapped[int] = mapped_column(BigInteger)
+    open: Mapped[Decimal] = mapped_column(Numeric)
+    high: Mapped[Decimal] = mapped_column(Numeric)
+    low: Mapped[Decimal] = mapped_column(Numeric)
+    close: Mapped[Decimal] = mapped_column(Numeric)
+    volume: Mapped[Decimal] = mapped_column(Numeric)
+    close_time: Mapped[int] = mapped_column(BigInteger)
+    quote_volume: Mapped[Decimal] = mapped_column(Numeric)
+    trades: Mapped[int] = mapped_column(BigInteger)
+    taker_buy_base: Mapped[Decimal] = mapped_column(Numeric)
+    taker_buy_quote: Mapped[Decimal] = mapped_column(Numeric)
+
+
+class KlineSyncState(Base):
+    """REST 同步进度：实时 WS 写入不更新这里，所以从 latest_open_time 往后追平即可补上断线缺口"""
+
+    __tablename__ = "kline_sync_state"
+    __table_args__ = (PrimaryKeyConstraint("market", "symbol", "interval"),)
+
+    market: Mapped[str] = mapped_column(String(8))
+    symbol: Mapped[str] = mapped_column(String(32))
+    interval: Mapped[str] = mapped_column(String(4))
+    earliest_open_time: Mapped[int | None] = mapped_column(BigInteger)
+    latest_open_time: Mapped[int | None] = mapped_column(BigInteger)
+    # 首次全历史补齐是否完成
+    backfill_done: Mapped[bool] = mapped_column(Boolean, default=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())

backend/src/alphapulse/models/news.py

@@ -0,0 +1,122 @@
+import uuid
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from alphapulse.core.db import Base
+
+
+class Source(Base):
+    __tablename__ = "sources"
+
+    code: Mapped[str] = mapped_column(String(32), primary_key=True)
+    name: Mapped[str] = mapped_column(String(64))
+    list_url: Mapped[str] = mapped_column(String(255))
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True)
+    interval_seconds: Mapped[int] = mapped_column(Integer)
+    backfill_hours: Mapped[int] = mapped_column(Integer, default=72)
+    consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
+    last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_error: Mapped[str | None] = mapped_column(Text)
+    paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class NewsItem(Base):
+    __tablename__ = "news_items"
+    __table_args__ = (
+        UniqueConstraint("source_code", "external_id"),
+        Index("ix_news_items_published_at", "published_at"),
+        Index("ix_news_items_source_published_at", "source_code", "published_at"),
+        Index("ix_news_items_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
+        Index("ix_news_items_content_trgm", "content_text", postgresql_using="gin", postgresql_ops={"content_text": "gin_trgm_ops"}),
+    )
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
+    external_id: Mapped[str] = mapped_column(String(64))
+    url: Mapped[str] = mapped_column(String(512))
+    title: Mapped[str] = mapped_column(Text, default="")
+    summary: Mapped[str] = mapped_column(Text, default="")
+    content_html: Mapped[str] = mapped_column(Text, default="")
+    content_text: Mapped[str] = mapped_column(Text, default="")
+    author: Mapped[str | None] = mapped_column(String(128))
+    tags: Mapped[list] = mapped_column(JSONB, default=list)
+    is_paywalled: Mapped[bool] = mapped_column(Boolean, default=False)
+    published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    content_hash: Mapped[str] = mapped_column(String(64))
+    revision_no: Mapped[int] = mapped_column(Integer, default=1)
+    extra: Mapped[dict] = mapped_column(JSONB, default=dict)
+
+
+class NewsRevision(Base):
+    __tablename__ = "news_revisions"
+    __table_args__ = (UniqueConstraint("news_id", "revision_no"),)
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    news_id: Mapped[int] = mapped_column(ForeignKey("news_items.id"))
+    revision_no: Mapped[int] = mapped_column(Integer)
+    title: Mapped[str] = mapped_column(Text)
+    summary: Mapped[str] = mapped_column(Text)
+    content_html: Mapped[str] = mapped_column(Text)
+    content_hash: Mapped[str] = mapped_column(String(64))
+    captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
+
+
+class RawBody(Base):
+    __tablename__ = "raw_bodies"
+
+    sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
+    # gzip 压缩后的原始响应体；同内容只存一份
+    body: Mapped[bytes] = mapped_column(LargeBinary)
+
+
+class RawResponse(Base):
+    __tablename__ = "raw_responses"
+    __table_args__ = (Index("ix_raw_responses_source_fetched_at", "source_code", "fetched_at"),)
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    run_id: Mapped[uuid.UUID | None] = mapped_column()
+    source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
+    url: Mapped[str] = mapped_column(Text)
+    method: Mapped[str] = mapped_column(String(8))
+    status: Mapped[int] = mapped_column(Integer)
+    headers: Mapped[dict] = mapped_column(JSONB)
+    body_sha256: Mapped[str] = mapped_column(ForeignKey("raw_bodies.sha256"))
+    fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class CrawlRun(Base):
+    __tablename__ = "crawl_runs"
+    __table_args__ = (Index("ix_crawl_runs_source_started_at", "source_code", "started_at"),)
+
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
+    source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
+    # schedule / startup / restart
+    trigger: Mapped[str] = mapped_column(String(16))
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    # running / success / failed / interrupted
+    status: Mapped[str] = mapped_column(String(16))
+    requests: Mapped[int] = mapped_column(Integer, default=0)
+    items_seen: Mapped[int] = mapped_column(Integer, default=0)
+    items_new: Mapped[int] = mapped_column(Integer, default=0)
+    items_revised: Mapped[int] = mapped_column(Integer, default=0)
+    error: Mapped[str | None] = mapped_column(Text)
+
+
+class CrawlLog(Base):
+    __tablename__ = "crawl_logs"
+    __table_args__ = (Index("ix_crawl_logs_source_id", "source_code", "id"),)
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    run_id: Mapped[uuid.UUID | None] = mapped_column()
+    source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
+    ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    level: Mapped[str] = mapped_column(String(8))
+    message: Mapped[str] = mapped_column(Text)