Initial AlphaPulse implementation: news crawler and Binance klines

- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth,
  news/crawler/market APIs with SSE and WebSocket streaming
- Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth,
  cls_telegraph spiders, idempotent upserts with revisions, raw response
  archiving, per-source subprocess supervision, restart and backoff
- Market worker: full-history Binance spot/USD-M kline backfill,
  realtime WS stream, hourly and on-reconnect catch-up
- Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed,
  crawler console with live logs, Binance-style chart page
- Deploy: docker compose, nginx site, migrate and daily backup scripts

backend/src/alphapulse/api/auth.py

@@ -0,0 +1,58 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from pydantic import BaseModel
+
+from alphapulse.core.auth import COOKIE_NAME, create_token, require_admin, verify_password
+from alphapulse.core.config import get_settings
+from alphapulse.core.redis import KEY_PREFIX, redis
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+MAX_FAILURES = 5
+LOCK_SECONDS = 15 * 60
+
+
+class LoginIn(BaseModel):
+    username: str
+    password: str
+
+
+class MeOut(BaseModel):
+    username: str
+
+
+@router.post("/login")
+async def login(body: LoginIn, request: Request, response: Response) -> MeOut:
+    settings = get_settings()
+    if not settings.admin_password_hash:
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "管理员密码未配置")
+    fail_key = f"{KEY_PREFIX}login:fail:{request.client.host}"
+    if int(await redis.get(fail_key) or 0) >= MAX_FAILURES:
+        raise HTTPException(status.HTTP_429_TOO_MANY_REQUESTS, "失败次数过多，请 15 分钟后再试")
+
+    if body.username != settings.admin_username or not verify_password(body.password, settings.admin_password_hash):
+        async with redis.pipeline() as pipe:
+            await pipe.incr(fail_key).expire(fail_key, LOCK_SECONDS).execute()
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "用户名或密码错误")
+
+    await redis.delete(fail_key)
+    response.set_cookie(
+        COOKIE_NAME,
+        create_token(body.username),
+        max_age=settings.jwt_expire_days * 86400,
+        httponly=True,
+        secure=True,
+        samesite="strict",
+    )
+    return MeOut(username=body.username)
+
+
+@router.post("/logout")
+async def logout(response: Response) -> None:
+    response.delete_cookie(COOKIE_NAME, httponly=True, secure=True, samesite="strict")
+
+
+@router.get("/me")
+async def me(username: Annotated[str, Depends(require_admin)]) -> MeOut:
+    return MeOut(username=username)

backend/src/alphapulse/api/crawler.py

@@ -0,0 +1,180 @@
+import json
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime
+from typing import Annotated, Literal
+from zoneinfo import ZoneInfo
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.sse import EventSourceResponse, ServerSentEvent
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import CONTROL_CHANNEL, LOG_CHANNEL, STATUS_KEY
+from alphapulse.models import CrawlLog, CrawlRun, NewsItem, Source
+
+router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
+
+# “今日”按北京时间计算
+LOCAL_TZ = ZoneInfo("Asia/Shanghai")
+
+
+class ProcessStatus(BaseModel):
+    state: Literal["running", "stopped", "restarting"]
+    pid: int | None
+    started_at: datetime | None
+    next_run_at: datetime
+
+
+class SourceOut(BaseModel):
+    code: str
+    name: str
+    list_url: str
+    enabled: bool
+    interval_seconds: int
+    backfill_hours: int
+    consecutive_failures: int
+    last_success_at: datetime | None
+    last_error: str | None
+    paused_until: datetime | None
+    # 管理进程未运行时为 None
+    process: ProcessStatus | None
+    today_new: int
+    today_revised: int
+    total_items: int
+
+
+class SourcesOut(BaseModel):
+    manager_alive: bool
+    db_size_bytes: int
+    sources: list[SourceOut]
+
+
+class SourcePatch(BaseModel):
+    enabled: bool | None = None
+    interval_seconds: Annotated[int | None, Field(ge=5, le=86400)] = None
+
+
+class RunOut(BaseModel):
+    id: uuid.UUID
+    source_code: str
+    trigger: str
+    started_at: datetime
+    finished_at: datetime | None
+    status: str
+    requests: int
+    items_seen: int
+    items_new: int
+    items_revised: int
+    error: str | None
+
+
+class LogOut(BaseModel):
+    id: int
+    source_code: str
+    run_id: uuid.UUID | None
+    ts: datetime
+    level: str
+    message: str
+
+
+class RestartIn(BaseModel):
+    # 来源 code 或 "all"
+    source: str
+
+
+@router.get("/sources")
+async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -> SourcesOut:
+    sources = (await session.scalars(select(Source).order_by(Source.code))).all()
+    today = datetime.now(LOCAL_TZ).replace(hour=0, minute=0, second=0, microsecond=0)
+    today_counts = {
+        code: (new or 0, revised or 0)
+        for code, new, revised in await session.execute(
+            select(CrawlRun.source_code, func.sum(CrawlRun.items_new), func.sum(CrawlRun.items_revised))
+            .where(CrawlRun.started_at >= today)
+            .group_by(CrawlRun.source_code)
+        )
+    }
+    totals = dict((await session.execute(
+        select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
+    )).all())
+    statuses = await redis.hgetall(STATUS_KEY)
+    db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
+    return SourcesOut(
+        manager_alive=bool(statuses),
+        db_size_bytes=db_size,
+        sources=[
+            SourceOut.model_validate({
+                **s.__dict__,
+                "process": json.loads(statuses[s.code]) if s.code in statuses else None,
+                "today_new": today_counts.get(s.code, (0, 0))[0],
+                "today_revised": today_counts.get(s.code, (0, 0))[1],
+                "total_items": totals.get(s.code, 0),
+            })
+            for s in sources
+        ],
+    )
+
+
+@router.patch("/sources/{code}")
+async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    source = await session.get(Source, code)
+    if source is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    for field, value in body.model_dump(exclude_none=True).items():
+        setattr(source, field, value)
+    await session.commit()
+
+
+@router.get("/runs")
+async def list_runs(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: str | None = None,
+    limit: Annotated[int, Query(ge=1, le=200)] = 50,
+) -> list[RunOut]:
+    stmt = select(CrawlRun).order_by(CrawlRun.started_at.desc()).limit(limit)
+    if source:
+        stmt = stmt.where(CrawlRun.source_code == source)
+    return [RunOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
+
+
+@router.get("/logs")
+async def list_logs(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: str | None = None,
+    before_id: int | None = None,
+    limit: Annotated[int, Query(ge=1, le=1000)] = 200,
+) -> list[LogOut]:
+    """按时间正序返回 before_id 之前最近的 limit 条"""
+    stmt = select(CrawlLog).order_by(CrawlLog.id.desc()).limit(limit)
+    if source:
+        stmt = stmt.where(CrawlLog.source_code == source)
+    if before_id:
+        stmt = stmt.where(CrawlLog.id < before_id)
+    rows = [LogOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
+    return rows[::-1]
+
+
+@router.get("/logs/stream", response_class=EventSourceResponse)
+async def stream_logs(source: str | None = None) -> AsyncIterator[ServerSentEvent]:
+    async with redis.pubsub() as pubsub:
+        await pubsub.subscribe(LOG_CHANNEL)
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            if source and json.loads(message["data"])["source_code"] != source:
+                continue
+            yield ServerSentEvent(raw_data=message["data"], event="log")
+
+
+@router.post("/restart")
+async def restart(body: RestartIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    if body.source != "all" and await session.get(Source, body.source) is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
+    receivers = await redis.publish(CONTROL_CHANNEL, json.dumps({"action": "restart", "source": body.source}))
+    if receivers == 0:
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "采集管理进程未运行")

backend/src/alphapulse/api/main.py

@@ -0,0 +1,29 @@
+from fastapi import Depends, FastAPI
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import HTMLResponse
+
+from alphapulse.api import auth, crawler, market, news
+from alphapulse.core.auth import require_admin
+
+# 关闭默认文档路由，改为登录后才能访问
+app = FastAPI(title="AlphaPulse", docs_url=None, redoc_url=None, openapi_url=None)
+app.include_router(auth.router)
+app.include_router(news.router)
+app.include_router(crawler.router)
+app.include_router(market.router)
+app.include_router(market.ws_router)
+
+
+@app.get("/api/health", include_in_schema=False)
+async def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/api/openapi.json", include_in_schema=False, dependencies=[Depends(require_admin)])
+async def openapi() -> dict:
+    return app.openapi()
+
+
+@app.get("/api/docs", include_in_schema=False, dependencies=[Depends(require_admin)])
+async def docs() -> HTMLResponse:
+    return get_swagger_ui_html(openapi_url="/api/openapi.json", title="AlphaPulse API")

backend/src/alphapulse/api/market.py

@@ -0,0 +1,202 @@
+import asyncio
+import json
+import time
+from datetime import datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect, status
+from pydantic import BaseModel
+from sqlalchemy import delete, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin, websocket_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import KEY_PREFIX, redis
+from alphapulse.market.binance import BinanceClient
+from alphapulse.market.channels import kline_channel, ticker_channel
+from alphapulse.models import Kline, KlineSyncState, MarketSymbol
+from alphapulse.models.market import INTERVALS
+
+router = APIRouter(prefix="/api/market", tags=["market"], dependencies=[Depends(require_admin)])
+# WebSocket 握手不能用基于 Request 的依赖，单独鉴权
+ws_router = APIRouter(prefix="/api/market", tags=["market"])
+
+Market = Literal["spot", "um"]
+SYMBOLS_CACHE_SECONDS = 3600
+MAX_KLINES = 1500
+
+client = BinanceClient()
+
+
+class IntervalProgress(BaseModel):
+    done: bool
+    percent: float
+
+
+class SymbolOut(BaseModel):
+    id: int
+    market: str
+    symbol: str
+    base: str
+    quote: str
+    price_precision: int
+    volume_precision: int
+    status: str
+    is_default: bool
+    added_at: datetime
+    last_error: str | None
+    progress: dict[str, IntervalProgress]
+
+
+class SymbolIn(BaseModel):
+    market: Market
+    symbol: str
+
+
+class TradableSymbol(BaseModel):
+    symbol: str
+    base: str
+    quote: str
+    price_precision: int
+    volume_precision: int
+
+
+async def tradable_symbols(market: str) -> list[dict]:
+    key = f"{KEY_PREFIX}market:symbols:{market}"
+    if cached := await redis.get(key):
+        return json.loads(cached)
+    symbols = await client.tradable_symbols(market)
+    await redis.set(key, json.dumps(symbols), ex=SYMBOLS_CACHE_SECONDS)
+    return symbols
+
+
+@router.get("/symbols")
+async def list_symbols(session: Annotated[AsyncSession, Depends(get_session)]) -> list[SymbolOut]:
+    symbols = (await session.scalars(select(MarketSymbol).order_by(MarketSymbol.is_default.desc(), MarketSymbol.added_at))).all()
+    states = {(s.market, s.symbol, s.interval): s for s in await session.scalars(select(KlineSyncState))}
+    now = int(time.time() * 1000)
+
+    def progress(s: MarketSymbol, interval: str) -> IntervalProgress:
+        state = states.get((s.market, s.symbol, interval))
+        if state is None or state.earliest_open_time is None:
+            return IntervalProgress(done=False, percent=0)
+        if state.backfill_done:
+            return IntervalProgress(done=True, percent=100)
+        span = max(now - state.earliest_open_time, 1)
+        return IntervalProgress(done=False, percent=round((state.latest_open_time - state.earliest_open_time) / span * 100, 1))
+
+    return [
+        SymbolOut.model_validate({**s.__dict__, "progress": {i: progress(s, i) for i in INTERVALS}})
+        for s in symbols
+    ]
+
+
+@router.post("/symbols", status_code=status.HTTP_201_CREATED)
+async def add_symbol(body: SymbolIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    symbol = body.symbol.strip().upper()
+    match = next((s for s in await tradable_symbols(body.market) if s["symbol"] == symbol), None)
+    if match is None:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"{body.market} 市场没有可交易的 {symbol}")
+    session.add(MarketSymbol(market=body.market, **match))
+    try:
+        await session.commit()
+    except IntegrityError:
+        raise HTTPException(status.HTTP_409_CONFLICT, f"{symbol} 已存在") from None
+
+
+@router.delete("/symbols/{symbol_id}")
+async def delete_symbol(symbol_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
+    """删除交易对及其 K 线（K 线可随时从币安重新补齐）"""
+    s = await session.get(MarketSymbol, symbol_id)
+    if s is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "交易对不存在")
+    await session.execute(delete(Kline).where(Kline.market == s.market, Kline.symbol == s.symbol))
+    await session.execute(delete(KlineSyncState).where(KlineSyncState.market == s.market, KlineSyncState.symbol == s.symbol))
+    await session.delete(s)
+    await session.commit()
+
+
+@router.get("/exchange-info")
+async def exchange_info(market: Market) -> list[TradableSymbol]:
+    return [TradableSymbol.model_validate(s) for s in await tradable_symbols(market)]
+
+
+@router.get("/klines")
+async def klines(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    market: Market,
+    symbol: str,
+    interval: str,
+    end: int | None = None,
+    limit: Annotated[int, Query(ge=1, le=MAX_KLINES)] = 500,
+) -> list[list[float]]:
+    """返回 open_time < end 的最近 limit 根，按时间正序：[open_time, open, high, low, close, volume, quote_volume]"""
+    if interval == "1s":
+        # 1s 不落库，直接取币安最近数据
+        if market != "spot":
+            raise HTTPException(status.HTTP_400_BAD_REQUEST, "合约没有 1s 周期")
+        params = {"symbol": symbol, "interval": "1s", "limit": limit, **({"endTime": end - 1} if end else {})}
+        rows = await client.get("spot", "klines", params)
+        return [[r[0], float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5]), float(r[7])] for r in rows]
+    if interval not in INTERVALS:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"不支持的周期：{interval}")
+    stmt = (
+        select(Kline.open_time, Kline.open, Kline.high, Kline.low, Kline.close, Kline.volume, Kline.quote_volume)
+        .where(Kline.market == market, Kline.symbol == symbol, Kline.interval == interval)
+        .order_by(Kline.open_time.desc())
+        .limit(limit)
+    )
+    if end:
+        stmt = stmt.where(Kline.open_time < end)
+    rows = (await session.execute(stmt)).all()
+    return [[r[0], *(float(v) for v in r[1:])] for r in reversed(rows)]
+
+
+@router.get("/ticker")
+async def ticker(market: Market, symbol: str) -> dict:
+    return await client.ticker(market, symbol)
+
+
+@ws_router.websocket("/ws")
+async def market_ws(websocket: WebSocket) -> None:
+    """客户端发送 {"market","symbol","interval"} 切换订阅；服务端推送 {"type":"kline"|"ticker", market, symbol, interval?, data}"""
+    if websocket_admin(websocket) is None:
+        await websocket.close(code=4401)
+        return
+    await websocket.accept()
+    pubsub = redis.pubsub()
+    subscribed = asyncio.Event()
+
+    async def forward() -> None:
+        await subscribed.wait()
+        while True:
+            message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
+            if message is None:
+                continue
+            # 频道格式：ap:pub:kline:<market>:<symbol>:<interval> 或 ap:pub:ticker:<market>:<symbol>
+            parts = message["channel"].split(":")
+            await websocket.send_text(json.dumps({
+                "type": parts[2],
+                "market": parts[3],
+                "symbol": parts[4],
+                "interval": parts[5] if parts[2] == "kline" else None,
+                "data": json.loads(message["data"]),
+            }))
+
+    task = asyncio.create_task(forward())
+    try:
+        while True:
+            sub = await websocket.receive_json()
+            if subscribed.is_set():
+                await pubsub.unsubscribe()
+            await pubsub.subscribe(
+                kline_channel(sub["market"], sub["symbol"], sub["interval"]),
+                ticker_channel(sub["market"], sub["symbol"]),
+            )
+            subscribed.set()
+    except WebSocketDisconnect:
+        pass
+    finally:
+        task.cancel()
+        await pubsub.aclose()

backend/src/alphapulse/api/news.py

@@ -0,0 +1,110 @@
+from collections.abc import AsyncIterator
+from datetime import datetime
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.sse import EventSourceResponse, ServerSentEvent
+from pydantic import BaseModel
+from sqlalchemy import func, or_, select, tuple_
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from alphapulse.core.auth import require_admin
+from alphapulse.core.db import get_session
+from alphapulse.core.redis import redis
+from alphapulse.crawler.events import NEWS_CHANNEL
+from alphapulse.models import NewsItem, NewsRevision
+
+router = APIRouter(prefix="/api/news", tags=["news"], dependencies=[Depends(require_admin)])
+
+EXCERPT_LENGTH = 300
+
+
+class NewsBrief(BaseModel):
+    id: int
+    source_code: str
+    url: str
+    title: str
+    summary: str
+    excerpt: str
+    author: str | None
+    tags: list
+    is_paywalled: bool
+    published_at: datetime
+    first_seen_at: datetime
+    revision_no: int
+
+
+class NewsPage(BaseModel):
+    items: list[NewsBrief]
+    # 形如 "<published_at ISO>|<id>"，原样传回即可取下一页
+    next_cursor: str | None
+
+
+class NewsDetail(NewsBrief):
+    content_html: str
+    source_updated_at: datetime | None
+    last_seen_at: datetime
+    extra: dict
+
+
+class Revision(BaseModel):
+    revision_no: int
+    title: str
+    summary: str
+    content_html: str
+    captured_at: datetime
+
+
+@router.get("")
+async def list_news(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    source: Annotated[list[str] | None, Query()] = None,
+    q: str | None = None,
+    start: datetime | None = None,
+    end: datetime | None = None,
+    cursor: str | None = None,
+    limit: Annotated[int, Query(ge=1, le=100)] = 30,
+) -> NewsPage:
+    stmt = select(NewsItem, func.left(NewsItem.content_text, EXCERPT_LENGTH).label("excerpt"))
+    if source:
+        stmt = stmt.where(NewsItem.source_code.in_(source))
+    if q:
+        pattern = f"%{q}%"
+        stmt = stmt.where(or_(NewsItem.title.ilike(pattern), NewsItem.content_text.ilike(pattern)))
+    if start:
+        stmt = stmt.where(NewsItem.published_at >= start)
+    if end:
+        stmt = stmt.where(NewsItem.published_at < end)
+    if cursor:
+        published_at, news_id = cursor.rsplit("|", 1)
+        stmt = stmt.where(tuple_(NewsItem.published_at, NewsItem.id) < (datetime.fromisoformat(published_at), int(news_id)))
+    rows = (await session.execute(stmt.order_by(NewsItem.published_at.desc(), NewsItem.id.desc()).limit(limit))).all()
+
+    items = [NewsBrief.model_validate({**news.__dict__, "excerpt": excerpt}) for news, excerpt in rows]
+    next_cursor = f"{items[-1].published_at.isoformat()}|{items[-1].id}" if len(items) == limit else None
+    return NewsPage(items=items, next_cursor=next_cursor)
+
+
+@router.get("/stream", response_class=EventSourceResponse)
+async def stream_news() -> AsyncIterator[ServerSentEvent]:
+    async with redis.pubsub() as pubsub:
+        await pubsub.subscribe(NEWS_CHANNEL)
+        async for message in pubsub.listen():
+            if message["type"] == "message":
+                yield ServerSentEvent(raw_data=message["data"], event="news")
+
+
+@router.get("/{news_id}")
+async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> NewsDetail:
+    news = await session.get(NewsItem, news_id)
+    if news is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
+    return NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]})
+
+
+@router.get("/{news_id}/revisions")
+async def list_revisions(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> list[Revision]:
+    rows = await session.scalars(
+        select(NewsRevision).where(NewsRevision.news_id == news_id).order_by(NewsRevision.revision_no.desc())
+    )
+    return [Revision.model_validate(r, from_attributes=True) for r in rows]