Initial AlphaPulse implementation: news crawler and Binance klines

- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth,
  news/crawler/market APIs with SSE and WebSocket streaming
- Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth,
  cls_telegraph spiders, idempotent upserts with revisions, raw response
  archiving, per-source subprocess supervision, restart and backoff
- Market worker: full-history Binance spot/USD-M kline backfill,
  realtime WS stream, hourly and on-reconnect catch-up
- Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed,
  crawler console with live logs, Binance-style chart page
- Deploy: docker compose, nginx site, migrate and daily backup scripts

backend/src/alphapulse/market/binance.py

@@ -0,0 +1,87 @@
+"""币安公开行情接口（无需 API Key）"""
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger("alphapulse.market.binance")
+
+REST = {
+    "spot": {"base": "https://api.binance.com", "klines": "/api/v3/klines", "exchange_info": "/api/v3/exchangeInfo",
+             "ticker": "/api/v3/ticker/24hr", "weight_limit": 6000, "kline_limit": 1000},
+    "um": {"base": "https://fapi.binance.com", "klines": "/fapi/v1/klines", "exchange_info": "/fapi/v1/exchangeInfo",
+           "ticker": "/fapi/v1/ticker/24hr", "weight_limit": 2400, "kline_limit": 1500},
+}
+WS = {"spot": "wss://stream.binance.com:9443/stream", "um": "wss://fstream.binance.com/stream"}
+
+# 权重用量超过上限的这个比例就等到下一分钟
+WEIGHT_RATIO = 0.5
+MAX_RETRIES = 3
+
+
+def decimals(step: str) -> int:
+    """"0.01000000" -> 2，"1.0" -> 0"""
+    return len(step.rstrip("0").split(".")[1]) if "." in step.rstrip("0") else 0
+
+
+class BinanceClient:
+    def __init__(self) -> None:
+        self.http = httpx.AsyncClient(timeout=20)
+
+    async def get(self, market: str, path_key: str, params: dict[str, Any] | None = None) -> Any:
+        conf = REST[market]
+        for attempt in range(1, MAX_RETRIES + 1):
+            try:
+                resp = await self.http.get(conf["base"] + conf[path_key], params=params)
+            except httpx.TransportError as e:
+                if attempt == MAX_RETRIES:
+                    raise
+                logger.warning("币安请求网络错误（第 %d 次）：%r", attempt, e)
+                await asyncio.sleep(2 * attempt)
+                continue
+            if resp.status_code in (418, 429):
+                # 418 表示 IP 已被临时封禁，必须按 Retry-After 等待
+                wait = int(resp.headers.get("Retry-After", "60"))
+                logger.warning("币安限流 %d，暂停 %ds", resp.status_code, wait)
+                await asyncio.sleep(wait)
+                continue
+            if resp.status_code >= 500 and attempt < MAX_RETRIES:
+                await asyncio.sleep(2 * attempt)
+                continue
+            resp.raise_for_status()
+            used = int(resp.headers.get("X-MBX-USED-WEIGHT-1M", "0"))
+            if used > conf["weight_limit"] * WEIGHT_RATIO:
+                wait = 61 - time.time() % 60
+                logger.info("%s 权重已用 %d，等待 %.0fs 到下一分钟", market, used, wait)
+                await asyncio.sleep(wait)
+            return resp.json()
+        raise RuntimeError(f"币安请求重试 {MAX_RETRIES} 次仍被限流：{path_key} {params}")
+
+    async def klines(self, market: str, symbol: str, interval: str, start_time: int, end_time: int | None = None) -> list[list]:
+        params: dict[str, Any] = {"symbol": symbol, "interval": interval, "startTime": start_time, "limit": REST[market]["kline_limit"]}
+        if end_time is not None:
+            params["endTime"] = end_time
+        return await self.get(market, "klines", params)
+
+    async def tradable_symbols(self, market: str) -> list[dict[str, Any]]:
+        """可交易的交易对；合约只保留永续（含 TRADIFI_PERPETUAL 等美股永续）"""
+        info = await self.get(market, "exchange_info")
+        result = []
+        for s in info["symbols"]:
+            if s["status"] != "TRADING" or (market == "um" and not s["contractType"].endswith("PERPETUAL")):
+                continue
+            filters = {f["filterType"]: f for f in s["filters"]}
+            result.append({
+                "symbol": s["symbol"],
+                "base": s["baseAsset"],
+                "quote": s["quoteAsset"],
+                "price_precision": decimals(filters["PRICE_FILTER"]["tickSize"]),
+                "volume_precision": decimals(filters["LOT_SIZE"]["stepSize"]),
+            })
+        return result
+
+    async def ticker(self, market: str, symbol: str) -> dict[str, Any]:
+        return await self.get(market, "ticker", {"symbol": symbol})

backend/src/alphapulse/market/channels.py

@@ -0,0 +1,9 @@
+from alphapulse.core.redis import KEY_PREFIX
+
+
+def kline_channel(market: str, symbol: str, interval: str) -> str:
+    return f"{KEY_PREFIX}pub:kline:{market}:{symbol}:{interval}"
+
+
+def ticker_channel(market: str, symbol: str) -> str:
+    return f"{KEY_PREFIX}pub:ticker:{market}:{symbol}"

backend/src/alphapulse/market/worker.py

@@ -0,0 +1,224 @@
+"""行情进程：全历史补齐、断线/定时追平、币安 WS 实时推送。
+
+用法：python -m alphapulse.market.worker
+"""
+
+import asyncio
+import json
+import logging
+import time
+
+import httpx
+import websockets
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from alphapulse.core.db import SessionLocal
+from alphapulse.core.redis import redis
+from alphapulse.market.binance import REST, WS, BinanceClient
+from alphapulse.market.channels import kline_channel, ticker_channel
+from alphapulse.models import Kline, KlineSyncState, MarketSymbol
+from alphapulse.models.market import INTERVALS, MARKETS
+
+logger = logging.getLogger("alphapulse.market.worker")
+
+# 每个补齐任务连续拉取的页数，之后重新挑选优先级最高的任务（新加的交易对能插队补大周期）
+PAGES_PER_TURN = 20
+CATCHUP_SECONDS = 3600
+SYMBOL_CHECK_SECONDS = 10
+ACTIVE_STATUSES = ("pending", "backfilling", "ready")
+
+client = BinanceClient()
+# WS（重新）连上后触发一次追平，补断线期间的缺口
+catchup_needed = asyncio.Event()
+
+
+def row_to_values(market: str, symbol: str, interval: str, row: list) -> dict:
+    return {
+        "market": market, "symbol": symbol, "interval": interval,
+        "open_time": row[0], "open": row[1], "high": row[2], "low": row[3], "close": row[4], "volume": row[5],
+        "close_time": row[6], "quote_volume": row[7], "trades": row[8], "taker_buy_base": row[9], "taker_buy_quote": row[10],
+    }
+
+
+async def upsert_klines(values: list[dict]) -> None:
+    stmt = pg_insert(Kline).values(values)
+    updatable = ("open", "high", "low", "close", "volume", "close_time", "quote_volume", "trades", "taker_buy_base", "taker_buy_quote")
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[Kline.market, Kline.symbol, Kline.interval, Kline.open_time],
+        set_={c: stmt.excluded[c] for c in updatable},
+    )
+    async with SessionLocal() as session:
+        await session.execute(stmt)
+        await session.commit()
+
+
+async def sync_page(market: str, symbol: str, interval: str) -> bool:
+    """从上次同步位置拉一页并落库，返回是否已追平到最新"""
+    async with SessionLocal() as session:
+        state = await session.get(KlineSyncState, (market, symbol, interval))
+        if state is None:
+            state = KlineSyncState(market=market, symbol=symbol, interval=interval)
+            session.add(state)
+        # 从最后一根（可能未收盘）重新拉起；首次从 0 开始，币安会返回上市后的第一根
+        rows = await client.klines(market, symbol, interval, state.latest_open_time or 0)
+        if rows:
+            await upsert_klines([row_to_values(market, symbol, interval, r) for r in rows])
+            state.earliest_open_time = state.earliest_open_time or rows[0][0]
+            state.latest_open_time = rows[-1][0]
+        caught_up = len(rows) < REST[market]["kline_limit"]
+        if caught_up:
+            state.backfill_done = True
+        await session.commit()
+    return caught_up
+
+
+async def next_backfill_task(market: str) -> tuple[MarketSymbol, str] | None:
+    async with SessionLocal() as session:
+        symbols = (await session.scalars(
+            select(MarketSymbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(("pending", "backfilling")))
+            .order_by(MarketSymbol.added_at)
+        )).all()
+        done = set((await session.execute(
+            select(KlineSyncState.symbol, KlineSyncState.interval)
+            .where(KlineSyncState.market == market, KlineSyncState.backfill_done)
+        )).all())
+    unfinished = []
+    for s in symbols:
+        if all((s.symbol, interval) in done for interval in INTERVALS):
+            await set_symbol_status(s.id, "ready")
+            logger.info("%s %s 全部周期补齐完成", market, s.symbol)
+        else:
+            unfinished.append(s)
+    for interval in INTERVALS:
+        for s in unfinished:
+            if (s.symbol, interval) not in done:
+                return s, interval
+    return None
+
+
+async def set_symbol_status(symbol_id: int, status: str, error: str | None = None) -> None:
+    async with SessionLocal() as session:
+        s = await session.get(MarketSymbol, symbol_id)
+        s.status = status
+        s.last_error = error
+        await session.commit()
+
+
+async def backfill_loop(market: str) -> None:
+    while True:
+        task = await next_backfill_task(market)
+        if task is None:
+            await asyncio.sleep(5)
+            continue
+        s, interval = task
+        if s.status == "pending":
+            await set_symbol_status(s.id, "backfilling")
+            logger.info("%s %s 开始补齐", market, s.symbol)
+        try:
+            for _ in range(PAGES_PER_TURN):
+                if await sync_page(market, s.symbol, interval):
+                    logger.info("%s %s %s 补齐完成", market, s.symbol, interval)
+                    break
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 400:
+                # 交易对不存在或已下架：标记错误，不再重试
+                await set_symbol_status(s.id, "error", e.response.text[:500])
+                logger.error("%s %s 补齐失败：%s", market, s.symbol, e.response.text[:200])
+            else:
+                logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
+                await asyncio.sleep(10)
+        except Exception:
+            logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
+            await asyncio.sleep(10)
+
+
+async def catchup_loop() -> None:
+    """已补齐的周期从上次 REST 同步位置追平到最新：每小时一次，WS 重连后立即一次"""
+    while True:
+        try:
+            await asyncio.wait_for(catchup_needed.wait(), timeout=CATCHUP_SECONDS)
+        except TimeoutError:
+            pass
+        catchup_needed.clear()
+        async with SessionLocal() as session:
+            states = (await session.execute(
+                select(KlineSyncState.market, KlineSyncState.symbol, KlineSyncState.interval)
+                .join(MarketSymbol, (MarketSymbol.market == KlineSyncState.market) & (MarketSymbol.symbol == KlineSyncState.symbol))
+                .where(KlineSyncState.backfill_done, MarketSymbol.status.in_(ACTIVE_STATUSES))
+            )).all()
+        started = time.monotonic()
+        for market, symbol, interval in states:
+            try:
+                while not await sync_page(market, symbol, interval):
+                    pass
+            except Exception:
+                logger.exception("%s %s %s 追平出错", market, symbol, interval)
+        logger.info("追平完成：%d 个周期，用时 %.1fs", len(states), time.monotonic() - started)
+
+
+async def active_symbols(market: str) -> list[str]:
+    async with SessionLocal() as session:
+        return list((await session.scalars(
+            select(MarketSymbol.symbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(ACTIVE_STATUSES))
+            .order_by(MarketSymbol.symbol)
+        )).all())
+
+
+async def handle_message(market: str, raw: str | bytes) -> None:
+    data = json.loads(raw)["data"]
+    symbol = data["s"]
+    if data["e"] == "kline":
+        k = data["k"]
+        payload = {"t": k["t"], "o": k["o"], "h": k["h"], "l": k["l"], "c": k["c"], "v": k["v"], "q": k["q"], "x": k["x"]}
+        await redis.publish(kline_channel(market, symbol, k["i"]), json.dumps(payload))
+        # 收盘 K 线入库；1s 只推送不落库
+        if k["x"] and k["i"] != "1s":
+            await upsert_klines([{
+                "market": market, "symbol": symbol, "interval": k["i"],
+                "open_time": k["t"], "open": k["o"], "high": k["h"], "low": k["l"], "close": k["c"], "volume": k["v"],
+                "close_time": k["T"], "quote_volume": k["q"], "trades": k["n"], "taker_buy_base": k["V"], "taker_buy_quote": k["Q"],
+            }])
+    elif data["e"] == "24hrMiniTicker":
+        payload = {"c": data["c"], "o": data["o"], "h": data["h"], "l": data["l"], "v": data["v"], "q": data["q"], "E": data["E"]}
+        await redis.publish(ticker_channel(market, symbol), json.dumps(payload))
+
+
+async def stream_loop(market: str) -> None:
+    while True:
+        symbols = await active_symbols(market)
+        if not symbols:
+            await asyncio.sleep(SYMBOL_CHECK_SECONDS)
+            continue
+        intervals = INTERVALS + (["1s"] if market == "spot" else [])
+        streams = [f"{s.lower()}@kline_{i}" for s in symbols for i in intervals] + [f"{s.lower()}@miniTicker" for s in symbols]
+        try:
+            async with websockets.connect(f"{WS[market]}?streams={'/'.join(streams)}", max_size=2**22) as ws:
+                logger.info("%s WS 已连接：%d 个交易对，%d 个流", market, len(symbols), len(streams))
+                catchup_needed.set()
+                next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
+                async for raw in ws:
+                    await handle_message(market, raw)
+                    if time.monotonic() >= next_check:
+                        next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
+                        if await active_symbols(market) != symbols:
+                            logger.info("%s 交易对列表变化，重新订阅", market)
+                            break
+        except Exception:
+            logger.exception("%s WS 断开，5s 后重连", market)
+            await asyncio.sleep(5)
+
+
+async def main() -> None:
+    await asyncio.gather(
+        *(backfill_loop(m) for m in MARKETS),
+        *(stream_loop(m) for m in MARKETS),
+        catchup_loop(),
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    # httpx 每个请求一行 INFO，补齐时刷屏
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    asyncio.run(main())