eezd/AlphaPulse
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden background loops and API edge cases from code review

Browse Source 
- crawler manager: retry transient errors inside each loop, never let
  log writes break supervision, run tasks in a TaskGroup so any escaped
  error restarts the container instead of leaving a half-dead process
- market worker: tolerate symbols deleted mid-backfill, retry task
  selection and symbol queries, run loops in a TaskGroup
- market WS: validate subscription messages and close the connection
  when either the receive or forward side ends
- news pipeline: make first insert concurrency-safe with ON CONFLICT
- market API: only proxy ticker/1s klines for added symbols, map Binance
  rate limits to 503 and upstream errors to 502 without blocking
- news API: escape LIKE wildcards in search, return 422 for bad cursors
- market page: cancel pending reconnect timer on unmount
This commit is contained in:
eezd committed 2026-10-13 13:51:00 +08:00
1 parent aabc81ba5b
commit 554d224be2
7 files changed
+259 -114
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/alphapulse/market/binance.py
+10 -2
View File
@@ -27,8 +27,14 @@ def decimals(step: str) -> int:
return len(step.rstrip("0").split(".")[1]) if "." in step.rstrip("0") else 0
class RateLimited(Exception):
"""非等待模式下被币安限流"""
class BinanceClient:
def __init__(self) -> None:
def __init__(self, wait: bool = True) -> None:
# wait=False 用于 API 请求：被限流时立即抛 RateLimited，不让 HTTP 请求挂起等待
self.wait = wait
self.http = httpx.AsyncClient(timeout=20)
async def get(self, market: str, path_key: str, params: dict[str, Any] | None = None) -> Any:
@@ -45,6 +51,8 @@ class BinanceClient:
if resp.status_code in (418, 429):
# 418 表示 IP 已被临时封禁，必须按 Retry-After 等待
wait = int(resp.headers.get("Retry-After", "60"))
if not self.wait:
raise RateLimited(f"币安限流 {resp.status_code}，需等待 {wait}s")
logger.warning("币安限流 %d，暂停 %ds", resp.status_code, wait)
await asyncio.sleep(wait)
continue
@@ -53,7 +61,7 @@ class BinanceClient:
continue
resp.raise_for_status()
used = int(resp.headers.get("X-MBX-USED-WEIGHT-1M", "0"))
if used > conf["weight_limit"] * WEIGHT_RATIO:
if self.wait and used > conf["weight_limit"] * WEIGHT_RATIO:
wait = 61 - time.time() % 60
logger.info("%s 权重已用 %d，等待 %.0fs 到下一分钟", market, used, wait)
await asyncio.sleep(wait)
backend/src/alphapulse/market/worker.py
+35 -22
View File
@@ -100,6 +100,9 @@ async def next_backfill_task(market: str) -> tuple[MarketSymbol, str] | None:
async def set_symbol_status(symbol_id: int, status: str, error: str | None = None) -> None:
async with SessionLocal() as session:
s = await session.get(MarketSymbol, symbol_id)
if s is None:
# 补齐过程中交易对被用户删除
return
s.status = status
s.last_error = error
await session.commit()
@@ -107,15 +110,20 @@ async def set_symbol_status(symbol_id: int, status: str, error: str | None = Non
async def backfill_loop(market: str) -> None:
while True:
task = await next_backfill_task(market)
try:
task = await next_backfill_task(market)
except Exception:
logger.exception("%s 选取补齐任务出错，稍后重试", market)
await asyncio.sleep(10)
continue
if task is None:
await asyncio.sleep(5)
continue
s, interval = task
if s.status == "pending":
await set_symbol_status(s.id, "backfilling")
logger.info("%s %s 开始补齐", market, s.symbol)
try:
if s.status == "pending":
await set_symbol_status(s.id, "backfilling")
logger.info("%s %s 开始补齐", market, s.symbol)
for _ in range(PAGES_PER_TURN):
if await sync_page(market, s.symbol, interval):
logger.info("%s %s %s 补齐完成", market, s.symbol, interval)
@@ -141,12 +149,16 @@ async def catchup_loop() -> None:
except TimeoutError:
pass
catchup_needed.clear()
async with SessionLocal() as session:
states = (await session.execute(
select(KlineSyncState.market, KlineSyncState.symbol, KlineSyncState.interval)
.join(MarketSymbol, (MarketSymbol.market == KlineSyncState.market) & (MarketSymbol.symbol == KlineSyncState.symbol))
.where(KlineSyncState.backfill_done, MarketSymbol.status.in_(ACTIVE_STATUSES))
)).all()
try:
async with SessionLocal() as session:
states = (await session.execute(
select(KlineSyncState.market, KlineSyncState.symbol, KlineSyncState.interval)
.join(MarketSymbol, (MarketSymbol.market == KlineSyncState.market) & (MarketSymbol.symbol == KlineSyncState.symbol))
.where(KlineSyncState.backfill_done, MarketSymbol.status.in_(ACTIVE_STATUSES))
)).all()
except Exception:
logger.exception("读取追平列表失败，下次再试")
continue
started = time.monotonic()
for market, symbol, interval in states:
try:
@@ -186,13 +198,13 @@ async def handle_message(market: str, raw: str | bytes) -> None:
async def stream_loop(market: str) -> None:
while True:
symbols = await active_symbols(market)
if not symbols:
await asyncio.sleep(SYMBOL_CHECK_SECONDS)
continue
intervals = INTERVALS + (["1s"] if market == "spot" else [])
streams = [f"{s.lower()}@kline_{i}" for s in symbols for i in intervals] + [f"{s.lower()}@miniTicker" for s in symbols]
try:
symbols = await active_symbols(market)
if not symbols:
await asyncio.sleep(SYMBOL_CHECK_SECONDS)
continue
intervals = INTERVALS + (["1s"] if market == "spot" else [])
streams = [f"{s.lower()}@kline_{i}" for s in symbols for i in intervals] + [f"{s.lower()}@miniTicker" for s in symbols]
async with websockets.connect(f"{WS[market]}?streams={'/'.join(streams)}", max_size=2**22) as ws:
logger.info("%s WS 已连接：%d 个交易对，%d 个流", market, len(symbols), len(streams))
catchup_needed.set()
@@ -205,16 +217,17 @@ async def stream_loop(market: str) -> None:
logger.info("%s 交易对列表变化，重新订阅", market)
break
except Exception:
logger.exception("%s WS 断开，5s 后重连", market)
logger.exception("%s WS 循环出错，5s 后重连", market)
await asyncio.sleep(5)
async def main() -> None:
await asyncio.gather(
*(backfill_loop(m) for m in MARKETS),
*(stream_loop(m) for m in MARKETS),
catchup_loop(),
)
# 各循环自行重试可恢复的错误；仍有异常逃出时整个进程退出，由 docker 重启
async with asyncio.TaskGroup() as tg:
for m in MARKETS:
tg.create_task(backfill_loop(m))
tg.create_task(stream_loop(m))
tg.create_task(catchup_loop())
if __name__ == "__main__":