Initial AlphaPulse implementation: news crawler and Binance klines

- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth,
  news/crawler/market APIs with SSE and WebSocket streaming
- Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth,
  cls_telegraph spiders, idempotent upserts with revisions, raw response
  archiving, per-source subprocess supervision, restart and backoff
- Market worker: full-history Binance spot/USD-M kline backfill,
  realtime WS stream, hourly and on-reconnect catch-up
- Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed,
  crawler console with live logs, Binance-style chart page
- Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed 2026-10-13 13:32:55 +08:00
commit aabc81ba5b
88 files changed
+12603

No files matched your search

+41
View File
@@ -0,0 +1,41 @@
import json
import uuid
from sqlalchemy import insert
from alphapulse.core.db import SessionLocal
from alphapulse.core.redis import KEY_PREFIX, redis
from alphapulse.models import CrawlLog
LOG_CHANNEL = f"{KEY_PREFIX}pub:logs"
NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
STATUS_KEY = f"{KEY_PREFIX}crawler:status"
def start_urls_key(source_code: str) -> str:
return f"{KEY_PREFIX}{source_code}:start_urls"
async def write_logs(rows: list[tuple[str, uuid.UUID | None, str, str]]) -> None:
"""rows: (source_code, run_id, level, message);落库后再发布,前端拿到的日志都带 id"""
if not rows:
return
async with SessionLocal() as session:
result = await session.execute(
insert(CrawlLog).returning(CrawlLog.id, CrawlLog.ts),
[{"source_code": s, "run_id": r, "level": lv, "message": m} for s, r, lv, m in rows],
)
saved = result.all()
await session.commit()
async with redis.pipeline(transaction=False) as pipe:
for (source_code, run_id, level, message), (log_id, ts) in zip(rows, saved, strict=True):
pipe.publish(LOG_CHANNEL, json.dumps({
"id": log_id,
"source_code": source_code,
"run_id": str(run_id) if run_id else None,
"ts": ts.isoformat(),
"level": level,
"message": message,
}, ensure_ascii=False))
await pipe.execute()