Initial AlphaPulse implementation: news crawler and Binance klines
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed
commit
aabc81ba5b
88 files changed
+12603
No files matched your search
@@ -0,0 +1,17 @@
|
||||
from alphapulse.core.db import Base
|
||||
from alphapulse.models.market import Kline, KlineSyncState, MarketSymbol
|
||||
from alphapulse.models.news import CrawlLog, CrawlRun, NewsItem, NewsRevision, RawBody, RawResponse, Source
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"CrawlLog",
|
||||
"CrawlRun",
|
||||
"Kline",
|
||||
"KlineSyncState",
|
||||
"MarketSymbol",
|
||||
"NewsItem",
|
||||
"NewsRevision",
|
||||
"RawBody",
|
||||
"RawResponse",
|
||||
"Source",
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, Integer, Numeric, PrimaryKeyConstraint, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from alphapulse.core.db import Base
|
||||
|
||||
# 落库的周期，顺序即补齐优先级（大周期先补，图表最先可看）；1s 只做实时推送不落库
|
||||
INTERVALS = ["1M", "1w", "3d", "1d", "12h", "8h", "6h", "4h", "2h", "1h", "30m", "15m", "5m", "3m", "1m"]
|
||||
MARKETS = ("spot", "um")
|
||||
|
||||
|
||||
class MarketSymbol(Base):
|
||||
__tablename__ = "market_symbols"
|
||||
__table_args__ = (UniqueConstraint("market", "symbol"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# spot 现货 / um U 本位永续
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
base: Mapped[str] = mapped_column(String(16))
|
||||
quote: Mapped[str] = mapped_column(String(16))
|
||||
# 由币安 tickSize / stepSize 推出的小数位，用于图表显示
|
||||
price_precision: Mapped[int] = mapped_column(Integer)
|
||||
volume_precision: Mapped[int] = mapped_column(Integer)
|
||||
# pending / backfilling / ready / error / delisted
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending")
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class Kline(Base):
|
||||
"""时间字段沿用币安的毫秒时间戳；表按 interval 做 LIST 分区（分区在迁移中创建）"""
|
||||
|
||||
__tablename__ = "klines"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("market", "symbol", "interval", "open_time"),
|
||||
{"postgresql_partition_by": "LIST (interval)"},
|
||||
)
|
||||
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
interval: Mapped[str] = mapped_column(String(4))
|
||||
open_time: Mapped[int] = mapped_column(BigInteger)
|
||||
open: Mapped[Decimal] = mapped_column(Numeric)
|
||||
high: Mapped[Decimal] = mapped_column(Numeric)
|
||||
low: Mapped[Decimal] = mapped_column(Numeric)
|
||||
close: Mapped[Decimal] = mapped_column(Numeric)
|
||||
volume: Mapped[Decimal] = mapped_column(Numeric)
|
||||
close_time: Mapped[int] = mapped_column(BigInteger)
|
||||
quote_volume: Mapped[Decimal] = mapped_column(Numeric)
|
||||
trades: Mapped[int] = mapped_column(BigInteger)
|
||||
taker_buy_base: Mapped[Decimal] = mapped_column(Numeric)
|
||||
taker_buy_quote: Mapped[Decimal] = mapped_column(Numeric)
|
||||
|
||||
|
||||
class KlineSyncState(Base):
|
||||
"""REST 同步进度：实时 WS 写入不更新这里，所以从 latest_open_time 往后追平即可补上断线缺口"""
|
||||
|
||||
__tablename__ = "kline_sync_state"
|
||||
__table_args__ = (PrimaryKeyConstraint("market", "symbol", "interval"),)
|
||||
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
interval: Mapped[str] = mapped_column(String(4))
|
||||
earliest_open_time: Mapped[int | None] = mapped_column(BigInteger)
|
||||
latest_open_time: Mapped[int | None] = mapped_column(BigInteger)
|
||||
# 首次全历史补齐是否完成
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -0,0 +1,122 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from alphapulse.core.db import Base
|
||||
|
||||
|
||||
class Source(Base):
|
||||
__tablename__ = "sources"
|
||||
|
||||
code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(64))
|
||||
list_url: Mapped[str] = mapped_column(String(255))
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
interval_seconds: Mapped[int] = mapped_column(Integer)
|
||||
backfill_hours: Mapped[int] = mapped_column(Integer, default=72)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class NewsItem(Base):
|
||||
__tablename__ = "news_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_code", "external_id"),
|
||||
Index("ix_news_items_published_at", "published_at"),
|
||||
Index("ix_news_items_source_published_at", "source_code", "published_at"),
|
||||
Index("ix_news_items_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
|
||||
Index("ix_news_items_content_trgm", "content_text", postgresql_using="gin", postgresql_ops={"content_text": "gin_trgm_ops"}),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
external_id: Mapped[str] = mapped_column(String(64))
|
||||
url: Mapped[str] = mapped_column(String(512))
|
||||
title: Mapped[str] = mapped_column(Text, default="")
|
||||
summary: Mapped[str] = mapped_column(Text, default="")
|
||||
content_html: Mapped[str] = mapped_column(Text, default="")
|
||||
content_text: Mapped[str] = mapped_column(Text, default="")
|
||||
author: Mapped[str | None] = mapped_column(String(128))
|
||||
tags: Mapped[list] = mapped_column(JSONB, default=list)
|
||||
is_paywalled: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
content_hash: Mapped[str] = mapped_column(String(64))
|
||||
revision_no: Mapped[int] = mapped_column(Integer, default=1)
|
||||
extra: Mapped[dict] = mapped_column(JSONB, default=dict)
|
||||
|
||||
|
||||
class NewsRevision(Base):
|
||||
__tablename__ = "news_revisions"
|
||||
__table_args__ = (UniqueConstraint("news_id", "revision_no"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
news_id: Mapped[int] = mapped_column(ForeignKey("news_items.id"))
|
||||
revision_no: Mapped[int] = mapped_column(Integer)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
summary: Mapped[str] = mapped_column(Text)
|
||||
content_html: Mapped[str] = mapped_column(Text)
|
||||
content_hash: Mapped[str] = mapped_column(String(64))
|
||||
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
|
||||
|
||||
|
||||
class RawBody(Base):
|
||||
__tablename__ = "raw_bodies"
|
||||
|
||||
sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# gzip 压缩后的原始响应体；同内容只存一份
|
||||
body: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class RawResponse(Base):
|
||||
__tablename__ = "raw_responses"
|
||||
__table_args__ = (Index("ix_raw_responses_source_fetched_at", "source_code", "fetched_at"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
method: Mapped[str] = mapped_column(String(8))
|
||||
status: Mapped[int] = mapped_column(Integer)
|
||||
headers: Mapped[dict] = mapped_column(JSONB)
|
||||
body_sha256: Mapped[str] = mapped_column(ForeignKey("raw_bodies.sha256"))
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class CrawlRun(Base):
|
||||
__tablename__ = "crawl_runs"
|
||||
__table_args__ = (Index("ix_crawl_runs_source_started_at", "source_code", "started_at"),)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
# schedule / startup / restart
|
||||
trigger: Mapped[str] = mapped_column(String(16))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# running / success / failed / interrupted
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
requests: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_seen: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_new: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_revised: Mapped[int] = mapped_column(Integer, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class CrawlLog(Base):
|
||||
__tablename__ = "crawl_logs"
|
||||
__table_args__ = (Index("ix_crawl_logs_source_id", "source_code", "id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
Reference in new issue
Block a user