Initial AlphaPulse implementation: news crawler and Binance klines
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed
commit
aabc81ba5b
88 files changed
+12603
No files matched your search
@@ -0,0 +1,161 @@
|
||||
"""news tables
|
||||
|
||||
Revision ID: ac70d5a07cf0
|
||||
Revises:
|
||||
Create Date: 2026-10-13 04:29:09.326501
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'ac70d5a07cf0'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('raw_bodies',
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('body', sa.LargeBinary(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('sha256')
|
||||
)
|
||||
op.create_table('sources',
|
||||
sa.Column('code', sa.String(length=32), nullable=False),
|
||||
sa.Column('name', sa.String(length=64), nullable=False),
|
||||
sa.Column('list_url', sa.String(length=255), nullable=False),
|
||||
sa.Column('enabled', sa.Boolean(), nullable=False),
|
||||
sa.Column('interval_seconds', sa.Integer(), nullable=False),
|
||||
sa.Column('backfill_hours', sa.Integer(), nullable=False),
|
||||
sa.Column('consecutive_failures', sa.Integer(), nullable=False),
|
||||
sa.Column('last_success_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('paused_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('code')
|
||||
)
|
||||
op.create_table('crawl_logs',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('run_id', sa.Uuid(), nullable=True),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('ts', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('level', sa.String(length=8), nullable=False),
|
||||
sa.Column('message', sa.Text(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_crawl_logs_source_id', 'crawl_logs', ['source_code', 'id'], unique=False)
|
||||
op.create_table('crawl_runs',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('trigger', sa.String(length=16), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('requests', sa.Integer(), nullable=False),
|
||||
sa.Column('items_seen', sa.Integer(), nullable=False),
|
||||
sa.Column('items_new', sa.Integer(), nullable=False),
|
||||
sa.Column('items_revised', sa.Integer(), nullable=False),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_crawl_runs_source_started_at', 'crawl_runs', ['source_code', 'started_at'], unique=False)
|
||||
op.create_table('news_items',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('external_id', sa.String(length=64), nullable=False),
|
||||
sa.Column('url', sa.String(length=512), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=False),
|
||||
sa.Column('summary', sa.Text(), nullable=False),
|
||||
sa.Column('content_html', sa.Text(), nullable=False),
|
||||
sa.Column('content_text', sa.Text(), nullable=False),
|
||||
sa.Column('author', sa.String(length=128), nullable=True),
|
||||
sa.Column('tags', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('is_paywalled', sa.Boolean(), nullable=False),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('source_updated_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('first_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('content_hash', sa.String(length=64), nullable=False),
|
||||
sa.Column('revision_no', sa.Integer(), nullable=False),
|
||||
sa.Column('extra', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('source_code', 'external_id')
|
||||
)
|
||||
op.create_index('ix_news_items_content_trgm', 'news_items', ['content_text'], unique=False, postgresql_using='gin', postgresql_ops={'content_text': 'gin_trgm_ops'})
|
||||
op.create_index('ix_news_items_published_at', 'news_items', ['published_at'], unique=False)
|
||||
op.create_index('ix_news_items_source_published_at', 'news_items', ['source_code', 'published_at'], unique=False)
|
||||
op.create_index('ix_news_items_title_trgm', 'news_items', ['title'], unique=False, postgresql_using='gin', postgresql_ops={'title': 'gin_trgm_ops'})
|
||||
op.create_table('raw_responses',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('run_id', sa.Uuid(), nullable=True),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('url', sa.Text(), nullable=False),
|
||||
sa.Column('method', sa.String(length=8), nullable=False),
|
||||
sa.Column('status', sa.Integer(), nullable=False),
|
||||
sa.Column('headers', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('body_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('fetched_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['body_sha256'], ['raw_bodies.sha256'], ),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_raw_responses_source_fetched_at', 'raw_responses', ['source_code', 'fetched_at'], unique=False)
|
||||
op.create_table('news_revisions',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('news_id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('revision_no', sa.Integer(), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=False),
|
||||
sa.Column('summary', sa.Text(), nullable=False),
|
||||
sa.Column('content_html', sa.Text(), nullable=False),
|
||||
sa.Column('content_hash', sa.String(length=64), nullable=False),
|
||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('raw_response_id', sa.BigInteger(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['news_id'], ['news_items.id'], ),
|
||||
sa.ForeignKeyConstraint(['raw_response_id'], ['raw_responses.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('news_id', 'revision_no')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
sources = sa.table(
|
||||
"sources",
|
||||
sa.column("code", sa.String), sa.column("name", sa.String), sa.column("list_url", sa.String),
|
||||
sa.column("enabled", sa.Boolean), sa.column("interval_seconds", sa.Integer),
|
||||
sa.column("backfill_hours", sa.Integer), sa.column("consecutive_failures", sa.Integer),
|
||||
)
|
||||
op.bulk_insert(sources, [
|
||||
{"code": code, "name": name, "list_url": url, "enabled": True, "interval_seconds": interval, "backfill_hours": 72, "consecutive_failures": 0}
|
||||
for code, name, url, interval in [
|
||||
("wscn_news", "华尔街见闻·文章", "https://wallstreetcn.com/news/global", 120),
|
||||
("wscn_live", "华尔街见闻·快讯", "https://wallstreetcn.com/live/global", 15),
|
||||
("cls_depth", "财联社·深度", "https://www.cls.cn/depth?id=1000", 120),
|
||||
("cls_telegraph", "财联社·电报", "https://www.cls.cn/telegraph", 15),
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('news_revisions')
|
||||
op.drop_index('ix_raw_responses_source_fetched_at', table_name='raw_responses')
|
||||
op.drop_table('raw_responses')
|
||||
op.drop_index('ix_news_items_title_trgm', table_name='news_items', postgresql_using='gin', postgresql_ops={'title': 'gin_trgm_ops'})
|
||||
op.drop_index('ix_news_items_source_published_at', table_name='news_items')
|
||||
op.drop_index('ix_news_items_published_at', table_name='news_items')
|
||||
op.drop_index('ix_news_items_content_trgm', table_name='news_items', postgresql_using='gin', postgresql_ops={'content_text': 'gin_trgm_ops'})
|
||||
op.drop_table('news_items')
|
||||
op.drop_index('ix_crawl_runs_source_started_at', table_name='crawl_runs')
|
||||
op.drop_table('crawl_runs')
|
||||
op.drop_index('ix_crawl_logs_source_id', table_name='crawl_logs')
|
||||
op.drop_table('crawl_logs')
|
||||
op.drop_table('sources')
|
||||
op.drop_table('raw_bodies')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,36 @@
|
||||
"""symbol precision
|
||||
|
||||
Revision ID: c7e1beb72c75
|
||||
Revises: fb4db40f2e86
|
||||
Create Date: 2026-10-13 05:00:25.535575
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'c7e1beb72c75'
|
||||
down_revision: Union[str, Sequence[str], None] = 'fb4db40f2e86'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
op.add_column('market_symbols', sa.Column('price_precision', sa.Integer(), nullable=True))
|
||||
op.add_column('market_symbols', sa.Column('volume_precision', sa.Integer(), nullable=True))
|
||||
# 默认三个交易对的精度（2026-10-13 币安 exchangeInfo：tickSize 均为 0.01，stepSize 分别为 0.00001/0.0001/0.001）
|
||||
for symbol, volume_precision in (("BTCUSDT", 5), ("ETHUSDT", 4), ("BNBUSDT", 3)):
|
||||
op.execute(sa.text("UPDATE market_symbols SET price_precision = 2, volume_precision = :v WHERE market = 'spot' AND symbol = :s")
|
||||
.bindparams(v=volume_precision, s=symbol))
|
||||
op.alter_column('market_symbols', 'price_precision', nullable=False)
|
||||
op.alter_column('market_symbols', 'volume_precision', nullable=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_column('market_symbols', 'volume_precision')
|
||||
op.drop_column('market_symbols', 'price_precision')
|
||||
@@ -0,0 +1,86 @@
|
||||
"""market tables
|
||||
|
||||
Revision ID: fb4db40f2e86
|
||||
Revises: ac70d5a07cf0
|
||||
Create Date: 2026-10-13 04:54:37.566401
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'fb4db40f2e86'
|
||||
down_revision: Union[str, Sequence[str], None] = 'ac70d5a07cf0'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('kline_sync_state',
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('interval', sa.String(length=4), nullable=False),
|
||||
sa.Column('earliest_open_time', sa.BigInteger(), nullable=True),
|
||||
sa.Column('latest_open_time', sa.BigInteger(), nullable=True),
|
||||
sa.Column('backfill_done', sa.Boolean(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('market', 'symbol', 'interval')
|
||||
)
|
||||
op.create_table('klines',
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('interval', sa.String(length=4), nullable=False),
|
||||
sa.Column('open_time', sa.BigInteger(), nullable=False),
|
||||
sa.Column('open', sa.Numeric(), nullable=False),
|
||||
sa.Column('high', sa.Numeric(), nullable=False),
|
||||
sa.Column('low', sa.Numeric(), nullable=False),
|
||||
sa.Column('close', sa.Numeric(), nullable=False),
|
||||
sa.Column('volume', sa.Numeric(), nullable=False),
|
||||
sa.Column('close_time', sa.BigInteger(), nullable=False),
|
||||
sa.Column('quote_volume', sa.Numeric(), nullable=False),
|
||||
sa.Column('trades', sa.BigInteger(), nullable=False),
|
||||
sa.Column('taker_buy_base', sa.Numeric(), nullable=False),
|
||||
sa.Column('taker_buy_quote', sa.Numeric(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('market', 'symbol', 'interval', 'open_time'),
|
||||
postgresql_partition_by='LIST (interval)'
|
||||
)
|
||||
op.create_table('market_symbols',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('base', sa.String(length=16), nullable=False),
|
||||
sa.Column('quote', sa.String(length=16), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('is_default', sa.Boolean(), nullable=False),
|
||||
sa.Column('added_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('market', 'symbol')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
# 每个周期一个分区；周期列表固定写在迁移里，不依赖运行时代码
|
||||
for interval in ["1M", "1w", "3d", "1d", "12h", "8h", "6h", "4h", "2h", "1h", "30m", "15m", "5m", "3m", "1m"]:
|
||||
op.execute(f"CREATE TABLE klines_{interval.lower() if interval != '1M' else '1mo'} PARTITION OF klines FOR VALUES IN ('{interval}')")
|
||||
symbols = sa.table(
|
||||
"market_symbols",
|
||||
sa.column("market", sa.String), sa.column("symbol", sa.String), sa.column("base", sa.String),
|
||||
sa.column("quote", sa.String), sa.column("status", sa.String), sa.column("is_default", sa.Boolean),
|
||||
)
|
||||
op.bulk_insert(symbols, [
|
||||
{"market": "spot", "symbol": f"{base}USDT", "base": base, "quote": "USDT", "status": "pending", "is_default": True}
|
||||
for base in ("BTC", "ETH", "BNB")
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('market_symbols')
|
||||
op.drop_table('klines')
|
||||
op.drop_table('kline_sync_state')
|
||||
# ### end Alembic commands ###
|
||||
Reference in new issue
Block a user