Initial AlphaPulse implementation: news crawler and Binance klines
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed
commit
aabc81ba5b
88 files changed
+12603
No files matched your search
@@ -0,0 +1,3 @@
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
@@ -0,0 +1 @@
|
||||
3.13
|
||||
@@ -0,0 +1,149 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s_%%(slug)s
|
||||
# Or organize into date-based subdirectories (requires recursive_version_locations = true)
|
||||
# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the tzdata library which can be installed by adding
|
||||
# `alembic[tz]` to the pip requirements.
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
# sqlalchemy.url 由 alembic/env.py 从环境变量设置
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1 @@
|
||||
Generic single-database configuration with an async dbapi.
|
||||
@@ -0,0 +1,95 @@
|
||||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
from alembic import context
|
||||
from alphapulse.core.config import get_settings
|
||||
from alphapulse.models import Base
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# 连接串只从环境变量读取，不写入 alembic.ini
|
||||
config.set_main_option("sqlalchemy.url", get_settings().database_url)
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def include_object(obj, name, type_, reflected, compare_to):
|
||||
# klines 的分区表由迁移手工创建，不在模型里；不过滤的话 autogenerate 会生成 drop_table
|
||||
return not (type_ == "table" and reflected and compare_to is None and name.startswith("klines_"))
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
include_object=include_object,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata, include_object=include_object)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
"""In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
|
||||
connectable = async_engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode."""
|
||||
|
||||
asyncio.run(run_async_migrations())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,161 @@
|
||||
"""news tables
|
||||
|
||||
Revision ID: ac70d5a07cf0
|
||||
Revises:
|
||||
Create Date: 2026-10-13 04:29:09.326501
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'ac70d5a07cf0'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('raw_bodies',
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('body', sa.LargeBinary(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('sha256')
|
||||
)
|
||||
op.create_table('sources',
|
||||
sa.Column('code', sa.String(length=32), nullable=False),
|
||||
sa.Column('name', sa.String(length=64), nullable=False),
|
||||
sa.Column('list_url', sa.String(length=255), nullable=False),
|
||||
sa.Column('enabled', sa.Boolean(), nullable=False),
|
||||
sa.Column('interval_seconds', sa.Integer(), nullable=False),
|
||||
sa.Column('backfill_hours', sa.Integer(), nullable=False),
|
||||
sa.Column('consecutive_failures', sa.Integer(), nullable=False),
|
||||
sa.Column('last_success_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('paused_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('code')
|
||||
)
|
||||
op.create_table('crawl_logs',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('run_id', sa.Uuid(), nullable=True),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('ts', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('level', sa.String(length=8), nullable=False),
|
||||
sa.Column('message', sa.Text(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_crawl_logs_source_id', 'crawl_logs', ['source_code', 'id'], unique=False)
|
||||
op.create_table('crawl_runs',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('trigger', sa.String(length=16), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('requests', sa.Integer(), nullable=False),
|
||||
sa.Column('items_seen', sa.Integer(), nullable=False),
|
||||
sa.Column('items_new', sa.Integer(), nullable=False),
|
||||
sa.Column('items_revised', sa.Integer(), nullable=False),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_crawl_runs_source_started_at', 'crawl_runs', ['source_code', 'started_at'], unique=False)
|
||||
op.create_table('news_items',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('external_id', sa.String(length=64), nullable=False),
|
||||
sa.Column('url', sa.String(length=512), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=False),
|
||||
sa.Column('summary', sa.Text(), nullable=False),
|
||||
sa.Column('content_html', sa.Text(), nullable=False),
|
||||
sa.Column('content_text', sa.Text(), nullable=False),
|
||||
sa.Column('author', sa.String(length=128), nullable=True),
|
||||
sa.Column('tags', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('is_paywalled', sa.Boolean(), nullable=False),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('source_updated_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('first_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('content_hash', sa.String(length=64), nullable=False),
|
||||
sa.Column('revision_no', sa.Integer(), nullable=False),
|
||||
sa.Column('extra', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('source_code', 'external_id')
|
||||
)
|
||||
op.create_index('ix_news_items_content_trgm', 'news_items', ['content_text'], unique=False, postgresql_using='gin', postgresql_ops={'content_text': 'gin_trgm_ops'})
|
||||
op.create_index('ix_news_items_published_at', 'news_items', ['published_at'], unique=False)
|
||||
op.create_index('ix_news_items_source_published_at', 'news_items', ['source_code', 'published_at'], unique=False)
|
||||
op.create_index('ix_news_items_title_trgm', 'news_items', ['title'], unique=False, postgresql_using='gin', postgresql_ops={'title': 'gin_trgm_ops'})
|
||||
op.create_table('raw_responses',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('run_id', sa.Uuid(), nullable=True),
|
||||
sa.Column('source_code', sa.String(length=32), nullable=False),
|
||||
sa.Column('url', sa.Text(), nullable=False),
|
||||
sa.Column('method', sa.String(length=8), nullable=False),
|
||||
sa.Column('status', sa.Integer(), nullable=False),
|
||||
sa.Column('headers', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('body_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('fetched_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['body_sha256'], ['raw_bodies.sha256'], ),
|
||||
sa.ForeignKeyConstraint(['source_code'], ['sources.code'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_raw_responses_source_fetched_at', 'raw_responses', ['source_code', 'fetched_at'], unique=False)
|
||||
op.create_table('news_revisions',
|
||||
sa.Column('id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('news_id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('revision_no', sa.Integer(), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=False),
|
||||
sa.Column('summary', sa.Text(), nullable=False),
|
||||
sa.Column('content_html', sa.Text(), nullable=False),
|
||||
sa.Column('content_hash', sa.String(length=64), nullable=False),
|
||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('raw_response_id', sa.BigInteger(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['news_id'], ['news_items.id'], ),
|
||||
sa.ForeignKeyConstraint(['raw_response_id'], ['raw_responses.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('news_id', 'revision_no')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
sources = sa.table(
|
||||
"sources",
|
||||
sa.column("code", sa.String), sa.column("name", sa.String), sa.column("list_url", sa.String),
|
||||
sa.column("enabled", sa.Boolean), sa.column("interval_seconds", sa.Integer),
|
||||
sa.column("backfill_hours", sa.Integer), sa.column("consecutive_failures", sa.Integer),
|
||||
)
|
||||
op.bulk_insert(sources, [
|
||||
{"code": code, "name": name, "list_url": url, "enabled": True, "interval_seconds": interval, "backfill_hours": 72, "consecutive_failures": 0}
|
||||
for code, name, url, interval in [
|
||||
("wscn_news", "华尔街见闻·文章", "https://wallstreetcn.com/news/global", 120),
|
||||
("wscn_live", "华尔街见闻·快讯", "https://wallstreetcn.com/live/global", 15),
|
||||
("cls_depth", "财联社·深度", "https://www.cls.cn/depth?id=1000", 120),
|
||||
("cls_telegraph", "财联社·电报", "https://www.cls.cn/telegraph", 15),
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('news_revisions')
|
||||
op.drop_index('ix_raw_responses_source_fetched_at', table_name='raw_responses')
|
||||
op.drop_table('raw_responses')
|
||||
op.drop_index('ix_news_items_title_trgm', table_name='news_items', postgresql_using='gin', postgresql_ops={'title': 'gin_trgm_ops'})
|
||||
op.drop_index('ix_news_items_source_published_at', table_name='news_items')
|
||||
op.drop_index('ix_news_items_published_at', table_name='news_items')
|
||||
op.drop_index('ix_news_items_content_trgm', table_name='news_items', postgresql_using='gin', postgresql_ops={'content_text': 'gin_trgm_ops'})
|
||||
op.drop_table('news_items')
|
||||
op.drop_index('ix_crawl_runs_source_started_at', table_name='crawl_runs')
|
||||
op.drop_table('crawl_runs')
|
||||
op.drop_index('ix_crawl_logs_source_id', table_name='crawl_logs')
|
||||
op.drop_table('crawl_logs')
|
||||
op.drop_table('sources')
|
||||
op.drop_table('raw_bodies')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,36 @@
|
||||
"""symbol precision
|
||||
|
||||
Revision ID: c7e1beb72c75
|
||||
Revises: fb4db40f2e86
|
||||
Create Date: 2026-10-13 05:00:25.535575
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'c7e1beb72c75'
|
||||
down_revision: Union[str, Sequence[str], None] = 'fb4db40f2e86'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
op.add_column('market_symbols', sa.Column('price_precision', sa.Integer(), nullable=True))
|
||||
op.add_column('market_symbols', sa.Column('volume_precision', sa.Integer(), nullable=True))
|
||||
# 默认三个交易对的精度（2026-10-13 币安 exchangeInfo：tickSize 均为 0.01，stepSize 分别为 0.00001/0.0001/0.001）
|
||||
for symbol, volume_precision in (("BTCUSDT", 5), ("ETHUSDT", 4), ("BNBUSDT", 3)):
|
||||
op.execute(sa.text("UPDATE market_symbols SET price_precision = 2, volume_precision = :v WHERE market = 'spot' AND symbol = :s")
|
||||
.bindparams(v=volume_precision, s=symbol))
|
||||
op.alter_column('market_symbols', 'price_precision', nullable=False)
|
||||
op.alter_column('market_symbols', 'volume_precision', nullable=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_column('market_symbols', 'volume_precision')
|
||||
op.drop_column('market_symbols', 'price_precision')
|
||||
@@ -0,0 +1,86 @@
|
||||
"""market tables
|
||||
|
||||
Revision ID: fb4db40f2e86
|
||||
Revises: ac70d5a07cf0
|
||||
Create Date: 2026-10-13 04:54:37.566401
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'fb4db40f2e86'
|
||||
down_revision: Union[str, Sequence[str], None] = 'ac70d5a07cf0'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('kline_sync_state',
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('interval', sa.String(length=4), nullable=False),
|
||||
sa.Column('earliest_open_time', sa.BigInteger(), nullable=True),
|
||||
sa.Column('latest_open_time', sa.BigInteger(), nullable=True),
|
||||
sa.Column('backfill_done', sa.Boolean(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('market', 'symbol', 'interval')
|
||||
)
|
||||
op.create_table('klines',
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('interval', sa.String(length=4), nullable=False),
|
||||
sa.Column('open_time', sa.BigInteger(), nullable=False),
|
||||
sa.Column('open', sa.Numeric(), nullable=False),
|
||||
sa.Column('high', sa.Numeric(), nullable=False),
|
||||
sa.Column('low', sa.Numeric(), nullable=False),
|
||||
sa.Column('close', sa.Numeric(), nullable=False),
|
||||
sa.Column('volume', sa.Numeric(), nullable=False),
|
||||
sa.Column('close_time', sa.BigInteger(), nullable=False),
|
||||
sa.Column('quote_volume', sa.Numeric(), nullable=False),
|
||||
sa.Column('trades', sa.BigInteger(), nullable=False),
|
||||
sa.Column('taker_buy_base', sa.Numeric(), nullable=False),
|
||||
sa.Column('taker_buy_quote', sa.Numeric(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('market', 'symbol', 'interval', 'open_time'),
|
||||
postgresql_partition_by='LIST (interval)'
|
||||
)
|
||||
op.create_table('market_symbols',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('market', sa.String(length=8), nullable=False),
|
||||
sa.Column('symbol', sa.String(length=32), nullable=False),
|
||||
sa.Column('base', sa.String(length=16), nullable=False),
|
||||
sa.Column('quote', sa.String(length=16), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('is_default', sa.Boolean(), nullable=False),
|
||||
sa.Column('added_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('market', 'symbol')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
# 每个周期一个分区；周期列表固定写在迁移里，不依赖运行时代码
|
||||
for interval in ["1M", "1w", "3d", "1d", "12h", "8h", "6h", "4h", "2h", "1h", "30m", "15m", "5m", "3m", "1m"]:
|
||||
op.execute(f"CREATE TABLE klines_{interval.lower() if interval != '1M' else '1mo'} PARTITION OF klines FOR VALUES IN ('{interval}')")
|
||||
symbols = sa.table(
|
||||
"market_symbols",
|
||||
sa.column("market", sa.String), sa.column("symbol", sa.String), sa.column("base", sa.String),
|
||||
sa.column("quote", sa.String), sa.column("status", sa.String), sa.column("is_default", sa.Boolean),
|
||||
)
|
||||
op.bulk_insert(symbols, [
|
||||
{"market": "spot", "symbol": f"{base}USDT", "base": base, "quote": "USDT", "status": "pending", "is_default": True}
|
||||
for base in ("BTC", "ETH", "BNB")
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('market_symbols')
|
||||
op.drop_table('klines')
|
||||
op.drop_table('kline_sync_state')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,26 @@
|
||||
[project]
|
||||
name = "alphapulse"
|
||||
version = "0.1.0"
|
||||
description = "AlphaPulse：新闻与行情信号研究"
|
||||
authors = [
|
||||
{ name = "eezd", email = "[email protected]" }
|
||||
]
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"alembic>=1.20.0",
|
||||
"argon2-cffi>=25.1.0",
|
||||
"asyncpg>=0.32.0",
|
||||
"fastapi>=0.143.0",
|
||||
"httpx>=0.28.1",
|
||||
"pydantic-settings>=2.15.0",
|
||||
"pyjwt>=2.15.1",
|
||||
"redis>=8.1.0",
|
||||
"scrapy>=2.19.0",
|
||||
"scrapy-redis>=0.9.1",
|
||||
"sqlalchemy[asyncio]>=2.1.4",
|
||||
"uvicorn[standard]>=0.54.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.12.23,<0.13.0"]
|
||||
build-backend = "uv_build"
|
||||
@@ -0,0 +1,2 @@
|
||||
[settings]
|
||||
default = alphapulse.crawler.settings
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,58 @@
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
from alphapulse.core.auth import COOKIE_NAME, create_token, require_admin, verify_password
|
||||
from alphapulse.core.config import get_settings
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
|
||||
MAX_FAILURES = 5
|
||||
LOCK_SECONDS = 15 * 60
|
||||
|
||||
|
||||
class LoginIn(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class MeOut(BaseModel):
|
||||
username: str
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
async def login(body: LoginIn, request: Request, response: Response) -> MeOut:
|
||||
settings = get_settings()
|
||||
if not settings.admin_password_hash:
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "管理员密码未配置")
|
||||
fail_key = f"{KEY_PREFIX}login:fail:{request.client.host}"
|
||||
if int(await redis.get(fail_key) or 0) >= MAX_FAILURES:
|
||||
raise HTTPException(status.HTTP_429_TOO_MANY_REQUESTS, "失败次数过多，请 15 分钟后再试")
|
||||
|
||||
if body.username != settings.admin_username or not verify_password(body.password, settings.admin_password_hash):
|
||||
async with redis.pipeline() as pipe:
|
||||
await pipe.incr(fail_key).expire(fail_key, LOCK_SECONDS).execute()
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "用户名或密码错误")
|
||||
|
||||
await redis.delete(fail_key)
|
||||
response.set_cookie(
|
||||
COOKIE_NAME,
|
||||
create_token(body.username),
|
||||
max_age=settings.jwt_expire_days * 86400,
|
||||
httponly=True,
|
||||
secure=True,
|
||||
samesite="strict",
|
||||
)
|
||||
return MeOut(username=body.username)
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(response: Response) -> None:
|
||||
response.delete_cookie(COOKIE_NAME, httponly=True, secure=True, samesite="strict")
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def me(username: Annotated[str, Depends(require_admin)]) -> MeOut:
|
||||
return MeOut(username=username)
|
||||
@@ -0,0 +1,180 @@
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Literal
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from fastapi.sse import EventSourceResponse, ServerSentEvent
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.core.auth import require_admin
|
||||
from alphapulse.core.db import get_session
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.crawler.events import CONTROL_CHANNEL, LOG_CHANNEL, STATUS_KEY
|
||||
from alphapulse.models import CrawlLog, CrawlRun, NewsItem, Source
|
||||
|
||||
router = APIRouter(prefix="/api/crawler", tags=["crawler"], dependencies=[Depends(require_admin)])
|
||||
|
||||
# “今日”按北京时间计算
|
||||
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
class ProcessStatus(BaseModel):
|
||||
state: Literal["running", "stopped", "restarting"]
|
||||
pid: int | None
|
||||
started_at: datetime | None
|
||||
next_run_at: datetime
|
||||
|
||||
|
||||
class SourceOut(BaseModel):
|
||||
code: str
|
||||
name: str
|
||||
list_url: str
|
||||
enabled: bool
|
||||
interval_seconds: int
|
||||
backfill_hours: int
|
||||
consecutive_failures: int
|
||||
last_success_at: datetime | None
|
||||
last_error: str | None
|
||||
paused_until: datetime | None
|
||||
# 管理进程未运行时为 None
|
||||
process: ProcessStatus | None
|
||||
today_new: int
|
||||
today_revised: int
|
||||
total_items: int
|
||||
|
||||
|
||||
class SourcesOut(BaseModel):
|
||||
manager_alive: bool
|
||||
db_size_bytes: int
|
||||
sources: list[SourceOut]
|
||||
|
||||
|
||||
class SourcePatch(BaseModel):
|
||||
enabled: bool | None = None
|
||||
interval_seconds: Annotated[int | None, Field(ge=5, le=86400)] = None
|
||||
|
||||
|
||||
class RunOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
source_code: str
|
||||
trigger: str
|
||||
started_at: datetime
|
||||
finished_at: datetime | None
|
||||
status: str
|
||||
requests: int
|
||||
items_seen: int
|
||||
items_new: int
|
||||
items_revised: int
|
||||
error: str | None
|
||||
|
||||
|
||||
class LogOut(BaseModel):
|
||||
id: int
|
||||
source_code: str
|
||||
run_id: uuid.UUID | None
|
||||
ts: datetime
|
||||
level: str
|
||||
message: str
|
||||
|
||||
|
||||
class RestartIn(BaseModel):
|
||||
# 来源 code 或 "all"
|
||||
source: str
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(session: Annotated[AsyncSession, Depends(get_session)]) -> SourcesOut:
|
||||
sources = (await session.scalars(select(Source).order_by(Source.code))).all()
|
||||
today = datetime.now(LOCAL_TZ).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_counts = {
|
||||
code: (new or 0, revised or 0)
|
||||
for code, new, revised in await session.execute(
|
||||
select(CrawlRun.source_code, func.sum(CrawlRun.items_new), func.sum(CrawlRun.items_revised))
|
||||
.where(CrawlRun.started_at >= today)
|
||||
.group_by(CrawlRun.source_code)
|
||||
)
|
||||
}
|
||||
totals = dict((await session.execute(
|
||||
select(NewsItem.source_code, func.count()).group_by(NewsItem.source_code)
|
||||
)).all())
|
||||
statuses = await redis.hgetall(STATUS_KEY)
|
||||
db_size = await session.scalar(select(func.pg_database_size(func.current_database())))
|
||||
return SourcesOut(
|
||||
manager_alive=bool(statuses),
|
||||
db_size_bytes=db_size,
|
||||
sources=[
|
||||
SourceOut.model_validate({
|
||||
**s.__dict__,
|
||||
"process": json.loads(statuses[s.code]) if s.code in statuses else None,
|
||||
"today_new": today_counts.get(s.code, (0, 0))[0],
|
||||
"today_revised": today_counts.get(s.code, (0, 0))[1],
|
||||
"total_items": totals.get(s.code, 0),
|
||||
})
|
||||
for s in sources
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/sources/{code}")
|
||||
async def patch_source(code: str, body: SourcePatch, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
|
||||
source = await session.get(Source, code)
|
||||
if source is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
|
||||
for field, value in body.model_dump(exclude_none=True).items():
|
||||
setattr(source, field, value)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
async def list_runs(
|
||||
session: Annotated[AsyncSession, Depends(get_session)],
|
||||
source: str | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=200)] = 50,
|
||||
) -> list[RunOut]:
|
||||
stmt = select(CrawlRun).order_by(CrawlRun.started_at.desc()).limit(limit)
|
||||
if source:
|
||||
stmt = stmt.where(CrawlRun.source_code == source)
|
||||
return [RunOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
async def list_logs(
|
||||
session: Annotated[AsyncSession, Depends(get_session)],
|
||||
source: str | None = None,
|
||||
before_id: int | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=1000)] = 200,
|
||||
) -> list[LogOut]:
|
||||
"""按时间正序返回 before_id 之前最近的 limit 条"""
|
||||
stmt = select(CrawlLog).order_by(CrawlLog.id.desc()).limit(limit)
|
||||
if source:
|
||||
stmt = stmt.where(CrawlLog.source_code == source)
|
||||
if before_id:
|
||||
stmt = stmt.where(CrawlLog.id < before_id)
|
||||
rows = [LogOut.model_validate(r, from_attributes=True) for r in await session.scalars(stmt)]
|
||||
return rows[::-1]
|
||||
|
||||
|
||||
@router.get("/logs/stream", response_class=EventSourceResponse)
|
||||
async def stream_logs(source: str | None = None) -> AsyncIterator[ServerSentEvent]:
|
||||
async with redis.pubsub() as pubsub:
|
||||
await pubsub.subscribe(LOG_CHANNEL)
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
if source and json.loads(message["data"])["source_code"] != source:
|
||||
continue
|
||||
yield ServerSentEvent(raw_data=message["data"], event="log")
|
||||
|
||||
|
||||
@router.post("/restart")
|
||||
async def restart(body: RestartIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
|
||||
if body.source != "all" and await session.get(Source, body.source) is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "来源不存在")
|
||||
receivers = await redis.publish(CONTROL_CHANNEL, json.dumps({"action": "restart", "source": body.source}))
|
||||
if receivers == 0:
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "采集管理进程未运行")
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.openapi.docs import get_swagger_ui_html
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from alphapulse.api import auth, crawler, market, news
|
||||
from alphapulse.core.auth import require_admin
|
||||
|
||||
# 关闭默认文档路由，改为登录后才能访问
|
||||
app = FastAPI(title="AlphaPulse", docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(news.router)
|
||||
app.include_router(crawler.router)
|
||||
app.include_router(market.router)
|
||||
app.include_router(market.ws_router)
|
||||
|
||||
|
||||
@app.get("/api/health", include_in_schema=False)
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/api/openapi.json", include_in_schema=False, dependencies=[Depends(require_admin)])
|
||||
async def openapi() -> dict:
|
||||
return app.openapi()
|
||||
|
||||
|
||||
@app.get("/api/docs", include_in_schema=False, dependencies=[Depends(require_admin)])
|
||||
async def docs() -> HTMLResponse:
|
||||
return get_swagger_ui_html(openapi_url="/api/openapi.json", title="AlphaPulse API")
|
||||
@@ -0,0 +1,202 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.core.auth import require_admin, websocket_admin
|
||||
from alphapulse.core.db import get_session
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.market.binance import BinanceClient
|
||||
from alphapulse.market.channels import kline_channel, ticker_channel
|
||||
from alphapulse.models import Kline, KlineSyncState, MarketSymbol
|
||||
from alphapulse.models.market import INTERVALS
|
||||
|
||||
router = APIRouter(prefix="/api/market", tags=["market"], dependencies=[Depends(require_admin)])
|
||||
# WebSocket 握手不能用基于 Request 的依赖，单独鉴权
|
||||
ws_router = APIRouter(prefix="/api/market", tags=["market"])
|
||||
|
||||
Market = Literal["spot", "um"]
|
||||
SYMBOLS_CACHE_SECONDS = 3600
|
||||
MAX_KLINES = 1500
|
||||
|
||||
client = BinanceClient()
|
||||
|
||||
|
||||
class IntervalProgress(BaseModel):
|
||||
done: bool
|
||||
percent: float
|
||||
|
||||
|
||||
class SymbolOut(BaseModel):
|
||||
id: int
|
||||
market: str
|
||||
symbol: str
|
||||
base: str
|
||||
quote: str
|
||||
price_precision: int
|
||||
volume_precision: int
|
||||
status: str
|
||||
is_default: bool
|
||||
added_at: datetime
|
||||
last_error: str | None
|
||||
progress: dict[str, IntervalProgress]
|
||||
|
||||
|
||||
class SymbolIn(BaseModel):
|
||||
market: Market
|
||||
symbol: str
|
||||
|
||||
|
||||
class TradableSymbol(BaseModel):
|
||||
symbol: str
|
||||
base: str
|
||||
quote: str
|
||||
price_precision: int
|
||||
volume_precision: int
|
||||
|
||||
|
||||
async def tradable_symbols(market: str) -> list[dict]:
|
||||
key = f"{KEY_PREFIX}market:symbols:{market}"
|
||||
if cached := await redis.get(key):
|
||||
return json.loads(cached)
|
||||
symbols = await client.tradable_symbols(market)
|
||||
await redis.set(key, json.dumps(symbols), ex=SYMBOLS_CACHE_SECONDS)
|
||||
return symbols
|
||||
|
||||
|
||||
@router.get("/symbols")
|
||||
async def list_symbols(session: Annotated[AsyncSession, Depends(get_session)]) -> list[SymbolOut]:
|
||||
symbols = (await session.scalars(select(MarketSymbol).order_by(MarketSymbol.is_default.desc(), MarketSymbol.added_at))).all()
|
||||
states = {(s.market, s.symbol, s.interval): s for s in await session.scalars(select(KlineSyncState))}
|
||||
now = int(time.time() * 1000)
|
||||
|
||||
def progress(s: MarketSymbol, interval: str) -> IntervalProgress:
|
||||
state = states.get((s.market, s.symbol, interval))
|
||||
if state is None or state.earliest_open_time is None:
|
||||
return IntervalProgress(done=False, percent=0)
|
||||
if state.backfill_done:
|
||||
return IntervalProgress(done=True, percent=100)
|
||||
span = max(now - state.earliest_open_time, 1)
|
||||
return IntervalProgress(done=False, percent=round((state.latest_open_time - state.earliest_open_time) / span * 100, 1))
|
||||
|
||||
return [
|
||||
SymbolOut.model_validate({**s.__dict__, "progress": {i: progress(s, i) for i in INTERVALS}})
|
||||
for s in symbols
|
||||
]
|
||||
|
||||
|
||||
@router.post("/symbols", status_code=status.HTTP_201_CREATED)
|
||||
async def add_symbol(body: SymbolIn, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
|
||||
symbol = body.symbol.strip().upper()
|
||||
match = next((s for s in await tradable_symbols(body.market) if s["symbol"] == symbol), None)
|
||||
if match is None:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, f"{body.market} 市场没有可交易的 {symbol}")
|
||||
session.add(MarketSymbol(market=body.market, **match))
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, f"{symbol} 已存在") from None
|
||||
|
||||
|
||||
@router.delete("/symbols/{symbol_id}")
|
||||
async def delete_symbol(symbol_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> None:
|
||||
"""删除交易对及其 K 线（K 线可随时从币安重新补齐）"""
|
||||
s = await session.get(MarketSymbol, symbol_id)
|
||||
if s is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "交易对不存在")
|
||||
await session.execute(delete(Kline).where(Kline.market == s.market, Kline.symbol == s.symbol))
|
||||
await session.execute(delete(KlineSyncState).where(KlineSyncState.market == s.market, KlineSyncState.symbol == s.symbol))
|
||||
await session.delete(s)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@router.get("/exchange-info")
|
||||
async def exchange_info(market: Market) -> list[TradableSymbol]:
|
||||
return [TradableSymbol.model_validate(s) for s in await tradable_symbols(market)]
|
||||
|
||||
|
||||
@router.get("/klines")
|
||||
async def klines(
|
||||
session: Annotated[AsyncSession, Depends(get_session)],
|
||||
market: Market,
|
||||
symbol: str,
|
||||
interval: str,
|
||||
end: int | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=MAX_KLINES)] = 500,
|
||||
) -> list[list[float]]:
|
||||
"""返回 open_time < end 的最近 limit 根，按时间正序：[open_time, open, high, low, close, volume, quote_volume]"""
|
||||
if interval == "1s":
|
||||
# 1s 不落库，直接取币安最近数据
|
||||
if market != "spot":
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "合约没有 1s 周期")
|
||||
params = {"symbol": symbol, "interval": "1s", "limit": limit, **({"endTime": end - 1} if end else {})}
|
||||
rows = await client.get("spot", "klines", params)
|
||||
return [[r[0], float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5]), float(r[7])] for r in rows]
|
||||
if interval not in INTERVALS:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, f"不支持的周期：{interval}")
|
||||
stmt = (
|
||||
select(Kline.open_time, Kline.open, Kline.high, Kline.low, Kline.close, Kline.volume, Kline.quote_volume)
|
||||
.where(Kline.market == market, Kline.symbol == symbol, Kline.interval == interval)
|
||||
.order_by(Kline.open_time.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if end:
|
||||
stmt = stmt.where(Kline.open_time < end)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [[r[0], *(float(v) for v in r[1:])] for r in reversed(rows)]
|
||||
|
||||
|
||||
@router.get("/ticker")
|
||||
async def ticker(market: Market, symbol: str) -> dict:
|
||||
return await client.ticker(market, symbol)
|
||||
|
||||
|
||||
@ws_router.websocket("/ws")
|
||||
async def market_ws(websocket: WebSocket) -> None:
|
||||
"""客户端发送 {"market","symbol","interval"} 切换订阅；服务端推送 {"type":"kline"|"ticker", market, symbol, interval?, data}"""
|
||||
if websocket_admin(websocket) is None:
|
||||
await websocket.close(code=4401)
|
||||
return
|
||||
await websocket.accept()
|
||||
pubsub = redis.pubsub()
|
||||
subscribed = asyncio.Event()
|
||||
|
||||
async def forward() -> None:
|
||||
await subscribed.wait()
|
||||
while True:
|
||||
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
|
||||
if message is None:
|
||||
continue
|
||||
# 频道格式：ap:pub:kline:<market>:<symbol>:<interval> 或 ap:pub:ticker:<market>:<symbol>
|
||||
parts = message["channel"].split(":")
|
||||
await websocket.send_text(json.dumps({
|
||||
"type": parts[2],
|
||||
"market": parts[3],
|
||||
"symbol": parts[4],
|
||||
"interval": parts[5] if parts[2] == "kline" else None,
|
||||
"data": json.loads(message["data"]),
|
||||
}))
|
||||
|
||||
task = asyncio.create_task(forward())
|
||||
try:
|
||||
while True:
|
||||
sub = await websocket.receive_json()
|
||||
if subscribed.is_set():
|
||||
await pubsub.unsubscribe()
|
||||
await pubsub.subscribe(
|
||||
kline_channel(sub["market"], sub["symbol"], sub["interval"]),
|
||||
ticker_channel(sub["market"], sub["symbol"]),
|
||||
)
|
||||
subscribed.set()
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
task.cancel()
|
||||
await pubsub.aclose()
|
||||
@@ -0,0 +1,110 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from fastapi.sse import EventSourceResponse, ServerSentEvent
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, or_, select, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from alphapulse.core.auth import require_admin
|
||||
from alphapulse.core.db import get_session
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.crawler.events import NEWS_CHANNEL
|
||||
from alphapulse.models import NewsItem, NewsRevision
|
||||
|
||||
router = APIRouter(prefix="/api/news", tags=["news"], dependencies=[Depends(require_admin)])
|
||||
|
||||
EXCERPT_LENGTH = 300
|
||||
|
||||
|
||||
class NewsBrief(BaseModel):
|
||||
id: int
|
||||
source_code: str
|
||||
url: str
|
||||
title: str
|
||||
summary: str
|
||||
excerpt: str
|
||||
author: str | None
|
||||
tags: list
|
||||
is_paywalled: bool
|
||||
published_at: datetime
|
||||
first_seen_at: datetime
|
||||
revision_no: int
|
||||
|
||||
|
||||
class NewsPage(BaseModel):
|
||||
items: list[NewsBrief]
|
||||
# 形如 "<published_at ISO>|<id>"，原样传回即可取下一页
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class NewsDetail(NewsBrief):
|
||||
content_html: str
|
||||
source_updated_at: datetime | None
|
||||
last_seen_at: datetime
|
||||
extra: dict
|
||||
|
||||
|
||||
class Revision(BaseModel):
|
||||
revision_no: int
|
||||
title: str
|
||||
summary: str
|
||||
content_html: str
|
||||
captured_at: datetime
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_news(
|
||||
session: Annotated[AsyncSession, Depends(get_session)],
|
||||
source: Annotated[list[str] | None, Query()] = None,
|
||||
q: str | None = None,
|
||||
start: datetime | None = None,
|
||||
end: datetime | None = None,
|
||||
cursor: str | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 30,
|
||||
) -> NewsPage:
|
||||
stmt = select(NewsItem, func.left(NewsItem.content_text, EXCERPT_LENGTH).label("excerpt"))
|
||||
if source:
|
||||
stmt = stmt.where(NewsItem.source_code.in_(source))
|
||||
if q:
|
||||
pattern = f"%{q}%"
|
||||
stmt = stmt.where(or_(NewsItem.title.ilike(pattern), NewsItem.content_text.ilike(pattern)))
|
||||
if start:
|
||||
stmt = stmt.where(NewsItem.published_at >= start)
|
||||
if end:
|
||||
stmt = stmt.where(NewsItem.published_at < end)
|
||||
if cursor:
|
||||
published_at, news_id = cursor.rsplit("|", 1)
|
||||
stmt = stmt.where(tuple_(NewsItem.published_at, NewsItem.id) < (datetime.fromisoformat(published_at), int(news_id)))
|
||||
rows = (await session.execute(stmt.order_by(NewsItem.published_at.desc(), NewsItem.id.desc()).limit(limit))).all()
|
||||
|
||||
items = [NewsBrief.model_validate({**news.__dict__, "excerpt": excerpt}) for news, excerpt in rows]
|
||||
next_cursor = f"{items[-1].published_at.isoformat()}|{items[-1].id}" if len(items) == limit else None
|
||||
return NewsPage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@router.get("/stream", response_class=EventSourceResponse)
|
||||
async def stream_news() -> AsyncIterator[ServerSentEvent]:
|
||||
async with redis.pubsub() as pubsub:
|
||||
await pubsub.subscribe(NEWS_CHANNEL)
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] == "message":
|
||||
yield ServerSentEvent(raw_data=message["data"], event="news")
|
||||
|
||||
|
||||
@router.get("/{news_id}")
|
||||
async def get_news(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> NewsDetail:
|
||||
news = await session.get(NewsItem, news_id)
|
||||
if news is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "新闻不存在")
|
||||
return NewsDetail.model_validate({**news.__dict__, "excerpt": news.content_text[:EXCERPT_LENGTH]})
|
||||
|
||||
|
||||
@router.get("/{news_id}/revisions")
|
||||
async def list_revisions(news_id: int, session: Annotated[AsyncSession, Depends(get_session)]) -> list[Revision]:
|
||||
rows = await session.scalars(
|
||||
select(NewsRevision).where(NewsRevision.news_id == news_id).order_by(NewsRevision.revision_no.desc())
|
||||
)
|
||||
return [Revision.model_validate(r, from_attributes=True) for r in rows]
|
||||
Whitespace-only changes.
@@ -0,0 +1,59 @@
|
||||
import getpass
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import jwt
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from fastapi import HTTPException, Request, WebSocket, status
|
||||
|
||||
from alphapulse.core.config import get_settings
|
||||
|
||||
COOKIE_NAME = "ap_token"
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
|
||||
def create_token(username: str) -> str:
|
||||
settings = get_settings()
|
||||
expire = datetime.now(UTC) + timedelta(days=settings.jwt_expire_days)
|
||||
return jwt.encode({"sub": username, "exp": expire}, settings.jwt_secret, algorithm="HS256")
|
||||
|
||||
|
||||
def _username_from_token(token: str | None) -> str | None:
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
payload = jwt.decode(token, get_settings().jwt_secret, algorithms=["HS256"])
|
||||
except jwt.InvalidTokenError:
|
||||
return None
|
||||
return payload["sub"]
|
||||
|
||||
|
||||
def require_admin(request: Request) -> str:
|
||||
username = _username_from_token(request.cookies.get(COOKIE_NAME))
|
||||
if username != get_settings().admin_username:
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "未登录或登录已过期")
|
||||
return username
|
||||
|
||||
|
||||
def websocket_admin(websocket: WebSocket) -> str | None:
|
||||
# WebSocket 握手阶段不能用 HTTPException，由调用方自行关闭连接
|
||||
username = _username_from_token(websocket.cookies.get(COOKIE_NAME))
|
||||
return username if username == get_settings().admin_username else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 用法：python -m alphapulse.core.auth hash-password，输出写入 .env 的 ADMIN_PASSWORD_HASH
|
||||
if sys.argv[1:] != ["hash-password"]:
|
||||
sys.exit("用法：python -m alphapulse.core.auth hash-password")
|
||||
password = getpass.getpass("新密码：")
|
||||
if password != getpass.getpass("再次输入："):
|
||||
sys.exit("两次输入不一致")
|
||||
print(_hasher.hash(password))
|
||||
@@ -0,0 +1,20 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# 容器内由 compose 的 env_file 注入；只有一套配置，见仓库根目录 .env
|
||||
model_config = SettingsConfigDict(extra="ignore")
|
||||
|
||||
database_url: str
|
||||
redis_url: str
|
||||
jwt_secret: str
|
||||
admin_username: str
|
||||
admin_password_hash: str = ""
|
||||
jwt_expire_days: int = 7
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,19 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from alphapulse.core.config import get_settings
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
engine = create_async_engine(get_settings().database_url, pool_pre_ping=True)
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def get_session() -> AsyncIterator[AsyncSession]:
|
||||
async with SessionLocal() as session:
|
||||
yield session
|
||||
@@ -0,0 +1,8 @@
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from alphapulse.core.config import get_settings
|
||||
|
||||
# DB 14 与 AFFiNE 共用：所有 key 必须带此前缀，禁止 FLUSHDB
|
||||
KEY_PREFIX = "ap:"
|
||||
|
||||
redis = Redis.from_url(get_settings().redis_url, decode_responses=True)
|
||||
Whitespace-only changes.
@@ -0,0 +1,41 @@
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import insert
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.models import CrawlLog
|
||||
|
||||
LOG_CHANNEL = f"{KEY_PREFIX}pub:logs"
|
||||
NEWS_CHANNEL = f"{KEY_PREFIX}pub:news"
|
||||
CONTROL_CHANNEL = f"{KEY_PREFIX}ctl:crawler"
|
||||
STATUS_KEY = f"{KEY_PREFIX}crawler:status"
|
||||
|
||||
|
||||
def start_urls_key(source_code: str) -> str:
|
||||
return f"{KEY_PREFIX}{source_code}:start_urls"
|
||||
|
||||
|
||||
async def write_logs(rows: list[tuple[str, uuid.UUID | None, str, str]]) -> None:
|
||||
"""rows: (source_code, run_id, level, message)；落库后再发布，前端拿到的日志都带 id"""
|
||||
if not rows:
|
||||
return
|
||||
async with SessionLocal() as session:
|
||||
result = await session.execute(
|
||||
insert(CrawlLog).returning(CrawlLog.id, CrawlLog.ts),
|
||||
[{"source_code": s, "run_id": r, "level": lv, "message": m} for s, r, lv, m in rows],
|
||||
)
|
||||
saved = result.all()
|
||||
await session.commit()
|
||||
async with redis.pipeline(transaction=False) as pipe:
|
||||
for (source_code, run_id, level, message), (log_id, ts) in zip(rows, saved, strict=True):
|
||||
pipe.publish(LOG_CHANNEL, json.dumps({
|
||||
"id": log_id,
|
||||
"source_code": source_code,
|
||||
"run_id": str(run_id) if run_id else None,
|
||||
"ts": ts.isoformat(),
|
||||
"level": level,
|
||||
"message": message,
|
||||
}, ensure_ascii=False))
|
||||
await pipe.execute()
|
||||
@@ -0,0 +1,186 @@
|
||||
"""采集管理进程：按间隔触发采集、守护每个来源的 scrapy 子进程、处理重启指令。
|
||||
|
||||
用法（工作目录需包含 scrapy.cfg）：python -m alphapulse.crawler.manager
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.crawler.events import CONTROL_CHANNEL, STATUS_KEY, start_urls_key, write_logs
|
||||
from alphapulse.models import CrawlRun, Source
|
||||
|
||||
logger = logging.getLogger("alphapulse.crawler.manager")
|
||||
|
||||
RESTART_DELAY_MIN = 5
|
||||
RESTART_DELAY_MAX = 300
|
||||
# 子进程稳定运行超过该时长后，异常退出的重启延迟重新从最小值开始
|
||||
STABLE_SECONDS = 60
|
||||
STOP_TIMEOUT = 15
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
code: str
|
||||
proc: asyncio.subprocess.Process | None = None
|
||||
started_at: datetime | None = None
|
||||
next_run_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
# 下一次触发的类型：schedule / startup / restart
|
||||
trigger: str = "schedule"
|
||||
restarting: bool = False
|
||||
restart_delay: int = RESTART_DELAY_MIN
|
||||
wake: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
async def log(code: str, level: str, message: str) -> None:
|
||||
logger.log(logging.getLevelName(level), "[%s] %s", code, message)
|
||||
await write_logs([(code, None, level, message)])
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self) -> None:
|
||||
self.workers: dict[str, Worker] = {}
|
||||
self.stopping = asyncio.Event()
|
||||
|
||||
async def run(self) -> None:
|
||||
async with SessionLocal() as session:
|
||||
codes = (await session.scalars(select(Source.code).order_by(Source.code))).all()
|
||||
self.workers = {code: Worker(code) for code in codes}
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, self.stopping.set)
|
||||
|
||||
tasks = [asyncio.create_task(self.supervise(w)) for w in self.workers.values()]
|
||||
tasks += [asyncio.create_task(c) for c in (self.schedule_loop(), self.control_loop(), self.status_loop())]
|
||||
await self.stopping.wait()
|
||||
logger.info("收到退出信号，停止全部子进程")
|
||||
await asyncio.gather(*(self.stop_process(w) for w in self.workers.values()))
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await redis.delete(STATUS_KEY)
|
||||
|
||||
async def supervise(self, w: Worker) -> None:
|
||||
while not self.stopping.is_set():
|
||||
# 上次进程遗留的请求和未收尾的轮次都已失效，清理后从干净状态开始
|
||||
await redis.delete(f"{KEY_PREFIX}{w.code}:requests", start_urls_key(w.code))
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
update(CrawlRun)
|
||||
.where(CrawlRun.source_code == w.code, CrawlRun.status == "running")
|
||||
.values(status="interrupted", finished_at=datetime.now(UTC), error="子进程退出时本轮未完成")
|
||||
)
|
||||
await session.commit()
|
||||
w.proc = await asyncio.create_subprocess_exec("scrapy", "crawl", w.code)
|
||||
w.started_at = datetime.now(UTC)
|
||||
# 新进程的第一轮立即执行，并做全量回溯扫描
|
||||
if w.trigger == "schedule":
|
||||
w.trigger = "startup"
|
||||
w.next_run_at = w.started_at
|
||||
await log(w.code, "INFO", f"子进程已启动 pid={w.proc.pid}")
|
||||
returncode = await w.proc.wait()
|
||||
ran_seconds = (datetime.now(UTC) - w.started_at).total_seconds()
|
||||
w.proc = None
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
if w.restarting:
|
||||
w.restarting = False
|
||||
w.restart_delay = RESTART_DELAY_MIN
|
||||
continue
|
||||
if ran_seconds >= STABLE_SECONDS:
|
||||
w.restart_delay = RESTART_DELAY_MIN
|
||||
await log(w.code, "ERROR", f"子进程异常退出 returncode={returncode}，{w.restart_delay}s 后重启")
|
||||
w.wake.clear()
|
||||
try:
|
||||
await asyncio.wait_for(w.wake.wait(), timeout=w.restart_delay)
|
||||
except TimeoutError:
|
||||
pass
|
||||
w.restart_delay = min(w.restart_delay * 2, RESTART_DELAY_MAX)
|
||||
|
||||
async def stop_process(self, w: Worker) -> None:
|
||||
if w.proc is None:
|
||||
return
|
||||
w.proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(w.proc.wait(), timeout=STOP_TIMEOUT)
|
||||
except TimeoutError:
|
||||
await log(w.code, "WARNING", f"子进程 {STOP_TIMEOUT}s 内未退出，强制结束")
|
||||
w.proc.kill()
|
||||
|
||||
async def schedule_loop(self) -> None:
|
||||
while True:
|
||||
async with SessionLocal() as session:
|
||||
sources = (await session.scalars(select(Source))).all()
|
||||
now = datetime.now(UTC)
|
||||
for s in sources:
|
||||
w = self.workers[s.code]
|
||||
if not s.enabled or w.proc is None or w.restarting:
|
||||
continue
|
||||
if s.paused_until and s.paused_until > now:
|
||||
w.next_run_at = max(w.next_run_at, s.paused_until)
|
||||
continue
|
||||
if now < w.next_run_at:
|
||||
continue
|
||||
# 子进程还没取走上一条触发时不再堆积
|
||||
if await redis.llen(start_urls_key(s.code)) == 0:
|
||||
await redis.lpush(start_urls_key(s.code), json.dumps({"trigger": w.trigger, "backfill_hours": s.backfill_hours}))
|
||||
w.trigger = "schedule"
|
||||
w.next_run_at = now + timedelta(seconds=s.interval_seconds)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def control_loop(self) -> None:
|
||||
async with redis.pubsub() as pubsub:
|
||||
await pubsub.subscribe(CONTROL_CHANNEL)
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
command = json.loads(message["data"])
|
||||
if command["action"] != "restart":
|
||||
logger.warning("未知指令：%s", command)
|
||||
continue
|
||||
codes = list(self.workers) if command["source"] == "all" else [command["source"]]
|
||||
await asyncio.gather(*(self.restart(self.workers[code]) for code in codes))
|
||||
|
||||
async def restart(self, w: Worker) -> None:
|
||||
await log(w.code, "INFO", "收到重启指令")
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
update(Source).where(Source.code == w.code).values(consecutive_failures=0, paused_until=None)
|
||||
)
|
||||
await session.commit()
|
||||
w.trigger = "restart"
|
||||
w.next_run_at = datetime.now(UTC)
|
||||
if w.proc is None:
|
||||
# 处于异常退出后的等待期：立即重启
|
||||
w.wake.set()
|
||||
return
|
||||
w.restarting = True
|
||||
await self.stop_process(w)
|
||||
|
||||
async def status_loop(self) -> None:
|
||||
while True:
|
||||
mapping = {
|
||||
w.code: json.dumps({
|
||||
"state": "restarting" if w.restarting else ("running" if w.proc else "stopped"),
|
||||
"pid": w.proc.pid if w.proc else None,
|
||||
"started_at": w.started_at.isoformat() if w.started_at else None,
|
||||
"next_run_at": w.next_run_at.isoformat(),
|
||||
})
|
||||
for w in self.workers.values()
|
||||
}
|
||||
async with redis.pipeline() as pipe:
|
||||
# 管理进程挂掉后状态自动过期，前端据此显示"管理进程未运行"
|
||||
await pipe.hset(STATUS_KEY, mapping=mapping).expire(STATUS_KEY, 10).execute()
|
||||
await asyncio.sleep(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
asyncio.run(Manager().run())
|
||||
@@ -0,0 +1,58 @@
|
||||
import asyncio
|
||||
import gzip
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Request, Response
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import KEY_PREFIX, redis
|
||||
from alphapulse.models import RawBody, RawResponse
|
||||
|
||||
|
||||
class RedisRateLimitMiddleware:
|
||||
"""按域名的跨进程限速：同一域名两次请求至少间隔 interval_ms"""
|
||||
|
||||
interval_ms = 1000
|
||||
|
||||
async def process_request(self, request: Request) -> None:
|
||||
key = f"{KEY_PREFIX}rl:{urlparse(request.url).hostname}"
|
||||
while not await redis.set(key, "1", px=self.interval_ms, nx=True):
|
||||
ttl = await redis.pttl(key)
|
||||
await asyncio.sleep(max(ttl, 50) / 1000)
|
||||
|
||||
|
||||
class RawResponseMiddleware:
|
||||
"""每次响应都写 raw_responses；响应体按 sha256 去重存 raw_bodies"""
|
||||
|
||||
def __init__(self, crawler: Crawler) -> None:
|
||||
self.crawler = crawler
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> "RawResponseMiddleware":
|
||||
return cls(crawler)
|
||||
|
||||
async def process_response(self, request: Request, response: Response) -> Response:
|
||||
sha256 = hashlib.sha256(response.body).hexdigest()
|
||||
headers = {k.decode(): b", ".join(v).decode("latin-1") for k, v in response.headers.items()}
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
pg_insert(RawBody).values(sha256=sha256, body=gzip.compress(response.body)).on_conflict_do_nothing()
|
||||
)
|
||||
raw_id = await session.scalar(
|
||||
insert(RawResponse).values(
|
||||
run_id=request.meta.get("run_id"),
|
||||
source_code=self.crawler.spider.name,
|
||||
url=response.url,
|
||||
method=request.method,
|
||||
status=response.status,
|
||||
headers=headers,
|
||||
body_sha256=sha256,
|
||||
).returning(RawResponse.id)
|
||||
)
|
||||
await session.commit()
|
||||
request.meta["raw_response_id"] = raw_id
|
||||
return response
|
||||
@@ -0,0 +1,74 @@
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.crawler.events import NEWS_CHANNEL
|
||||
from alphapulse.models import NewsItem, NewsRevision
|
||||
|
||||
UPDATABLE_FIELDS = ("url", "title", "summary", "content_html", "content_text", "author", "tags",
|
||||
"is_paywalled", "published_at", "source_updated_at", "extra")
|
||||
|
||||
|
||||
def content_hash(item: dict[str, Any]) -> str:
|
||||
return hashlib.sha256("\0".join((item["title"], item["summary"], item["content_html"])).encode()).hexdigest()
|
||||
|
||||
|
||||
class NewsPipeline:
|
||||
"""按 (source_code, external_id) 幂等落库；内容变化写一条修订"""
|
||||
|
||||
def __init__(self, crawler: Crawler) -> None:
|
||||
self.crawler = crawler
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> "NewsPipeline":
|
||||
return cls(crawler)
|
||||
|
||||
async def process_item(self, item: dict[str, Any]) -> dict[str, Any]:
|
||||
spider = self.crawler.spider
|
||||
digest = content_hash(item)
|
||||
raw_response_id = item.pop("raw_response_id")
|
||||
async with SessionLocal() as session:
|
||||
news = await session.scalar(
|
||||
select(NewsItem)
|
||||
.where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
|
||||
.with_for_update()
|
||||
)
|
||||
if news is None:
|
||||
kind = "new"
|
||||
news = NewsItem(**item, content_hash=digest, revision_no=1)
|
||||
session.add(news)
|
||||
await session.flush()
|
||||
elif news.content_hash != digest:
|
||||
kind = "revised"
|
||||
for field in UPDATABLE_FIELDS:
|
||||
setattr(news, field, item[field])
|
||||
news.content_hash = digest
|
||||
news.revision_no += 1
|
||||
news.last_seen_at = func.now()
|
||||
else:
|
||||
kind = None
|
||||
news.last_seen_at = func.now()
|
||||
|
||||
if kind:
|
||||
session.add(NewsRevision(
|
||||
news_id=news.id,
|
||||
revision_no=news.revision_no,
|
||||
title=item["title"],
|
||||
summary=item["summary"],
|
||||
content_html=item["content_html"],
|
||||
content_hash=digest,
|
||||
raw_response_id=raw_response_id,
|
||||
))
|
||||
news_id = news.id
|
||||
await session.commit()
|
||||
|
||||
spider.run.items_seen += 1
|
||||
if kind:
|
||||
setattr(spider.run, f"items_{kind}", getattr(spider.run, f"items_{kind}") + 1)
|
||||
await redis.publish(NEWS_CHANNEL, json.dumps({"id": news_id, "source_code": item["source_code"], "kind": kind}))
|
||||
return item
|
||||
@@ -0,0 +1,45 @@
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
from alphapulse.core.config import get_settings
|
||||
from alphapulse.core.redis import KEY_PREFIX
|
||||
|
||||
BOT_NAME = "alphapulse"
|
||||
SPIDER_MODULES = ["alphapulse.crawler.spiders"]
|
||||
|
||||
# 采集的是无需登录的公开 JSON 接口；限速由 RedisRateLimitMiddleware 统一控制
|
||||
ROBOTSTXT_OBEY = False
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
|
||||
DEFAULT_REQUEST_HEADERS = {"Accept": "application/json, text/html;q=0.9, */*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9"}
|
||||
COOKIES_ENABLED = False
|
||||
TELNETCONSOLE_ENABLED = False
|
||||
# 2.19 默认开启的远程控制 HTTP 服务用不上，关掉避免开放端口
|
||||
REMOTE_CONTROL_ENABLED = False
|
||||
LOG_LEVEL = "INFO"
|
||||
|
||||
CONCURRENT_REQUESTS = 4
|
||||
DOWNLOAD_TIMEOUT = 15
|
||||
RETRY_TIMES = 2
|
||||
# 429/403 不重试，交给来源退避
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
"alphapulse.crawler.middlewares.RedisRateLimitMiddleware": 100,
|
||||
# process_response 按优先级从大到小调用：排在 HttpCompression(590) 之后拿到解压后的响应体，
|
||||
# 排在 Retry(550) 之前，被重试的每次响应也都会记录
|
||||
"alphapulse.crawler.middlewares.RawResponseMiddleware": 580,
|
||||
}
|
||||
ITEM_PIPELINES = {"alphapulse.crawler.pipelines.NewsPipeline": 300}
|
||||
|
||||
# Scrapy-Redis：请求队列与起始任务放 Redis（DB 14，统一 ap: 前缀）
|
||||
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
|
||||
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
|
||||
SCHEDULER_PERSIST = True
|
||||
SCHEDULER_QUEUE_KEY = f"{KEY_PREFIX}%(spider)s:requests"
|
||||
SCHEDULER_DUPEFILTER_KEY = f"{KEY_PREFIX}%(spider)s:dupefilter"
|
||||
REDIS_START_URLS_KEY = f"{KEY_PREFIX}%(name)s:start_urls"
|
||||
REDIS_URL = get_settings().redis_url
|
||||
|
||||
# scrapy-redis 0.9.1 仍按旧签名给 stats 传 spider 参数，每个请求都告警，会刷屏实时日志
|
||||
warnings.filterwarnings("ignore", category=ScrapyDeprecationWarning, module="scrapy_redis")
|
||||
Whitespace-only changes.
@@ -0,0 +1,287 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Coroutine, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from scrapy import Request, signals
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.spidermiddlewares.httperror import HttpError
|
||||
from scrapy_redis.spiders import RedisSpider
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from twisted.python.failure import Failure
|
||||
from w3lib.html import remove_tags, replace_entities
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.crawler.events import write_logs
|
||||
from alphapulse.models import CrawlRun, NewsItem, Source
|
||||
|
||||
# 日志落库失败时只输出到 stdout，避免被自己的处理器再次捕获
|
||||
sink_logger = logging.getLogger("alphapulse.logsink")
|
||||
|
||||
MAX_PAGES = 200
|
||||
BACKOFF_BASE_SECONDS = 30
|
||||
BACKOFF_MAX_SECONDS = 15 * 60
|
||||
|
||||
|
||||
def ts(epoch: int) -> datetime:
|
||||
return datetime.fromtimestamp(epoch, UTC)
|
||||
|
||||
|
||||
def html_to_text(html: str) -> str:
|
||||
text = re.sub(r"(?i)<br\s*/?>|</p>|</div>|</h\d>|</li>", "\n", html)
|
||||
text = replace_entities(remove_tags(text))
|
||||
return re.sub(r"\n\s*\n+", "\n", text).strip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunState:
|
||||
id: uuid.UUID
|
||||
trigger: str
|
||||
started_at: datetime
|
||||
# 早于该时间的内容不再翻页回溯
|
||||
cutoff: datetime
|
||||
requests: int = 0
|
||||
items_seen: int = 0
|
||||
items_new: int = 0
|
||||
items_revised: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
# 遇到 403/429 后本轮不再发新请求
|
||||
blocked: bool = False
|
||||
|
||||
@property
|
||||
def full_scan(self) -> bool:
|
||||
# 进程启动后的第一轮一直翻到回溯时间，补上次被中断的回溯；常规轮次遇到整页已知即停
|
||||
return self.trigger in ("startup", "restart")
|
||||
|
||||
|
||||
class DbLogHandler(logging.Handler):
|
||||
"""把本来源的日志和全局 WARNING 以上日志收集起来，由刷新任务批量落库并发布"""
|
||||
|
||||
def __init__(self, spider: "NewsSpider") -> None:
|
||||
super().__init__(logging.INFO)
|
||||
self.spider = spider
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if record.name == sink_logger.name:
|
||||
return
|
||||
if record.name != self.spider.name and record.levelno < logging.WARNING:
|
||||
return
|
||||
message = record.getMessage()
|
||||
# scrapy-redis 每次取到起始任务都会打这一行，没有信息量
|
||||
if message.startswith("start req url"):
|
||||
return
|
||||
if record.exc_info:
|
||||
message += "\n" + logging.Formatter().formatException(record.exc_info)
|
||||
run_id = self.spider.run.id if self.spider.run else None
|
||||
self.spider.log_buffer.append((self.spider.name, run_id, record.levelname, message))
|
||||
|
||||
|
||||
class NewsSpider(RedisSpider):
|
||||
"""一个来源一个子进程。管理进程往 Redis 推一条触发消息即开始一轮采集，引擎空闲即本轮结束。"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.run: RunState | None = None
|
||||
self.log_buffer: list[tuple[str, uuid.UUID | None, str, str]] = []
|
||||
self._tasks: set[asyncio.Task] = set()
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> "NewsSpider":
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider._on_opened, signal=signals.spider_opened)
|
||||
crawler.signals.connect(spider._on_idle, signal=signals.spider_idle)
|
||||
crawler.signals.connect(spider._on_error, signal=signals.spider_error)
|
||||
crawler.signals.connect(spider._on_closed, signal=signals.spider_closed)
|
||||
return spider
|
||||
|
||||
# ---------- 子类实现 ----------
|
||||
|
||||
def first_requests(self) -> Iterable[Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---------- 运行生命周期 ----------
|
||||
|
||||
async def start(self):
|
||||
for request in self.next_requests():
|
||||
yield request
|
||||
|
||||
def make_request_from_data(self, data: bytes) -> list[Request]:
|
||||
tick = json.loads(data)
|
||||
if self.run is not None:
|
||||
# scrapy-redis 的 idle 处理先于 _on_idle 执行：引擎已空闲说明上一轮其实已结束
|
||||
if not self.crawler.engine.spider_is_idle():
|
||||
self.logger.warning("上一轮尚未结束，跳过本次触发（%s）", tick["trigger"])
|
||||
return []
|
||||
self._close_run()
|
||||
now = datetime.now(UTC)
|
||||
self.run = RunState(uuid.uuid4(), tick["trigger"], now, now - timedelta(hours=tick["backfill_hours"]))
|
||||
self._spawn(self._insert_run(self.run))
|
||||
self.logger.info("开始采集（触发方式：%s）", tick["trigger"])
|
||||
return list(self.first_requests())
|
||||
|
||||
def _on_opened(self) -> None:
|
||||
logging.getLogger().addHandler(DbLogHandler(self))
|
||||
self._spawn(self._flush_logs_forever())
|
||||
self.logger.info("采集进程已启动")
|
||||
|
||||
def _on_idle(self) -> None:
|
||||
# scrapy-redis 的 idle 处理先执行，可能刚取到新触发并已开始新一轮；引擎仍有待处理请求时不能结束
|
||||
if self.run is None or not self.crawler.engine.spider_is_idle():
|
||||
return
|
||||
self._close_run()
|
||||
|
||||
def _close_run(self) -> None:
|
||||
run, self.run = self.run, None
|
||||
self._spawn(self._finish_run(run))
|
||||
|
||||
def _on_error(self, failure: Failure, response: Response) -> None:
|
||||
if self.run:
|
||||
self.run.errors.append(f"解析异常 {response.url}：{failure.value!r}")
|
||||
|
||||
async def _on_closed(self) -> None:
|
||||
self.logger.info("采集进程退出")
|
||||
await self._flush_logs()
|
||||
|
||||
def _spawn(self, coro: Coroutine) -> None:
|
||||
task = asyncio.get_running_loop().create_task(coro)
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
|
||||
async def _insert_run(self, run: RunState) -> None:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
pg_insert(CrawlRun).values(
|
||||
id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, status="running"
|
||||
).on_conflict_do_nothing()
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def _finish_run(self, run: RunState) -> None:
|
||||
now = datetime.now(UTC)
|
||||
failed = bool(run.errors)
|
||||
values = {
|
||||
"finished_at": now,
|
||||
"status": "failed" if failed else "success",
|
||||
"requests": run.requests,
|
||||
"items_seen": run.items_seen,
|
||||
"items_new": run.items_new,
|
||||
"items_revised": run.items_revised,
|
||||
"error": "\n".join(run.errors)[:4000] if failed else None,
|
||||
}
|
||||
async with SessionLocal() as session:
|
||||
# 与 _insert_run 并发时谁先执行都正确
|
||||
await session.execute(
|
||||
pg_insert(CrawlRun)
|
||||
.values(id=run.id, source_code=self.name, trigger=run.trigger, started_at=run.started_at, **values)
|
||||
.on_conflict_do_update(index_elements=[CrawlRun.id], set_=values)
|
||||
)
|
||||
source = await session.get(Source, self.name, with_for_update=True)
|
||||
if failed:
|
||||
source.consecutive_failures += 1
|
||||
delay = min(BACKOFF_BASE_SECONDS * 2 ** (source.consecutive_failures - 1), BACKOFF_MAX_SECONDS)
|
||||
source.paused_until = now + timedelta(seconds=delay)
|
||||
source.last_error = run.errors[0][:1000]
|
||||
else:
|
||||
source.consecutive_failures = 0
|
||||
source.paused_until = None
|
||||
source.last_error = None
|
||||
source.last_success_at = now
|
||||
await session.commit()
|
||||
|
||||
elapsed = (now - run.started_at).total_seconds()
|
||||
summary = f"请求 {run.requests}，条目 {run.items_seen}，新增 {run.items_new}，修订 {run.items_revised}，用时 {elapsed:.1f}s"
|
||||
if failed:
|
||||
self.logger.error("本轮失败：%s；错误 %d 个，暂停 %ds", summary, len(run.errors), delay)
|
||||
else:
|
||||
self.logger.info("本轮完成：%s", summary)
|
||||
|
||||
async def _flush_logs(self) -> None:
|
||||
rows, self.log_buffer = self.log_buffer, []
|
||||
try:
|
||||
await write_logs(rows)
|
||||
except Exception:
|
||||
sink_logger.exception("日志落库失败，丢弃 %d 条", len(rows))
|
||||
|
||||
async def _flush_logs_forever(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
await self._flush_logs()
|
||||
|
||||
# ---------- 子类使用的工具 ----------
|
||||
|
||||
def request(self, url: str, callback: Callable, **meta: Any) -> Request:
|
||||
self.run.requests += 1
|
||||
return Request(
|
||||
url,
|
||||
callback=callback,
|
||||
errback=self.on_request_error,
|
||||
dont_filter=True,
|
||||
meta={"run_id": self.run.id, **meta},
|
||||
headers=getattr(self, "request_headers", None),
|
||||
)
|
||||
|
||||
def on_request_error(self, failure: Failure) -> None:
|
||||
status = failure.value.response.status if failure.check(HttpError) else None
|
||||
if status in (403, 429):
|
||||
self.run.blocked = True
|
||||
message = f"请求失败 {failure.request.url}：{status or repr(failure.value)}"
|
||||
self.run.errors.append(message)
|
||||
self.logger.error(message)
|
||||
|
||||
async def known_items(self, external_ids: list[str]) -> dict[str, tuple[str, str]]:
|
||||
"""已入库条目的 (title, summary)，用于判断是否新增、是否需要重抓详情、是否继续翻页"""
|
||||
async with SessionLocal() as session:
|
||||
rows = await session.execute(
|
||||
select(NewsItem.external_id, NewsItem.title, NewsItem.summary)
|
||||
.where(NewsItem.source_code == self.name, NewsItem.external_id.in_(external_ids))
|
||||
)
|
||||
return {external_id: (title, summary) for external_id, title, summary in rows}
|
||||
|
||||
def should_continue(self, external_ids: list[str], known: dict, oldest: datetime, page: int) -> bool:
|
||||
"""未越过回溯时间、未被限流，且本页有新条目（或本轮为全量扫描）时才翻下一页"""
|
||||
return (
|
||||
not self.run.blocked
|
||||
and page < MAX_PAGES
|
||||
and oldest > self.run.cutoff
|
||||
and (self.run.full_scan or any(external_id not in known for external_id in external_ids))
|
||||
)
|
||||
|
||||
def item(
|
||||
self,
|
||||
response: Response,
|
||||
*,
|
||||
external_id: str,
|
||||
url: str,
|
||||
title: str,
|
||||
summary: str,
|
||||
content_html: str,
|
||||
published_at: datetime,
|
||||
author: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
is_paywalled: bool = False,
|
||||
source_updated_at: datetime | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"source_code": self.name,
|
||||
"external_id": external_id,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"content_html": content_html,
|
||||
"content_text": html_to_text(content_html),
|
||||
"author": author or None,
|
||||
"tags": tags or [],
|
||||
"is_paywalled": is_paywalled,
|
||||
"published_at": published_at,
|
||||
"source_updated_at": source_updated_at,
|
||||
"extra": extra or {},
|
||||
"raw_response_id": response.meta["raw_response_id"],
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
from alphapulse.crawler.spiders.base import NewsSpider, ts
|
||||
|
||||
|
||||
def signed_url(path: str, params: dict[str, str]) -> str:
|
||||
# 财联社网页前端的公开签名算法：md5(sha1(按键排序的查询串))
|
||||
query = {"app": "CailianpressWeb", "os": "web", "sv": "8.4.6", **params}
|
||||
query["sign"] = hashlib.md5(hashlib.sha1(urlencode(sorted(query.items())).encode()).hexdigest().encode()).hexdigest()
|
||||
return f"https://www.cls.cn{path}?{urlencode(query)}"
|
||||
|
||||
|
||||
class ClsTelegraphSpider(NewsSpider):
|
||||
"""财联社·电报：列表即全文；翻页需 refresh_type=1"""
|
||||
|
||||
name = "cls_telegraph"
|
||||
request_headers = {"Referer": "https://www.cls.cn/telegraph"}
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.request(signed_url("/v1/roll/get_roll_list", {"category": "", "last_time": "", "rn": "20"}), self.parse_list, page=1)]
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
body = response.json()
|
||||
if body["errno"] != 0:
|
||||
raise ValueError(f"接口返回错误：{body}")
|
||||
rows = [x for x in body["data"]["roll_data"] if not x["is_ad"]]
|
||||
# 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(x["id"]),
|
||||
url=f"https://www.cls.cn/detail/{x['id']}",
|
||||
title=x["title"],
|
||||
summary=x["brief"],
|
||||
content_html=f"<p>{html.escape(x['content'])}</p>",
|
||||
published_at=ts(x["ctime"]),
|
||||
source_updated_at=ts(x["modified_time"]) if x.get("modified_time") else None,
|
||||
author=x.get("author"),
|
||||
tags=[s["subject_name"] for s in x.get("subjects") or []],
|
||||
extra={"level": x.get("level"), "stock_list": x.get("stock_list")},
|
||||
)
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
||||
last_time = str(min(x["ctime"] for x in rows))
|
||||
params = {"category": "", "last_time": last_time, "refresh_type": "1", "rn": "20"}
|
||||
yield self.request(signed_url("/v1/roll/get_roll_list", params), self.parse_list, page=page + 1)
|
||||
|
||||
|
||||
class ClsDepthSpider(NewsSpider):
|
||||
"""财联社·深度（头条）：列表只有摘要，正文从详情页 __NEXT_DATA__ 解析"""
|
||||
|
||||
name = "cls_depth"
|
||||
request_headers = {"Referer": "https://www.cls.cn/depth?id=1000"}
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
# 首页不带 last_time 时 rn 无效，返回约 30 条且不按时间排序
|
||||
return [self.request(signed_url("/v3/depth/list/1000", {}), self.parse_list, page=1)]
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
body = response.json()
|
||||
if body["errno"] != 0:
|
||||
raise ValueError(f"接口返回错误：{body}")
|
||||
rows = [x for x in body["data"] if not x["is_ad"]]
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
if known.get(str(x["id"])) != (x["title"], x["brief"]):
|
||||
# 详情页不一定有 brief，摘要以列表为准（也用于下次比较是否变化）
|
||||
yield self.request(f"https://www.cls.cn/detail/{x['id']}", self.parse_detail, summary=x["brief"])
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["ctime"] for x in rows)), page):
|
||||
params = {"last_time": str(min(x["ctime"] for x in rows)), "rn": "20"}
|
||||
yield self.request(signed_url("/v3/depth/list/1000", params), self.parse_list, page=page + 1)
|
||||
|
||||
async def parse_detail(self, response: Response):
|
||||
match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text, re.S)
|
||||
if match is None:
|
||||
raise ValueError("详情页缺少 __NEXT_DATA__，页面结构可能已变化")
|
||||
a = json.loads(match.group(1))["props"]["pageProps"]["articleDetail"]
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(a["id"]),
|
||||
url=f"https://www.cls.cn/detail/{a['id']}",
|
||||
title=a["title"],
|
||||
summary=response.meta["summary"],
|
||||
content_html=a["content"],
|
||||
published_at=ts(a["ctime"]),
|
||||
author=(a.get("author") or {}).get("name"),
|
||||
tags=[s["name"] for s in a.get("subject") or []],
|
||||
# 付费内容只保存公开部分
|
||||
is_paywalled=not a["isFree"],
|
||||
extra={"level": a.get("level")},
|
||||
)
|
||||
@@ -0,0 +1,81 @@
|
||||
from scrapy import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
from alphapulse.crawler.spiders.base import NewsSpider, ts
|
||||
|
||||
API = "https://api-one-wscn.awtmt.com/apiv1/content"
|
||||
|
||||
|
||||
class WscnNewsSpider(NewsSpider):
|
||||
"""华尔街见闻·文章：列表只有摘要，新条目或标题/摘要变化时再拉详情"""
|
||||
|
||||
name = "wscn_news"
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.list_request("", 1)]
|
||||
|
||||
def list_request(self, cursor: str, page: int) -> Request:
|
||||
return self.request(f"{API}/information-flow?channel=global&accept=article&limit=20&cursor={cursor}", self.parse_list, page=page)
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
data = response.json()["data"]
|
||||
# 信息流里混有快讯，快讯由 wscn_live 负责
|
||||
rows = [x["resource"] for x in data["items"] if x["resource_type"] == "article"]
|
||||
ids = [str(r["id"]) for r in rows]
|
||||
known = await self.known_items(ids)
|
||||
for r in rows:
|
||||
if known.get(str(r["id"])) != (r["title"], r["content_short"]):
|
||||
yield self.request(f"{API}/articles/{r['id']}?extract=0", self.parse_detail, summary=r["content_short"])
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(r["display_time"] for r in rows)), page):
|
||||
yield self.list_request(data["next_cursor"], page + 1)
|
||||
|
||||
async def parse_detail(self, response: Response):
|
||||
a = response.json()["data"]
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(a["id"]),
|
||||
url=f"https://wallstreetcn.com/articles/{a['id']}",
|
||||
title=a["title"],
|
||||
summary=response.meta["summary"],
|
||||
content_html=a["content"],
|
||||
published_at=ts(a["display_time"]),
|
||||
author=(a.get("author") or {}).get("display_name"),
|
||||
is_paywalled=bool(a.get("is_need_pay")),
|
||||
extra={"source_name": a.get("source_name"), "is_priced": a.get("is_priced")},
|
||||
)
|
||||
|
||||
|
||||
class WscnLiveSpider(NewsSpider):
|
||||
"""华尔街见闻·快讯：列表即全文"""
|
||||
|
||||
name = "wscn_live"
|
||||
|
||||
def first_requests(self) -> list[Request]:
|
||||
return [self.list_request("", 1)]
|
||||
|
||||
def list_request(self, cursor: str, page: int) -> Request:
|
||||
return self.request(f"{API}/lives?channel=global-channel&limit=50&cursor={cursor}", self.parse_list, page=page)
|
||||
|
||||
async def parse_list(self, response: Response):
|
||||
data = response.json()["data"]
|
||||
rows = data["items"]
|
||||
# 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页
|
||||
ids = [str(x["id"]) for x in rows]
|
||||
known = await self.known_items(ids)
|
||||
for x in rows:
|
||||
yield self.item(
|
||||
response,
|
||||
external_id=str(x["id"]),
|
||||
url=x["uri"],
|
||||
title=x["title"],
|
||||
summary="",
|
||||
content_html=x["content"] + (x.get("content_more") or ""),
|
||||
published_at=ts(x["display_time"]),
|
||||
author=(x.get("author") or {}).get("display_name"),
|
||||
tags=x.get("channels"),
|
||||
extra={"score": x.get("score"), "symbols": x.get("symbols")},
|
||||
)
|
||||
page = response.meta["page"]
|
||||
if rows and self.should_continue(ids, known, ts(min(x["display_time"] for x in rows)), page):
|
||||
yield self.list_request(data["next_cursor"], page + 1)
|
||||
Whitespace-only changes.
@@ -0,0 +1,87 @@
|
||||
"""币安公开行情接口（无需 API Key）"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger("alphapulse.market.binance")
|
||||
|
||||
REST = {
|
||||
"spot": {"base": "https://api.binance.com", "klines": "/api/v3/klines", "exchange_info": "/api/v3/exchangeInfo",
|
||||
"ticker": "/api/v3/ticker/24hr", "weight_limit": 6000, "kline_limit": 1000},
|
||||
"um": {"base": "https://fapi.binance.com", "klines": "/fapi/v1/klines", "exchange_info": "/fapi/v1/exchangeInfo",
|
||||
"ticker": "/fapi/v1/ticker/24hr", "weight_limit": 2400, "kline_limit": 1500},
|
||||
}
|
||||
WS = {"spot": "wss://stream.binance.com:9443/stream", "um": "wss://fstream.binance.com/stream"}
|
||||
|
||||
# 权重用量超过上限的这个比例就等到下一分钟
|
||||
WEIGHT_RATIO = 0.5
|
||||
MAX_RETRIES = 3
|
||||
|
||||
|
||||
def decimals(step: str) -> int:
|
||||
""""0.01000000" -> 2，"1.0" -> 0"""
|
||||
return len(step.rstrip("0").split(".")[1]) if "." in step.rstrip("0") else 0
|
||||
|
||||
|
||||
class BinanceClient:
|
||||
def __init__(self) -> None:
|
||||
self.http = httpx.AsyncClient(timeout=20)
|
||||
|
||||
async def get(self, market: str, path_key: str, params: dict[str, Any] | None = None) -> Any:
|
||||
conf = REST[market]
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
resp = await self.http.get(conf["base"] + conf[path_key], params=params)
|
||||
except httpx.TransportError as e:
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
logger.warning("币安请求网络错误（第 %d 次）：%r", attempt, e)
|
||||
await asyncio.sleep(2 * attempt)
|
||||
continue
|
||||
if resp.status_code in (418, 429):
|
||||
# 418 表示 IP 已被临时封禁，必须按 Retry-After 等待
|
||||
wait = int(resp.headers.get("Retry-After", "60"))
|
||||
logger.warning("币安限流 %d，暂停 %ds", resp.status_code, wait)
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
if resp.status_code >= 500 and attempt < MAX_RETRIES:
|
||||
await asyncio.sleep(2 * attempt)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
used = int(resp.headers.get("X-MBX-USED-WEIGHT-1M", "0"))
|
||||
if used > conf["weight_limit"] * WEIGHT_RATIO:
|
||||
wait = 61 - time.time() % 60
|
||||
logger.info("%s 权重已用 %d，等待 %.0fs 到下一分钟", market, used, wait)
|
||||
await asyncio.sleep(wait)
|
||||
return resp.json()
|
||||
raise RuntimeError(f"币安请求重试 {MAX_RETRIES} 次仍被限流：{path_key} {params}")
|
||||
|
||||
async def klines(self, market: str, symbol: str, interval: str, start_time: int, end_time: int | None = None) -> list[list]:
|
||||
params: dict[str, Any] = {"symbol": symbol, "interval": interval, "startTime": start_time, "limit": REST[market]["kline_limit"]}
|
||||
if end_time is not None:
|
||||
params["endTime"] = end_time
|
||||
return await self.get(market, "klines", params)
|
||||
|
||||
async def tradable_symbols(self, market: str) -> list[dict[str, Any]]:
|
||||
"""可交易的交易对；合约只保留永续（含 TRADIFI_PERPETUAL 等美股永续）"""
|
||||
info = await self.get(market, "exchange_info")
|
||||
result = []
|
||||
for s in info["symbols"]:
|
||||
if s["status"] != "TRADING" or (market == "um" and not s["contractType"].endswith("PERPETUAL")):
|
||||
continue
|
||||
filters = {f["filterType"]: f for f in s["filters"]}
|
||||
result.append({
|
||||
"symbol": s["symbol"],
|
||||
"base": s["baseAsset"],
|
||||
"quote": s["quoteAsset"],
|
||||
"price_precision": decimals(filters["PRICE_FILTER"]["tickSize"]),
|
||||
"volume_precision": decimals(filters["LOT_SIZE"]["stepSize"]),
|
||||
})
|
||||
return result
|
||||
|
||||
async def ticker(self, market: str, symbol: str) -> dict[str, Any]:
|
||||
return await self.get(market, "ticker", {"symbol": symbol})
|
||||
@@ -0,0 +1,9 @@
|
||||
from alphapulse.core.redis import KEY_PREFIX
|
||||
|
||||
|
||||
def kline_channel(market: str, symbol: str, interval: str) -> str:
|
||||
return f"{KEY_PREFIX}pub:kline:{market}:{symbol}:{interval}"
|
||||
|
||||
|
||||
def ticker_channel(market: str, symbol: str) -> str:
|
||||
return f"{KEY_PREFIX}pub:ticker:{market}:{symbol}"
|
||||
@@ -0,0 +1,224 @@
|
||||
"""行情进程：全历史补齐、断线/定时追平、币安 WS 实时推送。
|
||||
|
||||
用法：python -m alphapulse.market.worker
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import websockets
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.market.binance import REST, WS, BinanceClient
|
||||
from alphapulse.market.channels import kline_channel, ticker_channel
|
||||
from alphapulse.models import Kline, KlineSyncState, MarketSymbol
|
||||
from alphapulse.models.market import INTERVALS, MARKETS
|
||||
|
||||
logger = logging.getLogger("alphapulse.market.worker")
|
||||
|
||||
# 每个补齐任务连续拉取的页数，之后重新挑选优先级最高的任务（新加的交易对能插队补大周期）
|
||||
PAGES_PER_TURN = 20
|
||||
CATCHUP_SECONDS = 3600
|
||||
SYMBOL_CHECK_SECONDS = 10
|
||||
ACTIVE_STATUSES = ("pending", "backfilling", "ready")
|
||||
|
||||
client = BinanceClient()
|
||||
# WS（重新）连上后触发一次追平，补断线期间的缺口
|
||||
catchup_needed = asyncio.Event()
|
||||
|
||||
|
||||
def row_to_values(market: str, symbol: str, interval: str, row: list) -> dict:
|
||||
return {
|
||||
"market": market, "symbol": symbol, "interval": interval,
|
||||
"open_time": row[0], "open": row[1], "high": row[2], "low": row[3], "close": row[4], "volume": row[5],
|
||||
"close_time": row[6], "quote_volume": row[7], "trades": row[8], "taker_buy_base": row[9], "taker_buy_quote": row[10],
|
||||
}
|
||||
|
||||
|
||||
async def upsert_klines(values: list[dict]) -> None:
|
||||
stmt = pg_insert(Kline).values(values)
|
||||
updatable = ("open", "high", "low", "close", "volume", "close_time", "quote_volume", "trades", "taker_buy_base", "taker_buy_quote")
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[Kline.market, Kline.symbol, Kline.interval, Kline.open_time],
|
||||
set_={c: stmt.excluded[c] for c in updatable},
|
||||
)
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def sync_page(market: str, symbol: str, interval: str) -> bool:
|
||||
"""从上次同步位置拉一页并落库，返回是否已追平到最新"""
|
||||
async with SessionLocal() as session:
|
||||
state = await session.get(KlineSyncState, (market, symbol, interval))
|
||||
if state is None:
|
||||
state = KlineSyncState(market=market, symbol=symbol, interval=interval)
|
||||
session.add(state)
|
||||
# 从最后一根（可能未收盘）重新拉起；首次从 0 开始，币安会返回上市后的第一根
|
||||
rows = await client.klines(market, symbol, interval, state.latest_open_time or 0)
|
||||
if rows:
|
||||
await upsert_klines([row_to_values(market, symbol, interval, r) for r in rows])
|
||||
state.earliest_open_time = state.earliest_open_time or rows[0][0]
|
||||
state.latest_open_time = rows[-1][0]
|
||||
caught_up = len(rows) < REST[market]["kline_limit"]
|
||||
if caught_up:
|
||||
state.backfill_done = True
|
||||
await session.commit()
|
||||
return caught_up
|
||||
|
||||
|
||||
async def next_backfill_task(market: str) -> tuple[MarketSymbol, str] | None:
|
||||
async with SessionLocal() as session:
|
||||
symbols = (await session.scalars(
|
||||
select(MarketSymbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(("pending", "backfilling")))
|
||||
.order_by(MarketSymbol.added_at)
|
||||
)).all()
|
||||
done = set((await session.execute(
|
||||
select(KlineSyncState.symbol, KlineSyncState.interval)
|
||||
.where(KlineSyncState.market == market, KlineSyncState.backfill_done)
|
||||
)).all())
|
||||
unfinished = []
|
||||
for s in symbols:
|
||||
if all((s.symbol, interval) in done for interval in INTERVALS):
|
||||
await set_symbol_status(s.id, "ready")
|
||||
logger.info("%s %s 全部周期补齐完成", market, s.symbol)
|
||||
else:
|
||||
unfinished.append(s)
|
||||
for interval in INTERVALS:
|
||||
for s in unfinished:
|
||||
if (s.symbol, interval) not in done:
|
||||
return s, interval
|
||||
return None
|
||||
|
||||
|
||||
async def set_symbol_status(symbol_id: int, status: str, error: str | None = None) -> None:
|
||||
async with SessionLocal() as session:
|
||||
s = await session.get(MarketSymbol, symbol_id)
|
||||
s.status = status
|
||||
s.last_error = error
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def backfill_loop(market: str) -> None:
|
||||
while True:
|
||||
task = await next_backfill_task(market)
|
||||
if task is None:
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
s, interval = task
|
||||
if s.status == "pending":
|
||||
await set_symbol_status(s.id, "backfilling")
|
||||
logger.info("%s %s 开始补齐", market, s.symbol)
|
||||
try:
|
||||
for _ in range(PAGES_PER_TURN):
|
||||
if await sync_page(market, s.symbol, interval):
|
||||
logger.info("%s %s %s 补齐完成", market, s.symbol, interval)
|
||||
break
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 400:
|
||||
# 交易对不存在或已下架：标记错误，不再重试
|
||||
await set_symbol_status(s.id, "error", e.response.text[:500])
|
||||
logger.error("%s %s 补齐失败：%s", market, s.symbol, e.response.text[:200])
|
||||
else:
|
||||
logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
|
||||
await asyncio.sleep(10)
|
||||
except Exception:
|
||||
logger.exception("%s %s %s 补齐出错，稍后重试", market, s.symbol, interval)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
async def catchup_loop() -> None:
|
||||
"""已补齐的周期从上次 REST 同步位置追平到最新：每小时一次，WS 重连后立即一次"""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(catchup_needed.wait(), timeout=CATCHUP_SECONDS)
|
||||
except TimeoutError:
|
||||
pass
|
||||
catchup_needed.clear()
|
||||
async with SessionLocal() as session:
|
||||
states = (await session.execute(
|
||||
select(KlineSyncState.market, KlineSyncState.symbol, KlineSyncState.interval)
|
||||
.join(MarketSymbol, (MarketSymbol.market == KlineSyncState.market) & (MarketSymbol.symbol == KlineSyncState.symbol))
|
||||
.where(KlineSyncState.backfill_done, MarketSymbol.status.in_(ACTIVE_STATUSES))
|
||||
)).all()
|
||||
started = time.monotonic()
|
||||
for market, symbol, interval in states:
|
||||
try:
|
||||
while not await sync_page(market, symbol, interval):
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("%s %s %s 追平出错", market, symbol, interval)
|
||||
logger.info("追平完成：%d 个周期，用时 %.1fs", len(states), time.monotonic() - started)
|
||||
|
||||
|
||||
async def active_symbols(market: str) -> list[str]:
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.scalars(
|
||||
select(MarketSymbol.symbol).where(MarketSymbol.market == market, MarketSymbol.status.in_(ACTIVE_STATUSES))
|
||||
.order_by(MarketSymbol.symbol)
|
||||
)).all())
|
||||
|
||||
|
||||
async def handle_message(market: str, raw: str | bytes) -> None:
|
||||
data = json.loads(raw)["data"]
|
||||
symbol = data["s"]
|
||||
if data["e"] == "kline":
|
||||
k = data["k"]
|
||||
payload = {"t": k["t"], "o": k["o"], "h": k["h"], "l": k["l"], "c": k["c"], "v": k["v"], "q": k["q"], "x": k["x"]}
|
||||
await redis.publish(kline_channel(market, symbol, k["i"]), json.dumps(payload))
|
||||
# 收盘 K 线入库；1s 只推送不落库
|
||||
if k["x"] and k["i"] != "1s":
|
||||
await upsert_klines([{
|
||||
"market": market, "symbol": symbol, "interval": k["i"],
|
||||
"open_time": k["t"], "open": k["o"], "high": k["h"], "low": k["l"], "close": k["c"], "volume": k["v"],
|
||||
"close_time": k["T"], "quote_volume": k["q"], "trades": k["n"], "taker_buy_base": k["V"], "taker_buy_quote": k["Q"],
|
||||
}])
|
||||
elif data["e"] == "24hrMiniTicker":
|
||||
payload = {"c": data["c"], "o": data["o"], "h": data["h"], "l": data["l"], "v": data["v"], "q": data["q"], "E": data["E"]}
|
||||
await redis.publish(ticker_channel(market, symbol), json.dumps(payload))
|
||||
|
||||
|
||||
async def stream_loop(market: str) -> None:
|
||||
while True:
|
||||
symbols = await active_symbols(market)
|
||||
if not symbols:
|
||||
await asyncio.sleep(SYMBOL_CHECK_SECONDS)
|
||||
continue
|
||||
intervals = INTERVALS + (["1s"] if market == "spot" else [])
|
||||
streams = [f"{s.lower()}@kline_{i}" for s in symbols for i in intervals] + [f"{s.lower()}@miniTicker" for s in symbols]
|
||||
try:
|
||||
async with websockets.connect(f"{WS[market]}?streams={'/'.join(streams)}", max_size=2**22) as ws:
|
||||
logger.info("%s WS 已连接：%d 个交易对，%d 个流", market, len(symbols), len(streams))
|
||||
catchup_needed.set()
|
||||
next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
|
||||
async for raw in ws:
|
||||
await handle_message(market, raw)
|
||||
if time.monotonic() >= next_check:
|
||||
next_check = time.monotonic() + SYMBOL_CHECK_SECONDS
|
||||
if await active_symbols(market) != symbols:
|
||||
logger.info("%s 交易对列表变化，重新订阅", market)
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("%s WS 断开，5s 后重连", market)
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
await asyncio.gather(
|
||||
*(backfill_loop(m) for m in MARKETS),
|
||||
*(stream_loop(m) for m in MARKETS),
|
||||
catchup_loop(),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
# httpx 每个请求一行 INFO，补齐时刷屏
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,17 @@
|
||||
from alphapulse.core.db import Base
|
||||
from alphapulse.models.market import Kline, KlineSyncState, MarketSymbol
|
||||
from alphapulse.models.news import CrawlLog, CrawlRun, NewsItem, NewsRevision, RawBody, RawResponse, Source
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"CrawlLog",
|
||||
"CrawlRun",
|
||||
"Kline",
|
||||
"KlineSyncState",
|
||||
"MarketSymbol",
|
||||
"NewsItem",
|
||||
"NewsRevision",
|
||||
"RawBody",
|
||||
"RawResponse",
|
||||
"Source",
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, Integer, Numeric, PrimaryKeyConstraint, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from alphapulse.core.db import Base
|
||||
|
||||
# 落库的周期，顺序即补齐优先级（大周期先补，图表最先可看）；1s 只做实时推送不落库
|
||||
INTERVALS = ["1M", "1w", "3d", "1d", "12h", "8h", "6h", "4h", "2h", "1h", "30m", "15m", "5m", "3m", "1m"]
|
||||
MARKETS = ("spot", "um")
|
||||
|
||||
|
||||
class MarketSymbol(Base):
|
||||
__tablename__ = "market_symbols"
|
||||
__table_args__ = (UniqueConstraint("market", "symbol"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# spot 现货 / um U 本位永续
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
base: Mapped[str] = mapped_column(String(16))
|
||||
quote: Mapped[str] = mapped_column(String(16))
|
||||
# 由币安 tickSize / stepSize 推出的小数位，用于图表显示
|
||||
price_precision: Mapped[int] = mapped_column(Integer)
|
||||
volume_precision: Mapped[int] = mapped_column(Integer)
|
||||
# pending / backfilling / ready / error / delisted
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending")
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class Kline(Base):
|
||||
"""时间字段沿用币安的毫秒时间戳；表按 interval 做 LIST 分区（分区在迁移中创建）"""
|
||||
|
||||
__tablename__ = "klines"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("market", "symbol", "interval", "open_time"),
|
||||
{"postgresql_partition_by": "LIST (interval)"},
|
||||
)
|
||||
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
interval: Mapped[str] = mapped_column(String(4))
|
||||
open_time: Mapped[int] = mapped_column(BigInteger)
|
||||
open: Mapped[Decimal] = mapped_column(Numeric)
|
||||
high: Mapped[Decimal] = mapped_column(Numeric)
|
||||
low: Mapped[Decimal] = mapped_column(Numeric)
|
||||
close: Mapped[Decimal] = mapped_column(Numeric)
|
||||
volume: Mapped[Decimal] = mapped_column(Numeric)
|
||||
close_time: Mapped[int] = mapped_column(BigInteger)
|
||||
quote_volume: Mapped[Decimal] = mapped_column(Numeric)
|
||||
trades: Mapped[int] = mapped_column(BigInteger)
|
||||
taker_buy_base: Mapped[Decimal] = mapped_column(Numeric)
|
||||
taker_buy_quote: Mapped[Decimal] = mapped_column(Numeric)
|
||||
|
||||
|
||||
class KlineSyncState(Base):
|
||||
"""REST 同步进度：实时 WS 写入不更新这里，所以从 latest_open_time 往后追平即可补上断线缺口"""
|
||||
|
||||
__tablename__ = "kline_sync_state"
|
||||
__table_args__ = (PrimaryKeyConstraint("market", "symbol", "interval"),)
|
||||
|
||||
market: Mapped[str] = mapped_column(String(8))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
interval: Mapped[str] = mapped_column(String(4))
|
||||
earliest_open_time: Mapped[int | None] = mapped_column(BigInteger)
|
||||
latest_open_time: Mapped[int | None] = mapped_column(BigInteger)
|
||||
# 首次全历史补齐是否完成
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -0,0 +1,122 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from alphapulse.core.db import Base
|
||||
|
||||
|
||||
class Source(Base):
|
||||
__tablename__ = "sources"
|
||||
|
||||
code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(64))
|
||||
list_url: Mapped[str] = mapped_column(String(255))
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
interval_seconds: Mapped[int] = mapped_column(Integer)
|
||||
backfill_hours: Mapped[int] = mapped_column(Integer, default=72)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class NewsItem(Base):
|
||||
__tablename__ = "news_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_code", "external_id"),
|
||||
Index("ix_news_items_published_at", "published_at"),
|
||||
Index("ix_news_items_source_published_at", "source_code", "published_at"),
|
||||
Index("ix_news_items_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
|
||||
Index("ix_news_items_content_trgm", "content_text", postgresql_using="gin", postgresql_ops={"content_text": "gin_trgm_ops"}),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
external_id: Mapped[str] = mapped_column(String(64))
|
||||
url: Mapped[str] = mapped_column(String(512))
|
||||
title: Mapped[str] = mapped_column(Text, default="")
|
||||
summary: Mapped[str] = mapped_column(Text, default="")
|
||||
content_html: Mapped[str] = mapped_column(Text, default="")
|
||||
content_text: Mapped[str] = mapped_column(Text, default="")
|
||||
author: Mapped[str | None] = mapped_column(String(128))
|
||||
tags: Mapped[list] = mapped_column(JSONB, default=list)
|
||||
is_paywalled: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
content_hash: Mapped[str] = mapped_column(String(64))
|
||||
revision_no: Mapped[int] = mapped_column(Integer, default=1)
|
||||
extra: Mapped[dict] = mapped_column(JSONB, default=dict)
|
||||
|
||||
|
||||
class NewsRevision(Base):
|
||||
__tablename__ = "news_revisions"
|
||||
__table_args__ = (UniqueConstraint("news_id", "revision_no"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
news_id: Mapped[int] = mapped_column(ForeignKey("news_items.id"))
|
||||
revision_no: Mapped[int] = mapped_column(Integer)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
summary: Mapped[str] = mapped_column(Text)
|
||||
content_html: Mapped[str] = mapped_column(Text)
|
||||
content_hash: Mapped[str] = mapped_column(String(64))
|
||||
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
|
||||
|
||||
|
||||
class RawBody(Base):
|
||||
__tablename__ = "raw_bodies"
|
||||
|
||||
sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# gzip 压缩后的原始响应体；同内容只存一份
|
||||
body: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class RawResponse(Base):
|
||||
__tablename__ = "raw_responses"
|
||||
__table_args__ = (Index("ix_raw_responses_source_fetched_at", "source_code", "fetched_at"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
method: Mapped[str] = mapped_column(String(8))
|
||||
status: Mapped[int] = mapped_column(Integer)
|
||||
headers: Mapped[dict] = mapped_column(JSONB)
|
||||
body_sha256: Mapped[str] = mapped_column(ForeignKey("raw_bodies.sha256"))
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class CrawlRun(Base):
|
||||
__tablename__ = "crawl_runs"
|
||||
__table_args__ = (Index("ix_crawl_runs_source_started_at", "source_code", "started_at"),)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
# schedule / startup / restart
|
||||
trigger: Mapped[str] = mapped_column(String(16))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# running / success / failed / interrupted
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
requests: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_seen: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_new: Mapped[int] = mapped_column(Integer, default=0)
|
||||
items_revised: Mapped[int] = mapped_column(Integer, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class CrawlLog(Base):
|
||||
__tablename__ = "crawl_logs"
|
||||
__table_args__ = (Index("ix_crawl_logs_source_id", "source_code", "id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
||||
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
Generated
+2340
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user