Add official US and European collection and calendar storage
Collect official RSS and Atom summaries, preserve historical captures, and poll macro releases with bounded cadence and Retry-After handling. Separate news collectors from the economic calendar worker. Track statistical periods, publication stages, first observations, revisions, and current official series. Include explicit migration files and isolated archive, parsing, and import tests; no migration or import is executed by this commit. Keep dependency declarations and lockfile together for PDF archives and the upcoming captcha integration. Verify the staged snapshot with 90 focused tests.
This commit is contained in:
eezd committed
1 parent
5fa1e37fd4
commit
e8ca4daa0e
76 files changed
+18923
-4
No files matched your search
@@ -0,0 +1,37 @@
|
||||
"""新增国外新闻和宏观公告来源。
|
||||
|
||||
Revision ID: e6f8a2c4d109
|
||||
Revises: d2a4c6e8f103
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "e6f8a2c4d109"
|
||||
down_revision = "d2a4c6e8f103"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
sources = sa.table(
|
||||
"sources",
|
||||
sa.column("code", sa.String), sa.column("name", sa.String), sa.column("list_url", sa.String),
|
||||
sa.column("enabled", sa.Boolean), sa.column("interval_seconds", sa.Integer),
|
||||
sa.column("backfill_hours", sa.Integer), sa.column("consecutive_failures", sa.Integer),
|
||||
)
|
||||
op.bulk_insert(sources, [
|
||||
{"code": code, "name": name, "list_url": url, "enabled": enabled,
|
||||
"interval_seconds": 60, "backfill_hours": 72, "consecutive_failures": 0}
|
||||
for code, name, url, enabled in [
|
||||
("coindesk", "CoinDesk", "https://www.coindesk.com/arc/outboundfeeds/rss/", True),
|
||||
("fed", "美联储·政策与讲话", "https://www.federalreserve.gov/feeds/feeds.htm", True),
|
||||
("bls", "美国劳工统计局·CPI/PPI/就业", "https://www.bls.gov/feed/", True),
|
||||
("bea", "美国经济分析局", "https://apps.bea.gov/rss/rss.xml", True),
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 已有采集记录时由外键拒绝回退，不级联删除新闻、原始响应或日志。
|
||||
op.execute("DELETE FROM sources WHERE code IN ('coindesk', 'fed', 'bls', 'bea')")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""日历统计期、发布阶段及当前官方历史序列。
|
||||
|
||||
Revision ID: a8c1d4e7f230
|
||||
Revises: f7a9b3d5e210
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "a8c1d4e7f230"
|
||||
down_revision = "f7a9b3d5e210"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("economic_releases", sa.Column("period", sa.String(10)))
|
||||
op.add_column("economic_releases", sa.Column("stage", sa.String(16)))
|
||||
op.add_column("sources", sa.Column("collector", sa.String(16), nullable=False, server_default="news"))
|
||||
sources = sa.table("sources", *[sa.column(k, t) for k, t in [
|
||||
("code", sa.String), ("name", sa.String), ("list_url", sa.String), ("collector", sa.String),
|
||||
("enabled", sa.Boolean), ("interval_seconds", sa.Integer), ("backfill_hours", sa.Integer),
|
||||
("consecutive_failures", sa.Integer),
|
||||
]])
|
||||
op.bulk_insert(sources, [{"code": "umich", "name": "密歇根大学·经济日历", "collector": "calendar",
|
||||
"list_url": "https://www.sca.isr.umich.edu/", "enabled": True,
|
||||
"interval_seconds": 3600, "backfill_hours": 0, "consecutive_failures": 0}])
|
||||
op.create_table(
|
||||
"economic_series_values",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("indicator", sa.String(48), nullable=False),
|
||||
sa.Column("period", sa.String(10), nullable=False),
|
||||
sa.Column("stage", sa.String(16), nullable=False),
|
||||
sa.Column("value", sa.String(64), nullable=False),
|
||||
sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("raw_response_id", sa.BigInteger, sa.ForeignKey("raw_responses.id"), nullable=False),
|
||||
sa.UniqueConstraint("source", "indicator", "period", "stage"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 已有原始响应时外键拒绝删除来源，不级联删除用户数据。
|
||||
op.execute("DELETE FROM sources WHERE code = 'umich'")
|
||||
op.drop_table("economic_series_values")
|
||||
op.drop_column("sources", "collector")
|
||||
op.drop_column("economic_releases", "stage")
|
||||
op.drop_column("economic_releases", "period")
|
||||
@@ -0,0 +1,54 @@
|
||||
"""美国经济日历与原始公布修订。
|
||||
|
||||
Revision ID: f7a9b3d5e210
|
||||
Revises: e6f8a2c4d109
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "f7a9b3d5e210"
|
||||
down_revision = "e6f8a2c4d109"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"economic_releases",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("external_id", sa.String(160), nullable=False),
|
||||
sa.Column("family", sa.String(16), nullable=False),
|
||||
sa.Column("title", sa.Text, nullable=False),
|
||||
sa.Column("scheduled_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("schedule_status", sa.String(16), nullable=False),
|
||||
sa.Column("schedule_url", sa.Text, nullable=False),
|
||||
sa.Column("release_url", sa.Text),
|
||||
sa.Column("schedule_checked_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("first_received_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_checked_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("next_check_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("error", sa.Text),
|
||||
sa.Column("values", postgresql.JSONB, nullable=False),
|
||||
sa.Column("revision_no", sa.Integer, nullable=False),
|
||||
sa.UniqueConstraint("source", "external_id"),
|
||||
)
|
||||
op.create_index("ix_economic_releases_scheduled_at", "economic_releases", ["scheduled_at"])
|
||||
op.create_index("ix_economic_releases_next_check_at", "economic_releases", ["next_check_at"])
|
||||
op.create_table(
|
||||
"economic_revisions",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("release_id", sa.Integer, sa.ForeignKey("economic_releases.id"), nullable=False),
|
||||
sa.Column("revision_no", sa.Integer, nullable=False),
|
||||
sa.Column("payload", postgresql.JSONB, nullable=False),
|
||||
sa.Column("raw_response_id", sa.BigInteger, sa.ForeignKey("raw_responses.id"), nullable=False),
|
||||
sa.Column("captured_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.UniqueConstraint("release_id", "revision_no"),
|
||||
)
|
||||
op.create_index("ix_economic_revisions_release_id", "economic_revisions", ["release_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("economic_revisions")
|
||||
op.drop_table("economic_releases")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""欧洲官方新闻与经济日历来源。
|
||||
|
||||
Revision ID: b9d2e5f8a341
|
||||
Revises: a8c1d4e7f230
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "b9d2e5f8a341"
|
||||
down_revision = "a8c1d4e7f230"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
sources = sa.table("sources", *[sa.column(k, t) for k, t in [
|
||||
("code", sa.String), ("name", sa.String), ("list_url", sa.String), ("collector", sa.String),
|
||||
("enabled", sa.Boolean), ("interval_seconds", sa.Integer), ("backfill_hours", sa.Integer),
|
||||
("consecutive_failures", sa.Integer),
|
||||
]])
|
||||
op.bulk_insert(sources, [
|
||||
{"code": code, "name": name, "list_url": url, "collector": "news", "enabled": True,
|
||||
"interval_seconds": 60, "backfill_hours": 72, "consecutive_failures": 0}
|
||||
for code, name, url in [
|
||||
("ecb", "欧洲央行·政策与讲话", "https://www.ecb.europa.eu/home/html/rss.en.html"),
|
||||
("eurostat", "Eurostat·欧元区统计", "https://ec.europa.eu/eurostat/news/euro-indicators"),
|
||||
("boe", "英国央行·政策与讲话", "https://www.bankofengland.co.uk/rss"),
|
||||
("ons", "英国国家统计局", "https://www.ons.gov.uk/releasecalendar"),
|
||||
("destatis", "德国联邦统计局", "https://www.destatis.de/DE/Presse/_inhalt.html"),
|
||||
("insee", "法国国家统计局", "https://www.insee.fr/fr/statistiques?categorie=10"),
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 已有采集记录时由外键拒绝删除，不级联清理新闻或原始响应。
|
||||
op.execute("DELETE FROM sources WHERE code IN ('ecb','eurostat','boe','ons','destatis','insee')")
|
||||
Reference in new issue
Block a user