eezd
80d266dba7
Group original news into tagged fact packages, review each package with evidence and read-only research, and build complete 72-hour reading lists with independent model selections. Add paused RSS/Atom subscriptions, news-scoped Skills, versioned analysis records, and locked deployment support for SearXNG. Preserve original news and keep live migration and legacy AI cleanup separately authorized. Validate with 604 backend tests, 49 frontend tests, 19 CI control tests, lint, type checks, production build, and disposable PostgreSQL migration and three-stage smoke checks.
130 lines
6.5 KiB
Python
130 lines
6.5 KiB
Python
import uuid
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, func
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from alphapulse.core.db import Base
|
|
|
|
|
|
class Source(Base):
|
|
__tablename__ = "sources"
|
|
|
|
code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(64))
|
|
list_url: Mapped[str] = mapped_column(String(255))
|
|
# 新闻管理器只负责 news；日历专属来源不启动 Scrapy 子进程。
|
|
adapter: Mapped[str] = mapped_column(String(16), default="builtin", server_default="builtin")
|
|
collector: Mapped[str] = mapped_column(String(16), default="news", server_default="news")
|
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
interval_seconds: Mapped[int] = mapped_column(Integer)
|
|
backfill_hours: Mapped[int] = mapped_column(Integer, default=72)
|
|
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
|
|
last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
last_error: Mapped[str | None] = mapped_column(Text)
|
|
paused_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# 历史回补：目标时间（为空表示不回补）、已回补到的最早时间（断点）、是否已结束（到达目标或来源无更早数据）
|
|
history_target: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
history_position: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
history_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
|
|
|
|
class NewsItem(Base):
|
|
__tablename__ = "news_items"
|
|
__table_args__ = (
|
|
UniqueConstraint("source_code", "external_id"),
|
|
Index("ix_news_items_published_at", "published_at"),
|
|
Index("ix_news_items_source_published_at", "source_code", "published_at"),
|
|
Index("ix_news_items_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
|
|
Index("ix_news_items_content_trgm", "content_text", postgresql_using="gin", postgresql_ops={"content_text": "gin_trgm_ops"}),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
|
external_id: Mapped[str] = mapped_column(String(64))
|
|
url: Mapped[str] = mapped_column(String(512))
|
|
title: Mapped[str] = mapped_column(Text, default="")
|
|
summary: Mapped[str] = mapped_column(Text, default="")
|
|
content_html: Mapped[str] = mapped_column(Text, default="")
|
|
content_text: Mapped[str] = mapped_column(Text, default="")
|
|
author: Mapped[str | None] = mapped_column(String(128))
|
|
tags: Mapped[list] = mapped_column(JSONB, default=list)
|
|
is_paywalled: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
content_hash: Mapped[str] = mapped_column(String(64))
|
|
revision_no: Mapped[int] = mapped_column(Integer, default=1)
|
|
extra: Mapped[dict] = mapped_column(JSONB, default=dict)
|
|
|
|
|
|
class NewsRevision(Base):
|
|
__tablename__ = "news_revisions"
|
|
__table_args__ = (UniqueConstraint("news_id", "revision_no"),)
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
news_id: Mapped[int] = mapped_column(ForeignKey("news_items.id"))
|
|
revision_no: Mapped[int] = mapped_column(Integer)
|
|
title: Mapped[str] = mapped_column(Text)
|
|
summary: Mapped[str] = mapped_column(Text)
|
|
content_html: Mapped[str] = mapped_column(Text)
|
|
content_hash: Mapped[str] = mapped_column(String(64))
|
|
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
raw_response_id: Mapped[int | None] = mapped_column(ForeignKey("raw_responses.id"))
|
|
|
|
|
|
class RawBody(Base):
|
|
__tablename__ = "raw_bodies"
|
|
|
|
sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
# gzip 压缩后的原始响应体；同内容只存一份
|
|
body: Mapped[bytes] = mapped_column(LargeBinary)
|
|
|
|
|
|
class RawResponse(Base):
|
|
__tablename__ = "raw_responses"
|
|
__table_args__ = (Index("ix_raw_responses_source_fetched_at", "source_code", "fetched_at"),)
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
|
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
|
url: Mapped[str] = mapped_column(Text)
|
|
method: Mapped[str] = mapped_column(String(8))
|
|
status: Mapped[int] = mapped_column(Integer)
|
|
headers: Mapped[dict] = mapped_column(JSONB)
|
|
body_sha256: Mapped[str] = mapped_column(ForeignKey("raw_bodies.sha256"))
|
|
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
|
|
|
|
class CrawlRun(Base):
|
|
__tablename__ = "crawl_runs"
|
|
__table_args__ = (Index("ix_crawl_runs_source_started_at", "source_code", "started_at"),)
|
|
|
|
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
|
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
|
# schedule / startup / restart
|
|
trigger: Mapped[str] = mapped_column(String(16))
|
|
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# running / success / failed / interrupted
|
|
status: Mapped[str] = mapped_column(String(16))
|
|
requests: Mapped[int] = mapped_column(Integer, default=0)
|
|
items_seen: Mapped[int] = mapped_column(Integer, default=0)
|
|
items_new: Mapped[int] = mapped_column(Integer, default=0)
|
|
items_revised: Mapped[int] = mapped_column(Integer, default=0)
|
|
error: Mapped[str | None] = mapped_column(Text)
|
|
|
|
|
|
class CrawlLog(Base):
|
|
__tablename__ = "crawl_logs"
|
|
__table_args__ = (Index("ix_crawl_logs_source_id", "source_code", "id"),)
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
run_id: Mapped[uuid.UUID | None] = mapped_column()
|
|
source_code: Mapped[str] = mapped_column(ForeignKey("sources.code"))
|
|
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
level: Mapped[str] = mapped_column(String(8))
|
|
message: Mapped[str] = mapped_column(Text)
|