Prioritize realtime news and make historical backfill resumable
Development CI / check-and-build (push) Successful in 3m0s
GitHub Git mirror / mirror (push) Successful in 5s
Development CI / deploy-development (push) Successful in 14m7s

This commit is contained in:
eezd committed 2026-11-06 14:28:39 +08:00
1 parent c90691d9a0
commit 5f347f8ab4
17 files changed
+697 -143

No files matched your search

@@ -0,0 +1,24 @@
"""Persist realtime boundaries and enforce one active call per subject."""
from alembic import op
import sqlalchemy as sa
revision = 'dd4567890123'
down_revision = 'dc3456789012'
branch_labels = None
depends_on = None
def upgrade():
op.add_column('news_research_settings', sa.Column('realtime_from', sa.DateTime(timezone=True)))
op.add_column('news_research_events', sa.Column('realtime_pending', sa.Boolean(), nullable=False, server_default=sa.false()))
op.create_index('uq_news_attempt_active', 'news_research_attempts', ['package_id'], unique=True,
postgresql_where=sa.text("state = 'processing'"))
op.create_index('uq_news_version_active', 'news_research_versions', ['event_id'], unique=True,
postgresql_where=sa.text("state = 'processing'"))
def downgrade():
op.drop_index('uq_news_version_active', table_name='news_research_versions')
op.drop_index('uq_news_attempt_active', table_name='news_research_attempts')
op.drop_column('news_research_events', 'realtime_pending')
op.drop_column('news_research_settings', 'realtime_from')
+42 -16
View File
@@ -20,8 +20,9 @@ from alphapulse.news_research.models import (
Version, Version,
Reference, Reference,
BatchEvent, BatchEvent,
Batch,
) )
from alphapulse.news_research import service, batches, vocabulary as vocab from alphapulse.news_research import service, batches, scheduling, vocabulary as vocab
from alphapulse.news_research.models import VocabularyVersion from alphapulse.news_research.models import VocabularyVersion
from alphapulse.news_research.contracts import ( from alphapulse.news_research.contracts import (
INDEX_ASSIGN_SKILL, INDEX_ASSIGN_SKILL,
@@ -90,6 +91,8 @@ class SettingsInput(Strict):
index: ModelConfig index: ModelConfig
research: ModelConfig research: ModelConfig
concurrency: int = Field(default=3, ge=1, le=16) concurrency: int = Field(default=3, ge=1, le=16)
start_at: AwareDatetime | None = None
base_revision: int | None = Field(default=None, ge=0)
async def settings_row(session): async def settings_row(session):
@@ -101,7 +104,7 @@ async def settings_row(session):
id=1, id=1,
config={}, config={},
enabled=False, enabled=False,
start_at=datetime(2026, 10, 31, 16, tzinfo=UTC), start_at=datetime(2026, 11, 4, 16, tzinfo=UTC),
) )
.on_conflict_do_nothing(index_elements=["id"]) .on_conflict_do_nothing(index_elements=["id"])
) )
@@ -115,6 +118,8 @@ async def settings(session: DB):
row = await settings_row(session) row = await settings_row(session)
return success( return success(
{"config": row.config, "enabled": row.enabled, "start_at": row.start_at, {"config": row.config, "enabled": row.enabled, "start_at": row.start_at,
"control_revision": row.control_revision, "realtime_from": row.realtime_from,
"timezone": "Asia/Shanghai", "queue": await scheduling.queue_status(session, row),
"batch": await batches.progress(session)} "batch": await batches.progress(session)}
) )
@@ -125,21 +130,24 @@ async def coverage(session: DB, start_at: AwareDatetime | None = None):
start = (start_at if start_at is not None else row.start_at).astimezone(UTC) start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
if start > datetime.now(UTC): if start > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间") raise HTTPException(422, "回补起始时间不能晚于当前时间")
return success(await batches.coverage(session, start)) effective = min(start, row.realtime_from) if row.realtime_from else start
report = await batches.coverage(session, effective)
return success({**report, 'backfill_start_at': start, 'realtime_from': row.realtime_from})
@router.put("/settings") @router.put("/settings")
async def save(body: SettingsInput, session: DB): async def save(body: SettingsInput, session: DB):
await settings_row(session) await settings_row(session)
await session.execute( row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
update(ResearchSettings) if body.base_revision is not None and body.base_revision != row.control_revision:
.where(ResearchSettings.id == 1) raise HTTPException(409, "设置已变化,请重新读取后保存")
.values( if body.start_at is not None:
config=body.model_dump(), if body.start_at > datetime.now(UTC):
enabled=False, raise HTTPException(422, "回补起始时间不能晚于当前时间")
control_revision=ResearchSettings.control_revision + 1, row.start_at = body.start_at.astimezone(UTC)
) row.config = body.model_dump(exclude={"start_at", "base_revision"})
) row.enabled = False
row.control_revision += 1
await session.commit() await session.commit()
return success(None) return success(None)
@@ -164,6 +172,8 @@ class EnableInput(Strict):
async def enable(session: DB, body: EnableInput | None = None): async def enable(session: DB, body: EnableInput | None = None):
row = await settings_row(session) row = await settings_row(session)
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC) start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
if start_at != row.start_at:
raise HTTPException(422, "请在模型与 Skills 中保存回补时间并暂停后再启用")
if start_at > datetime.now(UTC): if start_at > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间") raise HTTPException(422, "回补起始时间不能晚于当前时间")
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")): if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
@@ -172,11 +182,11 @@ async def enable(session: DB, body: EnableInput | None = None):
raise HTTPException(409, "请先暂停两阶段处理") raise HTTPException(409, "请先暂停两阶段处理")
running = await session.scalar( running = await session.scalar(
select(func.count()) select(func.count())
.select_from(Package) .select_from(Attempt)
.where(Package.state.in_(["index_processing", "assign_processing"])) .where(Attempt.state == "processing")
) )
running += await session.scalar( running += await session.scalar(
select(func.count()).select_from(Event).where(Event.state == "processing") select(func.count()).select_from(Version).where(Version.state == "processing")
) )
if running: if running:
raise HTTPException(409, "请等待已接收任务完成后再验证") raise HTTPException(409, "请等待已接收任务完成后再验证")
@@ -259,7 +269,10 @@ async def enable(session: DB, body: EnableInput | None = None):
if row.control_revision != control_revision or row.config != original_config: if row.control_revision != control_revision or row.config != original_config:
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证") raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
report = await batches.coverage(session, start_at) report = await batches.coverage(session, start_at)
row.start_at = start_at if row.realtime_from is None:
row.realtime_from = datetime.now(UTC)
await session.execute(update(Batch).where(Batch.phase.in_(["assigning", "researching"]))
.values(phase="rescheduled", finished_at=datetime.now(UTC)))
row.enabled = True row.enabled = True
row.control_revision += 1 row.control_revision += 1
await session.commit() await session.commit()
@@ -480,6 +493,18 @@ async def event(identifier: int, session: DB):
.order_by(Version.id.desc()) .order_by(Version.id.desc())
) )
) )
wait_reason = None
if row.state == 'pending':
settings = await settings_row(session)
unfinished = [(p, n) for p, n in members if p.state != 'done' or p.content_hash != n.content_hash]
if any(n.published_at < scheduling.boundary(settings) for p, n in unfinished):
wait_reason = '范围外成员尚未完成,扩大回补范围后可继续研究'
elif any(p.state.endswith('_failed') for p, n in unfinished):
wait_reason = '成员包处理失败,需手动重试后才能研究'
elif unfinished:
wait_reason = '等待成员包完成关键词与事件归属'
elif row.due_at > datetime.now(UTC):
wait_reason = '正在汇集事件变化,聚合窗口结束后研究'
latest = next((v for v in versions if v.state == "done"), None) latest = next((v for v in versions if v.state == "done"), None)
references = await version_references(session, latest.id) if latest else [] references = await version_references(session, latest.id) if latest else []
return success( return success(
@@ -490,6 +515,7 @@ async def event(identifier: int, session: DB):
"result": row.result, "result": row.result,
"error": row.error, "error": row.error,
"revision": row.revision, "revision": row.revision,
"wait_reason": wait_reason,
"packages": [await package_out(session, p, n) for p, n in members], "packages": [await package_out(session, p, n) for p, n in members],
"references": references, "references": references,
"versions": [ "versions": [
+40 -16
View File
@@ -5,6 +5,7 @@ from sqlalchemy import select, or_, and_, exists, func, any_, case
from alphapulse.models import NewsItem from alphapulse.models import NewsItem
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
from alphapulse.news_research.service import enroll from alphapulse.news_research.service import enroll
from alphapulse.news_research.scheduling import scoped_event, ready_event
async def coverage(session, start_at, cutoff=None): async def coverage(session, start_at, cutoff=None):
@@ -48,13 +49,22 @@ async def coverage(session, start_at, cutoff=None):
async def freeze(session, settings): async def freeze(session, settings):
cutoff = datetime.now(UTC) cutoff = datetime.now(UTC)
ids = list(await session.scalars(select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where( scope = [NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff]
or_(and_(NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff, Package.id.is_(None)), realtime_from = getattr(settings, 'realtime_from', None)
and_(Package.id.is_not(None), Package.content_hash != NewsItem.content_hash))))) if realtime_from is not None:
existing_ids = list(await session.scalars(select(Package.id).where( scope.append(NewsItem.published_at < realtime_from)
Package.state.in_(['index_pending', 'assign_pending', 'index_failed', 'assign_failed'])))) candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
event_cutoff = list(await session.scalars(select(Event.id).where(Event.state == "pending"))) *scope, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash,
Package.state.in_(['index_pending', 'assign_pending'])))
ids = list(await session.scalars(candidates.order_by(NewsItem.published_at, NewsItem.id).limit(30)))
await enroll(session, ids) await enroll(session, ids)
existing_ids = list(await session.scalars(select(Package.id).where(Package.news_id.in_(ids))))
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
event_query = select(Event.id).where(Event.state == 'pending', scoped_event(settings), ready_event(), Event.due_at <= cutoff, ~deferred)
if realtime_from is not None:
event_query = event_query.where(~Event.realtime_pending)
event_cutoff = list(await session.scalars(event_query.order_by(Event.due_at, Event.id).limit(30)))
# Failed items are reported once; only explicit retries or new source revisions rerun them. # Failed items are reported once; only explicit retries or new source revisions rerun them.
previous_failure = exists(select(BatchPackage.batch_id).where( previous_failure = exists(select(BatchPackage.batch_id).where(
BatchPackage.package_id == Package.id, BatchPackage.content_hash == Package.content_hash, BatchPackage.package_id == Package.id, BatchPackage.content_hash == Package.content_hash,
@@ -63,8 +73,6 @@ async def freeze(session, settings):
or_(Package.id == any_(existing_ids), Package.news_id == any_(ids)), or_( or_(Package.id == any_(existing_ids), Package.news_id == any_(ids)), or_(
Package.state.in_(['index_pending', 'assign_pending']), Package.state.in_(['index_pending', 'assign_pending']),
Package.state.in_(['index_failed', 'assign_failed']) & ~previous_failure)))) Package.state.in_(['index_failed', 'assign_failed']) & ~previous_failure))))
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
events = list(await session.scalars(select(Event).where(Event.id == any_(event_cutoff), Event.state == 'pending', ~deferred))) events = list(await session.scalars(select(Event).where(Event.id == any_(event_cutoff), Event.state == 'pending', ~deferred)))
if not packages and not events: if not packages and not events:
return None return None
@@ -83,7 +91,7 @@ async def freeze(session, settings):
return batch return batch
async def actions(session, batch, active, slots): async def actions(session, batch, active, slots, settings=None):
"""Return stage/ID pairs; callers commit before launching and never overlap phases.""" """Return stage/ID pairs; callers commit before launching and never overlap phases."""
if batch.phase == 'assigning': if batch.phase == 'assigning':
rows = list(await session.execute(select(BatchPackage, Package, NewsItem).join( rows = list(await session.execute(select(BatchPackage, Package, NewsItem).join(
@@ -94,7 +102,9 @@ async def actions(session, batch, active, slots):
for item, package, news in rows: for item, package, news in rows:
if item.state != 'pending': if item.state != 'pending':
continue continue
if item.content_hash != news.content_hash or item.content_hash != package.content_hash: if settings is not None and news.published_at < settings.start_at:
item.state = 'range_paused'
elif item.content_hash != news.content_hash or item.content_hash != package.content_hash:
item.state = 'deferred' item.state = 'deferred'
elif package.state == 'done': elif package.state == 'done':
item.state = 'done' item.state = 'done'
@@ -102,10 +112,12 @@ async def actions(session, batch, active, slots):
item.state = 'failed' item.state = 'failed'
elif package.state in ('index_pending', 'assign_pending'): elif package.state in ('index_pending', 'assign_pending'):
stage = 'index' if package.state == 'index_pending' else 'assign' stage = 'index' if package.state == 'index_pending' else 'assign'
if (stage, package.id) not in active and len(result) < slots and (stage != 'assign' or not assigning): if not any(identifier == package.id and task != 'event' for task, identifier in active) and len(result) < slots and (stage != 'assign' or not assigning):
result.append((stage, package.id)) result.append((stage, package.id))
assigning |= stage == 'assign' assigning |= stage == 'assign'
if any(item.state == 'pending' for item, _, _ in rows) or active: package_ids = {package.id for _, package, _ in rows}
batch_active = any(stage != 'event' and identifier in package_ids for stage, identifier in active)
if any(item.state == 'pending' for item, _, _ in rows) or batch_active:
return result return result
# Memberships now include events created by this batch. Existing affected events # Memberships now include events created by this batch. Existing affected events
# were frozen up front, including memberships that have since been removed. # were frozen up front, including memberships that have since been removed.
@@ -131,7 +143,13 @@ async def actions(session, batch, active, slots):
for item, event in rows: for item, event in rows:
if item.state not in ('pending', 'urgent_pending', 'processing') or ('event', event.id) in active: if item.state not in ('pending', 'urgent_pending', 'processing') or ('event', event.id) in active:
continue continue
if item.revision != event.revision: if settings is not None and not await session.scalar(select(Event.id).where(Event.id == event.id, scoped_event(settings))):
item.state = 'range_paused'
elif settings is not None and event.realtime_pending:
item.state = 'realtime'
elif event.due_at > datetime.now(UTC):
item.state = 'aggregating'
elif item.revision != event.revision:
item.state = 'deferred' item.state = 'deferred'
elif event.state == 'done' or event.state == 'retired': elif event.state == 'done' or event.state == 'retired':
item.state = 'done' item.state = 'done'
@@ -143,11 +161,17 @@ async def actions(session, batch, active, slots):
invalid = await session.scalar(select(Membership.event_id).join(Package).join(NewsItem).where( invalid = await session.scalar(select(Membership.event_id).join(Package).join(NewsItem).where(
Membership.event_id == event.id, or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)).limit(1)) Membership.event_id == event.id, or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)).limit(1))
if invalid: if invalid:
item.state = 'deferred' item.state = 'blocked'
elif len(result) < slots: elif len(result) < slots:
item.state = 'processing' item.state = 'processing'
result.append(('event', event.id)) result.append(('event', event.id))
if not result and not active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows): event_ids = {event.id for _, event in rows}
batch_active = any(stage == 'event' and identifier in event_ids for stage, identifier in active)
active_packages = {identifier for stage, identifier in active if stage != 'event'}
if active_packages:
batch_active |= bool(await session.scalar(select(BatchPackage.package_id).where(
BatchPackage.batch_id == batch.id, BatchPackage.package_id.in_(active_packages)).limit(1)))
if not result and not batch_active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows):
bad = any(item.state in ('failed', 'deferred') for item, _ in rows) bad = any(item.state in ('failed', 'deferred') for item, _ in rows)
bad |= bool(await session.scalar(select(BatchPackage.batch_id).where( bad |= bool(await session.scalar(select(BatchPackage.batch_id).where(
BatchPackage.batch_id == batch.id, BatchPackage.state.in_(['failed', 'deferred'])).limit(1))) BatchPackage.batch_id == batch.id, BatchPackage.state.in_(['failed', 'deferred'])).limit(1)))
@@ -163,5 +187,5 @@ async def progress(session):
result = {'id': batch.id, 'phase': batch.phase, 'cutoff_at': batch.cutoff_at} result = {'id': batch.id, 'phase': batch.phase, 'cutoff_at': batch.cutoff_at}
for name, model in [('packages', BatchPackage), ('events', BatchEvent)]: for name, model in [('packages', BatchPackage), ('events', BatchEvent)]:
counts = Counter(await session.scalars(select(model.state).where(model.batch_id == batch.id))) counts = Counter(await session.scalars(select(model.state).where(model.batch_id == batch.id)))
result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred')}}) result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred', 'range_paused', 'blocked', 'realtime')}})
return result return result
@@ -6,6 +6,8 @@ from sqlalchemy import (
DateTime, DateTime,
ForeignKey, ForeignKey,
Integer, Integer,
Index,
text,
String, String,
Text, Text,
UniqueConstraint, UniqueConstraint,
@@ -32,6 +34,7 @@ class ResearchSettings(Base):
Integer, default=0, server_default="0" Integer, default=0, server_default="0"
) )
start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
realtime_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Package(Base): class Package(Base):
@@ -49,6 +52,7 @@ class Package(Base):
class Attempt(Base): class Attempt(Base):
__tablename__ = "news_research_attempts" __tablename__ = "news_research_attempts"
__table_args__ = (Index("uq_news_attempt_active", "package_id", unique=True, postgresql_where=text("state = 'processing'")),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True) id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
package_id: Mapped[int] = mapped_column( package_id: Mapped[int] = mapped_column(
ForeignKey("news_research_packages.id"), index=True ForeignKey("news_research_packages.id"), index=True
@@ -75,6 +79,7 @@ class Event(Base):
state: Mapped[str] = mapped_column(String(16), default="pending", index=True) state: Mapped[str] = mapped_column(String(16), default="pending", index=True)
result: Mapped[dict | None] = mapped_column(JSONB) result: Mapped[dict | None] = mapped_column(JSONB)
error: Mapped[str | None] = mapped_column(Text) error: Mapped[str | None] = mapped_column(Text)
realtime_pending: Mapped[bool] = mapped_column(default=False, server_default="false")
dirty_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) dirty_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
due_at: Mapped[datetime] = mapped_column( due_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), index=True DateTime(timezone=True), server_default=func.now(), index=True
@@ -99,6 +104,7 @@ class Membership(Base):
class Version(Base): class Version(Base):
__tablename__ = "news_research_versions" __tablename__ = "news_research_versions"
__table_args__ = (Index("uq_news_version_active", "event_id", unique=True, postgresql_where=text("state = 'processing'")),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True) id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
event_id: Mapped[int] = mapped_column( event_id: Mapped[int] = mapped_column(
ForeignKey("news_research_events.id"), index=True ForeignKey("news_research_events.id"), index=True
@@ -0,0 +1,103 @@
"""Priority selection over durable package and event states; no paid work here."""
from datetime import UTC, datetime
from sqlalchemy import select, exists, or_, and_, func, case
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
def boundary(settings):
realtime_from = getattr(settings, 'realtime_from', None)
return min(settings.start_at, realtime_from) if realtime_from else settings.start_at
def eligible(settings):
return and_(NewsItem.published_at >= boundary(settings), NewsItem.published_at <= datetime.now(UTC))
def members(event_id):
return select(Membership.package_id).where(Membership.event_id == event_id)
def ready_event():
return ~exists(select(Membership.package_id).join(Package).join(NewsItem).where(
Membership.event_id == Event.id,
or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)))
def scoped_event(settings):
# Empty events must still reach the existing no-model retirement path.
return or_(exists(select(Membership.package_id).join(Package).join(NewsItem).where(
Membership.event_id == Event.id, eligible(settings))),
~exists(select(Membership.package_id).where(Membership.event_id == Event.id)))
async def control(session, config):
"""The control lock spans the durable claim, never the remote model call."""
if '_control_revision' not in config:
return None # Explicit bounded/offline callers retain their existing contract.
row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
if not row.enabled or row.control_revision != config['_control_revision']:
return False
return row
async def mark_priority(session, event, news):
settings = await session.get(ResearchSettings, 1)
if settings and settings.realtime_from and news.published_at >= settings.realtime_from:
event.realtime_pending = True
async def realtime_actions(session, settings, active, slots):
if settings.realtime_from is None or slots <= 0:
return []
now = datetime.now(UTC)
busy_packages = {identifier for stage, identifier in active if stage != 'event'}
busy_events = {identifier for stage, identifier in active if stage == 'event'}
result = []
events = await session.scalars(select(Event).where(
Event.state == 'pending', Event.realtime_pending, Event.due_at <= now,
scoped_event(settings), ready_event(), Event.id.not_in(busy_events),
~exists(select(Version.id).where(Version.event_id == Event.id, Version.state == 'processing')),
).order_by(Event.due_at, Event.id).limit(slots))
result.extend(('event', event.id) for event in events)
# Known historical dependencies inherit the waiting realtime event's priority.
dependency = exists(select(Membership.package_id).join(Event).where(
Membership.package_id == Package.id, Event.realtime_pending, Event.state == 'pending'))
priority = or_(NewsItem.published_at >= settings.realtime_from, dependency)
candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
eligible(settings), priority, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash),
or_(Package.id.is_(None), Package.id.not_in(busy_packages))).order_by(NewsItem.published_at, NewsItem.id).limit(30)
ids = list(await session.scalars(candidates))
if ids:
from alphapulse.news_research.service import enroll
await enroll(session, ids)
assigning = any(stage == 'assign' for stage, _ in active)
for stage in ('assign', 'index'):
if stage == 'assign' and assigning:
continue
remaining = slots - len(result)
if remaining <= 0:
break
query = select(Package.id).join(NewsItem).where(
eligible(settings), priority, Package.state == stage + '_pending',
Package.id.not_in(busy_packages),
~exists(select(Attempt.id).where(Attempt.package_id == Package.id, Attempt.state == 'processing')),
).order_by(NewsItem.published_at, NewsItem.id).limit(1 if stage == 'assign' else remaining)
result.extend((stage, identifier) for identifier in await session.scalars(query))
return result
async def queue_status(session, settings):
running = await session.scalar(select(func.count()).select_from(Attempt).where(Attempt.state == 'processing'))
running += await session.scalar(select(func.count()).select_from(Version).where(Version.state == 'processing'))
realtime_from = settings.realtime_from
lane = case((NewsItem.published_at >= realtime_from, 'realtime'), else_='backfill') if realtime_from else 'backfill'
if realtime_from:
groups = (await session.execute(select(lane, Package.state, func.count()).join(NewsItem)
.where(eligible(settings)).group_by(lane, Package.state))).all()
else:
groups = [('backfill', state, count) for state, count in (await session.execute(
select(Package.state, func.count()).join(NewsItem).where(eligible(settings)).group_by(Package.state))).all()]
queues = {name: {state: count for selected, state, count in groups if selected == name}
for name in ('realtime', 'backfill')}
return {'running': running, 'draining': not settings.enabled and running > 0, **queues}
@@ -27,6 +27,7 @@ from alphapulse.news_research.models import (
Version, Version,
Reference, Reference,
) )
from alphapulse.news_research.scheduling import control, mark_priority, boundary
from alphapulse.news_research.contracts import ( from alphapulse.news_research.contracts import (
INDEX_ASSIGN_SKILL, INDEX_ASSIGN_SKILL,
task_skill, task_skill,
@@ -280,6 +281,7 @@ async def enroll(session, ids):
.with_for_update() .with_for_update()
): ):
dirty(event) dirty(event)
await mark_priority(session, event, row)
await session.commit() await session.commit()
@@ -429,6 +431,9 @@ async def process_package(identifier, config):
async def _process_package(identifier, config, expected_stage): async def _process_package(identifier, config, expected_stage):
async with SessionLocal() as session: async with SessionLocal() as session:
settings = await control(session, config)
if settings is False:
return
package = await session.get(Package, identifier, with_for_update=True) package = await session.get(Package, identifier, with_for_update=True)
if package is None: if package is None:
return return
@@ -438,6 +443,10 @@ async def _process_package(identifier, config, expected_stage):
if package.content_hash != news.content_hash: if package.content_hash != news.content_hash:
await enroll(session, [news.id]) await enroll(session, [news.id])
return return
if settings is not None and not boundary(settings) <= news.published_at <= datetime.now(UTC):
return
if await session.scalar(select(Attempt.id).where(Attempt.package_id == identifier, Attempt.state == 'processing').limit(1)):
return
source = article(news) source = article(news)
source_hash = package.content_hash source_hash = package.content_hash
stage = "index" if package.state == "index_pending" else "assign" stage = "index" if package.state == "index_pending" else "assign"
@@ -566,12 +575,14 @@ async def _process_package(identifier, config, expected_stage):
existing.change_kind = target.change_kind existing.change_kind = target.change_kind
if target.change_kind != "repost" or event.result is None: if target.change_kind != "repost" or event.result is None:
dirty(event, target.urgent) dirty(event, target.urgent)
await mark_priority(session, event, news)
for old in previous: for old in previous:
if old.event_id not in assigned: if old.event_id not in assigned:
event = await session.get( event = await session.get(
Event, old.event_id, with_for_update=True Event, old.event_id, with_for_update=True
) )
dirty(event) dirty(event)
await mark_priority(session, event, news)
await session.delete(old) await session.delete(old)
affected[event.id] = event affected[event.id] = event
package.state = "done" package.state = "done"
@@ -805,6 +816,9 @@ def validate_timeline(result, package_news):
async def process_event(identifier, config, force=False): async def process_event(identifier, config, force=False):
async with SessionLocal() as session: async with SessionLocal() as session:
settings = await control(session, config)
if settings is False:
return
event = await session.get(Event, identifier, with_for_update=True) event = await session.get(Event, identifier, with_for_update=True)
if ( if (
event is None event is None
@@ -812,6 +826,8 @@ async def process_event(identifier, config, force=False):
or (not force and event.due_at > datetime.now(UTC)) or (not force and event.due_at > datetime.now(UTC))
): ):
return return
if await session.scalar(select(Version.id).where(Version.event_id == identifier, Version.state == 'processing').limit(1)):
return
members = list( members = list(
await session.execute( await session.execute(
select(Package, Membership, NewsItem) select(Package, Membership, NewsItem)
@@ -821,6 +837,8 @@ async def process_event(identifier, config, force=False):
.order_by(Package.id) .order_by(Package.id)
) )
) )
if settings is not None and members and not any(boundary(settings) <= n.published_at <= datetime.now(UTC) for p, m, n in members):
return
if not members: if not members:
event.state = "retired" event.state = "retired"
await session.commit() await session.commit()
@@ -859,6 +877,7 @@ async def process_event(identifier, config, force=False):
) )
session.add(version) session.add(version)
event.state = "processing" event.state = "processing"
event.dirty_since = None
await session.commit() await session.commit()
version_id, revision = version.id, event.revision version_id, revision = version.id, event.revision
researcher = Research(event={"id": identifier, "revision": revision}) researcher = Research(event={"id": identifier, "revision": revision})
@@ -977,6 +996,7 @@ async def process_event(identifier, config, force=False):
current.title = output.title current.title = output.title
current.state = "done" current.state = "done"
current.dirty_since = None current.dirty_since = None
current.realtime_pending = False
current.error = None current.error = None
for ref in output.references: for ref in output.references:
session.add( session.add(
+38 -33
View File
@@ -13,38 +13,42 @@ from alphapulse.news_research.models import (
Batch, BatchEvent, Batch, BatchEvent,
) )
from alphapulse.news_research.service import process_package, process_event from alphapulse.news_research.service import process_package, process_event
from alphapulse.news_research import batches from alphapulse.news_research import batches, scheduling
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
async def recover_interrupted():
async with SessionLocal() as session:
for stage in ("index", "assign"):
await session.execute(
update(Package)
.where(Package.state == stage + "_processing")
.values(state=stage + "_failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Attempt)
.where(Attempt.state == "processing")
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Event).where(Event.state == "processing").values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Version)
.where(Version.state == "processing")
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
await session.commit()
async def run(): async def run():
async with engine.connect() as lock: async with engine.connect() as lock:
if not await lock.scalar(text("SELECT pg_try_advisory_lock(20261030, 1)")): if not await lock.scalar(text("SELECT pg_try_advisory_lock(20261030, 1)")):
raise RuntimeError("新闻AI worker 已运行") raise RuntimeError("新闻AI worker 已运行")
await lock.commit() await lock.commit()
async with SessionLocal() as session: await recover_interrupted()
for stage in ("index", "assign"):
await session.execute(
update(Package)
.where(Package.state == stage + "_processing")
.values(state=stage + "_pending")
)
await session.execute(
update(Attempt)
.where(Attempt.state == "processing")
.values(state="failed", error="进程中断,待重试")
)
await session.execute(
update(Event).where(Event.state == "processing").values(state="pending")
)
await session.execute(
update(Version)
.where(Version.state == "processing")
.values(state="failed", error="进程中断,待重试")
)
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
await session.commit()
tasks = {} tasks = {}
try: try:
while True: while True:
@@ -58,20 +62,21 @@ async def run():
async with SessionLocal() as session: async with SessionLocal() as session:
settings = await session.get(ResearchSettings, 1) settings = await session.get(ResearchSettings, 1)
if settings and settings.enabled: if settings and settings.enabled:
active = set(tasks.values())
slots = max(0, settings.config.get('concurrency', 3) - len(tasks))
actions = await scheduling.realtime_actions(session, settings, active, slots)
batch = await session.scalar(select(Batch).where( batch = await session.scalar(select(Batch).where(
Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1)) Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1))
if batch is None: if batch is None:
batch = await batches.freeze(session, settings) batch = await batches.freeze(session, settings)
settings = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True) if batch is not None:
if batch is not None and settings.enabled: actions += await batches.actions(session, batch, active | set(actions), slots - len(actions), settings)
active = set(tasks.values()) await session.commit()
actions = await batches.actions(session, batch, active, config = {**settings.config, '_control_revision': settings.control_revision}
max(0, settings.config.get('concurrency', 3) - len(tasks))) for stage, identifier in actions:
await session.commit() call = (process_event(identifier, config)
for stage, identifier in actions: if stage == 'event' else process_package(identifier, config))
call = (process_event(identifier, settings.config, force=True) tasks[asyncio.create_task(call)] = (stage, identifier)
if stage == 'event' else process_package(identifier, settings.config))
tasks[asyncio.create_task(call)] = (stage, identifier)
await asyncio.sleep(1) await asyncio.sleep(1)
finally: finally:
await asyncio.gather(*tasks, return_exceptions=True) await asyncio.gather(*tasks, return_exceptions=True)
+46 -1
View File
@@ -302,7 +302,7 @@ def test_failed_enable_does_not_save_the_selected_backfill_start(trading, monkey
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock()) session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用"))) monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
with pytest.raises(HTTPException) as error: with pytest.raises(HTTPException) as error:
asyncio.run(api.enable(session, api.EnableInput(start_at=previous - timedelta(days=30)))) asyncio.run(api.enable(session, api.EnableInput(start_at=previous)))
assert error.value.status_code == 502 assert error.value.status_code == 502
assert row.start_at == previous and not row.enabled and row.control_revision == 7 assert row.start_at == previous and not row.enabled and row.control_revision == 7
session.commit.assert_not_called() session.commit.assert_not_called()
@@ -842,6 +842,7 @@ def test_alias_adjustment_audit_survives_index_failures(trading, monkeypatch, fa
session.get = AsyncMock(side_effect=lambda model, *args, **kwargs: session.get = AsyncMock(side_effect=lambda model, *args, **kwargs:
attempts[0] if model is Attempt else package if model is Package else news) attempts[0] if model is Attempt else package if model is Package else news)
session.execute = AsyncMock() session.execute = AsyncMock()
session.scalar = AsyncMock(return_value=None)
monkeypatch.setattr(service, "SessionLocal", lambda: session) monkeypatch.setattr(service, "SessionLocal", lambda: session)
monkeypatch.setattr(service.vocab, "snapshot", AsyncMock(return_value={"entries": []})) monkeypatch.setattr(service.vocab, "snapshot", AsyncMock(return_value={"entries": []}))
monkeypatch.setattr(service, "model_config", AsyncMock(return_value=(None, "索引规则", {}))) monkeypatch.setattr(service, "model_config", AsyncMock(return_value=(None, "索引规则", {})))
@@ -903,3 +904,47 @@ def test_task_skill_keeps_custom_rules_and_accepts_crlf(trading):
content = "## 归属规则\r\n自定义归属\r\n### 细则\r\n保留规则\r\n## 索引规则\r\n自定义索引" content = "## 归属规则\r\n自定义归属\r\n### 细则\r\n保留规则\r\n## 索引规则\r\n自定义索引"
assert task_skill(content, "assign") == "自定义归属\n### 细则\n保留规则" assert task_skill(content, "assign") == "自定义归属\n### 细则\n保留规则"
assert task_skill(content, "index") == "自定义索引" assert task_skill(content, "index") == "自定义索引"
def test_settings_save_atomically_pauses_and_preserves_model_contract(trading):
import asyncio
from unittest.mock import AsyncMock
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=3)
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
model = dict(provider_id=1, model='test', skill_id=1)
body = api.SettingsInput(index=model, research=model, start_at='2026-10-20T00:00:00+08:00', base_revision=3)
asyncio.run(api.save(body, session))
assert not row.enabled and row.control_revision == 4
assert row.start_at == datetime(2026, 10, 19, 16, tzinfo=UTC)
assert set(row.config) == {'index', 'research', 'concurrency'}
session.commit.assert_awaited_once()
def test_settings_save_rejects_other_tabs_revision_without_changes(trading):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=4)
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
model = dict(provider_id=1, model='test', skill_id=1)
with pytest.raises(HTTPException) as error:
asyncio.run(api.save(api.SettingsInput(index=model, research=model, base_revision=3), session))
assert error.value.status_code == 409
assert row.enabled and row.control_revision == 4
session.commit.assert_not_awaited()
def test_enable_cannot_apply_an_unsaved_backfill_time(trading):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC))
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(), commit=AsyncMock())
with pytest.raises(HTTPException) as error:
asyncio.run(api.enable(session, api.EnableInput(start_at='2026-10-20T00:00:00+08:00')))
assert error.value.status_code == 422
session.scalar.assert_not_awaited()
session.commit.assert_not_awaited()
+39
View File
@@ -455,3 +455,42 @@ some documents were imported: ECB dynamic index version coverage, inaccessible
BoE speeches and CoinDesk access blocks are reported rather than labeled complete. BoE speeches and CoinDesk access blocks are reported rather than labeled complete.
The deployment lock serializes writes; crawler/calendar processes are restored in The deployment lock serializes writes; crawler/calendar processes are restored in
finally blocks. Do not start a second driver or remove its state to renew quotas. finally blocks. Do not start a second driver or remove its state to renew quotas.
### Realtime priority and historical AI backfill
Migration `dd4567890123` adds the persistent realtime boundary and pending-event
priority, plus partial unique indexes for active package attempts and event
versions. Pause and drain news AI before applying it; duplicate active records
must be investigated rather than deleted to force migration success. Preserve
originals, attempts and version history. Deployment requires the candidate's
schema head and must leave automatic processing paused during verification.
The model/Skills drawer saves the historical start time with model settings via
**Save and pause**. New settings default to 2026-11-05 00:00 Asia/Shanghai; existing
saved values remain intact. Settings responses expose `control_revision`,
`realtime_from`, `timezone` and queue counts. PUT accepts optional `start_at` and
`base_revision`; version conflicts return 409. Enable uses the saved start only;
a differing legacy enable body returns 422. The first successful enable fixes
`realtime_from`; later saves, pauses and restarts never move it.
Publication time, not ingestion time, determines realtime membership. Realtime
work takes the next available slot without cancelling accepted calls. Historical
work uses batches of at most 30 originals, ordered chronologically. Saving a new
start pauses dispatch; enabling after drain retires unfinished batch schedules
and rebuilds from durable stage states without repeating successes. Shrinking
history does not exclude realtime backlog. Known in-range dependencies inherit
a waiting realtime event's priority. Out-of-range or failed members block event
research without silently dropping evidence or retrying paid failures.
Ordinary event research uses the existing five-minute quiet window with a
15-minute maximum; urgent events become ready immediately. A running research
call is not duplicated when newer members arrive. New changes start another
aggregation window and obsolete output cannot overwrite current conclusions.
After a worker interruption, started calls become failed with an unknown-result
message and need manual retry; unclaimed work resumes normally. Strict realtime
priority can starve historical work under sustained overload.
CI additionally runs `scripts/smoke_priority_queue.py` in its disposable internal
network with synthetic credentials and no model calls. It covers preemption,
late historical ingestion, stable realtime eligibility, range changes,
dependencies, aggregation, control revisions and interrupted-call recovery.
+2
View File
@@ -182,6 +182,8 @@ def smoke(build, label):
'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)') 'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)')
print(run('docker', 'exec', '-i', names['api'], 'python', '-', print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True) input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_priority_queue.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-', print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True) input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-', print(run('docker', 'exec', '-i', names['api'], 'python', '-',
+103 -19
View File
@@ -90,6 +90,7 @@ async def main():
await backfill_regressions(now) await backfill_regressions(now)
await carryover_regressions(now) await carryover_regressions(now)
await regressions(now, settings) await regressions(now, settings)
await queue_review_regressions(now, settings)
await engine.dispose() await engine.dispose()
print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.') print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
@@ -208,7 +209,13 @@ async def regressions(now, settings):
async with SessionLocal() as s: async with SessionLocal() as s:
event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now) event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now)
previous = Batch(phase='partial', cutoff_at=now) previous = Batch(phase='partial', cutoff_at=now)
s.add_all([event, previous]) original = NewsItem(source_code='wscn_news', external_id='retry-member', title='retry', summary='', content_text='retry', content_hash='a'*64, url='https://example.invalid/retry', published_at=now)
s.add_all([event, previous, original])
await s.flush()
member = Package(news_id=original.id, content_hash=original.content_hash, state='done', keywords=[])
s.add(member)
await s.flush()
s.add(Membership(event_id=event.id, package_id=member.id, evidence='retry', reason='retry', change_kind='new_fact'))
await s.flush() await s.flush()
s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred')) s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred'))
await s.commit() await s.commit()
@@ -236,39 +243,116 @@ async def regressions(now, settings):
s.add_all([news, excluded]) s.add_all([news, excluded])
await s.commit() await s.commit()
await service.enroll(s, [news.id]) await service.enroll(s, [news.id])
batch = await batches.freeze(s, settings) assert await batches.freeze(s, settings) is None
package = await s.scalar(select(Package).where(Package.news_id == news.id)) package = await s.scalar(select(Package).where(Package.news_id == news.id))
news.content_hash = 'b'*64 news.content_hash = 'b'*64
await s.commit() await s.commit()
await batches.actions(s, batch, set(), 8) assert await batches.freeze(s, settings) is None
await s.commit() assert package.content_hash != news.content_hash
await batches.actions(s, batch, set(), 8) expanded = SimpleNamespace(start_at=news.published_at)
await s.commit() batch = await batches.freeze(s, expanded)
assert batch.phase == 'partial'
batch = await batches.freeze(s, settings)
assert package.content_hash == news.content_hash and package.state == 'index_pending'
assert await batches.actions(s, batch, set(), 8) == [('index', package.id)]
assert await s.scalar(select(func.count()).select_from(Package)) == 1
# A previously completed manual original also refreshes outside the automatic boundary.
package.state = 'done'
news.content_hash = 'c'*64
await s.commit()
await batches.freeze(s, settings)
assert package.content_hash == news.content_hash and package.state == 'index_pending' assert package.content_hash == news.content_hash and package.state == 'index_pending'
assert ('index', package.id) in await batches.actions(s, batch, set(), 8)
await clear(s) await clear(s)
count = 33000 count = 33000
await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)]) await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)])
await s.commit() await s.commit()
batch = await batches.freeze(s, settings) batch = await batches.freeze(s, settings)
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == count assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == 30
assert await s.scalar(select(func.count()).select_from(Package)) == count assert await s.scalar(select(func.count()).select_from(Package)) == 30
# Exercise the existing-pending-ID and membership arrays above the driver limit too. # Exercise the existing-pending-ID and membership arrays above the driver limit too.
next_batch = await batches.freeze(s, settings) next_batch = await batches.freeze(s, settings)
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == count assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == 30
assert len(await batches.actions(s, next_batch, set(), 8)) == 8 assert len(await batches.actions(s, next_batch, set(), 8)) == 8
await clear(s) await clear(s)
print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.') print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.')
async def queue_review_regressions(now, settings):
async def clear(session):
await session.execute(text('TRUNCATE news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
await session.commit()
async with SessionLocal() as s:
news = NewsItem(source_code='wscn_news', external_id='review-barrier', title='historical',
summary='', content_text='historical', content_hash='a'*64,
url='https://example.invalid/review-barrier', published_at=now)
s.add(news)
await s.flush()
package = Package(news_id=news.id, content_hash=news.content_hash, state='assign_pending', keywords=[])
event = Event(title='historical event', tags=[], revision=1, state='pending', due_at=now)
batch = Batch(phase='assigning', cutoff_at=now)
s.add_all([package, event, batch])
await s.flush()
s.add(Membership(event_id=event.id, package_id=package.id,
evidence='historical', reason='same event', change_kind='new_fact'))
s.add(BatchPackage(batch_id=batch.id, package_id=package.id, content_hash=package.content_hash, state='pending'))
await s.commit()
unrelated = package.id + 1000
# Assignment remains globally serial even when the other call is not in this batch.
assert await batches.actions(s, batch, {('assign', unrelated)}, 7) == []
assert batch.phase == 'assigning'
# Active work on this batch still holds its assignment barrier.
package.state = 'done'
await s.commit()
assert await batches.actions(s, batch, {('assign', package.id)}, 7) == []
assert batch.phase == 'assigning'
# Unrelated realtime package and event calls must not hold the barrier.
realtime = {('index', unrelated), ('event', event.id + 1000)}
assert await batches.actions(s, batch, realtime, 6) == []
await s.commit()
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime, 6) == [('event', event.id)]
await s.commit()
event.state = 'done'
await s.commit()
# A completed result can be visible before the owning task has returned.
assert await batches.actions(s, batch, realtime | {('event', event.id)}, 5) == []
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime | {('index', package.id)}, 5) == []
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime, 6) == []
await s.commit()
assert batch.phase == 'done'
# Completed batches release admission of the next historical originals.
news2 = NewsItem(source_code='wscn_news', external_id='review-next', title='next', summary='',
content_text='next', content_hash='b'*64,
url='https://example.invalid/review-next', published_at=now)
s.add(news2)
await s.commit()
assert (await batches.freeze(s, settings)).id != batch.id
await clear(s)
# Thirty manual-only deferred heads cannot consume the thirty runnable slots.
news = NewsItem(source_code='wscn_news', external_id='review-deferred', title='source', summary='',
content_text='source', content_hash='a'*64,
url='https://example.invalid/review-deferred', published_at=now)
s.add(news)
await s.flush()
package = Package(news_id=news.id, content_hash=news.content_hash, state='done', keywords=[])
previous = Batch(phase='partial', cutoff_at=now)
events = [Event(title='deferred-head-'+str(i), tags=[], revision=7, state='pending',
due_at=now-timedelta(minutes=35-i)) for i in range(35)]
s.add_all([package, previous, *events])
await s.flush()
for i, event in enumerate(events):
s.add(Membership(event_id=event.id, package_id=package.id, evidence='source', reason='same source', change_kind='new_fact'))
if i <= 30:
s.add(BatchEvent(batch_id=previous.id, event_id=event.id,
revision=7 if i < 30 else 6, state='deferred'))
await s.commit()
batch = await batches.freeze(s, settings)
assert batch is not None
selected = set(await s.scalars(select(BatchEvent.event_id).where(BatchEvent.batch_id == batch.id)))
assert selected == {event.id for event in events[30:]}, selected
assert all(event.state == 'pending' and event.revision == 7 for event in events[:30])
await batches.actions(s, batch, set(), 8)
await s.commit()
actions = await batches.actions(s, batch, set(), 8)
assert set(actions) == {('event', event.id) for event in events[30:]}
await clear(s)
print('Queue review regressions passed: unrelated realtime calls allow both batch barriers to advance, owning calls still block, assignment stays serial, and deferred heads cannot starve runnable events.')
asyncio.run(main()) asyncio.run(main())
+9 -2
View File
@@ -393,14 +393,20 @@ async def main():
assert report.json()["data"]["start_at"] == "2026-09-30T16:00:00Z" assert report.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert set(report.json()["data"]["outside"]) == {"unprocessed", "stale", "pending", "processing", "failed"} assert set(report.json()["data"]["outside"]) == {"unprocessed", "stale", "pending", "processing", "failed"}
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == previous_start assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == previous_start
assert (await client.post("/api/risk/enable", json={"start_at": selected_start})).status_code == 422
settings = (await client.get("/api/risk/settings")).json()["data"]
saved = await client.put("/api/risk/settings", json={**config, "start_at": selected_start, "base_revision": settings["control_revision"]})
assert saved.status_code == 200, saved.text
with patch.object(model_service, "complete", validation): with patch.object(model_service, "complete", validation):
response = await client.post("/api/risk/enable", json={"start_at": selected_start}) response = await client.post("/api/risk/enable")
assert response.status_code == 200, response.text assert response.status_code == 200, response.text
assert response.json()["data"]["start_at"] == "2026-09-30T16:00:00Z" assert response.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert (await client.get("/api/risk/settings")).json()["data"][ assert (await client.get("/api/risk/settings")).json()["data"][
"enabled" "enabled"
] is True ] is True
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z" assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
fixed_anchor = (await client.get("/api/risk/settings")).json()["data"]["realtime_from"]
assert fixed_anchor is not None
assert (await client.post("/api/risk/enable")).status_code == 409 assert (await client.post("/api/risk/enable")).status_code == 409
assert (await client.post("/api/risk/pause")).status_code == 200 assert (await client.post("/api/risk/pause")).status_code == 200
for action in ("pause", "save"): for action in ("pause", "save"):
@@ -421,12 +427,13 @@ async def main():
return await validation(provider, skill, payload, schema) return await validation(provider, skill, payload, schema)
with patch.object(model_service, "complete", interrupted_validation): with patch.object(model_service, "complete", interrupted_validation):
response = await client.post("/api/risk/enable", json={"start_at": "2026-09-01T00:00:00Z"}) response = await client.post("/api/risk/enable")
assert response.status_code == 409, response.text assert response.status_code == 409, response.text
assert (await client.get("/api/risk/settings")).json()["data"][ assert (await client.get("/api/risk/settings")).json()["data"][
"enabled" "enabled"
] is False ] is False
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z" assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert (await client.get("/api/risk/settings")).json()["data"]["realtime_from"] == fixed_anchor
package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[ package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[
"data" "data"
][0]["id"] ][0]["id"]
+107
View File
@@ -0,0 +1,107 @@
"""Exercise preemption and recovery using disposable PostgreSQL, never a model."""
import asyncio
import os
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from urllib.parse import urlparse
from sqlalchemy import select, text, func
from sqlalchemy.exc import IntegrityError
from unittest.mock import patch
from alphapulse.core.db import SessionLocal, engine
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
from alphapulse.news_research import scheduling, batches, service
from alphapulse.news_research.worker import recover_interrupted
async def main():
url = urlparse(os.environ['DATABASE_URL'])
assert (url.username, url.password, url.path) == ('ci', 'ci-only', '/ci')
now = datetime.now(UTC)
anchor = now - timedelta(hours=1)
config = SimpleNamespace(start_at=now-timedelta(days=5), realtime_from=anchor)
async with SessionLocal() as s:
assert await s.scalar(select(func.count()).select_from(Package)) == 0
def original(key, published):
return NewsItem(source_code='wscn_news', external_id='priority-'+key, title=key, summary='',
content_text=key, content_hash='a'*64, url='https://example.invalid/'+key, published_at=published)
old = original('old', now-timedelta(days=4))
live = original('live', now-timedelta(minutes=30))
late = original('late', now-timedelta(days=3))
s.add_all([old, live, late]); await s.commit()
batch = await batches.freeze(s, config)
packages = list(await s.scalars(select(Package)))
assert len(packages) == 2 and all(p.news_id != live.id for p in packages)
actions = await scheduling.realtime_actions(s, config, {('index', packages[0].id)}, 1)
live_package = await s.scalar(select(Package).where(Package.news_id == live.id))
assert actions == [('index', live_package.id)]
assert await scheduling.realtime_actions(s, config, set(), 0) == []
# Shrinking the history range must not exclude accumulated realtime news.
config.start_at = now-timedelta(minutes=5)
actions = await scheduling.realtime_actions(s, config, set(), 1)
assert actions == [('index', live_package.id)]
await batches.actions(s, batch, set(), 8, config)
assert not await batches.freeze(s, config)
config.start_at = now-timedelta(days=5)
assert await batches.freeze(s, config)
assert packages[0].keywords == []
# A realtime event's historical dependencies inherit priority.
event = Event(title='shared', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
s.add(event); await s.flush()
for p in [packages[0], live_package]:
s.add(Membership(event_id=event.id, package_id=p.id, evidence='fixture', reason='fixture', change_kind='new_fact'))
packages[0].state = 'assign_pending'
live_package.state = 'done'
await s.commit()
config.start_at = now-timedelta(minutes=5)
assert await scheduling.realtime_actions(s, config, set(), 1) == []
config.start_at = now-timedelta(days=5)
assert await scheduling.realtime_actions(s, config, set(), 1) == [('assign', packages[0].id)]
packages[0].state = 'done'; await s.commit()
assert await scheduling.realtime_actions(s, config, set(), 1) == [('event', event.id)]
event.due_at = now+timedelta(minutes=5); await s.commit()
assert ('event', event.id) not in await scheduling.realtime_actions(s, config, set(), 8)
# Pausing invalidates a selected but not-yet-claimed task.
s.add(ResearchSettings(id=1, config={}, enabled=False, control_revision=9, start_at=config.start_at, realtime_from=anchor))
await s.commit()
assert await scheduling.control(s, {'_control_revision': 8}) is False
await s.commit()
with patch.object(service, 'complete', side_effect=AssertionError('Paused selection must not call a model')):
await service.process_package(live_package.id, {'_control_revision': 8})
assert await s.scalar(select(func.count()).select_from(Attempt)) == 0
live_package = await s.get(Package, live_package.id)
live_package.state = 'index_processing'
s.add(Attempt(package_id=live_package.id, content_hash='a'*64, stage='index', state='processing', input={}, rules={}))
event = await s.get(Event, event.id); event.state='processing'
s.add(Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}))
await s.commit()
for duplicate in [
Attempt(package_id=live_package.id, content_hash='a'*64, stage='assign', state='processing', input={}, rules={}),
Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}),
]:
try:
async with s.begin_nested():
s.add(duplicate)
await s.flush()
except IntegrityError:
pass
else:
raise AssertionError('Concurrent duplicate claim was accepted')
await recover_interrupted()
async with SessionLocal() as s:
assert (await s.get(Package, live_package.id)).state == 'index_failed'
assert (await s.get(Event, event.id)).state == 'failed'
assert await scheduling.realtime_actions(s, config, set(), 8) == []
empty = Event(title='removed members', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
s.add(empty); await s.commit()
assert ('event', empty.id) in await scheduling.realtime_actions(s, config, set(), 8)
with patch.object(service.Research, 'run', side_effect=AssertionError('Empty events must not call a model')):
await service.process_event(empty.id, {})
await s.refresh(empty)
assert empty.state == 'retired'
await s.execute(text('TRUNCATE news_research_settings, news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
await s.commit()
await engine.dispose()
print('Priority queue passed: live preemption, stable boundary, late history, range changes, dependencies, aggregation, stale control and uncertain-call recovery.')
asyncio.run(main())
+2 -2
View File
@@ -2,8 +2,8 @@ import type * as Research from "./type"
import { request } from "@/http/axios" import { request } from "@/http/axios"
export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" }) export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" })
export const saveSettingsApi = (data: Research.Config) => request<null>({ url: "/risk/settings", method: "put", data }) export const saveSettingsApi = (data: Research.Config & { start_at: string, base_revision: number }) => request<null>({ url: "/risk/settings", method: "put", data })
export const enableApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/enable", method: "post", data: { start_at } }) export const enableApi = () => request<Research.Coverage>({ url: "/risk/enable", method: "post" })
export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } }) export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } })
export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" }) export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" })
export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" }) export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" })
+2 -2
View File
@@ -2,13 +2,13 @@ export interface Keyword { canonical: string, aliases: string[], quote: string }
export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null } export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null }
export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number } export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number }
export interface BatchProgress { id: number, phase: string, cutoff_at: string, packages_total: number, packages_done: number, packages_failed: number, packages_deferred: number, events_total: number, events_done: number, events_failed: number, events_deferred: number } export interface BatchProgress { id: number, phase: string, cutoff_at: string, packages_total: number, packages_done: number, packages_failed: number, packages_deferred: number, events_total: number, events_done: number, events_failed: number, events_deferred: number }
export interface Settings { batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string } export interface Settings { control_revision: number, realtime_from: string | null, queue: { running: number, draining: boolean, realtime: Record<string, number>, backfill: Record<string, number> }, batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string }
export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } } export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } }
export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] } export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] }
export interface Point { time_precision: "minute" | "day" | "month" | "unknown", planned_precision: "minute" | "day" | "month" | "unknown", package_ids: number[], time_text: string, description: string, occurred_at: string | null, planned_at: string | null, reported_at: string | null, period_text: string, evidence_refs: { article_id: number, quote: string }[] } export interface Point { time_precision: "minute" | "day" | "month" | "unknown", planned_precision: "minute" | "day" | "month" | "unknown", package_ids: number[], time_text: string, description: string, occurred_at: string | null, planned_at: string | null, reported_at: string | null, period_text: string, evidence_refs: { article_id: number, quote: string }[] }
export interface Result { title: string, summary: string, facts: string, reasoning: string, uncertainty: string, opinions: string, counter_evidence: string, watchpoints: string, asset_impacts: string, importance_stars: number, risk_stars: number, confidence: string, verification: string, timeline_points: Point[] } export interface Result { title: string, summary: string, facts: string, reasoning: string, uncertainty: string, opinions: string, counter_evidence: string, watchpoints: string, asset_impacts: string, importance_stars: number, risk_stars: number, confidence: string, verification: string, timeline_points: Point[] }
export interface Event { id: number, title: string, state: string, revision: number, error: string | null, tags: string[], result: Result | null } export interface Event { id: number, title: string, state: string, revision: number, error: string | null, tags: string[], result: Result | null }
export interface EventDetail extends Event { packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] } export interface EventDetail extends Event { wait_reason?: string | null, packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] }
export interface PackageDetail extends Package { article: { title: string, summary: string, content: string }, attempts: { id: number, stage: string, state: string, error: string | null, output: unknown, usage: unknown }[] } export interface PackageDetail extends Package { article: { title: string, summary: string, content: string }, attempts: { id: number, stage: string, state: string, error: string | null, output: unknown, usage: unknown }[] }
export interface VocabularyEntry { canonical: string, aliases: string[] } export interface VocabularyEntry { canonical: string, aliases: string[] }
+79 -45
View File
@@ -6,8 +6,15 @@ import * as api from "@@/apis/news-research"
import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings" import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings"
import { listSkillTreeApi } from "@@/apis/skills" import { listSkillTreeApi } from "@@/apis/skills"
import { EFFORTS } from "@@/constants/ai" import { EFFORTS } from "@@/constants/ai"
import dayjs from "dayjs"
import timezone from "dayjs/plugin/timezone"
import utc from "dayjs/plugin/utc"
import { useUserStore } from "@/pinia/stores/user" import { useUserStore } from "@/pinia/stores/user"
dayjs.extend(utc)
dayjs.extend(timezone)
const vocabularyOpened = ref(false) const vocabularyOpened = ref(false)
const vocabulary = ref<Vocabulary>({ revision: 0, entries: [] }) const vocabulary = ref<Vocabulary>({ revision: 0, entries: [] })
const vocabularyBusy = ref(false) const vocabularyBusy = ref(false)
@@ -90,8 +97,19 @@ const settingsOpened = ref(false)
const enabled = ref(false) const enabled = ref(false)
const startAt = ref<Date | null>(null) const startAt = ref<Date | null>(null)
const savedStartAt = ref<Date | null>(null) const savedStartAt = ref<Date | null>(null)
const settingsRevision = ref(0)
const queue = ref<{ running: number, draining: boolean, realtime?: Record<string, number>, backfill?: Record<string, number> } | null>(null)
function disabledBackfillDate(date: Date) {
return dayjs.tz(dayjs(date).format("YYYY-MM-DD"), "Asia/Shanghai").valueOf() > Date.now()
}
const startText = computed({
get: () => startAt.value ? dayjs(startAt.value).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : "",
set: (value: string) => {
startAt.value = value ? dayjs.tz(value, "Asia/Shanghai").toDate() : null
}
})
const selectedStartAt = computed({ const selectedStartAt = computed({
get: () => enabled.value ? savedStartAt.value : startAt.value, get: () => enabled.value && !settingsOpened.value ? savedStartAt.value : startAt.value,
set: (value: Date | null) => { set: (value: Date | null) => {
startAt.value = value startAt.value = value
} }
@@ -134,6 +152,7 @@ async function load() {
const [settings, rows] = await Promise.all([api.getSettingsApi(), tab.value === "events" ? api.listEventsApi(q.value, page.value) : api.listPackagesApi(q.value, page.value)]) const [settings, rows] = await Promise.all([api.getSettingsApi(), tab.value === "events" ? api.listEventsApi(q.value, page.value) : api.listPackagesApi(q.value, page.value)])
if (!valid() || seq !== sequence) return if (!valid() || seq !== sequence) return
enabled.value = settings.enabled enabled.value = settings.enabled
queue.value = settings.queue ?? null
if (!startAt.value || startAt.value.getTime() === savedStartAt.value?.getTime()) startAt.value = new Date(settings.start_at) if (!startAt.value || startAt.value.getTime() === savedStartAt.value?.getTime()) startAt.value = new Date(settings.start_at)
savedStartAt.value = new Date(settings.start_at) savedStartAt.value = new Date(settings.start_at)
batch.value = settings.batch ?? null batch.value = settings.batch ?? null
@@ -166,13 +185,7 @@ async function checkCoverage() {
} }
async function enable() { async function enable() {
if (!valid()) return if (!valid()) return
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) { await api.enableApi()
ElMessage.error("请选择不晚于当前时间的回补起始时间")
return
}
const start = startAt.value.toISOString()
const result = await api.enableApi(start)
if (valid() && selectedStartAt.value?.toISOString() === start) coverage.value = result
} }
async function openEvent(id: number) { async function openEvent(id: number) {
eventId = id eventId = id
@@ -248,6 +261,8 @@ async function settings() {
providers.value = choices providers.value = choices
skills.value = nodes.filter(s => s.scope === "news" && s.kind === "skill") skills.value = nodes.filter(s => s.scope === "news" && s.kind === "skill")
if (value.config.index && value.config.research) config.value = value.config as Config if (value.config.index && value.config.research) config.value = value.config as Config
startAt.value = new Date(value.start_at)
settingsRevision.value = value.control_revision
settingsOpened.value = true settingsOpened.value = true
} }
async function action(fn: () => Promise<unknown>) { async function action(fn: () => Promise<unknown>) {
@@ -275,7 +290,12 @@ async function retryPackage() {
if (id) await action(() => api.retryPackageApi(id)) if (id) await action(() => api.retryPackageApi(id))
} }
async function save() { async function save() {
await action(() => api.saveSettingsApi(config.value)) if (!valid() || busy.value) return
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) {
ElMessage.error("请选择不晚于当前时间的回补起始时间")
return
}
await action(() => api.saveSettingsApi({ ...config.value, start_at: startAt.value!.toISOString(), base_revision: settingsRevision.value }))
if (valid()) settingsOpened.value = false if (valid()) settingsOpened.value = false
} }
async function prepareSkills() { async function prepareSkills() {
@@ -364,7 +384,7 @@ onBeforeUnmount(() => {
<el-button @click="settings"> <el-button @click="settings">
模型与 Skills 模型与 Skills
</el-button> </el-button>
<el-button :loading="busy" :disabled="!enabled && (loading || error || !startAt)" @click="action(enabled ? api.pauseApi : enable)"> <el-button :loading="busy" :disabled="!enabled && (loading || error || queue?.draining)" @click="action(enabled ? api.pauseApi : enable)">
{{ enabled ? '暂停新任务' : '启用两阶段处理' }} {{ enabled ? '暂停新任务' : '启用两阶段处理' }}
</el-button> </el-button>
<el-button @click="openVocabulary"> <el-button @click="openVocabulary">
@@ -377,45 +397,20 @@ onBeforeUnmount(() => {
<p class="mb-4 text-sm text-slate-500"> <p class="mb-4 text-sm text-slate-500">
新闻包保存原文和关键词;事件汇集相关报道,持续更新判断与时间线。 新闻包保存原文和关键词;事件汇集相关报道,持续更新判断与时间线。
</p> </p>
<div class="mb-4 rounded border border-slate-200 p-3"> <p class="mb-4 text-sm text-slate-500">
<div class="flex flex-wrap items-center gap-3"> 实时任务优先取得空闲并发;历史回补至 {{ savedStartAt ? dayjs(savedStartAt).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : '—' }}(北京时间)。
<label for="research-start-at">回补起始时间</label> <span v-if="queue?.draining">正在完成 {{ queue.running }} 个已接收调用,收尾后可重新启用。</span>
<el-date-picker </p>
id="research-start-at" <p v-if="queue" class="mb-4 text-sm text-slate-500">
v-model="selectedStartAt" 实时队列:等待索引 {{ queue.realtime?.index_pending ?? 0 }},等待归属 {{ queue.realtime?.assign_pending ?? 0 }};
type="datetime" 回补队列:等待索引 {{ queue.backfill?.index_pending ?? 0 }},等待归属 {{ queue.backfill?.assign_pending ?? 0 }}。
:clearable="false" 正在调用 {{ queue.running }};范围外暂停、失败和中断结果不明的任务不会自动重试。
:disabled="enabled || busy" </p>
:disabled-date="(date: Date) => date.getTime() > Date.now()"
placeholder="选择回补起始时间"
/>
<span class="text-sm text-slate-500">按浏览器本地时区</span>
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
核查遗漏
</el-button>
</div>
<p class="mt-2 text-sm text-slate-500">
启用时保存起点,自动核查并补齐该时间以来已采集的新闻;先完成关键词与归属,再集中研究事件。调整起点不会取消已有队列。
</p>
<p v-if="coverage" class="mt-2 text-sm">
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
失败项需手动重试,当前批次以外的遗漏排入下一批。
</p>
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
范围外另有已入队但未完成的新闻 {{ coverage.outside.unprocessed }} 条:
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
已有任务继续按批次处理,失败项需手动重试。
</p>
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
核查失败,请重试。
</p>
</div>
<p v-if="batch" class="mb-4 text-sm text-slate-500"> <p v-if="batch" class="mb-4 text-sm text-slate-500">
批次 #{{ batch.id }} · {{ batchPhases[batch.phase] ?? batch.phase }} · 批次 #{{ batch.id }} · {{ batchPhases[batch.phase] ?? batch.phase }} ·
新闻归属 {{ batch.packages_done }}/{{ batch.packages_total }},失败 {{ batch.packages_failed }},延期 {{ batch.packages_deferred }} · 新闻归属 {{ batch.packages_done }}/{{ batch.packages_total }},失败 {{ batch.packages_failed }},延期 {{ batch.packages_deferred }} ·
事件研究 {{ batch.events_done }}/{{ batch.events_total }},失败 {{ batch.events_failed }},延期 {{ batch.events_deferred }}。 事件研究 {{ batch.events_done }}/{{ batch.events_total }},失败 {{ batch.events_failed }},延期 {{ batch.events_deferred }}。
新新闻和重试请求排入下一批,研究仅使用站内资料。 实时任务可中途插队;回补任务分批归属后集中研究,研究仅使用站内资料。
</p> </p>
<el-alert v-if="error" title="读取失败,保留上次结果,可刷新重试" type="error" :closable="false" /> <el-alert v-if="error" title="读取失败,保留上次结果,可刷新重试" type="error" :closable="false" />
<div class="mb-4 flex flex-wrap gap-3"> <div class="mb-4 flex flex-wrap gap-3">
@@ -474,6 +469,9 @@ onBeforeUnmount(() => {
<el-drawer v-model="opened" title="事件研究与时间线" size="min(820px, 100vw)"> <el-drawer v-model="opened" title="事件研究与时间线" size="min(820px, 100vw)">
<el-alert v-if="detailError" title="事件详情读取失败,请重新打开重试" type="error" :closable="false" /> <el-alert v-if="detailError" title="事件详情读取失败,请重新打开重试" type="error" :closable="false" />
<template v-if="detail"> <template v-if="detail">
<p v-if="detail.wait_reason" class="mb-3 text-amber-700">
{{ detail.wait_reason }}
</p>
<h2>{{ detail.title }}</h2><p>{{ states[detail.state] ?? detail.state }}</p> <h2>{{ detail.title }}</h2><p>{{ states[detail.state] ?? detail.state }}</p>
<el-button :loading="busy" @click="retryEvent"> <el-button :loading="busy" @click="retryEvent">
重新研究 重新研究
@@ -627,6 +625,42 @@ onBeforeUnmount(() => {
</el-drawer> </el-drawer>
<el-drawer v-model="settingsOpened" title="两阶段模型与 Skills" size="min(640px, 100vw)"> <el-drawer v-model="settingsOpened" title="两阶段模型与 Skills" size="min(640px, 100vw)">
<el-form label-position="top"> <el-form label-position="top">
<div class="mb-4 rounded border border-slate-200 p-3">
<div class="flex flex-wrap items-center gap-3">
<label for="research-start-at">回补起始时间</label>
<el-date-picker
id="research-start-at"
v-model="startText"
value-format="YYYY-MM-DD HH:mm:ss"
type="datetime"
:clearable="false"
:disabled="busy"
:disabled-date="disabledBackfillDate"
placeholder="选择回补起始时间"
/>
<span class="text-sm text-slate-500">北京时间(系统时区 UTC+8)</span>
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
核查遗漏
</el-button>
</div>
<p class="mt-2 text-sm text-slate-500">
点击“保存并暂停”后生效;重新启用时重排未完成任务。实时新闻优先插队,历史新闻分批补齐。
</p>
<p v-if="coverage" class="mt-2 text-sm">
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
失败项需手动重试,当前批次以外的遗漏排入下一批。
</p>
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
范围外已暂停自动处理的新闻 {{ coverage.outside.unprocessed }} 条:
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
已接收调用允许完成,扩大范围后恢复尚未完成的阶段;失败项需手动重试。
</p>
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
核查失败,请重试。
</p>
</div>
<template v-for="stage in (['index', 'research'] as const)" :key="stage"> <template v-for="stage in (['index', 'research'] as const)" :key="stage">
<h3>{{ stage === 'index' ? '第一阶段:关键词索引与事件归属' : '第二阶段:事件整体研究' }}</h3> <h3>{{ stage === 'index' ? '第一阶段:关键词索引与事件归属' : '第二阶段:事件整体研究' }}</h3>
<el-form-item label="模型接入"> <el-form-item label="模型接入">
+35 -7
View File
@@ -485,7 +485,7 @@ test("新闻列表只更新可见条目的匹配版本评级,保留滚动与
function researchPage(api: Record<string, unknown> = {}, user = vue.reactive({ sessionVersion: 1 }), extra: Record<string, unknown> = {}, dependencies: Record<string, unknown> = {}) { function researchPage(api: Record<string, unknown> = {}, user = vue.reactive({ sessionVersion: 1 }), extra: Record<string, unknown> = {}, dependencies: Record<string, unknown> = {}) {
const load = moduleLoader({ const load = moduleLoader({
"@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api }, "@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api },
"@@/apis/skills": { listSkillTreeApi: async () => [] }, "@@/apis/skills": { listSkillTreeApi: async () => [] },
"@@/apis/settings": { listProvidersApi: async () => [], listProviderModelsApi: async () => [] }, "@@/apis/settings": { listProvidersApi: async () => [], listProviderModelsApi: async () => [] },
"@/pinia/stores/user": { useUserStore: () => user }, "@/pinia/stores/user": { useUserStore: () => user },
@@ -516,11 +516,11 @@ test("回补时间按时区提交,刷新保留未提交的起点,核查不
assert.equal(enabled.length, 0) assert.equal(enabled.length, 0)
assert.equal(page.coverage.value.missing, 4) assert.equal(page.coverage.value.missing, 4)
await page.enable() await page.enable()
assert.deepEqual(enabled, ["2026-09-30T16:00:00.000Z"]) assert.deepEqual(enabled, [undefined])
}) })
test("其他标签页启用后按实际起点核查,暂停后恢复草稿并丢弃旧范围响应", async () => { test("其他标签页启用后按实际起点核查,暂停后恢复草稿并丢弃旧范围响应", async () => {
let server = { config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" } let server = { config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }
const pending = Promise.withResolvers() const pending = Promise.withResolvers()
const checked: string[] = [] const checked: string[] = []
let delayDraft = true let delayDraft = true
@@ -612,13 +612,13 @@ test("切换会话后,旧回补核查不能恢复统计结果", async () => {
assert.equal(page.startAt.value, null) assert.equal(page.startAt.value, null)
}) })
test("回补时间无效或晚于当前时间时禁止启用", async () => { test("回补时间无效或晚于当前时间时禁止保存", async () => {
let calls = 0 let calls = 0
const errors: string[] = [] const errors: string[] = []
const page = researchPage({ enableApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } }) const page = researchPage({ saveSettingsApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } })
for (const value of [null, new Date("invalid"), new Date(Date.now() + 86400000)]) { for (const value of [null, new Date("invalid"), new Date(Date.now() + 86400000)]) {
page.startAt.value = value page.startAt.value = value
await page.enable() await page.save()
} }
assert.equal(calls, 0) assert.equal(calls, 0)
assert.equal(errors.length, 3) assert.equal(errors.length, 3)
@@ -718,12 +718,15 @@ test("两阶段保存各自思考强度,不提交第三阶段配置", async ()
const page = researchPage({ saveSettingsApi: async (data: unknown) => { const page = researchPage({ saveSettingsApi: async (data: unknown) => {
saved = JSON.parse(JSON.stringify(data)) saved = JSON.parse(JSON.stringify(data))
} }) } })
await page.load()
page.config.value.index.reasoning_effort = "low" page.config.value.index.reasoning_effort = "low"
page.config.value.research.reasoning_effort = "high" page.config.value.research.reasoning_effort = "high"
await page.save() await page.save()
assert.equal(saved.index.reasoning_effort, "low") assert.equal(saved.index.reasoning_effort, "low")
assert.equal(saved.research.reasoning_effort, "high") assert.equal(saved.research.reasoning_effort, "high")
assert.equal(Object.keys(saved).length, 3) assert.equal(Object.keys(saved).length, 5)
assert.equal(saved.start_at, "2026-10-31T16:00:00.000Z")
assert.equal(saved.base_revision, 0)
page.config.value.research.reasoning_effort = null page.config.value.research.reasoning_effort = null
await page.save() await page.save()
assert.equal(saved.research.reasoning_effort, null) assert.equal(saved.research.reasoning_effort, null)
@@ -756,6 +759,7 @@ test("两阶段操作拒绝重复提交和旧会话操作,失败保留配置
calls++ calls++
return pending.promise return pending.promise
} }, user) } }, user)
await page.load()
page.config.value.index.model = "draft" page.config.value.index.model = "draft"
const first = page.save() const first = page.save()
await page.save() await page.save()
@@ -964,3 +968,27 @@ test("词库保存拒绝重复提交,关闭后的响应不覆盖草稿,失
assert.equal(page.vocabularyOpened.value, true) assert.equal(page.vocabularyOpened.value, true)
assert.equal(page.vocabularyBusy.value, false) assert.equal(page.vocabularyBusy.value, false)
}) })
test("侧栏时间统一使用北京时间,保存时间与配置版本但启用不提交草稿", async () => {
let saved: any
let enableArgs: unknown[] = []
const page = researchPage({
getSettingsApi: async () => ({ config: {}, enabled: true, control_revision: 7, start_at: "2026-11-04T16:00:00Z" }),
saveSettingsApi: async (value: unknown) => {
saved = value
},
enableApi: async (...args: unknown[]) => {
enableArgs = args
}
})
await page.settings()
assert.equal(page.startText.value, "2026-11-05 00:00:00")
page.startText.value = "2026-10-20 00:00:00"
await page.load()
assert.equal(page.startText.value, "2026-10-20 00:00:00")
await page.save()
assert.equal(saved.start_at, "2026-10-19T16:00:00.000Z")
assert.equal(saved.base_revision, 7)
await page.enable()
assert.deepEqual(enableArgs, [])
})