Prioritize realtime news and make historical backfill resumable
Development CI / check-and-build (push) Successful in 3m0s
GitHub Git mirror / mirror (push) Successful in 5s
Development CI / deploy-development (push) Successful in 14m7s

This commit is contained in:
eezd committed 2026-11-06 14:28:39 +08:00
1 parent c90691d9a0
commit 5f347f8ab4
17 files changed
+697 -143

No files matched your search

@@ -0,0 +1,24 @@
"""Persist realtime boundaries and enforce one active call per subject."""
from alembic import op
import sqlalchemy as sa
revision = 'dd4567890123'
down_revision = 'dc3456789012'
branch_labels = None
depends_on = None
def upgrade():
op.add_column('news_research_settings', sa.Column('realtime_from', sa.DateTime(timezone=True)))
op.add_column('news_research_events', sa.Column('realtime_pending', sa.Boolean(), nullable=False, server_default=sa.false()))
op.create_index('uq_news_attempt_active', 'news_research_attempts', ['package_id'], unique=True,
postgresql_where=sa.text("state = 'processing'"))
op.create_index('uq_news_version_active', 'news_research_versions', ['event_id'], unique=True,
postgresql_where=sa.text("state = 'processing'"))
def downgrade():
op.drop_index('uq_news_version_active', table_name='news_research_versions')
op.drop_index('uq_news_attempt_active', table_name='news_research_attempts')
op.drop_column('news_research_events', 'realtime_pending')
op.drop_column('news_research_settings', 'realtime_from')
+42 -16
View File
@@ -20,8 +20,9 @@ from alphapulse.news_research.models import (
Version,
Reference,
BatchEvent,
Batch,
)
from alphapulse.news_research import service, batches, vocabulary as vocab
from alphapulse.news_research import service, batches, scheduling, vocabulary as vocab
from alphapulse.news_research.models import VocabularyVersion
from alphapulse.news_research.contracts import (
INDEX_ASSIGN_SKILL,
@@ -90,6 +91,8 @@ class SettingsInput(Strict):
index: ModelConfig
research: ModelConfig
concurrency: int = Field(default=3, ge=1, le=16)
start_at: AwareDatetime | None = None
base_revision: int | None = Field(default=None, ge=0)
async def settings_row(session):
@@ -101,7 +104,7 @@ async def settings_row(session):
id=1,
config={},
enabled=False,
start_at=datetime(2026, 10, 31, 16, tzinfo=UTC),
start_at=datetime(2026, 11, 4, 16, tzinfo=UTC),
)
.on_conflict_do_nothing(index_elements=["id"])
)
@@ -115,6 +118,8 @@ async def settings(session: DB):
row = await settings_row(session)
return success(
{"config": row.config, "enabled": row.enabled, "start_at": row.start_at,
"control_revision": row.control_revision, "realtime_from": row.realtime_from,
"timezone": "Asia/Shanghai", "queue": await scheduling.queue_status(session, row),
"batch": await batches.progress(session)}
)
@@ -125,21 +130,24 @@ async def coverage(session: DB, start_at: AwareDatetime | None = None):
start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
if start > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间")
return success(await batches.coverage(session, start))
effective = min(start, row.realtime_from) if row.realtime_from else start
report = await batches.coverage(session, effective)
return success({**report, 'backfill_start_at': start, 'realtime_from': row.realtime_from})
@router.put("/settings")
async def save(body: SettingsInput, session: DB):
await settings_row(session)
await session.execute(
update(ResearchSettings)
.where(ResearchSettings.id == 1)
.values(
config=body.model_dump(),
enabled=False,
control_revision=ResearchSettings.control_revision + 1,
)
)
row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
if body.base_revision is not None and body.base_revision != row.control_revision:
raise HTTPException(409, "设置已变化,请重新读取后保存")
if body.start_at is not None:
if body.start_at > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间")
row.start_at = body.start_at.astimezone(UTC)
row.config = body.model_dump(exclude={"start_at", "base_revision"})
row.enabled = False
row.control_revision += 1
await session.commit()
return success(None)
@@ -164,6 +172,8 @@ class EnableInput(Strict):
async def enable(session: DB, body: EnableInput | None = None):
row = await settings_row(session)
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
if start_at != row.start_at:
raise HTTPException(422, "请在模型与 Skills 中保存回补时间并暂停后再启用")
if start_at > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间")
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
@@ -172,11 +182,11 @@ async def enable(session: DB, body: EnableInput | None = None):
raise HTTPException(409, "请先暂停两阶段处理")
running = await session.scalar(
select(func.count())
.select_from(Package)
.where(Package.state.in_(["index_processing", "assign_processing"]))
.select_from(Attempt)
.where(Attempt.state == "processing")
)
running += await session.scalar(
select(func.count()).select_from(Event).where(Event.state == "processing")
select(func.count()).select_from(Version).where(Version.state == "processing")
)
if running:
raise HTTPException(409, "请等待已接收任务完成后再验证")
@@ -259,7 +269,10 @@ async def enable(session: DB, body: EnableInput | None = None):
if row.control_revision != control_revision or row.config != original_config:
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
report = await batches.coverage(session, start_at)
row.start_at = start_at
if row.realtime_from is None:
row.realtime_from = datetime.now(UTC)
await session.execute(update(Batch).where(Batch.phase.in_(["assigning", "researching"]))
.values(phase="rescheduled", finished_at=datetime.now(UTC)))
row.enabled = True
row.control_revision += 1
await session.commit()
@@ -480,6 +493,18 @@ async def event(identifier: int, session: DB):
.order_by(Version.id.desc())
)
)
wait_reason = None
if row.state == 'pending':
settings = await settings_row(session)
unfinished = [(p, n) for p, n in members if p.state != 'done' or p.content_hash != n.content_hash]
if any(n.published_at < scheduling.boundary(settings) for p, n in unfinished):
wait_reason = '范围外成员尚未完成,扩大回补范围后可继续研究'
elif any(p.state.endswith('_failed') for p, n in unfinished):
wait_reason = '成员包处理失败,需手动重试后才能研究'
elif unfinished:
wait_reason = '等待成员包完成关键词与事件归属'
elif row.due_at > datetime.now(UTC):
wait_reason = '正在汇集事件变化,聚合窗口结束后研究'
latest = next((v for v in versions if v.state == "done"), None)
references = await version_references(session, latest.id) if latest else []
return success(
@@ -490,6 +515,7 @@ async def event(identifier: int, session: DB):
"result": row.result,
"error": row.error,
"revision": row.revision,
"wait_reason": wait_reason,
"packages": [await package_out(session, p, n) for p, n in members],
"references": references,
"versions": [
+40 -16
View File
@@ -5,6 +5,7 @@ from sqlalchemy import select, or_, and_, exists, func, any_, case
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
from alphapulse.news_research.service import enroll
from alphapulse.news_research.scheduling import scoped_event, ready_event
async def coverage(session, start_at, cutoff=None):
@@ -48,13 +49,22 @@ async def coverage(session, start_at, cutoff=None):
async def freeze(session, settings):
cutoff = datetime.now(UTC)
ids = list(await session.scalars(select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
or_(and_(NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff, Package.id.is_(None)),
and_(Package.id.is_not(None), Package.content_hash != NewsItem.content_hash)))))
existing_ids = list(await session.scalars(select(Package.id).where(
Package.state.in_(['index_pending', 'assign_pending', 'index_failed', 'assign_failed']))))
event_cutoff = list(await session.scalars(select(Event.id).where(Event.state == "pending")))
scope = [NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff]
realtime_from = getattr(settings, 'realtime_from', None)
if realtime_from is not None:
scope.append(NewsItem.published_at < realtime_from)
candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
*scope, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash,
Package.state.in_(['index_pending', 'assign_pending'])))
ids = list(await session.scalars(candidates.order_by(NewsItem.published_at, NewsItem.id).limit(30)))
await enroll(session, ids)
existing_ids = list(await session.scalars(select(Package.id).where(Package.news_id.in_(ids))))
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
event_query = select(Event.id).where(Event.state == 'pending', scoped_event(settings), ready_event(), Event.due_at <= cutoff, ~deferred)
if realtime_from is not None:
event_query = event_query.where(~Event.realtime_pending)
event_cutoff = list(await session.scalars(event_query.order_by(Event.due_at, Event.id).limit(30)))
# Failed items are reported once; only explicit retries or new source revisions rerun them.
previous_failure = exists(select(BatchPackage.batch_id).where(
BatchPackage.package_id == Package.id, BatchPackage.content_hash == Package.content_hash,
@@ -63,8 +73,6 @@ async def freeze(session, settings):
or_(Package.id == any_(existing_ids), Package.news_id == any_(ids)), or_(
Package.state.in_(['index_pending', 'assign_pending']),
Package.state.in_(['index_failed', 'assign_failed']) & ~previous_failure))))
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
events = list(await session.scalars(select(Event).where(Event.id == any_(event_cutoff), Event.state == 'pending', ~deferred)))
if not packages and not events:
return None
@@ -83,7 +91,7 @@ async def freeze(session, settings):
return batch
async def actions(session, batch, active, slots):
async def actions(session, batch, active, slots, settings=None):
"""Return stage/ID pairs; callers commit before launching and never overlap phases."""
if batch.phase == 'assigning':
rows = list(await session.execute(select(BatchPackage, Package, NewsItem).join(
@@ -94,7 +102,9 @@ async def actions(session, batch, active, slots):
for item, package, news in rows:
if item.state != 'pending':
continue
if item.content_hash != news.content_hash or item.content_hash != package.content_hash:
if settings is not None and news.published_at < settings.start_at:
item.state = 'range_paused'
elif item.content_hash != news.content_hash or item.content_hash != package.content_hash:
item.state = 'deferred'
elif package.state == 'done':
item.state = 'done'
@@ -102,10 +112,12 @@ async def actions(session, batch, active, slots):
item.state = 'failed'
elif package.state in ('index_pending', 'assign_pending'):
stage = 'index' if package.state == 'index_pending' else 'assign'
if (stage, package.id) not in active and len(result) < slots and (stage != 'assign' or not assigning):
if not any(identifier == package.id and task != 'event' for task, identifier in active) and len(result) < slots and (stage != 'assign' or not assigning):
result.append((stage, package.id))
assigning |= stage == 'assign'
if any(item.state == 'pending' for item, _, _ in rows) or active:
package_ids = {package.id for _, package, _ in rows}
batch_active = any(stage != 'event' and identifier in package_ids for stage, identifier in active)
if any(item.state == 'pending' for item, _, _ in rows) or batch_active:
return result
# Memberships now include events created by this batch. Existing affected events
# were frozen up front, including memberships that have since been removed.
@@ -131,7 +143,13 @@ async def actions(session, batch, active, slots):
for item, event in rows:
if item.state not in ('pending', 'urgent_pending', 'processing') or ('event', event.id) in active:
continue
if item.revision != event.revision:
if settings is not None and not await session.scalar(select(Event.id).where(Event.id == event.id, scoped_event(settings))):
item.state = 'range_paused'
elif settings is not None and event.realtime_pending:
item.state = 'realtime'
elif event.due_at > datetime.now(UTC):
item.state = 'aggregating'
elif item.revision != event.revision:
item.state = 'deferred'
elif event.state == 'done' or event.state == 'retired':
item.state = 'done'
@@ -143,11 +161,17 @@ async def actions(session, batch, active, slots):
invalid = await session.scalar(select(Membership.event_id).join(Package).join(NewsItem).where(
Membership.event_id == event.id, or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)).limit(1))
if invalid:
item.state = 'deferred'
item.state = 'blocked'
elif len(result) < slots:
item.state = 'processing'
result.append(('event', event.id))
if not result and not active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows):
event_ids = {event.id for _, event in rows}
batch_active = any(stage == 'event' and identifier in event_ids for stage, identifier in active)
active_packages = {identifier for stage, identifier in active if stage != 'event'}
if active_packages:
batch_active |= bool(await session.scalar(select(BatchPackage.package_id).where(
BatchPackage.batch_id == batch.id, BatchPackage.package_id.in_(active_packages)).limit(1)))
if not result and not batch_active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows):
bad = any(item.state in ('failed', 'deferred') for item, _ in rows)
bad |= bool(await session.scalar(select(BatchPackage.batch_id).where(
BatchPackage.batch_id == batch.id, BatchPackage.state.in_(['failed', 'deferred'])).limit(1)))
@@ -163,5 +187,5 @@ async def progress(session):
result = {'id': batch.id, 'phase': batch.phase, 'cutoff_at': batch.cutoff_at}
for name, model in [('packages', BatchPackage), ('events', BatchEvent)]:
counts = Counter(await session.scalars(select(model.state).where(model.batch_id == batch.id)))
result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred')}})
result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred', 'range_paused', 'blocked', 'realtime')}})
return result
@@ -6,6 +6,8 @@ from sqlalchemy import (
DateTime,
ForeignKey,
Integer,
Index,
text,
String,
Text,
UniqueConstraint,
@@ -32,6 +34,7 @@ class ResearchSettings(Base):
Integer, default=0, server_default="0"
)
start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
realtime_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Package(Base):
@@ -49,6 +52,7 @@ class Package(Base):
class Attempt(Base):
__tablename__ = "news_research_attempts"
__table_args__ = (Index("uq_news_attempt_active", "package_id", unique=True, postgresql_where=text("state = 'processing'")),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
package_id: Mapped[int] = mapped_column(
ForeignKey("news_research_packages.id"), index=True
@@ -75,6 +79,7 @@ class Event(Base):
state: Mapped[str] = mapped_column(String(16), default="pending", index=True)
result: Mapped[dict | None] = mapped_column(JSONB)
error: Mapped[str | None] = mapped_column(Text)
realtime_pending: Mapped[bool] = mapped_column(default=False, server_default="false")
dirty_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
due_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), index=True
@@ -99,6 +104,7 @@ class Membership(Base):
class Version(Base):
__tablename__ = "news_research_versions"
__table_args__ = (Index("uq_news_version_active", "event_id", unique=True, postgresql_where=text("state = 'processing'")),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
event_id: Mapped[int] = mapped_column(
ForeignKey("news_research_events.id"), index=True
@@ -0,0 +1,103 @@
"""Priority selection over durable package and event states; no paid work here."""
from datetime import UTC, datetime
from sqlalchemy import select, exists, or_, and_, func, case
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
def boundary(settings):
realtime_from = getattr(settings, 'realtime_from', None)
return min(settings.start_at, realtime_from) if realtime_from else settings.start_at
def eligible(settings):
return and_(NewsItem.published_at >= boundary(settings), NewsItem.published_at <= datetime.now(UTC))
def members(event_id):
return select(Membership.package_id).where(Membership.event_id == event_id)
def ready_event():
return ~exists(select(Membership.package_id).join(Package).join(NewsItem).where(
Membership.event_id == Event.id,
or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)))
def scoped_event(settings):
# Empty events must still reach the existing no-model retirement path.
return or_(exists(select(Membership.package_id).join(Package).join(NewsItem).where(
Membership.event_id == Event.id, eligible(settings))),
~exists(select(Membership.package_id).where(Membership.event_id == Event.id)))
async def control(session, config):
"""The control lock spans the durable claim, never the remote model call."""
if '_control_revision' not in config:
return None # Explicit bounded/offline callers retain their existing contract.
row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
if not row.enabled or row.control_revision != config['_control_revision']:
return False
return row
async def mark_priority(session, event, news):
settings = await session.get(ResearchSettings, 1)
if settings and settings.realtime_from and news.published_at >= settings.realtime_from:
event.realtime_pending = True
async def realtime_actions(session, settings, active, slots):
if settings.realtime_from is None or slots <= 0:
return []
now = datetime.now(UTC)
busy_packages = {identifier for stage, identifier in active if stage != 'event'}
busy_events = {identifier for stage, identifier in active if stage == 'event'}
result = []
events = await session.scalars(select(Event).where(
Event.state == 'pending', Event.realtime_pending, Event.due_at <= now,
scoped_event(settings), ready_event(), Event.id.not_in(busy_events),
~exists(select(Version.id).where(Version.event_id == Event.id, Version.state == 'processing')),
).order_by(Event.due_at, Event.id).limit(slots))
result.extend(('event', event.id) for event in events)
# Known historical dependencies inherit the waiting realtime event's priority.
dependency = exists(select(Membership.package_id).join(Event).where(
Membership.package_id == Package.id, Event.realtime_pending, Event.state == 'pending'))
priority = or_(NewsItem.published_at >= settings.realtime_from, dependency)
candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
eligible(settings), priority, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash),
or_(Package.id.is_(None), Package.id.not_in(busy_packages))).order_by(NewsItem.published_at, NewsItem.id).limit(30)
ids = list(await session.scalars(candidates))
if ids:
from alphapulse.news_research.service import enroll
await enroll(session, ids)
assigning = any(stage == 'assign' for stage, _ in active)
for stage in ('assign', 'index'):
if stage == 'assign' and assigning:
continue
remaining = slots - len(result)
if remaining <= 0:
break
query = select(Package.id).join(NewsItem).where(
eligible(settings), priority, Package.state == stage + '_pending',
Package.id.not_in(busy_packages),
~exists(select(Attempt.id).where(Attempt.package_id == Package.id, Attempt.state == 'processing')),
).order_by(NewsItem.published_at, NewsItem.id).limit(1 if stage == 'assign' else remaining)
result.extend((stage, identifier) for identifier in await session.scalars(query))
return result
async def queue_status(session, settings):
running = await session.scalar(select(func.count()).select_from(Attempt).where(Attempt.state == 'processing'))
running += await session.scalar(select(func.count()).select_from(Version).where(Version.state == 'processing'))
realtime_from = settings.realtime_from
lane = case((NewsItem.published_at >= realtime_from, 'realtime'), else_='backfill') if realtime_from else 'backfill'
if realtime_from:
groups = (await session.execute(select(lane, Package.state, func.count()).join(NewsItem)
.where(eligible(settings)).group_by(lane, Package.state))).all()
else:
groups = [('backfill', state, count) for state, count in (await session.execute(
select(Package.state, func.count()).join(NewsItem).where(eligible(settings)).group_by(Package.state))).all()]
queues = {name: {state: count for selected, state, count in groups if selected == name}
for name in ('realtime', 'backfill')}
return {'running': running, 'draining': not settings.enabled and running > 0, **queues}
@@ -27,6 +27,7 @@ from alphapulse.news_research.models import (
Version,
Reference,
)
from alphapulse.news_research.scheduling import control, mark_priority, boundary
from alphapulse.news_research.contracts import (
INDEX_ASSIGN_SKILL,
task_skill,
@@ -280,6 +281,7 @@ async def enroll(session, ids):
.with_for_update()
):
dirty(event)
await mark_priority(session, event, row)
await session.commit()
@@ -429,6 +431,9 @@ async def process_package(identifier, config):
async def _process_package(identifier, config, expected_stage):
async with SessionLocal() as session:
settings = await control(session, config)
if settings is False:
return
package = await session.get(Package, identifier, with_for_update=True)
if package is None:
return
@@ -438,6 +443,10 @@ async def _process_package(identifier, config, expected_stage):
if package.content_hash != news.content_hash:
await enroll(session, [news.id])
return
if settings is not None and not boundary(settings) <= news.published_at <= datetime.now(UTC):
return
if await session.scalar(select(Attempt.id).where(Attempt.package_id == identifier, Attempt.state == 'processing').limit(1)):
return
source = article(news)
source_hash = package.content_hash
stage = "index" if package.state == "index_pending" else "assign"
@@ -566,12 +575,14 @@ async def _process_package(identifier, config, expected_stage):
existing.change_kind = target.change_kind
if target.change_kind != "repost" or event.result is None:
dirty(event, target.urgent)
await mark_priority(session, event, news)
for old in previous:
if old.event_id not in assigned:
event = await session.get(
Event, old.event_id, with_for_update=True
)
dirty(event)
await mark_priority(session, event, news)
await session.delete(old)
affected[event.id] = event
package.state = "done"
@@ -805,6 +816,9 @@ def validate_timeline(result, package_news):
async def process_event(identifier, config, force=False):
async with SessionLocal() as session:
settings = await control(session, config)
if settings is False:
return
event = await session.get(Event, identifier, with_for_update=True)
if (
event is None
@@ -812,6 +826,8 @@ async def process_event(identifier, config, force=False):
or (not force and event.due_at > datetime.now(UTC))
):
return
if await session.scalar(select(Version.id).where(Version.event_id == identifier, Version.state == 'processing').limit(1)):
return
members = list(
await session.execute(
select(Package, Membership, NewsItem)
@@ -821,6 +837,8 @@ async def process_event(identifier, config, force=False):
.order_by(Package.id)
)
)
if settings is not None and members and not any(boundary(settings) <= n.published_at <= datetime.now(UTC) for p, m, n in members):
return
if not members:
event.state = "retired"
await session.commit()
@@ -859,6 +877,7 @@ async def process_event(identifier, config, force=False):
)
session.add(version)
event.state = "processing"
event.dirty_since = None
await session.commit()
version_id, revision = version.id, event.revision
researcher = Research(event={"id": identifier, "revision": revision})
@@ -977,6 +996,7 @@ async def process_event(identifier, config, force=False):
current.title = output.title
current.state = "done"
current.dirty_since = None
current.realtime_pending = False
current.error = None
for ref in output.references:
session.add(
+38 -33
View File
@@ -13,38 +13,42 @@ from alphapulse.news_research.models import (
Batch, BatchEvent,
)
from alphapulse.news_research.service import process_package, process_event
from alphapulse.news_research import batches
from alphapulse.news_research import batches, scheduling
logger = logging.getLogger(__name__)
async def recover_interrupted():
async with SessionLocal() as session:
for stage in ("index", "assign"):
await session.execute(
update(Package)
.where(Package.state == stage + "_processing")
.values(state=stage + "_failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Attempt)
.where(Attempt.state == "processing")
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Event).where(Event.state == "processing").values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(
update(Version)
.where(Version.state == "processing")
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
)
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
await session.commit()
async def run():
async with engine.connect() as lock:
if not await lock.scalar(text("SELECT pg_try_advisory_lock(20261030, 1)")):
raise RuntimeError("新闻AI worker 已运行")
await lock.commit()
async with SessionLocal() as session:
for stage in ("index", "assign"):
await session.execute(
update(Package)
.where(Package.state == stage + "_processing")
.values(state=stage + "_pending")
)
await session.execute(
update(Attempt)
.where(Attempt.state == "processing")
.values(state="failed", error="进程中断,待重试")
)
await session.execute(
update(Event).where(Event.state == "processing").values(state="pending")
)
await session.execute(
update(Version)
.where(Version.state == "processing")
.values(state="failed", error="进程中断,待重试")
)
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
await session.commit()
await recover_interrupted()
tasks = {}
try:
while True:
@@ -58,20 +62,21 @@ async def run():
async with SessionLocal() as session:
settings = await session.get(ResearchSettings, 1)
if settings and settings.enabled:
active = set(tasks.values())
slots = max(0, settings.config.get('concurrency', 3) - len(tasks))
actions = await scheduling.realtime_actions(session, settings, active, slots)
batch = await session.scalar(select(Batch).where(
Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1))
if batch is None:
batch = await batches.freeze(session, settings)
settings = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
if batch is not None and settings.enabled:
active = set(tasks.values())
actions = await batches.actions(session, batch, active,
max(0, settings.config.get('concurrency', 3) - len(tasks)))
await session.commit()
for stage, identifier in actions:
call = (process_event(identifier, settings.config, force=True)
if stage == 'event' else process_package(identifier, settings.config))
tasks[asyncio.create_task(call)] = (stage, identifier)
if batch is not None:
actions += await batches.actions(session, batch, active | set(actions), slots - len(actions), settings)
await session.commit()
config = {**settings.config, '_control_revision': settings.control_revision}
for stage, identifier in actions:
call = (process_event(identifier, config)
if stage == 'event' else process_package(identifier, config))
tasks[asyncio.create_task(call)] = (stage, identifier)
await asyncio.sleep(1)
finally:
await asyncio.gather(*tasks, return_exceptions=True)
+46 -1
View File
@@ -302,7 +302,7 @@ def test_failed_enable_does_not_save_the_selected_backfill_start(trading, monkey
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
with pytest.raises(HTTPException) as error:
asyncio.run(api.enable(session, api.EnableInput(start_at=previous - timedelta(days=30))))
asyncio.run(api.enable(session, api.EnableInput(start_at=previous)))
assert error.value.status_code == 502
assert row.start_at == previous and not row.enabled and row.control_revision == 7
session.commit.assert_not_called()
@@ -842,6 +842,7 @@ def test_alias_adjustment_audit_survives_index_failures(trading, monkeypatch, fa
session.get = AsyncMock(side_effect=lambda model, *args, **kwargs:
attempts[0] if model is Attempt else package if model is Package else news)
session.execute = AsyncMock()
session.scalar = AsyncMock(return_value=None)
monkeypatch.setattr(service, "SessionLocal", lambda: session)
monkeypatch.setattr(service.vocab, "snapshot", AsyncMock(return_value={"entries": []}))
monkeypatch.setattr(service, "model_config", AsyncMock(return_value=(None, "索引规则", {})))
@@ -903,3 +904,47 @@ def test_task_skill_keeps_custom_rules_and_accepts_crlf(trading):
content = "## 归属规则\r\n自定义归属\r\n### 细则\r\n保留规则\r\n## 索引规则\r\n自定义索引"
assert task_skill(content, "assign") == "自定义归属\n### 细则\n保留规则"
assert task_skill(content, "index") == "自定义索引"
def test_settings_save_atomically_pauses_and_preserves_model_contract(trading):
import asyncio
from unittest.mock import AsyncMock
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=3)
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
model = dict(provider_id=1, model='test', skill_id=1)
body = api.SettingsInput(index=model, research=model, start_at='2026-10-20T00:00:00+08:00', base_revision=3)
asyncio.run(api.save(body, session))
assert not row.enabled and row.control_revision == 4
assert row.start_at == datetime(2026, 10, 19, 16, tzinfo=UTC)
assert set(row.config) == {'index', 'research', 'concurrency'}
session.commit.assert_awaited_once()
def test_settings_save_rejects_other_tabs_revision_without_changes(trading):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=4)
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
model = dict(provider_id=1, model='test', skill_id=1)
with pytest.raises(HTTPException) as error:
asyncio.run(api.save(api.SettingsInput(index=model, research=model, base_revision=3), session))
assert error.value.status_code == 409
assert row.enabled and row.control_revision == 4
session.commit.assert_not_awaited()
def test_enable_cannot_apply_an_unsaved_backfill_time(trading):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC))
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(), commit=AsyncMock())
with pytest.raises(HTTPException) as error:
asyncio.run(api.enable(session, api.EnableInput(start_at='2026-10-20T00:00:00+08:00')))
assert error.value.status_code == 422
session.scalar.assert_not_awaited()
session.commit.assert_not_awaited()
+39
View File
@@ -455,3 +455,42 @@ some documents were imported: ECB dynamic index version coverage, inaccessible
BoE speeches and CoinDesk access blocks are reported rather than labeled complete.
The deployment lock serializes writes; crawler/calendar processes are restored in
finally blocks. Do not start a second driver or remove its state to renew quotas.
### Realtime priority and historical AI backfill
Migration `dd4567890123` adds the persistent realtime boundary and pending-event
priority, plus partial unique indexes for active package attempts and event
versions. Pause and drain news AI before applying it; duplicate active records
must be investigated rather than deleted to force migration success. Preserve
originals, attempts and version history. Deployment requires the candidate's
schema head and must leave automatic processing paused during verification.
The model/Skills drawer saves the historical start time with model settings via
**Save and pause**. New settings default to 2026-11-05 00:00 Asia/Shanghai; existing
saved values remain intact. Settings responses expose `control_revision`,
`realtime_from`, `timezone` and queue counts. PUT accepts optional `start_at` and
`base_revision`; version conflicts return 409. Enable uses the saved start only;
a differing legacy enable body returns 422. The first successful enable fixes
`realtime_from`; later saves, pauses and restarts never move it.
Publication time, not ingestion time, determines realtime membership. Realtime
work takes the next available slot without cancelling accepted calls. Historical
work uses batches of at most 30 originals, ordered chronologically. Saving a new
start pauses dispatch; enabling after drain retires unfinished batch schedules
and rebuilds from durable stage states without repeating successes. Shrinking
history does not exclude realtime backlog. Known in-range dependencies inherit
a waiting realtime event's priority. Out-of-range or failed members block event
research without silently dropping evidence or retrying paid failures.
Ordinary event research uses the existing five-minute quiet window with a
15-minute maximum; urgent events become ready immediately. A running research
call is not duplicated when newer members arrive. New changes start another
aggregation window and obsolete output cannot overwrite current conclusions.
After a worker interruption, started calls become failed with an unknown-result
message and need manual retry; unclaimed work resumes normally. Strict realtime
priority can starve historical work under sustained overload.
CI additionally runs `scripts/smoke_priority_queue.py` in its disposable internal
network with synthetic credentials and no model calls. It covers preemption,
late historical ingestion, stable realtime eligibility, range changes,
dependencies, aggregation, control revisions and interrupted-call recovery.
+2
View File
@@ -182,6 +182,8 @@ def smoke(build, label):
'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)')
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_priority_queue.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
+103 -19
View File
@@ -90,6 +90,7 @@ async def main():
await backfill_regressions(now)
await carryover_regressions(now)
await regressions(now, settings)
await queue_review_regressions(now, settings)
await engine.dispose()
print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
@@ -208,7 +209,13 @@ async def regressions(now, settings):
async with SessionLocal() as s:
event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now)
previous = Batch(phase='partial', cutoff_at=now)
s.add_all([event, previous])
original = NewsItem(source_code='wscn_news', external_id='retry-member', title='retry', summary='', content_text='retry', content_hash='a'*64, url='https://example.invalid/retry', published_at=now)
s.add_all([event, previous, original])
await s.flush()
member = Package(news_id=original.id, content_hash=original.content_hash, state='done', keywords=[])
s.add(member)
await s.flush()
s.add(Membership(event_id=event.id, package_id=member.id, evidence='retry', reason='retry', change_kind='new_fact'))
await s.flush()
s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred'))
await s.commit()
@@ -236,39 +243,116 @@ async def regressions(now, settings):
s.add_all([news, excluded])
await s.commit()
await service.enroll(s, [news.id])
batch = await batches.freeze(s, settings)
assert await batches.freeze(s, settings) is None
package = await s.scalar(select(Package).where(Package.news_id == news.id))
news.content_hash = 'b'*64
await s.commit()
await batches.actions(s, batch, set(), 8)
await s.commit()
await batches.actions(s, batch, set(), 8)
await s.commit()
assert batch.phase == 'partial'
batch = await batches.freeze(s, settings)
assert package.content_hash == news.content_hash and package.state == 'index_pending'
assert await batches.actions(s, batch, set(), 8) == [('index', package.id)]
assert await s.scalar(select(func.count()).select_from(Package)) == 1
# A previously completed manual original also refreshes outside the automatic boundary.
package.state = 'done'
news.content_hash = 'c'*64
await s.commit()
await batches.freeze(s, settings)
assert await batches.freeze(s, settings) is None
assert package.content_hash != news.content_hash
expanded = SimpleNamespace(start_at=news.published_at)
batch = await batches.freeze(s, expanded)
assert package.content_hash == news.content_hash and package.state == 'index_pending'
assert ('index', package.id) in await batches.actions(s, batch, set(), 8)
await clear(s)
count = 33000
await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)])
await s.commit()
batch = await batches.freeze(s, settings)
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == count
assert await s.scalar(select(func.count()).select_from(Package)) == count
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == 30
assert await s.scalar(select(func.count()).select_from(Package)) == 30
# Exercise the existing-pending-ID and membership arrays above the driver limit too.
next_batch = await batches.freeze(s, settings)
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == count
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == 30
assert len(await batches.actions(s, next_batch, set(), 8)) == 8
await clear(s)
print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.')
async def queue_review_regressions(now, settings):
async def clear(session):
await session.execute(text('TRUNCATE news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
await session.commit()
async with SessionLocal() as s:
news = NewsItem(source_code='wscn_news', external_id='review-barrier', title='historical',
summary='', content_text='historical', content_hash='a'*64,
url='https://example.invalid/review-barrier', published_at=now)
s.add(news)
await s.flush()
package = Package(news_id=news.id, content_hash=news.content_hash, state='assign_pending', keywords=[])
event = Event(title='historical event', tags=[], revision=1, state='pending', due_at=now)
batch = Batch(phase='assigning', cutoff_at=now)
s.add_all([package, event, batch])
await s.flush()
s.add(Membership(event_id=event.id, package_id=package.id,
evidence='historical', reason='same event', change_kind='new_fact'))
s.add(BatchPackage(batch_id=batch.id, package_id=package.id, content_hash=package.content_hash, state='pending'))
await s.commit()
unrelated = package.id + 1000
# Assignment remains globally serial even when the other call is not in this batch.
assert await batches.actions(s, batch, {('assign', unrelated)}, 7) == []
assert batch.phase == 'assigning'
# Active work on this batch still holds its assignment barrier.
package.state = 'done'
await s.commit()
assert await batches.actions(s, batch, {('assign', package.id)}, 7) == []
assert batch.phase == 'assigning'
# Unrelated realtime package and event calls must not hold the barrier.
realtime = {('index', unrelated), ('event', event.id + 1000)}
assert await batches.actions(s, batch, realtime, 6) == []
await s.commit()
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime, 6) == [('event', event.id)]
await s.commit()
event.state = 'done'
await s.commit()
# A completed result can be visible before the owning task has returned.
assert await batches.actions(s, batch, realtime | {('event', event.id)}, 5) == []
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime | {('index', package.id)}, 5) == []
assert batch.phase == 'researching'
assert await batches.actions(s, batch, realtime, 6) == []
await s.commit()
assert batch.phase == 'done'
# Completed batches release admission of the next historical originals.
news2 = NewsItem(source_code='wscn_news', external_id='review-next', title='next', summary='',
content_text='next', content_hash='b'*64,
url='https://example.invalid/review-next', published_at=now)
s.add(news2)
await s.commit()
assert (await batches.freeze(s, settings)).id != batch.id
await clear(s)
# Thirty manual-only deferred heads cannot consume the thirty runnable slots.
news = NewsItem(source_code='wscn_news', external_id='review-deferred', title='source', summary='',
content_text='source', content_hash='a'*64,
url='https://example.invalid/review-deferred', published_at=now)
s.add(news)
await s.flush()
package = Package(news_id=news.id, content_hash=news.content_hash, state='done', keywords=[])
previous = Batch(phase='partial', cutoff_at=now)
events = [Event(title='deferred-head-'+str(i), tags=[], revision=7, state='pending',
due_at=now-timedelta(minutes=35-i)) for i in range(35)]
s.add_all([package, previous, *events])
await s.flush()
for i, event in enumerate(events):
s.add(Membership(event_id=event.id, package_id=package.id, evidence='source', reason='same source', change_kind='new_fact'))
if i <= 30:
s.add(BatchEvent(batch_id=previous.id, event_id=event.id,
revision=7 if i < 30 else 6, state='deferred'))
await s.commit()
batch = await batches.freeze(s, settings)
assert batch is not None
selected = set(await s.scalars(select(BatchEvent.event_id).where(BatchEvent.batch_id == batch.id)))
assert selected == {event.id for event in events[30:]}, selected
assert all(event.state == 'pending' and event.revision == 7 for event in events[:30])
await batches.actions(s, batch, set(), 8)
await s.commit()
actions = await batches.actions(s, batch, set(), 8)
assert set(actions) == {('event', event.id) for event in events[30:]}
await clear(s)
print('Queue review regressions passed: unrelated realtime calls allow both batch barriers to advance, owning calls still block, assignment stays serial, and deferred heads cannot starve runnable events.')
asyncio.run(main())
+9 -2
View File
@@ -393,14 +393,20 @@ async def main():
assert report.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert set(report.json()["data"]["outside"]) == {"unprocessed", "stale", "pending", "processing", "failed"}
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == previous_start
assert (await client.post("/api/risk/enable", json={"start_at": selected_start})).status_code == 422
settings = (await client.get("/api/risk/settings")).json()["data"]
saved = await client.put("/api/risk/settings", json={**config, "start_at": selected_start, "base_revision": settings["control_revision"]})
assert saved.status_code == 200, saved.text
with patch.object(model_service, "complete", validation):
response = await client.post("/api/risk/enable", json={"start_at": selected_start})
response = await client.post("/api/risk/enable")
assert response.status_code == 200, response.text
assert response.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert (await client.get("/api/risk/settings")).json()["data"][
"enabled"
] is True
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
fixed_anchor = (await client.get("/api/risk/settings")).json()["data"]["realtime_from"]
assert fixed_anchor is not None
assert (await client.post("/api/risk/enable")).status_code == 409
assert (await client.post("/api/risk/pause")).status_code == 200
for action in ("pause", "save"):
@@ -421,12 +427,13 @@ async def main():
return await validation(provider, skill, payload, schema)
with patch.object(model_service, "complete", interrupted_validation):
response = await client.post("/api/risk/enable", json={"start_at": "2026-09-01T00:00:00Z"})
response = await client.post("/api/risk/enable")
assert response.status_code == 409, response.text
assert (await client.get("/api/risk/settings")).json()["data"][
"enabled"
] is False
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
assert (await client.get("/api/risk/settings")).json()["data"]["realtime_from"] == fixed_anchor
package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[
"data"
][0]["id"]
+107
View File
@@ -0,0 +1,107 @@
"""Exercise preemption and recovery using disposable PostgreSQL, never a model."""
import asyncio
import os
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from urllib.parse import urlparse
from sqlalchemy import select, text, func
from sqlalchemy.exc import IntegrityError
from unittest.mock import patch
from alphapulse.core.db import SessionLocal, engine
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
from alphapulse.news_research import scheduling, batches, service
from alphapulse.news_research.worker import recover_interrupted
async def main():
url = urlparse(os.environ['DATABASE_URL'])
assert (url.username, url.password, url.path) == ('ci', 'ci-only', '/ci')
now = datetime.now(UTC)
anchor = now - timedelta(hours=1)
config = SimpleNamespace(start_at=now-timedelta(days=5), realtime_from=anchor)
async with SessionLocal() as s:
assert await s.scalar(select(func.count()).select_from(Package)) == 0
def original(key, published):
return NewsItem(source_code='wscn_news', external_id='priority-'+key, title=key, summary='',
content_text=key, content_hash='a'*64, url='https://example.invalid/'+key, published_at=published)
old = original('old', now-timedelta(days=4))
live = original('live', now-timedelta(minutes=30))
late = original('late', now-timedelta(days=3))
s.add_all([old, live, late]); await s.commit()
batch = await batches.freeze(s, config)
packages = list(await s.scalars(select(Package)))
assert len(packages) == 2 and all(p.news_id != live.id for p in packages)
actions = await scheduling.realtime_actions(s, config, {('index', packages[0].id)}, 1)
live_package = await s.scalar(select(Package).where(Package.news_id == live.id))
assert actions == [('index', live_package.id)]
assert await scheduling.realtime_actions(s, config, set(), 0) == []
# Shrinking the history range must not exclude accumulated realtime news.
config.start_at = now-timedelta(minutes=5)
actions = await scheduling.realtime_actions(s, config, set(), 1)
assert actions == [('index', live_package.id)]
await batches.actions(s, batch, set(), 8, config)
assert not await batches.freeze(s, config)
config.start_at = now-timedelta(days=5)
assert await batches.freeze(s, config)
assert packages[0].keywords == []
# A realtime event's historical dependencies inherit priority.
event = Event(title='shared', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
s.add(event); await s.flush()
for p in [packages[0], live_package]:
s.add(Membership(event_id=event.id, package_id=p.id, evidence='fixture', reason='fixture', change_kind='new_fact'))
packages[0].state = 'assign_pending'
live_package.state = 'done'
await s.commit()
config.start_at = now-timedelta(minutes=5)
assert await scheduling.realtime_actions(s, config, set(), 1) == []
config.start_at = now-timedelta(days=5)
assert await scheduling.realtime_actions(s, config, set(), 1) == [('assign', packages[0].id)]
packages[0].state = 'done'; await s.commit()
assert await scheduling.realtime_actions(s, config, set(), 1) == [('event', event.id)]
event.due_at = now+timedelta(minutes=5); await s.commit()
assert ('event', event.id) not in await scheduling.realtime_actions(s, config, set(), 8)
# Pausing invalidates a selected but not-yet-claimed task.
s.add(ResearchSettings(id=1, config={}, enabled=False, control_revision=9, start_at=config.start_at, realtime_from=anchor))
await s.commit()
assert await scheduling.control(s, {'_control_revision': 8}) is False
await s.commit()
with patch.object(service, 'complete', side_effect=AssertionError('Paused selection must not call a model')):
await service.process_package(live_package.id, {'_control_revision': 8})
assert await s.scalar(select(func.count()).select_from(Attempt)) == 0
live_package = await s.get(Package, live_package.id)
live_package.state = 'index_processing'
s.add(Attempt(package_id=live_package.id, content_hash='a'*64, stage='index', state='processing', input={}, rules={}))
event = await s.get(Event, event.id); event.state='processing'
s.add(Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}))
await s.commit()
for duplicate in [
Attempt(package_id=live_package.id, content_hash='a'*64, stage='assign', state='processing', input={}, rules={}),
Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}),
]:
try:
async with s.begin_nested():
s.add(duplicate)
await s.flush()
except IntegrityError:
pass
else:
raise AssertionError('Concurrent duplicate claim was accepted')
await recover_interrupted()
async with SessionLocal() as s:
assert (await s.get(Package, live_package.id)).state == 'index_failed'
assert (await s.get(Event, event.id)).state == 'failed'
assert await scheduling.realtime_actions(s, config, set(), 8) == []
empty = Event(title='removed members', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
s.add(empty); await s.commit()
assert ('event', empty.id) in await scheduling.realtime_actions(s, config, set(), 8)
with patch.object(service.Research, 'run', side_effect=AssertionError('Empty events must not call a model')):
await service.process_event(empty.id, {})
await s.refresh(empty)
assert empty.state == 'retired'
await s.execute(text('TRUNCATE news_research_settings, news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
await s.commit()
await engine.dispose()
print('Priority queue passed: live preemption, stable boundary, late history, range changes, dependencies, aggregation, stale control and uncertain-call recovery.')
asyncio.run(main())
+2 -2
View File
@@ -2,8 +2,8 @@ import type * as Research from "./type"
import { request } from "@/http/axios"
export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" })
export const saveSettingsApi = (data: Research.Config) => request<null>({ url: "/risk/settings", method: "put", data })
export const enableApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/enable", method: "post", data: { start_at } })
export const saveSettingsApi = (data: Research.Config & { start_at: string, base_revision: number }) => request<null>({ url: "/risk/settings", method: "put", data })
export const enableApi = () => request<Research.Coverage>({ url: "/risk/enable", method: "post" })
export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } })
export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" })
export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" })
+2 -2
View File
@@ -2,13 +2,13 @@ export interface Keyword { canonical: string, aliases: string[], quote: string }
export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null }
export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number }
export interface BatchProgress { id: number, phase: string, cutoff_at: string, packages_total: number, packages_done: number, packages_failed: number, packages_deferred: number, events_total: number, events_done: number, events_failed: number, events_deferred: number }
export interface Settings { batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string }
export interface Settings { control_revision: number, realtime_from: string | null, queue: { running: number, draining: boolean, realtime: Record<string, number>, backfill: Record<string, number> }, batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string }
export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } }
export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] }
export interface Point { time_precision: "minute" | "day" | "month" | "unknown", planned_precision: "minute" | "day" | "month" | "unknown", package_ids: number[], time_text: string, description: string, occurred_at: string | null, planned_at: string | null, reported_at: string | null, period_text: string, evidence_refs: { article_id: number, quote: string }[] }
export interface Result { title: string, summary: string, facts: string, reasoning: string, uncertainty: string, opinions: string, counter_evidence: string, watchpoints: string, asset_impacts: string, importance_stars: number, risk_stars: number, confidence: string, verification: string, timeline_points: Point[] }
export interface Event { id: number, title: string, state: string, revision: number, error: string | null, tags: string[], result: Result | null }
export interface EventDetail extends Event { packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] }
export interface EventDetail extends Event { wait_reason?: string | null, packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] }
export interface PackageDetail extends Package { article: { title: string, summary: string, content: string }, attempts: { id: number, stage: string, state: string, error: string | null, output: unknown, usage: unknown }[] }
export interface VocabularyEntry { canonical: string, aliases: string[] }
+79 -45
View File
@@ -6,8 +6,15 @@ import * as api from "@@/apis/news-research"
import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings"
import { listSkillTreeApi } from "@@/apis/skills"
import { EFFORTS } from "@@/constants/ai"
import dayjs from "dayjs"
import timezone from "dayjs/plugin/timezone"
import utc from "dayjs/plugin/utc"
import { useUserStore } from "@/pinia/stores/user"
dayjs.extend(utc)
dayjs.extend(timezone)
const vocabularyOpened = ref(false)
const vocabulary = ref<Vocabulary>({ revision: 0, entries: [] })
const vocabularyBusy = ref(false)
@@ -90,8 +97,19 @@ const settingsOpened = ref(false)
const enabled = ref(false)
const startAt = ref<Date | null>(null)
const savedStartAt = ref<Date | null>(null)
const settingsRevision = ref(0)
const queue = ref<{ running: number, draining: boolean, realtime?: Record<string, number>, backfill?: Record<string, number> } | null>(null)
function disabledBackfillDate(date: Date) {
return dayjs.tz(dayjs(date).format("YYYY-MM-DD"), "Asia/Shanghai").valueOf() > Date.now()
}
const startText = computed({
get: () => startAt.value ? dayjs(startAt.value).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : "",
set: (value: string) => {
startAt.value = value ? dayjs.tz(value, "Asia/Shanghai").toDate() : null
}
})
const selectedStartAt = computed({
get: () => enabled.value ? savedStartAt.value : startAt.value,
get: () => enabled.value && !settingsOpened.value ? savedStartAt.value : startAt.value,
set: (value: Date | null) => {
startAt.value = value
}
@@ -134,6 +152,7 @@ async function load() {
const [settings, rows] = await Promise.all([api.getSettingsApi(), tab.value === "events" ? api.listEventsApi(q.value, page.value) : api.listPackagesApi(q.value, page.value)])
if (!valid() || seq !== sequence) return
enabled.value = settings.enabled
queue.value = settings.queue ?? null
if (!startAt.value || startAt.value.getTime() === savedStartAt.value?.getTime()) startAt.value = new Date(settings.start_at)
savedStartAt.value = new Date(settings.start_at)
batch.value = settings.batch ?? null
@@ -166,13 +185,7 @@ async function checkCoverage() {
}
async function enable() {
if (!valid()) return
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) {
ElMessage.error("请选择不晚于当前时间的回补起始时间")
return
}
const start = startAt.value.toISOString()
const result = await api.enableApi(start)
if (valid() && selectedStartAt.value?.toISOString() === start) coverage.value = result
await api.enableApi()
}
async function openEvent(id: number) {
eventId = id
@@ -248,6 +261,8 @@ async function settings() {
providers.value = choices
skills.value = nodes.filter(s => s.scope === "news" && s.kind === "skill")
if (value.config.index && value.config.research) config.value = value.config as Config
startAt.value = new Date(value.start_at)
settingsRevision.value = value.control_revision
settingsOpened.value = true
}
async function action(fn: () => Promise<unknown>) {
@@ -275,7 +290,12 @@ async function retryPackage() {
if (id) await action(() => api.retryPackageApi(id))
}
async function save() {
await action(() => api.saveSettingsApi(config.value))
if (!valid() || busy.value) return
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) {
ElMessage.error("请选择不晚于当前时间的回补起始时间")
return
}
await action(() => api.saveSettingsApi({ ...config.value, start_at: startAt.value!.toISOString(), base_revision: settingsRevision.value }))
if (valid()) settingsOpened.value = false
}
async function prepareSkills() {
@@ -364,7 +384,7 @@ onBeforeUnmount(() => {
<el-button @click="settings">
模型与 Skills
</el-button>
<el-button :loading="busy" :disabled="!enabled && (loading || error || !startAt)" @click="action(enabled ? api.pauseApi : enable)">
<el-button :loading="busy" :disabled="!enabled && (loading || error || queue?.draining)" @click="action(enabled ? api.pauseApi : enable)">
{{ enabled ? '暂停新任务' : '启用两阶段处理' }}
</el-button>
<el-button @click="openVocabulary">
@@ -377,45 +397,20 @@ onBeforeUnmount(() => {
<p class="mb-4 text-sm text-slate-500">
新闻包保存原文和关键词;事件汇集相关报道,持续更新判断与时间线。
</p>
<div class="mb-4 rounded border border-slate-200 p-3">
<div class="flex flex-wrap items-center gap-3">
<label for="research-start-at">回补起始时间</label>
<el-date-picker
id="research-start-at"
v-model="selectedStartAt"
type="datetime"
:clearable="false"
:disabled="enabled || busy"
:disabled-date="(date: Date) => date.getTime() > Date.now()"
placeholder="选择回补起始时间"
/>
<span class="text-sm text-slate-500">按浏览器本地时区</span>
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
核查遗漏
</el-button>
</div>
<p class="mt-2 text-sm text-slate-500">
启用时保存起点,自动核查并补齐该时间以来已采集的新闻;先完成关键词与归属,再集中研究事件。调整起点不会取消已有队列。
</p>
<p v-if="coverage" class="mt-2 text-sm">
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
失败项需手动重试,当前批次以外的遗漏排入下一批。
</p>
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
范围外另有已入队但未完成的新闻 {{ coverage.outside.unprocessed }} 条:
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
已有任务继续按批次处理,失败项需手动重试。
</p>
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
核查失败,请重试。
</p>
</div>
<p class="mb-4 text-sm text-slate-500">
实时任务优先取得空闲并发;历史回补至 {{ savedStartAt ? dayjs(savedStartAt).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : '—' }}(北京时间)。
<span v-if="queue?.draining">正在完成 {{ queue.running }} 个已接收调用,收尾后可重新启用。</span>
</p>
<p v-if="queue" class="mb-4 text-sm text-slate-500">
实时队列:等待索引 {{ queue.realtime?.index_pending ?? 0 }},等待归属 {{ queue.realtime?.assign_pending ?? 0 }};
回补队列:等待索引 {{ queue.backfill?.index_pending ?? 0 }},等待归属 {{ queue.backfill?.assign_pending ?? 0 }}。
正在调用 {{ queue.running }};范围外暂停、失败和中断结果不明的任务不会自动重试。
</p>
<p v-if="batch" class="mb-4 text-sm text-slate-500">
批次 #{{ batch.id }} · {{ batchPhases[batch.phase] ?? batch.phase }} ·
新闻归属 {{ batch.packages_done }}/{{ batch.packages_total }},失败 {{ batch.packages_failed }},延期 {{ batch.packages_deferred }} ·
事件研究 {{ batch.events_done }}/{{ batch.events_total }},失败 {{ batch.events_failed }},延期 {{ batch.events_deferred }}。
新新闻和重试请求排入下一批,研究仅使用站内资料。
实时任务可中途插队;回补任务分批归属后集中研究,研究仅使用站内资料。
</p>
<el-alert v-if="error" title="读取失败,保留上次结果,可刷新重试" type="error" :closable="false" />
<div class="mb-4 flex flex-wrap gap-3">
@@ -474,6 +469,9 @@ onBeforeUnmount(() => {
<el-drawer v-model="opened" title="事件研究与时间线" size="min(820px, 100vw)">
<el-alert v-if="detailError" title="事件详情读取失败,请重新打开重试" type="error" :closable="false" />
<template v-if="detail">
<p v-if="detail.wait_reason" class="mb-3 text-amber-700">
{{ detail.wait_reason }}
</p>
<h2>{{ detail.title }}</h2><p>{{ states[detail.state] ?? detail.state }}</p>
<el-button :loading="busy" @click="retryEvent">
重新研究
@@ -627,6 +625,42 @@ onBeforeUnmount(() => {
</el-drawer>
<el-drawer v-model="settingsOpened" title="两阶段模型与 Skills" size="min(640px, 100vw)">
<el-form label-position="top">
<div class="mb-4 rounded border border-slate-200 p-3">
<div class="flex flex-wrap items-center gap-3">
<label for="research-start-at">回补起始时间</label>
<el-date-picker
id="research-start-at"
v-model="startText"
value-format="YYYY-MM-DD HH:mm:ss"
type="datetime"
:clearable="false"
:disabled="busy"
:disabled-date="disabledBackfillDate"
placeholder="选择回补起始时间"
/>
<span class="text-sm text-slate-500">北京时间(系统时区 UTC+8)</span>
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
核查遗漏
</el-button>
</div>
<p class="mt-2 text-sm text-slate-500">
点击“保存并暂停”后生效;重新启用时重排未完成任务。实时新闻优先插队,历史新闻分批补齐。
</p>
<p v-if="coverage" class="mt-2 text-sm">
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
失败项需手动重试,当前批次以外的遗漏排入下一批。
</p>
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
范围外已暂停自动处理的新闻 {{ coverage.outside.unprocessed }} 条:
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
已接收调用允许完成,扩大范围后恢复尚未完成的阶段;失败项需手动重试。
</p>
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
核查失败,请重试。
</p>
</div>
<template v-for="stage in (['index', 'research'] as const)" :key="stage">
<h3>{{ stage === 'index' ? '第一阶段:关键词索引与事件归属' : '第二阶段:事件整体研究' }}</h3>
<el-form-item label="模型接入">
+35 -7
View File
@@ -485,7 +485,7 @@ test("新闻列表只更新可见条目的匹配版本评级,保留滚动与
function researchPage(api: Record<string, unknown> = {}, user = vue.reactive({ sessionVersion: 1 }), extra: Record<string, unknown> = {}, dependencies: Record<string, unknown> = {}) {
const load = moduleLoader({
"@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api },
"@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api },
"@@/apis/skills": { listSkillTreeApi: async () => [] },
"@@/apis/settings": { listProvidersApi: async () => [], listProviderModelsApi: async () => [] },
"@/pinia/stores/user": { useUserStore: () => user },
@@ -516,11 +516,11 @@ test("回补时间按时区提交,刷新保留未提交的起点,核查不
assert.equal(enabled.length, 0)
assert.equal(page.coverage.value.missing, 4)
await page.enable()
assert.deepEqual(enabled, ["2026-09-30T16:00:00.000Z"])
assert.deepEqual(enabled, [undefined])
})
test("其他标签页启用后按实际起点核查,暂停后恢复草稿并丢弃旧范围响应", async () => {
let server = { config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" }
let server = { config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }
const pending = Promise.withResolvers()
const checked: string[] = []
let delayDraft = true
@@ -612,13 +612,13 @@ test("切换会话后,旧回补核查不能恢复统计结果", async () => {
assert.equal(page.startAt.value, null)
})
test("回补时间无效或晚于当前时间时禁止启用", async () => {
test("回补时间无效或晚于当前时间时禁止保存", async () => {
let calls = 0
const errors: string[] = []
const page = researchPage({ enableApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } })
const page = researchPage({ saveSettingsApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } })
for (const value of [null, new Date("invalid"), new Date(Date.now() + 86400000)]) {
page.startAt.value = value
await page.enable()
await page.save()
}
assert.equal(calls, 0)
assert.equal(errors.length, 3)
@@ -718,12 +718,15 @@ test("两阶段保存各自思考强度,不提交第三阶段配置", async ()
const page = researchPage({ saveSettingsApi: async (data: unknown) => {
saved = JSON.parse(JSON.stringify(data))
} })
await page.load()
page.config.value.index.reasoning_effort = "low"
page.config.value.research.reasoning_effort = "high"
await page.save()
assert.equal(saved.index.reasoning_effort, "low")
assert.equal(saved.research.reasoning_effort, "high")
assert.equal(Object.keys(saved).length, 3)
assert.equal(Object.keys(saved).length, 5)
assert.equal(saved.start_at, "2026-10-31T16:00:00.000Z")
assert.equal(saved.base_revision, 0)
page.config.value.research.reasoning_effort = null
await page.save()
assert.equal(saved.research.reasoning_effort, null)
@@ -756,6 +759,7 @@ test("两阶段操作拒绝重复提交和旧会话操作,失败保留配置
calls++
return pending.promise
} }, user)
await page.load()
page.config.value.index.model = "draft"
const first = page.save()
await page.save()
@@ -964,3 +968,27 @@ test("词库保存拒绝重复提交,关闭后的响应不覆盖草稿,失
assert.equal(page.vocabularyOpened.value, true)
assert.equal(page.vocabularyBusy.value, false)
})
test("侧栏时间统一使用北京时间,保存时间与配置版本但启用不提交草稿", async () => {
let saved: any
let enableArgs: unknown[] = []
const page = researchPage({
getSettingsApi: async () => ({ config: {}, enabled: true, control_revision: 7, start_at: "2026-11-04T16:00:00Z" }),
saveSettingsApi: async (value: unknown) => {
saved = value
},
enableApi: async (...args: unknown[]) => {
enableArgs = args
}
})
await page.settings()
assert.equal(page.startText.value, "2026-11-05 00:00:00")
page.startText.value = "2026-10-20 00:00:00"
await page.load()
assert.equal(page.startText.value, "2026-10-20 00:00:00")
await page.save()
assert.equal(saved.start_at, "2026-10-19T16:00:00.000Z")
assert.equal(saved.base_revision, 7)
await page.enable()
assert.deepEqual(enableArgs, [])
})