Prioritize realtime news and make historical backfill resumable
This commit is contained in:
1 parent
c90691d9a0
commit
5f347f8ab4
17 files changed
+692
-138
No files matched your search
@@ -0,0 +1,24 @@
|
|||||||
|
"""Persist realtime boundaries and enforce one active call per subject."""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = 'dd4567890123'
|
||||||
|
down_revision = 'dc3456789012'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column('news_research_settings', sa.Column('realtime_from', sa.DateTime(timezone=True)))
|
||||||
|
op.add_column('news_research_events', sa.Column('realtime_pending', sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
|
op.create_index('uq_news_attempt_active', 'news_research_attempts', ['package_id'], unique=True,
|
||||||
|
postgresql_where=sa.text("state = 'processing'"))
|
||||||
|
op.create_index('uq_news_version_active', 'news_research_versions', ['event_id'], unique=True,
|
||||||
|
postgresql_where=sa.text("state = 'processing'"))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_index('uq_news_version_active', table_name='news_research_versions')
|
||||||
|
op.drop_index('uq_news_attempt_active', table_name='news_research_attempts')
|
||||||
|
op.drop_column('news_research_events', 'realtime_pending')
|
||||||
|
op.drop_column('news_research_settings', 'realtime_from')
|
||||||
@@ -20,8 +20,9 @@ from alphapulse.news_research.models import (
|
|||||||
Version,
|
Version,
|
||||||
Reference,
|
Reference,
|
||||||
BatchEvent,
|
BatchEvent,
|
||||||
|
Batch,
|
||||||
)
|
)
|
||||||
from alphapulse.news_research import service, batches, vocabulary as vocab
|
from alphapulse.news_research import service, batches, scheduling, vocabulary as vocab
|
||||||
from alphapulse.news_research.models import VocabularyVersion
|
from alphapulse.news_research.models import VocabularyVersion
|
||||||
from alphapulse.news_research.contracts import (
|
from alphapulse.news_research.contracts import (
|
||||||
INDEX_ASSIGN_SKILL,
|
INDEX_ASSIGN_SKILL,
|
||||||
@@ -90,6 +91,8 @@ class SettingsInput(Strict):
|
|||||||
index: ModelConfig
|
index: ModelConfig
|
||||||
research: ModelConfig
|
research: ModelConfig
|
||||||
concurrency: int = Field(default=3, ge=1, le=16)
|
concurrency: int = Field(default=3, ge=1, le=16)
|
||||||
|
start_at: AwareDatetime | None = None
|
||||||
|
base_revision: int | None = Field(default=None, ge=0)
|
||||||
|
|
||||||
|
|
||||||
async def settings_row(session):
|
async def settings_row(session):
|
||||||
@@ -101,7 +104,7 @@ async def settings_row(session):
|
|||||||
id=1,
|
id=1,
|
||||||
config={},
|
config={},
|
||||||
enabled=False,
|
enabled=False,
|
||||||
start_at=datetime(2026, 10, 31, 16, tzinfo=UTC),
|
start_at=datetime(2026, 11, 4, 16, tzinfo=UTC),
|
||||||
)
|
)
|
||||||
.on_conflict_do_nothing(index_elements=["id"])
|
.on_conflict_do_nothing(index_elements=["id"])
|
||||||
)
|
)
|
||||||
@@ -115,6 +118,8 @@ async def settings(session: DB):
|
|||||||
row = await settings_row(session)
|
row = await settings_row(session)
|
||||||
return success(
|
return success(
|
||||||
{"config": row.config, "enabled": row.enabled, "start_at": row.start_at,
|
{"config": row.config, "enabled": row.enabled, "start_at": row.start_at,
|
||||||
|
"control_revision": row.control_revision, "realtime_from": row.realtime_from,
|
||||||
|
"timezone": "Asia/Shanghai", "queue": await scheduling.queue_status(session, row),
|
||||||
"batch": await batches.progress(session)}
|
"batch": await batches.progress(session)}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -125,21 +130,24 @@ async def coverage(session: DB, start_at: AwareDatetime | None = None):
|
|||||||
start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
|
start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
|
||||||
if start > datetime.now(UTC):
|
if start > datetime.now(UTC):
|
||||||
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
||||||
return success(await batches.coverage(session, start))
|
effective = min(start, row.realtime_from) if row.realtime_from else start
|
||||||
|
report = await batches.coverage(session, effective)
|
||||||
|
return success({**report, 'backfill_start_at': start, 'realtime_from': row.realtime_from})
|
||||||
|
|
||||||
|
|
||||||
@router.put("/settings")
|
@router.put("/settings")
|
||||||
async def save(body: SettingsInput, session: DB):
|
async def save(body: SettingsInput, session: DB):
|
||||||
await settings_row(session)
|
await settings_row(session)
|
||||||
await session.execute(
|
row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
|
||||||
update(ResearchSettings)
|
if body.base_revision is not None and body.base_revision != row.control_revision:
|
||||||
.where(ResearchSettings.id == 1)
|
raise HTTPException(409, "设置已变化,请重新读取后保存")
|
||||||
.values(
|
if body.start_at is not None:
|
||||||
config=body.model_dump(),
|
if body.start_at > datetime.now(UTC):
|
||||||
enabled=False,
|
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
||||||
control_revision=ResearchSettings.control_revision + 1,
|
row.start_at = body.start_at.astimezone(UTC)
|
||||||
)
|
row.config = body.model_dump(exclude={"start_at", "base_revision"})
|
||||||
)
|
row.enabled = False
|
||||||
|
row.control_revision += 1
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return success(None)
|
return success(None)
|
||||||
|
|
||||||
@@ -164,6 +172,8 @@ class EnableInput(Strict):
|
|||||||
async def enable(session: DB, body: EnableInput | None = None):
|
async def enable(session: DB, body: EnableInput | None = None):
|
||||||
row = await settings_row(session)
|
row = await settings_row(session)
|
||||||
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
|
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
|
||||||
|
if start_at != row.start_at:
|
||||||
|
raise HTTPException(422, "请在模型与 Skills 中保存回补时间并暂停后再启用")
|
||||||
if start_at > datetime.now(UTC):
|
if start_at > datetime.now(UTC):
|
||||||
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
||||||
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
|
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
|
||||||
@@ -172,11 +182,11 @@ async def enable(session: DB, body: EnableInput | None = None):
|
|||||||
raise HTTPException(409, "请先暂停两阶段处理")
|
raise HTTPException(409, "请先暂停两阶段处理")
|
||||||
running = await session.scalar(
|
running = await session.scalar(
|
||||||
select(func.count())
|
select(func.count())
|
||||||
.select_from(Package)
|
.select_from(Attempt)
|
||||||
.where(Package.state.in_(["index_processing", "assign_processing"]))
|
.where(Attempt.state == "processing")
|
||||||
)
|
)
|
||||||
running += await session.scalar(
|
running += await session.scalar(
|
||||||
select(func.count()).select_from(Event).where(Event.state == "processing")
|
select(func.count()).select_from(Version).where(Version.state == "processing")
|
||||||
)
|
)
|
||||||
if running:
|
if running:
|
||||||
raise HTTPException(409, "请等待已接收任务完成后再验证")
|
raise HTTPException(409, "请等待已接收任务完成后再验证")
|
||||||
@@ -259,7 +269,10 @@ async def enable(session: DB, body: EnableInput | None = None):
|
|||||||
if row.control_revision != control_revision or row.config != original_config:
|
if row.control_revision != control_revision or row.config != original_config:
|
||||||
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
|
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
|
||||||
report = await batches.coverage(session, start_at)
|
report = await batches.coverage(session, start_at)
|
||||||
row.start_at = start_at
|
if row.realtime_from is None:
|
||||||
|
row.realtime_from = datetime.now(UTC)
|
||||||
|
await session.execute(update(Batch).where(Batch.phase.in_(["assigning", "researching"]))
|
||||||
|
.values(phase="rescheduled", finished_at=datetime.now(UTC)))
|
||||||
row.enabled = True
|
row.enabled = True
|
||||||
row.control_revision += 1
|
row.control_revision += 1
|
||||||
await session.commit()
|
await session.commit()
|
||||||
@@ -480,6 +493,18 @@ async def event(identifier: int, session: DB):
|
|||||||
.order_by(Version.id.desc())
|
.order_by(Version.id.desc())
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
wait_reason = None
|
||||||
|
if row.state == 'pending':
|
||||||
|
settings = await settings_row(session)
|
||||||
|
unfinished = [(p, n) for p, n in members if p.state != 'done' or p.content_hash != n.content_hash]
|
||||||
|
if any(n.published_at < scheduling.boundary(settings) for p, n in unfinished):
|
||||||
|
wait_reason = '范围外成员尚未完成,扩大回补范围后可继续研究'
|
||||||
|
elif any(p.state.endswith('_failed') for p, n in unfinished):
|
||||||
|
wait_reason = '成员包处理失败,需手动重试后才能研究'
|
||||||
|
elif unfinished:
|
||||||
|
wait_reason = '等待成员包完成关键词与事件归属'
|
||||||
|
elif row.due_at > datetime.now(UTC):
|
||||||
|
wait_reason = '正在汇集事件变化,聚合窗口结束后研究'
|
||||||
latest = next((v for v in versions if v.state == "done"), None)
|
latest = next((v for v in versions if v.state == "done"), None)
|
||||||
references = await version_references(session, latest.id) if latest else []
|
references = await version_references(session, latest.id) if latest else []
|
||||||
return success(
|
return success(
|
||||||
@@ -490,6 +515,7 @@ async def event(identifier: int, session: DB):
|
|||||||
"result": row.result,
|
"result": row.result,
|
||||||
"error": row.error,
|
"error": row.error,
|
||||||
"revision": row.revision,
|
"revision": row.revision,
|
||||||
|
"wait_reason": wait_reason,
|
||||||
"packages": [await package_out(session, p, n) for p, n in members],
|
"packages": [await package_out(session, p, n) for p, n in members],
|
||||||
"references": references,
|
"references": references,
|
||||||
"versions": [
|
"versions": [
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from sqlalchemy import select, or_, and_, exists, func, any_, case
|
|||||||
from alphapulse.models import NewsItem
|
from alphapulse.models import NewsItem
|
||||||
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
|
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
|
||||||
from alphapulse.news_research.service import enroll
|
from alphapulse.news_research.service import enroll
|
||||||
|
from alphapulse.news_research.scheduling import scoped_event, ready_event
|
||||||
|
|
||||||
|
|
||||||
async def coverage(session, start_at, cutoff=None):
|
async def coverage(session, start_at, cutoff=None):
|
||||||
@@ -48,13 +49,22 @@ async def coverage(session, start_at, cutoff=None):
|
|||||||
|
|
||||||
async def freeze(session, settings):
|
async def freeze(session, settings):
|
||||||
cutoff = datetime.now(UTC)
|
cutoff = datetime.now(UTC)
|
||||||
ids = list(await session.scalars(select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
|
scope = [NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff]
|
||||||
or_(and_(NewsItem.published_at >= settings.start_at, NewsItem.published_at <= cutoff, Package.id.is_(None)),
|
realtime_from = getattr(settings, 'realtime_from', None)
|
||||||
and_(Package.id.is_not(None), Package.content_hash != NewsItem.content_hash)))))
|
if realtime_from is not None:
|
||||||
existing_ids = list(await session.scalars(select(Package.id).where(
|
scope.append(NewsItem.published_at < realtime_from)
|
||||||
Package.state.in_(['index_pending', 'assign_pending', 'index_failed', 'assign_failed']))))
|
candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
|
||||||
event_cutoff = list(await session.scalars(select(Event.id).where(Event.state == "pending")))
|
*scope, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash,
|
||||||
|
Package.state.in_(['index_pending', 'assign_pending'])))
|
||||||
|
ids = list(await session.scalars(candidates.order_by(NewsItem.published_at, NewsItem.id).limit(30)))
|
||||||
await enroll(session, ids)
|
await enroll(session, ids)
|
||||||
|
existing_ids = list(await session.scalars(select(Package.id).where(Package.news_id.in_(ids))))
|
||||||
|
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
|
||||||
|
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
|
||||||
|
event_query = select(Event.id).where(Event.state == 'pending', scoped_event(settings), ready_event(), Event.due_at <= cutoff, ~deferred)
|
||||||
|
if realtime_from is not None:
|
||||||
|
event_query = event_query.where(~Event.realtime_pending)
|
||||||
|
event_cutoff = list(await session.scalars(event_query.order_by(Event.due_at, Event.id).limit(30)))
|
||||||
# Failed items are reported once; only explicit retries or new source revisions rerun them.
|
# Failed items are reported once; only explicit retries or new source revisions rerun them.
|
||||||
previous_failure = exists(select(BatchPackage.batch_id).where(
|
previous_failure = exists(select(BatchPackage.batch_id).where(
|
||||||
BatchPackage.package_id == Package.id, BatchPackage.content_hash == Package.content_hash,
|
BatchPackage.package_id == Package.id, BatchPackage.content_hash == Package.content_hash,
|
||||||
@@ -63,8 +73,6 @@ async def freeze(session, settings):
|
|||||||
or_(Package.id == any_(existing_ids), Package.news_id == any_(ids)), or_(
|
or_(Package.id == any_(existing_ids), Package.news_id == any_(ids)), or_(
|
||||||
Package.state.in_(['index_pending', 'assign_pending']),
|
Package.state.in_(['index_pending', 'assign_pending']),
|
||||||
Package.state.in_(['index_failed', 'assign_failed']) & ~previous_failure))))
|
Package.state.in_(['index_failed', 'assign_failed']) & ~previous_failure))))
|
||||||
deferred = exists(select(BatchEvent.batch_id).where(BatchEvent.event_id == Event.id,
|
|
||||||
BatchEvent.revision == Event.revision, BatchEvent.state == 'deferred'))
|
|
||||||
events = list(await session.scalars(select(Event).where(Event.id == any_(event_cutoff), Event.state == 'pending', ~deferred)))
|
events = list(await session.scalars(select(Event).where(Event.id == any_(event_cutoff), Event.state == 'pending', ~deferred)))
|
||||||
if not packages and not events:
|
if not packages and not events:
|
||||||
return None
|
return None
|
||||||
@@ -83,7 +91,7 @@ async def freeze(session, settings):
|
|||||||
return batch
|
return batch
|
||||||
|
|
||||||
|
|
||||||
async def actions(session, batch, active, slots):
|
async def actions(session, batch, active, slots, settings=None):
|
||||||
"""Return stage/ID pairs; callers commit before launching and never overlap phases."""
|
"""Return stage/ID pairs; callers commit before launching and never overlap phases."""
|
||||||
if batch.phase == 'assigning':
|
if batch.phase == 'assigning':
|
||||||
rows = list(await session.execute(select(BatchPackage, Package, NewsItem).join(
|
rows = list(await session.execute(select(BatchPackage, Package, NewsItem).join(
|
||||||
@@ -94,7 +102,9 @@ async def actions(session, batch, active, slots):
|
|||||||
for item, package, news in rows:
|
for item, package, news in rows:
|
||||||
if item.state != 'pending':
|
if item.state != 'pending':
|
||||||
continue
|
continue
|
||||||
if item.content_hash != news.content_hash or item.content_hash != package.content_hash:
|
if settings is not None and news.published_at < settings.start_at:
|
||||||
|
item.state = 'range_paused'
|
||||||
|
elif item.content_hash != news.content_hash or item.content_hash != package.content_hash:
|
||||||
item.state = 'deferred'
|
item.state = 'deferred'
|
||||||
elif package.state == 'done':
|
elif package.state == 'done':
|
||||||
item.state = 'done'
|
item.state = 'done'
|
||||||
@@ -102,10 +112,12 @@ async def actions(session, batch, active, slots):
|
|||||||
item.state = 'failed'
|
item.state = 'failed'
|
||||||
elif package.state in ('index_pending', 'assign_pending'):
|
elif package.state in ('index_pending', 'assign_pending'):
|
||||||
stage = 'index' if package.state == 'index_pending' else 'assign'
|
stage = 'index' if package.state == 'index_pending' else 'assign'
|
||||||
if (stage, package.id) not in active and len(result) < slots and (stage != 'assign' or not assigning):
|
if not any(identifier == package.id and task != 'event' for task, identifier in active) and len(result) < slots and (stage != 'assign' or not assigning):
|
||||||
result.append((stage, package.id))
|
result.append((stage, package.id))
|
||||||
assigning |= stage == 'assign'
|
assigning |= stage == 'assign'
|
||||||
if any(item.state == 'pending' for item, _, _ in rows) or active:
|
package_ids = {package.id for _, package, _ in rows}
|
||||||
|
batch_active = any(stage != 'event' and identifier in package_ids for stage, identifier in active)
|
||||||
|
if any(item.state == 'pending' for item, _, _ in rows) or batch_active:
|
||||||
return result
|
return result
|
||||||
# Memberships now include events created by this batch. Existing affected events
|
# Memberships now include events created by this batch. Existing affected events
|
||||||
# were frozen up front, including memberships that have since been removed.
|
# were frozen up front, including memberships that have since been removed.
|
||||||
@@ -131,7 +143,13 @@ async def actions(session, batch, active, slots):
|
|||||||
for item, event in rows:
|
for item, event in rows:
|
||||||
if item.state not in ('pending', 'urgent_pending', 'processing') or ('event', event.id) in active:
|
if item.state not in ('pending', 'urgent_pending', 'processing') or ('event', event.id) in active:
|
||||||
continue
|
continue
|
||||||
if item.revision != event.revision:
|
if settings is not None and not await session.scalar(select(Event.id).where(Event.id == event.id, scoped_event(settings))):
|
||||||
|
item.state = 'range_paused'
|
||||||
|
elif settings is not None and event.realtime_pending:
|
||||||
|
item.state = 'realtime'
|
||||||
|
elif event.due_at > datetime.now(UTC):
|
||||||
|
item.state = 'aggregating'
|
||||||
|
elif item.revision != event.revision:
|
||||||
item.state = 'deferred'
|
item.state = 'deferred'
|
||||||
elif event.state == 'done' or event.state == 'retired':
|
elif event.state == 'done' or event.state == 'retired':
|
||||||
item.state = 'done'
|
item.state = 'done'
|
||||||
@@ -143,11 +161,17 @@ async def actions(session, batch, active, slots):
|
|||||||
invalid = await session.scalar(select(Membership.event_id).join(Package).join(NewsItem).where(
|
invalid = await session.scalar(select(Membership.event_id).join(Package).join(NewsItem).where(
|
||||||
Membership.event_id == event.id, or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)).limit(1))
|
Membership.event_id == event.id, or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)).limit(1))
|
||||||
if invalid:
|
if invalid:
|
||||||
item.state = 'deferred'
|
item.state = 'blocked'
|
||||||
elif len(result) < slots:
|
elif len(result) < slots:
|
||||||
item.state = 'processing'
|
item.state = 'processing'
|
||||||
result.append(('event', event.id))
|
result.append(('event', event.id))
|
||||||
if not result and not active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows):
|
event_ids = {event.id for _, event in rows}
|
||||||
|
batch_active = any(stage == 'event' and identifier in event_ids for stage, identifier in active)
|
||||||
|
active_packages = {identifier for stage, identifier in active if stage != 'event'}
|
||||||
|
if active_packages:
|
||||||
|
batch_active |= bool(await session.scalar(select(BatchPackage.package_id).where(
|
||||||
|
BatchPackage.batch_id == batch.id, BatchPackage.package_id.in_(active_packages)).limit(1)))
|
||||||
|
if not result and not batch_active and all(item.state not in ('pending', 'urgent_pending', 'processing') for item, _ in rows):
|
||||||
bad = any(item.state in ('failed', 'deferred') for item, _ in rows)
|
bad = any(item.state in ('failed', 'deferred') for item, _ in rows)
|
||||||
bad |= bool(await session.scalar(select(BatchPackage.batch_id).where(
|
bad |= bool(await session.scalar(select(BatchPackage.batch_id).where(
|
||||||
BatchPackage.batch_id == batch.id, BatchPackage.state.in_(['failed', 'deferred'])).limit(1)))
|
BatchPackage.batch_id == batch.id, BatchPackage.state.in_(['failed', 'deferred'])).limit(1)))
|
||||||
@@ -163,5 +187,5 @@ async def progress(session):
|
|||||||
result = {'id': batch.id, 'phase': batch.phase, 'cutoff_at': batch.cutoff_at}
|
result = {'id': batch.id, 'phase': batch.phase, 'cutoff_at': batch.cutoff_at}
|
||||||
for name, model in [('packages', BatchPackage), ('events', BatchEvent)]:
|
for name, model in [('packages', BatchPackage), ('events', BatchEvent)]:
|
||||||
counts = Counter(await session.scalars(select(model.state).where(model.batch_id == batch.id)))
|
counts = Counter(await session.scalars(select(model.state).where(model.batch_id == batch.id)))
|
||||||
result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred')}})
|
result.update({name + '_total': sum(counts.values()), **{name + '_' + s: counts[s] for s in ('done', 'failed', 'deferred', 'range_paused', 'blocked', 'realtime')}})
|
||||||
return result
|
return result
|
||||||
@@ -6,6 +6,8 @@ from sqlalchemy import (
|
|||||||
DateTime,
|
DateTime,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
Integer,
|
Integer,
|
||||||
|
Index,
|
||||||
|
text,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
@@ -32,6 +34,7 @@ class ResearchSettings(Base):
|
|||||||
Integer, default=0, server_default="0"
|
Integer, default=0, server_default="0"
|
||||||
)
|
)
|
||||||
start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
realtime_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
class Package(Base):
|
class Package(Base):
|
||||||
@@ -49,6 +52,7 @@ class Package(Base):
|
|||||||
|
|
||||||
class Attempt(Base):
|
class Attempt(Base):
|
||||||
__tablename__ = "news_research_attempts"
|
__tablename__ = "news_research_attempts"
|
||||||
|
__table_args__ = (Index("uq_news_attempt_active", "package_id", unique=True, postgresql_where=text("state = 'processing'")),)
|
||||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||||
package_id: Mapped[int] = mapped_column(
|
package_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("news_research_packages.id"), index=True
|
ForeignKey("news_research_packages.id"), index=True
|
||||||
@@ -75,6 +79,7 @@ class Event(Base):
|
|||||||
state: Mapped[str] = mapped_column(String(16), default="pending", index=True)
|
state: Mapped[str] = mapped_column(String(16), default="pending", index=True)
|
||||||
result: Mapped[dict | None] = mapped_column(JSONB)
|
result: Mapped[dict | None] = mapped_column(JSONB)
|
||||||
error: Mapped[str | None] = mapped_column(Text)
|
error: Mapped[str | None] = mapped_column(Text)
|
||||||
|
realtime_pending: Mapped[bool] = mapped_column(default=False, server_default="false")
|
||||||
dirty_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
dirty_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
due_at: Mapped[datetime] = mapped_column(
|
due_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), server_default=func.now(), index=True
|
DateTime(timezone=True), server_default=func.now(), index=True
|
||||||
@@ -99,6 +104,7 @@ class Membership(Base):
|
|||||||
|
|
||||||
class Version(Base):
|
class Version(Base):
|
||||||
__tablename__ = "news_research_versions"
|
__tablename__ = "news_research_versions"
|
||||||
|
__table_args__ = (Index("uq_news_version_active", "event_id", unique=True, postgresql_where=text("state = 'processing'")),)
|
||||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||||
event_id: Mapped[int] = mapped_column(
|
event_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("news_research_events.id"), index=True
|
ForeignKey("news_research_events.id"), index=True
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Priority selection over durable package and event states; no paid work here."""
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from sqlalchemy import select, exists, or_, and_, func, case
|
||||||
|
from alphapulse.models import NewsItem
|
||||||
|
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
|
||||||
|
|
||||||
|
|
||||||
|
def boundary(settings):
|
||||||
|
realtime_from = getattr(settings, 'realtime_from', None)
|
||||||
|
return min(settings.start_at, realtime_from) if realtime_from else settings.start_at
|
||||||
|
|
||||||
|
|
||||||
|
def eligible(settings):
|
||||||
|
return and_(NewsItem.published_at >= boundary(settings), NewsItem.published_at <= datetime.now(UTC))
|
||||||
|
|
||||||
|
|
||||||
|
def members(event_id):
|
||||||
|
return select(Membership.package_id).where(Membership.event_id == event_id)
|
||||||
|
|
||||||
|
|
||||||
|
def ready_event():
|
||||||
|
return ~exists(select(Membership.package_id).join(Package).join(NewsItem).where(
|
||||||
|
Membership.event_id == Event.id,
|
||||||
|
or_(Package.state != 'done', Package.content_hash != NewsItem.content_hash)))
|
||||||
|
|
||||||
|
|
||||||
|
def scoped_event(settings):
|
||||||
|
# Empty events must still reach the existing no-model retirement path.
|
||||||
|
return or_(exists(select(Membership.package_id).join(Package).join(NewsItem).where(
|
||||||
|
Membership.event_id == Event.id, eligible(settings))),
|
||||||
|
~exists(select(Membership.package_id).where(Membership.event_id == Event.id)))
|
||||||
|
|
||||||
|
|
||||||
|
async def control(session, config):
|
||||||
|
"""The control lock spans the durable claim, never the remote model call."""
|
||||||
|
if '_control_revision' not in config:
|
||||||
|
return None # Explicit bounded/offline callers retain their existing contract.
|
||||||
|
row = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
|
||||||
|
if not row.enabled or row.control_revision != config['_control_revision']:
|
||||||
|
return False
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
async def mark_priority(session, event, news):
|
||||||
|
settings = await session.get(ResearchSettings, 1)
|
||||||
|
if settings and settings.realtime_from and news.published_at >= settings.realtime_from:
|
||||||
|
event.realtime_pending = True
|
||||||
|
|
||||||
|
|
||||||
|
async def realtime_actions(session, settings, active, slots):
|
||||||
|
if settings.realtime_from is None or slots <= 0:
|
||||||
|
return []
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
busy_packages = {identifier for stage, identifier in active if stage != 'event'}
|
||||||
|
busy_events = {identifier for stage, identifier in active if stage == 'event'}
|
||||||
|
result = []
|
||||||
|
events = await session.scalars(select(Event).where(
|
||||||
|
Event.state == 'pending', Event.realtime_pending, Event.due_at <= now,
|
||||||
|
scoped_event(settings), ready_event(), Event.id.not_in(busy_events),
|
||||||
|
~exists(select(Version.id).where(Version.event_id == Event.id, Version.state == 'processing')),
|
||||||
|
).order_by(Event.due_at, Event.id).limit(slots))
|
||||||
|
result.extend(('event', event.id) for event in events)
|
||||||
|
# Known historical dependencies inherit the waiting realtime event's priority.
|
||||||
|
dependency = exists(select(Membership.package_id).join(Event).where(
|
||||||
|
Membership.package_id == Package.id, Event.realtime_pending, Event.state == 'pending'))
|
||||||
|
priority = or_(NewsItem.published_at >= settings.realtime_from, dependency)
|
||||||
|
candidates = select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
|
||||||
|
eligible(settings), priority, or_(Package.id.is_(None), Package.content_hash != NewsItem.content_hash),
|
||||||
|
or_(Package.id.is_(None), Package.id.not_in(busy_packages))).order_by(NewsItem.published_at, NewsItem.id).limit(30)
|
||||||
|
ids = list(await session.scalars(candidates))
|
||||||
|
if ids:
|
||||||
|
from alphapulse.news_research.service import enroll
|
||||||
|
await enroll(session, ids)
|
||||||
|
assigning = any(stage == 'assign' for stage, _ in active)
|
||||||
|
for stage in ('assign', 'index'):
|
||||||
|
if stage == 'assign' and assigning:
|
||||||
|
continue
|
||||||
|
remaining = slots - len(result)
|
||||||
|
if remaining <= 0:
|
||||||
|
break
|
||||||
|
query = select(Package.id).join(NewsItem).where(
|
||||||
|
eligible(settings), priority, Package.state == stage + '_pending',
|
||||||
|
Package.id.not_in(busy_packages),
|
||||||
|
~exists(select(Attempt.id).where(Attempt.package_id == Package.id, Attempt.state == 'processing')),
|
||||||
|
).order_by(NewsItem.published_at, NewsItem.id).limit(1 if stage == 'assign' else remaining)
|
||||||
|
result.extend((stage, identifier) for identifier in await session.scalars(query))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def queue_status(session, settings):
|
||||||
|
running = await session.scalar(select(func.count()).select_from(Attempt).where(Attempt.state == 'processing'))
|
||||||
|
running += await session.scalar(select(func.count()).select_from(Version).where(Version.state == 'processing'))
|
||||||
|
realtime_from = settings.realtime_from
|
||||||
|
lane = case((NewsItem.published_at >= realtime_from, 'realtime'), else_='backfill') if realtime_from else 'backfill'
|
||||||
|
if realtime_from:
|
||||||
|
groups = (await session.execute(select(lane, Package.state, func.count()).join(NewsItem)
|
||||||
|
.where(eligible(settings)).group_by(lane, Package.state))).all()
|
||||||
|
else:
|
||||||
|
groups = [('backfill', state, count) for state, count in (await session.execute(
|
||||||
|
select(Package.state, func.count()).join(NewsItem).where(eligible(settings)).group_by(Package.state))).all()]
|
||||||
|
queues = {name: {state: count for selected, state, count in groups if selected == name}
|
||||||
|
for name in ('realtime', 'backfill')}
|
||||||
|
return {'running': running, 'draining': not settings.enabled and running > 0, **queues}
|
||||||
@@ -27,6 +27,7 @@ from alphapulse.news_research.models import (
|
|||||||
Version,
|
Version,
|
||||||
Reference,
|
Reference,
|
||||||
)
|
)
|
||||||
|
from alphapulse.news_research.scheduling import control, mark_priority, boundary
|
||||||
from alphapulse.news_research.contracts import (
|
from alphapulse.news_research.contracts import (
|
||||||
INDEX_ASSIGN_SKILL,
|
INDEX_ASSIGN_SKILL,
|
||||||
task_skill,
|
task_skill,
|
||||||
@@ -280,6 +281,7 @@ async def enroll(session, ids):
|
|||||||
.with_for_update()
|
.with_for_update()
|
||||||
):
|
):
|
||||||
dirty(event)
|
dirty(event)
|
||||||
|
await mark_priority(session, event, row)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
@@ -429,6 +431,9 @@ async def process_package(identifier, config):
|
|||||||
|
|
||||||
async def _process_package(identifier, config, expected_stage):
|
async def _process_package(identifier, config, expected_stage):
|
||||||
async with SessionLocal() as session:
|
async with SessionLocal() as session:
|
||||||
|
settings = await control(session, config)
|
||||||
|
if settings is False:
|
||||||
|
return
|
||||||
package = await session.get(Package, identifier, with_for_update=True)
|
package = await session.get(Package, identifier, with_for_update=True)
|
||||||
if package is None:
|
if package is None:
|
||||||
return
|
return
|
||||||
@@ -438,6 +443,10 @@ async def _process_package(identifier, config, expected_stage):
|
|||||||
if package.content_hash != news.content_hash:
|
if package.content_hash != news.content_hash:
|
||||||
await enroll(session, [news.id])
|
await enroll(session, [news.id])
|
||||||
return
|
return
|
||||||
|
if settings is not None and not boundary(settings) <= news.published_at <= datetime.now(UTC):
|
||||||
|
return
|
||||||
|
if await session.scalar(select(Attempt.id).where(Attempt.package_id == identifier, Attempt.state == 'processing').limit(1)):
|
||||||
|
return
|
||||||
source = article(news)
|
source = article(news)
|
||||||
source_hash = package.content_hash
|
source_hash = package.content_hash
|
||||||
stage = "index" if package.state == "index_pending" else "assign"
|
stage = "index" if package.state == "index_pending" else "assign"
|
||||||
@@ -566,12 +575,14 @@ async def _process_package(identifier, config, expected_stage):
|
|||||||
existing.change_kind = target.change_kind
|
existing.change_kind = target.change_kind
|
||||||
if target.change_kind != "repost" or event.result is None:
|
if target.change_kind != "repost" or event.result is None:
|
||||||
dirty(event, target.urgent)
|
dirty(event, target.urgent)
|
||||||
|
await mark_priority(session, event, news)
|
||||||
for old in previous:
|
for old in previous:
|
||||||
if old.event_id not in assigned:
|
if old.event_id not in assigned:
|
||||||
event = await session.get(
|
event = await session.get(
|
||||||
Event, old.event_id, with_for_update=True
|
Event, old.event_id, with_for_update=True
|
||||||
)
|
)
|
||||||
dirty(event)
|
dirty(event)
|
||||||
|
await mark_priority(session, event, news)
|
||||||
await session.delete(old)
|
await session.delete(old)
|
||||||
affected[event.id] = event
|
affected[event.id] = event
|
||||||
package.state = "done"
|
package.state = "done"
|
||||||
@@ -805,6 +816,9 @@ def validate_timeline(result, package_news):
|
|||||||
|
|
||||||
async def process_event(identifier, config, force=False):
|
async def process_event(identifier, config, force=False):
|
||||||
async with SessionLocal() as session:
|
async with SessionLocal() as session:
|
||||||
|
settings = await control(session, config)
|
||||||
|
if settings is False:
|
||||||
|
return
|
||||||
event = await session.get(Event, identifier, with_for_update=True)
|
event = await session.get(Event, identifier, with_for_update=True)
|
||||||
if (
|
if (
|
||||||
event is None
|
event is None
|
||||||
@@ -812,6 +826,8 @@ async def process_event(identifier, config, force=False):
|
|||||||
or (not force and event.due_at > datetime.now(UTC))
|
or (not force and event.due_at > datetime.now(UTC))
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
|
if await session.scalar(select(Version.id).where(Version.event_id == identifier, Version.state == 'processing').limit(1)):
|
||||||
|
return
|
||||||
members = list(
|
members = list(
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(Package, Membership, NewsItem)
|
select(Package, Membership, NewsItem)
|
||||||
@@ -821,6 +837,8 @@ async def process_event(identifier, config, force=False):
|
|||||||
.order_by(Package.id)
|
.order_by(Package.id)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if settings is not None and members and not any(boundary(settings) <= n.published_at <= datetime.now(UTC) for p, m, n in members):
|
||||||
|
return
|
||||||
if not members:
|
if not members:
|
||||||
event.state = "retired"
|
event.state = "retired"
|
||||||
await session.commit()
|
await session.commit()
|
||||||
@@ -859,6 +877,7 @@ async def process_event(identifier, config, force=False):
|
|||||||
)
|
)
|
||||||
session.add(version)
|
session.add(version)
|
||||||
event.state = "processing"
|
event.state = "processing"
|
||||||
|
event.dirty_since = None
|
||||||
await session.commit()
|
await session.commit()
|
||||||
version_id, revision = version.id, event.revision
|
version_id, revision = version.id, event.revision
|
||||||
researcher = Research(event={"id": identifier, "revision": revision})
|
researcher = Research(event={"id": identifier, "revision": revision})
|
||||||
@@ -977,6 +996,7 @@ async def process_event(identifier, config, force=False):
|
|||||||
current.title = output.title
|
current.title = output.title
|
||||||
current.state = "done"
|
current.state = "done"
|
||||||
current.dirty_since = None
|
current.dirty_since = None
|
||||||
|
current.realtime_pending = False
|
||||||
current.error = None
|
current.error = None
|
||||||
for ref in output.references:
|
for ref in output.references:
|
||||||
session.add(
|
session.add(
|
||||||
|
|||||||
@@ -13,38 +13,42 @@ from alphapulse.news_research.models import (
|
|||||||
Batch, BatchEvent,
|
Batch, BatchEvent,
|
||||||
)
|
)
|
||||||
from alphapulse.news_research.service import process_package, process_event
|
from alphapulse.news_research.service import process_package, process_event
|
||||||
from alphapulse.news_research import batches
|
from alphapulse.news_research import batches, scheduling
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def recover_interrupted():
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
for stage in ("index", "assign"):
|
||||||
|
await session.execute(
|
||||||
|
update(Package)
|
||||||
|
.where(Package.state == stage + "_processing")
|
||||||
|
.values(state=stage + "_failed", error="进程中断,调用结果不明,请手动重试")
|
||||||
|
)
|
||||||
|
await session.execute(
|
||||||
|
update(Attempt)
|
||||||
|
.where(Attempt.state == "processing")
|
||||||
|
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
|
||||||
|
)
|
||||||
|
await session.execute(
|
||||||
|
update(Event).where(Event.state == "processing").values(state="failed", error="进程中断,调用结果不明,请手动重试")
|
||||||
|
)
|
||||||
|
await session.execute(
|
||||||
|
update(Version)
|
||||||
|
.where(Version.state == "processing")
|
||||||
|
.values(state="failed", error="进程中断,调用结果不明,请手动重试")
|
||||||
|
)
|
||||||
|
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
async def run():
|
async def run():
|
||||||
async with engine.connect() as lock:
|
async with engine.connect() as lock:
|
||||||
if not await lock.scalar(text("SELECT pg_try_advisory_lock(20261030, 1)")):
|
if not await lock.scalar(text("SELECT pg_try_advisory_lock(20261030, 1)")):
|
||||||
raise RuntimeError("新闻AI worker 已运行")
|
raise RuntimeError("新闻AI worker 已运行")
|
||||||
await lock.commit()
|
await lock.commit()
|
||||||
async with SessionLocal() as session:
|
await recover_interrupted()
|
||||||
for stage in ("index", "assign"):
|
|
||||||
await session.execute(
|
|
||||||
update(Package)
|
|
||||||
.where(Package.state == stage + "_processing")
|
|
||||||
.values(state=stage + "_pending")
|
|
||||||
)
|
|
||||||
await session.execute(
|
|
||||||
update(Attempt)
|
|
||||||
.where(Attempt.state == "processing")
|
|
||||||
.values(state="failed", error="进程中断,待重试")
|
|
||||||
)
|
|
||||||
await session.execute(
|
|
||||||
update(Event).where(Event.state == "processing").values(state="pending")
|
|
||||||
)
|
|
||||||
await session.execute(
|
|
||||||
update(Version)
|
|
||||||
.where(Version.state == "processing")
|
|
||||||
.values(state="failed", error="进程中断,待重试")
|
|
||||||
)
|
|
||||||
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
|
|
||||||
await session.commit()
|
|
||||||
tasks = {}
|
tasks = {}
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
@@ -58,19 +62,20 @@ async def run():
|
|||||||
async with SessionLocal() as session:
|
async with SessionLocal() as session:
|
||||||
settings = await session.get(ResearchSettings, 1)
|
settings = await session.get(ResearchSettings, 1)
|
||||||
if settings and settings.enabled:
|
if settings and settings.enabled:
|
||||||
|
active = set(tasks.values())
|
||||||
|
slots = max(0, settings.config.get('concurrency', 3) - len(tasks))
|
||||||
|
actions = await scheduling.realtime_actions(session, settings, active, slots)
|
||||||
batch = await session.scalar(select(Batch).where(
|
batch = await session.scalar(select(Batch).where(
|
||||||
Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1))
|
Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1))
|
||||||
if batch is None:
|
if batch is None:
|
||||||
batch = await batches.freeze(session, settings)
|
batch = await batches.freeze(session, settings)
|
||||||
settings = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
|
if batch is not None:
|
||||||
if batch is not None and settings.enabled:
|
actions += await batches.actions(session, batch, active | set(actions), slots - len(actions), settings)
|
||||||
active = set(tasks.values())
|
|
||||||
actions = await batches.actions(session, batch, active,
|
|
||||||
max(0, settings.config.get('concurrency', 3) - len(tasks)))
|
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
config = {**settings.config, '_control_revision': settings.control_revision}
|
||||||
for stage, identifier in actions:
|
for stage, identifier in actions:
|
||||||
call = (process_event(identifier, settings.config, force=True)
|
call = (process_event(identifier, config)
|
||||||
if stage == 'event' else process_package(identifier, settings.config))
|
if stage == 'event' else process_package(identifier, config))
|
||||||
tasks[asyncio.create_task(call)] = (stage, identifier)
|
tasks[asyncio.create_task(call)] = (stage, identifier)
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -302,7 +302,7 @@ def test_failed_enable_does_not_save_the_selected_backfill_start(trading, monkey
|
|||||||
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
|
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
|
||||||
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
|
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
|
||||||
with pytest.raises(HTTPException) as error:
|
with pytest.raises(HTTPException) as error:
|
||||||
asyncio.run(api.enable(session, api.EnableInput(start_at=previous - timedelta(days=30))))
|
asyncio.run(api.enable(session, api.EnableInput(start_at=previous)))
|
||||||
assert error.value.status_code == 502
|
assert error.value.status_code == 502
|
||||||
assert row.start_at == previous and not row.enabled and row.control_revision == 7
|
assert row.start_at == previous and not row.enabled and row.control_revision == 7
|
||||||
session.commit.assert_not_called()
|
session.commit.assert_not_called()
|
||||||
@@ -842,6 +842,7 @@ def test_alias_adjustment_audit_survives_index_failures(trading, monkeypatch, fa
|
|||||||
session.get = AsyncMock(side_effect=lambda model, *args, **kwargs:
|
session.get = AsyncMock(side_effect=lambda model, *args, **kwargs:
|
||||||
attempts[0] if model is Attempt else package if model is Package else news)
|
attempts[0] if model is Attempt else package if model is Package else news)
|
||||||
session.execute = AsyncMock()
|
session.execute = AsyncMock()
|
||||||
|
session.scalar = AsyncMock(return_value=None)
|
||||||
monkeypatch.setattr(service, "SessionLocal", lambda: session)
|
monkeypatch.setattr(service, "SessionLocal", lambda: session)
|
||||||
monkeypatch.setattr(service.vocab, "snapshot", AsyncMock(return_value={"entries": []}))
|
monkeypatch.setattr(service.vocab, "snapshot", AsyncMock(return_value={"entries": []}))
|
||||||
monkeypatch.setattr(service, "model_config", AsyncMock(return_value=(None, "索引规则", {})))
|
monkeypatch.setattr(service, "model_config", AsyncMock(return_value=(None, "索引规则", {})))
|
||||||
@@ -903,3 +904,47 @@ def test_task_skill_keeps_custom_rules_and_accepts_crlf(trading):
|
|||||||
content = "## 归属规则\r\n自定义归属\r\n### 细则\r\n保留规则\r\n## 索引规则\r\n自定义索引"
|
content = "## 归属规则\r\n自定义归属\r\n### 细则\r\n保留规则\r\n## 索引规则\r\n自定义索引"
|
||||||
assert task_skill(content, "assign") == "自定义归属\n### 细则\n保留规则"
|
assert task_skill(content, "assign") == "自定义归属\n### 细则\n保留规则"
|
||||||
assert task_skill(content, "index") == "自定义索引"
|
assert task_skill(content, "index") == "自定义索引"
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_save_atomically_pauses_and_preserves_model_contract(trading):
|
||||||
|
import asyncio
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
from alphapulse.news_research import api
|
||||||
|
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=3)
|
||||||
|
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
|
||||||
|
model = dict(provider_id=1, model='test', skill_id=1)
|
||||||
|
body = api.SettingsInput(index=model, research=model, start_at='2026-10-20T00:00:00+08:00', base_revision=3)
|
||||||
|
asyncio.run(api.save(body, session))
|
||||||
|
assert not row.enabled and row.control_revision == 4
|
||||||
|
assert row.start_at == datetime(2026, 10, 19, 16, tzinfo=UTC)
|
||||||
|
assert set(row.config) == {'index', 'research', 'concurrency'}
|
||||||
|
session.commit.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_save_rejects_other_tabs_revision_without_changes(trading):
|
||||||
|
import asyncio
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from alphapulse.news_research import api
|
||||||
|
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC), config={}, enabled=True, control_revision=4)
|
||||||
|
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
|
||||||
|
model = dict(provider_id=1, model='test', skill_id=1)
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
asyncio.run(api.save(api.SettingsInput(index=model, research=model, base_revision=3), session))
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
assert row.enabled and row.control_revision == 4
|
||||||
|
session.commit.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
def test_enable_cannot_apply_an_unsaved_backfill_time(trading):
|
||||||
|
import asyncio
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from alphapulse.news_research import api
|
||||||
|
row = N(start_at=datetime(2026, 11, 4, 16, tzinfo=UTC))
|
||||||
|
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(), commit=AsyncMock())
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
asyncio.run(api.enable(session, api.EnableInput(start_at='2026-10-20T00:00:00+08:00')))
|
||||||
|
assert error.value.status_code == 422
|
||||||
|
session.scalar.assert_not_awaited()
|
||||||
|
session.commit.assert_not_awaited()
|
||||||
@@ -455,3 +455,42 @@ some documents were imported: ECB dynamic index version coverage, inaccessible
|
|||||||
BoE speeches and CoinDesk access blocks are reported rather than labeled complete.
|
BoE speeches and CoinDesk access blocks are reported rather than labeled complete.
|
||||||
The deployment lock serializes writes; crawler/calendar processes are restored in
|
The deployment lock serializes writes; crawler/calendar processes are restored in
|
||||||
finally blocks. Do not start a second driver or remove its state to renew quotas.
|
finally blocks. Do not start a second driver or remove its state to renew quotas.
|
||||||
|
|
||||||
|
### Realtime priority and historical AI backfill
|
||||||
|
|
||||||
|
Migration `dd4567890123` adds the persistent realtime boundary and pending-event
|
||||||
|
priority, plus partial unique indexes for active package attempts and event
|
||||||
|
versions. Pause and drain news AI before applying it; duplicate active records
|
||||||
|
must be investigated rather than deleted to force migration success. Preserve
|
||||||
|
originals, attempts and version history. Deployment requires the candidate's
|
||||||
|
schema head and must leave automatic processing paused during verification.
|
||||||
|
|
||||||
|
The model/Skills drawer saves the historical start time with model settings via
|
||||||
|
**Save and pause**. New settings default to 2026-11-05 00:00 Asia/Shanghai; existing
|
||||||
|
saved values remain intact. Settings responses expose `control_revision`,
|
||||||
|
`realtime_from`, `timezone` and queue counts. PUT accepts optional `start_at` and
|
||||||
|
`base_revision`; version conflicts return 409. Enable uses the saved start only;
|
||||||
|
a differing legacy enable body returns 422. The first successful enable fixes
|
||||||
|
`realtime_from`; later saves, pauses and restarts never move it.
|
||||||
|
|
||||||
|
Publication time, not ingestion time, determines realtime membership. Realtime
|
||||||
|
work takes the next available slot without cancelling accepted calls. Historical
|
||||||
|
work uses batches of at most 30 originals, ordered chronologically. Saving a new
|
||||||
|
start pauses dispatch; enabling after drain retires unfinished batch schedules
|
||||||
|
and rebuilds from durable stage states without repeating successes. Shrinking
|
||||||
|
history does not exclude realtime backlog. Known in-range dependencies inherit
|
||||||
|
a waiting realtime event's priority. Out-of-range or failed members block event
|
||||||
|
research without silently dropping evidence or retrying paid failures.
|
||||||
|
|
||||||
|
Ordinary event research uses the existing five-minute quiet window with a
|
||||||
|
15-minute maximum; urgent events become ready immediately. A running research
|
||||||
|
call is not duplicated when newer members arrive. New changes start another
|
||||||
|
aggregation window and obsolete output cannot overwrite current conclusions.
|
||||||
|
After a worker interruption, started calls become failed with an unknown-result
|
||||||
|
message and need manual retry; unclaimed work resumes normally. Strict realtime
|
||||||
|
priority can starve historical work under sustained overload.
|
||||||
|
|
||||||
|
CI additionally runs `scripts/smoke_priority_queue.py` in its disposable internal
|
||||||
|
network with synthetic credentials and no model calls. It covers preemption,
|
||||||
|
late historical ingestion, stable realtime eligibility, range changes,
|
||||||
|
dependencies, aggregation, control revisions and interrupted-call recovery.
|
||||||
@@ -182,6 +182,8 @@ def smoke(build, label):
|
|||||||
'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)')
|
'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)')
|
||||||
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
||||||
input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True)
|
input=(ROOT / 'scripts/smoke_news_batches.py').read_text()), flush=True)
|
||||||
|
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
||||||
|
input=(ROOT / 'scripts/smoke_priority_queue.py').read_text()), flush=True)
|
||||||
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
||||||
input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
|
input=(ROOT / 'scripts/smoke_news_research.py').read_text()), flush=True)
|
||||||
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
print(run('docker', 'exec', '-i', names['api'], 'python', '-',
|
||||||
|
|||||||
+103
-19
@@ -90,6 +90,7 @@ async def main():
|
|||||||
await backfill_regressions(now)
|
await backfill_regressions(now)
|
||||||
await carryover_regressions(now)
|
await carryover_regressions(now)
|
||||||
await regressions(now, settings)
|
await regressions(now, settings)
|
||||||
|
await queue_review_regressions(now, settings)
|
||||||
await engine.dispose()
|
await engine.dispose()
|
||||||
print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
|
print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
|
||||||
|
|
||||||
@@ -208,7 +209,13 @@ async def regressions(now, settings):
|
|||||||
async with SessionLocal() as s:
|
async with SessionLocal() as s:
|
||||||
event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now)
|
event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now)
|
||||||
previous = Batch(phase='partial', cutoff_at=now)
|
previous = Batch(phase='partial', cutoff_at=now)
|
||||||
s.add_all([event, previous])
|
original = NewsItem(source_code='wscn_news', external_id='retry-member', title='retry', summary='', content_text='retry', content_hash='a'*64, url='https://example.invalid/retry', published_at=now)
|
||||||
|
s.add_all([event, previous, original])
|
||||||
|
await s.flush()
|
||||||
|
member = Package(news_id=original.id, content_hash=original.content_hash, state='done', keywords=[])
|
||||||
|
s.add(member)
|
||||||
|
await s.flush()
|
||||||
|
s.add(Membership(event_id=event.id, package_id=member.id, evidence='retry', reason='retry', change_kind='new_fact'))
|
||||||
await s.flush()
|
await s.flush()
|
||||||
s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred'))
|
s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred'))
|
||||||
await s.commit()
|
await s.commit()
|
||||||
@@ -236,39 +243,116 @@ async def regressions(now, settings):
|
|||||||
s.add_all([news, excluded])
|
s.add_all([news, excluded])
|
||||||
await s.commit()
|
await s.commit()
|
||||||
await service.enroll(s, [news.id])
|
await service.enroll(s, [news.id])
|
||||||
batch = await batches.freeze(s, settings)
|
assert await batches.freeze(s, settings) is None
|
||||||
package = await s.scalar(select(Package).where(Package.news_id == news.id))
|
package = await s.scalar(select(Package).where(Package.news_id == news.id))
|
||||||
news.content_hash = 'b'*64
|
news.content_hash = 'b'*64
|
||||||
await s.commit()
|
await s.commit()
|
||||||
await batches.actions(s, batch, set(), 8)
|
assert await batches.freeze(s, settings) is None
|
||||||
await s.commit()
|
assert package.content_hash != news.content_hash
|
||||||
await batches.actions(s, batch, set(), 8)
|
expanded = SimpleNamespace(start_at=news.published_at)
|
||||||
await s.commit()
|
batch = await batches.freeze(s, expanded)
|
||||||
assert batch.phase == 'partial'
|
|
||||||
batch = await batches.freeze(s, settings)
|
|
||||||
assert package.content_hash == news.content_hash and package.state == 'index_pending'
|
|
||||||
assert await batches.actions(s, batch, set(), 8) == [('index', package.id)]
|
|
||||||
assert await s.scalar(select(func.count()).select_from(Package)) == 1
|
|
||||||
# A previously completed manual original also refreshes outside the automatic boundary.
|
|
||||||
package.state = 'done'
|
|
||||||
news.content_hash = 'c'*64
|
|
||||||
await s.commit()
|
|
||||||
await batches.freeze(s, settings)
|
|
||||||
assert package.content_hash == news.content_hash and package.state == 'index_pending'
|
assert package.content_hash == news.content_hash and package.state == 'index_pending'
|
||||||
|
assert ('index', package.id) in await batches.actions(s, batch, set(), 8)
|
||||||
await clear(s)
|
await clear(s)
|
||||||
|
|
||||||
count = 33000
|
count = 33000
|
||||||
await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)])
|
await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)])
|
||||||
await s.commit()
|
await s.commit()
|
||||||
batch = await batches.freeze(s, settings)
|
batch = await batches.freeze(s, settings)
|
||||||
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == count
|
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == 30
|
||||||
assert await s.scalar(select(func.count()).select_from(Package)) == count
|
assert await s.scalar(select(func.count()).select_from(Package)) == 30
|
||||||
# Exercise the existing-pending-ID and membership arrays above the driver limit too.
|
# Exercise the existing-pending-ID and membership arrays above the driver limit too.
|
||||||
next_batch = await batches.freeze(s, settings)
|
next_batch = await batches.freeze(s, settings)
|
||||||
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == count
|
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == 30
|
||||||
assert len(await batches.actions(s, next_batch, set(), 8)) == 8
|
assert len(await batches.actions(s, next_batch, set(), 8)) == 8
|
||||||
await clear(s)
|
await clear(s)
|
||||||
print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.')
|
print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.')
|
||||||
|
|
||||||
|
|
||||||
|
async def queue_review_regressions(now, settings):
|
||||||
|
async def clear(session):
|
||||||
|
await session.execute(text('TRUNCATE news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
async with SessionLocal() as s:
|
||||||
|
news = NewsItem(source_code='wscn_news', external_id='review-barrier', title='historical',
|
||||||
|
summary='', content_text='historical', content_hash='a'*64,
|
||||||
|
url='https://example.invalid/review-barrier', published_at=now)
|
||||||
|
s.add(news)
|
||||||
|
await s.flush()
|
||||||
|
package = Package(news_id=news.id, content_hash=news.content_hash, state='assign_pending', keywords=[])
|
||||||
|
event = Event(title='historical event', tags=[], revision=1, state='pending', due_at=now)
|
||||||
|
batch = Batch(phase='assigning', cutoff_at=now)
|
||||||
|
s.add_all([package, event, batch])
|
||||||
|
await s.flush()
|
||||||
|
s.add(Membership(event_id=event.id, package_id=package.id,
|
||||||
|
evidence='historical', reason='same event', change_kind='new_fact'))
|
||||||
|
s.add(BatchPackage(batch_id=batch.id, package_id=package.id, content_hash=package.content_hash, state='pending'))
|
||||||
|
await s.commit()
|
||||||
|
unrelated = package.id + 1000
|
||||||
|
# Assignment remains globally serial even when the other call is not in this batch.
|
||||||
|
assert await batches.actions(s, batch, {('assign', unrelated)}, 7) == []
|
||||||
|
assert batch.phase == 'assigning'
|
||||||
|
# Active work on this batch still holds its assignment barrier.
|
||||||
|
package.state = 'done'
|
||||||
|
await s.commit()
|
||||||
|
assert await batches.actions(s, batch, {('assign', package.id)}, 7) == []
|
||||||
|
assert batch.phase == 'assigning'
|
||||||
|
# Unrelated realtime package and event calls must not hold the barrier.
|
||||||
|
realtime = {('index', unrelated), ('event', event.id + 1000)}
|
||||||
|
assert await batches.actions(s, batch, realtime, 6) == []
|
||||||
|
await s.commit()
|
||||||
|
assert batch.phase == 'researching'
|
||||||
|
assert await batches.actions(s, batch, realtime, 6) == [('event', event.id)]
|
||||||
|
await s.commit()
|
||||||
|
event.state = 'done'
|
||||||
|
await s.commit()
|
||||||
|
# A completed result can be visible before the owning task has returned.
|
||||||
|
assert await batches.actions(s, batch, realtime | {('event', event.id)}, 5) == []
|
||||||
|
assert batch.phase == 'researching'
|
||||||
|
assert await batches.actions(s, batch, realtime | {('index', package.id)}, 5) == []
|
||||||
|
assert batch.phase == 'researching'
|
||||||
|
assert await batches.actions(s, batch, realtime, 6) == []
|
||||||
|
await s.commit()
|
||||||
|
assert batch.phase == 'done'
|
||||||
|
# Completed batches release admission of the next historical originals.
|
||||||
|
news2 = NewsItem(source_code='wscn_news', external_id='review-next', title='next', summary='',
|
||||||
|
content_text='next', content_hash='b'*64,
|
||||||
|
url='https://example.invalid/review-next', published_at=now)
|
||||||
|
s.add(news2)
|
||||||
|
await s.commit()
|
||||||
|
assert (await batches.freeze(s, settings)).id != batch.id
|
||||||
|
await clear(s)
|
||||||
|
|
||||||
|
# Thirty manual-only deferred heads cannot consume the thirty runnable slots.
|
||||||
|
news = NewsItem(source_code='wscn_news', external_id='review-deferred', title='source', summary='',
|
||||||
|
content_text='source', content_hash='a'*64,
|
||||||
|
url='https://example.invalid/review-deferred', published_at=now)
|
||||||
|
s.add(news)
|
||||||
|
await s.flush()
|
||||||
|
package = Package(news_id=news.id, content_hash=news.content_hash, state='done', keywords=[])
|
||||||
|
previous = Batch(phase='partial', cutoff_at=now)
|
||||||
|
events = [Event(title='deferred-head-'+str(i), tags=[], revision=7, state='pending',
|
||||||
|
due_at=now-timedelta(minutes=35-i)) for i in range(35)]
|
||||||
|
s.add_all([package, previous, *events])
|
||||||
|
await s.flush()
|
||||||
|
for i, event in enumerate(events):
|
||||||
|
s.add(Membership(event_id=event.id, package_id=package.id, evidence='source', reason='same source', change_kind='new_fact'))
|
||||||
|
if i <= 30:
|
||||||
|
s.add(BatchEvent(batch_id=previous.id, event_id=event.id,
|
||||||
|
revision=7 if i < 30 else 6, state='deferred'))
|
||||||
|
await s.commit()
|
||||||
|
batch = await batches.freeze(s, settings)
|
||||||
|
assert batch is not None
|
||||||
|
selected = set(await s.scalars(select(BatchEvent.event_id).where(BatchEvent.batch_id == batch.id)))
|
||||||
|
assert selected == {event.id for event in events[30:]}, selected
|
||||||
|
assert all(event.state == 'pending' and event.revision == 7 for event in events[:30])
|
||||||
|
await batches.actions(s, batch, set(), 8)
|
||||||
|
await s.commit()
|
||||||
|
actions = await batches.actions(s, batch, set(), 8)
|
||||||
|
assert set(actions) == {('event', event.id) for event in events[30:]}
|
||||||
|
await clear(s)
|
||||||
|
print('Queue review regressions passed: unrelated realtime calls allow both batch barriers to advance, owning calls still block, assignment stays serial, and deferred heads cannot starve runnable events.')
|
||||||
|
|
||||||
|
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
@@ -393,14 +393,20 @@ async def main():
|
|||||||
assert report.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
assert report.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
||||||
assert set(report.json()["data"]["outside"]) == {"unprocessed", "stale", "pending", "processing", "failed"}
|
assert set(report.json()["data"]["outside"]) == {"unprocessed", "stale", "pending", "processing", "failed"}
|
||||||
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == previous_start
|
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == previous_start
|
||||||
|
assert (await client.post("/api/risk/enable", json={"start_at": selected_start})).status_code == 422
|
||||||
|
settings = (await client.get("/api/risk/settings")).json()["data"]
|
||||||
|
saved = await client.put("/api/risk/settings", json={**config, "start_at": selected_start, "base_revision": settings["control_revision"]})
|
||||||
|
assert saved.status_code == 200, saved.text
|
||||||
with patch.object(model_service, "complete", validation):
|
with patch.object(model_service, "complete", validation):
|
||||||
response = await client.post("/api/risk/enable", json={"start_at": selected_start})
|
response = await client.post("/api/risk/enable")
|
||||||
assert response.status_code == 200, response.text
|
assert response.status_code == 200, response.text
|
||||||
assert response.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
assert response.json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
||||||
assert (await client.get("/api/risk/settings")).json()["data"][
|
assert (await client.get("/api/risk/settings")).json()["data"][
|
||||||
"enabled"
|
"enabled"
|
||||||
] is True
|
] is True
|
||||||
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
||||||
|
fixed_anchor = (await client.get("/api/risk/settings")).json()["data"]["realtime_from"]
|
||||||
|
assert fixed_anchor is not None
|
||||||
assert (await client.post("/api/risk/enable")).status_code == 409
|
assert (await client.post("/api/risk/enable")).status_code == 409
|
||||||
assert (await client.post("/api/risk/pause")).status_code == 200
|
assert (await client.post("/api/risk/pause")).status_code == 200
|
||||||
for action in ("pause", "save"):
|
for action in ("pause", "save"):
|
||||||
@@ -421,12 +427,13 @@ async def main():
|
|||||||
return await validation(provider, skill, payload, schema)
|
return await validation(provider, skill, payload, schema)
|
||||||
|
|
||||||
with patch.object(model_service, "complete", interrupted_validation):
|
with patch.object(model_service, "complete", interrupted_validation):
|
||||||
response = await client.post("/api/risk/enable", json={"start_at": "2026-09-01T00:00:00Z"})
|
response = await client.post("/api/risk/enable")
|
||||||
assert response.status_code == 409, response.text
|
assert response.status_code == 409, response.text
|
||||||
assert (await client.get("/api/risk/settings")).json()["data"][
|
assert (await client.get("/api/risk/settings")).json()["data"][
|
||||||
"enabled"
|
"enabled"
|
||||||
] is False
|
] is False
|
||||||
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
assert (await client.get("/api/risk/settings")).json()["data"]["start_at"] == "2026-09-30T16:00:00Z"
|
||||||
|
assert (await client.get("/api/risk/settings")).json()["data"]["realtime_from"] == fixed_anchor
|
||||||
package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[
|
package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[
|
||||||
"data"
|
"data"
|
||||||
][0]["id"]
|
][0]["id"]
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""Exercise preemption and recovery using disposable PostgreSQL, never a model."""
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from sqlalchemy import select, text, func
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from unittest.mock import patch
|
||||||
|
from alphapulse.core.db import SessionLocal, engine
|
||||||
|
from alphapulse.models import NewsItem
|
||||||
|
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version, ResearchSettings
|
||||||
|
from alphapulse.news_research import scheduling, batches, service
|
||||||
|
from alphapulse.news_research.worker import recover_interrupted
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
url = urlparse(os.environ['DATABASE_URL'])
|
||||||
|
assert (url.username, url.password, url.path) == ('ci', 'ci-only', '/ci')
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
anchor = now - timedelta(hours=1)
|
||||||
|
config = SimpleNamespace(start_at=now-timedelta(days=5), realtime_from=anchor)
|
||||||
|
async with SessionLocal() as s:
|
||||||
|
assert await s.scalar(select(func.count()).select_from(Package)) == 0
|
||||||
|
def original(key, published):
|
||||||
|
return NewsItem(source_code='wscn_news', external_id='priority-'+key, title=key, summary='',
|
||||||
|
content_text=key, content_hash='a'*64, url='https://example.invalid/'+key, published_at=published)
|
||||||
|
old = original('old', now-timedelta(days=4))
|
||||||
|
live = original('live', now-timedelta(minutes=30))
|
||||||
|
late = original('late', now-timedelta(days=3))
|
||||||
|
s.add_all([old, live, late]); await s.commit()
|
||||||
|
batch = await batches.freeze(s, config)
|
||||||
|
packages = list(await s.scalars(select(Package)))
|
||||||
|
assert len(packages) == 2 and all(p.news_id != live.id for p in packages)
|
||||||
|
actions = await scheduling.realtime_actions(s, config, {('index', packages[0].id)}, 1)
|
||||||
|
live_package = await s.scalar(select(Package).where(Package.news_id == live.id))
|
||||||
|
assert actions == [('index', live_package.id)]
|
||||||
|
assert await scheduling.realtime_actions(s, config, set(), 0) == []
|
||||||
|
# Shrinking the history range must not exclude accumulated realtime news.
|
||||||
|
config.start_at = now-timedelta(minutes=5)
|
||||||
|
actions = await scheduling.realtime_actions(s, config, set(), 1)
|
||||||
|
assert actions == [('index', live_package.id)]
|
||||||
|
await batches.actions(s, batch, set(), 8, config)
|
||||||
|
assert not await batches.freeze(s, config)
|
||||||
|
config.start_at = now-timedelta(days=5)
|
||||||
|
assert await batches.freeze(s, config)
|
||||||
|
assert packages[0].keywords == []
|
||||||
|
# A realtime event's historical dependencies inherit priority.
|
||||||
|
event = Event(title='shared', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
|
||||||
|
s.add(event); await s.flush()
|
||||||
|
for p in [packages[0], live_package]:
|
||||||
|
s.add(Membership(event_id=event.id, package_id=p.id, evidence='fixture', reason='fixture', change_kind='new_fact'))
|
||||||
|
packages[0].state = 'assign_pending'
|
||||||
|
live_package.state = 'done'
|
||||||
|
await s.commit()
|
||||||
|
config.start_at = now-timedelta(minutes=5)
|
||||||
|
assert await scheduling.realtime_actions(s, config, set(), 1) == []
|
||||||
|
config.start_at = now-timedelta(days=5)
|
||||||
|
assert await scheduling.realtime_actions(s, config, set(), 1) == [('assign', packages[0].id)]
|
||||||
|
packages[0].state = 'done'; await s.commit()
|
||||||
|
assert await scheduling.realtime_actions(s, config, set(), 1) == [('event', event.id)]
|
||||||
|
event.due_at = now+timedelta(minutes=5); await s.commit()
|
||||||
|
assert ('event', event.id) not in await scheduling.realtime_actions(s, config, set(), 8)
|
||||||
|
# Pausing invalidates a selected but not-yet-claimed task.
|
||||||
|
s.add(ResearchSettings(id=1, config={}, enabled=False, control_revision=9, start_at=config.start_at, realtime_from=anchor))
|
||||||
|
await s.commit()
|
||||||
|
assert await scheduling.control(s, {'_control_revision': 8}) is False
|
||||||
|
await s.commit()
|
||||||
|
with patch.object(service, 'complete', side_effect=AssertionError('Paused selection must not call a model')):
|
||||||
|
await service.process_package(live_package.id, {'_control_revision': 8})
|
||||||
|
assert await s.scalar(select(func.count()).select_from(Attempt)) == 0
|
||||||
|
live_package = await s.get(Package, live_package.id)
|
||||||
|
live_package.state = 'index_processing'
|
||||||
|
s.add(Attempt(package_id=live_package.id, content_hash='a'*64, stage='index', state='processing', input={}, rules={}))
|
||||||
|
event = await s.get(Event, event.id); event.state='processing'
|
||||||
|
s.add(Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}))
|
||||||
|
await s.commit()
|
||||||
|
for duplicate in [
|
||||||
|
Attempt(package_id=live_package.id, content_hash='a'*64, stage='assign', state='processing', input={}, rules={}),
|
||||||
|
Version(event_id=event.id, revision=event.revision, state='processing', input={}, rules={}),
|
||||||
|
]:
|
||||||
|
try:
|
||||||
|
async with s.begin_nested():
|
||||||
|
s.add(duplicate)
|
||||||
|
await s.flush()
|
||||||
|
except IntegrityError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError('Concurrent duplicate claim was accepted')
|
||||||
|
await recover_interrupted()
|
||||||
|
async with SessionLocal() as s:
|
||||||
|
assert (await s.get(Package, live_package.id)).state == 'index_failed'
|
||||||
|
assert (await s.get(Event, event.id)).state == 'failed'
|
||||||
|
assert await scheduling.realtime_actions(s, config, set(), 8) == []
|
||||||
|
empty = Event(title='removed members', tags=[], state='pending', revision=1, realtime_pending=True, due_at=now)
|
||||||
|
s.add(empty); await s.commit()
|
||||||
|
assert ('event', empty.id) in await scheduling.realtime_actions(s, config, set(), 8)
|
||||||
|
with patch.object(service.Research, 'run', side_effect=AssertionError('Empty events must not call a model')):
|
||||||
|
await service.process_event(empty.id, {})
|
||||||
|
await s.refresh(empty)
|
||||||
|
assert empty.state == 'retired'
|
||||||
|
await s.execute(text('TRUNCATE news_research_settings, news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
|
||||||
|
await s.commit()
|
||||||
|
await engine.dispose()
|
||||||
|
print('Priority queue passed: live preemption, stable boundary, late history, range changes, dependencies, aggregation, stale control and uncertain-call recovery.')
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
@@ -2,8 +2,8 @@ import type * as Research from "./type"
|
|||||||
import { request } from "@/http/axios"
|
import { request } from "@/http/axios"
|
||||||
|
|
||||||
export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" })
|
export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" })
|
||||||
export const saveSettingsApi = (data: Research.Config) => request<null>({ url: "/risk/settings", method: "put", data })
|
export const saveSettingsApi = (data: Research.Config & { start_at: string, base_revision: number }) => request<null>({ url: "/risk/settings", method: "put", data })
|
||||||
export const enableApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/enable", method: "post", data: { start_at } })
|
export const enableApi = () => request<Research.Coverage>({ url: "/risk/enable", method: "post" })
|
||||||
export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } })
|
export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } })
|
||||||
export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" })
|
export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" })
|
||||||
export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" })
|
export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" })
|
||||||
|
|||||||
@@ -2,13 +2,13 @@ export interface Keyword { canonical: string, aliases: string[], quote: string }
|
|||||||
export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null }
|
export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null }
|
||||||
export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number }
|
export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number }
|
||||||
export interface BatchProgress { id: number, phase: string, cutoff_at: string, packages_total: number, packages_done: number, packages_failed: number, packages_deferred: number, events_total: number, events_done: number, events_failed: number, events_deferred: number }
|
export interface BatchProgress { id: number, phase: string, cutoff_at: string, packages_total: number, packages_done: number, packages_failed: number, packages_deferred: number, events_total: number, events_done: number, events_failed: number, events_deferred: number }
|
||||||
export interface Settings { batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string }
|
export interface Settings { control_revision: number, realtime_from: string | null, queue: { running: number, draining: boolean, realtime: Record<string, number>, backfill: Record<string, number> }, batch?: BatchProgress | null, config: Partial<Config>, enabled: boolean, start_at: string }
|
||||||
export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } }
|
export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } }
|
||||||
export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] }
|
export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] }
|
||||||
export interface Point { time_precision: "minute" | "day" | "month" | "unknown", planned_precision: "minute" | "day" | "month" | "unknown", package_ids: number[], time_text: string, description: string, occurred_at: string | null, planned_at: string | null, reported_at: string | null, period_text: string, evidence_refs: { article_id: number, quote: string }[] }
|
export interface Point { time_precision: "minute" | "day" | "month" | "unknown", planned_precision: "minute" | "day" | "month" | "unknown", package_ids: number[], time_text: string, description: string, occurred_at: string | null, planned_at: string | null, reported_at: string | null, period_text: string, evidence_refs: { article_id: number, quote: string }[] }
|
||||||
export interface Result { title: string, summary: string, facts: string, reasoning: string, uncertainty: string, opinions: string, counter_evidence: string, watchpoints: string, asset_impacts: string, importance_stars: number, risk_stars: number, confidence: string, verification: string, timeline_points: Point[] }
|
export interface Result { title: string, summary: string, facts: string, reasoning: string, uncertainty: string, opinions: string, counter_evidence: string, watchpoints: string, asset_impacts: string, importance_stars: number, risk_stars: number, confidence: string, verification: string, timeline_points: Point[] }
|
||||||
export interface Event { id: number, title: string, state: string, revision: number, error: string | null, tags: string[], result: Result | null }
|
export interface Event { id: number, title: string, state: string, revision: number, error: string | null, tags: string[], result: Result | null }
|
||||||
export interface EventDetail extends Event { packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] }
|
export interface EventDetail extends Event { wait_reason?: string | null, packages: Package[], references: { event_id: number, title: string, version_id: number, role: string, reason: string, changed: boolean }[], versions: { id: number, revision: number, state: string, result: Result | null, error: string | null, usage: unknown, evidence: unknown }[] }
|
||||||
export interface PackageDetail extends Package { article: { title: string, summary: string, content: string }, attempts: { id: number, stage: string, state: string, error: string | null, output: unknown, usage: unknown }[] }
|
export interface PackageDetail extends Package { article: { title: string, summary: string, content: string }, attempts: { id: number, stage: string, state: string, error: string | null, output: unknown, usage: unknown }[] }
|
||||||
|
|
||||||
export interface VocabularyEntry { canonical: string, aliases: string[] }
|
export interface VocabularyEntry { canonical: string, aliases: string[] }
|
||||||
|
|||||||
@@ -6,8 +6,15 @@ import * as api from "@@/apis/news-research"
|
|||||||
import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings"
|
import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings"
|
||||||
import { listSkillTreeApi } from "@@/apis/skills"
|
import { listSkillTreeApi } from "@@/apis/skills"
|
||||||
import { EFFORTS } from "@@/constants/ai"
|
import { EFFORTS } from "@@/constants/ai"
|
||||||
|
import dayjs from "dayjs"
|
||||||
|
|
||||||
|
import timezone from "dayjs/plugin/timezone"
|
||||||
|
import utc from "dayjs/plugin/utc"
|
||||||
import { useUserStore } from "@/pinia/stores/user"
|
import { useUserStore } from "@/pinia/stores/user"
|
||||||
|
|
||||||
|
dayjs.extend(utc)
|
||||||
|
dayjs.extend(timezone)
|
||||||
|
|
||||||
const vocabularyOpened = ref(false)
|
const vocabularyOpened = ref(false)
|
||||||
const vocabulary = ref<Vocabulary>({ revision: 0, entries: [] })
|
const vocabulary = ref<Vocabulary>({ revision: 0, entries: [] })
|
||||||
const vocabularyBusy = ref(false)
|
const vocabularyBusy = ref(false)
|
||||||
@@ -90,8 +97,19 @@ const settingsOpened = ref(false)
|
|||||||
const enabled = ref(false)
|
const enabled = ref(false)
|
||||||
const startAt = ref<Date | null>(null)
|
const startAt = ref<Date | null>(null)
|
||||||
const savedStartAt = ref<Date | null>(null)
|
const savedStartAt = ref<Date | null>(null)
|
||||||
|
const settingsRevision = ref(0)
|
||||||
|
const queue = ref<{ running: number, draining: boolean, realtime?: Record<string, number>, backfill?: Record<string, number> } | null>(null)
|
||||||
|
function disabledBackfillDate(date: Date) {
|
||||||
|
return dayjs.tz(dayjs(date).format("YYYY-MM-DD"), "Asia/Shanghai").valueOf() > Date.now()
|
||||||
|
}
|
||||||
|
const startText = computed({
|
||||||
|
get: () => startAt.value ? dayjs(startAt.value).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : "",
|
||||||
|
set: (value: string) => {
|
||||||
|
startAt.value = value ? dayjs.tz(value, "Asia/Shanghai").toDate() : null
|
||||||
|
}
|
||||||
|
})
|
||||||
const selectedStartAt = computed({
|
const selectedStartAt = computed({
|
||||||
get: () => enabled.value ? savedStartAt.value : startAt.value,
|
get: () => enabled.value && !settingsOpened.value ? savedStartAt.value : startAt.value,
|
||||||
set: (value: Date | null) => {
|
set: (value: Date | null) => {
|
||||||
startAt.value = value
|
startAt.value = value
|
||||||
}
|
}
|
||||||
@@ -134,6 +152,7 @@ async function load() {
|
|||||||
const [settings, rows] = await Promise.all([api.getSettingsApi(), tab.value === "events" ? api.listEventsApi(q.value, page.value) : api.listPackagesApi(q.value, page.value)])
|
const [settings, rows] = await Promise.all([api.getSettingsApi(), tab.value === "events" ? api.listEventsApi(q.value, page.value) : api.listPackagesApi(q.value, page.value)])
|
||||||
if (!valid() || seq !== sequence) return
|
if (!valid() || seq !== sequence) return
|
||||||
enabled.value = settings.enabled
|
enabled.value = settings.enabled
|
||||||
|
queue.value = settings.queue ?? null
|
||||||
if (!startAt.value || startAt.value.getTime() === savedStartAt.value?.getTime()) startAt.value = new Date(settings.start_at)
|
if (!startAt.value || startAt.value.getTime() === savedStartAt.value?.getTime()) startAt.value = new Date(settings.start_at)
|
||||||
savedStartAt.value = new Date(settings.start_at)
|
savedStartAt.value = new Date(settings.start_at)
|
||||||
batch.value = settings.batch ?? null
|
batch.value = settings.batch ?? null
|
||||||
@@ -166,13 +185,7 @@ async function checkCoverage() {
|
|||||||
}
|
}
|
||||||
async function enable() {
|
async function enable() {
|
||||||
if (!valid()) return
|
if (!valid()) return
|
||||||
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) {
|
await api.enableApi()
|
||||||
ElMessage.error("请选择不晚于当前时间的回补起始时间")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
const start = startAt.value.toISOString()
|
|
||||||
const result = await api.enableApi(start)
|
|
||||||
if (valid() && selectedStartAt.value?.toISOString() === start) coverage.value = result
|
|
||||||
}
|
}
|
||||||
async function openEvent(id: number) {
|
async function openEvent(id: number) {
|
||||||
eventId = id
|
eventId = id
|
||||||
@@ -248,6 +261,8 @@ async function settings() {
|
|||||||
providers.value = choices
|
providers.value = choices
|
||||||
skills.value = nodes.filter(s => s.scope === "news" && s.kind === "skill")
|
skills.value = nodes.filter(s => s.scope === "news" && s.kind === "skill")
|
||||||
if (value.config.index && value.config.research) config.value = value.config as Config
|
if (value.config.index && value.config.research) config.value = value.config as Config
|
||||||
|
startAt.value = new Date(value.start_at)
|
||||||
|
settingsRevision.value = value.control_revision
|
||||||
settingsOpened.value = true
|
settingsOpened.value = true
|
||||||
}
|
}
|
||||||
async function action(fn: () => Promise<unknown>) {
|
async function action(fn: () => Promise<unknown>) {
|
||||||
@@ -275,7 +290,12 @@ async function retryPackage() {
|
|||||||
if (id) await action(() => api.retryPackageApi(id))
|
if (id) await action(() => api.retryPackageApi(id))
|
||||||
}
|
}
|
||||||
async function save() {
|
async function save() {
|
||||||
await action(() => api.saveSettingsApi(config.value))
|
if (!valid() || busy.value) return
|
||||||
|
if (!startAt.value || !Number.isFinite(startAt.value.getTime()) || startAt.value.getTime() > Date.now()) {
|
||||||
|
ElMessage.error("请选择不晚于当前时间的回补起始时间")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await action(() => api.saveSettingsApi({ ...config.value, start_at: startAt.value!.toISOString(), base_revision: settingsRevision.value }))
|
||||||
if (valid()) settingsOpened.value = false
|
if (valid()) settingsOpened.value = false
|
||||||
}
|
}
|
||||||
async function prepareSkills() {
|
async function prepareSkills() {
|
||||||
@@ -364,7 +384,7 @@ onBeforeUnmount(() => {
|
|||||||
<el-button @click="settings">
|
<el-button @click="settings">
|
||||||
模型与 Skills
|
模型与 Skills
|
||||||
</el-button>
|
</el-button>
|
||||||
<el-button :loading="busy" :disabled="!enabled && (loading || error || !startAt)" @click="action(enabled ? api.pauseApi : enable)">
|
<el-button :loading="busy" :disabled="!enabled && (loading || error || queue?.draining)" @click="action(enabled ? api.pauseApi : enable)">
|
||||||
{{ enabled ? '暂停新任务' : '启用两阶段处理' }}
|
{{ enabled ? '暂停新任务' : '启用两阶段处理' }}
|
||||||
</el-button>
|
</el-button>
|
||||||
<el-button @click="openVocabulary">
|
<el-button @click="openVocabulary">
|
||||||
@@ -377,45 +397,20 @@ onBeforeUnmount(() => {
|
|||||||
<p class="mb-4 text-sm text-slate-500">
|
<p class="mb-4 text-sm text-slate-500">
|
||||||
新闻包保存原文和关键词;事件汇集相关报道,持续更新判断与时间线。
|
新闻包保存原文和关键词;事件汇集相关报道,持续更新判断与时间线。
|
||||||
</p>
|
</p>
|
||||||
<div class="mb-4 rounded border border-slate-200 p-3">
|
<p class="mb-4 text-sm text-slate-500">
|
||||||
<div class="flex flex-wrap items-center gap-3">
|
实时任务优先取得空闲并发;历史回补至 {{ savedStartAt ? dayjs(savedStartAt).tz("Asia/Shanghai").format("YYYY-MM-DD HH:mm:ss") : '—' }}(北京时间)。
|
||||||
<label for="research-start-at">回补起始时间</label>
|
<span v-if="queue?.draining">正在完成 {{ queue.running }} 个已接收调用,收尾后可重新启用。</span>
|
||||||
<el-date-picker
|
|
||||||
id="research-start-at"
|
|
||||||
v-model="selectedStartAt"
|
|
||||||
type="datetime"
|
|
||||||
:clearable="false"
|
|
||||||
:disabled="enabled || busy"
|
|
||||||
:disabled-date="(date: Date) => date.getTime() > Date.now()"
|
|
||||||
placeholder="选择回补起始时间"
|
|
||||||
/>
|
|
||||||
<span class="text-sm text-slate-500">按浏览器本地时区</span>
|
|
||||||
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
|
|
||||||
核查遗漏
|
|
||||||
</el-button>
|
|
||||||
</div>
|
|
||||||
<p class="mt-2 text-sm text-slate-500">
|
|
||||||
启用时保存起点,自动核查并补齐该时间以来已采集的新闻;先完成关键词与归属,再集中研究事件。调整起点不会取消已有队列。
|
|
||||||
</p>
|
</p>
|
||||||
<p v-if="coverage" class="mt-2 text-sm">
|
<p v-if="queue" class="mb-4 text-sm text-slate-500">
|
||||||
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
|
实时队列:等待索引 {{ queue.realtime?.index_pending ?? 0 }},等待归属 {{ queue.realtime?.assign_pending ?? 0 }};
|
||||||
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
|
回补队列:等待索引 {{ queue.backfill?.index_pending ?? 0 }},等待归属 {{ queue.backfill?.assign_pending ?? 0 }}。
|
||||||
失败项需手动重试,当前批次以外的遗漏排入下一批。
|
正在调用 {{ queue.running }};范围外暂停、失败和中断结果不明的任务不会自动重试。
|
||||||
</p>
|
</p>
|
||||||
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
|
|
||||||
范围外另有已入队但未完成的新闻 {{ coverage.outside.unprocessed }} 条:
|
|
||||||
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
|
|
||||||
已有任务继续按批次处理,失败项需手动重试。
|
|
||||||
</p>
|
|
||||||
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
|
|
||||||
核查失败,请重试。
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<p v-if="batch" class="mb-4 text-sm text-slate-500">
|
<p v-if="batch" class="mb-4 text-sm text-slate-500">
|
||||||
批次 #{{ batch.id }} · {{ batchPhases[batch.phase] ?? batch.phase }} ·
|
批次 #{{ batch.id }} · {{ batchPhases[batch.phase] ?? batch.phase }} ·
|
||||||
新闻归属 {{ batch.packages_done }}/{{ batch.packages_total }},失败 {{ batch.packages_failed }},延期 {{ batch.packages_deferred }} ·
|
新闻归属 {{ batch.packages_done }}/{{ batch.packages_total }},失败 {{ batch.packages_failed }},延期 {{ batch.packages_deferred }} ·
|
||||||
事件研究 {{ batch.events_done }}/{{ batch.events_total }},失败 {{ batch.events_failed }},延期 {{ batch.events_deferred }}。
|
事件研究 {{ batch.events_done }}/{{ batch.events_total }},失败 {{ batch.events_failed }},延期 {{ batch.events_deferred }}。
|
||||||
新新闻和重试请求排入下一批,研究仅使用站内资料。
|
实时任务可中途插队;回补任务分批归属后集中研究,研究仅使用站内资料。
|
||||||
</p>
|
</p>
|
||||||
<el-alert v-if="error" title="读取失败,保留上次结果,可刷新重试" type="error" :closable="false" />
|
<el-alert v-if="error" title="读取失败,保留上次结果,可刷新重试" type="error" :closable="false" />
|
||||||
<div class="mb-4 flex flex-wrap gap-3">
|
<div class="mb-4 flex flex-wrap gap-3">
|
||||||
@@ -474,6 +469,9 @@ onBeforeUnmount(() => {
|
|||||||
<el-drawer v-model="opened" title="事件研究与时间线" size="min(820px, 100vw)">
|
<el-drawer v-model="opened" title="事件研究与时间线" size="min(820px, 100vw)">
|
||||||
<el-alert v-if="detailError" title="事件详情读取失败,请重新打开重试" type="error" :closable="false" />
|
<el-alert v-if="detailError" title="事件详情读取失败,请重新打开重试" type="error" :closable="false" />
|
||||||
<template v-if="detail">
|
<template v-if="detail">
|
||||||
|
<p v-if="detail.wait_reason" class="mb-3 text-amber-700">
|
||||||
|
{{ detail.wait_reason }}
|
||||||
|
</p>
|
||||||
<h2>{{ detail.title }}</h2><p>{{ states[detail.state] ?? detail.state }}</p>
|
<h2>{{ detail.title }}</h2><p>{{ states[detail.state] ?? detail.state }}</p>
|
||||||
<el-button :loading="busy" @click="retryEvent">
|
<el-button :loading="busy" @click="retryEvent">
|
||||||
重新研究
|
重新研究
|
||||||
@@ -627,6 +625,42 @@ onBeforeUnmount(() => {
|
|||||||
</el-drawer>
|
</el-drawer>
|
||||||
<el-drawer v-model="settingsOpened" title="两阶段模型与 Skills" size="min(640px, 100vw)">
|
<el-drawer v-model="settingsOpened" title="两阶段模型与 Skills" size="min(640px, 100vw)">
|
||||||
<el-form label-position="top">
|
<el-form label-position="top">
|
||||||
|
<div class="mb-4 rounded border border-slate-200 p-3">
|
||||||
|
<div class="flex flex-wrap items-center gap-3">
|
||||||
|
<label for="research-start-at">回补起始时间</label>
|
||||||
|
<el-date-picker
|
||||||
|
id="research-start-at"
|
||||||
|
v-model="startText"
|
||||||
|
value-format="YYYY-MM-DD HH:mm:ss"
|
||||||
|
type="datetime"
|
||||||
|
:clearable="false"
|
||||||
|
:disabled="busy"
|
||||||
|
:disabled-date="disabledBackfillDate"
|
||||||
|
placeholder="选择回补起始时间"
|
||||||
|
/>
|
||||||
|
<span class="text-sm text-slate-500">北京时间(系统时区 UTC+8)</span>
|
||||||
|
<el-button :loading="coverageLoading" :disabled="!startAt || busy" @click="checkCoverage">
|
||||||
|
核查遗漏
|
||||||
|
</el-button>
|
||||||
|
</div>
|
||||||
|
<p class="mt-2 text-sm text-slate-500">
|
||||||
|
点击“保存并暂停”后生效;重新启用时重排未完成任务。实时新闻优先插队,历史新闻分批补齐。
|
||||||
|
</p>
|
||||||
|
<p v-if="coverage" class="mt-2 text-sm">
|
||||||
|
范围内 {{ coverage.total }} 条,全部阶段已完成 {{ coverage.completed }} 条,未完成 {{ coverage.unprocessed }} 条:
|
||||||
|
未入队 {{ coverage.missing }},原文更新需重判 {{ coverage.stale }},待处理 {{ coverage.pending }},处理中 {{ coverage.processing }},失败 {{ coverage.failed }}。
|
||||||
|
失败项需手动重试,当前批次以外的遗漏排入下一批。
|
||||||
|
</p>
|
||||||
|
<p v-if="coverage && coverage.outside.unprocessed" class="mt-2 text-sm text-amber-700">
|
||||||
|
范围外已暂停自动处理的新闻 {{ coverage.outside.unprocessed }} 条:
|
||||||
|
原文更新需重判 {{ coverage.outside.stale }},待处理 {{ coverage.outside.pending }},处理中 {{ coverage.outside.processing }},失败 {{ coverage.outside.failed }}。
|
||||||
|
已接收调用允许完成,扩大范围后恢复尚未完成的阶段;失败项需手动重试。
|
||||||
|
</p>
|
||||||
|
<p v-if="coverageError" class="mt-2 text-sm text-red-600">
|
||||||
|
核查失败,请重试。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<template v-for="stage in (['index', 'research'] as const)" :key="stage">
|
<template v-for="stage in (['index', 'research'] as const)" :key="stage">
|
||||||
<h3>{{ stage === 'index' ? '第一阶段:关键词索引与事件归属' : '第二阶段:事件整体研究' }}</h3>
|
<h3>{{ stage === 'index' ? '第一阶段:关键词索引与事件归属' : '第二阶段:事件整体研究' }}</h3>
|
||||||
<el-form-item label="模型接入">
|
<el-form-item label="模型接入">
|
||||||
|
|||||||
@@ -485,7 +485,7 @@ test("新闻列表只更新可见条目的匹配版本评级,保留滚动与
|
|||||||
|
|
||||||
function researchPage(api: Record<string, unknown> = {}, user = vue.reactive({ sessionVersion: 1 }), extra: Record<string, unknown> = {}, dependencies: Record<string, unknown> = {}) {
|
function researchPage(api: Record<string, unknown> = {}, user = vue.reactive({ sessionVersion: 1 }), extra: Record<string, unknown> = {}, dependencies: Record<string, unknown> = {}) {
|
||||||
const load = moduleLoader({
|
const load = moduleLoader({
|
||||||
"@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api },
|
"@@/apis/news-research": { getSettingsApi: async () => ({ config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }), getCoverageApi: async (start_at: string) => ({ start_at, total: 0 }), listEventsApi: async () => ({ items: [], total: 0 }), listPackagesApi: async () => ({ items: [], total: 0 }), ...api },
|
||||||
"@@/apis/skills": { listSkillTreeApi: async () => [] },
|
"@@/apis/skills": { listSkillTreeApi: async () => [] },
|
||||||
"@@/apis/settings": { listProvidersApi: async () => [], listProviderModelsApi: async () => [] },
|
"@@/apis/settings": { listProvidersApi: async () => [], listProviderModelsApi: async () => [] },
|
||||||
"@/pinia/stores/user": { useUserStore: () => user },
|
"@/pinia/stores/user": { useUserStore: () => user },
|
||||||
@@ -516,11 +516,11 @@ test("回补时间按时区提交,刷新保留未提交的起点,核查不
|
|||||||
assert.equal(enabled.length, 0)
|
assert.equal(enabled.length, 0)
|
||||||
assert.equal(page.coverage.value.missing, 4)
|
assert.equal(page.coverage.value.missing, 4)
|
||||||
await page.enable()
|
await page.enable()
|
||||||
assert.deepEqual(enabled, ["2026-09-30T16:00:00.000Z"])
|
assert.deepEqual(enabled, [undefined])
|
||||||
})
|
})
|
||||||
|
|
||||||
test("其他标签页启用后按实际起点核查,暂停后恢复草稿并丢弃旧范围响应", async () => {
|
test("其他标签页启用后按实际起点核查,暂停后恢复草稿并丢弃旧范围响应", async () => {
|
||||||
let server = { config: {}, enabled: false, start_at: "2026-10-31T16:00:00Z" }
|
let server = { config: {}, control_revision: 0, enabled: false, start_at: "2026-10-31T16:00:00Z" }
|
||||||
const pending = Promise.withResolvers()
|
const pending = Promise.withResolvers()
|
||||||
const checked: string[] = []
|
const checked: string[] = []
|
||||||
let delayDraft = true
|
let delayDraft = true
|
||||||
@@ -612,13 +612,13 @@ test("切换会话后,旧回补核查不能恢复统计结果", async () => {
|
|||||||
assert.equal(page.startAt.value, null)
|
assert.equal(page.startAt.value, null)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("回补时间无效或晚于当前时间时禁止启用", async () => {
|
test("回补时间无效或晚于当前时间时禁止保存", async () => {
|
||||||
let calls = 0
|
let calls = 0
|
||||||
const errors: string[] = []
|
const errors: string[] = []
|
||||||
const page = researchPage({ enableApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } })
|
const page = researchPage({ saveSettingsApi: async () => calls++ }, undefined, { ElMessage: { error: (message: string) => errors.push(message) } })
|
||||||
for (const value of [null, new Date("invalid"), new Date(Date.now() + 86400000)]) {
|
for (const value of [null, new Date("invalid"), new Date(Date.now() + 86400000)]) {
|
||||||
page.startAt.value = value
|
page.startAt.value = value
|
||||||
await page.enable()
|
await page.save()
|
||||||
}
|
}
|
||||||
assert.equal(calls, 0)
|
assert.equal(calls, 0)
|
||||||
assert.equal(errors.length, 3)
|
assert.equal(errors.length, 3)
|
||||||
@@ -718,12 +718,15 @@ test("两阶段保存各自思考强度,不提交第三阶段配置", async ()
|
|||||||
const page = researchPage({ saveSettingsApi: async (data: unknown) => {
|
const page = researchPage({ saveSettingsApi: async (data: unknown) => {
|
||||||
saved = JSON.parse(JSON.stringify(data))
|
saved = JSON.parse(JSON.stringify(data))
|
||||||
} })
|
} })
|
||||||
|
await page.load()
|
||||||
page.config.value.index.reasoning_effort = "low"
|
page.config.value.index.reasoning_effort = "low"
|
||||||
page.config.value.research.reasoning_effort = "high"
|
page.config.value.research.reasoning_effort = "high"
|
||||||
await page.save()
|
await page.save()
|
||||||
assert.equal(saved.index.reasoning_effort, "low")
|
assert.equal(saved.index.reasoning_effort, "low")
|
||||||
assert.equal(saved.research.reasoning_effort, "high")
|
assert.equal(saved.research.reasoning_effort, "high")
|
||||||
assert.equal(Object.keys(saved).length, 3)
|
assert.equal(Object.keys(saved).length, 5)
|
||||||
|
assert.equal(saved.start_at, "2026-10-31T16:00:00.000Z")
|
||||||
|
assert.equal(saved.base_revision, 0)
|
||||||
page.config.value.research.reasoning_effort = null
|
page.config.value.research.reasoning_effort = null
|
||||||
await page.save()
|
await page.save()
|
||||||
assert.equal(saved.research.reasoning_effort, null)
|
assert.equal(saved.research.reasoning_effort, null)
|
||||||
@@ -756,6 +759,7 @@ test("两阶段操作拒绝重复提交和旧会话操作,失败保留配置
|
|||||||
calls++
|
calls++
|
||||||
return pending.promise
|
return pending.promise
|
||||||
} }, user)
|
} }, user)
|
||||||
|
await page.load()
|
||||||
page.config.value.index.model = "draft"
|
page.config.value.index.model = "draft"
|
||||||
const first = page.save()
|
const first = page.save()
|
||||||
await page.save()
|
await page.save()
|
||||||
@@ -964,3 +968,27 @@ test("词库保存拒绝重复提交,关闭后的响应不覆盖草稿,失
|
|||||||
assert.equal(page.vocabularyOpened.value, true)
|
assert.equal(page.vocabularyOpened.value, true)
|
||||||
assert.equal(page.vocabularyBusy.value, false)
|
assert.equal(page.vocabularyBusy.value, false)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("侧栏时间统一使用北京时间,保存时间与配置版本但启用不提交草稿", async () => {
|
||||||
|
let saved: any
|
||||||
|
let enableArgs: unknown[] = []
|
||||||
|
const page = researchPage({
|
||||||
|
getSettingsApi: async () => ({ config: {}, enabled: true, control_revision: 7, start_at: "2026-11-04T16:00:00Z" }),
|
||||||
|
saveSettingsApi: async (value: unknown) => {
|
||||||
|
saved = value
|
||||||
|
},
|
||||||
|
enableApi: async (...args: unknown[]) => {
|
||||||
|
enableArgs = args
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await page.settings()
|
||||||
|
assert.equal(page.startText.value, "2026-11-05 00:00:00")
|
||||||
|
page.startText.value = "2026-10-20 00:00:00"
|
||||||
|
await page.load()
|
||||||
|
assert.equal(page.startText.value, "2026-10-20 00:00:00")
|
||||||
|
await page.save()
|
||||||
|
assert.equal(saved.start_at, "2026-10-19T16:00:00.000Z")
|
||||||
|
assert.equal(saved.base_revision, 7)
|
||||||
|
await page.enable()
|
||||||
|
assert.deepEqual(enableArgs, [])
|
||||||
|
})
|
||||||
Reference in new issue
Block a user