Add configurable news research backfill and coverage reporting
Development CI / check-and-build (push) Successful in 4m8s
GitHub Git mirror / mirror (push) Successful in 6s
Development CI / deploy-development (push) Successful in 2m29s

Allow selecting a historical start time and checking unfinished news before enabling analysis. Report enrolled work outside the selected range and use the active server boundary while retaining paused drafts. Cover date validation, queue carryover, task completion and cross-tab responses.
This commit is contained in:
eezd committed 2026-11-06 12:49:20 +08:00
1 parent 80a1aa389c
commit c90691d9a0
9 files changed
+451 -10

No files matched your search

+21 -3
View File
@@ -3,7 +3,7 @@
from datetime import UTC, datetime
from typing import Annotated, Literal
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import Field
from pydantic import AwareDatetime, Field
from sqlalchemy import select, update, func, or_, text
from sqlalchemy.dialects.postgresql import insert
from alphapulse.core.db import get_session
@@ -119,6 +119,15 @@ async def settings(session: DB):
)
@router.get("/coverage")
async def coverage(session: DB, start_at: AwareDatetime | None = None):
row = await settings_row(session)
start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
if start > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间")
return success(await batches.coverage(session, start))
@router.put("/settings")
async def save(body: SettingsInput, session: DB):
await settings_row(session)
@@ -147,9 +156,16 @@ async def pause(session: DB):
return success(None)
class EnableInput(Strict):
start_at: AwareDatetime
@router.post("/enable")
async def enable(session: DB):
async def enable(session: DB, body: EnableInput | None = None):
row = await settings_row(session)
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
if start_at > datetime.now(UTC):
raise HTTPException(422, "回补起始时间不能晚于当前时间")
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
raise HTTPException(409, "模型验证已在进行中")
if row.enabled:
@@ -242,10 +258,12 @@ async def enable(session: DB):
)
if row.control_revision != control_revision or row.config != original_config:
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
report = await batches.coverage(session, start_at)
row.start_at = start_at
row.enabled = True
row.control_revision += 1
await session.commit()
return success(None)
return success(report)
@router.post("/skill-templates")
@@ -1,12 +1,51 @@
"""Durable admission barriers: finish this batch's assignments before any research."""
from collections import Counter
from datetime import UTC, datetime
from sqlalchemy import select, or_, and_, exists, func, any_
from sqlalchemy import select, or_, and_, exists, func, any_, case
from alphapulse.models import NewsItem
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
from alphapulse.news_research.service import enroll
async def coverage(session, start_at, cutoff=None):
"""Count each original once, including unfinished research after assignment."""
cutoff = cutoff or datetime.now(UTC)
research = select(
Membership.package_id,
func.bool_or(Event.state == 'failed').label('failed'),
func.bool_or(Event.state == 'processing').label('processing'),
func.bool_or(Event.state.not_in(['done', 'retired'])).label('unfinished'),
).join(Event, Event.id == Membership.event_id).group_by(Membership.package_id).subquery()
state = case(
(Package.id.is_(None), 'missing'),
(Package.content_hash != NewsItem.content_hash, 'stale'),
(Package.state.in_(['index_failed', 'assign_failed']), 'failed'),
(Package.state.in_(['index_processing', 'assign_processing']), 'processing'),
(Package.state != 'done', 'pending'),
(research.c.failed, 'failed'),
(research.c.processing, 'processing'),
(or_(research.c.package_id.is_(None), research.c.unfinished), 'pending'),
else_='completed',
)
in_range = and_(NewsItem.published_at >= start_at, NewsItem.published_at <= cutoff)
rows = (await session.execute(
select(in_range, state, func.count()).select_from(NewsItem)
.outerjoin(Package, Package.news_id == NewsItem.id)
.outerjoin(research, research.c.package_id == Package.id)
.where(or_(in_range, Package.id.is_not(None)))
.group_by(in_range, state)
)).all()
counts = {status: count for selected, status, count in rows if selected}
outside_counts = {status: count for selected, status, count in rows if not selected}
outside = {name: outside_counts.get(name, 0) for name in
('stale', 'pending', 'processing', 'failed')}
result = {name: counts.get(name, 0) for name in
('missing', 'stale', 'pending', 'processing', 'failed', 'completed')}
return {'start_at': start_at, 'checked_at': cutoff, 'total': sum(result.values()),
'unprocessed': sum(result.values()) - result['completed'], **result,
'outside': {'unprocessed': sum(outside.values()), **outside}}
async def freeze(session, settings):
cutoff = datetime.now(UTC)
ids = list(await session.scalars(select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
@@ -260,6 +260,54 @@ def test_settings_contract_rejects_third_stage_and_unknown_efforts(trading):
)
def test_backfill_start_requires_an_explicit_timezone(trading):
from alphapulse.news_research.api import EnableInput
selected = EnableInput(start_at="2026-10-01T00:00:00+08:00")
assert selected.start_at == datetime(2026, 9, 30, 16, tzinfo=UTC)
with pytest.raises(ValueError, match="timezone_aware"):
EnableInput(start_at="2026-10-01T00:00:00")
@pytest.mark.parametrize("endpoint", ["coverage", "enable"])
def test_future_backfill_is_rejected_before_model_calls(trading, monkeypatch, endpoint):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api, service
future = datetime.now(UTC) + timedelta(days=1)
row = N(start_at=datetime.now(UTC), enabled=False)
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
config = AsyncMock(side_effect=AssertionError("不应调用模型"))
monkeypatch.setattr(service, "model_config", config)
call = (api.enable(session, api.EnableInput(start_at=future)) if endpoint == "enable"
else api.coverage(session, start_at=future))
with pytest.raises(HTTPException) as error:
asyncio.run(call)
assert error.value.status_code == 422
assert not row.enabled
config.assert_not_called()
session.commit.assert_not_called()
def test_failed_enable_does_not_save_the_selected_backfill_start(trading, monkeypatch):
import asyncio
from unittest.mock import AsyncMock
from fastapi import HTTPException
from alphapulse.news_research import api, service
previous = datetime(2026, 10, 1, tzinfo=UTC)
row = N(start_at=previous, enabled=False, config={}, control_revision=7)
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
with pytest.raises(HTTPException) as error:
asyncio.run(api.enable(session, api.EnableInput(start_at=previous - timedelta(days=30))))
assert error.value.status_code == 502
assert row.start_at == previous and not row.enabled and row.control_revision == 7
session.commit.assert_not_called()
@pytest.mark.parametrize("tool", ["read_news", "read_package"])
def test_compressed_members_read_frozen_positive_sources_and_reject_old_cache(
trading, tool