Add configurable news research backfill and coverage reporting
Allow selecting a historical start time and checking unfinished news before enabling analysis. Report enrolled work outside the selected range and use the active server boundary while retaining paused drafts. Cover date validation, queue carryover, task completion and cross-tab responses.
This commit is contained in:
1 parent
80a1aa389c
commit
c90691d9a0
9 files changed
+451
-10
No files matched your search
@@ -3,7 +3,7 @@
|
||||
from datetime import UTC, datetime
|
||||
from typing import Annotated, Literal
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import Field
|
||||
from pydantic import AwareDatetime, Field
|
||||
from sqlalchemy import select, update, func, or_, text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from alphapulse.core.db import get_session
|
||||
@@ -119,6 +119,15 @@ async def settings(session: DB):
|
||||
)
|
||||
|
||||
|
||||
@router.get("/coverage")
|
||||
async def coverage(session: DB, start_at: AwareDatetime | None = None):
|
||||
row = await settings_row(session)
|
||||
start = (start_at if start_at is not None else row.start_at).astimezone(UTC)
|
||||
if start > datetime.now(UTC):
|
||||
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
||||
return success(await batches.coverage(session, start))
|
||||
|
||||
|
||||
@router.put("/settings")
|
||||
async def save(body: SettingsInput, session: DB):
|
||||
await settings_row(session)
|
||||
@@ -147,9 +156,16 @@ async def pause(session: DB):
|
||||
return success(None)
|
||||
|
||||
|
||||
class EnableInput(Strict):
|
||||
start_at: AwareDatetime
|
||||
|
||||
|
||||
@router.post("/enable")
|
||||
async def enable(session: DB):
|
||||
async def enable(session: DB, body: EnableInput | None = None):
|
||||
row = await settings_row(session)
|
||||
start_at = (body.start_at if body is not None else row.start_at).astimezone(UTC)
|
||||
if start_at > datetime.now(UTC):
|
||||
raise HTTPException(422, "回补起始时间不能晚于当前时间")
|
||||
if not await session.scalar(text("SELECT pg_try_advisory_xact_lock(20261103, 2)")):
|
||||
raise HTTPException(409, "模型验证已在进行中")
|
||||
if row.enabled:
|
||||
@@ -242,10 +258,12 @@ async def enable(session: DB):
|
||||
)
|
||||
if row.control_revision != control_revision or row.config != original_config:
|
||||
raise HTTPException(409, "验证期间已暂停或设置变化,请重新验证")
|
||||
report = await batches.coverage(session, start_at)
|
||||
row.start_at = start_at
|
||||
row.enabled = True
|
||||
row.control_revision += 1
|
||||
await session.commit()
|
||||
return success(None)
|
||||
return success(report)
|
||||
|
||||
|
||||
@router.post("/skill-templates")
|
||||
|
||||
@@ -1,12 +1,51 @@
|
||||
"""Durable admission barriers: finish this batch's assignments before any research."""
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from sqlalchemy import select, or_, and_, exists, func, any_
|
||||
from sqlalchemy import select, or_, and_, exists, func, any_, case
|
||||
from alphapulse.models import NewsItem
|
||||
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
|
||||
from alphapulse.news_research.service import enroll
|
||||
|
||||
|
||||
async def coverage(session, start_at, cutoff=None):
|
||||
"""Count each original once, including unfinished research after assignment."""
|
||||
cutoff = cutoff or datetime.now(UTC)
|
||||
research = select(
|
||||
Membership.package_id,
|
||||
func.bool_or(Event.state == 'failed').label('failed'),
|
||||
func.bool_or(Event.state == 'processing').label('processing'),
|
||||
func.bool_or(Event.state.not_in(['done', 'retired'])).label('unfinished'),
|
||||
).join(Event, Event.id == Membership.event_id).group_by(Membership.package_id).subquery()
|
||||
state = case(
|
||||
(Package.id.is_(None), 'missing'),
|
||||
(Package.content_hash != NewsItem.content_hash, 'stale'),
|
||||
(Package.state.in_(['index_failed', 'assign_failed']), 'failed'),
|
||||
(Package.state.in_(['index_processing', 'assign_processing']), 'processing'),
|
||||
(Package.state != 'done', 'pending'),
|
||||
(research.c.failed, 'failed'),
|
||||
(research.c.processing, 'processing'),
|
||||
(or_(research.c.package_id.is_(None), research.c.unfinished), 'pending'),
|
||||
else_='completed',
|
||||
)
|
||||
in_range = and_(NewsItem.published_at >= start_at, NewsItem.published_at <= cutoff)
|
||||
rows = (await session.execute(
|
||||
select(in_range, state, func.count()).select_from(NewsItem)
|
||||
.outerjoin(Package, Package.news_id == NewsItem.id)
|
||||
.outerjoin(research, research.c.package_id == Package.id)
|
||||
.where(or_(in_range, Package.id.is_not(None)))
|
||||
.group_by(in_range, state)
|
||||
)).all()
|
||||
counts = {status: count for selected, status, count in rows if selected}
|
||||
outside_counts = {status: count for selected, status, count in rows if not selected}
|
||||
outside = {name: outside_counts.get(name, 0) for name in
|
||||
('stale', 'pending', 'processing', 'failed')}
|
||||
result = {name: counts.get(name, 0) for name in
|
||||
('missing', 'stale', 'pending', 'processing', 'failed', 'completed')}
|
||||
return {'start_at': start_at, 'checked_at': cutoff, 'total': sum(result.values()),
|
||||
'unprocessed': sum(result.values()) - result['completed'], **result,
|
||||
'outside': {'unprocessed': sum(outside.values()), **outside}}
|
||||
|
||||
|
||||
async def freeze(session, settings):
|
||||
cutoff = datetime.now(UTC)
|
||||
ids = list(await session.scalars(select(NewsItem.id).outerjoin(Package, Package.news_id == NewsItem.id).where(
|
||||
|
||||
@@ -260,6 +260,54 @@ def test_settings_contract_rejects_third_stage_and_unknown_efforts(trading):
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_start_requires_an_explicit_timezone(trading):
|
||||
from alphapulse.news_research.api import EnableInput
|
||||
|
||||
selected = EnableInput(start_at="2026-10-01T00:00:00+08:00")
|
||||
assert selected.start_at == datetime(2026, 9, 30, 16, tzinfo=UTC)
|
||||
with pytest.raises(ValueError, match="timezone_aware"):
|
||||
EnableInput(start_at="2026-10-01T00:00:00")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["coverage", "enable"])
|
||||
def test_future_backfill_is_rejected_before_model_calls(trading, monkeypatch, endpoint):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock
|
||||
from fastapi import HTTPException
|
||||
from alphapulse.news_research import api, service
|
||||
|
||||
future = datetime.now(UTC) + timedelta(days=1)
|
||||
row = N(start_at=datetime.now(UTC), enabled=False)
|
||||
session = N(get=AsyncMock(return_value=row), commit=AsyncMock())
|
||||
config = AsyncMock(side_effect=AssertionError("不应调用模型"))
|
||||
monkeypatch.setattr(service, "model_config", config)
|
||||
call = (api.enable(session, api.EnableInput(start_at=future)) if endpoint == "enable"
|
||||
else api.coverage(session, start_at=future))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(call)
|
||||
assert error.value.status_code == 422
|
||||
assert not row.enabled
|
||||
config.assert_not_called()
|
||||
session.commit.assert_not_called()
|
||||
|
||||
|
||||
def test_failed_enable_does_not_save_the_selected_backfill_start(trading, monkeypatch):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock
|
||||
from fastapi import HTTPException
|
||||
from alphapulse.news_research import api, service
|
||||
|
||||
previous = datetime(2026, 10, 1, tzinfo=UTC)
|
||||
row = N(start_at=previous, enabled=False, config={}, control_revision=7)
|
||||
session = N(get=AsyncMock(return_value=row), scalar=AsyncMock(side_effect=[True, 0, 0]), commit=AsyncMock())
|
||||
monkeypatch.setattr(service, "model_config", AsyncMock(side_effect=ValueError("Skill 不可用")))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(api.enable(session, api.EnableInput(start_at=previous - timedelta(days=30))))
|
||||
assert error.value.status_code == 502
|
||||
assert row.start_at == previous and not row.enabled and row.control_revision == 7
|
||||
session.commit.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", ["read_news", "read_package"])
|
||||
def test_compressed_members_read_frozen_positive_sources_and_reject_old_cache(
|
||||
trading, tool
|
||||
|
||||
Reference in new issue
Block a user