feat(news): index news packages and research persistent events

Extract reusable literal keywords once per source revision, assign complete news packages to long-lived events, and research all event members with immutable evidence and references.

Add additive research storage, vocabulary snapshots, the two-stage workspace, isolated acceptance coverage, and a paused rollout with an initial fixed 50-news production trial.
This commit is contained in:
eezd committed 2026-11-04 12:09:58 +08:00
1 parent 07252fea0f
commit 4929a32a0a
69 files changed
+8215 -2092

No files matched your search

+3 -9
View File
@@ -3,7 +3,7 @@ import json
from datetime import UTC, datetime
from sqlalchemy import select, func, or_, exists, BigInteger
from alphapulse.models import NewsItem
from alphapulse.models.risk import (ReprocessItem, ReprocessTarget, ArticleAnalysis, NewsSummary, NewsEvent, RiskSnapshot, RiskSettings, ReprocessJob)
from alphapulse.models.risk import (ReprocessItem, ReprocessTarget, ArticleAnalysis, NewsEvent, RiskSnapshot, RiskSettings, ReprocessJob)
from alphapulse.core.redis import redis
@@ -13,7 +13,7 @@ async def job_out(session, job):
counts = {state:sum(i.get('state','pending')==state for i in job.items) for state in ('pending','processing','done','failed')}
return {'id':job.id,'state':job.state,'total':len(job.items),'completed':job.position,'failures':job.failures,'request':None,'counts':counts}
stages = {}
for kind, model in (('analysis',NewsSummary),('events',NewsEvent)):
for kind, model in (('events',NewsEvent),):
stages[kind] = dict((await session.execute(select(model.state,func.count()).join(
ReprocessTarget,ReprocessTarget.object_id==model.id).where(ReprocessTarget.job_id==job.id,
ReprocessTarget.kind==kind).group_by(model.state))).all())
@@ -31,7 +31,7 @@ async def listing(session, state, stage, origin, job_id, page, limit):
entries=[]
total=0
offset=(page-1)*limit
for kind,model in [('grouping',ArticleAnalysis),('analysis',NewsSummary),('events',NewsEvent),('outlook',RiskSnapshot),('backfill',ReprocessItem)]:
for kind,model in [('grouping',ArticleAnalysis),('events',NewsEvent),('outlook',RiskSnapshot),('backfill',ReprocessItem)]:
if stage and stage!=kind:continue
query=select(model)
busy = [a['object_id'] for a in (active or []) if a['stage']==kind]
@@ -45,12 +45,6 @@ async def listing(session, state, stage, origin, job_id, page, limit):
pending = model.state.in_(['pending','processing','reprocess'])
due = getattr(model,'due_at',None)
waiting = due > now if due is not None else False
if kind=='events':
from alphapulse.models.risk import EventMember
blocked=select(EventMember.event_id).join(NewsSummary,NewsSummary.id==EventMember.package_id).where(
NewsSummary.replacement_ids==[],NewsSummary.state!='withdrawn',
or_(NewsSummary.state!='done',NewsSummary.result==None))
waiting=or_(waiting,model.id.in_(blocked))
if state=='running': query=query.where(model.id.in_(busy) if active is not None else model.state=='processing')
elif state=='waiting': query=query.where(pending,model.id.not_in(busy),waiting)
elif state=='queued': query=query.where(pending,model.state!='processing' if active is None else True,model.id.not_in(busy),~waiting if due is not None else True)