feat: bound news AI concurrency with immediate task refill
This commit is contained in:
1 parent
f1d060d475
commit
cbd2618a20
15 files changed
+603
-129
No files matched your search
@@ -91,11 +91,11 @@ async def associate(session, package, result, researched_ids, researched_events=
|
||||
event.updated_at = max(p.updated_at for p in members)
|
||||
|
||||
|
||||
async def organize():
|
||||
async def organize(target_id=None):
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
if not settings or not settings.enabled: return
|
||||
event = await session.scalar(select(NewsEvent).where(NewsEvent.state == 'pending', NewsEvent.due_at <= datetime.now(UTC)).order_by(NewsEvent.due_at).limit(1))
|
||||
event = await session.scalar(select(NewsEvent).where(NewsEvent.state == 'pending', NewsEvent.due_at <= datetime.now(UTC), NewsEvent.id == target_id if target_id else True).order_by(NewsEvent.due_at).limit(1))
|
||||
if event is None: return
|
||||
members = (await session.scalars(select(NewsSummary).join(EventMember, EventMember.package_id == NewsSummary.id)
|
||||
.where(EventMember.event_id == event.id))).all()
|
||||
@@ -125,18 +125,21 @@ async def organize():
|
||||
result, usage = await complete(provider, system, {'task': 'event_timeline', 'previous': event.result, 'packages': packages}, EventResult)
|
||||
validate_timeline(result, packages)
|
||||
version.result = result.model_dump(mode='json'); version.usage = usage; version.state = 'done'
|
||||
current_members = (await session.scalars(select(NewsSummary).join(EventMember, EventMember.package_id == NewsSummary.id)
|
||||
.where(EventMember.event_id == event_id).order_by(NewsSummary.id).with_for_update(of=NewsSummary).execution_options(populate_existing=True))).all()
|
||||
current = await session.get(NewsEvent, event_id, populate_existing=True, with_for_update=True)
|
||||
current_members = [p for p in current_members if not p.replacement_ids and p.state != 'withdrawn']
|
||||
unchanged = {(p.id, p.published_revision) for p in current_members} == {(p['id'], p['revision']) for p in packages}
|
||||
unchanged = unchanged and all(p.state == 'done' and not p.replacement_ids for p in current_members)
|
||||
if current.revision == version.revision and unchanged:
|
||||
current.published_revision = version.revision
|
||||
current.result = version.result; current.title = result.title; current.state = 'done'; current.error = None
|
||||
else:
|
||||
version.state = 'superseded'
|
||||
current.state = 'pending'
|
||||
from alphapulse.news_risk.worker import publication_lock
|
||||
async with publication_lock:
|
||||
current_members = (await session.scalars(select(NewsSummary).join(EventMember, EventMember.package_id == NewsSummary.id)
|
||||
.where(EventMember.event_id == event_id).order_by(NewsSummary.id).with_for_update(of=NewsSummary).execution_options(populate_existing=True))).all()
|
||||
current = await session.get(NewsEvent, event_id, populate_existing=True, with_for_update=True)
|
||||
current_members = [p for p in current_members if not p.replacement_ids and p.state != 'withdrawn']
|
||||
unchanged = {(p.id, p.published_revision) for p in current_members} == {(p['id'], p['revision']) for p in packages}
|
||||
unchanged = unchanged and all(p.state == 'done' and not p.replacement_ids for p in current_members)
|
||||
if current.revision == version.revision and unchanged:
|
||||
current.published_revision = version.revision
|
||||
current.result = version.result; current.title = result.title; current.state = 'done'; current.error = None
|
||||
else:
|
||||
version.state = 'superseded'
|
||||
current.state = 'pending'
|
||||
await session.commit()
|
||||
except Exception as error:
|
||||
await session.rollback()
|
||||
version = await session.get(EventVersion, version_id)
|
||||
|
||||
@@ -21,6 +21,7 @@ async def capture(session, settings):
|
||||
from alphapulse.api.risk import config_out
|
||||
value = config_out(settings)
|
||||
value.pop('start_at')
|
||||
value.pop('max_concurrency', None)
|
||||
configurations = {}
|
||||
for stage in ('grouping', 'analysis', 'outlook'):
|
||||
provider, system, rules, fingerprint = await configuration(session, settings, stage)
|
||||
@@ -39,16 +40,17 @@ async def frozen_configuration(session, frozen, stage):
|
||||
return provider, item['system'], item['rules']
|
||||
|
||||
|
||||
async def process_job():
|
||||
async def process_job(item_index=None):
|
||||
from alphapulse.news_risk import worker
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
if not settings or not settings.enabled: return
|
||||
if await session.scalar(select(ArticleAnalysis.id).where(ArticleAnalysis.state == 'pending').limit(1)): return
|
||||
if await session.scalar(select(NewsSummary.id).where(NewsSummary.state.in_(['pending', 'processing'])).limit(1)): return
|
||||
if item_index is None and await session.scalar(select(ArticleAnalysis.id).where(ArticleAnalysis.state == 'pending').limit(1)): return
|
||||
if item_index is None and await session.scalar(select(NewsSummary.id).where(NewsSummary.state.in_(['pending', 'processing'])).limit(1)): return
|
||||
job = await session.scalar(select(ReprocessJob).where(ReprocessJob.state.in_(['pending', 'running'])).order_by(ReprocessJob.id).limit(1).with_for_update())
|
||||
if job is None: return
|
||||
if job.position >= len(job.items):
|
||||
if await session.scalar(select(NewsSummary.id).where(NewsSummary.state.in_(['pending', 'processing'])).limit(1)): return
|
||||
from alphapulse.models.risk import NewsEvent
|
||||
if await session.scalar(select(NewsEvent.id).where(NewsEvent.state.in_(['pending','processing'])).limit(1)): return
|
||||
from alphapulse.models.risk import RiskSnapshot
|
||||
@@ -72,7 +74,11 @@ async def process_job():
|
||||
failures.append({'stage': 'outlook', 'id': snapshot.id, 'error': (snapshot.error or '阅读编排失败')[:1000]})
|
||||
job.failures = [*job.failures, *failures]
|
||||
job.state = 'failed' if job.failures else 'done'; await session.commit(); return
|
||||
item = job.items[job.position]
|
||||
index = job.position if item_index is None else item_index
|
||||
if index < 0: return
|
||||
item = job.items[index]
|
||||
if item.get('state') not in (None, 'pending'): return
|
||||
job.items = [{**entry, 'state': 'processing'} if n == index else entry for n, entry in enumerate(job.items)]
|
||||
job_id = job.id
|
||||
try:
|
||||
for stage in ('grouping', 'analysis', 'outlook'):
|
||||
@@ -89,16 +95,22 @@ async def process_job():
|
||||
session.add(row); await session.flush()
|
||||
row_id = row.id; job_id = job.id; job.state = 'running'; await session.commit()
|
||||
outcome = await worker.analyze(row_id, job.settings)
|
||||
if outcome == 'deferred': return
|
||||
if outcome == 'deferred':
|
||||
job = await session.get(ReprocessJob, job_id, populate_existing=True, with_for_update=True)
|
||||
job.items = [{**entry, 'state': 'pending'} if n == index else entry for n, entry in enumerate(job.items)]
|
||||
await session.commit(); return
|
||||
row = await session.get(ArticleAnalysis, row_id, populate_existing=True)
|
||||
if outcome != 'done' or row.state != 'done' or row.error: raise ValueError(row.error or '原文已变化')
|
||||
terminal = 'done'
|
||||
except Exception as error:
|
||||
terminal = 'failed'
|
||||
await session.rollback()
|
||||
job = await session.get(ReprocessJob, job_id, populate_existing=True)
|
||||
job = await session.get(ReprocessJob, job_id, populate_existing=True, with_for_update=True)
|
||||
job.failures = [*job.failures, {'stage': 'grouping', 'id': item['id'], 'error': str(error)[:1000]}]
|
||||
await session.flush()
|
||||
job = await session.get(ReprocessJob, job_id, populate_existing=True, with_for_update=True)
|
||||
job.position += 1
|
||||
job.items = [{**entry, 'state': terminal} if n == index else entry for n, entry in enumerate(job.items)]
|
||||
job.position = sum(entry.get('state') in ('done', 'failed') for entry in job.items)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@@ -111,7 +123,7 @@ async def start_job(session, start, end):
|
||||
if settings is None: raise ValueError('请先配置新闻AI')
|
||||
counts, items = await preview(session, start, end)
|
||||
if not items: raise ValueError('所选范围没有新闻')
|
||||
job = ReprocessJob(state='pending', settings=await capture(session, settings), items=items, position=0, failures=[])
|
||||
job = ReprocessJob(state='pending', settings=await capture(session, settings), items=[{**i, 'state': 'pending'} for i in items], position=0, failures=[])
|
||||
session.add(job); await session.flush()
|
||||
job.settings = {**job.settings, 'job_id': job.id}
|
||||
await session.commit()
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Bounded, completion-driven dispatch for every background news AI stage."""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from sqlalchemy import select, text, func
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.models.risk import RiskSettings, NewsSummary, NewsEvent, ArticleAnalysis, ReprocessJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
GATE = 20261102
|
||||
|
||||
|
||||
class Pool:
|
||||
def __init__(self):
|
||||
self.tasks = {}
|
||||
|
||||
def start(self, key, action):
|
||||
self.tasks[asyncio.create_task(action())] = key
|
||||
|
||||
async def reap(self, timeout=1):
|
||||
if not self.tasks:
|
||||
await asyncio.sleep(timeout)
|
||||
return
|
||||
done, _ = await asyncio.wait(self.tasks, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
|
||||
for task in done:
|
||||
del self.tasks[task]
|
||||
try:
|
||||
task.result()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception('新闻AI任务异常')
|
||||
|
||||
async def close(self):
|
||||
for task in self.tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*self.tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def candidate(active, cursor, allow_outlook=True):
|
||||
from alphapulse.news_risk import worker, events, reprocess
|
||||
from alphapulse.models import NewsItem
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
if not settings or not settings.enabled or not settings.start_at:
|
||||
return None
|
||||
await worker.enqueue(session, settings)
|
||||
history_news = []
|
||||
for kind, identity in active:
|
||||
if kind == 'history' and identity[1] >= 0:
|
||||
job = await session.get(ReprocessJob, identity[0])
|
||||
history_news.append(job.items[identity[1]]['id'])
|
||||
stages = ('grouping', 'analysis', 'events', 'outlook')
|
||||
for historical in (False, True):
|
||||
for offset in range(len(stages)):
|
||||
stage = stages[(cursor + offset) % len(stages)]
|
||||
if historical and stage in ('grouping', 'outlook'): continue
|
||||
busy = [key[1] for key in active if key[0] == stage]
|
||||
if stage == 'outlook':
|
||||
if not busy and allow_outlook:
|
||||
return (stage, 0), worker.outlook, (cursor + offset + 1) % 4
|
||||
continue
|
||||
model = {'grouping': ArticleAnalysis, 'analysis': NewsSummary, 'events': NewsEvent}[stage]
|
||||
query = select(model.id).where(model.state == 'pending', model.id.not_in(busy))
|
||||
if stage == 'grouping':
|
||||
query = query.join(NewsItem).where(model.content_hash == NewsItem.content_hash, NewsItem.id.not_in(history_news)).order_by(NewsItem.published_at.desc())
|
||||
else:
|
||||
realtime = model.review_config.is_(None) | (func.jsonb_typeof(model.review_config) == 'null')
|
||||
query = query.where(~realtime if historical else realtime, model.due_at <= datetime.now(UTC)).order_by(model.due_at, model.id)
|
||||
if stage == 'events':
|
||||
from alphapulse.models.risk import EventMember
|
||||
blocked = select(EventMember.event_id).join(NewsSummary, NewsSummary.id == EventMember.package_id).where(
|
||||
NewsSummary.replacement_ids == [], NewsSummary.state != 'withdrawn',
|
||||
(NewsSummary.state != 'done') | NewsSummary.result.is_(None))
|
||||
query = query.where(model.id.not_in(blocked))
|
||||
identifier = await session.scalar(query.limit(1))
|
||||
if identifier is not None:
|
||||
action = {'grouping': worker.analyze, 'analysis': worker.summarize, 'events': events.organize}[stage]
|
||||
return (stage, identifier), lambda action=action, identifier=identifier: action(identifier), (cursor + offset + 1) % 4
|
||||
job = await session.scalar(select(ReprocessJob).where(ReprocessJob.state.in_(['pending', 'running'])).order_by(ReprocessJob.id).limit(1))
|
||||
if job:
|
||||
for index, item in enumerate(job.items):
|
||||
key = ('history', (job.id, index))
|
||||
if item.get('state', 'pending') == 'pending' and key not in active:
|
||||
# Real-time and historical grouping share the article identity.
|
||||
row_id = await session.scalar(select(ArticleAnalysis.id).where(ArticleAnalysis.news_id == item['id'], ArticleAnalysis.content_hash == item['hash']))
|
||||
if row_id is not None and ('grouping', row_id) in active:
|
||||
continue
|
||||
return key, lambda index=index: reprocess.process_job(index), cursor
|
||||
if not any(k[0] == 'history' for k in active):
|
||||
return ('history', (job.id, -1)), lambda: reprocess.process_job(-1), cursor
|
||||
return None
|
||||
|
||||
|
||||
async def run(lock):
|
||||
pool = Pool()
|
||||
cursor = 0
|
||||
gate = False
|
||||
next_outlook = 0.0
|
||||
# Interrupted historical items are reclaimed independently of completion order.
|
||||
async with SessionLocal() as session:
|
||||
jobs = (await session.scalars(select(ReprocessJob).where(ReprocessJob.state.in_(['pending', 'running'])))).all()
|
||||
for job in jobs:
|
||||
for item in job.items:
|
||||
if item['state'] == 'processing':
|
||||
from sqlalchemy import update
|
||||
await session.execute(update(ArticleAnalysis).where(ArticleAnalysis.news_id == item['id'],
|
||||
ArticleAnalysis.content_hash == item['hash'], ArticleAnalysis.state == 'pending').values(state='reprocess'))
|
||||
job.items = [{**i, 'state': 'pending' if i.get('state') == 'processing' else i['state']} for i in job.items]
|
||||
await session.commit()
|
||||
try:
|
||||
while True:
|
||||
if not pool.tasks and gate:
|
||||
await lock.execute(text('SELECT pg_advisory_unlock_shared(:gate, 1)'), {'gate': GATE})
|
||||
await lock.commit(); gate = False
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
limit = settings.max_concurrency if settings else 5
|
||||
enabled = bool(settings and settings.enabled)
|
||||
while enabled and len(pool.tasks) < limit:
|
||||
if not gate:
|
||||
gate = bool(await lock.scalar(text('SELECT pg_try_advisory_lock_shared(:gate, 1)'), {'gate': GATE}))
|
||||
await lock.commit()
|
||||
if not gate:
|
||||
break
|
||||
picked = await candidate(set(pool.tasks.values()), cursor, time.monotonic() >= next_outlook)
|
||||
if picked is None:
|
||||
break
|
||||
key, action, cursor = picked
|
||||
pool.start(key, action)
|
||||
if key[0] == 'outlook': next_outlook = time.monotonic() + 1
|
||||
# Refresh operational settings for each admission, including pause.
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
limit, enabled = settings.max_concurrency, settings.enabled
|
||||
now = datetime.now(UTC).isoformat()
|
||||
counts = {stage: sum(k[0] == stage for k in pool.tasks.values()) for stage in ('grouping', 'analysis', 'events', 'outlook', 'history')}
|
||||
try:
|
||||
await redis.set('ap:risk:concurrency', json.dumps({'running': len(pool.tasks), 'stages': counts, 'updated_at': now}), ex=15)
|
||||
for stage in ('', 'analysis:', 'events:', 'outlook:', 'reprocess:'):
|
||||
await redis.set('ap:risk:' + stage + 'heartbeat', now, ex=15)
|
||||
except Exception:
|
||||
logger.exception('新闻AI运行状态暂时无法更新')
|
||||
await pool.reap()
|
||||
finally:
|
||||
await pool.close()
|
||||
if gate:
|
||||
await lock.execute(text('SELECT pg_advisory_unlock_shared(:gate, 1)'), {'gate': GATE})
|
||||
await lock.commit()
|
||||
@@ -1,11 +1,10 @@
|
||||
"""Independent, durable grouping and event analysis loops."""
|
||||
"""Durable news AI stages dispatched through the shared concurrency pool."""
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from sqlalchemy import select, update, func, exists, text, case
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from alphapulse.core.db import SessionLocal, engine
|
||||
from alphapulse.core.redis import redis
|
||||
from alphapulse.models import NewsItem
|
||||
from alphapulse.models.risk import RiskSettings, RiskSnapshot, ArticleAnalysis, NewsSummary, SummaryArticle, SummaryVersion, PackageEvidence, GroupingVersion
|
||||
from alphapulse.news_risk.service import complete
|
||||
@@ -14,8 +13,8 @@ from alphapulse.news_risk.research import Research
|
||||
from alphapulse.news_risk import briefing
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
analysis_lock = asyncio.Lock()
|
||||
grouping_lock = asyncio.Lock()
|
||||
publication_lock = asyncio.Lock()
|
||||
|
||||
|
||||
|
||||
def eligible(start):
|
||||
@@ -44,8 +43,7 @@ async def enqueue(session, settings):
|
||||
|
||||
|
||||
async def analyze(target_id=None, frozen=None):
|
||||
async with grouping_lock:
|
||||
return await _analyze(target_id, frozen)
|
||||
return await _analyze(target_id, frozen)
|
||||
|
||||
|
||||
async def _analyze(target_id=None, frozen=None):
|
||||
@@ -64,6 +62,7 @@ async def _analyze(target_id=None, frozen=None):
|
||||
provider, system, rules = await frozen_configuration(session, frozen, 'grouping')
|
||||
else:
|
||||
provider, system, rules, _ = await configuration(session, settings, 'grouping')
|
||||
catalog = await session.scalar(select(func.max(NewsSummary.id)))
|
||||
tag_score = sum((case((NewsSummary.tags.contains([tag.strip().casefold()]), 1), else_=0) for tag in row.input.get('source_tags', [])[:8] if isinstance(tag, str)), 0)
|
||||
candidates = (await session.scalars(select(NewsSummary).where(NewsSummary.replacement_ids == [], NewsSummary.state != 'withdrawn').order_by(
|
||||
(func.greatest(func.similarity(NewsSummary.title, row.input['title']), func.similarity(NewsSummary.fact, row.input['title'])) + tag_score).desc(), NewsSummary.updated_at.desc()).limit(80))).all()
|
||||
@@ -88,53 +87,56 @@ async def _analyze(target_id=None, frozen=None):
|
||||
if attempt:
|
||||
raise
|
||||
payload = pack_candidates(provider, system, {**payload, 'correction': str(error)}, payload['candidates'])
|
||||
# Check the article again before changing event membership.
|
||||
current = await session.get(NewsItem, row.news_id, populate_existing=True)
|
||||
if current.content_hash != row.content_hash:
|
||||
row.state = 'superseded'; await session.commit(); return 'superseded'
|
||||
now = datetime.now(UTC)
|
||||
old_ids = select(SummaryArticle.summary_id).join(ArticleAnalysis).where(ArticleAnalysis.news_id == row.news_id)
|
||||
old_events = (await session.scalars(select(NewsSummary).where(NewsSummary.id.in_(old_ids))
|
||||
.with_for_update().execution_options(populate_existing=True))).all()
|
||||
touched = {e.id: e for e in old_events}
|
||||
previous_ids = set(touched)
|
||||
await session.execute(update(SummaryArticle).where(SummaryArticle.analysis_id == row.id).values(active=False))
|
||||
groups = {}
|
||||
changed = set()
|
||||
for match in result.packages:
|
||||
event = await session.get(NewsSummary, match.package_id, with_for_update=True, populate_existing=True) if match.package_id else groups.get(match.group_key)
|
||||
if event is not None and (event.replacement_ids or event.state == 'withdrawn'):
|
||||
raise ValueError('候选包已被纠错或撤回,请重试初筛')
|
||||
progress = datetime.fromisoformat(row.input['progress_at'])
|
||||
published = datetime.fromisoformat(row.input['published_at'])
|
||||
if event is None:
|
||||
event = NewsSummary(title=match.title, category=match.category, entities=match.entities,
|
||||
published_at=published, updated_at=progress, changed_at=now, due_at=now,
|
||||
revision=0, published_revision=0, state='pending')
|
||||
session.add(event); await session.flush(); groups[match.group_key] = event
|
||||
event.tags = sorted(set(event.tags or []) | {t.strip().casefold() for t in match.tags if t.strip()})
|
||||
if match.change_kind != 'repost' or not event.fact:
|
||||
event.content_kind = match.content_kind; event.fact = match.fact
|
||||
event.published_at = min(event.published_at, published)
|
||||
if match.change_kind != 'repost' or row.input['revision_no'] > 1:
|
||||
event.updated_at = max(event.updated_at, progress)
|
||||
link = await session.scalar(select(SummaryArticle).where(SummaryArticle.summary_id == event.id, SummaryArticle.analysis_id == row.id))
|
||||
if link is None:
|
||||
link = SummaryArticle(summary_id=event.id, analysis_id=row.id); session.add(link)
|
||||
link.active = True; link.unit = match.model_dump()
|
||||
if match.change_kind != 'repost' or event.result is None or row.input['revision_no'] > 1:
|
||||
changed.add(event.id)
|
||||
touched[event.id] = event
|
||||
if target_id or row.input['revision_no'] > 1:
|
||||
changed.update(previous_ids)
|
||||
for event in touched.values():
|
||||
if event.id in changed:
|
||||
event.review_config = frozen
|
||||
schedule(event, now, any(m.urgent for m in result.packages))
|
||||
session.add(GroupingVersion(analysis_id=row.id, input=row.input, result=result.model_dump(mode='json'), rules=rules, state='done'))
|
||||
row.result = result.model_dump(mode='json'); row.usage = {'attempts': usages}; row.state = 'done'
|
||||
row.error = None; row.finished_at = now
|
||||
await session.commit()
|
||||
async with publication_lock:
|
||||
# Check the article again before changing event membership.
|
||||
current = await session.get(NewsItem, row.news_id, populate_existing=True)
|
||||
if current.content_hash != row.content_hash:
|
||||
row.state = 'superseded'; await session.commit(); return 'superseded'
|
||||
if any(m.package_id is None for m in result.packages) and catalog != await session.scalar(select(func.max(NewsSummary.id))):
|
||||
row.state = 'reprocess' if frozen else 'pending'; await session.commit(); return 'deferred'
|
||||
now = datetime.now(UTC)
|
||||
old_ids = select(SummaryArticle.summary_id).join(ArticleAnalysis).where(ArticleAnalysis.news_id == row.news_id)
|
||||
old_events = (await session.scalars(select(NewsSummary).where(NewsSummary.id.in_(old_ids))
|
||||
.order_by(NewsSummary.id).with_for_update().execution_options(populate_existing=True))).all()
|
||||
touched = {e.id: e for e in old_events}
|
||||
previous_ids = set(touched)
|
||||
await session.execute(update(SummaryArticle).where(SummaryArticle.analysis_id == row.id).values(active=False))
|
||||
groups = {}
|
||||
changed = set()
|
||||
for match in result.packages:
|
||||
event = await session.get(NewsSummary, match.package_id, with_for_update=True, populate_existing=True) if match.package_id else groups.get(match.group_key)
|
||||
if event is not None and (event.replacement_ids or event.state == 'withdrawn'):
|
||||
raise ValueError('候选包已被纠错或撤回,请重试初筛')
|
||||
progress = datetime.fromisoformat(row.input['progress_at'])
|
||||
published = datetime.fromisoformat(row.input['published_at'])
|
||||
if event is None:
|
||||
event = NewsSummary(title=match.title, category=match.category, entities=match.entities,
|
||||
published_at=published, updated_at=progress, changed_at=now, due_at=now,
|
||||
revision=0, published_revision=0, state='pending')
|
||||
session.add(event); await session.flush(); groups[match.group_key] = event
|
||||
event.tags = sorted(set(event.tags or []) | {t.strip().casefold() for t in match.tags if t.strip()})
|
||||
if match.change_kind != 'repost' or not event.fact:
|
||||
event.content_kind = match.content_kind; event.fact = match.fact
|
||||
event.published_at = min(event.published_at, published)
|
||||
if match.change_kind != 'repost' or row.input['revision_no'] > 1:
|
||||
event.updated_at = max(event.updated_at, progress)
|
||||
link = await session.scalar(select(SummaryArticle).where(SummaryArticle.summary_id == event.id, SummaryArticle.analysis_id == row.id))
|
||||
if link is None:
|
||||
link = SummaryArticle(summary_id=event.id, analysis_id=row.id); session.add(link)
|
||||
link.active = True; link.unit = match.model_dump()
|
||||
if match.change_kind != 'repost' or event.result is None or row.input['revision_no'] > 1:
|
||||
changed.add(event.id)
|
||||
touched[event.id] = event
|
||||
if target_id or row.input['revision_no'] > 1:
|
||||
changed.update(previous_ids)
|
||||
for event in touched.values():
|
||||
if event.id in changed:
|
||||
event.review_config = frozen
|
||||
schedule(event, now, any(m.urgent for m in result.packages))
|
||||
session.add(GroupingVersion(analysis_id=row.id, input=row.input, result=result.model_dump(mode='json'), rules=rules, state='done'))
|
||||
row.result = result.model_dump(mode='json'); row.usage = {'attempts': usages}; row.state = 'done'
|
||||
row.error = None; row.finished_at = now
|
||||
await session.commit()
|
||||
return 'done'
|
||||
except Exception as error:
|
||||
await session.rollback()
|
||||
@@ -146,13 +148,14 @@ async def _analyze(target_id=None, frozen=None):
|
||||
return 'failed'
|
||||
|
||||
|
||||
async def summarize():
|
||||
async with analysis_lock, SessionLocal() as session:
|
||||
async def summarize(target_id=None):
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
if not settings or not settings.enabled:
|
||||
return
|
||||
event = await session.scalar(select(NewsSummary).where(NewsSummary.state == 'pending',
|
||||
NewsSummary.due_at <= datetime.now(UTC)).order_by(NewsSummary.due_at, NewsSummary.id).limit(1).with_for_update())
|
||||
NewsSummary.due_at <= datetime.now(UTC),
|
||||
NewsSummary.id == target_id if target_id else True).order_by(NewsSummary.due_at, NewsSummary.id).limit(1).with_for_update())
|
||||
if event is None:
|
||||
return
|
||||
config_error = None
|
||||
@@ -194,45 +197,51 @@ async def summarize():
|
||||
research = Research()
|
||||
evidence_rows = (await session.scalars(select(PackageEvidence).where(PackageEvidence.version_id == identifier).order_by(PackageEvidence.id))).all()
|
||||
research.restore([r.data for r in evidence_rows])
|
||||
await session.commit()
|
||||
try:
|
||||
if config_error: raise ValueError(config_error)
|
||||
result, usage = await summarize_articles(provider, system, articles, package, research)
|
||||
event = await session.get(NewsSummary, event_id, populate_existing=True, with_for_update=True)
|
||||
if result.correction is not None:
|
||||
from alphapulse.news_risk.corrections import apply_correction
|
||||
from alphapulse.news_risk.contracts import Correction
|
||||
await apply_correction(session, Correction.model_validate(result.correction), event_id, research.read_package_ids, rules)
|
||||
version.result = result.model_dump(mode='json'); version.usage = usage; version.state = 'corrected'; version.finished_at = datetime.now(UTC)
|
||||
async with publication_lock:
|
||||
event = await session.get(NewsSummary, event_id, populate_existing=True, with_for_update=True)
|
||||
if event.replacement_ids or event.state == 'withdrawn':
|
||||
version.state = 'superseded'; await session.commit(); return
|
||||
if result.correction is not None and event.revision != revision:
|
||||
version.state = 'superseded'; await session.commit(); return
|
||||
if result.correction is not None:
|
||||
from alphapulse.news_risk.corrections import apply_correction
|
||||
from alphapulse.news_risk.contracts import Correction
|
||||
await apply_correction(session, Correction.model_validate(result.correction), event_id, research.read_package_ids, rules)
|
||||
version.result = result.model_dump(mode='json'); version.usage = usage; version.state = 'corrected'; version.finished_at = datetime.now(UTC)
|
||||
for record in research.records:
|
||||
session.add(PackageEvidence(summary_id=event_id, version_id=identifier, kind=record['tool'], reference=str(record['arguments']), data=record))
|
||||
await session.commit(); return
|
||||
version.result = result.model_dump(mode='json'); version.usage = usage; version.state = 'done'; version.finished_at = datetime.now(UTC)
|
||||
if event.revision == revision and revision > event.published_revision:
|
||||
event.result = result.model_dump(mode='json')
|
||||
event.stars = result.risk_stars; event.peak_stars = max(event.peak_stars, result.risk_stars)
|
||||
event.importance_stars = result.importance_stars
|
||||
event.published_revision = revision; event.analyzed_at = version.finished_at; event.error = None
|
||||
event.state = 'done' if event.revision == revision else 'pending'
|
||||
if event.state == 'done': event.dirty_since = None
|
||||
for record in research.records:
|
||||
session.add(PackageEvidence(summary_id=event_id, version_id=identifier, kind=record['tool'], reference=str(record['arguments']), data=record))
|
||||
await session.commit(); return
|
||||
version.result = result.model_dump(mode='json'); version.usage = usage; version.state = 'done'; version.finished_at = datetime.now(UTC)
|
||||
if event.revision == revision and revision > event.published_revision:
|
||||
event.result = result.model_dump(mode='json')
|
||||
event.stars = result.risk_stars; event.peak_stars = max(event.peak_stars, result.risk_stars)
|
||||
event.importance_stars = result.importance_stars
|
||||
event.published_revision = revision; event.analyzed_at = version.finished_at; event.error = None
|
||||
event.state = 'done' if event.revision == revision else 'pending'
|
||||
if event.state == 'done': event.dirty_since = None
|
||||
for record in research.records:
|
||||
session.add(PackageEvidence(summary_id=event_id, version_id=identifier, kind=record['tool'], reference=str(record['arguments']), data=record))
|
||||
from alphapulse.news_risk.events import associate
|
||||
if event.revision == revision:
|
||||
await associate(session, event, result, research.read_package_ids, research.read_event_ids)
|
||||
await session.commit()
|
||||
from alphapulse.news_risk.events import associate
|
||||
if event.revision == revision:
|
||||
await associate(session, event, result, research.read_package_ids, research.read_event_ids)
|
||||
await session.commit()
|
||||
except Exception as error:
|
||||
await session.rollback()
|
||||
for record in research.records:
|
||||
session.add(PackageEvidence(summary_id=event_id, version_id=identifier, kind=record['tool'], reference=str(record['arguments']), data=record))
|
||||
await session.execute(update(SummaryVersion).where(SummaryVersion.id == identifier).values(
|
||||
state='failed', error=str(error)[:2000], finished_at=datetime.now(UTC)))
|
||||
await session.execute(update(NewsSummary).where(NewsSummary.id == event_id).values(
|
||||
await session.execute(update(NewsSummary).where(NewsSummary.id == event_id, NewsSummary.replacement_ids == [], NewsSummary.state != 'withdrawn').values(
|
||||
state=case((NewsSummary.revision > revision, 'pending'), else_='failed'), error=str(error)[:2000]))
|
||||
await session.commit(); logger.exception('事件总结失败')
|
||||
|
||||
|
||||
async def outlook():
|
||||
async with analysis_lock, SessionLocal() as session:
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(RiskSettings, 1)
|
||||
if not settings or not settings.enabled: return
|
||||
now = datetime.now(UTC)
|
||||
@@ -293,16 +302,6 @@ async def outlook():
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def loop(action, heartbeat):
|
||||
while True:
|
||||
try:
|
||||
await redis.set(heartbeat, datetime.now(UTC).isoformat(), ex=1800)
|
||||
await action()
|
||||
except Exception:
|
||||
logger.exception('新闻AI处理异常')
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
async def main():
|
||||
async with engine.connect() as lock:
|
||||
if not await lock.scalar(text('SELECT pg_try_advisory_lock(20261030, 1)')):
|
||||
@@ -319,10 +318,8 @@ async def main():
|
||||
await session.execute(update(NewsEvent).where(NewsEvent.state == 'processing').values(state='pending'))
|
||||
await session.execute(update(EventVersion).where(EventVersion.state == 'processing').values(state='failed', error='进程中断'))
|
||||
await session.commit()
|
||||
from alphapulse.news_risk.events import organize
|
||||
from alphapulse.news_risk.reprocess import process_job
|
||||
await asyncio.gather(loop(organize, 'ap:risk:events:heartbeat'), loop(process_job, 'ap:risk:reprocess:heartbeat'), loop(analyze, 'ap:risk:heartbeat'), loop(summarize, 'ap:risk:analysis:heartbeat'),
|
||||
loop(outlook, 'ap:risk:outlook:heartbeat'))
|
||||
from alphapulse.news_risk.scheduler import run
|
||||
await run(lock)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user