feat(news): restrict research to local sources and process frozen batches
This commit is contained in:
eezd committed
1 parent
d79a10c2e6
commit
89da4df785
28 files changed
+360
-409
No files matched your search
@@ -2,19 +2,18 @@
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from sqlalchemy import select, text, update, exists
|
||||
from sqlalchemy import select, text, update
|
||||
from alphapulse.core.db import SessionLocal, engine
|
||||
from alphapulse.models import NewsItem
|
||||
from alphapulse.news_research.models import (
|
||||
ResearchSettings,
|
||||
Package,
|
||||
Event,
|
||||
Attempt,
|
||||
Version,
|
||||
Membership,
|
||||
Batch, BatchEvent,
|
||||
)
|
||||
from alphapulse.news_research.service import enroll, process_package, process_event
|
||||
from alphapulse.news_research.service import process_package, process_event
|
||||
from alphapulse.news_research import batches
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,6 +43,7 @@ async def run():
|
||||
.where(Version.state == "processing")
|
||||
.values(state="failed", error="进程中断，待重试")
|
||||
)
|
||||
await session.execute(update(BatchEvent).where(BatchEvent.state == 'processing').values(state='pending'))
|
||||
await session.commit()
|
||||
tasks = {}
|
||||
try:
|
||||
@@ -58,91 +58,20 @@ async def run():
|
||||
async with SessionLocal() as session:
|
||||
settings = await session.get(ResearchSettings, 1)
|
||||
if settings and settings.enabled:
|
||||
ids = list(
|
||||
await session.scalars(
|
||||
select(NewsItem.id)
|
||||
.outerjoin(Package, Package.news_id == NewsItem.id)
|
||||
.where(
|
||||
NewsItem.published_at >= settings.start_at,
|
||||
NewsItem.published_at <= datetime.now(UTC),
|
||||
(Package.id.is_(None))
|
||||
| (Package.content_hash != NewsItem.content_hash),
|
||||
)
|
||||
.order_by(NewsItem.published_at.desc())
|
||||
.limit(100)
|
||||
)
|
||||
)
|
||||
await enroll(session, ids)
|
||||
active = set(tasks.values())
|
||||
assigning = any(key[0] == "assign" for key in active)
|
||||
slots = settings.config.get("concurrency", 3) - len(tasks)
|
||||
packages = list(
|
||||
await session.scalars(
|
||||
select(Package)
|
||||
.where(Package.state == "index_pending")
|
||||
.order_by(Package.id.desc())
|
||||
.limit(max(0, slots))
|
||||
)
|
||||
)
|
||||
if not assigning and slots > 0:
|
||||
assignment = await session.scalar(
|
||||
select(Package)
|
||||
.where(Package.state == "assign_pending")
|
||||
.order_by(Package.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if assignment is not None:
|
||||
packages.insert(0, assignment)
|
||||
events = list(
|
||||
await session.scalars(
|
||||
select(Event)
|
||||
.where(
|
||||
Event.state == "pending",
|
||||
Event.due_at <= datetime.now(UTC),
|
||||
~exists(
|
||||
select(Membership.event_id)
|
||||
.join(
|
||||
Package, Package.id == Membership.package_id
|
||||
)
|
||||
.join(NewsItem, NewsItem.id == Package.news_id)
|
||||
.where(
|
||||
Membership.event_id == Event.id,
|
||||
(Package.state != "done")
|
||||
| (
|
||||
Package.content_hash
|
||||
!= NewsItem.content_hash
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
.order_by(Event.due_at)
|
||||
.limit(max(0, slots))
|
||||
)
|
||||
)
|
||||
# Interleave ready event work with new indexing to avoid starvation.
|
||||
actions = []
|
||||
for i in range(max(len(packages), len(events))):
|
||||
if i < len(events):
|
||||
actions.append((("event", events[i].id), process_event))
|
||||
if i < len(packages):
|
||||
actions.append(
|
||||
(
|
||||
(
|
||||
"assign"
|
||||
if packages[i].state == "assign_pending"
|
||||
else "index",
|
||||
packages[i].id,
|
||||
),
|
||||
process_package,
|
||||
)
|
||||
)
|
||||
for key, action in actions:
|
||||
if len(tasks) >= settings.config.get("concurrency", 3):
|
||||
break
|
||||
if key not in active:
|
||||
tasks[
|
||||
asyncio.create_task(action(key[1], settings.config))
|
||||
] = key
|
||||
batch = await session.scalar(select(Batch).where(
|
||||
Batch.phase.in_(['assigning', 'researching'])).order_by(Batch.id).limit(1))
|
||||
if batch is None:
|
||||
batch = await batches.freeze(session, settings)
|
||||
settings = await session.get(ResearchSettings, 1, populate_existing=True, with_for_update=True)
|
||||
if batch is not None and settings.enabled:
|
||||
active = set(tasks.values())
|
||||
actions = await batches.actions(session, batch, active,
|
||||
max(0, settings.config.get('concurrency', 3) - len(tasks)))
|
||||
await session.commit()
|
||||
for stage, identifier in actions:
|
||||
call = (process_event(identifier, settings.config, force=True)
|
||||
if stage == 'event' else process_package(identifier, settings.config))
|
||||
tasks[asyncio.create_task(call)] = (stage, identifier)
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
Reference in new issue
Block a user