169 lines
8.4 KiB
Python
169 lines
8.4 KiB
Python
"""Exercise durable batch barriers in the disposable CI database without model calls."""
|
|
import asyncio
|
|
import os
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import SimpleNamespace
|
|
from urllib.parse import urlparse
|
|
from sqlalchemy import select, delete, insert, func, text
|
|
from alphapulse.core.db import SessionLocal, engine
|
|
from alphapulse.models import NewsItem
|
|
from alphapulse.news_research.models import Batch, BatchPackage, BatchEvent, Package, Event, Membership, Attempt
|
|
from alphapulse.news_research import batches, service
|
|
from alphapulse.news_research.api import retry_event
|
|
|
|
|
|
async def main():
|
|
url = urlparse(os.environ['DATABASE_URL'])
|
|
assert (url.username, url.password, url.path) == ('ci', 'ci-only', '/ci')
|
|
now = datetime.now(UTC)
|
|
settings = SimpleNamespace(start_at=now - timedelta(days=1))
|
|
async with SessionLocal() as s:
|
|
assert await s.scalar(select(Batch.id).limit(1)) is None
|
|
def news(key):
|
|
return NewsItem(source_code='wscn_news', external_id='batch-'+key,
|
|
title=key, summary='', content_text=key, content_hash='a'*64,
|
|
url='https://example.invalid/'+key, published_at=now)
|
|
originals = [news(str(i)) for i in range(3)]
|
|
s.add_all(originals)
|
|
await s.commit()
|
|
batch = await batches.freeze(s, settings)
|
|
identifier = batch.id
|
|
assert batch.phase == 'assigning'
|
|
packages = list(await s.scalars(select(Package).order_by(Package.id)))
|
|
assert len(packages) == 3
|
|
later = news('later')
|
|
s.add(later)
|
|
await s.commit()
|
|
actions = await batches.actions(s, batch, set(), 8)
|
|
assert len(actions) == 3 and all(stage == 'index' for stage, _ in actions)
|
|
for p in packages:
|
|
p.state = 'assign_pending'
|
|
await s.commit()
|
|
actions = await batches.actions(s, batch, set(), 8)
|
|
assert len(actions) == 1 and actions[0][0] == 'assign'
|
|
assert not await batches.actions(s, batch, {actions[0]}, 8)
|
|
event = Event(title='One shared event', tags=[], revision=3, state='pending', due_at=now)
|
|
s.add(event)
|
|
await s.flush()
|
|
for p in packages[:2]:
|
|
p.state = 'done'
|
|
s.add(Membership(event_id=event.id, package_id=p.id, evidence='original', reason='same event', change_kind='new_fact'))
|
|
s.add(Attempt(package_id=packages[0].id, content_hash=packages[0].content_hash, stage='assign', state='done', input={}, rules={'membership_targets': {str(event.id): {'urgent': True}}}))
|
|
packages[2].state = 'index_failed'
|
|
await s.commit()
|
|
assert not await batches.actions(s, batch, {('assign', packages[0].id)}, 8)
|
|
assert batch.phase == 'assigning'
|
|
assert not await batches.actions(s, batch, set(), 8)
|
|
await s.commit()
|
|
assert batch.phase == 'researching'
|
|
assert (await s.get(BatchEvent, (batch.id, event.id))).state == 'urgent_pending'
|
|
# Reopen the session to prove the admission barrier is durable.
|
|
async with SessionLocal() as s:
|
|
batch = await s.get(Batch, identifier)
|
|
actions = await batches.actions(s, batch, set(), 8)
|
|
assert actions == [('event', event.id)]
|
|
await s.commit()
|
|
assert not await batches.actions(s, batch, set(actions), 8)
|
|
e = await s.get(Event, event.id)
|
|
e.state = 'done'
|
|
await s.commit()
|
|
assert not await batches.actions(s, batch, set(), 8)
|
|
await s.commit()
|
|
assert batch.phase == 'partial'
|
|
progress = await batches.progress(s)
|
|
assert progress['packages_failed'] == 1 and progress['events_done'] == 1
|
|
next_batch = await batches.freeze(s, settings)
|
|
items = list(await s.scalars(select(BatchPackage).where(BatchPackage.batch_id == next_batch.id)))
|
|
assert len(items) == 1
|
|
p = await s.get(Package, items[0].package_id)
|
|
assert p.news_id == later.id
|
|
original = await s.get(NewsItem, later.id)
|
|
original.content_hash = 'b'*64
|
|
await s.commit()
|
|
assert not await batches.actions(s, next_batch, set(), 8)
|
|
await s.commit()
|
|
assert items[0].state == 'deferred'
|
|
# Leave no fixtures for other isolated acceptance scripts.
|
|
for model in (BatchEvent, BatchPackage, Batch, Membership, Attempt, Package, Event, NewsItem):
|
|
await s.execute(delete(model))
|
|
await s.commit()
|
|
await regressions(now, settings)
|
|
await engine.dispose()
|
|
print('Frozen batches passed: assignment barrier, single research, failures, new arrivals, restart and source revisions.')
|
|
|
|
|
|
async def regressions(now, settings):
|
|
async def clear(session):
|
|
# This script runs first in an empty, disposable CI database. TRUNCATE avoids
|
|
# quadratic foreign-key checks when removing the 33,000-row stress fixture.
|
|
await session.execute(text('TRUNCATE news_research_batches, news_research_packages, news_research_events, news_items CASCADE'))
|
|
await session.commit()
|
|
|
|
async with SessionLocal() as s:
|
|
event = Event(title='Deferred retry', tags=[], revision=7, state='pending', due_at=now)
|
|
previous = Batch(phase='partial', cutoff_at=now)
|
|
s.add_all([event, previous])
|
|
await s.flush()
|
|
s.add(BatchEvent(batch_id=previous.id, event_id=event.id, revision=7, state='deferred'))
|
|
await s.commit()
|
|
assert await batches.freeze(s, settings) is None
|
|
await retry_event(event.id, s)
|
|
assert event.revision == 8
|
|
batch = await batches.freeze(s, settings)
|
|
assert batch is not None
|
|
await batches.actions(s, batch, set(), 8)
|
|
await s.commit()
|
|
assert await batches.actions(s, batch, set(), 8) == [('event', event.id)]
|
|
# Failed retries retain their revision so cached evidence can still be reused.
|
|
event.state = 'failed'
|
|
await s.commit()
|
|
await retry_event(event.id, s)
|
|
assert event.revision == 8 and event.state == 'pending'
|
|
await clear(s)
|
|
|
|
def original(key, published_at):
|
|
return dict(source_code='wscn_news', external_id=key, title=key, summary='',
|
|
content_text='Original', content_hash='a'*64, url='https://example.invalid/'+key,
|
|
published_at=published_at)
|
|
news = NewsItem(**original('manual-history', now-timedelta(days=365)))
|
|
excluded = NewsItem(**original('unselected-history', now-timedelta(days=365)))
|
|
s.add_all([news, excluded])
|
|
await s.commit()
|
|
await service.enroll(s, [news.id])
|
|
batch = await batches.freeze(s, settings)
|
|
package = await s.scalar(select(Package).where(Package.news_id == news.id))
|
|
news.content_hash = 'b'*64
|
|
await s.commit()
|
|
await batches.actions(s, batch, set(), 8)
|
|
await s.commit()
|
|
await batches.actions(s, batch, set(), 8)
|
|
await s.commit()
|
|
assert batch.phase == 'partial'
|
|
batch = await batches.freeze(s, settings)
|
|
assert package.content_hash == news.content_hash and package.state == 'index_pending'
|
|
assert await batches.actions(s, batch, set(), 8) == [('index', package.id)]
|
|
assert await s.scalar(select(func.count()).select_from(Package)) == 1
|
|
# A previously completed manual original also refreshes outside the automatic boundary.
|
|
package.state = 'done'
|
|
news.content_hash = 'c'*64
|
|
await s.commit()
|
|
await batches.freeze(s, settings)
|
|
assert package.content_hash == news.content_hash and package.state == 'index_pending'
|
|
await clear(s)
|
|
|
|
count = 33000
|
|
await s.execute(insert(NewsItem), [original('bulk-'+str(i), now) for i in range(count)])
|
|
await s.commit()
|
|
batch = await batches.freeze(s, settings)
|
|
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == batch.id)) == count
|
|
assert await s.scalar(select(func.count()).select_from(Package)) == count
|
|
# Exercise the existing-pending-ID and membership arrays above the driver limit too.
|
|
next_batch = await batches.freeze(s, settings)
|
|
assert await s.scalar(select(func.count()).select_from(BatchPackage).where(BatchPackage.batch_id == next_batch.id)) == count
|
|
assert len(await batches.actions(s, next_batch, set(), 8)) == 8
|
|
await clear(s)
|
|
print('Batch regressions passed: deferred retries, failed-evidence reuse, manual source revisions, and 33,000 originals without parameter overflow.')
|
|
|
|
|
|
asyncio.run(main())
|