AlphaPulse/scripts/smoke_news_research.py

"""Verify two-stage behavior in a disposable CI database using mock model I/O."""

import asyncio
from copy import deepcopy
from datetime import UTC, datetime, timedelta
import os
from unittest.mock import patch
from types import SimpleNamespace
from urllib.parse import urlparse
from sqlalchemy import select, func, text
from alphapulse.core.db import SessionLocal, engine
from alphapulse.models import NewsItem
from alphapulse.news_research import service
from alphapulse.news_research.models import Package, Event, Membership, Attempt, Version
from alphapulse.news_research.contracts import IndexResult, Assignment, ResearchResult


async def main():
    url = urlparse(os.environ["DATABASE_URL"])
    assert url.username == "ci" and url.password == "ci-only" and url.path == "/ci"
    now = datetime.now(UTC)
    async with SessionLocal() as s:
        news = NewsItem(
            source_code="wscn_news",
            external_id="ci-research-" + str(now.timestamp()),
            title="Original incident and response",
            summary="",
            content_text="Original incident and response",
            content_hash="c" * 64,
            url="https://example.invalid/ci",
            published_at=now,
        )
        s.add(news)
        await s.commit()
        news_id = news.id
        await service.enroll(s, [news_id])
        await service.enroll(s, [news_id])
        package = await s.scalar(select(Package).where(Package.news_id == news_id))
        identifier = package.id
        assert (
            await s.scalar(
                select(func.count())
                .select_from(Package)
                .where(Package.news_id == news_id)
            )
            == 1
        )
    calls = []
    model_inputs = {}

    async def config(*args):
        return SimpleNamespace(context_window=1000000, max_tokens=1000), "test", {}

    async def complete(provider, skill, payload, schema):
        calls.append(schema.__name__)
        model_inputs[schema.__name__] = deepcopy(payload)
        if schema is IndexResult:
            return IndexResult(
                keywords=[
                    {
                        "canonical": "incident",
                        "aliases": ["incident"],
                        "quote": "Original incident and response",
                    }
                ]
            ), {}
        return Assignment(
            targets=[
                {
                    "event_id": None,
                    "new_key": str(i),
                    "title": "Incident " + str(i),
                    "reason": "A related part",
                    "quote": "Original incident and response",
                    "change_kind": "new_fact",
                    "urgent": False,
                }
                for i in (1, 2)
            ]
        ), {}

    with (
        patch.object(service, "model_config", config),
        patch.object(service, "complete", complete),
    ):
        await service.process_package(identifier, {})
        await service.process_package(identifier, {})
    assert calls == ["IndexResult", "Assignment"]
    async with SessionLocal() as s:
        attempts = list(
            await s.scalars(select(Attempt).where(Attempt.package_id == identifier))
        )
        assert {a.stage: a.input for a in attempts} == {
            "index": model_inputs["IndexResult"],
            "assign": model_inputs["Assignment"],
        }, "Persist the exact model input, including assignment keywords and candidates"
        for event in await s.scalars(select(Event)):
            assert event.due_at - event.dirty_since == timedelta(minutes=5)
        assert (
            await s.scalar(
                select(func.count())
                .select_from(Membership)
                .where(Membership.package_id == identifier)
            )
            == 2
        )
        assert (
            await s.get(NewsItem, news_id)
        ).content_text == "Original incident and response"
        package = await s.get(Package, identifier)
        package.state = "assign_pending"
        await s.commit()
    with (
        patch.object(service, "model_config", config),
        patch.object(
            service, "complete", side_effect=ValueError("classification failure")
        ),
    ):
        await service.process_package(identifier, {})
    async with SessionLocal() as s:
        package = await s.get(Package, identifier)
        assert package.state == "assign_failed" and package.keywords
        failed = await s.scalar(
            select(Attempt)
            .where(Attempt.package_id == identifier, Attempt.stage == "assign")
            .order_by(Attempt.id.desc())
        )
        assert failed.input["keywords"] == package.keywords
        assert len(failed.input["events"]) == 2
        assert (
            await s.scalar(
                select(func.count())
                .select_from(Attempt)
                .where(Attempt.package_id == identifier, Attempt.stage == "index")
            )
            == 1
        )
        # Evidence validation failure is retained as a failed version, never a success.
        event = await s.scalar(
            select(Event)
            .join(Membership)
            .where(Membership.package_id == identifier)
            .limit(1)
        )
        package.state = "done"
        event.state = "pending"
        event.due_at = now
        await s.commit()
        event_id = event.id

    async def run(self, *args):
        self.records.append(
            {
                "tool": "read_news",
                "arguments": {"id": 2147483647},
                "result": {
                    "article_id": -1,
                    "news_id": 2147483647,
                    "content": "Previously verified background",
                },
            }
        )
        result = ResearchResult(
            id=event_id,
            title="Incident",
            summary="Summary",
            category="安全",
            entities=[],
            risk_stars=1,
            importance_stars=2,
            direction="unknown",
            verification="insufficient",
            confidence="low",
            reasoning="Unknown",
            uncertainty="Unknown",
            facts="Original",
            timeline="Unknown",
            asset_impacts="Unknown",
            opinions="Unknown",
            counter_evidence="Unknown",
            watchpoints="Unknown",
            claims=[
                {
                    "statement": "Original",
                    "verdict": "insufficient",
                    "evidence_refs": [],
                }
            ],
            evidence_refs=[{"article_id": news_id, "quote": "Invented"}],
            timeline_points=[
                {
                    "package_ids": [identifier],
                    "occurred_at": None,
                    "time_precision": "unknown",
                    "time_text": "",
                    "reported_at": now,
                    "description": "Original",
                    "verification": "partial",
                    "evidence_refs": [
                        {
                            "article_id": news_id,
                            "quote": "Original incident and response",
                        }
                    ],
                }
            ],
            references=[],
            excluded_article_ids=[],
        )
        return result, {}

    with (
        patch.object(service, "model_config", config),
        patch.object(service.Research, "run", run),
    ):
        await service.process_event(event_id, {}, force=True)
    async with SessionLocal() as s:
        event = await s.get(Event, event_id)
        assert event.state == "failed" and event.result is None
        version = await s.scalar(select(Version).where(Version.event_id == event_id))
        assert (
            version.state == "failed"
            and version.output["evidence_refs"][0]["quote"] == "Invented"
        )
        assert (
            await s.scalar(
                text(
                    "SELECT count(*) FROM information_schema.tables WHERE table_name LIKE 'news_research_%'"
                )
            )
            == 8
        )
    # Exercise actual routes and published-Skill resolution against isolated storage.
    import httpx
    from fastapi import FastAPI
    from alphapulse.news_research import api
    from alphapulse.models import AiProvider, SkillNode, SkillVersion
    from alphapulse.core.crypto import encrypt
    from alphapulse.news_risk import service as legacy_service

    app = FastAPI()
    app.include_router(api.router)
    app.dependency_overrides[api.require_admin] = lambda: None
    async with httpx.AsyncClient(
        transport=httpx.ASGITransport(app), base_url="http://ci"
    ) as client:
        assert (await client.get("/api/risk/events?page=0")).status_code == 422
        assert (await client.get("/api/risk/packages/2147483647")).status_code == 404
        assert (
            len((await client.get(f"/api/risk/news/{news_id}/packages")).json()["data"])
            == 1
        )
        assert (
            await client.get(f"/api/risk/news/{news_id}/packages?revision_no=99")
        ).json()["data"] == []
        async with SessionLocal() as s:
            published = dict(
                (
                    await s.execute(
                        select(SkillNode.id, SkillNode.published_version_id)
                    )
                ).all()
            )
        ids = (await client.post("/api/risk/skill-templates")).json()["data"]
        async with SessionLocal() as s:
            provider = AiProvider(
                name="ci-only-" + str(now.timestamp()),
                protocol="openai",
                base_url="https://example.invalid",
                api_key_encrypted=encrypt("ci-only"),
                api_key_hint="ci",
                model="ci-only",
                max_tokens=1000,
                context_window=100000,
                reasoning_effort=None,
            )
            s.add(provider)
            for identifier in ids.values():
                node = await s.get(SkillNode, identifier)
                assert (
                    node.scope == "news"
                    and node.published_version_id == published.get(identifier)
                )
                node.published_version_id = await s.scalar(
                    select(SkillVersion.id)
                    .where(SkillVersion.skill_id == identifier)
                    .order_by(SkillVersion.id.desc())
                    .limit(1)
                )
            await s.commit()
            provider_id = provider.id
        config = {
            stage: {
                "provider_id": provider_id,
                "model": "ci-only",
                "reasoning_effort": None,
                "skill_id": ids[stage],
            }
            for stage in ("index", "research")
        }
        config["concurrency"] = 2
        assert (await client.put("/api/risk/settings", json=config)).status_code == 200
        assert (await client.get("/api/risk/settings")).json()["data"][
            "enabled"
        ] is False
        async with SessionLocal() as s:
            for stage in ("index", "research"):
                provider, skill, rules = await service.model_config(s, config, stage)
                assert (
                    rules["skill_id"] == ids[stage]
                    and skill
                    and provider.model == "ci-only"
                )

        async def validation(provider, skill, payload, schema):
            source = payload.get("article") or payload["articles"][0]
            quote = source["content"]
            if schema is IndexResult:
                return schema(
                    keywords=[
                        {
                            "canonical": "示例公司",
                            "aliases": ["示例公司"],
                            "quote": quote,
                        }
                    ]
                ), {}
            if schema is Assignment:
                return schema(
                    targets=[
                        {
                            "event_id": None,
                            "new_key": "ci",
                            "title": "Test",
                            "reason": "Test",
                            "quote": quote,
                            "change_kind": "new_fact",
                            "urgent": False,
                        }
                    ]
                ), {}
            result = dict(version.output)
            result.update(
                id=1,
                evidence_refs=[{"article_id": 1, "quote": quote}],
                timeline_points=[
                    {
                        **result["timeline_points"][0],
                        "package_ids": [1],
                        "evidence_refs": [{"article_id": 1, "quote": quote}],
                    }
                ],
            )
            return schema.model_validate(result), {}

        with patch.object(legacy_service, "complete", validation):
            response = await client.post("/api/risk/enable")
            assert response.status_code == 200, response.text
        assert (await client.get("/api/risk/settings")).json()["data"][
            "enabled"
        ] is True
        assert (await client.post("/api/risk/enable")).status_code == 409
        assert (await client.post("/api/risk/pause")).status_code == 200
        for action in ("pause", "save"):
            interrupted = False

            async def interrupted_validation(
                provider, skill, payload, schema, *, action=action
            ):
                nonlocal interrupted
                if not interrupted:
                    interrupted = True
                    response = (
                        await client.post("/api/risk/pause")
                        if action == "pause"
                        else await client.put("/api/risk/settings", json=config)
                    )
                    assert response.status_code == 200
                return await validation(provider, skill, payload, schema)

            with patch.object(legacy_service, "complete", interrupted_validation):
                response = await client.post("/api/risk/enable")
                assert response.status_code == 409, response.text
            assert (await client.get("/api/risk/settings")).json()["data"][
                "enabled"
            ] is False
        package_id = (await client.get(f"/api/risk/news/{news_id}/packages")).json()[
            "data"
        ][0]["id"]
        assert (
            await client.post(f"/api/risk/packages/{package_id}/retry")
        ).status_code == 200
        async with SessionLocal() as s:
            row = await s.get(Package, package_id)
            assert row.state == "assign_pending" and row.keywords
        event_detail = (await client.get(f"/api/risk/events/{event_id}")).json()["data"]
        assert (
            event_detail["packages"]
            and event_detail["versions"][0]["state"] == "failed"
        )
        assert (
            await client.post(f"/api/risk/events/{event_id}/retry")
        ).status_code == 200
        async with SessionLocal() as s:
            assert (await s.get(Event, event_id)).revision == version.revision
            package = await s.get(Package, package_id)
            package.state = "done"
            await s.commit()
            other_id = await s.scalar(
                select(Membership.event_id).where(
                    Membership.package_id == package_id, Membership.event_id != event_id
                )
            )

        async def success(self, provider, skill, payload, schema):
            if payload["id"] == event_id:
                cached = await self.tool("read_news", {"id": 2147483647})
                assert cached["content"] == "Previously verified background"
                assert self.counts["read"] == 0
                assert any(r.get("reused") for r in self.records)
            result = dict(version.output)
            quote = payload["articles"][0]["content"]
            result.update(
                id=payload["id"],
                evidence_refs=[{"article_id": news_id, "quote": quote}],
                references=[],
                timeline_points=[
                    {
                        **result["timeline_points"][0],
                        "package_ids": [package_id],
                        "evidence_refs": [{"article_id": news_id, "quote": quote}],
                    }
                ],
            )
            if payload["id"] == other_id:
                target = await self.tool("read_event", {"id": event_id})
                result["references"] = [
                    {
                        "version_id": target["version_id"],
                        "role": "background",
                        "reason": "Original source already provided",
                    }
                ]
            return schema.model_validate(result), {}

        with patch.object(service.Research, "run", success):
            await service.process_event(event_id, config, force=True)
            await service.process_event(other_id, config, force=True)
        detail = (await client.get(f"/api/risk/events/{other_id}")).json()["data"]
        assert detail["state"] == "done" and detail["references"][0]["changed"] is False
        assert (
            await client.post(f"/api/risk/events/{event_id}/retry")
        ).status_code == 200
        detail = (await client.get(f"/api/risk/events/{other_id}")).json()["data"]
        assert detail["state"] == "done" and detail["references"][0]["changed"] is True
        # Referenced changes do not automatically enqueue the referring event.

        # Old source failures cannot replace work for a newly enrolled source revision.
        async with SessionLocal() as s:
            changed_news = NewsItem(
                source_code="wscn_news",
                external_id="ci-source-race-" + str(now.timestamp()),
                title="Original incident and response",
                content_text="Original incident and response",
                content_hash="d" * 64,
                url="https://example.invalid/source-race",
                published_at=now,
            )
            s.add(changed_news)
            await s.commit()
            changed_news_id = changed_news.id
            await service.enroll(s, [changed_news_id])
            changed_package_id = await s.scalar(
                select(Package.id).where(Package.news_id == changed_news_id)
            )
        with patch.object(service, "complete", complete):
            await service.process_package(changed_package_id, config)

        async def old_request_failure(*args):
            async with SessionLocal() as s:
                changed_news = await s.get(NewsItem, changed_news_id)
                changed_news.content_text += " updated"
                changed_news.content_hash = "e" * 64
                changed_news.revision_no = 2
                await s.commit()
            response = await client.get(
                f"/api/risk/news/{changed_news_id}/packages?revision_no=2"
            )
            assert response.json()["data"] == []
            detail = (
                await client.get(f"/api/risk/packages/{changed_package_id}")
            ).json()["data"]
            assert not detail["keywords"] and not detail["events"]
            assert detail["state"] == "index_pending"
            async with SessionLocal() as s:
                await service.enroll(s, [changed_news_id])
            raise TimeoutError("Old source request timed out")

        with patch.object(service, "complete", old_request_failure):
            await service.process_package(changed_package_id, config)
        async with SessionLocal() as s:
            package = await s.get(Package, changed_package_id)
            assert package.content_hash == "e" * 64
            assert package.state == "index_pending" and package.error is None
            attempt = await s.scalar(
                select(Attempt)
                .where(Attempt.package_id == changed_package_id)
                .order_by(Attempt.id.desc())
                .limit(1)
            )
            assert attempt.state == "failed" and attempt.content_hash == "d" * 64
        with patch.object(service, "complete", complete):
            await service.process_package(changed_package_id, config)
        async with SessionLocal() as s:
            assert (await s.get(Package, changed_package_id)).state == "assign_pending"

        # Date-free titles still have bounded original evidence across a long history.
        async with SessionLocal() as s:
            for days in (180, 60, 5, 2, 1):
                published_at = now - timedelta(days=days)
                quote = f"{published_at.date()} Original incident and response"
                historical = NewsItem(
                    source_code="wscn_news",
                    external_id=f"ci-candidate-{days}-{now.timestamp()}",
                    title="Recurring report",
                    content_text=quote,
                    content_hash=str(days).zfill(64),
                    url="https://example.invalid/candidate",
                    published_at=published_at,
                )
                s.add(historical)
                await s.flush()
                package = Package(
                    news_id=historical.id,
                    content_hash=historical.content_hash,
                    state="done",
                    keywords=[],
                )
                s.add(package)
                await s.flush()
                s.add(
                    Membership(
                        event_id=event_id,
                        package_id=package.id,
                        evidence=quote,
                        reason="Same incident follow-up",
                        change_kind="new_fact",
                    )
                )
            await s.commit()
            event = await s.get(Event, event_id)
            context = (await service.candidate_context(s, [event]))[0]
            assert context["reporting_range"] == {
                "first": (now - timedelta(days=180)).isoformat(),
                "latest": now.isoformat(),
            }
            assert len(context["evidence"]) == 3
            assert [e["published_at"] for e in context["evidence"]] == [
                (now - timedelta(days=180)).isoformat(),
                (now - timedelta(days=1)).isoformat(),
                now.isoformat(),
            ]
            assert (
                str((now - timedelta(days=180)).date())
                in context["evidence"][0]["quote"]
            )
            assert (await s.get(NewsItem, news_id)).content_hash == "c" * 64
    await review_regressions(config, complete, changed_package_id, changed_news_id)
    await enrollment_regressions()
    await phrase_search_regressions()
    await event_search_regressions()
    await compound_candidate_regressions()
    await vocabulary_regressions()
    await retrieval_revision_regressions()
    await engine.dispose()
    print(
        "Two-stage smoke passed: indexing, membership, evidence reuse, pause cancellation, source races and dated event candidates."
    )


async def review_regressions(config, complete, package_id, news_id):
    # Publication must not overwrite another session's newer source enrollment.
    async def superseded_success(*args):
        result = await complete(*args)
        async with SessionLocal() as session:
            news = await session.get(NewsItem, news_id)
            news.content_text += " newer revision"
            news.content_hash = "f" * 64
            await session.commit()
            await service.enroll(session, [news_id])
        return result

    with patch.object(service, "complete", superseded_success):
        await service.process_package(package_id, config)
    async with SessionLocal() as session:
        package = await session.get(Package, package_id)
        assert (
            package.content_hash == "f" * 64
            and package.state == "index_pending"
            and package.keywords == []
        )
        attempt = await session.scalar(
            select(Attempt)
            .where(Attempt.package_id == package_id)
            .order_by(Attempt.id.desc())
            .limit(1)
        )
        assert attempt.state == "superseded"
        package.state = "assign_pending"
        await session.commit()
    with patch.object(service, "complete", complete):
        await service.process_package(package_id, config)
    async with SessionLocal() as session:
        package = await session.get(Package, package_id)
        assert package.state == "assign_failed" and "关键词" in package.error
        from alphapulse.news_research.api import retry_package

        await retry_package(package_id, session)
        assert package.state == "index_pending"
        # One canonical group cannot gain ranking weight through its aliases.
        old = Event(
            title="Specific old incident",
            tags=["binance", "hack"],
            updated_at=datetime.now(UTC) - timedelta(days=180),
        )
        session.add(old)
        session.add_all(
            [
                Event(
                    title=f"Recent company item {i}",
                    tags=["binance"],
                    updated_at=datetime.now(UTC),
                )
                for i in range(65)
            ]
        )
        package.keywords = [
            {"canonical": "币安", "aliases": ["Binance"], "quote": "Binance"}
        ]
        await session.commit()
        catalog = await service.candidates(
            session,
            [
                {"canonical": "binance", "aliases": ["BINANCE"]},
                {"canonical": "hack", "aliases": ["HACK"]},
            ],
        )
        assert len(catalog) == 60 and catalog[0].id == old.id
        for term in ("Binance", " binANCE ", "币安"):
            condition, score = service.package_matching([term])
            matches = list(
                await session.scalars(
                    select(Package.id)
                    .join(NewsItem)
                    .where(condition)
                    .order_by(score.desc())
                )
            )
            assert package_id in matches
        assert package.keywords[0]["aliases"] == ["Binance"]
    # Two simultaneous callers must classify against consecutive committed catalogs.
    package_ids = []
    async with SessionLocal() as session:
        for i in range(2):
            news = NewsItem(
                source_code="wscn_news",
                external_id=f"ci-serial-{datetime.now(UTC).timestamp()}-{i}",
                title="Serial incident",
                url="https://example.invalid/serial",
                content_text="serialincident frozen evidence",
                content_hash=str(i) * 64,
                published_at=datetime.now(UTC),
            )
            session.add(news)
            await session.flush()
            package = Package(news_id=news.id, content_hash=news.content_hash)
            session.add(package)
            await session.flush()
            package_ids.append(package.id)
        await session.commit()
    submissions = []

    async def serial_complete(provider, skill, payload, schema):
        submissions.append(schema.__name__)
        if schema is IndexResult:
            return schema(
                keywords=[
                    {
                        "canonical": "serialincident",
                        "aliases": ["serialincident"],
                        "quote": "serialincident",
                    }
                ]
            ), {}
        existing = next(
            (e for e in payload["events"] if e["title"] == "Serial incident"), None
        )
        await asyncio.sleep(0.01)
        return schema(
            targets=[
                {
                    "event_id": existing["id"] if existing else None,
                    "new_key": None if existing else "serial",
                    "title": "Serial incident",
                    "reason": "Same incident",
                    "quote": "serialincident frozen evidence",
                    "change_kind": "new_fact",
                    "urgent": False,
                }
            ]
        ), {}

    with patch.object(service, "complete", serial_complete):
        await asyncio.gather(*(service.process_package(i, config) for i in package_ids))
        await asyncio.gather(*(service.process_package(i, config) for i in package_ids))
    assert (
        submissions.count("IndexResult") == 2 and submissions.count("Assignment") == 2
    )
    async with SessionLocal() as session:
        memberships = list(
            await session.scalars(
                select(Membership).where(Membership.package_id.in_(package_ids))
            )
        )
        assert len(memberships) == 2 and len({m.event_id for m in memberships}) == 1
        event_id = memberships[0].event_id
        template = (
            await session.scalar(
                select(Version)
                .where(Version.output.is_not(None))
                .order_by(Version.id)
                .limit(1)
            )
        ).output

    # Force compression, then exercise both member read tools with frozen originals.
    async def extract(provider, rules, articles):
        return [
            {
                "evidence_refs": [
                    {"article_id": a["id"], "quote": "serialincident"} for a in articles
                ]
            }
        ], []

    async def compact(provider, rules, request, schema, fits):
        return request, []

    async def compressed_run(self, provider, skill, payload, schema):
        assert payload["articles"] == [] and "extractions" in payload
        assert not self.provided_news
        refs = []
        for i, package in enumerate(payload["packages"]):
            result = await self.tool(
                "read_news" if i == 0 else "read_package",
                {"id": package["news_id"] if i == 0 else package["id"]},
            )
            assert (
                result["article_id"] == package["news_id"]
                and result["content"] == "serialincident frozen evidence"
            )
            refs.append(
                {"article_id": result["article_id"], "quote": result["content"]}
            )
        point = {
            **template["timeline_points"][0],
            "package_ids": payload["package_ids"],
            "evidence_refs": refs,
            "description": "serialincident",
            "time_text": "",
        }
        return schema(
            **{
                **template,
                "id": event_id,
                "evidence_refs": refs,
                "timeline_points": [point],
                "references": [],
            }
        ), {}

    with (
        patch.object(service.Research, "fits", return_value=False),
        patch.object(service, "extract_articles", extract),
        patch.object(service, "compact_extractions", compact),
        patch.object(service.Research, "run", compressed_run),
    ):
        await service.process_event(event_id, config, force=True)
    async with SessionLocal() as session:
        event = await session.get(Event, event_id)
        assert event.state == "done", event.error
    print("Serial assignment and compressed member tool integration passed.")
    print(
        "Review regressions passed: superseded success, durable empty-index failure, long-lived ranking and literal alias search."
    )


async def enrollment_regressions():
    for cache in ("news", "package", "both"):
        async with SessionLocal() as session:
            news = NewsItem(
                source_code="wscn_news",
                external_id=f"ci-cache-{cache}-{datetime.now(UTC).timestamp()}",
                title="Cache fixture",
                url="https://example.invalid/cache",
                content_text="Second revision",
                content_hash="2" * 64,
                published_at=datetime.now(UTC),
            )
            session.add(news)
            await session.flush()
            package = Package(
                news_id=news.id,
                content_hash=news.content_hash,
                state="done",
                keywords=[
                    {
                        "canonical": "Second",
                        "aliases": ["Second"],
                        "quote": "Second revision",
                    }
                ],
            )
            event = Event(title="Cache event", revision=7, state="done")
            session.add_all([package, event])
            await session.flush()
            session.add(
                Membership(
                    package_id=package.id,
                    event_id=event.id,
                    evidence="Second revision",
                    reason="Same event",
                    change_kind="new_fact",
                )
            )
            await session.commit()
            news_id, package_id, event_id = news.id, package.id, event.id
        async with SessionLocal() as stale:
            # Keep strong references so ORM identity-map entries remain cached.
            cached_news = (
                await stale.get(NewsItem, news_id)
                if cache in ("news", "both")
                else None
            )
            cached_package = (
                await stale.get(Package, package_id)
                if cache in ("package", "both")
                else None
            )
            async with SessionLocal() as fresh:
                news = await fresh.get(NewsItem, news_id)
                package = await fresh.get(Package, package_id)
                news.content_hash, news.content_text = "3" * 64, "Third revision"
                package.content_hash, package.state = (
                    news.content_hash,
                    "assign_processing",
                )
                package.keywords = [
                    {
                        "canonical": "Third",
                        "aliases": ["Third"],
                        "quote": "Third revision",
                    }
                ]
                await fresh.commit()
            await service.enroll(stale, [news_id])
            assert cached_news is not None or cached_package is not None
        async with SessionLocal() as session:
            package = await session.get(Package, package_id)
            event = await session.get(Event, event_id)
            assert (
                package.content_hash == "3" * 64
                and package.state == "assign_processing"
                and package.keywords[0]["canonical"] == "Third"
            ), cache
            assert event.revision == 7, cache
            news = await session.get(NewsItem, news_id)
            news.content_hash, news.content_text = "4" * 64, "Fourth revision"
            await session.commit()
            await service.enroll(session, [news_id])
            await session.refresh(package)
            await session.refresh(event)
            assert (
                package.content_hash == "4" * 64
                and package.state == "index_pending"
                and package.keywords == []
            )
            assert event.revision == 8
            await service.enroll(session, [news_id])
            await session.refresh(event)
            assert event.revision == 8
    print("Enrollment cached-source and cached-package regressions passed.")


async def phrase_search_regressions():
    async with SessionLocal() as session:
        target_id = None
        for i in range(23):
            news = NewsItem(
                source_code="wscn_news",
                external_id=f"ci-phrase-{datetime.now(UTC).timestamp()}-{i}",
                title="Generic report" if i == 0 else "Federal policy Reserve outlook",
                url="https://example.invalid/phrase",
                content_text="Federal Reserve announcement",
                content_hash="5" * 64,
                published_at=datetime.now(UTC),
            )
            session.add(news)
            await session.flush()
            package = Package(
                news_id=news.id,
                content_hash=news.content_hash,
                state="done",
                keywords=[
                    {
                        "canonical": "美联储",
                        "aliases": ["Federal Reserve"],
                        "quote": news.content_text,
                    }
                ]
                if i == 0
                else [],
            )
            session.add(package)
            await session.flush()
            if i == 0:
                target_id = package.id
        await session.commit()
    for args in (
        {"query": "Federal Reserve"},
        {"query": "  fEDERAL rESERVE  "},
        {
            "query": "Federal Reserve",
            "keywords": ["one", "two", "three", "four", "five", "six"],
        },
        {"query": "unrelated", "keywords": ["Federal Reserve"]},
        {
            "query": "absent",
            "keywords": ["one", "two", "three", "four", "five", "Federal Reserve"],
        },
        {"query": "美联储"},
    ):
        rows = await service.Research(event={"id": 0}).tool("search_packages", args)
        assert rows and rows[0]["id"] == target_id, args
        assert len(rows) <= 20
    for args in (
        {"query": "Federal policy"},
        {"query": "absent", "keywords": ["Federal"]},
    ):
        assert await service.Research(event={"id": 0}).tool("search_packages", args)
    async with SessionLocal() as session:
        package = await session.get(Package, target_id)
        assert package.keywords[0]["aliases"] == ["Federal Reserve"]
    print("Phrase retrieval and top-20 ranking regressions passed.")


async def event_search_regressions():
    async with SessionLocal() as session:
        target = Event(
            title="加拿大央行利率决议",
            tags=["bank of canada", "加拿大央行"],
            updated_at=datetime.now(UTC) - timedelta(days=180),
        )
        own = Event(title="Current event", tags=["bank of canada"])
        retired = Event(
            title="Retired event", tags=["bank of canada"], state="retired"
        )
        session.add_all([target, own, retired])
        session.add_all(
            Event(title=f"Bank lending of Canada {i}", tags=[]) for i in range(23)
        )
        await session.commit()
        target_id, own_id, retired_id = target.id, own.id, retired.id
    for args in (
        {"query": "Bank of Canada"},
        {"query": "  bANK oF cANADA  "},
        {
            "query": "Bank of Canada",
            "keywords": ["one", "two", "three", "four", "five", "six"],
        },
        {"query": "absent", "keywords": ["Bank of Canada"]},
        {"query": "加拿大央行"},
    ):
        rows = await service.Research(event={"id": own_id}).tool("search_events", args)
        assert rows and rows[0]["id"] == target_id, args
        assert len(rows) <= 20
        assert not {own_id, retired_id} & {row["id"] for row in rows}
    print("Event phrase retrieval, relevance ranking and visibility regressions passed.")



async def compound_candidate_regressions():
    now = datetime.now(UTC)
    async with SessionLocal() as session:
        target = Event(title="Historical exchange incident", tags=["bybit"], updated_at=now-timedelta(days=180))
        title_only = Event(title="Bybit frozen-funds follow-up", tags=[])
        unrelated = Event(title="Bybitx unrelated announcement", tags=["bybitx"])
        retired = Event(title="Retired Bybit incident", tags=["bybit"], state="retired")
        currency = Event(title="USDC reserve", tags=["usdc"])
        exact = Event(title="Earlier full-phrase match", tags=["bybit hack"], updated_at=now-timedelta(days=400))
        chinese_old = Event(title="隔离中文候选", tags=[], updated_at=now-timedelta(days=10))
        chinese_new = Event(title="隔离中文候选", tags=[], updated_at=now)
        session.add_all([target, title_only, unrelated, retired, currency, exact, chinese_old, chinese_new])
        await session.flush()
        keys = [{"canonical": "Bybit黑客事件", "aliases": ["Bybit Hack"]}]
        rows = await service.candidates(session, keys)
        ids = {row.id for row in rows}
        assert {target.id, title_only.id, exact.id} <= ids
        assert not {unrelated.id, retired.id} & ids
        assert rows[0].id == exact.id
        rows = await service.candidates(session, [{"canonical": "美元抵押", "aliases": ["USD collateral"]}])
        assert currency.id not in {row.id for row in rows}
        rows = await service.candidates(session, [{"canonical": "隔离中文候选", "aliases": ["隔离中文候选"]}])
        assert [row.id for row in rows] == [chinese_new.id, chinese_old.id]
        session.add_all(Event(title=f"Recent Bybit update {i}", tags=["bybit"]) for i in range(65))
        await session.flush()
        rows = await service.candidates(session, keys)
        assert len(rows) == 60 and rows[0].id == exact.id
        await session.rollback()
    print("Compound-alias candidate recall, whole-token boundaries, ranking and sixty-result cap passed.")


async def vocabulary_regressions():
    from alphapulse.news_research import api
    now = datetime.now(UTC)
    # Vocabulary changes affect recall without rewriting prior evidence or paid work.
    from alphapulse.news_research import vocabulary as vocab
    from alphapulse.news_research.models import Reference
    async with SessionLocal() as session:
        saved = await api.save_vocabulary(vocab.VocabularyInput(base_revision=0, entries=[{"canonical": "币安", "aliases": ["Binance"]}]), session)
        assert saved.data["revision"] == 1
        old = Event(title="Historical incident A", tags=["binance", "incident-a"], revision=1, state="done", updated_at=now-timedelta(days=400))
        session.add(old)
        session.add_all([Event(title=f"Unrelated subject occurrence {i}", tags=["binance"], revision=1, state="done") for i in range(65)])
        story = NewsItem(source_code="wscn_news", external_id=f"ci-vocab-{now.timestamp()}", title="Original evidence", content_text="Binance incident-a", content_hash="f"*64, published_at=now-timedelta(days=400), url="https://example.invalid/vocab")
        session.add(story)
        await session.flush()
        package = Package(news_id=story.id, content_hash=story.content_hash, state="done", keywords=[{"canonical": "Binance", "aliases": ["Binance"], "quote": "Binance incident-a"}])
        session.add(package)
        await session.flush()
        frozen = service.article(story)
        target = Version(event_id=old.id, revision=1, state="done", input={"packages": [{"id": package.id, "news_id": story.id}], "articles": [frozen]}, output={"title": "Historical title", "summary": "Frozen summary"}, rules={})
        session.add(target)
        await session.commit()
        keys = [{"canonical": "币安", "aliases": ["Binance"]}, {"canonical": "incident-a", "aliases": ["incident-a"]}]
        assert (await service.candidates(session, keys))[0].id == old.id
        rows = await api.packages(session, q="币安", page=1)
        assert package.id in {row["id"] for row in rows.data["items"]}
        await api.save_vocabulary(vocab.VocabularyInput(base_revision=1, entries=[{"canonical": "币安", "aliases": ["Binance", "BNC"]}]), session)
        rows = await api.packages(session, q=" BNC ", page=1)
        assert package.id in {row["id"] for row in rows.data["items"]}
        assert package.keywords[0]["canonical"] == "Binance"
        snapshot = await api.event_version(old.id, target.id, session)
        assert snapshot.data["articles"][0]["content"] == "Binance incident-a"
        consumers = []
        for i in range(2):
            consumer = Event(title=f"Independent rule {i}", tags=[], revision=1, state="done")
            session.add(consumer)
            await session.flush()
            version = Version(event_id=consumer.id, revision=1, state="done", input={}, output={}, rules={})
            session.add(version)
            await session.flush()
            session.add(Reference(version_id=version.id, target_version_id=target.id, role="support", reason="Independent source confirms context"))
            consumers.append(version)
        await session.commit()
        old.title = "Current different title"
        service.dirty(old)
        story.content_text = "Updated original"
        await session.commit()
        for consumer in consumers:
            refs = await api.version_references(session, consumer.id)
            assert refs[0]["changed"] and refs[0]["title"] == "Historical title"
            assert refs[0]["version_id"] == target.id
        assert (await api.event_version(old.id, target.id, session)).data["articles"][0]["content"] == "Binance incident-a"
        assert (await session.get(Package, package.id)).state == "done"
        old_id = old.id
    results = await service.Research(event={"id": 0}).tool("search_events", {"query": "BNC", "keywords": ["incident-a"]})
    assert results[0]["id"] == old_id
    print("Vocabulary API, cross-year recall above 60 distractors, alias updates and two immutable references passed.")


async def retrieval_revision_regressions():
    from alphapulse.news_research import api, vocabulary as vocab

    now = datetime.now(UTC)
    async with SessionLocal() as session:
        news = NewsItem(
            source_code="wscn_news",
            external_id=f"ci-stale-tags-{now.timestamp()}",
            title="Corrected headline for retrieval regression",
            content_text="Binance corrected company announcement",
            content_hash="8" * 64,
            published_at=now,
            url="https://example.invalid/retrieval-revision",
        )
        session.add(news)
        await session.flush()
        package = Package(
            news_id=news.id,
            content_hash="7" * 64,
            state="done",
            keywords=[{"canonical": "obsolete-retrieval-tag", "aliases": ["obsolete-retrieval-tag"], "quote": "obsolete-retrieval-tag"}],
        )
        session.add(package)
        await session.commit()
        package_id = package.id
        stale = await api.packages(session, q="obsolete-retrieval-tag", page=1)
        assert stale.data["total"] == 0
        current = await api.packages(session, q=news.title, page=1)
        assert current.data["items"][0]["id"] == package_id
        assert current.data["items"][0]["keywords"] == []
    query = {"query": "obsolete-retrieval-tag"}
    assert await service.Research(event={"id": 0}).tool("search_packages", query) == []
    current = await service.Research(event={"id": 0}).tool("search_packages", {"query": "Corrected headline for retrieval regression"})
    assert current[0]["id"] == package_id and current[0]["keywords"] == []
    async with SessionLocal() as session:
        package = await session.get(Package, package_id)
        package.content_hash = "8" * 64
        package.keywords = [{"canonical": "Binance", "aliases": ["Binance"], "quote": "Binance"}]
        current = await vocab.snapshot(session)
        before = {"revision": current["revision"], "entries": current["entries"]}
        entries = [{**e, "aliases": [*e["aliases"], "BNC-reviewed"]} if e["canonical"] == "币安" else e for e in current["entries"]]
        await api.save_vocabulary(vocab.VocabularyInput(base_revision=current["revision"], entries=entries), session)
        after = await vocab.snapshot(session)
        event = Event(title="Reviewed independent incident", tags=["binance"])
        session.add(event)
        await session.commit()
        event_id = event.id
    for tool, target in (("search_packages", package_id), ("search_events", event_id)):
        args = {"query": "BNC-reviewed", "keywords": []}
        prior = service.Research(event={"id": 0})
        prior.vocabulary = before
        assert await prior.tool(tool, args) == []
        retry = service.Research(event={"id": 0})
        retry.vocabulary = after
        retry.restore(prior.records)
        rows = await retry.tool(tool, args)
        assert target in {row["id"] for row in rows}
        assert len(retry.records) == 1 and not retry.records[0].get("reused")
    async with SessionLocal() as session:
        fresh = await api.packages(session, q="BNC-reviewed", page=1)
        assert package_id in {row["id"] for row in fresh.data["items"]}
    print("Stale TAG exclusion, current-title lookup and vocabulary retry refresh passed.")


asyncio.run(main())