feat(news): index news packages and research persistent events
Extract reusable literal keywords once per source revision, assign complete news packages to long-lived events, and research all event members with immutable evidence and references. Add additive research storage, vocabulary snapshots, the two-stage workspace, isolated acceptance coverage, and a paused rollout with an initial fixed 50-news production trial.
This commit is contained in:
eezd committed
1 parent
07252fea0f
commit
4929a32a0a
69 files changed
+8215
-2092
No files matched your search
@@ -0,0 +1,383 @@
|
||||
"""Bounded two-stage acceptance in an empty disposable CI database only."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from contextlib import ExitStack
|
||||
from datetime import datetime
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlparse
|
||||
from sqlalchemy import select, func
|
||||
from alphapulse.ai import llm
|
||||
from alphapulse.core.crypto import encrypt
|
||||
from alphapulse.core.db import SessionLocal, engine
|
||||
from alphapulse.models import AiProvider, NewsItem
|
||||
from alphapulse.news_research import service
|
||||
from alphapulse.news_research.models import (
|
||||
Package,
|
||||
Attempt,
|
||||
Event,
|
||||
Version,
|
||||
Membership,
|
||||
Reference,
|
||||
VocabularyVersion,
|
||||
)
|
||||
|
||||
|
||||
class BudgetExceeded(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Budget:
|
||||
def __init__(self, path, limit=240):
|
||||
self.path, self.limit = path, limit
|
||||
|
||||
def take(self):
|
||||
with self.path.open("a+") as file:
|
||||
fcntl.flock(file, fcntl.LOCK_EX)
|
||||
file.seek(0)
|
||||
count = int(file.read() or "0")
|
||||
if count >= self.limit:
|
||||
raise BudgetExceeded(f"真实模型请求已达{self.limit}次上限")
|
||||
file.seek(0)
|
||||
file.truncate()
|
||||
file.write(str(count + 1))
|
||||
file.flush()
|
||||
os.fsync(file.fileno())
|
||||
|
||||
def wrap(self, original):
|
||||
async def stream(*args, **kwargs):
|
||||
self.take()
|
||||
async for event in original(*args, **kwargs):
|
||||
yield event
|
||||
|
||||
return stream
|
||||
|
||||
|
||||
async def main(args):
|
||||
if args.output.exists():
|
||||
raise ValueError("报告已存在，请使用新路径，保留既有验收记录")
|
||||
url = urlparse(os.environ["DATABASE_URL"])
|
||||
if not (
|
||||
url.username == "ci"
|
||||
and url.password == "ci-only"
|
||||
and url.path == "/ci"
|
||||
and url.hostname in ("localhost", "127.0.0.1")
|
||||
):
|
||||
raise ValueError("只能写入独立本地CI数据库")
|
||||
raw = args.sample.read_bytes()
|
||||
source = json.loads(raw)
|
||||
news = source["news"] if args.synthetic else source["original_news"]
|
||||
if len(news) != (12 if args.synthetic else 30):
|
||||
raise ValueError("样本数量不符合冻结范围")
|
||||
credentials = json.loads(args.credentials.read_text())
|
||||
if not args.synthetic and credentials.get("searxng_url"):
|
||||
from alphapulse.core.config import get_settings
|
||||
|
||||
get_settings().searxng_url = credentials["searxng_url"]
|
||||
config = {"isolated_acceptance": True, "concurrency": 3}
|
||||
async with SessionLocal() as session:
|
||||
if await session.scalar(select(func.count()).select_from(NewsItem)):
|
||||
raise ValueError("验收数据库必须为空")
|
||||
for stage in ("index", "research"):
|
||||
fields = credentials[stage]
|
||||
provider = AiProvider(
|
||||
name="accept-" + stage,
|
||||
protocol=fields["protocol"],
|
||||
base_url=fields["base_url"],
|
||||
api_key_encrypted=encrypt(fields["api_key"]),
|
||||
api_key_hint="isolated",
|
||||
model=fields["model"],
|
||||
max_tokens=fields["max_tokens"],
|
||||
context_window=fields["context_window"],
|
||||
)
|
||||
session.add(provider)
|
||||
await session.flush()
|
||||
config[stage] = {
|
||||
"provider_id": provider.id,
|
||||
"model": "gpt-6.1-terra",
|
||||
"reasoning_effort": "medium" if stage == "index" else "xhigh",
|
||||
"skill_id": None,
|
||||
}
|
||||
session.add(
|
||||
VocabularyVersion(
|
||||
revision=1,
|
||||
entries=[
|
||||
{"canonical": "币安", "aliases": ["Binance"]},
|
||||
{"canonical": "星桥交易所", "aliases": ["XBridge"]},
|
||||
{"canonical": "中国外交部", "aliases": ["中华人民共和国外交部"]},
|
||||
],
|
||||
)
|
||||
)
|
||||
columns = set(NewsItem.__table__.columns.keys())
|
||||
for item in news:
|
||||
values = {k: v for k, v in item.items() if k in columns}
|
||||
for k, v in list(values.items()):
|
||||
if k.endswith("_at") and v:
|
||||
values[k] = datetime.fromisoformat(v)
|
||||
session.add(NewsItem(**values))
|
||||
await session.commit()
|
||||
budget = Budget(args.budget)
|
||||
batches = (
|
||||
source["batches"]
|
||||
if args.synthetic
|
||||
else [
|
||||
[n["id"] for n in sorted(news, key=lambda n: (n["published_at"], n["id"]))]
|
||||
]
|
||||
)
|
||||
semaphore = asyncio.Semaphore(3)
|
||||
checkpoints = []
|
||||
|
||||
async def indexed(identifier):
|
||||
async with semaphore:
|
||||
await service.process_package(identifier, config)
|
||||
|
||||
async def researched(identifier):
|
||||
async with semaphore:
|
||||
await service.process_event(identifier, config, force=True)
|
||||
|
||||
async def controlled_web(self, name, arguments):
|
||||
if name in ("search_web", "read_web"):
|
||||
return {"status": "unavailable", "reason": "虚构验收组不使用真实外部网络"}
|
||||
return await original_tool(self, name, arguments)
|
||||
|
||||
original_tool = service.Research.tool
|
||||
with ExitStack() as stack:
|
||||
for name in ("_stream_openai", "_stream_anthropic"):
|
||||
stack.enter_context(
|
||||
patch.object(llm, name, budget.wrap(getattr(llm, name)))
|
||||
)
|
||||
if args.synthetic:
|
||||
stack.enter_context(patch.object(service.Research, "tool", controlled_web))
|
||||
for batch in batches:
|
||||
if args.budget.exists() and int(args.budget.read_text() or "0") >= 240:
|
||||
break
|
||||
async with SessionLocal() as session:
|
||||
await service.enroll(session, batch)
|
||||
packages = list(
|
||||
await session.scalars(
|
||||
select(Package).where(Package.news_id.in_(batch))
|
||||
)
|
||||
)
|
||||
ordered = {p.news_id: p.id for p in packages}
|
||||
await asyncio.gather(*(indexed(ordered[n]) for n in batch))
|
||||
for n in batch:
|
||||
await service.process_package(ordered[n], config)
|
||||
async with SessionLocal() as session:
|
||||
pending = list(
|
||||
await session.scalars(
|
||||
select(Event.id).where(Event.state == "pending")
|
||||
)
|
||||
)
|
||||
await asyncio.gather(*(researched(i) for i in pending))
|
||||
async with SessionLocal() as session:
|
||||
checkpoints.append(
|
||||
{
|
||||
"batch": batch,
|
||||
"versions": [
|
||||
(v.event_id, v.id, v.revision, v.state)
|
||||
for v in await session.scalars(select(Version))
|
||||
],
|
||||
}
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"group": "synthetic" if args.synthetic else "real",
|
||||
"batch": batch,
|
||||
"requests": int(args.budget.read_text()),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
async with SessionLocal() as session:
|
||||
tables = {}
|
||||
for model in (
|
||||
Package,
|
||||
Attempt,
|
||||
Event,
|
||||
Version,
|
||||
Membership,
|
||||
Reference,
|
||||
VocabularyVersion,
|
||||
):
|
||||
tables[model.__tablename__] = [
|
||||
{c.name: getattr(row, c.name) for c in model.__table__.columns}
|
||||
for row in await session.scalars(select(model))
|
||||
]
|
||||
packages = tables["news_research_packages"]
|
||||
memberships = tables["news_research_memberships"]
|
||||
links = {
|
||||
p["news_id"]: {m["event_id"] for m in memberships if m["package_id"] == p["id"]}
|
||||
for p in packages
|
||||
}
|
||||
for item in news:
|
||||
links.setdefault(item["id"], set())
|
||||
assertions = []
|
||||
must_link = (
|
||||
source["must_link"]
|
||||
if args.synthetic
|
||||
else [
|
||||
[463197, 463198, 463199, 463205],
|
||||
[463206, 463208, 463218],
|
||||
[463192, 463193],
|
||||
[463201, 463202],
|
||||
[463200, 463204],
|
||||
[463209, 463212],
|
||||
[463194, 463213],
|
||||
[463216, 463220],
|
||||
]
|
||||
)
|
||||
for group in must_link:
|
||||
assertions.append(
|
||||
{
|
||||
"name": "same_event",
|
||||
"news_ids": group,
|
||||
"passed": bool(set.intersection(*(links[n] for n in group))),
|
||||
}
|
||||
)
|
||||
if args.synthetic:
|
||||
from itertools import combinations
|
||||
|
||||
for a, b in combinations(source["must_separate"], 2):
|
||||
assertions.append(
|
||||
{
|
||||
"name": "different_events",
|
||||
"news_ids": [a, b],
|
||||
"passed": not (links[a] & links[b]),
|
||||
}
|
||||
)
|
||||
versions = {v["id"]: v for v in tables["news_research_versions"]}
|
||||
for a, b in source["references"]:
|
||||
matching = [
|
||||
ref
|
||||
for ref in tables["news_research_references"]
|
||||
if versions[ref["version_id"]]["event_id"] in links[a]
|
||||
and versions[ref["target_version_id"]]["event_id"] in links[b]
|
||||
]
|
||||
assertions.append(
|
||||
{
|
||||
"name": "immutable_reference",
|
||||
"news_ids": [a, b],
|
||||
"passed": bool(matching)
|
||||
and all(
|
||||
versions[ref["target_version_id"]]["state"] == "done"
|
||||
and versions[ref["target_version_id"]]["id"]
|
||||
< max(
|
||||
v["id"]
|
||||
for v in versions.values()
|
||||
if v["event_id"] in links[b] and v["state"] == "done"
|
||||
)
|
||||
for ref in matching
|
||||
),
|
||||
}
|
||||
)
|
||||
main_events = links[1001]
|
||||
good = [
|
||||
v
|
||||
for v in versions.values()
|
||||
if v["event_id"] in main_events and v["state"] == "done"
|
||||
]
|
||||
assertions.append(
|
||||
{"name": "three_incremental_versions", "passed": len(good) == 3}
|
||||
)
|
||||
assertions.append(
|
||||
{
|
||||
"name": "repost_no_research",
|
||||
"passed": len(checkpoints) >= 2
|
||||
and checkpoints[0]["versions"] == checkpoints[1]["versions"],
|
||||
}
|
||||
)
|
||||
if not args.synthetic:
|
||||
from itertools import combinations
|
||||
|
||||
for a, b in combinations([463194, 463200, 463203, 463209, 463214], 2):
|
||||
assertions.append(
|
||||
{
|
||||
"name": "independent_foreign_ministry_topics",
|
||||
"news_ids": [a, b],
|
||||
"passed": not (links[a] & links[b]),
|
||||
}
|
||||
)
|
||||
report = {
|
||||
"quality_accepted": False,
|
||||
"automated_passed": all(a["passed"] for a in assertions)
|
||||
and all(p["state"] == "done" for p in packages)
|
||||
and all(e["state"] == "done" for e in tables["news_research_events"]),
|
||||
"human_review_required": [
|
||||
"事实与边界",
|
||||
"时间精度与统计期",
|
||||
"评级与冲突保留",
|
||||
"引用原文证据",
|
||||
],
|
||||
"sample_sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"group": "synthetic" if args.synthetic else "real",
|
||||
"original_news": news,
|
||||
"model_choices": {
|
||||
k: {f: v for f, v in config[k].items() if f != "provider_id"}
|
||||
for k in ("index", "research")
|
||||
},
|
||||
"request_count_total": int(args.budget.read_text()),
|
||||
"request_cap": 240,
|
||||
"not_completed_news_ids": [
|
||||
n["id"]
|
||||
for n in news
|
||||
if not any(
|
||||
p["news_id"] == n["id"] and p["state"] == "done" for p in packages
|
||||
)
|
||||
],
|
||||
"budget_blocked": {
|
||||
name: [
|
||||
row["id"]
|
||||
for row in tables[name]
|
||||
if "真实模型请求已达240次上限" in (row.get("error") or "")
|
||||
]
|
||||
for name in ("news_research_attempts", "news_research_versions")
|
||||
},
|
||||
"failed_processing": {
|
||||
name: [
|
||||
row["id"]
|
||||
for row in tables[name]
|
||||
if row["state"] == "failed"
|
||||
and "真实模型请求已达240次上限" not in (row.get("error") or "")
|
||||
]
|
||||
for name in ("news_research_attempts", "news_research_versions")
|
||||
},
|
||||
"assertions": assertions,
|
||||
"checkpoints": checkpoints,
|
||||
"tables": tables,
|
||||
"usage_note": "缺失usage不等于零消耗；请求数不是费用；人工复核后才可质量放行。",
|
||||
}
|
||||
args.output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, default=str)
|
||||
)
|
||||
args.output.chmod(0o600)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||
inputs.add_argument("--sample", type=Path)
|
||||
inputs.add_argument("--scenario-manifest", type=Path)
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
parser.add_argument("--finish", action="store_true")
|
||||
parser.add_argument("--request-limit", type=int)
|
||||
parser.add_argument("--repair-references", action="store_true")
|
||||
parser.add_argument("--credentials", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--budget", type=Path, required=True)
|
||||
parser.add_argument("--synthetic", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.scenario_manifest:
|
||||
from accept_news_scenario import run
|
||||
asyncio.run(run(args))
|
||||
else:
|
||||
if args.resume or args.finish or args.request_limit is not None:
|
||||
parser.error("续跑选项仅用于场景验收")
|
||||
asyncio.run(main(args))
|
||||
Reference in new issue
Block a user