Clarify news risk evidence validation
This commit is contained in:
1 parent
b801c83c60
commit
b54863f40a
3 files changed
+79
-5
No files matched your search
@@ -58,7 +58,10 @@ async def enable(body: SettingsIn, session: DB):
|
|||||||
provider = await provider_for(session, settings)
|
provider = await provider_for(session, settings)
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise HTTPException(400, str(error)) from error
|
raise HTTPException(400, str(error)) from error
|
||||||
fixture = {'news': [{'id': 1, 'title': '测试消息:暂无新事件', 'summary': '', 'content': ''}], 'candidates': []}
|
fixture = {'news': [{'id': 1, 'title': '虚构测试新闻:示例交易平台发布维护通知',
|
||||||
|
'summary': '示例交易平台计划进行例行维护,期间暂停充提服务。',
|
||||||
|
'content': '本条为虚构测试样例,不代表真实市场事件。示例交易平台计划进行例行维护,'
|
||||||
|
'期间暂停充提服务。通知未提及资产损失,恢复时间尚未公布。'}], 'candidates': []}
|
||||||
from alphapulse.news_risk.service import validate_labels
|
from alphapulse.news_risk.service import validate_labels
|
||||||
try:
|
try:
|
||||||
result, _ = await complete(provider, SYSTEM, fixture, Labels)
|
result, _ = await complete(provider, SYSTEM, fixture, Labels)
|
||||||
|
|||||||
@@ -15,7 +15,10 @@ SYSTEM = '''你是加密市场新闻风险分析员,只使用输入证据。
|
|||||||
重大传闻可以高星,但必须标为待核实;正文不足用 unknown,不能冒充无风险。
|
重大传闻可以高星,但必须标为待核实;正文不足用 unknown,不能冒充无风险。
|
||||||
分别记录方向、证实状态、依据和不确定性。不得声称已独立核实;转载数量不等于独立证实。
|
分别记录方向、证实状态、依据和不确定性。不得声称已独立核实;转载数量不等于独立证实。
|
||||||
official 仅限输入明确来自相关官方机构的公告,媒体转述不能当作官方原文。
|
official 仅限输入明确来自相关官方机构的公告,媒体转述不能当作官方原文。
|
||||||
summary 用中文简要归纳;evidence 是输入原文的短引用;entities 使用统一名称。
|
summary 用中文简要归纳;entities 使用统一名称。
|
||||||
|
evidence 必须逐字复制对应新闻 title、summary 或 content 中一段非空、连续的原文(最多500字)。
|
||||||
|
只返回引用文字本身,不加外层引号、说明、翻译或省略号,不拼接多处片段;不能引用其他新闻或候选事件。
|
||||||
|
即使没有明显风险或信息不足,也必须从该条新闻复制原文作为 evidence;分析说明写入 reasoning。
|
||||||
只有相同实体、事件类型、发生时段及具体事实一致才归并到候选 event_id,否则 null。
|
只有相同实体、事件类型、发生时段及具体事实一致才归并到候选 event_id,否则 null。
|
||||||
事件 followup/refutation 使用候选 id;更正或辟谣可降低当前星级。不要仅因同名机构合并不同事件。
|
事件 followup/refutation 使用候选 id;更正或辟谣可降低当前星级。不要仅因同名机构合并不同事件。
|
||||||
所有输出都是研究判断,不下单、不修改任何交易设置。'''
|
所有输出都是研究判断,不下单、不修改任何交易设置。'''
|
||||||
@@ -34,7 +37,8 @@ class Label(BaseModel):
|
|||||||
confidence: Literal['low', 'medium', 'high']
|
confidence: Literal['low', 'medium', 'high']
|
||||||
reasoning: str = Field(min_length=1, max_length=2000)
|
reasoning: str = Field(min_length=1, max_length=2000)
|
||||||
uncertainty: str = Field(min_length=1, max_length=1000)
|
uncertainty: str = Field(min_length=1, max_length=1000)
|
||||||
evidence: str = Field(min_length=1, max_length=500)
|
evidence: str = Field(min_length=1, max_length=500,
|
||||||
|
description='逐字复制本条新闻 title、summary 或 content 中一段连续原文;不加外层引号,不改写、翻译或拼接')
|
||||||
event_id: int | None
|
event_id: int | None
|
||||||
same_event_as: int | None = Field(default=None, description="同批较早输入的新闻 id,仅同一事件时填写")
|
same_event_as: int | None = Field(default=None, description="同批较早输入的新闻 id,仅同一事件时填写")
|
||||||
|
|
||||||
@@ -100,8 +104,15 @@ def validate_labels(result, inputs, candidates):
|
|||||||
or positions[item.same_event_as] >= positions[item.id]):
|
or positions[item.same_event_as] >= positions[item.id]):
|
||||||
raise ValueError("同批事件引用必须指向较早的输入,且不能同时指定已有事件")
|
raise ValueError("同批事件引用必须指向较早的输入,且不能同时指定已有事件")
|
||||||
source = inputs[item.id]
|
source = inputs[item.id]
|
||||||
if item.evidence not in '\n'.join(str(source.get(key, '')) for key in ('title', 'summary', 'content')):
|
texts = [source.get(key) or '' for key in ('title', 'summary', 'content')]
|
||||||
raise ValueError('模型证据并非输入原文引用')
|
evidence = item.evidence.strip()
|
||||||
|
# Accept presentation quotes only; the stored evidence must remain a literal source substring.
|
||||||
|
if not any(evidence in text for text in texts) and len(evidence) >= 2 and (evidence[0], evidence[-1]) in (
|
||||||
|
('“', '”'), ('「', '」'), ('‘', '’'), ('"', '"'), ("'", "'")):
|
||||||
|
evidence = evidence[1:-1].strip()
|
||||||
|
if not evidence or not any(evidence in text for text in texts):
|
||||||
|
raise ValueError(f'模型证据并非输入原文引用(新闻 ID {item.id});请逐字复制该条新闻中的连续原文,不要改写或拼接')
|
||||||
|
item.evidence = evidence
|
||||||
if item.event_id is not None and item.event_id not in candidates:
|
if item.event_id is not None and item.event_id not in candidates:
|
||||||
raise ValueError('模型引用未知候选事件')
|
raise ValueError('模型引用未知候选事件')
|
||||||
if item.verification == 'insufficient' and item.direction != 'unknown':
|
if item.verification == 'insufficient' and item.direction != 'unknown':
|
||||||
|
|||||||
@@ -27,6 +27,66 @@ def test_extreme_unverified_event_preserves_uncertainty(risk):
|
|||||||
assert item.stars == 5 and item.verification == 'unverified'
|
assert item.stars == 5 and item.verification == 'unverified'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('evidence', ['安全事件', ' 安全事件\n', '“安全事件”', '「安全事件」', '‘安全事件’', '"安全事件"', "'安全事件'"])
|
||||||
|
def test_evidence_presentation_quotes_preserve_literal_source(risk, evidence):
|
||||||
|
item = label(risk).model_copy(update={'evidence': evidence})
|
||||||
|
risk.validate_labels(risk.Labels(items=[item]), {1: {'content': '媒体报道安全事件,规模待核实。'}}, set())
|
||||||
|
assert item.evidence == '安全事件'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('evidence', ['“资产已被盗”', '安全…事件', 'security incident', ' ', '“”', '标题\n正文'])
|
||||||
|
def test_evidence_must_be_nonempty_and_from_one_source_field(risk, evidence):
|
||||||
|
item = label(risk).model_copy(update={'evidence': evidence})
|
||||||
|
with pytest.raises(ValueError, match='模型证据并非输入原文引用'):
|
||||||
|
risk.validate_labels(risk.Labels(items=[item]), {1: {'title': '标题', 'summary': '正文', 'content': '安全事件'}}, set())
|
||||||
|
|
||||||
|
|
||||||
|
def test_evidence_retains_quotes_present_in_source(risk):
|
||||||
|
item = label(risk).model_copy(update={'evidence': '“安全事件”'})
|
||||||
|
risk.validate_labels(risk.Labels(items=[item]), {1: {'content': '媒体称这是“安全事件”。'}}, set())
|
||||||
|
assert item.evidence == '“安全事件”'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('valid', [True, False])
|
||||||
|
def test_enable_validates_evidence_before_saving_settings(risk, monkeypatch, valid):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
from fastapi import FastAPI
|
||||||
|
import httpx
|
||||||
|
from alphapulse.api import risk as api
|
||||||
|
|
||||||
|
session = SimpleNamespace(merge=AsyncMock(), commit=AsyncMock())
|
||||||
|
provider = SimpleNamespace(model='news-only')
|
||||||
|
monkeypatch.setattr(api, 'provider_for', AsyncMock(return_value=provider))
|
||||||
|
|
||||||
|
async def complete(actual_provider, system, payload, schema):
|
||||||
|
assert actual_provider is provider
|
||||||
|
news = payload['news'][0]
|
||||||
|
assert news['content'] and news['summary']
|
||||||
|
evidence = '“'+news['summary']+'”' if valid else '虚构的资产损失6%'
|
||||||
|
return risk.Labels(items=[label(risk).model_copy(update={'evidence': evidence})]), {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(api, 'complete', complete)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(api.router)
|
||||||
|
app.dependency_overrides[api.require_admin] = lambda: None
|
||||||
|
app.dependency_overrides[api.get_session] = lambda: session
|
||||||
|
|
||||||
|
async def check():
|
||||||
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url='http://test') as client:
|
||||||
|
response = await client.post('/api/risk/enable', json={'provider_id': 1, 'model': 'news-only'})
|
||||||
|
assert response.status_code == (200 if valid else 502), response.text
|
||||||
|
if valid:
|
||||||
|
saved = session.merge.call_args.args[0]
|
||||||
|
assert saved.enabled and saved.model == 'news-only'
|
||||||
|
session.commit.assert_awaited_once()
|
||||||
|
else:
|
||||||
|
assert '模型证据并非输入原文引用' in response.json()['detail']
|
||||||
|
session.merge.assert_not_awaited()
|
||||||
|
session.commit.assert_not_awaited()
|
||||||
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('kind', ['missing', 'duplicate', 'unknown', 'evidence', 'candidate', 'insufficient'])
|
@pytest.mark.parametrize('kind', ['missing', 'duplicate', 'unknown', 'evidence', 'candidate', 'insufficient'])
|
||||||
def test_invalid_results_are_rejected(risk, kind):
|
def test_invalid_results_are_rejected(risk, kind):
|
||||||
item = label(risk)
|
item = label(risk)
|
||||||
|
|||||||
Reference in new issue
Block a user