Skip whitespace-only news extraction fragments
This commit is contained in:
1 parent
a208dfabe9
commit
4f4db88333
2 files changed
+45
No files matched your search
@@ -106,6 +106,11 @@ async def extract_articles(provider, system, articles, *, scopes=None):
|
||||
identifier = article['id']
|
||||
if not passages:
|
||||
raise ValueError('文章缺少可读取原文')
|
||||
if not any(p['text'].strip() for p in passages):
|
||||
if not any((article.get(field) or '').strip() for field in ('title', 'summary', 'content')):
|
||||
raise ValueError('文章缺少可读取原文')
|
||||
# Keep source characters intact, but never spend a call on whitespace alone.
|
||||
continue
|
||||
part = {'task': 'extract', 'article_id': identifier,
|
||||
'packages': [p for p in (scopes or []) if p['news_id'] == identifier],
|
||||
'passages': [{'id': i, **passage} for i, passage in enumerate(passages)]}
|
||||
|
||||
@@ -126,6 +126,46 @@ def test_single_long_sentence_splits_without_losing_characters(pipeline, monkeyp
|
||||
assert len(received) > 1 and ''.join(received) == article['content']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('content', ['正文。\n\n', ' 正文。 ', ' ' * 500 + '正文。' + '\n' * 500])
|
||||
def test_whitespace_fragments_never_trigger_model_calls(pipeline, monkeypatch, content):
|
||||
article = {'id': 9, 'title': '标题。', 'summary': '\t\n', 'content': content}
|
||||
original = article.copy()
|
||||
passages = pipeline.source_passages(article)
|
||||
for field in ('title', 'summary', 'content'):
|
||||
assert ''.join(p['text'] for p in passages if p['field'] == field) == article[field]
|
||||
# Force every sentence and long-sentence fragment through the splitting path.
|
||||
monkeypatch.setattr(pipeline, 'fits_context', lambda _p, _s, payload, _schema:
|
||||
len(payload['passages']) == 1 and len(payload['passages'][0]['text']) <= 2)
|
||||
received = []
|
||||
|
||||
async def complete(provider, rules, payload, schema):
|
||||
assert any(p['text'].strip() for p in payload['passages'])
|
||||
received.extend(payload['passages'])
|
||||
return schema(facts='保留新闻事实。', evidence_ids=[0]), {'input_tokens': 10}
|
||||
|
||||
monkeypatch.setattr(pipeline, 'complete', complete)
|
||||
extracts, usage = asyncio.run(pipeline.extract_articles(SimpleNamespace(), '规则', [article]))
|
||||
assert len(usage) == len(extracts) == len(received)
|
||||
assert {ref['article_id'] for part in extracts for ref in part['evidence_refs']} == {9}
|
||||
for field in ('title', 'summary', 'content'):
|
||||
sent = ''.join(p['text'] for p in received if p['field'] == field)
|
||||
assert ''.join(sent.split()) == ''.join(article[field].split())
|
||||
assert article == original
|
||||
pipeline.validate_evidence([pipeline.Evidence.model_validate(ref) for part in extracts
|
||||
for ref in part['evidence_refs']], {9: article})
|
||||
|
||||
|
||||
@pytest.mark.parametrize('content', ['', ' \n\t' * 500])
|
||||
def test_unreadable_article_fails_without_model_calls(pipeline, monkeypatch, content):
|
||||
from unittest.mock import AsyncMock
|
||||
complete = AsyncMock()
|
||||
monkeypatch.setattr(pipeline, 'complete', complete)
|
||||
with pytest.raises(pipeline.SubmissionFailure, match='文章9.*缺少可读取原文'):
|
||||
asyncio.run(pipeline.extract_articles(SimpleNamespace(), '规则',
|
||||
[{'id': 9, 'title': '', 'summary': '', 'content': content}]))
|
||||
complete.assert_not_awaited()
|
||||
|
||||
|
||||
def test_oversized_scope_fails_before_model_call_with_article_identity(pipeline, monkeypatch):
|
||||
from unittest.mock import AsyncMock
|
||||
complete = AsyncMock()
|
||||
|
||||
Reference in new issue
Block a user