Skip whitespace-only news extraction fragments
Development CI / check-and-build (push) Successful in 3m22s
GitHub Git mirror / mirror (push) Successful in 6s
Development CI / deploy-development (push) Successful in 2m30s

This commit is contained in:
eezd committed 2026-11-05 23:35:12 +08:00
1 parent a208dfabe9
commit 4f4db88333
2 files changed
+45

No files matched your search

@@ -106,6 +106,11 @@ async def extract_articles(provider, system, articles, *, scopes=None):
identifier = article['id']
if not passages:
raise ValueError('文章缺少可读取原文')
if not any(p['text'].strip() for p in passages):
if not any((article.get(field) or '').strip() for field in ('title', 'summary', 'content')):
raise ValueError('文章缺少可读取原文')
# Keep source characters intact, but never spend a call on whitespace alone.
continue
part = {'task': 'extract', 'article_id': identifier,
'packages': [p for p in (scopes or []) if p['news_id'] == identifier],
'passages': [{'id': i, **passage} for i, passage in enumerate(passages)]}
+40
View File
@@ -126,6 +126,46 @@ def test_single_long_sentence_splits_without_losing_characters(pipeline, monkeyp
assert len(received) > 1 and ''.join(received) == article['content']
@pytest.mark.parametrize('content', ['正文。\n\n', ' 正文。 ', ' ' * 500 + '正文。' + '\n' * 500])
def test_whitespace_fragments_never_trigger_model_calls(pipeline, monkeypatch, content):
article = {'id': 9, 'title': '标题。', 'summary': '\t\n', 'content': content}
original = article.copy()
passages = pipeline.source_passages(article)
for field in ('title', 'summary', 'content'):
assert ''.join(p['text'] for p in passages if p['field'] == field) == article[field]
# Force every sentence and long-sentence fragment through the splitting path.
monkeypatch.setattr(pipeline, 'fits_context', lambda _p, _s, payload, _schema:
len(payload['passages']) == 1 and len(payload['passages'][0]['text']) <= 2)
received = []
async def complete(provider, rules, payload, schema):
assert any(p['text'].strip() for p in payload['passages'])
received.extend(payload['passages'])
return schema(facts='保留新闻事实。', evidence_ids=[0]), {'input_tokens': 10}
monkeypatch.setattr(pipeline, 'complete', complete)
extracts, usage = asyncio.run(pipeline.extract_articles(SimpleNamespace(), '规则', [article]))
assert len(usage) == len(extracts) == len(received)
assert {ref['article_id'] for part in extracts for ref in part['evidence_refs']} == {9}
for field in ('title', 'summary', 'content'):
sent = ''.join(p['text'] for p in received if p['field'] == field)
assert ''.join(sent.split()) == ''.join(article[field].split())
assert article == original
pipeline.validate_evidence([pipeline.Evidence.model_validate(ref) for part in extracts
for ref in part['evidence_refs']], {9: article})
@pytest.mark.parametrize('content', ['', ' \n\t' * 500])
def test_unreadable_article_fails_without_model_calls(pipeline, monkeypatch, content):
from unittest.mock import AsyncMock
complete = AsyncMock()
monkeypatch.setattr(pipeline, 'complete', complete)
with pytest.raises(pipeline.SubmissionFailure, match='文章9.*缺少可读取原文'):
asyncio.run(pipeline.extract_articles(SimpleNamespace(), '规则',
[{'id': 9, 'title': '', 'summary': '', 'content': content}]))
complete.assert_not_awaited()
def test_oversized_scope_fails_before_model_call_with_article_identity(pipeline, monkeypatch):
from unittest.mock import AsyncMock
complete = AsyncMock()