Fix long-input evidence extraction and trim assignment context

backend/src/alphapulse/news_research/pipeline.py

@@ -1,6 +1,7 @@
 """Evidence contracts and lossless context preparation for event research."""
 import json
-from typing import Literal
+import re
+from typing import Annotated, Literal
 from pydantic import BaseModel, Field, ConfigDict
 from alphapulse.news_research.time_contracts import PackageTimePoint
 from alphapulse.news_research.model import complete, fits_context, SubmissionFailure
@@ -68,55 +69,110 @@ def validate_event(result, articles):
         if claim.verdict != 'insufficient' and not claim.evidence_refs:
             raise ValueError('核验结论缺少原文证据')
 
-async def extract_articles(provider, system, articles):
-    pending = [(a['id'], a['title']+'\n'+(a.get('summary') or '')+'\n'+(a.get('content') or '')) for a in articles]
+class EvidenceSelection(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    facts: str = Field(min_length=1)
+    evidence_ids: list[Annotated[int, Field(ge=0, strict=True)]] = Field(min_length=1)
+
+
+SELECTION_RULES = '\n本次只输出中文facts和evidence_ids，选择本次输入提供的证据编号；不生成quote或article_id。程序将按编号回填逐字原文，不得选择无关证据。'
+
+
+def selected_extraction(selection, evidence):
+    ids = selection.evidence_ids
+    if len(ids) != len(set(ids)) or not set(ids) <= evidence.keys():
+        raise ValueError('提取结果含未知或重复证据编号')
+    return Extraction(facts=selection.facts, evidence_refs=[evidence[i] for i in ids])
+
+
+def source_passages(article):
+    # Separate source fields so no quote can span a synthesized title/body boundary.
+    # Sentence boundaries keep evidence readable; long sentences still retain every character.
+    passages = []
+    for field in ('title', 'summary', 'content'):
+        for sentence in re.split(r'(?<=[。！？])|(?<=[.!?])(?=[ \t])', article.get(field) or ''):
+            passages.extend({'field': field, 'text': sentence[start:start+500]}
+                            for start in range(0, len(sentence), 500))
+    return passages
+
+
+async def extract_articles(provider, system, articles, *, scopes=None):
+    pending = [(article, source_passages(article)) for article in articles]
     extracts, usages = [], []
+    rules = system + SELECTION_RULES
     try:
-        schema = Extraction
         while pending:
-            identifier, content = pending.pop(0)
-            part = {'task': 'extract', 'article_id': identifier, 'content': content}
-            if not fits_context(provider, system, part, schema):
-                if len(content) < 2:
-                    raise ValueError('模型上下文不足以读取分段')
-                middle = len(content)//2
-                pending[0:0] = [(identifier, content[:middle]), (identifier, content[middle:])]
+            article, passages = pending.pop(0)
+            identifier = article['id']
+            if not passages:
+                raise ValueError('文章缺少可读取原文')
+            part = {'task': 'extract', 'article_id': identifier,
+                    'packages': [p for p in (scopes or []) if p['news_id'] == identifier],
+                    'passages': [{'id': i, **passage} for i, passage in enumerate(passages)]}
+            if not fits_context(provider, rules, part, EvidenceSelection):
+                if len(passages) > 1:
+                    middle = len(passages)//2
+                    halves = passages[:middle], passages[middle:]
+                elif passages and len(passages[0]['text']) > 1:
+                    passage = passages[0]
+                    middle = len(passage['text'])//2
+                    halves = ([{**passage, 'text': passage['text'][:middle]}],
+                              [{**passage, 'text': passage['text'][middle:]}])
+                else:
+                    raise ValueError(f'文章{identifier}的归属边界及最小原文片段超出模型上下文')
+                pending[0:0] = [(article, half) for half in halves]
                 continue
-            extract, used = await complete(provider, system, part, schema)
+            selection, used = await complete(provider, rules, part, EvidenceSelection)
             usages.append(used)
-            validate_evidence(extract.evidence_refs, {a['id']: a for a in articles})
-            if any(r.article_id != identifier or r.quote not in content for r in extract.evidence_refs):
-                raise ValueError('分段证据越界')
-            if extract.evidence_refs:
-                extracts.append(extract.model_dump())
+            evidence = {i: Evidence(article_id=identifier, quote=p['text']) for i, p in enumerate(passages)}
+            extract = selected_extraction(selection, evidence)
+            validate_evidence(extract.evidence_refs, {identifier: article})
+            extracts.append(extract.model_dump())
         return extracts, usages
     except Exception as error:
-        raise SubmissionFailure(str(error), {'parts': usages, 'failed': getattr(error, 'usage', None)}) from error
+        raise SubmissionFailure(f'文章{identifier}长输入原文提取失败：{error}', {'parts': usages, 'failed': getattr(error, 'usage', None)}) from error
+
+
+def consolidation_input(extracts):
+    evidence, identities, parts = {}, {}, []
+    for extract in extracts:
+        ids = []
+        for ref in extract['evidence_refs']:
+            key = (ref['article_id'], ref['quote'])
+            if key not in identities:
+                identifier = len(evidence)
+                identities[key] = identifier
+                evidence[identifier] = Evidence.model_validate(ref)
+            ids.append(identities[key])
+        parts.append({'facts': extract['facts'], 'evidence_ids': ids})
+    return {'task': 'consolidate', 'extractions': parts,
+            'evidence': [{'id': i, **ref.model_dump()} for i, ref in evidence.items()]}, evidence
+
 
 async def compact_extractions(provider, system, payload, schema, fits=fits_context):
     usages = []
     try:
-        rules = system + '\nconsolidate任务只合并给定分段事实，保留冲突和时间线；每个article_id至少保留一条给定原文引用，不新增引用，压缩重复表述。'
+        rules = system + '\nconsolidate任务只合并给定分段事实，保留冲突和时间线；每个article_id至少保留一条给定原文证据，压缩重复表述。' + SELECTION_RULES
         while not fits(provider, system, payload, schema):
             extracts = payload['extractions']
             groups, group = [], []
             for extract in extracts:
-                if not fits_context(provider, rules, {'task': 'consolidate', 'extractions': [*group, extract]}, Extraction):
+                if not fits_context(provider, rules, consolidation_input([*group, extract])[0], EvidenceSelection):
                     if not group:
                         raise ValueError('模型上下文不足以归纳分段结果，请增加上下文配置')
                     groups.append(group); group = []
-                    if not fits_context(provider, rules, {'task': 'consolidate', 'extractions': [extract]}, Extraction):
+                    if not fits_context(provider, rules, consolidation_input([extract])[0], EvidenceSelection):
                         raise ValueError('模型上下文不足以归纳分段结果，请增加上下文配置')
                 group.append(extract)
             if group: groups.append(group)
             compacted = []
             for group in groups:
-                result, used = await complete(provider, rules, {'task': 'consolidate', 'extractions': group}, Extraction)
+                request, evidence = consolidation_input(group)
+                selection, used = await complete(provider, rules, request, EvidenceSelection)
                 usages.append(used)
-                refs = {(r['article_id'], r['quote']) for part in group for r in part['evidence_refs']}
-                actual = {(r.article_id, r.quote) for r in result.evidence_refs}
-                if not actual <= refs or {r[0] for r in actual} != {r[0] for r in refs}:
-                    raise ValueError('分层归纳遗漏来源或改写原文引用')
+                result = selected_extraction(selection, evidence)
+                if {ref.article_id for ref in result.evidence_refs} != {ref.article_id for ref in evidence.values()}:
+                    raise ValueError('分层归纳遗漏来源')
                 compacted.append(result.model_dump())
             if len(json.dumps(compacted, ensure_ascii=False).encode()) >= len(json.dumps(extracts, ensure_ascii=False).encode()):
                 raise ValueError('分层归纳未缩减输入，无法在模型上下文内完整汇总')