Replace temperature with reasoning effort and retry in-stream errors
- Providers get a default reasoning effort (low..max, empty = not sent); chat can override it per message. OpenAI sends reasoning_effort, Claude sends output_config.effort with adaptive thinking, and thinking blocks are replayed during the tool loop. - Show a thinking indicator while waiting for model output. - Surface in-stream upstream errors (e.g. overloaded) instead of breaking the SSE stream, and retry them up to 3 times when nothing has been emitted yet.
This commit is contained in:
1 parent
5db3ead883
commit
b549f3f441
9 files changed
+190
-51
No files matched your search
@@ -53,6 +53,8 @@ class SessionOut(BaseModel):
|
||||
class SendIn(BaseModel):
|
||||
text: Annotated[str, Field(min_length=1, max_length=20000)]
|
||||
provider_id: int | None = None
|
||||
# 本轮思考强度;为空时用模型配置里的默认值
|
||||
reasoning_effort: llm.Effort | None = None
|
||||
|
||||
|
||||
class ResetIn(BaseModel):
|
||||
@@ -121,7 +123,7 @@ async def get_chat_session(session: Annotated[AsyncSession, Depends(get_session)
|
||||
|
||||
@router.post("/messages", response_class=EventSourceResponse)
|
||||
async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
||||
"""事件:prompt、delta、tool_call、tool_result、skill_draft、context_updated、usage、error、done"""
|
||||
"""事件:prompt、thinking、delta、tool_call、tool_result、skill_draft、context_updated、usage、error、done"""
|
||||
if generating.locked():
|
||||
yield event("error", {"message": "上一轮还在生成,请等待完成或先停止"})
|
||||
return
|
||||
@@ -148,14 +150,17 @@ async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
||||
await save_messages(ChatMessage(session_id=current.id, role="user", content={"text": body.text}))
|
||||
|
||||
specs = tool_specs()
|
||||
effort = body.reasoning_effort or provider.reasoning_effort
|
||||
for round_no in range(MAX_TOOL_ROUNDS + 1):
|
||||
text = ""
|
||||
completion: llm.Completion | None = None
|
||||
try:
|
||||
async for item in llm.stream(provider, system, messages, specs if round_no < MAX_TOOL_ROUNDS else []):
|
||||
async for item in llm.stream(provider, system, messages, specs if round_no < MAX_TOOL_ROUNDS else [], effort):
|
||||
if isinstance(item, llm.TextDelta):
|
||||
text += item.text
|
||||
yield event("delta", {"text": item.text})
|
||||
elif isinstance(item, llm.ThinkingStarted):
|
||||
yield event("thinking", {})
|
||||
else:
|
||||
completion = item
|
||||
except llm.LlmError as e:
|
||||
@@ -170,6 +175,8 @@ async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
||||
raise
|
||||
|
||||
assistant = {"text": completion.text, "tool_calls": completion.tool_calls}
|
||||
if completion.thinking_blocks:
|
||||
assistant["thinking_blocks"] = completion.thinking_blocks
|
||||
await save_messages(ChatMessage(session_id=current.id, role="assistant", content=assistant, provider_id=provider.id, usage=completion.usage))
|
||||
messages.append({"role": "assistant", **assistant})
|
||||
yield event("usage", completion.usage)
|
||||
|
||||
@@ -30,7 +30,7 @@ class ProviderOut(BaseModel):
|
||||
model: str
|
||||
max_tokens: int
|
||||
context_window: int
|
||||
temperature: float | None
|
||||
reasoning_effort: llm.Effort | None
|
||||
is_default: bool
|
||||
updated_at: datetime
|
||||
|
||||
@@ -43,7 +43,7 @@ class ProviderIn(BaseModel):
|
||||
model: Annotated[str, Field(min_length=1, max_length=128)]
|
||||
max_tokens: Annotated[int, Field(ge=256, le=128000)] = 8192
|
||||
context_window: Annotated[int, Field(ge=4096, le=2000000)] = 200000
|
||||
temperature: Annotated[float | None, Field(ge=0, le=2)] = None
|
||||
reasoning_effort: llm.Effort | None = None
|
||||
is_default: bool = False
|
||||
|
||||
|
||||
@@ -56,7 +56,8 @@ class ProviderPatch(BaseModel):
|
||||
model: Annotated[str | None, Field(min_length=1, max_length=128)] = None
|
||||
max_tokens: Annotated[int | None, Field(ge=256, le=128000)] = None
|
||||
context_window: Annotated[int | None, Field(ge=4096, le=2000000)] = None
|
||||
temperature: Annotated[float | None, Field(ge=0, le=2)] = None
|
||||
# 显式传 null 表示不传思考强度
|
||||
reasoning_effort: llm.Effort | None = None
|
||||
is_default: bool | None = None
|
||||
|
||||
|
||||
@@ -111,9 +112,9 @@ async def create_provider(body: ProviderIn, session: Annotated[AsyncSession, Dep
|
||||
async def patch_provider(provider_id: int, body: ProviderPatch, session: Annotated[AsyncSession, Depends(get_session)]) -> ProviderOut:
|
||||
provider = await get_provider(session, provider_id)
|
||||
values = body.model_dump(exclude_unset=True, exclude={"api_key", "base_url", "is_default"})
|
||||
# temperature 显式传 null 表示清除,其他字段为 null 时忽略
|
||||
# reasoning_effort 显式传 null 表示清除,其他字段为 null 时忽略
|
||||
for field, value in values.items():
|
||||
if value is not None or field == "temperature":
|
||||
if value is not None or field == "reasoning_effort":
|
||||
setattr(provider, field, value)
|
||||
if body.base_url is not None:
|
||||
provider.base_url = str(body.base_url)
|
||||
@@ -147,7 +148,7 @@ async def test_provider(provider_id: int, session: Annotated[AsyncSession, Depen
|
||||
started = time.monotonic()
|
||||
try:
|
||||
completion = None
|
||||
async for event in llm.stream(provider, "只回复 OK。", [{"role": "user", "text": "ping"}], []):
|
||||
async for event in llm.stream(provider, "只回复 OK。", [{"role": "user", "text": "ping"}], [], provider.reasoning_effort):
|
||||
if isinstance(event, llm.Completion):
|
||||
completion = event
|
||||
except llm.LlmError as e:
|
||||
|
||||
Reference in new issue
Block a user