Replace temperature with reasoning effort and retry in-stream errors
- Providers get a default reasoning effort (low..max, empty = not sent); chat can override it per message. OpenAI sends reasoning_effort, Claude sends output_config.effort with adaptive thinking, and thinking blocks are replayed during the tool loop. - Show a thinking indicator while waiting for model output. - Surface in-stream upstream errors (e.g. overloaded) instead of breaking the SSE stream, and retry them up to 3 times when nothing has been emitted yet.
This commit is contained in:
1 parent
5db3ead883
commit
b549f3f441
9 files changed
+187
-48
No files matched your search
@@ -0,0 +1,31 @@
|
|||||||
|
"""provider reasoning effort replaces temperature
|
||||||
|
|
||||||
|
Revision ID: 5c2e8f1a7b3d
|
||||||
|
Revises: 8349ae014ddd
|
||||||
|
Create Date: 2026-10-14 14:00:00.000000
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '5c2e8f1a7b3d'
|
||||||
|
down_revision: Union[str, Sequence[str], None] = '8349ae014ddd'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Upgrade schema."""
|
||||||
|
# 用户决定不再设置 temperature(2026-10-14),改为思考强度
|
||||||
|
op.drop_column('ai_providers', 'temperature')
|
||||||
|
op.add_column('ai_providers', sa.Column('reasoning_effort', sa.String(length=8), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Downgrade schema."""
|
||||||
|
op.drop_column('ai_providers', 'reasoning_effort')
|
||||||
|
op.add_column('ai_providers', sa.Column('temperature', sa.Float(), nullable=True))
|
||||||
@@ -2,14 +2,16 @@
|
|||||||
|
|
||||||
消息使用与协议无关的格式(与 ChatMessage.content 一致,外加 role):
|
消息使用与协议无关的格式(与 ChatMessage.content 一致,外加 role):
|
||||||
{"role": "user", "text"}
|
{"role": "user", "text"}
|
||||||
{"role": "assistant", "text", "tool_calls": [{"id", "name", "arguments"}]}
|
{"role": "assistant", "text", "tool_calls": [{"id", "name", "arguments"}], "thinking_blocks"?: [...]}
|
||||||
{"role": "tool", "tool_call_id", "name", "result"}
|
{"role": "tool", "tool_call_id", "name", "result"}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any
|
from typing import Any, Literal
|
||||||
|
|
||||||
import anthropic
|
import anthropic
|
||||||
import openai
|
import openai
|
||||||
@@ -17,9 +19,17 @@ import openai
|
|||||||
from alphapulse.core.crypto import decrypt
|
from alphapulse.core.crypto import decrypt
|
||||||
from alphapulse.models import AiProvider
|
from alphapulse.models import AiProvider
|
||||||
|
|
||||||
TIMEOUT_SECONDS = 120
|
# 高思考强度时 OpenAI 格式在推理期间不推送任何内容,读超时要留足
|
||||||
|
TIMEOUT_SECONDS = 600
|
||||||
|
# 思考强度:两种协议共有的取值。OpenAI 另有 none/minimal,但实测经 Sub2API 会卡住后 502,不提供
|
||||||
|
Effort = Literal["low", "medium", "high", "xhigh", "max"]
|
||||||
# SDK 自带重试只针对连接错误、429 和 5xx
|
# SDK 自带重试只针对连接错误、429 和 5xx
|
||||||
MAX_RETRIES = 2
|
MAX_RETRIES = 2
|
||||||
|
# 流内错误(上游过载等)SDK 不重试,由这里重试;只在还没有输出内容时重试
|
||||||
|
STREAM_RETRIES = 3
|
||||||
|
STREAM_RETRY_DELAY_SECONDS = 2
|
||||||
|
|
||||||
|
logger = logging.getLogger("alphapulse.ai.llm")
|
||||||
|
|
||||||
|
|
||||||
class LlmError(Exception):
|
class LlmError(Exception):
|
||||||
@@ -39,6 +49,8 @@ class Completion:
|
|||||||
text: str = ""
|
text: str = ""
|
||||||
tool_calls: list[dict] = field(default_factory=list)
|
tool_calls: list[dict] = field(default_factory=list)
|
||||||
usage: dict = field(default_factory=dict)
|
usage: dict = field(default_factory=dict)
|
||||||
|
# Anthropic 的 thinking / redacted_thinking 块:工具调用循环中必须原样回传,否则接口报错
|
||||||
|
thinking_blocks: list[dict] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -46,27 +58,60 @@ class TextDelta:
|
|||||||
text: str
|
text: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ThinkingStarted:
|
||||||
|
"""模型开始思考(只有 Claude 格式会推送);前端据此显示「思考中」"""
|
||||||
|
|
||||||
|
|
||||||
def api_root(base_url: str) -> str:
|
def api_root(base_url: str) -> str:
|
||||||
"""用户可能填 https://host 或 https://host/v1,统一成不带 /v1 的根地址"""
|
"""用户可能填 https://host 或 https://host/v1,统一成不带 /v1 的根地址"""
|
||||||
root = base_url.rstrip("/")
|
root = base_url.rstrip("/")
|
||||||
return root.removesuffix("/v1")
|
return root.removesuffix("/v1")
|
||||||
|
|
||||||
|
|
||||||
async def stream(provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec]) -> AsyncIterator[TextDelta | Completion]:
|
async def stream(
|
||||||
"""先逐段产出 TextDelta,最后产出一个 Completion"""
|
provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec], effort: Effort | None = None,
|
||||||
|
) -> AsyncIterator[TextDelta | ThinkingStarted | Completion]:
|
||||||
|
"""先逐段产出 TextDelta / ThinkingStarted,最后产出一个 Completion。effort 为空表示不传,由模型决定"""
|
||||||
|
for attempt in range(STREAM_RETRIES + 1):
|
||||||
|
output = False
|
||||||
try:
|
try:
|
||||||
if provider.protocol == "openai":
|
if provider.protocol == "openai":
|
||||||
async for event in _stream_openai(provider, system, messages, tools):
|
events = _stream_openai(provider, system, messages, tools, effort)
|
||||||
yield event
|
|
||||||
else:
|
else:
|
||||||
async for event in _stream_anthropic(provider, system, messages, tools):
|
events = _stream_anthropic(provider, system, messages, tools, effort)
|
||||||
|
async for event in events:
|
||||||
|
output = True
|
||||||
yield event
|
yield event
|
||||||
except (openai.APIStatusError, anthropic.APIStatusError) as e:
|
return
|
||||||
raise LlmError(f"模型接口返回 {e.status_code}:{e.message}") from e
|
except (openai.APIError, anthropic.APIError) as e:
|
||||||
except (openai.APITimeoutError, anthropic.APITimeoutError) as e:
|
# 已经推送过内容再重试会重复输出,只重试还没有输出时的流内错误
|
||||||
raise LlmError(f"模型接口超时({TIMEOUT_SECONDS}s)") from e
|
if output or attempt == STREAM_RETRIES or not _in_stream_error(e):
|
||||||
except (openai.APIConnectionError, anthropic.APIConnectionError) as e:
|
raise _llm_error(e) from e
|
||||||
raise LlmError(f"无法连接模型接口:{e}") from e
|
logger.warning("模型流内出错,%ss 后第 %s 次重试:%s", STREAM_RETRY_DELAY_SECONDS, attempt + 1, _llm_error(e))
|
||||||
|
await asyncio.sleep(STREAM_RETRY_DELAY_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
def _in_stream_error(e: Exception) -> bool:
|
||||||
|
"""连接建立后上游在流内返回的错误事件(如 overloaded)。
|
||||||
|
OpenAI SDK 抛 APIError 基类;Anthropic SDK 抛 APIStatusError,状态码是流本身的 200"""
|
||||||
|
if isinstance(e, openai.APIError):
|
||||||
|
return not isinstance(e, (openai.APIStatusError, openai.APIConnectionError))
|
||||||
|
return isinstance(e, anthropic.APIStatusError) and e.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def _llm_error(e: Exception) -> LlmError:
|
||||||
|
if _in_stream_error(e):
|
||||||
|
error = e.body.get("error") if isinstance(e.body, dict) else None
|
||||||
|
message = error.get("message") if isinstance(error, dict) else None
|
||||||
|
return LlmError(f"模型接口出错:{message or e.message}")
|
||||||
|
if isinstance(e, (openai.APIStatusError, anthropic.APIStatusError)):
|
||||||
|
return LlmError(f"模型接口返回 {e.status_code}:{e.message}")
|
||||||
|
if isinstance(e, (openai.APITimeoutError, anthropic.APITimeoutError)):
|
||||||
|
return LlmError(f"模型接口超时({TIMEOUT_SECONDS}s)")
|
||||||
|
if isinstance(e, (openai.APIConnectionError, anthropic.APIConnectionError)):
|
||||||
|
return LlmError(f"无法连接模型接口:{e}")
|
||||||
|
return LlmError(f"模型接口出错:{e}")
|
||||||
|
|
||||||
|
|
||||||
# ---------- OpenAI ----------
|
# ---------- OpenAI ----------
|
||||||
@@ -92,7 +137,7 @@ def _openai_messages(system: str, messages: list[dict]) -> list[dict]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def _stream_openai(provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec]) -> AsyncIterator[TextDelta | Completion]:
|
async def _stream_openai(provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec], effort: Effort | None) -> AsyncIterator[TextDelta | Completion]:
|
||||||
client = openai.AsyncOpenAI(
|
client = openai.AsyncOpenAI(
|
||||||
api_key=decrypt(provider.api_key_encrypted), base_url=f"{api_root(provider.base_url)}/v1",
|
api_key=decrypt(provider.api_key_encrypted), base_url=f"{api_root(provider.base_url)}/v1",
|
||||||
timeout=TIMEOUT_SECONDS, max_retries=MAX_RETRIES,
|
timeout=TIMEOUT_SECONDS, max_retries=MAX_RETRIES,
|
||||||
@@ -100,8 +145,8 @@ async def _stream_openai(provider: AiProvider, system: str, messages: list[dict]
|
|||||||
kwargs: dict[str, Any] = {}
|
kwargs: dict[str, Any] = {}
|
||||||
if tools:
|
if tools:
|
||||||
kwargs["tools"] = [{"type": "function", "function": {"name": t.name, "description": t.description, "parameters": t.parameters}} for t in tools]
|
kwargs["tools"] = [{"type": "function", "function": {"name": t.name, "description": t.description, "parameters": t.parameters}} for t in tools]
|
||||||
if provider.temperature is not None:
|
if effort:
|
||||||
kwargs["temperature"] = provider.temperature
|
kwargs["reasoning_effort"] = effort
|
||||||
result = Completion()
|
result = Completion()
|
||||||
# 按 index 拼接分片到达的工具调用
|
# 按 index 拼接分片到达的工具调用
|
||||||
calls: dict[int, dict] = {}
|
calls: dict[int, dict] = {}
|
||||||
@@ -139,7 +184,10 @@ def _anthropic_messages(messages: list[dict]) -> list[dict]:
|
|||||||
if m["role"] == "user":
|
if m["role"] == "user":
|
||||||
out.append({"role": "user", "content": m["text"]})
|
out.append({"role": "user", "content": m["text"]})
|
||||||
elif m["role"] == "assistant":
|
elif m["role"] == "assistant":
|
||||||
blocks: list[dict] = [{"type": "text", "text": m["text"]}] if m["text"] else []
|
# 思考块必须排在最前,原样回传
|
||||||
|
blocks: list[dict] = list(m.get("thinking_blocks") or [])
|
||||||
|
if m["text"]:
|
||||||
|
blocks.append({"type": "text", "text": m["text"]})
|
||||||
blocks += [{"type": "tool_use", "id": c["id"], "name": c["name"], "input": c["arguments"]} for c in m.get("tool_calls") or []]
|
blocks += [{"type": "tool_use", "id": c["id"], "name": c["name"], "input": c["arguments"]} for c in m.get("tool_calls") or []]
|
||||||
# 空的 assistant 消息会被接口拒绝
|
# 空的 assistant 消息会被接口拒绝
|
||||||
if blocks:
|
if blocks:
|
||||||
@@ -154,7 +202,7 @@ def _anthropic_messages(messages: list[dict]) -> list[dict]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def _stream_anthropic(provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec]) -> AsyncIterator[TextDelta | Completion]:
|
async def _stream_anthropic(provider: AiProvider, system: str, messages: list[dict], tools: list[ToolSpec], effort: Effort | None) -> AsyncIterator[TextDelta | ThinkingStarted | Completion]:
|
||||||
client = anthropic.AsyncAnthropic(
|
client = anthropic.AsyncAnthropic(
|
||||||
api_key=decrypt(provider.api_key_encrypted), base_url=api_root(provider.base_url),
|
api_key=decrypt(provider.api_key_encrypted), base_url=api_root(provider.base_url),
|
||||||
timeout=TIMEOUT_SECONDS, max_retries=MAX_RETRIES,
|
timeout=TIMEOUT_SECONDS, max_retries=MAX_RETRIES,
|
||||||
@@ -162,11 +210,13 @@ async def _stream_anthropic(provider: AiProvider, system: str, messages: list[di
|
|||||||
kwargs: dict[str, Any] = {}
|
kwargs: dict[str, Any] = {}
|
||||||
if tools:
|
if tools:
|
||||||
kwargs["tools"] = [{"name": t.name, "description": t.description, "input_schema": t.parameters} for t in tools]
|
kwargs["tools"] = [{"name": t.name, "description": t.description, "input_schema": t.parameters} for t in tools]
|
||||||
if provider.temperature is not None:
|
if effort:
|
||||||
# 当前 SDK 不再提供 temperature 参数,按原始字段透传
|
# 思考强度通过 output_config.effort 控制,配合自适应思考(由模型决定是否、思考多久)
|
||||||
kwargs["extra_body"] = {"temperature": provider.temperature}
|
kwargs["output_config"] = {"effort": effort}
|
||||||
|
kwargs["thinking"] = {"type": "adaptive"}
|
||||||
result = Completion()
|
result = Completion()
|
||||||
blocks: dict[int, dict] = {}
|
blocks: dict[int, dict] = {}
|
||||||
|
thinking: dict[int, dict] = {}
|
||||||
async with client:
|
async with client:
|
||||||
events = await client.messages.create(
|
events = await client.messages.create(
|
||||||
model=provider.model, system=system, messages=_anthropic_messages(messages), max_tokens=provider.max_tokens,
|
model=provider.model, system=system, messages=_anthropic_messages(messages), max_tokens=provider.max_tokens,
|
||||||
@@ -177,15 +227,26 @@ async def _stream_anthropic(provider: AiProvider, system: str, messages: list[di
|
|||||||
result.usage["input_tokens"] = event.message.usage.input_tokens
|
result.usage["input_tokens"] = event.message.usage.input_tokens
|
||||||
elif event.type == "content_block_start" and event.content_block.type == "tool_use":
|
elif event.type == "content_block_start" and event.content_block.type == "tool_use":
|
||||||
blocks[event.index] = {"id": event.content_block.id, "name": event.content_block.name, "arguments": ""}
|
blocks[event.index] = {"id": event.content_block.id, "name": event.content_block.name, "arguments": ""}
|
||||||
|
elif event.type == "content_block_start" and event.content_block.type == "thinking":
|
||||||
|
thinking[event.index] = {"type": "thinking", "thinking": event.content_block.thinking, "signature": event.content_block.signature}
|
||||||
|
yield ThinkingStarted()
|
||||||
|
elif event.type == "content_block_start" and event.content_block.type == "redacted_thinking":
|
||||||
|
thinking[event.index] = {"type": "redacted_thinking", "data": event.content_block.data}
|
||||||
|
yield ThinkingStarted()
|
||||||
elif event.type == "content_block_delta":
|
elif event.type == "content_block_delta":
|
||||||
if event.delta.type == "text_delta":
|
if event.delta.type == "text_delta":
|
||||||
result.text += event.delta.text
|
result.text += event.delta.text
|
||||||
yield TextDelta(event.delta.text)
|
yield TextDelta(event.delta.text)
|
||||||
elif event.delta.type == "input_json_delta":
|
elif event.delta.type == "input_json_delta":
|
||||||
blocks[event.index]["arguments"] += event.delta.partial_json
|
blocks[event.index]["arguments"] += event.delta.partial_json
|
||||||
|
elif event.delta.type == "thinking_delta":
|
||||||
|
thinking[event.index]["thinking"] += event.delta.thinking
|
||||||
|
elif event.delta.type == "signature_delta":
|
||||||
|
thinking[event.index]["signature"] += event.delta.signature
|
||||||
elif event.type == "message_delta":
|
elif event.type == "message_delta":
|
||||||
result.usage["output_tokens"] = event.usage.output_tokens
|
result.usage["output_tokens"] = event.usage.output_tokens
|
||||||
result.tool_calls = [_parse_call(b["id"], b["name"], b["arguments"]) for _, b in sorted(blocks.items())]
|
result.tool_calls = [_parse_call(b["id"], b["name"], b["arguments"]) for _, b in sorted(blocks.items())]
|
||||||
|
result.thinking_blocks = [b for _, b in sorted(thinking.items())]
|
||||||
yield result
|
yield result
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -53,6 +53,8 @@ class SessionOut(BaseModel):
|
|||||||
class SendIn(BaseModel):
|
class SendIn(BaseModel):
|
||||||
text: Annotated[str, Field(min_length=1, max_length=20000)]
|
text: Annotated[str, Field(min_length=1, max_length=20000)]
|
||||||
provider_id: int | None = None
|
provider_id: int | None = None
|
||||||
|
# 本轮思考强度;为空时用模型配置里的默认值
|
||||||
|
reasoning_effort: llm.Effort | None = None
|
||||||
|
|
||||||
|
|
||||||
class ResetIn(BaseModel):
|
class ResetIn(BaseModel):
|
||||||
@@ -121,7 +123,7 @@ async def get_chat_session(session: Annotated[AsyncSession, Depends(get_session)
|
|||||||
|
|
||||||
@router.post("/messages", response_class=EventSourceResponse)
|
@router.post("/messages", response_class=EventSourceResponse)
|
||||||
async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
||||||
"""事件:prompt、delta、tool_call、tool_result、skill_draft、context_updated、usage、error、done"""
|
"""事件:prompt、thinking、delta、tool_call、tool_result、skill_draft、context_updated、usage、error、done"""
|
||||||
if generating.locked():
|
if generating.locked():
|
||||||
yield event("error", {"message": "上一轮还在生成,请等待完成或先停止"})
|
yield event("error", {"message": "上一轮还在生成,请等待完成或先停止"})
|
||||||
return
|
return
|
||||||
@@ -148,14 +150,17 @@ async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
|||||||
await save_messages(ChatMessage(session_id=current.id, role="user", content={"text": body.text}))
|
await save_messages(ChatMessage(session_id=current.id, role="user", content={"text": body.text}))
|
||||||
|
|
||||||
specs = tool_specs()
|
specs = tool_specs()
|
||||||
|
effort = body.reasoning_effort or provider.reasoning_effort
|
||||||
for round_no in range(MAX_TOOL_ROUNDS + 1):
|
for round_no in range(MAX_TOOL_ROUNDS + 1):
|
||||||
text = ""
|
text = ""
|
||||||
completion: llm.Completion | None = None
|
completion: llm.Completion | None = None
|
||||||
try:
|
try:
|
||||||
async for item in llm.stream(provider, system, messages, specs if round_no < MAX_TOOL_ROUNDS else []):
|
async for item in llm.stream(provider, system, messages, specs if round_no < MAX_TOOL_ROUNDS else [], effort):
|
||||||
if isinstance(item, llm.TextDelta):
|
if isinstance(item, llm.TextDelta):
|
||||||
text += item.text
|
text += item.text
|
||||||
yield event("delta", {"text": item.text})
|
yield event("delta", {"text": item.text})
|
||||||
|
elif isinstance(item, llm.ThinkingStarted):
|
||||||
|
yield event("thinking", {})
|
||||||
else:
|
else:
|
||||||
completion = item
|
completion = item
|
||||||
except llm.LlmError as e:
|
except llm.LlmError as e:
|
||||||
@@ -170,6 +175,8 @@ async def send_message(body: SendIn) -> AsyncIterator[ServerSentEvent]:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
assistant = {"text": completion.text, "tool_calls": completion.tool_calls}
|
assistant = {"text": completion.text, "tool_calls": completion.tool_calls}
|
||||||
|
if completion.thinking_blocks:
|
||||||
|
assistant["thinking_blocks"] = completion.thinking_blocks
|
||||||
await save_messages(ChatMessage(session_id=current.id, role="assistant", content=assistant, provider_id=provider.id, usage=completion.usage))
|
await save_messages(ChatMessage(session_id=current.id, role="assistant", content=assistant, provider_id=provider.id, usage=completion.usage))
|
||||||
messages.append({"role": "assistant", **assistant})
|
messages.append({"role": "assistant", **assistant})
|
||||||
yield event("usage", completion.usage)
|
yield event("usage", completion.usage)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class ProviderOut(BaseModel):
|
|||||||
model: str
|
model: str
|
||||||
max_tokens: int
|
max_tokens: int
|
||||||
context_window: int
|
context_window: int
|
||||||
temperature: float | None
|
reasoning_effort: llm.Effort | None
|
||||||
is_default: bool
|
is_default: bool
|
||||||
updated_at: datetime
|
updated_at: datetime
|
||||||
|
|
||||||
@@ -43,7 +43,7 @@ class ProviderIn(BaseModel):
|
|||||||
model: Annotated[str, Field(min_length=1, max_length=128)]
|
model: Annotated[str, Field(min_length=1, max_length=128)]
|
||||||
max_tokens: Annotated[int, Field(ge=256, le=128000)] = 8192
|
max_tokens: Annotated[int, Field(ge=256, le=128000)] = 8192
|
||||||
context_window: Annotated[int, Field(ge=4096, le=2000000)] = 200000
|
context_window: Annotated[int, Field(ge=4096, le=2000000)] = 200000
|
||||||
temperature: Annotated[float | None, Field(ge=0, le=2)] = None
|
reasoning_effort: llm.Effort | None = None
|
||||||
is_default: bool = False
|
is_default: bool = False
|
||||||
|
|
||||||
|
|
||||||
@@ -56,7 +56,8 @@ class ProviderPatch(BaseModel):
|
|||||||
model: Annotated[str | None, Field(min_length=1, max_length=128)] = None
|
model: Annotated[str | None, Field(min_length=1, max_length=128)] = None
|
||||||
max_tokens: Annotated[int | None, Field(ge=256, le=128000)] = None
|
max_tokens: Annotated[int | None, Field(ge=256, le=128000)] = None
|
||||||
context_window: Annotated[int | None, Field(ge=4096, le=2000000)] = None
|
context_window: Annotated[int | None, Field(ge=4096, le=2000000)] = None
|
||||||
temperature: Annotated[float | None, Field(ge=0, le=2)] = None
|
# 显式传 null 表示不传思考强度
|
||||||
|
reasoning_effort: llm.Effort | None = None
|
||||||
is_default: bool | None = None
|
is_default: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
@@ -111,9 +112,9 @@ async def create_provider(body: ProviderIn, session: Annotated[AsyncSession, Dep
|
|||||||
async def patch_provider(provider_id: int, body: ProviderPatch, session: Annotated[AsyncSession, Depends(get_session)]) -> ProviderOut:
|
async def patch_provider(provider_id: int, body: ProviderPatch, session: Annotated[AsyncSession, Depends(get_session)]) -> ProviderOut:
|
||||||
provider = await get_provider(session, provider_id)
|
provider = await get_provider(session, provider_id)
|
||||||
values = body.model_dump(exclude_unset=True, exclude={"api_key", "base_url", "is_default"})
|
values = body.model_dump(exclude_unset=True, exclude={"api_key", "base_url", "is_default"})
|
||||||
# temperature 显式传 null 表示清除,其他字段为 null 时忽略
|
# reasoning_effort 显式传 null 表示清除,其他字段为 null 时忽略
|
||||||
for field, value in values.items():
|
for field, value in values.items():
|
||||||
if value is not None or field == "temperature":
|
if value is not None or field == "reasoning_effort":
|
||||||
setattr(provider, field, value)
|
setattr(provider, field, value)
|
||||||
if body.base_url is not None:
|
if body.base_url is not None:
|
||||||
provider.base_url = str(body.base_url)
|
provider.base_url = str(body.base_url)
|
||||||
@@ -147,7 +148,7 @@ async def test_provider(provider_id: int, session: Annotated[AsyncSession, Depen
|
|||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
try:
|
try:
|
||||||
completion = None
|
completion = None
|
||||||
async for event in llm.stream(provider, "只回复 OK。", [{"role": "user", "text": "ping"}], []):
|
async for event in llm.stream(provider, "只回复 OK。", [{"role": "user", "text": "ping"}], [], provider.reasoning_effort):
|
||||||
if isinstance(event, llm.Completion):
|
if isinstance(event, llm.Completion):
|
||||||
completion = event
|
completion = event
|
||||||
except llm.LlmError as e:
|
except llm.LlmError as e:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import BigInteger, Boolean, DateTime, Float, ForeignKey, Integer, LargeBinary, String, Text, UniqueConstraint, func
|
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Integer, LargeBinary, String, Text, UniqueConstraint, func
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -23,7 +23,8 @@ class AiProvider(Base):
|
|||||||
max_tokens: Mapped[int] = mapped_column(Integer, default=8192)
|
max_tokens: Mapped[int] = mapped_column(Integer, default=8192)
|
||||||
# 上下文上限(token),超过时拒绝发送而不是静默截断
|
# 上下文上限(token),超过时拒绝发送而不是静默截断
|
||||||
context_window: Mapped[int] = mapped_column(Integer, default=200000)
|
context_window: Mapped[int] = mapped_column(Integer, default=200000)
|
||||||
temperature: Mapped[float | None] = mapped_column(Float)
|
# 默认思考强度;为空表示不传,由模型决定。Chat 可按轮覆盖
|
||||||
|
reasoning_effort: Mapped[str | None] = mapped_column(String(8))
|
||||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|||||||
@@ -2,6 +2,11 @@ import { http } from "@/http"
|
|||||||
|
|
||||||
export type Protocol = "openai" | "anthropic"
|
export type Protocol = "openai" | "anthropic"
|
||||||
|
|
||||||
|
/** 思考强度,与后端 llm.Effort 一致 */
|
||||||
|
export const EFFORTS = ["low", "medium", "high", "xhigh", "max"] as const
|
||||||
|
|
||||||
|
export type Effort = typeof EFFORTS[number]
|
||||||
|
|
||||||
export interface Provider {
|
export interface Provider {
|
||||||
id: number
|
id: number
|
||||||
name: string
|
name: string
|
||||||
@@ -11,7 +16,8 @@ export interface Provider {
|
|||||||
model: string
|
model: string
|
||||||
max_tokens: number
|
max_tokens: number
|
||||||
context_window: number
|
context_window: number
|
||||||
temperature: number | null
|
/** 为空表示不传,由模型决定 */
|
||||||
|
reasoning_effort: Effort | null
|
||||||
is_default: boolean
|
is_default: boolean
|
||||||
updated_at: string
|
updated_at: string
|
||||||
}
|
}
|
||||||
@@ -25,7 +31,8 @@ export interface ProviderForm {
|
|||||||
model: string
|
model: string
|
||||||
max_tokens: number
|
max_tokens: number
|
||||||
context_window: number
|
context_window: number
|
||||||
temperature: number | null
|
/** 为空表示不传,由模型决定 */
|
||||||
|
reasoning_effort: Effort | null
|
||||||
is_default: boolean
|
is_default: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,6 +60,6 @@ export function deleteProvider(id: number): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function testProvider(id: number): Promise<TestResult> {
|
export function testProvider(id: number): Promise<TestResult> {
|
||||||
// 模型接口超时 120s 且 SDK 会重试,前端放宽等待时间
|
// 模型接口读超时 600s 且 SDK 会重试 2 次,前端放宽等待时间
|
||||||
return http.post(`/settings/providers/${id}/test`, null, { timeout: 400000 })
|
return http.post(`/settings/providers/${id}/test`, null, { timeout: 1900000 })
|
||||||
}
|
}
|
||||||
@@ -18,6 +18,8 @@ export interface DisplayItem {
|
|||||||
error?: string
|
error?: string
|
||||||
/** 生成中:这次模型调用已结束,后续文字另起一条 */
|
/** 生成中:这次模型调用已结束,后续文字另起一条 */
|
||||||
closed?: boolean
|
closed?: boolean
|
||||||
|
/** 生成中:模型正在思考,还没有输出文字 */
|
||||||
|
thinking?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
defineProps<{ items: DisplayItem[] }>()
|
defineProps<{ items: DisplayItem[] }>()
|
||||||
@@ -53,6 +55,9 @@ function prettyResult(result: string) {
|
|||||||
</div>
|
</div>
|
||||||
<div v-else class="flex flex-col gap-2">
|
<div v-else class="flex flex-col gap-2">
|
||||||
<!-- markdown-it 已关闭 html,模型输出中的标签按文本显示 -->
|
<!-- markdown-it 已关闭 html,模型输出中的标签按文本显示 -->
|
||||||
|
<div v-if="item.thinking" class="text-xs text-[var(--el-text-color-secondary)]">
|
||||||
|
思考中…
|
||||||
|
</div>
|
||||||
<div v-if="item.text" class="markdown-body" v-html="renderMarkdown(item.text)" />
|
<div v-if="item.text" class="markdown-body" v-html="renderMarkdown(item.text)" />
|
||||||
<el-collapse v-if="item.toolCalls.length" class="tool-cards">
|
<el-collapse v-if="item.toolCalls.length" class="tool-cards">
|
||||||
<el-collapse-item v-for="call in item.toolCalls" :key="call.id" :name="call.id">
|
<el-collapse-item v-for="call in item.toolCalls" :key="call.id" :name="call.id">
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
<script lang="ts" setup>
|
<script lang="ts" setup>
|
||||||
import type { ChatMessage } from "@/apis/chat"
|
import type { ChatMessage } from "@/apis/chat"
|
||||||
import type { Provider } from "@/apis/settings"
|
import type { Effort, Provider } from "@/apis/settings"
|
||||||
import type { SkillNode } from "@/apis/skills"
|
import type { SkillNode } from "@/apis/skills"
|
||||||
import type { DisplayItem } from "@/pages/chat/components/MessageList.vue"
|
import type { DisplayItem } from "@/pages/chat/components/MessageList.vue"
|
||||||
import { ElNotification } from "element-plus"
|
import { ElNotification } from "element-plus"
|
||||||
import { getChatSession, resetChat } from "@/apis/chat"
|
import { getChatSession, resetChat } from "@/apis/chat"
|
||||||
import { listProviders } from "@/apis/settings"
|
import { EFFORTS, listProviders } from "@/apis/settings"
|
||||||
import { listSkillTree } from "@/apis/skills"
|
import { listSkillTree } from "@/apis/skills"
|
||||||
import SkillTree from "@/components/SkillTree.vue"
|
import SkillTree from "@/components/SkillTree.vue"
|
||||||
import SkillVersionDrawer from "@/components/SkillVersionDrawer.vue"
|
import SkillVersionDrawer from "@/components/SkillVersionDrawer.vue"
|
||||||
@@ -22,6 +22,8 @@ const contextTab = ref<InstanceType<typeof ContextTab>>()
|
|||||||
const providers = ref<Provider[]>([])
|
const providers = ref<Provider[]>([])
|
||||||
const providerId = usePersisted<number | null>("chat.providerId", null)
|
const providerId = usePersisted<number | null>("chat.providerId", null)
|
||||||
const provider = computed(() => providers.value.find(p => p.id === providerId.value))
|
const provider = computed(() => providers.value.find(p => p.id === providerId.value))
|
||||||
|
// 为空表示跟随模型配置里的默认值
|
||||||
|
const effort = usePersisted<Effort | null>("chat.effort", null)
|
||||||
|
|
||||||
const messages = ref<ChatMessage[]>([])
|
const messages = ref<ChatMessage[]>([])
|
||||||
const estimatedTokens = ref(0)
|
const estimatedTokens = ref(0)
|
||||||
@@ -94,11 +96,19 @@ function currentAssistant(): DisplayItem {
|
|||||||
function onEvent(name: string, data: any) {
|
function onEvent(name: string, data: any) {
|
||||||
if (name === "prompt") {
|
if (name === "prompt") {
|
||||||
estimatedTokens.value = data.estimated_tokens
|
estimatedTokens.value = data.estimated_tokens
|
||||||
|
// OpenAI 格式推理期间没有任何推送,先显示等待状态直到出字
|
||||||
|
currentAssistant().thinking = true
|
||||||
|
} else if (name === "thinking") {
|
||||||
|
currentAssistant().thinking = true
|
||||||
} else if (name === "delta") {
|
} else if (name === "delta") {
|
||||||
currentAssistant().text += data.text
|
const item = currentAssistant()
|
||||||
|
item.thinking = false
|
||||||
|
item.text += data.text
|
||||||
} else if (name === "usage") {
|
} else if (name === "usage") {
|
||||||
// 一次模型调用结束;之后的工具调用挂在这条回复上,新的文字另起一条
|
// 一次模型调用结束;之后的工具调用挂在这条回复上,新的文字另起一条
|
||||||
currentAssistant().closed = true
|
const item = currentAssistant()
|
||||||
|
item.closed = true
|
||||||
|
item.thinking = false
|
||||||
} else if (name === "tool_call") {
|
} else if (name === "tool_call") {
|
||||||
const last = [...liveItems.value].reverse().find(i => i.kind === "assistant") ?? currentAssistant()
|
const last = [...liveItems.value].reverse().find(i => i.kind === "assistant") ?? currentAssistant()
|
||||||
last.toolCalls.push({ id: data.id, name: data.name, arguments: data.arguments, result: null })
|
last.toolCalls.push({ id: data.id, name: data.name, arguments: data.arguments, result: null })
|
||||||
@@ -108,6 +118,10 @@ function onEvent(name: string, data: any) {
|
|||||||
if (call)
|
if (call)
|
||||||
call.result = data.preview
|
call.result = data.preview
|
||||||
}
|
}
|
||||||
|
// 工具都返回后进入下一次模型调用
|
||||||
|
const last = liveItems.value.at(-1)
|
||||||
|
if (last?.closed && last.toolCalls.every(c => c.result !== null))
|
||||||
|
currentAssistant().thinking = true
|
||||||
} else if (name === "skill_draft") {
|
} else if (name === "skill_draft") {
|
||||||
highlightIds.value = [...new Set([...highlightIds.value, data.skill_id])]
|
highlightIds.value = [...new Set([...highlightIds.value, data.skill_id])]
|
||||||
loadSkills()
|
loadSkills()
|
||||||
@@ -115,7 +129,9 @@ function onEvent(name: string, data: any) {
|
|||||||
} else if (name === "context_updated") {
|
} else if (name === "context_updated") {
|
||||||
contextTab.value?.refreshFromChat()
|
contextTab.value?.refreshFromChat()
|
||||||
} else if (name === "error") {
|
} else if (name === "error") {
|
||||||
currentAssistant().error = data.message
|
const item = currentAssistant()
|
||||||
|
item.thinking = false
|
||||||
|
item.error = data.message
|
||||||
}
|
}
|
||||||
scrollToBottom()
|
scrollToBottom()
|
||||||
}
|
}
|
||||||
@@ -131,11 +147,11 @@ async function submit() {
|
|||||||
input.value = ""
|
input.value = ""
|
||||||
liveItems.value = [{ key: `live-user${Date.now()}`, kind: "user", text, toolCalls: [] }]
|
liveItems.value = [{ key: `live-user${Date.now()}`, kind: "user", text, toolCalls: [] }]
|
||||||
scrollToBottom()
|
scrollToBottom()
|
||||||
await send({ text, provider_id: provider.value.id }, onEvent)
|
await send({ text, provider_id: provider.value.id, reasoning_effort: effort.value }, onEvent)
|
||||||
// 无论完成、出错还是停止,都以服务端保存的会话为准;保留本地的错误提示
|
// 无论完成、出错还是停止,都以服务端保存的会话为准;保留本地的错误提示
|
||||||
const errors = liveItems.value.filter(i => i.error)
|
const errors = liveItems.value.filter(i => i.error)
|
||||||
await loadSession()
|
await loadSession()
|
||||||
liveItems.value = errors.map(e => ({ ...e, text: "", toolCalls: [] }))
|
liveItems.value = errors.map(e => ({ ...e, text: "", toolCalls: [], thinking: false }))
|
||||||
}
|
}
|
||||||
|
|
||||||
function onKeydown(e: Event | KeyboardEvent) {
|
function onKeydown(e: Event | KeyboardEvent) {
|
||||||
@@ -222,6 +238,9 @@ loadSkills()
|
|||||||
<el-select v-model="providerId" placeholder="选择模型配置" class="w-60" :disabled="streaming">
|
<el-select v-model="providerId" placeholder="选择模型配置" class="w-60" :disabled="streaming">
|
||||||
<el-option v-for="p in providers" :key="p.id" :value="p.id" :label="`${p.name}(${p.model})`" />
|
<el-option v-for="p in providers" :key="p.id" :value="p.id" :label="`${p.name}(${p.model})`" />
|
||||||
</el-select>
|
</el-select>
|
||||||
|
<el-select v-model="effort" class="w-44" :disabled="streaming" clearable :value-on-clear="null" :placeholder="`思考强度:跟随配置(${provider?.reasoning_effort ?? '不设置'})`">
|
||||||
|
<el-option v-for="e in EFFORTS" :key="e" :value="e" :label="`思考强度:${e}`" />
|
||||||
|
</el-select>
|
||||||
<el-tooltip content="下一轮发给模型的 system prompt(含已发布 skills、CONTEXT.md)与历史消息的估算大小" placement="bottom">
|
<el-tooltip content="下一轮发给模型的 system prompt(含已发布 skills、CONTEXT.md)与历史消息的估算大小" placement="bottom">
|
||||||
<span class="text-xs" :class="tokenRatio > 0.8 ? 'text-red-500 font-bold' : 'text-[var(--el-text-color-secondary)]'">
|
<span class="text-xs" :class="tokenRatio > 0.8 ? 'text-red-500 font-bold' : 'text-[var(--el-text-color-secondary)]'">
|
||||||
上下文约 {{ estimatedTokens.toLocaleString() }}{{ provider ? ` / ${provider.context_window.toLocaleString()}` : "" }} token
|
上下文约 {{ estimatedTokens.toLocaleString() }}{{ provider ? ` / ${provider.context_window.toLocaleString()}` : "" }} token
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
<script lang="ts" setup>
|
<script lang="ts" setup>
|
||||||
import type { Provider, ProviderForm } from "@/apis/settings"
|
import type { Provider, ProviderForm } from "@/apis/settings"
|
||||||
import { createProvider, deleteProvider, listProviders, patchProvider, testProvider } from "@/apis/settings"
|
import { createProvider, deleteProvider, EFFORTS, listProviders, patchProvider, testProvider } from "@/apis/settings"
|
||||||
import { formatTime } from "@/utils/text"
|
import { formatTime } from "@/utils/text"
|
||||||
|
|
||||||
const PROTOCOL_LABELS = { openai: "OpenAI 格式(/v1/chat/completions)", anthropic: "Claude 格式(/v1/messages)" } as const
|
const PROTOCOL_LABELS = { openai: "OpenAI 格式(/v1/chat/completions)", anthropic: "Claude 格式(/v1/messages)" } as const
|
||||||
@@ -12,7 +12,7 @@ const saving = ref(false)
|
|||||||
const testing = ref<number | null>(null)
|
const testing = ref<number | null>(null)
|
||||||
|
|
||||||
function emptyForm(): ProviderForm {
|
function emptyForm(): ProviderForm {
|
||||||
return { name: "", protocol: "anthropic", base_url: "", api_key: "", model: "", max_tokens: 8192, context_window: 200000, temperature: null, is_default: false }
|
return { name: "", protocol: "anthropic", base_url: "", api_key: "", model: "", max_tokens: 8192, context_window: 200000, reasoning_effort: null, is_default: false }
|
||||||
}
|
}
|
||||||
const form = ref<ProviderForm>(emptyForm())
|
const form = ref<ProviderForm>(emptyForm())
|
||||||
|
|
||||||
@@ -106,6 +106,11 @@ load()
|
|||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="base_url" label="Base URL" min-width="200" />
|
<el-table-column prop="base_url" label="Base URL" min-width="200" />
|
||||||
<el-table-column prop="model" label="模型" min-width="150" />
|
<el-table-column prop="model" label="模型" min-width="150" />
|
||||||
|
<el-table-column label="思考强度" width="90">
|
||||||
|
<template #default="{ row }">
|
||||||
|
{{ (row as Provider).reasoning_effort ?? "-" }}
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="API Key" width="100">
|
<el-table-column label="API Key" width="100">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
****{{ row.api_key_hint }}
|
****{{ row.api_key_hint }}
|
||||||
@@ -169,9 +174,11 @@ load()
|
|||||||
<el-input-number v-model="form.context_window" :min="4096" :max="2000000" :step="10000" controls-position="right" />
|
<el-input-number v-model="form.context_window" :min="4096" :max="2000000" :step="10000" controls-position="right" />
|
||||||
<span class="ml-2 text-xs">token,超过时拒绝发送</span>
|
<span class="ml-2 text-xs">token,超过时拒绝发送</span>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="Temperature">
|
<el-form-item label="思考强度">
|
||||||
<el-input-number v-model="form.temperature" :min="0" :max="2" :step="0.1" :value-on-clear="null" controls-position="right" placeholder="默认" />
|
<el-select v-model="form.reasoning_effort" clearable placeholder="不设置" class="w-40" :value-on-clear="null">
|
||||||
<span class="ml-2 text-xs">留空使用模型默认值</span>
|
<el-option v-for="e in EFFORTS" :key="e" :value="e" :label="e" />
|
||||||
|
</el-select>
|
||||||
|
<span class="ml-2 text-xs">Chat 默认值,可在 Chat 页按轮调整;留空则不传,由模型决定。越高越慢、越贵</span>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="设为默认">
|
<el-form-item label="设为默认">
|
||||||
<el-switch v-model="form.is_default" />
|
<el-switch v-model="form.is_default" />
|
||||||
|
|||||||
Reference in new issue
Block a user