Coordinate AI retries and recover shared provider outages
Retry transient requests after one and three minutes and coordinate provider recovery across processes with bounded probe intervals. Preserve completed results when recovery bookkeeping fails, keep concurrent failure accounting accurate, and expose retry state in the UI.
This commit is contained in:
1 parent
3fcd4c7fb8
commit
8766e829f9
27 files changed
+670
-108
No files matched your search
@@ -58,7 +58,7 @@ def test_model_requires_protocol_end_marker(trading, monkeypatch, protocol, comp
|
||||
asyncio.run(check())
|
||||
|
||||
|
||||
def test_connection_failure_uses_only_sdk_retry_budget(trading, monkeypatch):
|
||||
def test_connection_failure_has_one_retry_owner(trading, monkeypatch):
|
||||
from alphapulse.ai import llm
|
||||
attempts = 0
|
||||
def handle(request):
|
||||
@@ -66,12 +66,14 @@ def test_connection_failure_uses_only_sdk_retry_budget(trading, monkeypatch):
|
||||
attempts += 1
|
||||
raise httpx.ConnectError("测试连接失败", request=request)
|
||||
provider = install_model_transport(llm, monkeypatch, "openai", handle)
|
||||
from unittest.mock import AsyncMock
|
||||
monkeypatch.setattr(llm, "_wait_local", AsyncMock())
|
||||
async def check():
|
||||
with pytest.raises(llm.LlmError, match="无法连接"):
|
||||
async for _ in llm.stream(provider, "system", [], []):
|
||||
pass
|
||||
asyncio.run(check())
|
||||
assert attempts == llm.MAX_RETRIES + 1
|
||||
assert attempts == 3
|
||||
|
||||
|
||||
def test_pipeline_error_keeps_history_checkpoint_and_marks_run_failed(trading, monkeypatch):
|
||||
|
||||
Reference in new issue
Block a user