Coordinate AI retries and recover shared provider outages

Retry transient requests after one and three minutes and coordinate provider recovery across processes with bounded probe intervals. Preserve completed results when recovery bookkeeping fails, keep concurrent failure accounting accurate, and expose retry state in the UI.

backend/tests/conftest.py

@@ -11,4 +11,13 @@ def trading(monkeypatch):
     }.items():
         monkeypatch.setenv(key, value)
     from alphapulse.trading import executor
+    from unittest.mock import AsyncMock
+    from alphapulse.ai import llm, recovery
+    class HealthyGate:
+        async def call(self, operation, request_id='', epoch=-1):
+            return {'epoch': 0, 'level': 0, 'retry_at': 0, 'probe': False, 'admitted': True, 'now': 0}
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: HealthyGate())
+    monkeypatch.setattr(recovery, 'mark_waiting', AsyncMock())
+    monkeypatch.setattr(recovery, 'clear_waiting', AsyncMock())
+    monkeypatch.setattr(recovery, 'waiting_count', AsyncMock(return_value=0))
     return executor

backend/tests/test_ai_recovery.py

@@ -0,0 +1,160 @@
+"""隔离 Redis 验证实际 Lua 原子操作；不访问生产接入。"""
+import asyncio
+from types import SimpleNamespace as N
+from unittest.mock import AsyncMock
+
+import httpx
+import openai
+import pytest
+
+
+
+def test_global_failure_takes_over_local_retry_and_recovers(trading, monkeypatch):
+    from alphapulse.ai import llm
+    states = iter([
+        dict(level=0, epoch=0, admitted=True, probe=False, now=0, retry_at=0),
+        dict(level=1, epoch=1, admitted=False, probe=False, now=0, retry_at=180),
+        dict(level=1, epoch=1, admitted=False, probe=False, now=0, retry_at=180),
+        dict(level=1, epoch=1, admitted=True, probe=True, now=180, retry_at=180),
+    ])
+    async def coordinate(operation, *args):
+        if operation in ('enter', 'failure'):
+            return next(states)
+        return dict(level=0)
+    gate = N(call=AsyncMock(side_effect=coordinate))
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: gate)
+    monkeypatch.setattr(llm.asyncio, 'sleep', AsyncMock())
+    local = AsyncMock()
+    monkeypatch.setattr(llm, '_wait_local', local)
+    calls = []
+    async def adapter(*args):
+        calls.append(args)
+        if len(calls) == 1:
+            raise openai.RateLimitError('limited', response=httpx.Response(429, request=httpx.Request('POST', 'https://test.invalid')), body={})
+        yield llm.Completion(text='done')
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    async def run():
+        return [x async for x in llm.stream(N(protocol='openai'), '', [], [])]
+    result = asyncio.run(run())
+    assert result[-1].text == 'done' and len(calls) == 2
+    assert any(isinstance(x, llm.RetryStatus) and x.scope == 'upstream' for x in result)
+    local.assert_not_awaited()
+
+
+def test_coordinator_failure_does_not_send(trading, monkeypatch):
+    from alphapulse.ai import llm
+    adapter = AsyncMock()
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: N(call=AsyncMock(side_effect=RuntimeError('offline'))))
+    async def run():
+        with pytest.raises(llm.LlmError, match='协调不可用'):
+            async for _ in llm.stream(N(protocol='openai'), '', [], []):
+                pass
+    asyncio.run(run())
+    adapter.assert_not_called()
+
+
+def test_fail_fast_does_not_send_or_wait(trading, monkeypatch):
+    from alphapulse.ai import llm
+    gate = N(call=AsyncMock(return_value=dict(level=2, epoch=1, admitted=False, probe=False, now=100, retry_at=700)))
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: gate)
+    adapter = AsyncMock()
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    async def run():
+        token = llm.fail_fast.set(True)
+        try:
+            with pytest.raises(llm.LlmError, match='下次尝试'):
+                async for _ in llm.stream(N(protocol='openai'), '', [], []):
+                    pass
+        finally:
+            llm.fail_fast.reset(token)
+    asyncio.run(run())
+    adapter.assert_not_called()
+
+
+def test_completed_probe_is_confirmed_before_caller_returns(trading, monkeypatch):
+    from alphapulse.ai import llm
+    gate = N(call=AsyncMock(return_value=dict(level=0, epoch=1, admitted=True, probe=True, now=1, retry_at=0)))
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: gate)
+    async def adapter(*args):
+        yield llm.Completion(text='done')
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    async def run():
+        stream = llm.stream(N(protocol='openai'), '', [], [])
+        assert (await anext(stream)).text == 'done'
+        assert any(call.args[0] == 'success' for call in gate.call.await_args_list)
+        await stream.aclose()
+    asyncio.run(run())
+
+
+def test_retry_refreshes_only_current_request(trading, monkeypatch):
+    from alphapulse.ai import llm
+    calls = []
+    async def adapter(provider, system, messages, tools, effort):
+        calls.append(list(messages))
+        if len(calls) == 1:
+            raise httpx.ConnectError('unavailable')
+        yield llm.Completion(text='ok')
+    messages = [{'role': 'tool', 'tool_call_id': 'already-done', 'result': 'retained'}]
+    async def refresh():
+        messages.append({'role': 'user', 'text': 'fresh snapshot'})
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    monkeypatch.setattr(llm, '_wait_local', AsyncMock())
+    async def run():
+        return [x async for x in llm.stream(N(protocol='openai'), '', messages, [], before_retry=refresh)]
+    assert asyncio.run(run())[-1].text == 'ok'
+    assert len(calls) == 2 and calls[1][0] == calls[0][0]
+    assert calls[1][-1]['text'] == 'fresh snapshot'
+
+
+@pytest.mark.parametrize('failed_operation', ['success', 'release', 'both'])
+def test_completed_result_survives_coordination_write_failure(trading, monkeypatch, failed_operation):
+    from alphapulse.ai import llm
+    from alphapulse.news_research.model import complete
+    from alphapulse.news_research.contracts import IndexResult
+    calls = []
+
+    async def coordinate(operation, *args):
+        calls.append(operation)
+        if operation == failed_operation or failed_operation == 'both' and operation in ('success', 'release'):
+            raise RuntimeError('synthetic Redis outage')
+        return dict(level=0, epoch=0, admitted=True, probe=False, now=0, retry_at=0)
+
+    async def adapter(*args):
+        yield llm.Completion(tool_calls=[{'name': 'submit', 'id': '1', 'arguments': {
+            'keywords': [{'canonical': '新闻', 'aliases': ['news'], 'quote': 'news'}],
+        }}], usage={'input_tokens': 100, 'output_tokens': 10})
+
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: N(call=coordinate))
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    async def run():
+        return await complete(N(protocol='openai', max_tokens=100, context_window=250000,
+                                reasoning_effort=None), '', {}, IndexResult)
+    output, usage = asyncio.run(run())
+    assert output.keywords[0].canonical == '新闻'
+    assert usage == {'input_tokens': 100, 'output_tokens': 10}
+    assert calls.count('enter') == 1 and 'release' in calls
+
+
+def test_cancellation_survives_lease_release_failure(trading, monkeypatch):
+    from alphapulse.ai import llm
+    started = asyncio.Event()
+    async def coordinate(operation, *args):
+        if operation == 'release':
+            raise RuntimeError('synthetic Redis outage')
+        return dict(level=0, epoch=0, admitted=True, probe=False, now=0, retry_at=0)
+    async def adapter(*args):
+        started.set()
+        await asyncio.Event().wait()
+        yield
+    monkeypatch.setattr(llm, 'recovery_gate', lambda _: N(call=coordinate))
+    monkeypatch.setattr(llm, '_stream_openai', adapter)
+    async def run():
+        async def consume():
+            return [x async for x in llm.stream(N(protocol='openai'), '', [], [])]
+        task = asyncio.create_task(consume())
+        await started.wait()
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+    asyncio.run(run())

backend/tests/test_failure_paths.py

@@ -58,7 +58,7 @@ def test_model_requires_protocol_end_marker(trading, monkeypatch, protocol, comp
     asyncio.run(check())
 
 
-def test_connection_failure_uses_only_sdk_retry_budget(trading, monkeypatch):
+def test_connection_failure_has_one_retry_owner(trading, monkeypatch):
     from alphapulse.ai import llm
     attempts = 0
     def handle(request):
@@ -66,12 +66,14 @@ def test_connection_failure_uses_only_sdk_retry_budget(trading, monkeypatch):
         attempts += 1
         raise httpx.ConnectError("测试连接失败", request=request)
     provider = install_model_transport(llm, monkeypatch, "openai", handle)
+    from unittest.mock import AsyncMock
+    monkeypatch.setattr(llm, "_wait_local", AsyncMock())
     async def check():
         with pytest.raises(llm.LlmError, match="无法连接"):
             async for _ in llm.stream(provider, "system", [], []):
                 pass
     asyncio.run(check())
-    assert attempts == llm.MAX_RETRIES + 1
+    assert attempts == 3
 
 
 def test_pipeline_error_keeps_history_checkpoint_and_marks_run_failed(trading, monkeypatch):

backend/tests/test_news_transport.py

@@ -17,7 +17,7 @@ def test_news_transport_retry_budget_and_output_guard(trading, monkeypatch, mode
         calls.append(args)
         if mode == 'partial': yield llm.TextDelta('started')
         if mode == 'tool_fragment': llm._news_output()
-        if mode == 'success' and len(calls) == 3:
+        if mode in ('success', 'long_wait') and len(calls) == 3:
             yield llm.Completion(usage={'input_tokens': 1})
             return
         request = httpx.Request('POST', 'https://example.invalid/v1/chat/completions')
@@ -27,31 +27,22 @@ def test_news_transport_retry_budget_and_output_guard(trading, monkeypatch, mode
         raise openai.APIError('Concurrency limit exceeded for account', request, body={'error': {'message': 'Concurrency limit exceeded for account'}})
     monkeypatch.setattr(llm, '_stream_openai', adapter)
     sleep = AsyncMock()
-    monkeypatch.setattr(llm.asyncio, 'sleep', sleep)
+    monkeypatch.setattr(llm, '_wait_local', sleep)
     record = {}
     async def run():
         token = current.set(record)
         try: return [item async for item in model_stream(N(protocol='openai'), '', [], [], 'max')]
         finally: current.reset(token)
-    if mode == 'success': assert asyncio.run(run())[-1].usage == {'input_tokens': 1}
+    if mode in ('success', 'long_wait'): assert asyncio.run(run())[-1].usage == {'input_tokens': 1}
     else:
         with pytest.raises(llm.LlmError): asyncio.run(run())
-    retries = mode in ('success', 'exhausted')
+    retries = mode in ('success', 'exhausted', 'long_wait')
     assert len(calls) == (3 if retries else 1)
-    assert [call.args[0] for call in sleep.await_args_list] == ([30, 60] if retries else [])
+    assert [call.args[0] for call in sleep.await_args_list] == ([60, 180] if retries else [])
     assert llm.news_retry.get() is None
-    if retries: assert '2/2' in record['phase']
+    if retries: assert record['retry_attempt'] == 2
 
 
-def test_retry_after_seconds_and_dates(trading):
-    from alphapulse.ai import llm
-    from datetime import UTC, datetime, timedelta
-    from email.utils import format_datetime
-    request = httpx.Request('POST', 'https://example.invalid')
-    for value in ('90', format_datetime(datetime.now(UTC) + timedelta(seconds=95))):
-        error = openai.RateLimitError('wait', response=httpx.Response(429, headers={'retry-after': value}, request=request), body={})
-        assert 89 <= llm._news_retry_delay(error, 0) <= 96
-
 
 def test_cancelling_backoff_releases_context_and_progress(trading, monkeypatch):
     from alphapulse.ai import llm
@@ -61,11 +52,11 @@ def test_cancelling_backoff_releases_context_and_progress(trading, monkeypatch):
     async def adapter(*args):
         raise openai.APIError('Concurrency limit exceeded', httpx.Request('POST', 'https://example.invalid'), body={})
         yield
-    async def sleep(delay):
+    async def sleep(delay, coordinate):
         entered.set()
         await asyncio.Event().wait()
     monkeypatch.setattr(llm, '_stream_openai', adapter)
-    monkeypatch.setattr(llm.asyncio, 'sleep', sleep)
+    monkeypatch.setattr(llm, '_wait_local', sleep)
     async def run():
         async def consume():
             try:
@@ -82,7 +73,7 @@ def test_cancelling_backoff_releases_context_and_progress(trading, monkeypatch):
 
 @pytest.mark.parametrize('protocol', ['openai', 'anthropic'])
 @pytest.mark.parametrize('news', [True, False])
-def test_sdk_retry_is_disabled_only_for_news(trading, monkeypatch, protocol, news):
+def test_sdk_retry_is_disabled_for_every_request(trading, monkeypatch, protocol, news):
     from alphapulse.ai import llm
     options=[]
     class Client:
@@ -107,4 +98,4 @@ def test_sdk_retry_is_disabled_only_for_news(trading, monkeypatch, protocol, new
         try: return [item async for item in llm.stream(provider,'rules',[],[],None)]
         finally: llm.news_retry.reset(token)
     assert asyncio.run(run())[-1] == llm.Completion()
-    assert options[0]['max_retries'] == (0 if news else llm.MAX_RETRIES)
+    assert options[0]['max_retries'] == 0