Coordinate AI retries and recover shared provider outages

Retry transient requests after one and three minutes and coordinate provider recovery across processes with bounded probe intervals. Preserve completed results when recovery bookkeeping fails, keep concurrent failure accounting accurate, and expose retry state in the UI.

web/src/common/apis/chat/type.ts

@@ -51,7 +51,8 @@ export interface SendChatRequestData {
 }
 
 export type ChatStreamEvent
-  = | { name: "prompt", data: { estimated_tokens: number, context_window: number, provider: string } }
+  = | { name: "retry", data: { scope: string, attempt: number, retry_at: string, reason: string } }
+    | { name: "prompt", data: { estimated_tokens: number, context_window: number, provider: string } }
     | { name: "thinking", data: Record<string, never> }
     | { name: "done", data: Record<string, never> }
     | { name: "context_updated", data: { id: number } }

web/src/common/apis/news-research/index.ts

@@ -3,7 +3,7 @@ import { request } from "@/http/axios"
 
 export const getSettingsApi = () => request<Research.Settings>({ url: "/risk/settings", method: "get" })
 export const saveSettingsApi = (data: Research.Config & { start_at: string, base_revision: number }) => request<null>({ url: "/risk/settings", method: "put", data })
-export const enableApi = () => request<Research.Coverage>({ url: "/risk/enable", method: "post" })
+export const enableApi = () => request<Research.Coverage>({ url: "/risk/enable", method: "post", timeout: 6200000 })
 export const getCoverageApi = (start_at: string) => request<Research.Coverage>({ url: "/risk/coverage", method: "get", params: { start_at } })
 export const pauseApi = () => request<null>({ url: "/risk/pause", method: "post" })
 export const createSkillsApi = () => request<{ index: number, research: number }>({ url: "/risk/skill-templates", method: "post" })

web/src/common/apis/news-research/type.ts

@@ -1,7 +1,7 @@
 export interface Keyword { canonical: string, aliases: string[], quote: string }
 export interface ModelConfig { provider_id: number | null, model: string, reasoning_effort: string | null, skill_id: number | null }
 export interface Config { index: ModelConfig, research: ModelConfig, concurrency: number }
-export interface QueueStatus { running: number, capacity: number, running_by_stage: { index: number, assign: number, research: number }, draining: boolean, realtime: Record<string, number>, backfill: Record<string, number>, events: { realtime: Record<string, number>, backfill: Record<string, number> }, idle_reasons: string[] }
+export interface QueueStatus { waiting_retry?: number, running: number, capacity: number, running_by_stage: { index: number, assign: number, research: number }, draining: boolean, realtime: Record<string, number>, backfill: Record<string, number>, events: { realtime: Record<string, number>, backfill: Record<string, number> }, idle_reasons: string[] }
 export interface Settings { control_revision: number, realtime_from: string | null, queue: QueueStatus, batch?: null, config: Partial<Config>, enabled: boolean, start_at: string }
 export interface Coverage { start_at: string, checked_at: string, total: number, unprocessed: number, missing: number, stale: number, pending: number, processing: number, failed: number, completed: number, outside: { unprocessed: number, stale: number, pending: number, processing: number, failed: number } }
 export interface Package { id: number, news_id: number, title: string, url: string, state: string, error: string | null, keywords: Keyword[], events: { id: number, title: string, reason: string, quote: string }[] }

web/src/common/apis/settings/index.ts

@@ -45,13 +45,13 @@ export function deleteProviderApi(id: number) {
   })
 }
 
-/** 测试模型连接；按后端 SDK 重试上限等待 */
+/** 测试模型连接；按单请求重试上限等待 */
 export function testProviderApi(id: number) {
   return request<Settings.TestProviderResponseData>({
     url: `/settings/providers/${id}/test`,
     method: "post",
     data: null,
-    timeout: 1900000
+    timeout: 2100000
   })
 }
 
@@ -89,3 +89,6 @@ export function testExchangeApi() {
     timeout: 30000
   })
 }
+
+export interface ProviderAvailability { provider_id: number, name: string, state: string, retry_at: string | null, level: number }
+export const listProviderAvailabilityApi = () => request<ProviderAvailability[]>({ url: "/settings/provider-availability", method: "get", notify: false })

web/src/common/components/AiAvailability/index.vue

@@ -0,0 +1,40 @@
+<script setup lang="ts">
+import type { ProviderAvailability } from "@@/apis/settings"
+import { listProviderAvailabilityApi } from "@@/apis/settings"
+import { useSessionScope } from "@@/composables/useSessionScope"
+
+const scope = useSessionScope()
+const states = ref<ProviderAvailability[]>([])
+const failed = ref(false)
+let timer: ReturnType<typeof setTimeout> | undefined
+let closed = false
+async function refresh() {
+  try {
+    const rows = await listProviderAvailabilityApi()
+    if (closed) return
+    scope.assertCurrent()
+    states.value = rows.filter(row => row.state !== "normal")
+    failed.value = false
+  } catch (error) {
+    if (!closed) {
+      failed.value = true
+      console.error(error)
+    }
+  } finally {
+    if (!closed) timer = setTimeout(refresh, 5000)
+  }
+}
+onMounted(refresh)
+onBeforeUnmount(() => {
+  closed = true
+  clearTimeout(timer)
+})
+</script>
+
+<template>
+  <el-alert v-if="failed" class="mb-3" type="warning" :closable="false" title="模型接入状态读取失败" />
+  <el-alert v-for="row in states" :key="row.provider_id" class="mb-3" type="warning" :closable="false">
+    {{ row.name }}：{{ row.state === 'probing' ? '正在单请求试探恢复' : '接入暂时不可用，等待恢复' }}。
+    下次尝试 {{ row.retry_at ? new Date(row.retry_at).toLocaleString() : '—' }}；等待期间保留任务。
+  </el-alert>
+</template>

web/src/pages/chat/components/MessageList.vue

@@ -51,6 +51,9 @@ function prettyResult(result: string) {
             <span class="text-sm text-[var(--el-color-primary)] font-semibold">AlphaPulse AI</span>
             <span class="num text-slate-400">{{ item.time ? formatTime(item.time) : "" }}</span>
           </div>
+          <div v-if="item.retryNotice" class="text-xs text-amber-600">
+            {{ item.retryNotice }}
+          </div>
           <div v-if="item.thinking" class="flex items-center gap-2 text-xs text-slate-500">
             <span class="h-2 w-2 animate-pulse rounded-full bg-amber-500" />
             思考中…

web/src/pages/chat/index.vue

@@ -91,15 +91,21 @@ const onEvent: ChatStreamHandler = (event) => {
     estimatedTokens.value = data.estimated_tokens
     // OpenAI 格式推理期间没有任何推送，先显示等待状态直到出字
     currentAssistant().thinking = true
+  } else if (name === "retry") {
+    const item = currentAssistant()
+    item.thinking = false
+    item.retryNotice = `${data.reason}；下次尝试 ${new Date(data.retry_at).toLocaleString()}`
   } else if (name === "thinking") {
     currentAssistant().thinking = true
   } else if (name === "delta") {
     const item = currentAssistant()
     item.thinking = false
+    item.retryNotice = undefined
     item.text += data.text
   } else if (name === "usage") {
     // 一次模型调用结束；之后的工具调用挂在这条回复上，新的文字另起一条
     const item = currentAssistant()
+    item.retryNotice = undefined
     item.closed = true
     item.thinking = false
   } else if (name === "tool_call") {
@@ -124,6 +130,7 @@ const onEvent: ChatStreamHandler = (event) => {
   } else if (name === "error") {
     const item = currentAssistant()
     item.thinking = false
+    item.retryNotice = undefined
     item.error = data.message
   }
   scrollToBottom()

web/src/pages/chat/type.ts

@@ -17,6 +17,7 @@ export interface DisplayItem {
   closed?: boolean
   /** 生成中：模型正在思考，还没有输出文字 */
   thinking?: boolean
+  retryNotice?: string
   /** 已落库的消息时间；生成中的没有 */
   time?: string
 }

web/src/pages/home/index.vue

@@ -5,6 +5,7 @@ import type { SkillNode } from "@@/apis/skills/type"
 import * as api from "@@/apis/news-research"
 import { listProviderModelsApi, listProvidersApi } from "@@/apis/settings"
 import { listSkillTreeApi } from "@@/apis/skills"
+import AiAvailability from "@@/components/AiAvailability/index.vue"
 import { EFFORTS } from "@@/constants/ai"
 import dayjs from "dayjs"
 
@@ -372,6 +373,7 @@ onBeforeUnmount(() => {
 
 <template>
   <div class="p-4">
+    <AiAvailability />
     <div class="mb-4 flex flex-wrap items-center gap-3">
       <h1 class="text-xl font-semibold">
         新闻与事件研究
@@ -401,6 +403,7 @@ onBeforeUnmount(() => {
     </p>
     <p v-if="queue" class="mb-4 text-sm text-slate-500">
       正在处理 {{ queue.running }} / {{ queue.capacity }} 个任务：关键词提取 {{ queue.running_by_stage.index }}，事件归属 {{ queue.running_by_stage.assign }}，事件研究 {{ queue.running_by_stage.research }}。
+      其中等待重试 {{ queue.waiting_retry ?? 0 }} 个，等待不代表正在调用模型。
       实时队列：待索引 {{ (queue.realtime.missing ?? 0) + (queue.realtime.stale ?? 0) + (queue.realtime.index_pending ?? 0) }}，待归属 {{ queue.realtime.assign_pending ?? 0 }}，就绪研究 {{ queue.events.realtime.ready ?? 0 }}；
       回补队列：待索引 {{ (queue.backfill.missing ?? 0) + (queue.backfill.stale ?? 0) + (queue.backfill.index_pending ?? 0) }}，待归属 {{ queue.backfill.assign_pending ?? 0 }}，就绪研究 {{ queue.events.backfill.ready ?? 0 }}。
       <span v-if="queue.idle_reasons.length">{{ queue.idle_reasons.map(reason => idleReasonLabels[reason] ?? reason).join('；') }}。</span>

web/src/pages/settings/components/ProviderSettings.vue

@@ -2,6 +2,7 @@
 import type { Provider, ProviderForm } from "@@/apis/settings/type"
 import type { FormInstance, FormRules } from "element-plus"
 import { createProviderApi, deleteProviderApi, listProvidersApi, patchProviderApi, testProviderApi } from "@@/apis/settings"
+import AiAvailability from "@@/components/AiAvailability/index.vue"
 import { useSessionScope } from "@@/composables/useSessionScope"
 import { EFFORTS } from "@@/constants/ai"
 import { formatTime } from "@@/utils/text"
@@ -134,6 +135,7 @@ onMounted(load)
 
 <template>
   <section class="min-w-0">
+    <AiAvailability />
     <el-card shadow="never" body-class="p-0">
       <template #header>
         <div class="flex items-center justify-between">

web/src/pages/trading/index.vue

@@ -3,6 +3,7 @@ import type { Provider } from "@@/apis/settings/type"
 import type { SnapshotPosition, TradingStatus, TradingSymbol } from "@@/apis/trading/type"
 import { listProvidersApi } from "@@/apis/settings"
 import { deleteSymbolApi, getStatusApi, manualActionApi, patchSymbolApi, requestAnalysisApi, startTradingApi, stopTradingApi } from "@@/apis/trading"
+import AiAvailability from "@@/components/AiAvailability/index.vue"
 import { useSessionScope } from "@@/composables/useSessionScope"
 import { formatTime } from "@@/utils/text"
 import { Clock, Plus, Reading } from "@element-plus/icons-vue"
@@ -244,6 +245,7 @@ onMounted(() => listProvidersApi().then(list => providers.value = list))
 </script>
 
 <template>
+  <AiAvailability />
   <div v-if="!status" v-loading="loading" class="min-h-40">
     <el-result v-if="failed" icon="error" title="无法读取模拟盘状态">
       <template #extra>

web/tests/ai-availability.test.ts

@@ -0,0 +1,45 @@
+import assert from "node:assert/strict"
+import test from "node:test"
+import * as vue from "vue"
+import { moduleLoader } from "./module-loader.ts"
+
+test("接入状态卸载后丢弃延迟响应", async () => {
+  let unmount = () => {}
+  const pending = Promise.withResolvers<unknown[]>()
+  const load = moduleLoader({
+    "@@/apis/settings": { listProviderAvailabilityApi: () => pending.promise },
+    "@@/composables/useSessionScope": { useSessionScope: () => ({ assertCurrent() {} }) }
+  }, { ...vue, onMounted() {}, onBeforeUnmount(fn: () => void) {
+    unmount = fn
+  } })
+  const page = load("@@/components/AiAvailability/index.vue").default.setup({}, { expose() {} })
+  const request = page.refresh()
+  unmount()
+  pending.resolve([{ provider_id: 1, state: "waiting" }])
+  await request
+  assert.equal(page.states.value.length, 0)
+})
+
+test("接入状态只展示等待和试探，正常接入不显示故障", async () => {
+  let unmount = () => {}
+  const load = moduleLoader({
+    "@@/apis/settings": { listProviderAvailabilityApi: async () => [
+      { provider_id: 1, state: "normal" },
+      { provider_id: 2, state: "waiting", retry_at: "2026-11-06T10:00:00Z" },
+      { provider_id: 3, state: "probing", retry_at: "2026-11-06T10:00:00Z" }
+    ] },
+    "@@/composables/useSessionScope": { useSessionScope: () => ({ assertCurrent() {} }) }
+  }, { ...vue, onMounted() {}, onBeforeUnmount(fn: () => void) {
+    unmount = fn
+  } })
+  const page = load("@@/components/AiAvailability/index.vue").default.setup({}, { expose() {} })
+  try {
+    await page.refresh()
+    assert.equal(page.states.value.length, 2)
+    assert.equal(page.states.value[0].state, "waiting")
+    assert.equal(page.states.value[1].state, "probing")
+    assert.equal(page.failed.value, false)
+  } finally {
+    unmount()
+  }
+})