Initial AlphaPulse implementation: news crawler and Binance klines
- Backend (FastAPI, SQLAlchemy 2, Alembic): single-admin cookie auth, news/crawler/market APIs with SSE and WebSocket streaming - Crawler (Scrapy + Scrapy-Redis): wscn_news, wscn_live, cls_depth, cls_telegraph spiders, idempotent upserts with revisions, raw response archiving, per-source subprocess supervision, restart and backoff - Market worker: full-history Binance spot/USD-M kline backfill, realtime WS stream, hourly and on-reconnect catch-up - Frontend (Vue 3, Element Plus, UnoCSS, KLineChart): news feed, crawler console with live logs, Binance-style chart page - Deploy: docker compose, nginx site, migrate and daily backup scripts
This commit is contained in:
eezd committed
commit
aabc81ba5b
88 files changed
+12603
No files matched your search
@@ -0,0 +1,153 @@
|
||||
<script lang="ts" setup>
|
||||
import type { CrawlLog, SourceInfo } from "@/apis/crawler"
|
||||
import type { NewsBrief } from "@/apis/news"
|
||||
import { patchSource } from "@/apis/crawler"
|
||||
import { listNews } from "@/apis/news"
|
||||
import { formatTime } from "@/utils/text"
|
||||
|
||||
const props = defineProps<{
|
||||
source: SourceInfo
|
||||
logs: CrawlLog[]
|
||||
restarting: boolean
|
||||
}>()
|
||||
const emit = defineEmits<{
|
||||
restart: []
|
||||
changed: []
|
||||
}>()
|
||||
|
||||
const LEVELS = ["INFO", "WARNING", "ERROR"]
|
||||
const levels = ref([...LEVELS])
|
||||
const autoScroll = ref(true)
|
||||
const logBox = ref<HTMLElement>()
|
||||
const latest = ref<NewsBrief[]>([])
|
||||
const interval = ref(props.source.interval_seconds)
|
||||
|
||||
const visibleLogs = computed(() => props.logs.filter(log => levels.value.includes(log.level) || (log.level === "CRITICAL" && levels.value.includes("ERROR"))))
|
||||
|
||||
watch(() => props.logs.length, async () => {
|
||||
if (!autoScroll.value)
|
||||
return
|
||||
await nextTick()
|
||||
logBox.value?.scrollTo({ top: logBox.value.scrollHeight })
|
||||
}, { immediate: true })
|
||||
|
||||
async function loadLatest() {
|
||||
latest.value = (await listNews({ source: [props.source.code], limit: 15 })).items
|
||||
}
|
||||
|
||||
// 本来源有新的“本轮完成”日志时刷新最新入库列表
|
||||
watch(() => props.logs.at(-1), (log) => {
|
||||
if (log?.message.startsWith("本轮完成") && !log.message.includes("新增 0，修订 0"))
|
||||
loadLatest()
|
||||
})
|
||||
|
||||
async function toggleEnabled(enabled: boolean | string | number) {
|
||||
await patchSource(props.source.code, { enabled: Boolean(enabled) })
|
||||
emit("changed")
|
||||
}
|
||||
|
||||
async function saveInterval() {
|
||||
if (interval.value === props.source.interval_seconds)
|
||||
return
|
||||
await patchSource(props.source.code, { interval_seconds: interval.value })
|
||||
ElMessage.success("已保存，下一轮起生效")
|
||||
emit("changed")
|
||||
}
|
||||
|
||||
const statusTag = computed(() => {
|
||||
const process = props.source.process
|
||||
if (props.restarting || process?.state === "restarting")
|
||||
return { type: "warning", text: "重启中" } as const
|
||||
if (!process)
|
||||
return { type: "danger", text: "管理进程未运行" } as const
|
||||
if (process.state === "stopped")
|
||||
return { type: "danger", text: "子进程未运行" } as const
|
||||
if (!props.source.enabled)
|
||||
return { type: "info", text: "已停用" } as const
|
||||
if (props.source.paused_until && new Date(props.source.paused_until) > new Date())
|
||||
return { type: "warning", text: "失败退避中" } as const
|
||||
return { type: "success", text: "运行中" } as const
|
||||
})
|
||||
|
||||
loadLatest()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="h-full flex flex-col gap-3">
|
||||
<el-descriptions :column="4" border size="small">
|
||||
<template #extra>
|
||||
<el-button type="warning" :loading="restarting" @click="emit('restart')">
|
||||
重启该来源
|
||||
</el-button>
|
||||
</template>
|
||||
<el-descriptions-item label="状态">
|
||||
<el-tag :type="statusTag.type" size="small">
|
||||
{{ statusTag.text }}
|
||||
</el-tag>
|
||||
<span v-if="source.process?.pid" class="ml-2 text-xs text-[var(--el-text-color-secondary)]">pid {{ source.process.pid }}</span>
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="上次成功">
|
||||
{{ formatTime(source.last_success_at) }}
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="连续失败">
|
||||
<span :class="{ 'text-red-500 font-bold': source.consecutive_failures > 0 }">{{ source.consecutive_failures }}</span>
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="下次运行">
|
||||
{{ formatTime(source.paused_until && source.paused_until > (source.process?.next_run_at ?? "") ? source.paused_until : source.process?.next_run_at) }}
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="今日新增 / 修订">
|
||||
{{ source.today_new }} / {{ source.today_revised }}
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="累计条目">
|
||||
{{ source.total_items }}
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="启用">
|
||||
<el-switch :model-value="source.enabled" @change="toggleEnabled" />
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="采集间隔">
|
||||
<el-input-number v-model="interval" :min="5" :max="86400" size="small" controls-position="right" class="w-28" @change="saveInterval" />
|
||||
秒
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item v-if="source.last_error" label="最近错误" :span="4">
|
||||
<span class="whitespace-pre-wrap text-xs text-red-500">{{ source.last_error }}</span>
|
||||
</el-descriptions-item>
|
||||
</el-descriptions>
|
||||
|
||||
<div class="min-h-0 flex flex-1 gap-3">
|
||||
<el-card class="w-2/5 flex flex-col" body-class="min-h-0 flex-1 overflow-auto" shadow="never">
|
||||
<template #header>
|
||||
<span>最新入库</span>
|
||||
<el-link :href="source.list_url" target="_blank" type="primary" class="ml-2">
|
||||
原站
|
||||
</el-link>
|
||||
</template>
|
||||
<div v-for="item in latest" :key="item.id" class="mb-2 text-sm">
|
||||
<span class="mr-2 text-xs text-[var(--el-text-color-secondary)]">{{ formatTime(item.published_at, "MM-DD HH:mm") }}</span>
|
||||
<el-link :href="item.url" target="_blank">
|
||||
{{ item.title || item.excerpt.slice(0, 60) }}
|
||||
</el-link>
|
||||
</div>
|
||||
</el-card>
|
||||
|
||||
<el-card class="w-3/5 flex flex-col" body-class="min-h-0 flex-1 flex flex-col" shadow="never">
|
||||
<template #header>
|
||||
<div class="flex items-center gap-3">
|
||||
<span>实时日志</span>
|
||||
<el-checkbox-group v-model="levels" size="small">
|
||||
<el-checkbox-button v-for="level in LEVELS" :key="level" :value="level">
|
||||
{{ level }}
|
||||
</el-checkbox-button>
|
||||
</el-checkbox-group>
|
||||
<el-switch v-model="autoScroll" active-text="自动滚动" size="small" />
|
||||
</div>
|
||||
</template>
|
||||
<div ref="logBox" class="min-h-0 flex-1 overflow-auto rounded bg-[#1e1e1e] p-2 text-xs text-[#d4d4d4] font-mono leading-5">
|
||||
<div v-for="log in visibleLogs" :key="log.id" class="whitespace-pre-wrap break-all">
|
||||
<span class="text-[#808080]">{{ formatTime(log.ts) }}</span>
|
||||
<span :class="{ 'text-yellow-400': log.level === 'WARNING', 'text-red-400': log.level === 'ERROR' || log.level === 'CRITICAL' }" class="mx-2">{{ log.level }}</span>{{ log.message }}
|
||||
</div>
|
||||
</div>
|
||||
</el-card>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,114 @@
|
||||
<script lang="ts" setup>
|
||||
import type { CrawlLog, SourcesInfo } from "@/apis/crawler"
|
||||
import { listLogs, listSources, restartCrawler } from "@/apis/crawler"
|
||||
import { useEventSource } from "@/composables/useEventSource"
|
||||
import SourcePanel from "@/pages/crawler/components/SourcePanel.vue"
|
||||
|
||||
const MAX_LOGS = 1000
|
||||
// 连续失败达到该次数时 TAB 标红
|
||||
const FAILURE_ALERT = 3
|
||||
const RESTART_TIMEOUT_MS = 60_000
|
||||
|
||||
const info = ref<SourcesInfo>()
|
||||
const activeTab = ref<string>()
|
||||
const logs = reactive<Record<string, CrawlLog[]>>({})
|
||||
// 正在等待重新上线的来源
|
||||
const restarting = reactive(new Set<string>())
|
||||
|
||||
async function refresh() {
|
||||
info.value = await listSources()
|
||||
activeTab.value ??= info.value.sources[0]?.code
|
||||
}
|
||||
|
||||
function appendLog(log: CrawlLog) {
|
||||
const list = (logs[log.source_code] ??= [])
|
||||
// SSE 可能早于历史日志到达，按 id 去重
|
||||
if (list.length && list.at(-1)!.id >= log.id)
|
||||
return
|
||||
list.push(log)
|
||||
if (list.length > MAX_LOGS)
|
||||
list.splice(0, list.length - MAX_LOGS)
|
||||
if (log.message.startsWith("子进程已启动"))
|
||||
restarting.delete(log.source_code)
|
||||
}
|
||||
|
||||
const { connected } = useEventSource<CrawlLog>("/api/crawler/logs/stream", "log", appendLog)
|
||||
|
||||
async function loadHistory(code: string) {
|
||||
const history = await listLogs(code, 200)
|
||||
const live = logs[code] ?? []
|
||||
const firstLiveId = live[0]?.id ?? Infinity
|
||||
logs[code] = [...history.filter(log => log.id < firstLiveId), ...live]
|
||||
}
|
||||
|
||||
async function restart(source: string) {
|
||||
const label = source === "all" ? "全部来源" : info.value?.sources.find(s => s.code === source)?.name
|
||||
await ElMessageBox.confirm(`确定重启 ${label} 的采集进程？`, "重启确认", { type: "warning" })
|
||||
const codes = source === "all" ? info.value!.sources.map(s => s.code) : [source]
|
||||
codes.forEach(code => restarting.add(code))
|
||||
try {
|
||||
await restartCrawler(source)
|
||||
} catch (error) {
|
||||
codes.forEach(code => restarting.delete(code))
|
||||
throw error
|
||||
}
|
||||
setTimeout(() => {
|
||||
const stuck = codes.filter(code => restarting.has(code))
|
||||
if (stuck.length) {
|
||||
stuck.forEach(code => restarting.delete(code))
|
||||
ElMessage.warning(`${stuck.join("、")} ${RESTART_TIMEOUT_MS / 1000}s 内未收到上线日志，请查看日志`)
|
||||
}
|
||||
}, RESTART_TIMEOUT_MS)
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number) {
|
||||
return bytes > 1024 ** 3 ? `${(bytes / 1024 ** 3).toFixed(2)} GB` : `${(bytes / 1024 ** 2).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
const timer = setInterval(refresh, 3000)
|
||||
onBeforeUnmount(() => clearInterval(timer))
|
||||
|
||||
refresh().then(() => Promise.all(info.value!.sources.map(s => loadHistory(s.code))))
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div v-if="info" class="h-full flex flex-col gap-3">
|
||||
<div class="flex items-center gap-3">
|
||||
<el-tag :type="info.manager_alive ? 'success' : 'danger'">
|
||||
{{ info.manager_alive ? "管理进程运行中" : "管理进程未运行" }}
|
||||
</el-tag>
|
||||
<el-tag :type="connected ? 'success' : 'danger'">
|
||||
{{ connected ? "日志推送已连接" : "日志推送断开" }}
|
||||
</el-tag>
|
||||
<span class="text-sm text-[var(--el-text-color-secondary)]">数据库 {{ formatBytes(info.db_size_bytes) }}</span>
|
||||
<div class="flex-1" />
|
||||
<el-button type="danger" :loading="restarting.size > 0" @click="restart('all')">
|
||||
重启全部采集
|
||||
</el-button>
|
||||
</div>
|
||||
<el-tabs v-model="activeTab" type="border-card" class="crawler-tabs min-h-0 flex flex-1 flex-col">
|
||||
<el-tab-pane v-for="source in info.sources" :key="source.code" :name="source.code" class="h-full">
|
||||
<template #label>
|
||||
<span :class="{ 'text-red-500 font-bold': source.consecutive_failures >= FAILURE_ALERT }">
|
||||
{{ source.name }}
|
||||
</span>
|
||||
<el-badge v-if="source.consecutive_failures >= FAILURE_ALERT" :value="source.consecutive_failures" class="ml-1" />
|
||||
</template>
|
||||
<SourcePanel
|
||||
:source="source"
|
||||
:logs="logs[source.code] ?? []"
|
||||
:restarting="restarting.has(source.code)"
|
||||
@restart="restart(source.code)"
|
||||
@changed="refresh"
|
||||
/>
|
||||
</el-tab-pane>
|
||||
</el-tabs>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.crawler-tabs :deep(.el-tabs__content) {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
}
|
||||
</style>
|
||||
Reference in new issue
Block a user