Harden background loops and API edge cases from code review

- crawler manager: retry transient errors inside each loop, never let
  log writes break supervision, run tasks in a TaskGroup so any escaped
  error restarts the container instead of leaving a half-dead process
- market worker: tolerate symbols deleted mid-backfill, retry task
  selection and symbol queries, run loops in a TaskGroup
- market WS: validate subscription messages and close the connection
  when either the receive or forward side ends
- news pipeline: make first insert concurrency-safe with ON CONFLICT
- market API: only proxy ticker/1s klines for added symbols, map Binance
  rate limits to 503 and upstream errors to 502 without blocking
- news API: escape LIKE wildcards in search, return 422 for bad cursors
- market page: cancel pending reconnect timer on unmount
This commit is contained in:
eezd committed 2026-10-13 13:51:00 +08:00
1 parent aabc81ba5b
commit 554d224be2
7 files changed
+259 -114

No files matched your search

+18 -8
View File
@@ -4,6 +4,7 @@ from typing import Any
from scrapy.crawler import Crawler
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from alphapulse.core.db import SessionLocal
from alphapulse.core.redis import redis
@@ -32,17 +33,26 @@ class NewsPipeline:
spider = self.crawler.spider
digest = content_hash(item)
raw_response_id = item.pop("raw_response_id")
locked = (
select(NewsItem)
.where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
.with_for_update()
)
async with SessionLocal() as session:
news = await session.scalar(
select(NewsItem)
.where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
.with_for_update()
)
news = await session.scalar(locked)
inserted_id = None
if news is None:
# 行不存在时 FOR UPDATE 锁不住任何东西;用 ON CONFLICT 处理同一条目被并发首次写入
inserted_id = await session.scalar(
pg_insert(NewsItem)
.values(**item, content_hash=digest, revision_no=1)
.on_conflict_do_nothing(index_elements=[NewsItem.source_code, NewsItem.external_id])
.returning(NewsItem.id)
)
# 插入成功取新行;冲突说明另一处刚写入,按已存在条目走比较/修订流程
news = await session.scalar(locked)
if inserted_id is not None:
kind = "new"
news = NewsItem(**item, content_hash=digest, revision_no=1)
session.add(news)
await session.flush()
elif news.content_hash != digest:
kind = "revised"
for field in UPDATABLE_FIELDS: