Harden background loops and API edge cases from code review
- crawler manager: retry transient errors inside each loop, never let log writes break supervision, run tasks in a TaskGroup so any escaped error restarts the container instead of leaving a half-dead process - market worker: tolerate symbols deleted mid-backfill, retry task selection and symbol queries, run loops in a TaskGroup - market WS: validate subscription messages and close the connection when either the receive or forward side ends - news pipeline: make first insert concurrency-safe with ON CONFLICT - market API: only proxy ticker/1s klines for added symbols, map Binance rate limits to 503 and upstream errors to 502 without blocking - news API: escape LIKE wildcards in search, return 422 for bad cursors - market page: cancel pending reconnect timer on unmount
This commit is contained in:
1 parent
aabc81ba5b
commit
554d224be2
7 files changed
+259
-114
No files matched your search
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from alphapulse.core.db import SessionLocal
|
||||
from alphapulse.core.redis import redis
|
||||
@@ -32,17 +33,26 @@ class NewsPipeline:
|
||||
spider = self.crawler.spider
|
||||
digest = content_hash(item)
|
||||
raw_response_id = item.pop("raw_response_id")
|
||||
locked = (
|
||||
select(NewsItem)
|
||||
.where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
|
||||
.with_for_update()
|
||||
)
|
||||
async with SessionLocal() as session:
|
||||
news = await session.scalar(
|
||||
select(NewsItem)
|
||||
.where(NewsItem.source_code == item["source_code"], NewsItem.external_id == item["external_id"])
|
||||
.with_for_update()
|
||||
)
|
||||
news = await session.scalar(locked)
|
||||
inserted_id = None
|
||||
if news is None:
|
||||
# 行不存在时 FOR UPDATE 锁不住任何东西;用 ON CONFLICT 处理同一条目被并发首次写入
|
||||
inserted_id = await session.scalar(
|
||||
pg_insert(NewsItem)
|
||||
.values(**item, content_hash=digest, revision_no=1)
|
||||
.on_conflict_do_nothing(index_elements=[NewsItem.source_code, NewsItem.external_id])
|
||||
.returning(NewsItem.id)
|
||||
)
|
||||
# 插入成功取新行;冲突说明另一处刚写入,按已存在条目走比较/修订流程
|
||||
news = await session.scalar(locked)
|
||||
if inserted_id is not None:
|
||||
kind = "new"
|
||||
news = NewsItem(**item, content_hash=digest, revision_no=1)
|
||||
session.add(news)
|
||||
await session.flush()
|
||||
elif news.content_hash != digest:
|
||||
kind = "revised"
|
||||
for field in UPDATABLE_FIELDS:
|
||||
|
||||
Reference in new issue
Block a user