From 5fa1e37fd478071e37f8470fad2c7360dbb56178 Mon Sep 17 00:00:00 2001 From: eezd Date: Mon, 26 Oct 2026 12:01:29 +0800 Subject: [PATCH] Preserve public summaries for unavailable historical paid articles Validate Wallstreetcn business responses and retain list provenance for historical paid articles whose detail API returns code 50000. Keep other failures explicit and cover the boundary with eight isolated tests. --- .../src/alphapulse/crawler/spiders/wscn.py | 43 ++++++- backend/tests/test_wscn.py | 111 ++++++++++++++++++ 2 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_wscn.py diff --git a/backend/src/alphapulse/crawler/spiders/wscn.py b/backend/src/alphapulse/crawler/spiders/wscn.py index 4c514aa..0368263 100644 --- a/backend/src/alphapulse/crawler/spiders/wscn.py +++ b/backend/src/alphapulse/crawler/spiders/wscn.py @@ -1,6 +1,7 @@ import base64 import json from datetime import datetime +from html import escape from scrapy import Request from scrapy.http import Response @@ -10,6 +11,12 @@ from alphapulse.crawler.spiders.base import NewsSpider, ts API = "https://api-one-wscn.awtmt.com/apiv1/content" +def _api_data(body: dict) -> dict: + if body["code"] != 20000: + raise ValueError(f"华尔街见闻接口业务错误（code={body['code']}）：{body['message']}") + return body["data"] + + class WscnNewsSpider(NewsSpider): """华尔街见闻·文章：列表只有摘要，新条目或标题/摘要变化时再拉详情""" @@ -27,14 +34,21 @@ class WscnNewsSpider(NewsSpider): return self.list_request(base64.b64encode(cursor.encode()).decode(), 1, history=True) async def parse_list(self, response: Response): - data = response.json()["data"] + data = _api_data(response.json()) # 信息流里混有快讯，快讯由 wscn_live 负责 rows = [x["resource"] for x in data["items"] if x["resource_type"] == "article"] ids = [str(r["id"]) for r in rows] known = await self.known_items(ids) for r in rows: if known.get(str(r["id"])) != (r["title"], r["content_short"]): - yield self.request(f"{API}/articles/{r['id']}?extract=0", self.parse_detail, summary=r["content_short"]) + listed = {key: r[key] for key in ("id", "uri", "title", "display_time", "is_priced")} + listed["author"] = (r.get("author") or {}).get("display_name") + listed["source_name"] = r.get("source_name") + yield self.request( + f"{API}/articles/{r['id']}?extract=0", self.parse_detail, + summary=r["content_short"], listed_article=listed, + list_raw_response_id=response.meta["raw_response_id"], history=response.meta.get("history", False), + ) page = response.meta["page"] oldest = ts(min(r["display_time"] for r in rows)) if rows else None if response.meta.get("history"): @@ -44,11 +58,30 @@ class WscnNewsSpider(NewsSpider): yield self.list_request(data["next_cursor"], page + 1) async def parse_detail(self, response: Response): - a = response.json()["data"] + body = response.json() + listed = response.meta["listed_article"] + if body["code"] == 50000 and response.meta["history"] and listed["is_priced"]: + # 旧付费课程的详情和官网页面均可能失效；保存列表公开信息，不伪装成已取得正文。 + self.logger.warning("历史付费文章 %s 详情接口失败（code=%s）：%s；仅保存列表公开摘要", + listed["id"], body["code"], body["message"]) + summary = response.meta["summary"] + item = self.item( + response, external_id=str(listed["id"]), url=listed["uri"], title=listed["title"], + summary=summary, content_html=f"

{escape(summary)}

", + published_at=ts(listed["display_time"]), author=listed["author"], is_paywalled=True, + extra={"source_name": listed["source_name"], "is_priced": True, + "content_scope": "public_summary", "historical_backfill": True, + "detail_error": {"code": body["code"], "message": body["message"], + "raw_response_id": response.meta["raw_response_id"]}}, + ) + item["raw_response_id"] = response.meta["list_raw_response_id"] + yield item + return + a = _api_data(body) yield self.item( response, external_id=str(a["id"]), - url=f"https://wallstreetcn.com/articles/{a['id']}", + url=listed["uri"], title=a["title"], summary=response.meta["summary"], content_html=a["content"], @@ -75,7 +108,7 @@ class WscnLiveSpider(NewsSpider): return self.list_request(str(int(before.timestamp())), 1, history=True) async def parse_list(self, response: Response): - data = response.json()["data"] + data = _api_data(response.json()) rows = data["items"] # 先查已入库集合，再产出条目：否则管道可能先落库，导致误判为没有新条目而停止翻页 ids = [str(x["id"]) for x in rows] diff --git a/backend/tests/test_wscn.py b/backend/tests/test_wscn.py new file mode 100644 index 0000000..d30f1eb --- /dev/null +++ b/backend/tests/test_wscn.py @@ -0,0 +1,111 @@ +import asyncio +import json +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from scrapy.http import Request, TextResponse + + +@pytest.fixture +def wscn(trading): + from alphapulse.crawler.spiders.base import RunState + from alphapulse.crawler.spiders.wscn import WscnNewsSpider + + spider = WscnNewsSpider() + now = datetime(2026, 10, 22, tzinfo=UTC) + spider.run = RunState(uuid.uuid4(), "schedule", now, now - timedelta(hours=72), + history_target=datetime(2025, 1, 1, tzinfo=UTC), + history_position=datetime(2025, 12, 10, tzinfo=UTC)) + return spider + + +def json_response(request, body, raw_id): + request.meta["raw_response_id"] = raw_id + return TextResponse(request.url, request=request, body=json.dumps(body).encode(), encoding="utf-8") + + +async def detail_request(spider, monkeypatch, *, history=True, priced=True): + async def known_items(ids): + return {} + monkeypatch.setattr(spider, "known_items", known_items) + response = json_response(spider.list_request("", 1, history=history), { + "code": 20000, "data": {"items": [{"resource_type": "article", "resource": { + "id": 3759372, "uri": "https://wallstreetcn.com/premium/articles/3759372?layout=wscn-layout", + "title": "公开课程标题", "content_short": "公开摘要