Preserve received calendar results when latest pages roll over

Keep previously captured BLS and Michigan releases valid when their rolling latest pages advance to a newer period or stage. Continue reporting missing historical releases explicitly and never substitute newer values.

Verify unchanged values, first-arrival timestamps, revisions, and cleared source errors with isolated worker tests. Run all 74 calendar and history tests against the staged snapshot.
This commit is contained in:
eezd committed 2026-10-26 14:24:23 +08:00
1 parent 641351a002
commit 0886e74e21
3 files changed
+48 -2

No files matched your search

@@ -179,6 +179,8 @@ class CalendarWorker:
return response, raw_id return response, raw_id
if period < row.period or (period == row.period and stage == "preliminary" and row.stage == "final"): if period < row.period or (period == row.period and stage == "preliminary" and row.stage == "final"):
return None return None
if row.first_received_at is not None:
return None # 最新页已换期;保留已取得的当期值,不误报历史缺口。
raise ValueError("密歇根当前页已晚于所选历史期次,未取得当期公告") raise ValueError("密歇根当前页已晚于所选历史期次,未取得当期公告")
if row.source == "bls": if row.source == "bls":
code = "empsit" if row.family == "jobs" else row.family code = "empsit" if row.family == "jobs" else row.family
@@ -202,6 +204,8 @@ class CalendarWorker:
published = release_date(response.text).date() published = release_date(response.text).date()
if row.source == "bls" and published < date: if row.source == "bls" and published < date:
return None # 到点后官网仍缓存上一期,等待当期公告,不填旧值。 return None # 到点后官网仍缓存上一期,等待当期公告,不填旧值。
if row.source == "bls" and published > date and row.first_received_at is not None:
return None # 滚动最新页不再包含本期,不能据此否认已取得的结果。
if published != date: if published != date:
raise ValueError("官方公告日期与日历不符,历史公布值尚未取得") raise ValueError("官方公告日期与日历不符,历史公布值尚未取得")
return response, raw_id return response, raw_id
+42 -1
View File
@@ -158,7 +158,8 @@ def test_result_date_mismatch_waits_without_using_stale_release(trading):
response = httpx.Response(200, text='<html><body>October 14, 2026</body></html>', response = httpx.Response(200, text='<html><body>October 14, 2026</body></html>',
request=httpx.Request("GET", "https://www.bls.gov/example")) request=httpx.Request("GET", "https://www.bls.gov/example"))
worker.fetch = AsyncMock(return_value=(response, 1)) worker.fetch = AsyncMock(return_value=(response, 1))
row = SimpleNamespace(source="bls", family="cpi", scheduled_at=datetime(2026, 11, 10, 13, 30, tzinfo=UTC)) row = SimpleNamespace(source="bls", family="cpi", scheduled_at=datetime(2026, 11, 10, 13, 30, tzinfo=UTC),
first_received_at=None)
assert await worker.result_page(row) is None assert await worker.result_page(row) is None
row.scheduled_at = datetime(2026, 9, 10, 12, 30, tzinfo=UTC) row.scheduled_at = datetime(2026, 9, 10, 12, 30, tzinfo=UTC)
with pytest.raises(ValueError, match="日期与日历不符"): with pytest.raises(ValueError, match="日期与日历不符"):
@@ -169,6 +170,46 @@ def test_result_date_mismatch_waits_without_using_stale_release(trading):
asyncio.run(check()) asyncio.run(check())
@pytest.mark.parametrize("source", ["bls", "umich"])
def test_rolling_page_rollover_preserves_received_result_without_source_error(trading, monkeypatch, source):
import httpx
from alphapulse.calendar import worker as module
async def check():
first_received = datetime(2026, 9, 24, 14, tzinfo=UTC)
values = {"cpi_yoy" if source == "bls" else "umich_sentiment": {"actual": "2.5", "previous": "2.4"}}
row = SimpleNamespace(id=1, source=source, family="cpi" if source == "bls" else "umich",
scheduled_at=first_received, first_received_at=first_received,
period="2026-09", stage="final" if source == "umich" else None,
values=values.copy(), revision_no=3, error="旧的采集错误")
session = SimpleNamespace(scalar=AsyncMock(return_value=row), commit=AsyncMock())
class Context:
async def __aenter__(self):
return session
async def __aexit__(self, *args):
pass
monkeypatch.setattr(module, "SessionLocal", Context)
html = ('<html><body>October 22, 2026</body></html>' if source == "bls" else
'<h1>Preliminary Results for October 2026</h1><table id="front_table">'
'<tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr>'
'<tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>')
worker = module.CalendarWorker()
worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html,
request=httpx.Request("GET", "https://www.example.com/")), 9))
try:
await worker.poll_one(1)
assert row.error is None
assert row.first_received_at == first_received
assert row.values == values and row.revision_no == 3
session.commit.assert_awaited_once()
finally:
await worker.http.aclose()
await worker.bls.aclose()
asyncio.run(check())
def test_schedule_reschedule_and_withdrawal_preserve_past_and_revisions(trading): def test_schedule_reschedule_and_withdrawal_preserve_past_and_revisions(trading):
from alphapulse.calendar.parsers import Scheduled from alphapulse.calendar.parsers import Scheduled
from alphapulse.calendar.service import save_schedule from alphapulse.calendar.service import save_schedule
+2 -1
View File
@@ -170,7 +170,8 @@ def test_umich_poll_waits_for_final_instead_of_reusing_preliminary(trading):
<tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr> <tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr>
<tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>""" <tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>"""
worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html, request=httpx.Request("GET", "https://www.sca.isr.umich.edu/")), 1)) worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html, request=httpx.Request("GET", "https://www.sca.isr.umich.edu/")), 1))
row = SimpleNamespace(source="umich", family="umich", period="2026-10", stage="final", scheduled_at=datetime.now(UTC)) row = SimpleNamespace(source="umich", family="umich", period="2026-10", stage="final", scheduled_at=datetime.now(UTC),
first_received_at=None)
try: try:
assert await worker.result_page(row) is None assert await worker.result_page(row) is None
row.period = "2026-09" row.period = "2026-09"