Preserve received calendar results when latest pages roll over
Keep previously captured BLS and Michigan releases valid when their rolling latest pages advance to a newer period or stage. Continue reporting missing historical releases explicitly and never substitute newer values. Verify unchanged values, first-arrival timestamps, revisions, and cleared source errors with isolated worker tests. Run all 74 calendar and history tests against the staged snapshot.
This commit is contained in:
1 parent
641351a002
commit
0886e74e21
3 files changed
+48
-2
No files matched your search
@@ -179,6 +179,8 @@ class CalendarWorker:
|
|||||||
return response, raw_id
|
return response, raw_id
|
||||||
if period < row.period or (period == row.period and stage == "preliminary" and row.stage == "final"):
|
if period < row.period or (period == row.period and stage == "preliminary" and row.stage == "final"):
|
||||||
return None
|
return None
|
||||||
|
if row.first_received_at is not None:
|
||||||
|
return None # 最新页已换期;保留已取得的当期值,不误报历史缺口。
|
||||||
raise ValueError("密歇根当前页已晚于所选历史期次,未取得当期公告")
|
raise ValueError("密歇根当前页已晚于所选历史期次,未取得当期公告")
|
||||||
if row.source == "bls":
|
if row.source == "bls":
|
||||||
code = "empsit" if row.family == "jobs" else row.family
|
code = "empsit" if row.family == "jobs" else row.family
|
||||||
@@ -202,6 +204,8 @@ class CalendarWorker:
|
|||||||
published = release_date(response.text).date()
|
published = release_date(response.text).date()
|
||||||
if row.source == "bls" and published < date:
|
if row.source == "bls" and published < date:
|
||||||
return None # 到点后官网仍缓存上一期,等待当期公告,不填旧值。
|
return None # 到点后官网仍缓存上一期,等待当期公告,不填旧值。
|
||||||
|
if row.source == "bls" and published > date and row.first_received_at is not None:
|
||||||
|
return None # 滚动最新页不再包含本期,不能据此否认已取得的结果。
|
||||||
if published != date:
|
if published != date:
|
||||||
raise ValueError("官方公告日期与日历不符,历史公布值尚未取得")
|
raise ValueError("官方公告日期与日历不符,历史公布值尚未取得")
|
||||||
return response, raw_id
|
return response, raw_id
|
||||||
|
|||||||
@@ -158,7 +158,8 @@ def test_result_date_mismatch_waits_without_using_stale_release(trading):
|
|||||||
response = httpx.Response(200, text='<html><body>October 14, 2026</body></html>',
|
response = httpx.Response(200, text='<html><body>October 14, 2026</body></html>',
|
||||||
request=httpx.Request("GET", "https://www.bls.gov/example"))
|
request=httpx.Request("GET", "https://www.bls.gov/example"))
|
||||||
worker.fetch = AsyncMock(return_value=(response, 1))
|
worker.fetch = AsyncMock(return_value=(response, 1))
|
||||||
row = SimpleNamespace(source="bls", family="cpi", scheduled_at=datetime(2026, 11, 10, 13, 30, tzinfo=UTC))
|
row = SimpleNamespace(source="bls", family="cpi", scheduled_at=datetime(2026, 11, 10, 13, 30, tzinfo=UTC),
|
||||||
|
first_received_at=None)
|
||||||
assert await worker.result_page(row) is None
|
assert await worker.result_page(row) is None
|
||||||
row.scheduled_at = datetime(2026, 9, 10, 12, 30, tzinfo=UTC)
|
row.scheduled_at = datetime(2026, 9, 10, 12, 30, tzinfo=UTC)
|
||||||
with pytest.raises(ValueError, match="日期与日历不符"):
|
with pytest.raises(ValueError, match="日期与日历不符"):
|
||||||
@@ -169,6 +170,46 @@ def test_result_date_mismatch_waits_without_using_stale_release(trading):
|
|||||||
asyncio.run(check())
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("source", ["bls", "umich"])
|
||||||
|
def test_rolling_page_rollover_preserves_received_result_without_source_error(trading, monkeypatch, source):
|
||||||
|
import httpx
|
||||||
|
from alphapulse.calendar import worker as module
|
||||||
|
|
||||||
|
async def check():
|
||||||
|
first_received = datetime(2026, 9, 24, 14, tzinfo=UTC)
|
||||||
|
values = {"cpi_yoy" if source == "bls" else "umich_sentiment": {"actual": "2.5", "previous": "2.4"}}
|
||||||
|
row = SimpleNamespace(id=1, source=source, family="cpi" if source == "bls" else "umich",
|
||||||
|
scheduled_at=first_received, first_received_at=first_received,
|
||||||
|
period="2026-09", stage="final" if source == "umich" else None,
|
||||||
|
values=values.copy(), revision_no=3, error="旧的采集错误")
|
||||||
|
session = SimpleNamespace(scalar=AsyncMock(return_value=row), commit=AsyncMock())
|
||||||
|
|
||||||
|
class Context:
|
||||||
|
async def __aenter__(self):
|
||||||
|
return session
|
||||||
|
async def __aexit__(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "SessionLocal", Context)
|
||||||
|
html = ('<html><body>October 22, 2026</body></html>' if source == "bls" else
|
||||||
|
'<h1>Preliminary Results for October 2026</h1><table id="front_table">'
|
||||||
|
'<tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr>'
|
||||||
|
'<tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>')
|
||||||
|
worker = module.CalendarWorker()
|
||||||
|
worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html,
|
||||||
|
request=httpx.Request("GET", "https://www.example.com/")), 9))
|
||||||
|
try:
|
||||||
|
await worker.poll_one(1)
|
||||||
|
assert row.error is None
|
||||||
|
assert row.first_received_at == first_received
|
||||||
|
assert row.values == values and row.revision_no == 3
|
||||||
|
session.commit.assert_awaited_once()
|
||||||
|
finally:
|
||||||
|
await worker.http.aclose()
|
||||||
|
await worker.bls.aclose()
|
||||||
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
def test_schedule_reschedule_and_withdrawal_preserve_past_and_revisions(trading):
|
def test_schedule_reschedule_and_withdrawal_preserve_past_and_revisions(trading):
|
||||||
from alphapulse.calendar.parsers import Scheduled
|
from alphapulse.calendar.parsers import Scheduled
|
||||||
from alphapulse.calendar.service import save_schedule
|
from alphapulse.calendar.service import save_schedule
|
||||||
|
|||||||
@@ -170,7 +170,8 @@ def test_umich_poll_waits_for_final_instead_of_reusing_preliminary(trading):
|
|||||||
<tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr>
|
<tr><td></td><td>Oct</td><td>Sep</td></tr><tr><td></td><td>2026</td><td>2026</td></tr>
|
||||||
<tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>"""
|
<tr><td>Index of Consumer Sentiment</td><td>49.0</td><td>48.1</td></tr></table>"""
|
||||||
worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html, request=httpx.Request("GET", "https://www.sca.isr.umich.edu/")), 1))
|
worker.fetch = AsyncMock(return_value=(httpx.Response(200, text=html, request=httpx.Request("GET", "https://www.sca.isr.umich.edu/")), 1))
|
||||||
row = SimpleNamespace(source="umich", family="umich", period="2026-10", stage="final", scheduled_at=datetime.now(UTC))
|
row = SimpleNamespace(source="umich", family="umich", period="2026-10", stage="final", scheduled_at=datetime.now(UTC),
|
||||||
|
first_received_at=None)
|
||||||
try:
|
try:
|
||||||
assert await worker.result_page(row) is None
|
assert await worker.result_page(row) is None
|
||||||
row.period = "2026-09"
|
row.period = "2026-09"
|
||||||
|
|||||||
Reference in new issue
Block a user