fix: harden news analysis and deploy verified image tags

backend/tests/test_calendar_recovery.py

@@ -0,0 +1,205 @@
+"""Offline regression coverage for official-source recovery and throttling."""
+import asyncio
+from datetime import UTC, date, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+
+
+@pytest.mark.parametrize('source', ['eia', 'dol'])
+def test_archive_resolution_uses_discovered_links(trading, source):
+    from alphapulse.calendar.worker import CalendarWorker
+    async def run():
+        worker = CalendarWorker()
+        try:
+            archive = ('https://www.eia.gov/petroleum/supply/weekly/archive/2026/2026_10_28/wpsr_2026_10_28.php' if source == 'eia'
+                       else 'https://oui.doleta.gov/press/2026/102826.pdf')
+            response = lambda body, url: httpx.Response(200, text=body, request=httpx.Request('GET', url))
+            worker.index = AsyncMock(return_value=(response(f'<a href="{archive}">Report</a>', archive), 1))
+            pages = [(response('<a href="download/table1.csv">CSV</a>', archive), 2),
+                     (response('synthetic CSV', archive + '/download/table1.csv'), 3)] if source == 'eia' else [(response('synthetic PDF', archive), 2)]
+            worker.fetch = AsyncMock(side_effect=pages)
+            page, _ = await worker.archive_page(SimpleNamespace(source=source, scheduled_at=datetime(2026, 10, 28, 14, tzinfo=UTC)))
+            canonical = archive.rsplit('/', 1)[0] + '/download/table1.csv' if source == 'eia' else archive
+            assert page.extensions['calendar_url'] == canonical
+            assert worker.fetch.await_args.args == (source, canonical)
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+@pytest.mark.parametrize('status', [403, 429, 503])
+def test_host_cooldown_blocks_network_but_not_other_hosts(trading, monkeypatch, status):
+    from alphapulse.calendar import worker as module
+    async def run():
+        worker = module.CalendarWorker()
+        session = AsyncMock()
+        session.__aenter__.return_value = session
+        monkeypatch.setattr(module, 'SessionLocal', lambda: session)
+        monkeypatch.setattr(module, 'save_raw', AsyncMock(return_value=1))
+        worker.http.get = AsyncMock(side_effect=lambda url: httpx.Response(status if 'dol.gov' in url else 200,
+            headers={'Retry-After': '7200'}, request=httpx.Request('GET', url)))
+        try:
+            await worker.fetch('dol', 'https://www.dol.gov/ui/data.pdf')
+            with pytest.raises(module.SourceCooling) as error:
+                await worker.fetch('dol', 'https://www.dol.gov/ui/other.pdf')
+            assert error.value.retry_at >= datetime.now(UTC) + timedelta(seconds=7100)
+            await worker.fetch('dol', 'https://oui.doleta.gov/unemploy/archive.asp')
+            assert worker.http.get.await_count == 2
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+def test_adp_follows_weekly_page_to_monthly_and_stops_cycles(trading):
+    from alphapulse.calendar import worker as module, adp
+    async def run():
+        worker = module.CalendarWorker()
+        def page(title, next_link=''):
+            return httpx.Response(200, text=f'<div class="wd_item_list"><div class="wd_item"><span class="wd_date">Oct 01, 2026</span><div class="wd_title"><a href="/2026-10-01-ADP-National-Employment-Report-Test">{title}</a></div></div></div>{next_link}', request=httpx.Request('GET', adp.INDEX))
+        try:
+            worker.index = AsyncMock(side_effect=[(page('Weekly estimate', '<div class="wd_page_next"><a href="?o=9">Next</a></div>'), 1),
+                (page('ADP National Employment Report: Private-Sector Employment Increased by 1 Jobs in September'), 2)])
+            assert len(await worker.adp_links(date(2026, 9, 1), date(2026, 11, 1))) == 1
+            assert worker.index.await_count == 2
+            worker.index = AsyncMock(return_value=(page('Weekly estimate', '<div class="wd_page_next"><a href="/press-releases">Next</a></div>'), 1))
+            with pytest.raises(ValueError, match='循环'):
+                await worker.adp_links(date(2026, 9, 1), date(2026, 11, 1))
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+def test_same_poll_requests_are_coalesced_and_next_poll_refetches(trading):
+    from alphapulse.calendar.worker import CalendarWorker
+    async def run():
+        worker = CalendarWorker()
+        worker._fetch = AsyncMock(return_value=(object(), 7))
+        try:
+            worker.poll_cache = {}
+            pages = await asyncio.gather(*(worker.fetch('dol', 'https://www.dol.gov/ui/data.pdf') for _ in range(5)))
+            assert all(page == pages[0] for page in pages)
+            assert worker._fetch.await_count == 1
+            worker.poll_cache = {}
+            await worker.fetch('dol', 'https://www.dol.gov/ui/data.pdf')
+            assert worker._fetch.await_count == 2
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+@pytest.mark.parametrize('source', ['dol', 'eia'])
+def test_historical_rows_with_rolling_url_resolve_archives(trading, monkeypatch, source):
+    from alphapulse.calendar import worker as module, dol, eia
+    async def run():
+        worker = module.CalendarWorker()
+        try:
+            worker.archive_page = AsyncMock(side_effect=ValueError('archive requested'))
+            worker.fetch = AsyncMock(side_effect=AssertionError('rolling URL must not be fetched'))
+            worker.eia_schedule_body = 'already verified'
+            row = SimpleNamespace(source=source, scheduled_at=datetime(2020, 1, 1, tzinfo=UTC),
+                                  release_url=dol.LATEST if source == 'dol' else eia.LATEST + '?Signature=expired')
+            with pytest.raises(ValueError, match='archive requested'):
+                await worker.result_page(row)
+            worker.archive_page.assert_awaited_once_with(row)
+            worker.fetch.assert_not_awaited()
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+def test_missing_retry_after_uses_bounded_exponential_delay(trading, monkeypatch):
+    from alphapulse.calendar import worker as module
+    clock = [datetime(2026, 11, 2, tzinfo=UTC)]
+    class Clock(datetime):
+        @classmethod
+        def now(cls, tz=None):
+            return clock[0]
+    monkeypatch.setattr(module, 'datetime', Clock)
+    async def run():
+        worker = module.CalendarWorker()
+        session = AsyncMock()
+        session.__aenter__.return_value = session
+        monkeypatch.setattr(module, 'SessionLocal', lambda: session)
+        monkeypatch.setattr(module, 'save_raw', AsyncMock(return_value=1))
+        worker.http.get = AsyncMock(side_effect=lambda url: httpx.Response(429, request=httpx.Request('GET', url)))
+        try:
+            for expected in (60, 120, 240, 480, 960, 1920, 3600, 3600):
+                await worker.fetch('eurostat', 'https://ec.europa.eu/eurostat/')
+                retry = worker.cooldowns['ec.europa.eu']
+                assert (retry - clock[0]).total_seconds() == expected
+                clock[0] = retry + timedelta(seconds=1)
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+def test_adp_publication_is_visible_on_next_poll_even_with_old_cache(trading):
+    from alphapulse.calendar import adp
+    from alphapulse.calendar.worker import CalendarWorker
+    async def run():
+        worker = CalendarWorker()
+        def page(title):
+            return httpx.Response(200, text=f'<div class="wd_item_list"><div class="wd_item"><span class="wd_date">Nov 02, 2026</span><div class="wd_title"><a href="/2026-11-02-ADP-National-Employment-Report-Test">{title}</a></div></div></div>', request=httpx.Request('GET', adp.INDEX))
+        pending = (page('Weekly estimate'), 1)
+        published = (page('ADP National Employment Report: Private-Sector Employment Increased by 1 Jobs in October'), 2)
+        worker.index_cache['adp', adp.INDEX] = (datetime.now(UTC) + timedelta(hours=1), pending)
+        worker.fetch = AsyncMock(side_effect=[pending, published])
+        try:
+            assert await worker.adp_links(date(2026, 11, 2), date(2026, 11, 2)) == []
+            assert len(await worker.adp_links(date(2026, 11, 2), date(2026, 11, 2))) == 1
+            assert worker.fetch.await_count == 2
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+@pytest.mark.parametrize('source', ['dol', 'eia'])
+@pytest.mark.parametrize('outcome', ['match', 'wrong_period', 'unavailable', 'invalid_archive'])
+def test_missing_archive_checks_latest_without_accepting_wrong_week(trading, monkeypatch, source, outcome):
+    from alphapulse.calendar import worker as module, dol, eia
+    from alphapulse.calendar.archives import ArchiveMissing
+    from test_dol import REPORT
+    from test_eia import SCHEDULE, CSV
+    async def run():
+        worker = module.CalendarWorker()
+        worker.eia_schedule_body = SCHEDULE
+        try:
+            at = datetime(2026, 10, 29, 12, 30, tzinfo=UTC) if source == 'dol' else datetime(2026, 10, 28, 14, 30, tzinfo=UTC)
+            row = SimpleNamespace(source=source, family='initial_claims' if source == 'dol' else 'eia_stocks',
+                scheduled_at=at - timedelta(days=7) if outcome == 'wrong_period' else at,
+                release_url=None, first_received_at=None, period='2026-10-23')
+            worker.archive_page = AsyncMock(side_effect=ValueError('malformed index') if outcome == 'invalid_archive' else ArchiveMissing('archive missing'))
+            url = dol.LATEST if source == 'dol' else eia.LATEST
+            response = httpx.Response(403 if outcome == 'unavailable' else 200, text=REPORT if source == 'dol' else CSV, request=httpx.Request('GET', url))
+            worker.fetch = AsyncMock(return_value=(response, 7))
+            monkeypatch.setattr(module, 'pdf_text', lambda _: REPORT)
+            if outcome == 'match':
+                page, raw = await worker.result_page(row)
+                assert raw == 7
+                assert page is response
+            elif outcome == 'wrong_period':
+                with pytest.raises(ValueError, match='所选日程不符'):
+                    await worker.result_page(row)
+            elif outcome == 'unavailable':
+                with pytest.raises(httpx.HTTPStatusError):
+                    await worker.result_page(row)
+            else:
+                with pytest.raises(ValueError, match='malformed index'):
+                    await worker.result_page(row)
+                worker.fetch.assert_not_awaited()
+                return
+            worker.fetch.assert_awaited_once_with(source, url)
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())

backend/tests/test_dol.py

@@ -104,7 +104,7 @@ def test_live_result_never_applies_another_week(trading, monkeypatch, offset, re
             worker.fetch = AsyncMock(return_value=(response, 7))
             monkeypatch.setattr(module, "pdf_text", lambda body: REPORT)
             at = datetime(2026, 10, 29, 12, 30, tzinfo=UTC) + timedelta(days=offset)
-            row = SimpleNamespace(source="dol", family="initial_claims", release_url=None, scheduled_at=at,
+            row = SimpleNamespace(source="dol", family="initial_claims", release_url="https://oui.doleta.gov/press/2026/102926.pdf", scheduled_at=at,
                                   first_received_at=at if received else None)
             if expected == "error":
                 with pytest.raises(ValueError, match="公告时间"):

backend/tests/test_eia.py

@@ -80,7 +80,7 @@ def test_worker_does_not_apply_latest_values_to_another_week(trading, offset, re
             response = httpx.Response(200, text=CSV, request=httpx.Request("GET", eia.LATEST))
             worker.fetch = AsyncMock(return_value=(response, 1))
             at = datetime(2026, 10, 28, 14, 30, tzinfo=UTC) + timedelta(days=offset)
-            row = SimpleNamespace(source="eia", family="eia_stocks", period="2026-10-23", release_url=None,
+            row = SimpleNamespace(source="eia", family="eia_stocks", period="2026-10-23", release_url="https://www.eia.gov/petroleum/supply/weekly/archive/2026/2026_10_28/download/table1.csv",
                                   scheduled_at=at, first_received_at=at if received else None)
             if expected == "error":
                 with pytest.raises(ValueError, match="所选日程不符"):

backend/tests/test_jin10_calendar.py

@@ -311,3 +311,43 @@ def test_failed_days_do_not_starve_live_or_history_and_retry_after_backoff(tradi
         for _ in range(30): await collector.step()
         assert attempted.count(yesterday) == attempted.count(first_history) == 2
     asyncio.run(verify())
+
+
+def test_cross_day_release_uses_actual_date_and_keeps_source_day():
+    from alphapulse.calendar.jin10_view import jin10_item
+    body = deepcopy(FIXTURE['2026-11-04'])
+    body['list'] = body['list'][:1]
+    raw = body['list'][0]
+    at = datetime.fromisoformat('2026-11-05T00:20:43+08:00')
+    raw.update(pub_time='2026-11-05 00:20', pub_time_unix=int(at.timestamp()))
+    entry, = normalize(body, 'data', date(2026, 11, 4))
+    assert entry.at == at
+    assert entry.values['_jin10']['date'] == '2026-11-04'
+    assert jin10_item(stored(entry), datetime(2026, 11, 6, tzinfo=UTC))['calendar_date'] == '2026-11-05'
+    raw['pub_time_unix'] += 60
+    with pytest.raises(ValueError, match='时间戳'):
+        normalize(body, 'data', date(2026, 11, 4))
+
+
+def test_cross_day_membership_does_not_withdraw_until_all_buckets_remove_it(trading):
+    from alphapulse.calendar.jin10 import Entry
+    from alphapulse.calendar.jin10_store import save_entry, withdraw_missing
+    original = entries('2026-11-04')[0]
+    row = stored(original)
+    values = deepcopy(original.values)
+    values['_jin10']['date'] = '2026-11-05'
+    moved = Entry(original.identity, original.at, original.title, values)
+    revisions = []
+    session = SimpleNamespace(scalars=AsyncMock(return_value=[row]), scalar=AsyncMock(return_value=row), add=revisions.append)
+    now = datetime(2026, 11, 6, tzinfo=UTC)
+    async def run():
+        await save_entry(session, moved, 1, now)
+        assert row.values['_jin10']['source_dates'] == ['2026-11-04', '2026-11-05']
+        await withdraw_missing(session, date(2026, 11, 4), 'data', [], 2, now)
+        assert row.schedule_status == 'scheduled'
+        assert row.values['_jin10']['source_dates'] == ['2026-11-05']
+        await withdraw_missing(session, date(2026, 11, 5), 'data', [], 3, now)
+        assert row.schedule_status == 'withdrawn'
+        await save_entry(session, original, 4, now)
+        assert row.schedule_status == 'scheduled'
+    asyncio.run(run())

backend/tests/test_time_repair.py

@@ -0,0 +1,137 @@
+"""Time repairs use synthetic completions and never call a provider."""
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from pydantic import BaseModel
+
+from alphapulse.news_risk.contracts import PackageTimePoint
+
+
+class Result(BaseModel):
+    timeline_points: list[PackageTimePoint]
+
+
+@pytest.mark.parametrize('outcome', ['success', 'invalid', 'query', 'truncated', 'context'])
+def test_time_repair_is_bounded_and_accounts_for_calls(trading, monkeypatch, outcome):
+    from alphapulse.news_risk import service
+    from alphapulse.ai.llm import Completion
+    payload = {'timeline_points': [{'occurred_at': None, 'time_precision': 'day', 'time_text': '',
+        'reported_at': None, 'description': 'Unknown occurrence time', 'verification': 'insufficient',
+        'evidence_refs': [{'article_id': 1, 'quote': 'Source'}]}]}
+    calls = []
+
+    async def stream(provider, system, messages, tools, effort):
+        calls.append((messages, tools, effort))
+        if len(calls) == 2 and outcome == 'success':
+            payload['timeline_points'][0]['time_precision'] = 'unknown'
+        yield Completion(tool_calls=[{'id': str(len(calls)), 'name': 'read_news' if len(calls) == 2 and outcome == 'query' else 'submit', 'arguments': payload}],
+            usage={'input_tokens': 10}, truncated=len(calls) == 2 and outcome == 'truncated')
+
+    monkeypatch.setattr(service.llm, 'stream', stream)
+    provider = SimpleNamespace(model='synthetic', reasoning_effort='low', max_tokens=100, context_window=200000)
+    if outcome == 'context':
+        fits = iter([True, False])
+        monkeypatch.setattr(service, 'fits_context', lambda *args: next(fits))
+    if outcome == 'success':
+        result, usage = asyncio.run(service.complete(provider, 'rules', {}, Result))
+        assert result.timeline_points[0].occurred_at is None
+    else:
+        with pytest.raises(service.SubmissionFailure) as error:
+            asyncio.run(service.complete(provider, 'rules', {}, Result))
+        usage = error.value.usage
+    assert len(calls) == (1 if outcome == 'context' else 2)
+    assert len(usage['rounds']) == len(calls)
+    if len(calls) == 2:
+        assert [tool.name for tool in calls[1][1]] == ['submit']
+        assert calls[1][2] == 'low'
+        assert 'timeline_points' in calls[1][0][-1]['result']
+
+
+def test_unrelated_validation_does_not_retry(trading, monkeypatch):
+    from alphapulse.news_risk import service
+    from alphapulse.ai.llm import Completion
+    calls = []
+    async def stream(*args):
+        calls.append(args)
+        yield Completion(tool_calls=[{'id': '1', 'name': 'submit', 'arguments': {}}], usage={'output_tokens': 4})
+    monkeypatch.setattr(service.llm, 'stream', stream)
+    provider = SimpleNamespace(model='synthetic', reasoning_effort='low', max_tokens=100, context_window=200000)
+    with pytest.raises(service.SubmissionFailure):
+        asyncio.run(service.complete(provider, 'rules', {}, Result))
+    assert len(calls) == 1
+
+
+def test_research_time_repair_preserves_round_budget_and_thinking(trading, monkeypatch):
+    from alphapulse.news_risk import service
+    from alphapulse.news_risk.research import Research
+    from alphapulse.ai.llm import Completion
+    calls = []
+    args = {'timeline_points': [{'occurred_at': None, 'time_precision': 'day', 'time_text': '', 'reported_at': None,
+        'description': 'Source', 'verification': 'insufficient', 'evidence_refs': [{'quote': 'Source'}]}]}
+    async def stream(provider, system, messages, tools, effort):
+        calls.append((messages, tools))
+        if len(calls) <= 8:
+            yield Completion(tool_calls=[{'id': str(len(calls)), 'name': 'unknown_tool', 'arguments': {}}], usage={'output_tokens': 1})
+        else:
+            if len(calls) == 10:
+                assert messages[-2]['thinking_blocks'] == [{'type': 'thinking', 'thinking': 'synthetic'}]
+                assert [tool.name for tool in tools] == ['submit']
+                args['timeline_points'][0]['time_precision'] = 'unknown'
+            yield Completion(tool_calls=[{'id': str(len(calls)), 'name': 'submit', 'arguments': args}],
+                usage={'output_tokens': 2}, thinking_blocks=[{'type': 'thinking', 'thinking': 'synthetic'}])
+    monkeypatch.setattr(service.llm, 'stream', stream)
+    provider = SimpleNamespace(model='synthetic', reasoning_effort='high', max_tokens=100, context_window=200000)
+    research = Research()
+    result, usage = asyncio.run(research.run(provider, 'rules', {}, Result))
+    assert result.timeline_points[0].time_precision == 'unknown'
+    assert len(calls) == len(usage['rounds']) == 10
+    assert len(research.records) == 8
+
+
+def usage_tokens(usage):
+    if isinstance(usage, dict):
+        return usage.get('input_tokens', 0) + sum(usage_tokens(v) for k, v in usage.items() if k != 'input_tokens')
+    if isinstance(usage, list):
+        return sum(usage_tokens(v) for v in usage)
+    return 0
+
+
+@pytest.mark.parametrize('failure', ['second_call', 'evidence'])
+def test_extraction_failure_keeps_completed_and_failed_call_usage(trading, monkeypatch, failure):
+    from unittest.mock import AsyncMock
+    from alphapulse.news_risk import pipeline, service
+    extract = pipeline.Extraction(facts='Source', evidence_refs=[pipeline.Evidence(article_id=1, quote='Source')])
+    complete = AsyncMock(side_effect=[(extract, {'input_tokens': 10}), service.SubmissionFailure('synthetic failure', {'input_tokens': 40})])
+    monkeypatch.setattr(pipeline, 'complete', complete)
+    monkeypatch.setattr(pipeline, 'fits_context', lambda *args: True)
+    articles = [{'id': 1, 'title': 'Source' if failure == 'second_call' else 'Different'}, {'id': 2, 'title': 'Source'}]
+    with pytest.raises(service.SubmissionFailure) as error:
+        asyncio.run(pipeline.extract_articles(SimpleNamespace(), 'rules', articles))
+    assert usage_tokens(error.value.usage) == (50 if failure == 'second_call' else 10)
+
+
+@pytest.mark.parametrize('failure', ['second_call', 'evidence', 'no_shrink'])
+def test_compaction_failure_keeps_every_paid_call(trading, monkeypatch, failure):
+    from unittest.mock import AsyncMock
+    from alphapulse.news_risk import pipeline, service
+    group = {'facts': 'Very long source fact ' * 50, 'evidence_refs': [{'article_id': 1, 'quote': 'Source'}]}
+    extract = pipeline.Extraction(facts='Source' if failure != 'no_shrink' else 'Longer' * 1000,
+        evidence_refs=[pipeline.Evidence(article_id=1, quote='Changed' if failure == 'evidence' else 'Source')])
+    complete = AsyncMock(side_effect=[(extract, {'input_tokens': 10}), service.SubmissionFailure('synthetic failure', {'input_tokens': 40})])
+    monkeypatch.setattr(pipeline, 'complete', complete)
+    monkeypatch.setattr(pipeline, 'fits_context', lambda *args: True)
+    with pytest.raises(service.SubmissionFailure) as error:
+        asyncio.run(pipeline.compact_extractions(SimpleNamespace(), 'rules', {'extractions': [group]}, pipeline.EventResult, fits=lambda *args: False))
+    assert usage_tokens(error.value.usage) == (50 if failure == 'second_call' else 10)
+
+
+def test_summary_keeps_extraction_usage_when_compaction_fails(trading, monkeypatch):
+    from unittest.mock import AsyncMock
+    from alphapulse.news_risk import pipeline, service
+    monkeypatch.setattr(pipeline, 'fits_context', lambda *args: False)
+    monkeypatch.setattr(pipeline, 'extract_articles', AsyncMock(return_value=([{'facts': 'Source'}], [{'input_tokens': 123}])))
+    monkeypatch.setattr(pipeline, 'compact_extractions', AsyncMock(side_effect=service.SubmissionFailure('synthetic failure', {'input_tokens': 456})))
+    with pytest.raises(service.SubmissionFailure) as error:
+        asyncio.run(pipeline.summarize_articles(SimpleNamespace(), 'rules', [], {'id': 1}))
+    assert usage_tokens(error.value.usage) == 579