Repair official collection and retire exhausted material tasks

backend/tests/test_calendar.py

@@ -178,7 +178,7 @@ def test_rolling_page_rollover_preserves_received_result_without_source_error(tr
     async def check():
         first_received = datetime(2026, 9, 24, 14, tzinfo=UTC)
         values = {"cpi_yoy" if source == "bls" else "umich_sentiment": {"actual": "2.5", "previous": "2.4"}}
-        row = SimpleNamespace(id=1, source=source, family="cpi" if source == "bls" else "umich",
+        row = SimpleNamespace(id=1, collection_failures=0, collection_retired_at=None, source=source, family="cpi" if source == "bls" else "umich",
                               scheduled_at=first_received, first_received_at=first_received,
                               period="2026-09", stage="final" if source == "umich" else None,
                               values=values.copy(), revision_no=3, error="旧的采集错误")
@@ -275,7 +275,7 @@ def test_calendar_api_requires_auth_validates_range_and_exposes_unavailable_work
     asyncio.run(check())
 
 
-def test_worker_initializes_all_sources_and_drops_expired_month_status(trading):
+def test_worker_initializes_all_sources_and_drops_expired_month_status(trading, schedule_session):
     import httpx
     from alphapulse.calendar.worker import CalendarWorker
     async def check():
@@ -313,7 +313,7 @@ def test_limited_result_preserves_response_and_defers_single_poll(trading, monke
 
     async def check():
         now = datetime.now(UTC)
-        row = SimpleNamespace(id=1, source="eurostat", family="ea_hicp", scheduled_at=now,
+        row = SimpleNamespace(id=1, collection_failures=0, collection_retired_at=None, source="eurostat", family="ea_hicp", scheduled_at=now,
                               first_received_at=None, release_url="https://ec.europa.eu/eurostat/product?code=2-16102026-ap")
         session = SimpleNamespace(scalar=AsyncMock(return_value=row), commit=AsyncMock())
         class Context:
@@ -344,7 +344,7 @@ def test_limited_result_preserves_response_and_defers_single_poll(trading, monke
     asyncio.run(check())
 
 
-def test_schedule_honors_server_pause_without_blocking_other_sources(trading):
+def test_schedule_honors_server_pause_without_blocking_other_sources(trading, schedule_session):
     import httpx
     from alphapulse.calendar.worker import CalendarWorker
     async def check():
@@ -367,3 +367,14 @@ def test_schedule_honors_server_pause_without_blocking_other_sources(trading):
             await worker.http.aclose()
             await worker.bls.aclose()
     asyncio.run(check())
+
+
+@pytest.fixture
+def schedule_session(trading, monkeypatch):
+    from alphapulse.calendar import worker as module
+    session = AsyncMock()
+    session.scalars.return_value = []
+    session.get.return_value = None
+    session.__aenter__.return_value = session
+    monkeypatch.setattr(module, 'SessionLocal', lambda: session)
+    return session

backend/tests/test_calendar_history.py

@@ -220,7 +220,8 @@ def test_umich_poll_waits_for_final_instead_of_reusing_preliminary(trading):
         try:
             assert await worker.result_page(row) is None
             row.period = "2026-09"
-            with pytest.raises(ValueError, match="历史期次"):
+            worker.index = AsyncMock(return_value=(httpx.Response(200, text='<div id="report_heading"><span class="title">2026 Reports</span></div><a href="fetchdoc.php?docid=1">August Final Results</a>', request=httpx.Request('GET', 'https://data.sca.isr.umich.edu/reports.php?year=2026')), 2))
+            with pytest.raises(ValueError, match="年度归档"):
                 await worker.result_page(row)
         finally:
             await worker.http.aclose()

backend/tests/test_calendar_recovery.py

@@ -203,3 +203,49 @@ def test_missing_archive_checks_latest_without_accepting_wrong_week(trading, mon
             await worker.http.aclose()
             await worker.bls.aclose()
     asyncio.run(run())
+
+
+def test_result_retirement_is_persistent_and_cooling_does_not_spend_budget(trading, monkeypatch):
+    from alphapulse.calendar import worker as module
+
+    async def run():
+        row = SimpleNamespace(source='umich', collection_failures=0, collection_retired_at=None,
+                              scheduled_at=datetime(2026, 10, 9, tzinfo=UTC), first_received_at=None,
+                              family='umich', error=None)
+        session = AsyncMock()
+        session.scalar.return_value = row
+        session.__aenter__.return_value = session
+        monkeypatch.setattr(module, 'SessionLocal', lambda: session)
+        worker = module.CalendarWorker()
+        try:
+            worker.result_page = AsyncMock(side_effect=module.SourceCooling('example', datetime.now(UTC) + timedelta(hours=2)))
+            await worker.poll_one(1)
+            assert row.collection_failures == 0
+            assert row.next_check_at > datetime.now(UTC) + timedelta(hours=1)
+            worker.result_page.side_effect = ValueError('official archive missing')
+            for _ in range(3):
+                await worker.poll_one(1)
+            assert row.collection_failures == 3 and row.collection_retired_at is not None
+            calls = worker.result_page.await_count
+            await worker.poll_one(1)
+            assert worker.result_page.await_count == calls
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())
+
+
+def test_dol_lagging_index_uses_only_expected_official_archive(trading):
+    from alphapulse.calendar.worker import CalendarWorker
+    async def run():
+        worker = CalendarWorker()
+        response = lambda url, body: httpx.Response(200, text=body, request=httpx.Request('GET', url))
+        worker.index = AsyncMock(return_value=(response('https://oui.doleta.gov/unemploy/archive.asp', '<a href="/press/2025/010225.pdf">old</a>'), 1))
+        worker.fetch = AsyncMock(return_value=(response('https://oui.doleta.gov/press/2026/102926.pdf', 'PDF'), 2))
+        try:
+            await worker.archive_page(SimpleNamespace(source='dol', scheduled_at=datetime(2026, 10, 29, 12, 30, tzinfo=UTC)))
+            worker.fetch.assert_awaited_once_with('dol', 'https://oui.doleta.gov/press/2026/102926.pdf')
+        finally:
+            await worker.http.aclose()
+            await worker.bls.aclose()
+    asyncio.run(run())

backend/tests/test_collection_repair.py

@@ -0,0 +1,30 @@
+"""Repair initialization never renews an already initialized retry budget."""
+import asyncio
+import copy
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+
+def test_preview_and_repeated_execution_preserve_retry_evidence(trading, monkeypatch):
+    from alphapulse.crawler import repair_collection as module
+    original = {'document_retries': {'https://example.invalid/doc': {
+        'document': {'url': 'https://example.invalid/doc'}, 'attempts': 3,
+        'next_at': 1, 'error': 'HTTP 503'}}}
+    source = SimpleNamespace(code='doj', history_cursor=copy.deepcopy(original))
+    session = AsyncMock()
+    session.scalars.return_value = [source]
+    session.__aenter__.return_value = session
+    monkeypatch.setattr(module, 'SessionLocal', lambda: session)
+    monkeypatch.setattr(module, 'engine', SimpleNamespace(dispose=AsyncMock()))
+    async def check():
+        await module.run()
+        assert source.history_cursor == original
+        session.commit.assert_not_awaited()
+        await module.run(True)
+        entry = source.history_cursor['document_retries']['https://example.invalid/doc']
+        assert entry['attempts'] == 0 and entry['repair_previous']['attempts'] == 3
+        entry['attempts'] = 2
+        result = await module.run(True)
+        assert result['sources'][0]['already_initialized']
+        assert entry['attempts'] == 2
+    asyncio.run(check())

backend/tests/test_eia.py

@@ -123,3 +123,10 @@ def test_history_preview_requires_matching_archive_date_without_writes(trading,
     assert result["announcements"].get("eia_stocks", 0) == int(valid)
     assert bool(result["errors"]) != valid
     save.assert_not_awaited()
+
+
+def test_current_official_archive_csv_directory():
+    url = 'https://www.eia.gov/petroleum/supply/weekly/archive/2026/2026_10_15/wpsr_2026_10_15.php'
+    assert eia.csv_link('<a href="csv/table1.csv">CSV</a>', url) == url.rsplit('/', 1)[0] + '/csv/table1.csv'
+    with pytest.raises(ValueError):
+        eia.csv_link('<a href="csv/table1.csv">A</a><a href="download/table1.csv">B</a>', url)

backend/tests/test_us_official_news.py

@@ -1343,3 +1343,36 @@ def test_real_congress_api_fixture_replays_with_visible_source_gap(trading, tmp_
     assert sum(a['status'] == 'complete' for a in attachments) == 14
     assert sum(a['status'] == 'unavailable' for a in attachments) == 1
     assert any(item['extra']['content_scope'] == 'official_action' for item in items)
+
+
+def test_terminal_news_gap_survives_rediscovery(trading):
+    obj = spider(trading, 'doj')
+    url = 'https://www.justice.gov/opa/pr/example'
+    obj.cursor['attachment_retries'] = {url: {'attempts': 3, 'error': 'unsupported image'}}
+    obj.retire_exhausted()
+    assert obj.cursor['attachment_retries'][url]['retired_at']
+    obj.complete = lambda lane: []
+    assert list(obj.attach_or_finish({'url': url}, 'realtime')) == []
+
+
+def test_selected_news_requests_have_one_retry_owner(trading):
+    obj = spider(trading, 'doj')
+    req = obj.request_for(SOURCES['doj'].entries[0], obj.parse_listing, 'realtime')
+    assert req.meta['dont_retry'] is True
+
+
+def test_doj_captioned_photo_is_not_a_document_gap(trading):
+    obj = spider(trading, 'doj')
+    obj.run.history_deadline = time.monotonic() - 1
+    identifier = str(uuid.uuid4())
+    doc = {'url': f'https://www.justice.gov/api/v1/press_releases/{identifier}.json', 'kind': 'doj'}
+    obj.cursor['history']['pending'] = [doc]
+    record = {'uuid': identifier, 'url': 'https://www.justice.gov/opa/pr/test', 'date': '1735732800',
+              'title': 'Cryptocurrency exchange settlement', 'body': '<p>The cryptocurrency exchange pleaded guilty to violations of financial regulations.</p>',
+              'image': '<a href="/usao-sd/media/1470086">award.png</a>',
+              'attachment': [{'file': {'id': '1470086'}, 'alt': 'Photo- Left to right: award recipients.'}]}
+    async def collect():
+        return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'metadata': {'responseInfo': {'status': 200}}, 'results': [record]}), lane='history', document=doc))]
+    item, = asyncio.run(collect())
+    assert item['extra']['attachments'] == []
+    assert not obj.cursor.get('attachment_retries')