Add resumable official news and calendar backfill to January 2025

backend/tests/test_history_capture.py

@@ -0,0 +1,31 @@
+from datetime import UTC, datetime
+import pytest
+
+
+
+def test_timeline_preserves_actual_publication_and_provider_identity(trading):
+    from alphapulse.crawler.history_capture import timeline_articles
+    result = timeline_articles({'hasMore': False, 'articles': [{'_id': 'original', 'pathname': '/markets/story',
+        'title': 'Official title', 'description': 'Public summary',
+        'articleDates': {'displayDate': '2024-12-31T17:00:00Z'}}]})
+    assert result[0].guid == 'original'
+    assert result[0].published_at == datetime(2024, 12, 31, 17, tzinfo=UTC)
+    assert result[0].summary == 'Public summary'
+
+
+def test_unknown_timeline_shape_is_not_empty_success(trading):
+    from alphapulse.crawler.history_capture import timeline_articles
+    with pytest.raises(ValueError):
+        timeline_articles({'articles': []})
+
+
+def test_ecb_index_keeps_existing_news_scope(trading):
+    from alphapulse.crawler.history_capture import ecb_articles
+    row = [1, 1735732800, 2025, 0, 1, None, None, None, None,
+           ['/press/key/date/2025/html/example.en.html', '/stats/example.html'], {'Title': 'Speech'}, [], []]
+    result = ecb_articles(row)
+    assert len(result) == 1
+    assert '/press/key/' in str(result[0].url)
+    assert result[0].published_at == datetime.fromtimestamp(1735732800, UTC)
+    with pytest.raises(ValueError):
+        ecb_articles(row[:-1])

backend/tests/test_us_official_news.py

@@ -1411,3 +1411,17 @@ def test_attachment_http_failure_spends_durable_retry_budget(trading):
     assert item['extra']['content_scope'] == 'partial'
     assert obj.cursor['attachment_retries'][url]['attempts'] == 1
     assert '503' in obj.cursor['attachment_retries'][url]['error']
+
+
+@pytest.mark.parametrize('dated', [True, False])
+def test_sec_historical_listing_stops_only_with_verified_dates(trading, dated, monkeypatch):
+    monkeypatch.setattr('alphapulse.crawler.spiders.us_official.configuration_error', lambda code: None)
+    obj = spider(trading, 'sec')
+    stamp = '<time datetime="2024-12-30T12:00:00Z">Dec 30</time>' if dated else ''
+    body = '<h1>Press releases</h1><table><tr><td>' + stamp + '</td><td><a href="/newsroom/press-releases/2024-199">Release</a></td></tr></table><a rel="next" href="?page=1">Next</a>'
+    requests = list(obj.parse_listing(response(SOURCES['sec'].entries[0], body, lane='history')))
+    if dated:
+        assert all('/2024-199' not in r.url and '?page=1' not in r.url for r in requests)
+    else:
+        assert any('/2024-199' in r.url for r in requests)
+        assert obj.cursor['history']['next'].endswith('?page=1')