Fix official news retries, date boundaries and collection status

Bound document retries across realtime and history lanes, preserve safe completed reads on failed rounds, and retain unresolved gaps. Honor the longest access-block cooldown and measure retries from the actual failure time.

Filter dated official listings at the collection boundary, handle empty Federal Register days, and distinguish active rounds from waiting or failed processes in the console. Add captured-response regressions and isolated checkpoint validation.

backend/tests/test_us_official_news.py

@@ -9,6 +9,8 @@ from types import SimpleNamespace
 import pytest
 from scrapy import Request
 from scrapy.http import HtmlResponse, Response
+from scrapy.spidermiddlewares.httperror import HttpError
+from twisted.python.failure import Failure
 
 from alphapulse.core.us_news import SOURCES, article, official_url, published, selection
 
@@ -31,6 +33,314 @@ def response(url, body, **meta):
     return HtmlResponse(url, body=body.encode(), encoding='utf-8', request=Request(url, meta={'raw_response_id': 12, **meta}))
 
 
+def collection_sample(source, suffix=''):
+    import gzip
+    from pathlib import Path
+    samples = json.load(gzip.open(Path(__file__).parent / 'fixtures/us_news/collection_failures.json.gz', 'rt'))
+    return next(s for s in samples if s['source'] == source and s['url'].endswith(suffix))
+
+
+def test_real_empty_register_day_continues_to_previous_day(trading):
+    obj = spider(trading, 'federal_register')
+    obj.cursor['realtime']['day'] = '2026-11-05'
+    sample = collection_sample('federal_register')
+    request, = obj.parse_listing(response(sample['url'], sample['body'], lane='realtime'))
+    assert '2026-11-04' in request.url
+    assert not obj.run.errors
+
+
+@pytest.mark.parametrize('data', [{'count': 2}, {'count': '0', 'description': 'x'}, {'count': 0}, {'count': 0, 'description': 'x', 'results': None}])
+def test_register_does_not_hide_malformed_responses(trading, data):
+    obj = spider(trading, 'federal_register')
+    with pytest.raises(ValueError, match='列表结构'):
+        list(obj.parse_listing(response(SOURCES[obj.name].entries[0], json.dumps(data), lane='realtime')))
+
+
+@pytest.mark.parametrize('source', ['treasury', 'cftc', 'fincen', 'whitehouse'])
+def test_real_realtime_lists_filter_dated_old_documents(trading, source):
+    obj = spider(trading, source)
+    sample = collection_sample(source, '/press-releases' if source == 'fincen' else '')
+    # All sample pages precede January 2027; no old body should be requested.
+    obj.run.cutoff = datetime(2027, 1, 1, tzinfo=UTC)
+    requests = list(obj.parse_listing(response(sample['url'], sample['body'], lane='realtime')))
+    pending = obj.cursor['realtime'].get('pending', [])
+    assert not pending
+    assert all(not r.meta.get('document') for r in requests)
+
+
+def test_treasury_real_realtime_stops_at_date_boundary_before_broken_pagination(trading):
+    obj = spider(trading, 'treasury')
+    sample = collection_sample('treasury')
+    obj.cursor['realtime']['page'] = sample['url']
+    requests = list(obj.parse_listing(response(sample['url'], sample['body'], lane='realtime')))
+    assert requests and requests[0].meta['document']['kind'] == 'article'
+    assert obj.cursor['realtime']['next'] is None
+    assert all(published(d['listing_date'])[0].date() >= obj.run.cutoff.date() for d in obj.cursor['realtime']['pending'])
+    # The same response cannot establish historical coverage through 2025.
+    with pytest.raises(ValueError, match='重复当前页面'):
+        list(obj.parse_listing(response(sample['url'], sample['body'], lane='history')))
+
+
+def test_undated_or_unordered_lists_do_not_stop_early(trading):
+    obj = spider(trading, 'treasury')
+    url = SOURCES[obj.name].entries[0]
+    body = '<h1>News</h1><div><time datetime="2025-01-01"></time><h3><a href="/news/press-releases/sb0001">Old</a></h3></div><a href="/news/press-releases/sb0002">Undated</a><a rel="next" href="?page=2">Next</a>'
+    list(obj.parse_listing(response(url, body, lane='realtime')))
+    assert obj.cursor['realtime']['next'].endswith('?page=2')
+    assert obj.cursor['realtime']['pending'][0]['url'].endswith('sb0002')
+
+
+def test_failed_document_does_not_block_next_document_or_retry_immediately(trading):
+    obj = spider(trading, 'doj')
+    docs = [{'url': f'https://www.justice.gov/api/v1/press_releases/{uuid.uuid4()}.json', 'kind': 'doj'} for _ in range(2)]
+    obj.cursor['realtime']['pending'] = copy.deepcopy(docs)
+    request = Request(docs[0]['url'], meta={'document': docs[0], 'lane': 'realtime'})
+    failed = Failure(HttpError(Response(request.url, status=503, request=request)))
+    failed.request = request
+    following, = obj.on_request_error(failed)
+    assert following.url == docs[1]['url']
+    assert obj.run.errors and docs[0]['url'] not in obj.cursor.get('checked', {})
+    gap = obj.cursor['document_retries'][docs[0]['url']]
+    assert gap['document'] == docs[0] and gap['attempts'] == 1
+    assert gap['next_at'] > obj.run.started_at.timestamp()
+
+
+@pytest.mark.parametrize('attempts,due,expected', [(1, False, False), (3, True, False), (1, True, True)])
+def test_document_retry_is_bounded_and_survives_rounds(trading, attempts, due, expected):
+    obj = spider(trading, 'ofac')
+    url = 'https://ofac.treasury.gov/recent-actions/20261104'
+    obj.cursor['document_retries'] = {url: {'document': {'url': url, 'kind': 'article'}, 'attempts': attempts, 'next_at': obj.run.started_at.timestamp() + (-60 if due else 3600), 'error': 'HTTP503'}}
+    listing = '<h1>Actions</h1><a href="'+url+'">Sanctions</a>'
+    requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], listing, lane='realtime')))
+    assert any(r.url == url for r in requests) is expected
+    assert url in obj.cursor['document_retries']
+
+
+def test_old_pending_cursor_cannot_bypass_exhausted_document_retry(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [doc]
+    obj.cursor['document_retries'] = {doc['url']: {'document': doc, 'attempts': 3, 'next_at': 0, 'error': 'unavailable'}}
+    requests = obj.advance('realtime')
+    assert all(r.url != doc['url'] for r in requests)
+    assert doc['url'] in obj.cursor['document_retries']
+
+
+def test_due_retry_in_old_pending_cursor_is_not_discarded_by_moving_window(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article', 'listing_date': '2025-01-01'}
+    obj.cursor['realtime']['pending'] = [doc]
+    obj.cursor['document_retries'] = {doc['url']: {'document': copy.deepcopy(doc), 'attempts': 1, 'next_at': 0, 'error': 'unavailable'}}
+    request, = obj.first_requests()
+    assert request.meta['document']['document_retry']
+    body = '<h1>Crypto sanctions</h1><main><article class="node--type-ofac-recent-action"><div class="field--name-field-body"><p>New sanctions against a virtual currency exchange institution.</p></div></article></main>'
+    async def collect():
+        return [item async for item in obj.parse_document(response(request.url, body, **request.meta))]
+    items = asyncio.run(collect())
+    assert any(isinstance(item, dict) and item['url'] == doc['url'] for item in items)
+    assert not obj.cursor['document_retries']
+    assert doc['url'] in obj.cursor['checked']
+
+
+@pytest.mark.parametrize('restored_pending', [False, True])
+def test_attachment_retry_outside_realtime_window_fetches_and_completes_pdf(trading, monkeypatch, restored_pending):
+    from alphapulse.crawler.spiders import us_official
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20250101', 'kind': 'article', 'listing_date': '2025-01-01'}
+    obj.cursor['attachment_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0}}
+    if restored_pending:
+        obj.cursor['realtime']['pending'] = [copy.deepcopy(doc)]
+    request, = obj.first_requests()
+    body = '<h1>Crypto sanctions</h1><main><article class="node--type-ofac-recent-action"><div class="field--name-field-body"><p>New sanctions against a virtual currency exchange institution.</p><a href="/media/123/download">Order</a></div></article></main>'
+    async def collect():
+        return [item async for item in obj.parse_document(response(request.url, body, **request.meta))]
+    attachment, = asyncio.run(collect())
+    assert isinstance(attachment, Request) and attachment.url.endswith('/media/123/download')
+    assert obj.cursor['attachment_retries'][doc['url']]['attempts'] == 1
+    assert doc['url'] not in obj.cursor.get('checked', {})
+    monkeypatch.setattr(us_official, 'pdf_text', lambda body: 'The official sanctions order applies to a virtual currency exchange.')
+    items = list(obj.parse_attachment(Response(attachment.url, body=b'%PDF-test', request=attachment.replace(meta={**attachment.meta, 'raw_response_id': 13}))))
+    item = next(item for item in items if isinstance(item, dict))
+    assert item['extra']['content_scope'] == 'article'
+    assert item['extra']['attachments'][0]['status'] == 'complete'
+    assert not obj.cursor['attachment_retries']
+    assert doc['url'] in obj.cursor['checked']
+
+
+def test_due_attachment_is_queued_alongside_document_retry_without_duplicates(trading):
+    obj = spider(trading)
+    documents = [{'url': f'https://ofac.treasury.gov/recent-actions/2026110{day}', 'kind': 'article'} for day in (3, 4)]
+    obj.cursor['document_retries'] = {documents[0]['url']: {'document': documents[0], 'attempts': 1, 'next_at': 0, 'error': 'HTTP503'}}
+    obj.cursor['attachment_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0} for doc in documents}
+    request, = obj.first_requests()
+    pending = obj.cursor['realtime']['pending']
+    assert [doc['url'] for doc in pending] == [doc['url'] for doc in documents]
+    assert request.meta['document']['document_retry'] and request.meta['document']['attachment_retry']
+    requests = obj.complete('realtime')
+    assert requests[0].url == documents[1]['url'] and requests[0].meta['document']['attachment_retry']
+
+
+@pytest.mark.parametrize('first_lane', ['realtime', 'history'])
+def test_successful_retry_is_not_requeued_by_other_lane(trading, first_lane):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    obj.cursor['document_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0, 'error': 'HTTP503'}}
+    for lane in ('realtime', 'history'):
+        obj.cursor[lane]['pending'] = [copy.deepcopy(doc)]
+    request, = obj.advance(first_lane)
+    body = '<h1>Crypto sanctions</h1><main><article class="node--type-ofac-recent-action"><div class="field--name-field-body"><p>New sanctions against a virtual currency exchange institution.</p></div></article></main>'
+    async def collect():
+        return [item async for item in obj.parse_document(response(request.url, body, **request.meta))]
+    items = asyncio.run(collect())
+    assert any(isinstance(item, dict) for item in items)
+    assert not obj.cursor['document_retries']
+    other_lane = 'history' if first_lane == 'realtime' else 'realtime'
+    assert all(request.url != doc['url'] for request in obj.advance(other_lane))
+
+
+@pytest.mark.parametrize('first_lane', ['realtime', 'history'])
+def test_document_retry_is_requested_once_across_both_lanes(trading, first_lane):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    obj.cursor['document_retries'] = {doc['url']: {'document': doc, 'attempts': 1, 'next_at': 0, 'error': 'HTTP503'}}
+    for lane in ('realtime', 'history'):
+        obj.cursor[lane]['pending'] = [copy.deepcopy(doc)]
+    request, = obj.advance(first_lane)
+    other_lane = 'history' if first_lane == 'realtime' else 'realtime'
+    following = obj.advance(other_lane)
+    assert all(r.url != doc['url'] for r in following)
+    failure = Failure(HttpError(Response(request.url, status=503, request=request)))
+    failure.request = request
+    obj.on_request_error(failure)
+    assert obj.cursor['document_retries'][doc['url']]['attempts'] == 2
+    assert doc['url'] not in obj.cursor.get('checked', {})
+    # Completing a concurrent listing must not re-queue the same retry.
+    listing = '<h1>Actions</h1><a href="' + doc['url'] + '">Sanctions</a>'
+    requests = list(obj.parse_listing(response(SOURCES[obj.name].entries[0], listing, lane=other_lane)))
+    assert all(r.url != doc['url'] for r in requests)
+
+
+def test_concurrent_initial_document_failures_keep_one_attempt_and_all_errors(trading):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    for lane in ('realtime', 'history'):
+        obj.cursor[lane]['pending'] = [copy.deepcopy(doc)]
+    requests = [obj.advance(lane)[0] for lane in ('realtime', 'history')]
+    for request in requests:
+        failure = Failure(HttpError(Response(request.url, status=503, request=request)))
+        failure.request = request
+        obj.on_request_error(failure)
+    assert obj.cursor['document_retries'][doc['url']]['attempts'] == 1
+    assert len(obj.run.errors) == 2
+
+
+def test_document_retry_waits_one_hour_after_failure_in_long_round(trading):
+    obj = spider(trading)
+    obj.run.started_at = datetime.now(UTC) - timedelta(hours=2)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [doc]
+    request, = obj.advance('realtime')
+    failure = Failure(HttpError(Response(request.url, status=503, request=request)))
+    failure.request = request
+    before = datetime.now(UTC)
+    obj.on_request_error(failure)
+    after = datetime.now(UTC)
+    due_at = obj.cursor['document_retries'][doc['url']]['next_at']
+    assert (before + timedelta(hours=1)).timestamp() <= due_at <= (after + timedelta(hours=1)).timestamp()
+
+
+def test_pipeline_error_prevents_completed_read_checkpoint(trading):
+    obj = spider(trading)
+    obj.run.completed_documents_safe = True
+    obj._on_item_error(Failure(ValueError('test persistence failure')))
+    assert not obj.run.completed_documents_safe
+    assert '落库失败' in obj.run.errors[0]
+
+
+def test_completed_document_error_cannot_remove_next_pending_document(trading, monkeypatch):
+    obj = spider(trading)
+    doc = {'url': 'https://ofac.treasury.gov/recent-actions/20261104', 'kind': 'article'}
+    following = {'url': 'https://ofac.treasury.gov/recent-actions/20261103', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [following]
+    async def scheduling_failed(res):
+        raise ValueError('next request scheduling failed')
+        yield
+    monkeypatch.setattr(obj, '_parse_document', scheduling_failed)
+    async def collect():
+        return [item async for item in obj.parse_document(response(doc['url'], '', document=doc, lane='realtime'))]
+    with pytest.raises(ValueError, match='scheduling failed'):
+        asyncio.run(collect())
+    assert obj.cursor['realtime']['pending'] == [following]
+    assert not obj.cursor.get('document_retries')
+
+
+@pytest.mark.parametrize('safe', [False, True])
+def test_failed_round_saves_only_safe_completed_reads_and_explicit_gaps(trading, monkeypatch, safe):
+    from alphapulse.crawler.spiders import base
+    obj = spider(trading)
+    source = SimpleNamespace(history_target=obj.run.history_target, history_cursor={'history': {'entry': 0}, 'realtime': {'entry': 0}}, consecutive_failures=0)
+    class Session:
+        async def __aenter__(self): return self
+        async def __aexit__(self, *args): pass
+        async def execute(self, *args): pass
+        async def get(self, *args, **kwargs): return source
+        async def commit(self): pass
+    monkeypatch.setattr(base, 'SessionLocal', Session)
+    obj.run.errors = ['failed document']
+    obj.run.completed_documents_safe = safe
+    obj.run.official_cursor = {'history': {'entry': 5}, 'realtime': {'entry': 8}, 'checked': {'good': 123}, 'document_retries': {'bad': {'attempts': 1}}}
+    asyncio.run(obj._finish_run(obj.run))
+    assert source.history_cursor['history'] == {'entry': 0}
+    assert source.history_cursor['realtime'] == {'entry': 0}
+    assert ('checked' in source.history_cursor) is safe
+    assert ('document_retries' in source.history_cursor) is safe
+    if safe:
+        assert source.history_cursor['checked'] == {'good': 123}
+        assert 'bad' in source.history_cursor['document_retries']
+
+
+@pytest.mark.parametrize('status', [403, 429])
+def test_access_block_stops_round_and_honors_retry_after(trading, status):
+    obj = spider(trading, 'cftc')
+    doc = {'url': 'https://www.cftc.gov/PressRoom/PressReleases/9321-26', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [doc]
+    request = Request(doc['url'], meta={'document': doc, 'lane': 'realtime'})
+    failure = Failure(HttpError(Response(request.url, status=status, headers={'Retry-After': '3600'}, request=request)))
+    failure.request = request
+    obj.on_request_error(failure)
+    assert obj.run.blocked
+    assert obj.run.cooldown_until > datetime.now(UTC) + timedelta(minutes=59)
+    assert obj.advance('realtime') == []
+    assert obj.cursor['realtime']['pending'] == [doc]
+    assert not obj.cursor.get('document_retries')
+
+
+@pytest.mark.parametrize('later_block', ['403', '429', 'listing_challenge', 'document_challenge'])
+def test_later_block_does_not_shorten_existing_retry_after(trading, later_block):
+    obj = spider(trading, 'cftc')
+    doc = {'url': 'https://www.cftc.gov/PressRoom/PressReleases/9321-26', 'kind': 'article'}
+    obj.cursor['realtime']['pending'] = [doc]
+    request = Request(doc['url'], meta={'document': doc, 'lane': 'realtime'})
+    failure = Failure(HttpError(Response(request.url, status=429, headers={'Retry-After': '3600'}, request=request)))
+    failure.request = request
+    obj.on_request_error(failure)
+    until = obj.run.cooldown_until
+    if later_block in ('403', '429'):
+        failure = Failure(HttpError(Response(request.url, status=int(later_block), request=request)))
+        failure.request = request
+        obj.on_request_error(failure)
+    else:
+        body = '<h1>Just a moment...</h1><p>Enable JavaScript and cookies to continue.</p>'
+        if later_block == 'listing_challenge':
+            with pytest.raises(ValueError, match='验证页面'):
+                list(obj.parse_listing(response(SOURCES[obj.name].entries[0], body, lane='history')))
+        else:
+            async def collect():
+                return [item async for item in obj.parse_document(response(doc['url'], body, document=doc, lane='realtime'))]
+            assert asyncio.run(collect()) == []
+    assert obj.run.blocked and obj.run.cooldown_until == until
+
+
 @pytest.mark.parametrize('source,title,text,expected', [
     ('doj', 'Man sentenced for bitcoin theft', 'Personal bitcoin theft worth $300', False),
     ('doj', 'Exchange charged', 'Cryptocurrency exchange charged with money laundering', True),
@@ -130,9 +440,9 @@ def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeyp
     from alphapulse.models import Source
 
     source = Source(code='doj', name='DOJ', list_url=SOURCES['doj'].entries[0], enabled=False, interval_seconds=300, backfill_hours=72,
-                    consecutive_failures=0, last_success_at=None, last_error=None, paused_until=None, history_cursor={'attachment_retries': {'a': {'attempts': 1}, 'b': {'attempts': 2}, 'c': {'attempts': 3}}},
+                    consecutive_failures=0, last_success_at=None, last_error=None, paused_until=None, history_cursor={'attachment_retries': {'a': {'attempts': 1}, 'b': {'attempts': 2}, 'c': {'attempts': 3}}, 'document_retries': {'d': {'attempts': 1}, 'e': {'attempts': 3}}},
                     history_target=None, history_position=None, history_done=False)
-    latest = SimpleNamespace(source_code='doj', stats={'discovered': 2, 'attachment_gaps': 999})
+    latest = SimpleNamespace(source_code='doj', status='success', stats={'discovered': 2, 'attachment_gaps': 999})
 
     class Session:
         async def scalars(self, statement):
@@ -151,6 +461,10 @@ def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeyp
         assert result.collection_stats['attachment_gaps'] == 3
         assert result.collection_stats['attachment_retry_pending'] == 2
         assert result.collection_stats['attachment_retry_exhausted'] == 1
+        assert result.collection_stats['document_gaps'] == 2
+        assert result.collection_stats['document_retry_pending'] == 1
+        assert result.collection_stats['document_retry_exhausted'] == 1
+        assert result.collection_stats['round_running'] == 0
     source.history_cursor['attachment_retries'].pop('a')
     result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
     assert result.collection_stats['attachment_gaps'] == 2
@@ -160,6 +474,10 @@ def test_source_api_keeps_attachment_gaps_visible_across_rounds(trading, monkeyp
     result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
     assert result.collection_stats['attachment_gaps'] == 0
     assert result.collection_stats['attachment_retry_exhausted'] == 0
+    assert result.collection_stats['document_gaps'] == 0
+    latest.status = 'running'
+    result = asyncio.run(crawler.list_sources(Session())).data.sources[0]
+    assert result.collection_stats['round_running'] == 1
 
 
 def test_challenge_is_failure_not_empty_success(trading):
@@ -367,10 +685,13 @@ def test_attachment_failure_is_visible_and_keeps_a_bounded_retry(trading):
     obj.cursor['history']['pending'] = [doc]
     parsed = {'url': url, 'title': 'Crypto sanctions', 'external_id': 'x', 'content_html': '<p>Virtual currency exchange sanctions.</p>', 'published_at': datetime(2025, 1, 1, tzinfo=UTC), 'raw_response_id': 12, 'retry_document': doc,
               'extra': {'attachments': [{'url': 'https://ofac.treasury.gov/a.pdf', 'title': 'License', 'status': 'unavailable'}]}}
+    before = datetime.now(UTC)
     item, = obj.attach_or_finish(parsed, 'history')
+    after = datetime.now(UTC)
     assert item['extra']['content_scope'] == 'partial'
     assert obj.cursor['attachment_retries'][url]['attempts'] == 1
     assert obj.cursor['attachment_retries'][url]['document'] == doc
+    assert (before + timedelta(hours=1)).timestamp() <= obj.cursor['attachment_retries'][url]['next_at'] <= (after + timedelta(hours=1)).timestamp()
 
 
 @pytest.mark.parametrize('canceled,failed', [(False, False), (True, False), (False, True)])
@@ -566,10 +887,13 @@ def test_missing_bill_material_is_a_retryable_gap(trading, monkeypatch):
     asyncio.run(parse({'bill': {'title': 'GENIUS Act'}}))
     asyncio.run(parse({'subjects': {}}))
     asyncio.run(parse({'summaries': []}))
+    before = datetime.now(UTC)
     asyncio.run(parse({'textVersions': []}))
+    after = datetime.now(UTC)
     assert obj.cursor['bill_retries'][base]['attempts'] == 1
     assert obj.run.stats['unresolved_bills'] == 1
     assert not obj.run.stats.get('excluded')
+    assert (before + timedelta(hours=1)).timestamp() <= obj.cursor['bill_retries'][base]['next_at'] <= (after + timedelta(hours=1)).timestamp()
 
 
 @pytest.mark.parametrize('attempts,version,expected_kind', [(3, 'v1', 'listing'), (1, 'v1', 'listing'), (3, 'v2', 'subjects')])
@@ -761,8 +1085,10 @@ def test_doj_body_identity_date_and_validation(trading, bad_body):
     async def collect():
         return [v async for v in obj.parse_document(response(doc['url'], json.dumps({'metadata': {'responseInfo': {'status': 200}}, 'results': [record]}), lane='history', document=doc))]
     if bad_body:
-        with pytest.raises(ValueError): asyncio.run(collect())
-        assert obj.cursor['history']['pending'] == [doc]
+        assert asyncio.run(collect()) == []
+        assert obj.run.blocked and obj.run.errors
+        assert obj.cursor['document_retries'][doc['url']]['attempts'] == 1
+        assert obj.cursor['history']['pending'] == []
     else:
         item, = asyncio.run(collect())
         assert item['url'] == 'https://www.justice.gov/opa/pr/test'