Add resumable news history backfill
Let the crawler console set a backfill target per source. Each scheduled run pages backwards from a checkpoint for at most 20s so real-time crawling is not blocked. The checkpoint is saved only after a successful run and is also kept in process memory, because the next trigger may be queued before the database write lands. cls_depth is excluded since its list API cannot page into history.
This commit is contained in:
1 parent
0831971e83
commit
5c8d773ce1
10 files changed
+229
-17
No files matched your search
@@ -0,0 +1,32 @@
|
||||
"""source history backfill
|
||||
|
||||
Revision ID: d41b7c2e9a10
|
||||
Revises: bf0ffbfcaf9a
|
||||
Create Date: 2026-10-13 15:00:00.000000
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'd41b7c2e9a10'
|
||||
down_revision: Union[str, Sequence[str], None] = 'bf0ffbfcaf9a'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
op.add_column('sources', sa.Column('history_target', sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column('sources', sa.Column('history_position', sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column('sources', sa.Column('history_done', sa.Boolean(), server_default='false', nullable=False))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_column('sources', 'history_done')
|
||||
op.drop_column('sources', 'history_position')
|
||||
op.drop_column('sources', 'history_target')
|
||||
Reference in new issue
Block a user