:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): continue the cursor of a job that worker restarts stopped (#109760)
posthog[bot]32f01c2c2ce7
diff --git a/products/data_warehouse/backend/logic/external_data_source/jobs.py b/products/data_warehouse/backend/logic/external_data_source/jobs.pyindex 1532099b68f7..cda841547560 100644--- a/products/data_warehouse/backend/logic/external_data_source/jobs.py+++ b/products/data_warehouse/backend/logic/external_data_source/jobs.py@@ -10,7 +10,11 @@ from structlog.types import FilteringBoundLoggerfrom posthog.exceptions_capture import capture_exceptionfrom products.data_warehouse.backend.tasks.tasks import schedule_external_data_failure_digest-from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema+from products.warehouse_sources.backend.facade.models import (+ WORKER_RESTART_ERROR_MESSAGE,+ ExternalDataJob,+ ExternalDataSchema,+)from products.warehouse_sources.backend.facade.pipelines import (LOCK_TAKEOVER_LATEST_ERROR,TERMINAL_JOB_STATUSES,@@ -180,10 +184,18 @@ def update_external_job_status(# later can stale-overwrite newer data or flip this job's status. A normal v3 completion# has drained the queue by construction, so the common completion path stays queue-free.# on_commit: the queue is a separate Postgres; never call it holding these row locks.+ # Worker restarts stop a run without a final batch, and Failed stays absorbing, so a later+ # batch can neither complete the job nor promote its cursor. The loader drains those batches+ # for a merge sync and fails them for any other, and the next job continues from the cursor+ # that this run committed. Sweeping them would leave a gap of rows that no run reads again.+ stopped_by_worker_restarts = (+ status == ExternalDataJobStatus.FAILED and error_to_persist == WORKER_RESTART_ERROR_MESSAGE+ )should_sweep_queue = (model.pipeline_version == ExternalDataJobPipelineVersion.V3and status in TERMINAL_JOB_STATUSESand (status != ExternalDataJobStatus.COMPLETED or is_takeover_recovery)+ and not stopped_by_worker_restarts)if should_sweep_queue:transaction.on_commit(diff --git a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.pyindex 36880fb8f865..c162b81e7945 100644--- a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py+++ b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py@@ -9,7 +9,12 @@ from temporalio.exceptions import CancelledErrorfrom posthog.models import Organization, Teamfrom products.data_warehouse.backend.logic.external_data_source.jobs import update_external_job_status-from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema, ExternalDataSource+from products.warehouse_sources.backend.facade.models import (+ WORKER_RESTART_ERROR_MESSAGE,+ ExternalDataJob,+ ExternalDataSchema,+ ExternalDataSource,+)from products.warehouse_sources.backend.facade.pipelines import LOCK_TAKEOVER_LATEST_ERRORfrom products.warehouse_sources.backend.facade.types import (ExternalDataJobPipelineVersion,@@ -524,14 +529,15 @@ class TestUpdateExternalJobStatus:class TestFinalizeQueueSweep:@pytest.mark.parametrize(- "status",+ "status,latest_error",[- ExternalDataJobStatus.FAILED,- ExternalDataJobStatus.BILLING_LIMIT_REACHED,- ExternalDataJobStatus.BILLING_LIMIT_TOO_LOW,+ (ExternalDataJobStatus.FAILED, "boom"),+ (ExternalDataJobStatus.FAILED, "Sync cancelled by user"),+ (ExternalDataJobStatus.BILLING_LIMIT_REACHED, "boom"),+ (ExternalDataJobStatus.BILLING_LIMIT_TOO_LOW, "boom"),],)- def test_v3_risky_terminal_sweeps_queue_batches(self, status, django_capture_on_commit_callbacks):+ def test_v3_risky_terminal_sweeps_queue_batches(self, status, latest_error, django_capture_on_commit_callbacks):# Any non-Completed terminal can leave enqueued batches behind; the sweep must fire# after commit or the batches stay claimable until the retention prune.team, _source, _schema, job = _create_org_team_source_schema_job(@@ -553,26 +559,29 @@ class TestFinalizeQueueSweep:team_id=team.pk,status=status,logger=MagicMock(),- latest_error="boom",+ latest_error=latest_error,)mock_sweep.assert_called_once()assert mock_sweep.call_args.kwargs["job_id"] == str(job.id)@pytest.mark.parametrize(- "pipeline_version,status",+ "pipeline_version,status,latest_error",[# A normally-completed v3 run has drained its queue by construction; the fleet's# hottest status write must not gain a cross-DB call.- (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.COMPLETED),+ (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.COMPLETED, None),+ # The next job continues the cursor of a run that worker restarts stopped, so the loader+ # has to drain what that run queued.+ (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.FAILED, WORKER_RESTART_ERROR_MESSAGE),# v1/v2 have no batch queue at all.- (ExternalDataJobPipelineVersion.V2, ExternalDataJobStatus.FAILED),- (ExternalDataJobPipelineVersion.V1, ExternalDataJobStatus.FAILED),- (None, ExternalDataJobStatus.FAILED),+ (ExternalDataJobPipelineVersion.V2, ExternalDataJobStatus.FAILED, "boom"),+ (ExternalDataJobPipelineVersion.V1, ExternalDataJobStatus.FAILED, "boom"),+ (None, ExternalDataJobStatus.FAILED, "boom"),],)def test_no_queue_sweep_for_normal_completion_or_non_v3(- self, pipeline_version, status, django_capture_on_commit_callbacks+ self, pipeline_version, status, latest_error, django_capture_on_commit_callbacks):team, _source, _schema, job = _create_org_team_source_schema_job(pipeline_version=pipeline_version)@@ -591,7 +600,7 @@ class TestFinalizeQueueSweep:team_id=team.pk,status=status,logger=MagicMock(),- latest_error="boom" if status == ExternalDataJobStatus.FAILED else None,+ latest_error=latest_error,)mock_sweep.assert_not_called()diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.pyindex 97dffee054a0..8dc1ba4d8d07 100644--- a/products/warehouse_sources/backend/facade/models.py+++ b/products/warehouse_sources/backend/facade/models.py@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (MAX_FULL_REFRESH_INTERVAL_DAYS,SCHEDULED_FULL_REFRESH_SYNC_TYPES,SCHEMA_RESOURCE_ID_METADATA_KEY,+ WORKER_RESTART_ERROR_MESSAGE,ExternalDataSchema,auto_enable_new_schemas,discovered_schema_finder,@@ -104,6 +105,7 @@ __all__ = ["SCHEMA_RESOURCE_ID_METADATA_KEY","SCHEDULED_FULL_REFRESH_SYNC_TYPES","SERIALIZED_FIELD_TO_CLICKHOUSE_MAPPING",+ "WORKER_RESTART_ERROR_MESSAGE","WarehouseColumnAnnotation","WarehouseColumnStatistics","acreate_datawarehousetable",diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex 238d733f957f..e8cbe8be3ca6 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -83,6 +83,10 @@ def resolve_sync_type(value: str | None) -> ExternalDataSchemaSyncType | None:# Recorded as the job's latest_error, which the syncs UI shows to the customer.SYNC_DISABLED_JOB_ERROR = "Sync stopped because syncing was turned off"SCHEMA_DELETED_JOB_ERROR = "Sync stopped because the table was deleted"+WORKER_RESTART_ERROR_MESSAGE = (+ "This sync run was interrupted too many times by restarts on PostHog's side, so it did not finish. "+ "It will run again automatically. No action is needed."+)AUTO_DISABLED_JOB_ERROR = "Sync stopped because of an error that retrying would not fix"# `Any_Source_Errors` rewrites the raised exception into this copy, so a blocked schema carries itdiff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex 12c370d6dd5b..2378d2dd254a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -50,6 +50,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (MISSING_PRIMARY_KEY_DISABLED_MESSAGE,UNSUPPORTED_SYNC_TYPE_DISABLED_MESSAGE,UNSUPPORTED_SYNC_TYPE_ERROR,+ WORKER_RESTART_ERROR_MESSAGE,ExternalDataSchema,update_should_sync,)@@ -393,11 +394,6 @@ CANCELLED_RUN_MESSAGE = ("it or the source is paused. It will run again on its next schedule.")-WORKER_RESTART_ERROR_MESSAGE = (- "This sync run was interrupted too many times by restarts on PostHog's side, so it did not finish. "- "It will run again automatically. No action is needed."-)-TRANSIENT_SOURCE_ERROR_MESSAGE = ("The source's API kept returning temporary errors, such as rate limits or server errors, so this ""sync run did not finish. This is usually a short problem on the source's side. The sync will run "diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.pyindex 3279780a3882..ace3fb8de0c4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py@@ -99,6 +99,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,kept_rows_synced: int | None = None,+ continued_workflow_run_id: str | None = None,+ continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,) -> None:@@ -120,6 +122,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ continued_workflow_run_id=continued_workflow_run_id,+ continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex d3d341c74863..77bff085a6ee 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -616,6 +616,7 @@ def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:before_batch_index=export_signal.batch_index,)if unfinished.count == 0:+ _refuse_unloaded_continued_jobs(export_signal)returnINCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()@@ -633,6 +634,37 @@ def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:raise IncompleteRunError()+def _refuse_unloaded_continued_jobs(export_signal: ExportSignalMessage) -> None:+ """Raise when a batch of a job whose cursor this run continues did not load.++ Those batches hold the rows before the continued cursor, and no later run reads them again+ once this run completes. A failed job keeps the committed cursor, and the next job does not+ continue it, so that job reads those rows again.+ """+ if not export_signal.continued_job_ids:+ return+ unfinished = BatchQueue.get_unfinished_batches_of_jobs_sync(+ settings.WAREHOUSE_SOURCES_DATABASE_URL,+ team_id=export_signal.team_id,+ schema_id=export_signal.schema_id,+ job_ids=export_signal.continued_job_ids,+ )+ if unfinished.count == 0:+ return++ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()+ logger.error(+ "run_completion_refused_unfinished_continued_job_batches",+ team_id=export_signal.team_id,+ external_data_schema_id=export_signal.schema_id,+ external_data_job_id=export_signal.job_id,+ continued_job_ids=export_signal.continued_job_ids,+ unfinished_batches=unfinished.count,+ unfinished_states=list(unfinished.states),+ )+ raise IncompleteRunError()++def _mark_job_completed(export_signal: ExportSignalMessage) -> None:_refuse_incomplete_run(export_signal)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex dfa47674020b..7e41e66eb19f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -49,6 +49,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_promote_staged_cursor,_read_existing_rows_by_first_pk,_rebuild_compaction_due,+ _refuse_incomplete_run,_resolve_cdc_positions,_run_post_load_for_already_processed_batch,_trigger_post_import_workflow,@@ -83,10 +84,16 @@ def _no_close_old_connections():@pytest.fixture(autouse=True)def _unfinished_earlier_batches() -> Iterator[MagicMock]:# The completion check reads the queue DB, which these unit tests do not have.- with patch(- "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",- return_value=_COMPLETE_RUN,- ) as check:+ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",+ return_value=_COMPLETE_RUN,+ ) as check,+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_batches_of_jobs_sync",+ return_value=_COMPLETE_RUN,+ ),+ ):yield check@@ -590,6 +597,32 @@ class TestMarkJobCompleted:# The consumer fails the run on this error, and its fail path releases the lock.mock_release.assert_not_called()+ @parameterized.expand(+ [+ ("a_continued_job_batch_failed", ["job-stopped"], 1, True),+ ("every_continued_job_batch_loaded", ["job-stopped"], 0, False),+ ("no_continued_job", [], 1, False),+ ]+ )+ def test_a_run_that_continues_a_stopped_job_completes_only_once_that_job_loaded(+ self, _case: str, continued_job_ids: list[str], unfinished_count: int, refused: bool+ ) -> None:+ signal = self._make_signal()+ signal.continued_job_ids = continued_job_ids+ unfinished = UnfinishedEarlierBatches(count=unfinished_count, first_batch_index=0, states=("failed",))++ with patch(f"{_PROCESSOR}.BatchQueue.get_unfinished_batches_of_jobs_sync", return_value=unfinished) as check:+ if refused:+ with pytest.raises(IncompleteRunError, match=INCOMPLETE_RUN_ERROR):+ _refuse_incomplete_run(signal)+ else:+ _refuse_incomplete_run(signal)++ if continued_job_ids:+ assert check.call_args.kwargs == {"team_id": 1, "schema_id": "schema-1", "job_ids": ["job-stopped"]}+ else:+ check.assert_not_called()+class TestRedeliveredFinalBatchPostLoad:@parameterized.expand([("companion", "scd2_append"), ("consolidated", "incremental_merge"), ("snapshot", None)])diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.pyindex b55e873f5c7d..a0ca931a10ca 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py@@ -82,6 +82,9 @@ class ExportSignalMessage:# rather than as none. Resolved once per run, because the consumer reads it per batch.external_destination_ids: Optional[list[str]] = Nonereplace_scope: Optional[ReplaceScopeRef] = None+ # Jobs that worker restarts stopped and whose cursor this run continues. Their queued batches+ # hold the rows before that cursor, so this run must not complete while one of them is unloaded.+ continued_job_ids: list[str] = field(default_factory=list)def to_dict(self) -> dict[str, Any]:return asdict(self)@@ -120,4 +123,5 @@ class ExportSignalMessage:destination_ids=data.get("destination_ids") or [],external_destination_ids=data.get("external_destination_ids"),replace_scope=ReplaceScopeRef.from_dict(data.get("replace_scope")),+ continued_job_ids=list(data.get("continued_job_ids") or []),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex e50877be504d..478b7e8aac0b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -226,6 +226,8 @@ class PipelineV3(Generic[ResumableData]):resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,kept_rows_synced: int | None = None,+ continued_workflow_run_id: str | None = None,+ continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,) -> None:@@ -386,6 +388,7 @@ class PipelineV3(Generic[ResumableData]):"partition_format": partition_format,"partition_mode": partition_mode,"is_first_ever_sync": is_first_ever_sync,+ "continued_job_ids": list(continued_job_ids),}self._pg_producer = self._build_producer(self._s3_batch_writer, resource_name=self._resource_name, cdc_write_mode=self._resource.cdc_write_mode@@ -423,9 +426,13 @@ class PipelineV3(Generic[ResumableData]):# value of the earlier attempts of this run. Without their newest value, the run promotes the# largest value of its own rows as the watermark, and the next run reads every newer row again.# `models.schema` is loaded when this attempt starts, so it holds what the earlier attempts staged.- self._last_incremental_field_value = staged_run_last_value(- models.schema.sync_type_config or {}, self._job.workflow_run_id- )+ staged_config = models.schema.sync_type_config or {}+ self._last_incremental_field_value = staged_run_last_value(staged_config, self._job.workflow_run_id)+ if self._last_incremental_field_value is None and continued_workflow_run_id is not None:+ # The first attempt of a job that continues the cursor of an interrupted job has the same+ # gap: the interrupted run read the newest rows. A later attempt of this run staged that+ # value again, so the lookup above finds it.+ self._last_incremental_field_value = staged_run_last_value(staged_config, continued_workflow_run_id)if self._last_incremental_field_value is None and self._schema.should_use_incremental_field:self._logger.warning("V3 Pipeline: Resumed non-ascending extract found no staged last value of this run",diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.pyindex 3b263cd69f07..7ce4308a5a6f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py@@ -1105,6 +1105,10 @@ class DeltaBatchConsumerAdapter:only promotes on a Completed job (``load/processor.py``). The job stays Failed here,so the cursor does not advance, the next run re-extracts the same window, and itsprimary key makes that merge idempotent. Loading is therefore strictly progress.++ The exception is a job that worker restarts stopped: the next job continues its source+ cursor and does not read those rows again, so this drain is the only way they land. That+ job may not complete while one of these batches is unloaded (``continued_job_ids``)."""if batch.sync_type != "incremental":return Falsediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.pyindex a87ed7836e5b..10626d4c5092 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py@@ -95,6 +95,7 @@ class PostgresProducer:workflow_run_id: str | None = None,destination_ids: list[str] | None = None,external_destination_ids: list[str] | None = None,+ continued_job_ids: list[str] | None = None,) -> None:self._team_id = team_idself._job_id = job_id@@ -125,6 +126,7 @@ class PostgresProducer:)self._replace_scope: ReplaceScopeRef | None = None+ self._continued_job_ids: list[str] = list(continued_job_ids or [])with _queue_db_errors():self._conn = _connect_with_retry(database_url)@@ -355,6 +357,8 @@ class PostgresProducer:metadata[LOADED_PREFIX_CURSOR_KEY] = loaded_prefix_cursorif self._replace_scope is not None:metadata["replace_scope"] = self._replace_scope.to_dict()+ if self._continued_job_ids:+ metadata["continued_job_ids"] = self._continued_job_idswith _queue_db_errors():self._conn.execute(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.pyindex 5c8d1ba70641..d5c7daf101ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py@@ -1651,6 +1651,31 @@ class TestUnfinishedEarlierBatchesSync:assert self._check(_db_url).count == 0+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "states,expected_count",+ [+ (["succeeded", "succeeded"], 0),+ (["succeeded", "failed"], 1),+ (["pending", "succeeded"], 1),+ ],+ )+ async def test_reports_the_batches_of_continued_jobs_that_did_not_succeed(+ self, conn, _db_url, states, expected_count+ ):+ for index, state in enumerate(states):+ bid = await _insert_batch(conn, batch_index=index, job_id="job-stopped", run_uuid="run-stopped-a1")+ if state != "pending":+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)+ await _insert_batch(conn, batch_index=0, job_id="job-other", run_uuid="run-other-a1")+ await _insert_batch(conn, batch_index=0, job_id="job-stopped", run_uuid="run-stopped-a1", schema_id="schema-2")++ unfinished = BatchQueue.get_unfinished_batches_of_jobs_sync(+ _db_url, team_id=1, schema_id="schema-1", job_ids=["job-stopped"], connect_timeout_seconds=5+ )++ assert unfinished.count == expected_count+@pytest.mark.asyncioasync def test_reports_the_aged_out_batch_that_the_claim_skipped(self, conn, _db_url):# The run gates do not read an aged-out 'pending' batch, so the claim hands out the finaldiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 2efb16398243..eb75a1fc6254 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1474,7 +1474,15 @@ class TestDescWatermarkAcrossResumedAttempts:},)- def _attempt(self, schema: ExternalDataSchema, attempt: int, sort_mode: str | None = "desc") -> PipelineV3:+ def _attempt(+ self,+ schema: ExternalDataSchema,+ attempt: int,+ sort_mode: str | None = "desc",+ *,+ workflow_run_id: str = "wfrun-abc",+ continued_workflow_run_id: str | None = None,+ ) -> PipelineV3:resource = MagicMock(name="events",primary_keys=["id"],@@ -1489,11 +1497,11 @@ class TestDescWatermarkAcrossResumedAttempts:webhook_only=False,sort_mode=sort_mode,)- manager = MagicMock(can_resume=MagicMock(return_value=attempt > 1))+ manager = MagicMock(can_resume=MagicMock(return_value=attempt > 1 or continued_workflow_run_id is not None))with (patch(f"{_PIPELINE}.current_import_attempt", return_value=attempt),patch(f"{_PIPELINE}.current_workflow_id", return_value="wf-1"),- patch(f"{_PIPELINE}.current_workflow_run_id", return_value="wfrun-abc"),+ patch(f"{_PIPELINE}.current_workflow_run_id", return_value=workflow_run_id),patch(f"{_PIPELINE}.S3BatchWriter"),patch(f"{_PIPELINE}.PostgresProducer"),patch(f"{_PIPELINE}.DeltaTableRef"),@@ -1507,20 +1515,35 @@ class TestDescWatermarkAcrossResumedAttempts:shutdown_monitor=MagicMock(),resumable_source_manager=manager,models=ImportJobModels(- job=MagicMock(team_id=1, workflow_run_id="wfrun-abc", billable=False, id="job-1"),+ job=MagicMock(team_id=1, workflow_run_id=workflow_run_id, billable=False, id="job-1"),schema=schema,source=MagicMock(source_type="Langfuse"),table=MagicMock(),),+ continued_workflow_run_id=continued_workflow_run_id,)async def _extract(- self, schema: ExternalDataSchema, attempt: int, *, final: bool, sort_mode: str | None = "desc"+ self,+ schema: ExternalDataSchema,+ attempt: int,+ *,+ final: bool,+ sort_mode: str | None = "desc",+ page: int | None = None,+ workflow_run_id: str = "wfrun-abc",+ continued_workflow_run_id: str | None = None,) -> str:- pipeline = self._attempt(schema, attempt, sort_mode)+ pipeline = self._attempt(+ schema,+ attempt,+ sort_mode,+ workflow_run_id=workflow_run_id,+ continued_workflow_run_id=continued_workflow_run_id,+ )run_uuid = cast(str, pipeline._run_uuid)# Each attempt resumes on an older page than the attempt before it read.- newest_row = self._NEWEST - timedelta(hours=10 * (attempt - 1))+ newest_row = self._NEWEST - timedelta(hours=10 * ((page or attempt) - 1))table = pa.table({"created_at": pa.array([newest_row, newest_row - timedelta(hours=5)], pa.timestamp("us", tz="UTC"))})@@ -1584,6 +1607,27 @@ class TestDescWatermarkAcrossResumedAttempts:assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()+ @pytest.mark.asyncio+ async def test_a_job_that_continues_an_interrupted_job_keeps_its_largest_value(self) -> None:+ schema = self._schema()++ with (+ patch.object(schema_models, "update_sync_type_config_keys", side_effect=_in_memory_config_update(schema)),+ patch.object(ExternalDataSchema, "refresh_from_db"),+ ):+ await self._extract(schema, 1, final=False, workflow_run_id="wfrun-interrupted")+ run_uuid = await self._extract(+ schema,+ 1,+ final=True,+ page=2,+ workflow_run_id="wfrun-next",+ continued_workflow_run_id="wfrun-interrupted",+ )+ assert schema.promote_staged_incremental_values(run_uuid)++ assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()+class TestZeroBatchRunStampsTheFullRunMarker:@pytest.mark.asynciodiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.pyindex 44addcf2c27b..ac99c00e01fe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py@@ -25,6 +25,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typSourceResponse,)+RESUMABLE_STATE_TTL_SECONDS = 60 * 60 * 24+class _WriteFence:"""Stops the writes of a source that the pipeline abandoned.@@ -127,9 +129,12 @@ class ResumableSourceManager(Generic[ResumableData]):yield redis+ def _job_key(self, job_id: str) -> str:+ return f"posthog:data_warehouse:resumable_source:{self._inputs.team_id}:{job_id}"+@propertydef _key(self) -> str:- base = f"posthog:data_warehouse:resumable_source:{self._inputs.team_id}:{self._inputs.job_id}"+ base = self._job_key(self._inputs.job_id)return f"{base}:{self._namespace}" if self._namespace else basedef _dump_json(self, data: ResumableData) -> str:@@ -220,7 +225,7 @@ class ResumableSourceManager(Generic[ResumableData]):self._logger.debug(f"Saving resumable source state. key={key}")self._write_with_stale_replica_retry(redis_client,- functools.partial(redis_client.set, key, json_data, ex=60 * 60 * 24), # 24 hours expiration+ functools.partial(redis_client.set, key, json_data, ex=RESUMABLE_STATE_TTL_SECONDS),)del self._staged[key]wrote = True@@ -264,6 +269,37 @@ class ResumableSourceManager(Generic[ResumableData]):self._logger.debug(f"Clearing resumable source state. key={self._key}")self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(self._key))+ def continue_from_job(self, job_id: str) -> bool:+ """Copy the resume state that `job_id` committed to this job, and return whether `job_id` had any.++ A cursor is keyed by job, and the run after an interrupted job is a new job. Without the copy+ that run reads the source from the start again, so a table whose first load takes longer than+ one job can run never finishes. The copy covers every namespace. It copies nothing once this+ job has state of its own, in any namespace: every execution of this job calls it, and a+ later one must not bring back a namespace that this job already cleared. The caller must+ make sure that every row before the copied cursor reaches the table.+ """+ source_base = self._job_key(job_id)+ target_base = self._job_key(self._inputs.job_id)+ found = 0+ with self._write_fence.writing(), self._get_redis() as redis_client:+ if redis_client.exists(target_base) or next(iter(redis_client.scan_iter(match=f"{target_base}:*")), None):+ self._logger.debug(f"Not continuing another job: this job has resume state. key={target_base}")+ return False+ for source_key in [source_base, *redis_client.scan_iter(match=f"{source_base}:*")]:+ source_key = source_key.decode() if isinstance(source_key, bytes) else source_key+ data = redis_client.get(source_key)+ if not data:+ continue+ target_key = target_base + source_key.removeprefix(source_base)+ self._write_with_stale_replica_retry(+ redis_client,+ functools.partial(redis_client.set, target_key, data, ex=RESUMABLE_STATE_TTL_SECONDS, nx=True),+ )+ found += 1+ self._logger.debug(f"Continuing the resumable source state of another job. key={source_base}, keys={found}")+ return found > 0+def can_resume(self) -> bool:with self._get_redis() as redis:exists = redis.exists(self._key) == 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.pyindex f733640b0fea..cb72b8796367 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py@@ -4,6 +4,7 @@ import threadingimport pytestfrom unittest.mock import MagicMock, patch+import fakeredisimport redis.exceptions as redis_exceptionsfrom posthog.dataclasses import frozen@@ -26,8 +27,14 @@ class _SweepPosition:cursor: str | None = None-def _manager() -> ResumableSourceManager[_SweepPosition]:- return ResumableSourceManager[_SweepPosition](MagicMock(team_id=1, job_id="job-1"), _SweepPosition)+def _manager(job_id: str = "job-1") -> ResumableSourceManager[_SweepPosition]:+ return ResumableSourceManager[_SweepPosition](MagicMock(team_id=1, job_id=job_id), _SweepPosition)+++def _commit(manager: ResumableSourceManager[_SweepPosition], cursor: str) -> None:+ manager.save_state(_SweepPosition(cursor=cursor))+ manager.confirm()+ manager.commit()class TestResolveResumeManager:@@ -195,6 +202,51 @@ class TestResumableSourceManager:assert redis.delete.call_count == 2+class TestContinueFromJob:+ @pytest.fixture(autouse=True)+ def _fake_redis(self):+ client = fakeredis.FakeRedis()+ with patch.object(ResumableSourceManager, "_get_redis") as get_redis:+ get_redis.return_value.__enter__.return_value = client+ yield++ def test_the_next_job_resumes_from_every_cursor_the_interrupted_job_committed(self):+ interrupted = _manager("job-1")+ _commit(interrupted, "ch_100")+ _commit(interrupted.with_namespace("refunds"), "re_7")++ next_job = _manager("job-2")+ assert next_job.continue_from_job("job-1") is True++ assert next_job.load_state() == _SweepPosition(cursor="ch_100")+ assert next_job.with_namespace("refunds").load_state() == _SweepPosition(cursor="re_7")++ def test_a_cursor_the_job_committed_itself_is_not_replaced(self):+ _commit(_manager("job-1"), "ch_100")+ next_job = _manager("job-2")+ next_job.continue_from_job("job-1")+ _commit(next_job, "ch_50")++ assert next_job.continue_from_job("job-1") is False++ assert next_job.load_state() == _SweepPosition(cursor="ch_50")++ def test_a_later_execution_does_not_bring_back_a_namespace_the_job_cleared(self):+ interrupted = _manager("job-1")+ _commit(interrupted, "ch_100")+ _commit(interrupted.with_namespace("refunds"), "re_7")+ next_job = _manager("job-2")+ next_job.continue_from_job("job-1")+ next_job.with_namespace("refunds").clear_state()+ next_job.clear_state()+ _commit(next_job.with_namespace("disputes"), "dp_3")++ assert next_job.continue_from_job("job-1") is False++ assert next_job.with_namespace("refunds").load_state() is None+ assert next_job.load_state() is None++class TestResumeCoversRun:@staticmethoddef _resume_state_of(source) -> type | None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex cffca5cbb481..9e00df807df3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -9,7 +9,8 @@ from typing import TYPE_CHECKING, Any, NoReturn, Optionalfrom django.conf import settingsfrom django.db import InterfaceError, InternalError, OperationalError-from django.db.models import Prefetch+from django.db.models import F, Prefetch+from django.db.models.functions import Greatestimport redis.exceptions as redis_exceptionsfrom jsonpath_ng.exceptions import JSONPathError@@ -33,6 +34,7 @@ from posthog.temporal.common.utils import is_stale_connection_read_only_errorfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import (+ WORKER_RESTART_ERROR_MESSAGE,ExternalDataSchema,UnsupportedSyncTypeError,apply_incremental_lookback,@@ -205,6 +207,68 @@ def _has_completed_schema_job(schema_id: uuid.UUID, team_id: int) -> bool:).exists()+# The v3 pipeline queues these sync types as `incremental`, and the loader still loads the queued+# batches of a failed `incremental` run (`_drainable_after_failure` in the v3 consumer). Only these+# can continue a failed job's cursor without losing the rows that job extracted.+CONTINUABLE_SYNC_TYPES = frozenset(+ {+ ExternalDataSchema.SyncType.INCREMENTAL,+ ExternalDataSchema.SyncType.WEBHOOK,+ ExternalDataSchema.SyncType.XMIN,+ }+)+++# Bounds the walk back over a chain of jobs that each continued the one before. The loader only sees+# batches of the last 14 days anyway, which is a handful of jobs at the hand-off cap.+MAX_CONTINUED_JOBS = 20+++@database_sync_to_async_pool+def _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[ExternalDataJob]:+ """The jobs before this one that worker restarts stopped, newest first, when this job can continue their cursor.++ This job continues the cursor of the first one. Each job of the chain continued the one before+ it, so the rows before that cursor are in the queued batches of all of them. The job finalizer+ leaves those batches to the loader (see `update_external_job_status`), and the loader does not+ complete this job while one of them is unloaded (`continued_job_ids`).++ A reset in the chain is not visible on the job row, so the chain can include jobs before it.+ Their batches then only make the loader refuse to complete this job once, and the next job+ reads the source from the start.+ """+ if schema.sync_type not in CONTINUABLE_SYNC_TYPES:+ return []+ previous_jobs = (+ ExternalDataJob.objects.filter(team_id=job.team_id, schema_id=schema.id, created_at__lt=job.created_at)+ .order_by("-created_at")+ .only("id", "status", "latest_error", "schema_snapshot", "workflow_run_id", "rows_synced")[:MAX_CONTINUED_JOBS]+ )+ chain: list[ExternalDataJob] = []+ for previous in previous_jobs:+ if previous.status != ExternalDataJob.Status.FAILED or previous.latest_error != WORKER_RESTART_ERROR_MESSAGE:+ break+ # A cursor belongs to the query that wrote it, and another sync type reads the source with another query.+ if (previous.schema_snapshot or {}).get("sync_type") != schema.sync_type:+ break+ chain.append(previous)+ return chain+++@database_sync_to_async_pool+def _carry_rows_synced(job: ExternalDataJob, continued: ExternalDataJob) -> None:+ """Start the row count of `job` from the count of the job it continues.++ Only a Completed job bills, and the continued job failed, so without this the rows it read+ before the cursor never bill. `Greatest` keeps a repeated call from counting them twice.+ """+ carried = continued.rows_synced or 0+ if carried <= 0:+ return+ ExternalDataJob.objects.filter(id=job.id).update(rows_synced=Greatest(F("rows_synced"), carried))+ job.rows_synced = max(job.rows_synced or 0, carried)++# An allow-list, not a deny-list: every sync type here leaves one row per key, and the reader# streams the table with no dedupe state. Append keeps a row per sync and CDC keeps change# history, so either would fan the child out once per duplicate. Webhook qualifies because its@@ -741,6 +805,8 @@ async def _import_data_with_reporting()resumable_source_manager: ResumableSourceManager | None = None+ continued_workflow_run_id: str | None = None+ continued_job_ids: tuple[str, ...] = ()try:# Some source setup functions bridge back to async code. Keep their outer blocking# call off the shared default executor so that nested work cannot deadlock behind it.@@ -748,6 +814,22 @@ async def _import_data_with_reporting(try:if isinstance(new_source, ResumableSource):resumable_source_manager = new_source.get_resumable_source_manager(source_inputs)+ jobs_to_continue = await _jobs_to_continue(model, schema) if use_stored_cursors else []+ if jobs_to_continue:+ job_to_continue = jobs_to_continue[0]+ continued_workflow_run_id = job_to_continue.workflow_run_id+ continued_job_ids = tuple(str(continued.id) for continued in jobs_to_continue)+ # The copy runs only while this job has no state of its own, so a later+ # execution never brings back a namespace this job already cleared.+ if await database_sync_to_async_pool(resumable_source_manager.continue_from_job)(+ str(job_to_continue.id)+ ):+ await _carry_rows_synced(model, job_to_continue)+ await logger.ainfo(+ "Continuing the resumable cursor of the previous job, which stopped before it finished",+ previous_job_id=str(job_to_continue.id),+ continued_job_count=len(jobs_to_continue),+ )source_response = await database_sync_to_async_pool(new_source.source_for_pipeline, executor=executor)(config, resumable_source_manager, source_inputs)@@ -800,6 +882,8 @@ async def _import_data_with_reporting(resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ continued_workflow_run_id=continued_workflow_run_id,+ continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,)@@ -1190,6 +1274,8 @@ async def _run(preemption: PreemptionConfig | None = None,on_resumable_retry_budget: bool = False,kept_rows_synced: int | None = None,+ continued_workflow_run_id: str | None = None,+ continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,) -> PipelineResult:@@ -1213,6 +1299,8 @@ async def _run(resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ continued_workflow_run_id=continued_workflow_run_id,+ continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex eae58a94d940..20b7e087a7e9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -30,7 +30,10 @@ from posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.common.shutdown import WorkerShuttingDownErrorfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_schema import (+ WORKER_RESTART_ERROR_MESSAGE,+ ExternalDataSchema,+)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.external_data_job import (@@ -51,6 +54,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpersfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline import PipelineV3from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTSfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (+ ResumableSource,SimpleSource,SourceExtractionNotImplementedError,)@@ -305,6 +309,114 @@ class TestSchemaSyncHistory(BaseTest):assert has_completed_job is expected+class TestJobToContinue(BaseTest):+ @parameterized.expand(+ [+ (+ "worker_restarts",+ ExternalDataJob.Status.FAILED,+ WORKER_RESTART_ERROR_MESSAGE,+ "webhook",+ "webhook",+ True,+ ),+ ("user_cancel", ExternalDataJob.Status.FAILED, "Sync cancelled by user", "webhook", "webhook", False),+ ("source_error", ExternalDataJob.Status.FAILED, "401 Unauthorized", "webhook", "webhook", False),+ ("completed", ExternalDataJob.Status.COMPLETED, None, "webhook", "webhook", False),+ (+ "sync_type_changed",+ ExternalDataJob.Status.FAILED,+ WORKER_RESTART_ERROR_MESSAGE,+ "incremental",+ "webhook",+ False,+ ),+ (+ "full_refresh",+ ExternalDataJob.Status.FAILED,+ WORKER_RESTART_ERROR_MESSAGE,+ "full_refresh",+ "full_refresh",+ False,+ ),+ ]+ )+ def test_only_a_job_stopped_by_worker_restarts_hands_its_cursor_to_the_next_job(+ self, _name: str, status: str, latest_error: str | None, previous_sync_type: str, sync_type: str, expected: bool+ ) -> None:+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"+ )+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type=sync_type)+ previous = ExternalDataJob.objects.create(+ team=self.team,+ pipeline=source,+ schema=schema,+ status=status,+ latest_error=latest_error,+ schema_snapshot={"sync_type": previous_sync_type},+ )+ job = ExternalDataJob.objects.create(+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING+ )+ ExternalDataJob.objects.filter(id=previous.id).update(created_at=job.created_at - timedelta(minutes=1))++ jobs_to_continue = cast(Any, module._jobs_to_continue).func(job, schema)++ assert [continued.id for continued in jobs_to_continue] == ([previous.id] if expected else [])++ def test_the_chain_stops_at_the_first_job_that_worker_restarts_did_not_stop(self) -> None:+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"+ )+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type="webhook")++ def stopped() -> ExternalDataJob:+ return ExternalDataJob.objects.create(+ team=self.team,+ pipeline=source,+ schema=schema,+ status=ExternalDataJob.Status.FAILED,+ latest_error=WORKER_RESTART_ERROR_MESSAGE,+ schema_snapshot={"sync_type": "webhook"},+ )++ before_completed = stopped()+ completed = ExternalDataJob.objects.create(+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.COMPLETED+ )+ older, newer = stopped(), stopped()+ job = ExternalDataJob.objects.create(+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING+ )+ for minutes, previous in enumerate([newer, older, completed, before_completed], start=1):+ ExternalDataJob.objects.filter(id=previous.id).update(+ created_at=job.created_at - timedelta(minutes=minutes)+ )++ jobs_to_continue = cast(Any, module._jobs_to_continue).func(job, schema)++ assert [continued.id for continued in jobs_to_continue] == [newer.id, older.id]++ def test_the_continuing_job_counts_the_rows_of_the_job_it_continues_once(self) -> None:+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"+ )+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type="webhook")+ stopped = ExternalDataJob.objects.create(+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.FAILED, rows_synced=500+ )+ job = ExternalDataJob.objects.create(+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING, rows_synced=0+ )++ cast(Any, module._carry_rows_synced).func(job, stopped)+ cast(Any, module._carry_rows_synced).func(job, stopped)++ job.refresh_from_db()+ assert job.rows_synced == 500++@pytest.mark.asyncioasync def test_non_retryable_setup_error_routes_through_handler():# A MongoDB mongodb+srv:// URI resolves DNS in the MongoClient constructor, so a deleted@@ -1215,6 +1327,42 @@ async def test_reset_run_drops_cursors_but_preserves_sync_history(assert source_inputs.schema_has_ever_synced is expected_has_ever_synced+ "reset_pipeline,copied,continues",+ [(False, True, True), (False, False, True), (True, False, False)],+ ids=["first_execution", "later_execution", "reset"],+)+async def test_a_resumable_run_continues_the_cursor_of_the_interrupted_job_unless_it_resets(+ reset_pipeline: bool, copied: bool, continues: bool+) -> None:+ source = mock.MagicMock(spec=ResumableSource)+ source.parse_config.return_value = {}+ source.source_for_pipeline.return_value = mock.MagicMock()+ manager = source.get_resumable_source_manager.return_value+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)++ interrupted = mock.MagicMock(id="interrupted-job", workflow_run_id="interrupted-run")+ earlier = mock.MagicMock(id="earlier-job", workflow_run_id="earlier-run")+ manager.continue_from_job.return_value = copied++ with (+ _patched_activity_reaching_run(source, schema) as run_mock,+ mock.patch.object(module, "_jobs_to_continue", new=mock.AsyncMock(return_value=[interrupted, earlier])),+ mock.patch.object(module, "_carry_rows_synced", new=mock.AsyncMock()) as carry,+ ):+ await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), reset_pipeline=reset_pipeline))++ if continues:+ manager.continue_from_job.assert_called_once_with("interrupted-job")+ else:+ manager.continue_from_job.assert_not_called()+ # A later execution of the continuing job copies nothing, and still holds the loader to the stopped jobs.+ assert carry.await_count == (1 if continues and copied else 0)+ assert run_mock.call_args.kwargs["continued_workflow_run_id"] == ("interrupted-run" if continues else None)+ assert run_mock.call_args.kwargs["continued_job_ids"] == (("interrupted-job", "earlier-job") if continues else ())++@pytest.mark.asyncioasync def test_pending_delta_revive_extracts_full_table_not_incremental_slice():source = mock.MagicMock(spec=SimpleSource)diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.pyindex 952116c4f620..e5ce7d797109 100644--- a/products/warehouse_sources_queue/backend/core/jobs_db.py+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py@@ -1000,6 +1000,16 @@ _UNFINISHED_EARLIER_BATCHES_SQL = f"""AND b.latest_state <> 'succeeded'"""+_UNFINISHED_BATCHES_OF_JOBS_SQL = f"""+ SELECT count(*), MIN(b.batch_index), COALESCE(array_agg(DISTINCT b.latest_state), '{{}}')+ FROM {BATCH_TABLE} b+ WHERE b.job_id = ANY(%(job_ids)s)+ AND b.team_id = %(team_id)s+ AND b.schema_id = %(schema_id)s+ AND b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+ AND b.latest_state <> 'succeeded'+"""+# Batch metadata key for the watermark that the loaded batches of a run cover once this batch and# every earlier batch of the run are loaded. Only the producer writes it (see `get_loaded_prefix_sync`).@@ -1218,6 +1228,7 @@ class PendingBatch:"destination_ids": self.destination_ids or [],"external_destination_ids": self.metadata.get("external_destination_ids"),"replace_scope": self.metadata.get("replace_scope"),+ "continued_job_ids": self.metadata.get("continued_job_ids"),}@@ -2132,6 +2143,52 @@ class BatchQueue:)raise AssertionError("unreachable")+ @staticmethod+ def get_unfinished_batches_of_jobs_sync(+ database_url: str,+ *,+ team_id: int,+ schema_id: str,+ job_ids: list[str],+ connect_timeout_seconds: int = 10,+ statement_timeout_ms: int = UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS,+ ) -> UnfinishedEarlierBatches:+ """The batches of ``job_ids`` that did not reach 'succeeded'.++ The loader asks this before a run that continues the cursor of these jobs+ completes. Their batches hold the rows before that cursor, so one that did+ not load means the continuing run is not complete. Same pool and retry as+ :meth:`get_unfinished_earlier_batches_sync`.++ Raises ``psycopg.errors.QueryCanceled`` past ``statement_timeout_ms``.+ """+ pool = _sync_connection_pool(database_url, connect_timeout_seconds)+ params = {+ "team_id": team_id,+ "schema_id": schema_id,+ "job_ids": job_ids,+ }+ for attempt in (1, 2):+ with pool.connection() as conn:+ try:+ # The timeout is transaction-local, so it never leaks into the pooled session.+ with conn.transaction():+ conn.execute("SELECT set_config('statement_timeout', %s, true)", (str(statement_timeout_ms),))+ row = conn.execute(_UNFINISHED_BATCHES_OF_JOBS_SQL, params).fetchone()+ except psycopg.errors.QueryCanceled:+ raise+ except psycopg.OperationalError:+ pool.discard(conn)+ if attempt == 2:+ raise+ continue+ assert row is not None+ count, first_batch_index, states = row+ return UnfinishedEarlierBatches(+ count=count, first_batch_index=first_batch_index, states=tuple(sorted(states))+ )+ raise AssertionError("unreachable")+@staticmethoddef get_loaded_prefix_sync(database_url: str,