Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): continue the cursor of a job that worker restarts stopped (#109760)

posthog[bot]32f01c2c2ce7
diff --git a/products/data_warehouse/backend/logic/external_data_source/jobs.py b/products/data_warehouse/backend/logic/external_data_source/jobs.py
index 1532099b68f7..cda841547560 100644
--- a/products/data_warehouse/backend/logic/external_data_source/jobs.py
+++ b/products/data_warehouse/backend/logic/external_data_source/jobs.py
@@ -10,7 +10,11 @@ from structlog.types import FilteringBoundLogger
from posthog.exceptions_capture import capture_exception
from products.data_warehouse.backend.tasks.tasks import schedule_external_data_failure_digest
-from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema
+from products.warehouse_sources.backend.facade.models import (
+ WORKER_RESTART_ERROR_MESSAGE,
+ ExternalDataJob,
+ ExternalDataSchema,
+)
from products.warehouse_sources.backend.facade.pipelines import (
LOCK_TAKEOVER_LATEST_ERROR,
TERMINAL_JOB_STATUSES,
@@ -180,10 +184,18 @@ def update_external_job_status(
# later can stale-overwrite newer data or flip this job's status. A normal v3 completion
# has drained the queue by construction, so the common completion path stays queue-free.
# on_commit: the queue is a separate Postgres; never call it holding these row locks.
+ # Worker restarts stop a run without a final batch, and Failed stays absorbing, so a later
+ # batch can neither complete the job nor promote its cursor. The loader drains those batches
+ # for a merge sync and fails them for any other, and the next job continues from the cursor
+ # that this run committed. Sweeping them would leave a gap of rows that no run reads again.
+ stopped_by_worker_restarts = (
+ status == ExternalDataJobStatus.FAILED and error_to_persist == WORKER_RESTART_ERROR_MESSAGE
+ )
should_sweep_queue = (
model.pipeline_version == ExternalDataJobPipelineVersion.V3
and status in TERMINAL_JOB_STATUSES
and (status != ExternalDataJobStatus.COMPLETED or is_takeover_recovery)
+ and not stopped_by_worker_restarts
)
if should_sweep_queue:
transaction.on_commit(
diff --git a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
index 36880fb8f865..c162b81e7945 100644
--- a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
+++ b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
@@ -9,7 +9,12 @@ from temporalio.exceptions import CancelledError
from posthog.models import Organization, Team
from products.data_warehouse.backend.logic.external_data_source.jobs import update_external_job_status
-from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema, ExternalDataSource
+from products.warehouse_sources.backend.facade.models import (
+ WORKER_RESTART_ERROR_MESSAGE,
+ ExternalDataJob,
+ ExternalDataSchema,
+ ExternalDataSource,
+)
from products.warehouse_sources.backend.facade.pipelines import LOCK_TAKEOVER_LATEST_ERROR
from products.warehouse_sources.backend.facade.types import (
ExternalDataJobPipelineVersion,
@@ -524,14 +529,15 @@ class TestUpdateExternalJobStatus:
class TestFinalizeQueueSweep:
@pytest.mark.parametrize(
- "status",
+ "status,latest_error",
[
- ExternalDataJobStatus.FAILED,
- ExternalDataJobStatus.BILLING_LIMIT_REACHED,
- ExternalDataJobStatus.BILLING_LIMIT_TOO_LOW,
+ (ExternalDataJobStatus.FAILED, "boom"),
+ (ExternalDataJobStatus.FAILED, "Sync cancelled by user"),
+ (ExternalDataJobStatus.BILLING_LIMIT_REACHED, "boom"),
+ (ExternalDataJobStatus.BILLING_LIMIT_TOO_LOW, "boom"),
],
)
- def test_v3_risky_terminal_sweeps_queue_batches(self, status, django_capture_on_commit_callbacks):
+ def test_v3_risky_terminal_sweeps_queue_batches(self, status, latest_error, django_capture_on_commit_callbacks):
# Any non-Completed terminal can leave enqueued batches behind; the sweep must fire
# after commit or the batches stay claimable until the retention prune.
team, _source, _schema, job = _create_org_team_source_schema_job(
@@ -553,26 +559,29 @@ class TestFinalizeQueueSweep:
team_id=team.pk,
status=status,
logger=MagicMock(),
- latest_error="boom",
+ latest_error=latest_error,
)
mock_sweep.assert_called_once()
assert mock_sweep.call_args.kwargs["job_id"] == str(job.id)
@pytest.mark.parametrize(
- "pipeline_version,status",
+ "pipeline_version,status,latest_error",
[
# A normally-completed v3 run has drained its queue by construction; the fleet's
# hottest status write must not gain a cross-DB call.
- (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.COMPLETED),
+ (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.COMPLETED, None),
+ # The next job continues the cursor of a run that worker restarts stopped, so the loader
+ # has to drain what that run queued.
+ (ExternalDataJobPipelineVersion.V3, ExternalDataJobStatus.FAILED, WORKER_RESTART_ERROR_MESSAGE),
# v1/v2 have no batch queue at all.
- (ExternalDataJobPipelineVersion.V2, ExternalDataJobStatus.FAILED),
- (ExternalDataJobPipelineVersion.V1, ExternalDataJobStatus.FAILED),
- (None, ExternalDataJobStatus.FAILED),
+ (ExternalDataJobPipelineVersion.V2, ExternalDataJobStatus.FAILED, "boom"),
+ (ExternalDataJobPipelineVersion.V1, ExternalDataJobStatus.FAILED, "boom"),
+ (None, ExternalDataJobStatus.FAILED, "boom"),
],
)
def test_no_queue_sweep_for_normal_completion_or_non_v3(
- self, pipeline_version, status, django_capture_on_commit_callbacks
+ self, pipeline_version, status, latest_error, django_capture_on_commit_callbacks
):
team, _source, _schema, job = _create_org_team_source_schema_job(pipeline_version=pipeline_version)
@@ -591,7 +600,7 @@ class TestFinalizeQueueSweep:
team_id=team.pk,
status=status,
logger=MagicMock(),
- latest_error="boom" if status == ExternalDataJobStatus.FAILED else None,
+ latest_error=latest_error,
)
mock_sweep.assert_not_called()
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.py
index 97dffee054a0..8dc1ba4d8d07 100644
--- a/products/warehouse_sources/backend/facade/models.py
+++ b/products/warehouse_sources/backend/facade/models.py
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
MAX_FULL_REFRESH_INTERVAL_DAYS,
SCHEDULED_FULL_REFRESH_SYNC_TYPES,
SCHEMA_RESOURCE_ID_METADATA_KEY,
+ WORKER_RESTART_ERROR_MESSAGE,
ExternalDataSchema,
auto_enable_new_schemas,
discovered_schema_finder,
@@ -104,6 +105,7 @@ __all__ = [
"SCHEMA_RESOURCE_ID_METADATA_KEY",
"SCHEDULED_FULL_REFRESH_SYNC_TYPES",
"SERIALIZED_FIELD_TO_CLICKHOUSE_MAPPING",
+ "WORKER_RESTART_ERROR_MESSAGE",
"WarehouseColumnAnnotation",
"WarehouseColumnStatistics",
"acreate_datawarehousetable",
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.py
index 238d733f957f..e8cbe8be3ca6 100644
--- a/products/warehouse_sources/backend/models/external_data_schema.py
+++ b/products/warehouse_sources/backend/models/external_data_schema.py
@@ -83,6 +83,10 @@ def resolve_sync_type(value: str | None) -> ExternalDataSchemaSyncType | None:
# Recorded as the job's latest_error, which the syncs UI shows to the customer.
SYNC_DISABLED_JOB_ERROR = "Sync stopped because syncing was turned off"
SCHEMA_DELETED_JOB_ERROR = "Sync stopped because the table was deleted"
+WORKER_RESTART_ERROR_MESSAGE = (
+ "This sync run was interrupted too many times by restarts on PostHog's side, so it did not finish. "
+ "It will run again automatically. No action is needed."
+)
AUTO_DISABLED_JOB_ERROR = "Sync stopped because of an error that retrying would not fix"
# `Any_Source_Errors` rewrites the raised exception into this copy, so a blocked schema carries it
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index 12c370d6dd5b..2378d2dd254a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -50,6 +50,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
MISSING_PRIMARY_KEY_DISABLED_MESSAGE,
UNSUPPORTED_SYNC_TYPE_DISABLED_MESSAGE,
UNSUPPORTED_SYNC_TYPE_ERROR,
+ WORKER_RESTART_ERROR_MESSAGE,
ExternalDataSchema,
update_should_sync,
)
@@ -393,11 +394,6 @@ CANCELLED_RUN_MESSAGE = (
"it or the source is paused. It will run again on its next schedule."
)
-WORKER_RESTART_ERROR_MESSAGE = (
- "This sync run was interrupted too many times by restarts on PostHog's side, so it did not finish. "
- "It will run again automatically. No action is needed."
-)
-
TRANSIENT_SOURCE_ERROR_MESSAGE = (
"The source's API kept returning temporary errors, such as rate limits or server errors, so this "
"sync run did not finish. This is usually a short problem on the source's side. The sync will run "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
index 3279780a3882..ace3fb8de0c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
@@ -99,6 +99,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
resumed_incremental_value: Any = None,
preemption: PreemptionConfig | None = None,
kept_rows_synced: int | None = None,
+ continued_workflow_run_id: str | None = None,
+ continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
) -> None:
@@ -120,6 +122,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ continued_workflow_run_id=continued_workflow_run_id,
+ continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index d3d341c74863..77bff085a6ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -616,6 +616,7 @@ def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:
before_batch_index=export_signal.batch_index,
)
if unfinished.count == 0:
+ _refuse_unloaded_continued_jobs(export_signal)
return
INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()
@@ -633,6 +634,37 @@ def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:
raise IncompleteRunError()
+def _refuse_unloaded_continued_jobs(export_signal: ExportSignalMessage) -> None:
+ """Raise when a batch of a job whose cursor this run continues did not load.
+
+ Those batches hold the rows before the continued cursor, and no later run reads them again
+ once this run completes. A failed job keeps the committed cursor, and the next job does not
+ continue it, so that job reads those rows again.
+ """
+ if not export_signal.continued_job_ids:
+ return
+ unfinished = BatchQueue.get_unfinished_batches_of_jobs_sync(
+ settings.WAREHOUSE_SOURCES_DATABASE_URL,
+ team_id=export_signal.team_id,
+ schema_id=export_signal.schema_id,
+ job_ids=export_signal.continued_job_ids,
+ )
+ if unfinished.count == 0:
+ return
+
+ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()
+ logger.error(
+ "run_completion_refused_unfinished_continued_job_batches",
+ team_id=export_signal.team_id,
+ external_data_schema_id=export_signal.schema_id,
+ external_data_job_id=export_signal.job_id,
+ continued_job_ids=export_signal.continued_job_ids,
+ unfinished_batches=unfinished.count,
+ unfinished_states=list(unfinished.states),
+ )
+ raise IncompleteRunError()
+
+
def _mark_job_completed(export_signal: ExportSignalMessage) -> None:
_refuse_incomplete_run(export_signal)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index dfa47674020b..7e41e66eb19f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -49,6 +49,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
_promote_staged_cursor,
_read_existing_rows_by_first_pk,
_rebuild_compaction_due,
+ _refuse_incomplete_run,
_resolve_cdc_positions,
_run_post_load_for_already_processed_batch,
_trigger_post_import_workflow,
@@ -83,10 +84,16 @@ def _no_close_old_connections():
@pytest.fixture(autouse=True)
def _unfinished_earlier_batches() -> Iterator[MagicMock]:
# The completion check reads the queue DB, which these unit tests do not have.
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",
- return_value=_COMPLETE_RUN,
- ) as check:
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",
+ return_value=_COMPLETE_RUN,
+ ) as check,
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_batches_of_jobs_sync",
+ return_value=_COMPLETE_RUN,
+ ),
+ ):
yield check
@@ -590,6 +597,32 @@ class TestMarkJobCompleted:
# The consumer fails the run on this error, and its fail path releases the lock.
mock_release.assert_not_called()
+ @parameterized.expand(
+ [
+ ("a_continued_job_batch_failed", ["job-stopped"], 1, True),
+ ("every_continued_job_batch_loaded", ["job-stopped"], 0, False),
+ ("no_continued_job", [], 1, False),
+ ]
+ )
+ def test_a_run_that_continues_a_stopped_job_completes_only_once_that_job_loaded(
+ self, _case: str, continued_job_ids: list[str], unfinished_count: int, refused: bool
+ ) -> None:
+ signal = self._make_signal()
+ signal.continued_job_ids = continued_job_ids
+ unfinished = UnfinishedEarlierBatches(count=unfinished_count, first_batch_index=0, states=("failed",))
+
+ with patch(f"{_PROCESSOR}.BatchQueue.get_unfinished_batches_of_jobs_sync", return_value=unfinished) as check:
+ if refused:
+ with pytest.raises(IncompleteRunError, match=INCOMPLETE_RUN_ERROR):
+ _refuse_incomplete_run(signal)
+ else:
+ _refuse_incomplete_run(signal)
+
+ if continued_job_ids:
+ assert check.call_args.kwargs == {"team_id": 1, "schema_id": "schema-1", "job_ids": ["job-stopped"]}
+ else:
+ check.assert_not_called()
+
class TestRedeliveredFinalBatchPostLoad:
@parameterized.expand([("companion", "scd2_append"), ("consolidated", "incremental_merge"), ("snapshot", None)])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py
index b55e873f5c7d..a0ca931a10ca 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/messages.py
@@ -82,6 +82,9 @@ class ExportSignalMessage:
# rather than as none. Resolved once per run, because the consumer reads it per batch.
external_destination_ids: Optional[list[str]] = None
replace_scope: Optional[ReplaceScopeRef] = None
+ # Jobs that worker restarts stopped and whose cursor this run continues. Their queued batches
+ # hold the rows before that cursor, so this run must not complete while one of them is unloaded.
+ continued_job_ids: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@@ -120,4 +123,5 @@ class ExportSignalMessage:
destination_ids=data.get("destination_ids") or [],
external_destination_ids=data.get("external_destination_ids"),
replace_scope=ReplaceScopeRef.from_dict(data.get("replace_scope")),
+ continued_job_ids=list(data.get("continued_job_ids") or []),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index e50877be504d..478b7e8aac0b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -226,6 +226,8 @@ class PipelineV3(Generic[ResumableData]):
resumed_incremental_value: Any = None,
preemption: PreemptionConfig | None = None,
kept_rows_synced: int | None = None,
+ continued_workflow_run_id: str | None = None,
+ continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
) -> None:
@@ -386,6 +388,7 @@ class PipelineV3(Generic[ResumableData]):
"partition_format": partition_format,
"partition_mode": partition_mode,
"is_first_ever_sync": is_first_ever_sync,
+ "continued_job_ids": list(continued_job_ids),
}
self._pg_producer = self._build_producer(
self._s3_batch_writer, resource_name=self._resource_name, cdc_write_mode=self._resource.cdc_write_mode
@@ -423,9 +426,13 @@ class PipelineV3(Generic[ResumableData]):
# value of the earlier attempts of this run. Without their newest value, the run promotes the
# largest value of its own rows as the watermark, and the next run reads every newer row again.
# `models.schema` is loaded when this attempt starts, so it holds what the earlier attempts staged.
- self._last_incremental_field_value = staged_run_last_value(
- models.schema.sync_type_config or {}, self._job.workflow_run_id
- )
+ staged_config = models.schema.sync_type_config or {}
+ self._last_incremental_field_value = staged_run_last_value(staged_config, self._job.workflow_run_id)
+ if self._last_incremental_field_value is None and continued_workflow_run_id is not None:
+ # The first attempt of a job that continues the cursor of an interrupted job has the same
+ # gap: the interrupted run read the newest rows. A later attempt of this run staged that
+ # value again, so the lookup above finds it.
+ self._last_incremental_field_value = staged_run_last_value(staged_config, continued_workflow_run_id)
if self._last_incremental_field_value is None and self._schema.should_use_incremental_field:
self._logger.warning(
"V3 Pipeline: Resumed non-ascending extract found no staged last value of this run",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
index 3b263cd69f07..7ce4308a5a6f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
@@ -1105,6 +1105,10 @@ class DeltaBatchConsumerAdapter:
only promotes on a Completed job (``load/processor.py``). The job stays Failed here,
so the cursor does not advance, the next run re-extracts the same window, and its
primary key makes that merge idempotent. Loading is therefore strictly progress.
+
+ The exception is a job that worker restarts stopped: the next job continues its source
+ cursor and does not read those rows again, so this drain is the only way they land. That
+ job may not complete while one of these batches is unloaded (``continued_job_ids``).
"""
if batch.sync_type != "incremental":
return False
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
index a87ed7836e5b..10626d4c5092 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
@@ -95,6 +95,7 @@ class PostgresProducer:
workflow_run_id: str | None = None,
destination_ids: list[str] | None = None,
external_destination_ids: list[str] | None = None,
+ continued_job_ids: list[str] | None = None,
) -> None:
self._team_id = team_id
self._job_id = job_id
@@ -125,6 +126,7 @@ class PostgresProducer:
)
self._replace_scope: ReplaceScopeRef | None = None
+ self._continued_job_ids: list[str] = list(continued_job_ids or [])
with _queue_db_errors():
self._conn = _connect_with_retry(database_url)
@@ -355,6 +357,8 @@ class PostgresProducer:
metadata[LOADED_PREFIX_CURSOR_KEY] = loaded_prefix_cursor
if self._replace_scope is not None:
metadata["replace_scope"] = self._replace_scope.to_dict()
+ if self._continued_job_ids:
+ metadata["continued_job_ids"] = self._continued_job_ids
with _queue_db_errors():
self._conn.execute(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
index 5c8d1ba70641..d5c7daf101ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
@@ -1651,6 +1651,31 @@ class TestUnfinishedEarlierBatchesSync:
assert self._check(_db_url).count == 0
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "states,expected_count",
+ [
+ (["succeeded", "succeeded"], 0),
+ (["succeeded", "failed"], 1),
+ (["pending", "succeeded"], 1),
+ ],
+ )
+ async def test_reports_the_batches_of_continued_jobs_that_did_not_succeed(
+ self, conn, _db_url, states, expected_count
+ ):
+ for index, state in enumerate(states):
+ bid = await _insert_batch(conn, batch_index=index, job_id="job-stopped", run_uuid="run-stopped-a1")
+ if state != "pending":
+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)
+ await _insert_batch(conn, batch_index=0, job_id="job-other", run_uuid="run-other-a1")
+ await _insert_batch(conn, batch_index=0, job_id="job-stopped", run_uuid="run-stopped-a1", schema_id="schema-2")
+
+ unfinished = BatchQueue.get_unfinished_batches_of_jobs_sync(
+ _db_url, team_id=1, schema_id="schema-1", job_ids=["job-stopped"], connect_timeout_seconds=5
+ )
+
+ assert unfinished.count == expected_count
+
@pytest.mark.asyncio
async def test_reports_the_aged_out_batch_that_the_claim_skipped(self, conn, _db_url):
# The run gates do not read an aged-out 'pending' batch, so the claim hands out the final
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 2efb16398243..eb75a1fc6254 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1474,7 +1474,15 @@ class TestDescWatermarkAcrossResumedAttempts:
},
)
- def _attempt(self, schema: ExternalDataSchema, attempt: int, sort_mode: str | None = "desc") -> PipelineV3:
+ def _attempt(
+ self,
+ schema: ExternalDataSchema,
+ attempt: int,
+ sort_mode: str | None = "desc",
+ *,
+ workflow_run_id: str = "wfrun-abc",
+ continued_workflow_run_id: str | None = None,
+ ) -> PipelineV3:
resource = MagicMock(
name="events",
primary_keys=["id"],
@@ -1489,11 +1497,11 @@ class TestDescWatermarkAcrossResumedAttempts:
webhook_only=False,
sort_mode=sort_mode,
)
- manager = MagicMock(can_resume=MagicMock(return_value=attempt > 1))
+ manager = MagicMock(can_resume=MagicMock(return_value=attempt > 1 or continued_workflow_run_id is not None))
with (
patch(f"{_PIPELINE}.current_import_attempt", return_value=attempt),
patch(f"{_PIPELINE}.current_workflow_id", return_value="wf-1"),
- patch(f"{_PIPELINE}.current_workflow_run_id", return_value="wfrun-abc"),
+ patch(f"{_PIPELINE}.current_workflow_run_id", return_value=workflow_run_id),
patch(f"{_PIPELINE}.S3BatchWriter"),
patch(f"{_PIPELINE}.PostgresProducer"),
patch(f"{_PIPELINE}.DeltaTableRef"),
@@ -1507,20 +1515,35 @@ class TestDescWatermarkAcrossResumedAttempts:
shutdown_monitor=MagicMock(),
resumable_source_manager=manager,
models=ImportJobModels(
- job=MagicMock(team_id=1, workflow_run_id="wfrun-abc", billable=False, id="job-1"),
+ job=MagicMock(team_id=1, workflow_run_id=workflow_run_id, billable=False, id="job-1"),
schema=schema,
source=MagicMock(source_type="Langfuse"),
table=MagicMock(),
),
+ continued_workflow_run_id=continued_workflow_run_id,
)
async def _extract(
- self, schema: ExternalDataSchema, attempt: int, *, final: bool, sort_mode: str | None = "desc"
+ self,
+ schema: ExternalDataSchema,
+ attempt: int,
+ *,
+ final: bool,
+ sort_mode: str | None = "desc",
+ page: int | None = None,
+ workflow_run_id: str = "wfrun-abc",
+ continued_workflow_run_id: str | None = None,
) -> str:
- pipeline = self._attempt(schema, attempt, sort_mode)
+ pipeline = self._attempt(
+ schema,
+ attempt,
+ sort_mode,
+ workflow_run_id=workflow_run_id,
+ continued_workflow_run_id=continued_workflow_run_id,
+ )
run_uuid = cast(str, pipeline._run_uuid)
# Each attempt resumes on an older page than the attempt before it read.
- newest_row = self._NEWEST - timedelta(hours=10 * (attempt - 1))
+ newest_row = self._NEWEST - timedelta(hours=10 * ((page or attempt) - 1))
table = pa.table(
{"created_at": pa.array([newest_row, newest_row - timedelta(hours=5)], pa.timestamp("us", tz="UTC"))}
)
@@ -1584,6 +1607,27 @@ class TestDescWatermarkAcrossResumedAttempts:
assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()
+ @pytest.mark.asyncio
+ async def test_a_job_that_continues_an_interrupted_job_keeps_its_largest_value(self) -> None:
+ schema = self._schema()
+
+ with (
+ patch.object(schema_models, "update_sync_type_config_keys", side_effect=_in_memory_config_update(schema)),
+ patch.object(ExternalDataSchema, "refresh_from_db"),
+ ):
+ await self._extract(schema, 1, final=False, workflow_run_id="wfrun-interrupted")
+ run_uuid = await self._extract(
+ schema,
+ 1,
+ final=True,
+ page=2,
+ workflow_run_id="wfrun-next",
+ continued_workflow_run_id="wfrun-interrupted",
+ )
+ assert schema.promote_staged_incremental_values(run_uuid)
+
+ assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()
+
class TestZeroBatchRunStampsTheFullRunMarker:
@pytest.mark.asyncio
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
index 44addcf2c27b..ac99c00e01fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
@@ -25,6 +25,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
SourceResponse,
)
+RESUMABLE_STATE_TTL_SECONDS = 60 * 60 * 24
+
class _WriteFence:
"""Stops the writes of a source that the pipeline abandoned.
@@ -127,9 +129,12 @@ class ResumableSourceManager(Generic[ResumableData]):
yield redis
+ def _job_key(self, job_id: str) -> str:
+ return f"posthog:data_warehouse:resumable_source:{self._inputs.team_id}:{job_id}"
+
@property
def _key(self) -> str:
- base = f"posthog:data_warehouse:resumable_source:{self._inputs.team_id}:{self._inputs.job_id}"
+ base = self._job_key(self._inputs.job_id)
return f"{base}:{self._namespace}" if self._namespace else base
def _dump_json(self, data: ResumableData) -> str:
@@ -220,7 +225,7 @@ class ResumableSourceManager(Generic[ResumableData]):
self._logger.debug(f"Saving resumable source state. key={key}")
self._write_with_stale_replica_retry(
redis_client,
- functools.partial(redis_client.set, key, json_data, ex=60 * 60 * 24), # 24 hours expiration
+ functools.partial(redis_client.set, key, json_data, ex=RESUMABLE_STATE_TTL_SECONDS),
)
del self._staged[key]
wrote = True
@@ -264,6 +269,37 @@ class ResumableSourceManager(Generic[ResumableData]):
self._logger.debug(f"Clearing resumable source state. key={self._key}")
self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(self._key))
+ def continue_from_job(self, job_id: str) -> bool:
+ """Copy the resume state that `job_id` committed to this job, and return whether `job_id` had any.
+
+ A cursor is keyed by job, and the run after an interrupted job is a new job. Without the copy
+ that run reads the source from the start again, so a table whose first load takes longer than
+ one job can run never finishes. The copy covers every namespace. It copies nothing once this
+ job has state of its own, in any namespace: every execution of this job calls it, and a
+ later one must not bring back a namespace that this job already cleared. The caller must
+ make sure that every row before the copied cursor reaches the table.
+ """
+ source_base = self._job_key(job_id)
+ target_base = self._job_key(self._inputs.job_id)
+ found = 0
+ with self._write_fence.writing(), self._get_redis() as redis_client:
+ if redis_client.exists(target_base) or next(iter(redis_client.scan_iter(match=f"{target_base}:*")), None):
+ self._logger.debug(f"Not continuing another job: this job has resume state. key={target_base}")
+ return False
+ for source_key in [source_base, *redis_client.scan_iter(match=f"{source_base}:*")]:
+ source_key = source_key.decode() if isinstance(source_key, bytes) else source_key
+ data = redis_client.get(source_key)
+ if not data:
+ continue
+ target_key = target_base + source_key.removeprefix(source_base)
+ self._write_with_stale_replica_retry(
+ redis_client,
+ functools.partial(redis_client.set, target_key, data, ex=RESUMABLE_STATE_TTL_SECONDS, nx=True),
+ )
+ found += 1
+ self._logger.debug(f"Continuing the resumable source state of another job. key={source_base}, keys={found}")
+ return found > 0
+
def can_resume(self) -> bool:
with self._get_redis() as redis:
exists = redis.exists(self._key) == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
index f733640b0fea..cb72b8796367 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
@@ -4,6 +4,7 @@ import threading
import pytest
from unittest.mock import MagicMock, patch
+import fakeredis
import redis.exceptions as redis_exceptions
from posthog.dataclasses import frozen
@@ -26,8 +27,14 @@ class _SweepPosition:
cursor: str | None = None
-def _manager() -> ResumableSourceManager[_SweepPosition]:
- return ResumableSourceManager[_SweepPosition](MagicMock(team_id=1, job_id="job-1"), _SweepPosition)
+def _manager(job_id: str = "job-1") -> ResumableSourceManager[_SweepPosition]:
+ return ResumableSourceManager[_SweepPosition](MagicMock(team_id=1, job_id=job_id), _SweepPosition)
+
+
+def _commit(manager: ResumableSourceManager[_SweepPosition], cursor: str) -> None:
+ manager.save_state(_SweepPosition(cursor=cursor))
+ manager.confirm()
+ manager.commit()
class TestResolveResumeManager:
@@ -195,6 +202,51 @@ class TestResumableSourceManager:
assert redis.delete.call_count == 2
+class TestContinueFromJob:
+ @pytest.fixture(autouse=True)
+ def _fake_redis(self):
+ client = fakeredis.FakeRedis()
+ with patch.object(ResumableSourceManager, "_get_redis") as get_redis:
+ get_redis.return_value.__enter__.return_value = client
+ yield
+
+ def test_the_next_job_resumes_from_every_cursor_the_interrupted_job_committed(self):
+ interrupted = _manager("job-1")
+ _commit(interrupted, "ch_100")
+ _commit(interrupted.with_namespace("refunds"), "re_7")
+
+ next_job = _manager("job-2")
+ assert next_job.continue_from_job("job-1") is True
+
+ assert next_job.load_state() == _SweepPosition(cursor="ch_100")
+ assert next_job.with_namespace("refunds").load_state() == _SweepPosition(cursor="re_7")
+
+ def test_a_cursor_the_job_committed_itself_is_not_replaced(self):
+ _commit(_manager("job-1"), "ch_100")
+ next_job = _manager("job-2")
+ next_job.continue_from_job("job-1")
+ _commit(next_job, "ch_50")
+
+ assert next_job.continue_from_job("job-1") is False
+
+ assert next_job.load_state() == _SweepPosition(cursor="ch_50")
+
+ def test_a_later_execution_does_not_bring_back_a_namespace_the_job_cleared(self):
+ interrupted = _manager("job-1")
+ _commit(interrupted, "ch_100")
+ _commit(interrupted.with_namespace("refunds"), "re_7")
+ next_job = _manager("job-2")
+ next_job.continue_from_job("job-1")
+ next_job.with_namespace("refunds").clear_state()
+ next_job.clear_state()
+ _commit(next_job.with_namespace("disputes"), "dp_3")
+
+ assert next_job.continue_from_job("job-1") is False
+
+ assert next_job.with_namespace("refunds").load_state() is None
+ assert next_job.load_state() is None
+
+
class TestResumeCoversRun:
@staticmethod
def _resume_state_of(source) -> type | None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index cffca5cbb481..9e00df807df3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -9,7 +9,8 @@ from typing import TYPE_CHECKING, Any, NoReturn, Optional
from django.conf import settings
from django.db import InterfaceError, InternalError, OperationalError
-from django.db.models import Prefetch
+from django.db.models import F, Prefetch
+from django.db.models.functions import Greatest
import redis.exceptions as redis_exceptions
from jsonpath_ng.exceptions import JSONPathError
@@ -33,6 +34,7 @@ from posthog.temporal.common.utils import is_stale_connection_read_only_error
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import (
+ WORKER_RESTART_ERROR_MESSAGE,
ExternalDataSchema,
UnsupportedSyncTypeError,
apply_incremental_lookback,
@@ -205,6 +207,68 @@ def _has_completed_schema_job(schema_id: uuid.UUID, team_id: int) -> bool:
).exists()
+# The v3 pipeline queues these sync types as `incremental`, and the loader still loads the queued
+# batches of a failed `incremental` run (`_drainable_after_failure` in the v3 consumer). Only these
+# can continue a failed job's cursor without losing the rows that job extracted.
+CONTINUABLE_SYNC_TYPES = frozenset(
+ {
+ ExternalDataSchema.SyncType.INCREMENTAL,
+ ExternalDataSchema.SyncType.WEBHOOK,
+ ExternalDataSchema.SyncType.XMIN,
+ }
+)
+
+
+# Bounds the walk back over a chain of jobs that each continued the one before. The loader only sees
+# batches of the last 14 days anyway, which is a handful of jobs at the hand-off cap.
+MAX_CONTINUED_JOBS = 20
+
+
+@database_sync_to_async_pool
+def _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[ExternalDataJob]:
+ """The jobs before this one that worker restarts stopped, newest first, when this job can continue their cursor.
+
+ This job continues the cursor of the first one. Each job of the chain continued the one before
+ it, so the rows before that cursor are in the queued batches of all of them. The job finalizer
+ leaves those batches to the loader (see `update_external_job_status`), and the loader does not
+ complete this job while one of them is unloaded (`continued_job_ids`).
+
+ A reset in the chain is not visible on the job row, so the chain can include jobs before it.
+ Their batches then only make the loader refuse to complete this job once, and the next job
+ reads the source from the start.
+ """
+ if schema.sync_type not in CONTINUABLE_SYNC_TYPES:
+ return []
+ previous_jobs = (
+ ExternalDataJob.objects.filter(team_id=job.team_id, schema_id=schema.id, created_at__lt=job.created_at)
+ .order_by("-created_at")
+ .only("id", "status", "latest_error", "schema_snapshot", "workflow_run_id", "rows_synced")[:MAX_CONTINUED_JOBS]
+ )
+ chain: list[ExternalDataJob] = []
+ for previous in previous_jobs:
+ if previous.status != ExternalDataJob.Status.FAILED or previous.latest_error != WORKER_RESTART_ERROR_MESSAGE:
+ break
+ # A cursor belongs to the query that wrote it, and another sync type reads the source with another query.
+ if (previous.schema_snapshot or {}).get("sync_type") != schema.sync_type:
+ break
+ chain.append(previous)
+ return chain
+
+
+@database_sync_to_async_pool
+def _carry_rows_synced(job: ExternalDataJob, continued: ExternalDataJob) -> None:
+ """Start the row count of `job` from the count of the job it continues.
+
+ Only a Completed job bills, and the continued job failed, so without this the rows it read
+ before the cursor never bill. `Greatest` keeps a repeated call from counting them twice.
+ """
+ carried = continued.rows_synced or 0
+ if carried <= 0:
+ return
+ ExternalDataJob.objects.filter(id=job.id).update(rows_synced=Greatest(F("rows_synced"), carried))
+ job.rows_synced = max(job.rows_synced or 0, carried)
+
+
# An allow-list, not a deny-list: every sync type here leaves one row per key, and the reader
# streams the table with no dedupe state. Append keeps a row per sync and CDC keeps change
# history, so either would fan the child out once per duplicate. Webhook qualifies because its
@@ -741,6 +805,8 @@ async def _import_data_with_reporting(
)
resumable_source_manager: ResumableSourceManager | None = None
+ continued_workflow_run_id: str | None = None
+ continued_job_ids: tuple[str, ...] = ()
try:
# Some source setup functions bridge back to async code. Keep their outer blocking
# call off the shared default executor so that nested work cannot deadlock behind it.
@@ -748,6 +814,22 @@ async def _import_data_with_reporting(
try:
if isinstance(new_source, ResumableSource):
resumable_source_manager = new_source.get_resumable_source_manager(source_inputs)
+ jobs_to_continue = await _jobs_to_continue(model, schema) if use_stored_cursors else []
+ if jobs_to_continue:
+ job_to_continue = jobs_to_continue[0]
+ continued_workflow_run_id = job_to_continue.workflow_run_id
+ continued_job_ids = tuple(str(continued.id) for continued in jobs_to_continue)
+ # The copy runs only while this job has no state of its own, so a later
+ # execution never brings back a namespace this job already cleared.
+ if await database_sync_to_async_pool(resumable_source_manager.continue_from_job)(
+ str(job_to_continue.id)
+ ):
+ await _carry_rows_synced(model, job_to_continue)
+ await logger.ainfo(
+ "Continuing the resumable cursor of the previous job, which stopped before it finished",
+ previous_job_id=str(job_to_continue.id),
+ continued_job_count=len(jobs_to_continue),
+ )
source_response = await database_sync_to_async_pool(
new_source.source_for_pipeline, executor=executor
)(config, resumable_source_manager, source_inputs)
@@ -800,6 +882,8 @@ async def _import_data_with_reporting(
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ continued_workflow_run_id=continued_workflow_run_id,
+ continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
)
@@ -1190,6 +1274,8 @@ async def _run(
preemption: PreemptionConfig | None = None,
on_resumable_retry_budget: bool = False,
kept_rows_synced: int | None = None,
+ continued_workflow_run_id: str | None = None,
+ continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
) -> PipelineResult:
@@ -1213,6 +1299,8 @@ async def _run(
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ continued_workflow_run_id=continued_workflow_run_id,
+ continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index eae58a94d940..20b7e087a7e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -30,7 +30,10 @@ from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.shutdown import WorkerShuttingDownError
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_schema import (
+ WORKER_RESTART_ERROR_MESSAGE,
+ ExternalDataSchema,
+)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import (
@@ -51,6 +54,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline import PipelineV3
from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTS
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+ ResumableSource,
SimpleSource,
SourceExtractionNotImplementedError,
)
@@ -305,6 +309,114 @@ class TestSchemaSyncHistory(BaseTest):
assert has_completed_job is expected
+class TestJobToContinue(BaseTest):
+ @parameterized.expand(
+ [
+ (
+ "worker_restarts",
+ ExternalDataJob.Status.FAILED,
+ WORKER_RESTART_ERROR_MESSAGE,
+ "webhook",
+ "webhook",
+ True,
+ ),
+ ("user_cancel", ExternalDataJob.Status.FAILED, "Sync cancelled by user", "webhook", "webhook", False),
+ ("source_error", ExternalDataJob.Status.FAILED, "401 Unauthorized", "webhook", "webhook", False),
+ ("completed", ExternalDataJob.Status.COMPLETED, None, "webhook", "webhook", False),
+ (
+ "sync_type_changed",
+ ExternalDataJob.Status.FAILED,
+ WORKER_RESTART_ERROR_MESSAGE,
+ "incremental",
+ "webhook",
+ False,
+ ),
+ (
+ "full_refresh",
+ ExternalDataJob.Status.FAILED,
+ WORKER_RESTART_ERROR_MESSAGE,
+ "full_refresh",
+ "full_refresh",
+ False,
+ ),
+ ]
+ )
+ def test_only_a_job_stopped_by_worker_restarts_hands_its_cursor_to_the_next_job(
+ self, _name: str, status: str, latest_error: str | None, previous_sync_type: str, sync_type: str, expected: bool
+ ) -> None:
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"
+ )
+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type=sync_type)
+ previous = ExternalDataJob.objects.create(
+ team=self.team,
+ pipeline=source,
+ schema=schema,
+ status=status,
+ latest_error=latest_error,
+ schema_snapshot={"sync_type": previous_sync_type},
+ )
+ job = ExternalDataJob.objects.create(
+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING
+ )
+ ExternalDataJob.objects.filter(id=previous.id).update(created_at=job.created_at - timedelta(minutes=1))
+
+ jobs_to_continue = cast(Any, module._jobs_to_continue).func(job, schema)
+
+ assert [continued.id for continued in jobs_to_continue] == ([previous.id] if expected else [])
+
+ def test_the_chain_stops_at_the_first_job_that_worker_restarts_did_not_stop(self) -> None:
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"
+ )
+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type="webhook")
+
+ def stopped() -> ExternalDataJob:
+ return ExternalDataJob.objects.create(
+ team=self.team,
+ pipeline=source,
+ schema=schema,
+ status=ExternalDataJob.Status.FAILED,
+ latest_error=WORKER_RESTART_ERROR_MESSAGE,
+ schema_snapshot={"sync_type": "webhook"},
+ )
+
+ before_completed = stopped()
+ completed = ExternalDataJob.objects.create(
+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.COMPLETED
+ )
+ older, newer = stopped(), stopped()
+ job = ExternalDataJob.objects.create(
+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING
+ )
+ for minutes, previous in enumerate([newer, older, completed, before_completed], start=1):
+ ExternalDataJob.objects.filter(id=previous.id).update(
+ created_at=job.created_at - timedelta(minutes=minutes)
+ )
+
+ jobs_to_continue = cast(Any, module._jobs_to_continue).func(job, schema)
+
+ assert [continued.id for continued in jobs_to_continue] == [newer.id, older.id]
+
+ def test_the_continuing_job_counts_the_rows_of_the_job_it_continues_once(self) -> None:
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=self.team, source_type="Stripe"
+ )
+ schema = ExternalDataSchema.objects.create(name="Charge", team=self.team, source=source, sync_type="webhook")
+ stopped = ExternalDataJob.objects.create(
+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.FAILED, rows_synced=500
+ )
+ job = ExternalDataJob.objects.create(
+ team=self.team, pipeline=source, schema=schema, status=ExternalDataJob.Status.RUNNING, rows_synced=0
+ )
+
+ cast(Any, module._carry_rows_synced).func(job, stopped)
+ cast(Any, module._carry_rows_synced).func(job, stopped)
+
+ job.refresh_from_db()
+ assert job.rows_synced == 500
+
+
@pytest.mark.asyncio
async def test_non_retryable_setup_error_routes_through_handler():
# A MongoDB mongodb+srv:// URI resolves DNS in the MongoClient constructor, so a deleted
@@ -1215,6 +1327,42 @@ async def test_reset_run_drops_cursors_but_preserves_sync_history(
assert source_inputs.schema_has_ever_synced is expected_has_ever_synced
+ "reset_pipeline,copied,continues",
+ [(False, True, True), (False, False, True), (True, False, False)],
+ ids=["first_execution", "later_execution", "reset"],
+)
+async def test_a_resumable_run_continues_the_cursor_of_the_interrupted_job_unless_it_resets(
+ reset_pipeline: bool, copied: bool, continues: bool
+) -> None:
+ source = mock.MagicMock(spec=ResumableSource)
+ source.parse_config.return_value = {}
+ source.source_for_pipeline.return_value = mock.MagicMock()
+ manager = source.get_resumable_source_manager.return_value
+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)
+
+ interrupted = mock.MagicMock(id="interrupted-job", workflow_run_id="interrupted-run")
+ earlier = mock.MagicMock(id="earlier-job", workflow_run_id="earlier-run")
+ manager.continue_from_job.return_value = copied
+
+ with (
+ _patched_activity_reaching_run(source, schema) as run_mock,
+ mock.patch.object(module, "_jobs_to_continue", new=mock.AsyncMock(return_value=[interrupted, earlier])),
+ mock.patch.object(module, "_carry_rows_synced", new=mock.AsyncMock()) as carry,
+ ):
+ await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), reset_pipeline=reset_pipeline))
+
+ if continues:
+ manager.continue_from_job.assert_called_once_with("interrupted-job")
+ else:
+ manager.continue_from_job.assert_not_called()
+ # A later execution of the continuing job copies nothing, and still holds the loader to the stopped jobs.
+ assert carry.await_count == (1 if continues and copied else 0)
+ assert run_mock.call_args.kwargs["continued_workflow_run_id"] == ("interrupted-run" if continues else None)
+ assert run_mock.call_args.kwargs["continued_job_ids"] == (("interrupted-job", "earlier-job") if continues else ())
+
+
@pytest.mark.asyncio
async def test_pending_delta_revive_extracts_full_table_not_incremental_slice():
source = mock.MagicMock(spec=SimpleSource)
diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.py
index 952116c4f620..e5ce7d797109 100644
--- a/products/warehouse_sources_queue/backend/core/jobs_db.py
+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py
@@ -1000,6 +1000,16 @@ _UNFINISHED_EARLIER_BATCHES_SQL = f"""
AND b.latest_state <> 'succeeded'
"""
+_UNFINISHED_BATCHES_OF_JOBS_SQL = f"""
+ SELECT count(*), MIN(b.batch_index), COALESCE(array_agg(DISTINCT b.latest_state), '{{}}')
+ FROM {BATCH_TABLE} b
+ WHERE b.job_id = ANY(%(job_ids)s)
+ AND b.team_id = %(team_id)s
+ AND b.schema_id = %(schema_id)s
+ AND b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+ AND b.latest_state <> 'succeeded'
+"""
+
# Batch metadata key for the watermark that the loaded batches of a run cover once this batch and
# every earlier batch of the run are loaded. Only the producer writes it (see `get_loaded_prefix_sync`).
@@ -1218,6 +1228,7 @@ class PendingBatch:
"destination_ids": self.destination_ids or [],
"external_destination_ids": self.metadata.get("external_destination_ids"),
"replace_scope": self.metadata.get("replace_scope"),
+ "continued_job_ids": self.metadata.get("continued_job_ids"),
}
@@ -2132,6 +2143,52 @@ class BatchQueue:
)
raise AssertionError("unreachable")
+ @staticmethod
+ def get_unfinished_batches_of_jobs_sync(
+ database_url: str,
+ *,
+ team_id: int,
+ schema_id: str,
+ job_ids: list[str],
+ connect_timeout_seconds: int = 10,
+ statement_timeout_ms: int = UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS,
+ ) -> UnfinishedEarlierBatches:
+ """The batches of ``job_ids`` that did not reach 'succeeded'.
+
+ The loader asks this before a run that continues the cursor of these jobs
+ completes. Their batches hold the rows before that cursor, so one that did
+ not load means the continuing run is not complete. Same pool and retry as
+ :meth:`get_unfinished_earlier_batches_sync`.
+
+ Raises ``psycopg.errors.QueryCanceled`` past ``statement_timeout_ms``.
+ """
+ pool = _sync_connection_pool(database_url, connect_timeout_seconds)
+ params = {
+ "team_id": team_id,
+ "schema_id": schema_id,
+ "job_ids": job_ids,
+ }
+ for attempt in (1, 2):
+ with pool.connection() as conn:
+ try:
+ # The timeout is transaction-local, so it never leaks into the pooled session.
+ with conn.transaction():
+ conn.execute("SELECT set_config('statement_timeout', %s, true)", (str(statement_timeout_ms),))
+ row = conn.execute(_UNFINISHED_BATCHES_OF_JOBS_SQL, params).fetchone()
+ except psycopg.errors.QueryCanceled:
+ raise
+ except psycopg.OperationalError:
+ pool.discard(conn)
+ if attempt == 2:
+ raise
+ continue
+ assert row is not None
+ count, first_batch_index, states = row
+ return UnfinishedEarlierBatches(
+ count=count, first_batch_index=first_batch_index, states=tuple(sorted(states))
+ )
+ raise AssertionError("unreachable")
+
@staticmethod
def get_loaded_prefix_sync(
database_url: str,