:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): resume ad-hoc sync schedule when v3 load completes (#125552)
Daniel RC846f95dfd311
diff --git a/products/warehouse_sources/backend/ad_hoc_sync.py b/products/warehouse_sources/backend/ad_hoc_sync.pyindex 48aa620e42f7..803b3c66205a 100644--- a/products/warehouse_sources/backend/ad_hoc_sync.py+++ b/products/warehouse_sources/backend/ad_hoc_sync.py@@ -11,6 +11,7 @@ behavior rather than just their shape."""import time+import uuidfrom typing import Anyfrom django.conf import settings@@ -22,7 +23,12 @@ from temporalio.common import WorkflowIDReusePolicyfrom posthog.dataclasses import frozenfrom posthog.temporal.utils import ExternalDataWorkflowInputs-from products.data_warehouse.backend.facade.api import pause_external_data_schedule, unpause_external_data_schedule+from products.data_warehouse.backend.facade.api import (+ a_unpause_external_data_schedule,+ pause_external_data_schedule,+ unpause_external_data_schedule,+)+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import (CDC_SNAPSHOT_LANE_KEY,ExternalDataSchema,@@ -131,8 +137,11 @@ def trigger_ad_hoc_sync(updates["cdc_mode"] = "snapshot"removes.append("cdc_last_log_position")extra_model_fields["initial_sync_complete"] = False- if paused_now:- updates["admin_unpause_schedule_after_run"] = True+ workflow_id = f"{schema.id}-{workflow_id_prefix}-{int(time.time())}"+ # A schedule that an earlier ad-hoc run paused carries its marker, so this run takes it over.+ # Otherwise only the earlier run could resume the schedule, and that run already ended.+ if paused_now or config_before.get("admin_unpause_schedule_after_run"):+ updates["admin_unpause_schedule_after_run"] = workflow_idstaged = bool(updates or removes or extra_model_fields)if staged:# Merged under the row lock: capture writes this config concurrently, and saving the copy@@ -153,7 +162,6 @@ def trigger_ad_hoc_sync(billable=billable,reset_pipeline=None,)- workflow_id = f"{schema.id}-{workflow_id_prefix}-{int(time.time())}"try:start_external_data_workflow(client, workflow_id, inputs)except Exception as e:@@ -184,3 +192,30 @@ def trigger_ad_hoc_sync(raise WorkflowStartError(str(e)) from ereturn AdHocSyncTrigger(workflow_id=workflow_id, schedule_paused_now=paused_now)+++def resume_schedule_after_ad_hoc_run(schema_id: str | uuid.UUID, team_id: int, job_id: str | uuid.UUID) -> bool:+ """Unpause the schedule that an ad-hoc run paused, and return whether it did.++ Call this only after the job commits its Completed status. Any other outcome keeps the schedule+ paused, so a person looks at the schema before syncs resume. It is safe to repeat: unpausing a+ running schedule does nothing, and the marker goes only after the unpause succeeds.+ """+ schema = ExternalDataSchema.objects.filter(id=schema_id, team_id=team_id).only("sync_type_config").first()+ marker = (schema.sync_type_config or {}).get("admin_unpause_schedule_after_run") if schema else None+ if not marker:+ return False+ # The marker holds the ad-hoc workflow id, so a run that was already loading when the pause landed+ # cannot resume the schedule and hand the staged reset to the next billable scheduled run. A+ # legacy `True` marker has no id and matches any run.+ if isinstance(marker, str):+ job_workflow_id = (+ ExternalDataJob.objects.filter(id=job_id, team_id=team_id).values_list("workflow_id", flat=True).first()+ )+ if job_workflow_id != marker:+ return False+ # The async variant connects and unpauses in one coroutine, so the Temporal client stays on the+ # event loop that created it. The sync variant runs those two steps on separate loops.+ async_to_sync(a_unpause_external_data_schedule)(str(schema_id))+ update_sync_type_config_keys(schema_id, team_id, removes=["admin_unpause_schedule_after_run"])+ return Truediff --git a/products/warehouse_sources/backend/admin/external_data_schema_admin.py b/products/warehouse_sources/backend/admin/external_data_schema_admin.pyindex 12690cbbbee2..ceeaaf027043 100644--- a/products/warehouse_sources/backend/admin/external_data_schema_admin.py+++ b/products/warehouse_sources/backend/admin/external_data_schema_admin.py@@ -304,7 +304,7 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):return redirect(_change_url(schema.id))# Single save: stage the partition update(s), reset_pipeline, and the auto-unpause- # marker (read by `update_external_data_job_model` at workflow end) in one round-trip.+ # marker (read by `resume_schedule_after_ad_hoc_run` when the run completes) in one round-trip.# reset_pipeline goes on sync_type_config rather than the workflow input so the pipeline# can pop it after the first reset; passing it on inputs makes every activity retry# re-read True and wipe Delta + cursor, restarting from row 0.@@ -312,8 +312,10 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):schema.sync_type_config[key] = valueif reset_pipeline:schema.sync_type_config["reset_pipeline"] = True- if admin_paused_now:- schema.sync_type_config["admin_unpause_schedule_after_run"] = True+ workflow_id = f"{schema.id}-admin-{workflow_kind}-{int(time.time())}"+ previous_marker = schema.sync_type_config.get("admin_unpause_schedule_after_run")+ if admin_paused_now or previous_marker:+ schema.sync_type_config["admin_unpause_schedule_after_run"] = workflow_idschema.save(update_fields=["sync_type_config"])inputs = ExternalDataWorkflowInputs(@@ -323,7 +325,6 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):billable=False,reset_pipeline=None,)- workflow_id = f"{schema.id}-admin-{workflow_kind}-{int(time.time())}"try:_start_external_data_workflow(client, workflow_id, inputs)except Exception as e:@@ -338,6 +339,12 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):schema.save(update_fields=["sync_type_config"])except Exception:pass+ elif previous_marker:+ try:+ schema.sync_type_config["admin_unpause_schedule_after_run"] = previous_marker+ schema.save(update_fields=["sync_type_config"])+ except Exception:+ passmessages.error(request,f"Saved {change_label}, but failed to trigger run: {e}. "@@ -366,7 +373,7 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):# the schedule's stored input cannot override `billable` per-trigger. To avoid# racing with the regular scheduled run, mirror repartition_view: pause the# schedule (if not already paused), record the auto-unpause marker, and let- # `update_external_data_job_model` resume the schedule on COMPLETED.+ # `resume_schedule_after_ad_hoc_run` resume the schedule on COMPLETED.if request.method != "POST":return redirect(_change_url(schema_id))diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex 27d2a84c4d58..72a4e7683d12 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -30,11 +30,7 @@ from posthog.temporal.utils import CDPProducerWorkflowInputs, ExternalDataWorkflfrom posthog.usage_ingestion.client import UsageRecord, areport_usagefrom posthog.utils import get_machine_id-from products.data_warehouse.backend.facade.api import (- a_unpause_external_data_schedule,- create_warehouse_templates_for_source,- update_external_job_status,-)+from products.data_warehouse.backend.facade.api import create_warehouse_templates_for_source, update_external_job_statusfrom products.managed_warehouse.backend.facade.temporal import (DataImportsDuckLakeCopyInputs,DuckLakeCopyDataImportsWorkflow,@@ -42,6 +38,7 @@ from products.managed_warehouse.backend.facade.temporal import (DuckLakeRegisterDataImportsWorkflow,build_register_data_imports_workflow_id,)+from products.warehouse_sources.backend.ad_hoc_sync import resume_schedule_after_ad_hoc_runfrom products.warehouse_sources.backend.billing import billed_usage_for_jobfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import (@@ -758,29 +755,15 @@ async def _update_job_status(inputs: UpdateExternalDataJobStatusInputs, logger:# non-COMPLETED outcome (FAILED, BILLING_LIMIT_REACHED, …) the flag stays# set and the schedule stays paused — a human looks at it before resuming.if inputs.status == ExternalDataJob.Status.COMPLETED:- await _maybe_unpause_schedule_after_admin_run(inputs.schema_id, logger)---async def _maybe_unpause_schedule_after_admin_run(schema_id: str, logger) -> None:- try:- schema = await database_sync_to_async_pool(ExternalDataSchema.objects.get)(id=schema_id)- except ExternalDataSchema.DoesNotExist:- return-- sync_type_config = schema.sync_type_config or {}- if not sync_type_config.get("admin_unpause_schedule_after_run"):- return-- try:- await a_unpause_external_data_schedule(schema_id)- except Exception:- logger.exception(f"Failed to auto-unpause schedule for schema {schema_id} after admin run")- return-- sync_type_config.pop("admin_unpause_schedule_after_run", None)- schema.sync_type_config = sync_type_config- await database_sync_to_async_pool(schema.save)(update_fields=["sync_type_config"])- logger.info(f"Auto-unpaused schedule for schema {schema_id} after successful admin-triggered run")+ try:+ if await database_sync_to_async_pool(resume_schedule_after_ad_hoc_run)(+ inputs.schema_id, inputs.team_id, job_id+ ):+ logger.info(+ f"Auto-unpaused schedule for schema {inputs.schema_id} after successful admin-triggered run"+ )+ except Exception:+ logger.exception(f"Failed to auto-unpause schedule for schema {inputs.schema_id} after admin run")@dataclasses.dataclassdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex 631e49332494..b7112f04080e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -23,6 +23,7 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.utils import get_machine_idfrom products.data_warehouse.backend.facade.api import update_external_job_status+from products.warehouse_sources.backend.ad_hoc_sync import resume_schedule_after_ad_hoc_runfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.table import DataWarehouseTable@@ -732,6 +733,9 @@ def _mark_job_completed(export_signal: ExportSignalMessage) -> None:_release_pipeline_lock_for_job(export_signal)+ if job_completed:+ _resume_schedule_after_ad_hoc_run(export_signal)+# tonic's timeout layer cancels a call that outruns the client's per-request RPC deadline and# surfaces it as status CANCELLED with the message "Timeout expired" — the client-side analog of@@ -761,6 +765,37 @@ def _is_retryable_temporal_rpc_error(exc: BaseException) -> bool:return isinstance(exc, RuntimeError) and str(exc).startswith("Failed client connect:")+@retry(+ retry=retry_if_exception(_is_retryable_temporal_rpc_error),+ stop=stop_after_attempt(3),+ wait=wait_exponential_jitter(initial=1, max=5),+ reraise=True,+)+def _resume_schedule_with_retry(export_signal: ExportSignalMessage) -> bool:+ return resume_schedule_after_ad_hoc_run(export_signal.schema_id, export_signal.team_id, export_signal.job_id)+++def _resume_schedule_after_ad_hoc_run(export_signal: ExportSignalMessage) -> None:+ try:+ if _resume_schedule_with_retry(export_signal):+ logger.info(+ "schedule_resumed_after_ad_hoc_run",+ team_id=export_signal.team_id,+ external_data_schema_id=export_signal.schema_id,+ external_data_job_id=export_signal.job_id,+ )+ except Exception as e:+ # The job is already Completed, so a failed unpause must not fail and retry the batch.+ logger.error(+ "failed_to_resume_schedule_after_ad_hoc_run",+ team_id=export_signal.team_id,+ external_data_schema_id=export_signal.schema_id,+ external_data_job_id=export_signal.job_id,+ exc_info=True,+ )+ capture_exception(e)++def _trigger_ducklake_register_data_imports(export_signal: ExportSignalMessage, prepared_queryable_folder: str) -> None:"""Fire-and-forget start of `ducklake-register.data-imports` after a V3 final batch lands.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 533b57b0f4a7..13f1504d0504 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -464,10 +464,12 @@ class TestMarkJobCompleted:# If the job was cancelled (absorbing Failed) after the final batch passed# should_process_batch, the Completed write is rejected and the staged# incremental cursor must NOT be promoted.- ("completed_write_lands", "Completed", True),- ("completed_write_absorbed_by_failed", "Failed", False),+ ("completed_write_lands", "Completed", True, None),+ ("completed_write_absorbed_by_failed", "Failed", False, None),+ ("schedule_resume_fails", "Completed", True, RuntimeError("temporal down")),])+ @patch(f"{_PROCESSOR}.resume_schedule_after_ad_hoc_run")@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.transaction")@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.release_v3_pipeline_lock"@@ -490,13 +492,16 @@ class TestMarkJobCompleted:_case: str,resulting_status: str,expect_promotion: bool,+ resume_error: Exception | None,mock_update: MagicMock,mock_schema_objects: MagicMock,mock_finish_row_tracking: AsyncMock,mock_job_objects: MagicMock,mock_release: MagicMock,mock_transaction: MagicMock,+ mock_resume: MagicMock,) -> None:+ mock_resume.side_effect = resume_errormodel = MagicMock()model.status = resulting_statusmock_update.return_value = model@@ -511,9 +516,11 @@ class TestMarkJobCompleted:if expect_promotion:schema.promote_staged_incremental_values.assert_called_once_with("run-abc-a1", merge_source_cursors=ANY)+ mock_resume.assert_called_once_with("schema-1", 1, "job-1")else:schema.promote_staged_incremental_values.assert_not_called()mock_finish_row_tracking.assert_not_awaited()+ mock_resume.assert_not_called()# The pipeline lock must be released either way, or the next sync is blocked.mock_release.assert_called_once_with(team_id=1, schema_id="schema-1", token="wf-run-1")diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.pyindex 9948e39176a9..4f2882b9f627 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py@@ -39,7 +39,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.s_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,)from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSource-from products.warehouse_sources.backend.types import ExternalDataSourceType+from products.warehouse_sources.backend.types import ExternalDataJobStatus, ExternalDataSourceTypeclass TestCustomerFacingError(SimpleTestCase):@@ -193,6 +193,50 @@ class TestUpdateExternalDataJobModelActivity(BaseTest):mock_finish_row_tracking.assert_called_once_with(self.team.id, inputs.schema_id)mock_update_job_status.assert_called_once()+ @parameterized.expand(+ [+ ("completed", ExternalDataJob.Status.COMPLETED, True),+ ("failed", ExternalDataJob.Status.FAILED, False),+ ("billing_limit", ExternalDataJob.Status.BILLING_LIMIT_REACHED, False),+ ]+ )+ def test_resumes_a_schedule_paused_for_an_ad_hoc_run_only_on_completion(+ self, _name: str, status: ExternalDataJobStatus, expect_resume: bool+ ) -> None:+ env = ActivityEnvironment()+ inputs = UpdateExternalDataJobStatusInputs(+ team_id=self.team.id,+ job_id="019fde98-0727-0000-3f05-9991b4c84155",+ schema_id="019fde98-0727-0000-3f05-9991b4c84156",+ source_id="019fde98-0727-0000-3f05-9991b4c84157",+ status=status,+ internal_error=None,+ latest_error=None,+ )++ with (+ mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.get_rows",+ return_value=0,+ ),+ mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.finish_row_tracking"+ ),+ mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.update_external_job_status"+ ),+ mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.resume_schedule_after_ad_hoc_run",+ return_value=True,+ ) as mock_resume,+ ):+ asyncio.run(env.run(update_external_data_job_model, inputs))++ if expect_resume:+ mock_resume.assert_called_once_with(inputs.schema_id, self.team.id, inputs.job_id)+ else:+ mock_resume.assert_not_called()+@parameterized.expand([("handed_a_token", "run-token", ["update", "release"]),diff --git a/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py b/products/warehouse_sources/backend/tests/test_ad_hoc_sync.pyindex 9646fd5d77f4..37c83b110c9a 100644--- a/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py+++ b/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py@@ -7,8 +7,10 @@ from posthog.api.test.test_team import create_teamfrom products.warehouse_sources.backend.ad_hoc_sync import (SyncStillRunningError,WorkflowStartError,+ resume_schedule_after_ad_hoc_run,trigger_ad_hoc_sync,)+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import (ExternalDataSchema,update_sync_type_config_keys,@@ -62,7 +64,60 @@ def test_failed_start_rolls_back_the_pause_it_took(schema):assert "admin_unpause_schedule_after_run" not in schema.sync_type_config-def test_a_schedule_already_paused_is_left_paused(schema):+ "marker,unpause_error,expect_resume",+ [+ ("wf-ad-hoc", None, True),+ (True, None, True),+ (None, None, False),+ ("wf-other-run", None, False),+ # The marker is the only record of the pause, so it stays until the unpause succeeds.+ ("wf-ad-hoc", RuntimeError("temporal down"), True),+ ],+ ids=["own_run", "legacy_marker", "no_marker", "other_run", "unpause_fails"],+)+def test_resume_schedule_after_ad_hoc_run(schema, marker, unpause_error, expect_resume):+ schema.sync_type_config = {+ "reset_pipeline": True,+ **({"admin_unpause_schedule_after_run": marker} if marker else {}),+ }+ schema.save()+ job = ExternalDataJob.objects.create(+ team_id=schema.team_id, pipeline=schema.source, schema=schema, status="Completed", workflow_id="wf-ad-hoc"+ )++ with patch(f"{MODULE}.a_unpause_external_data_schedule", side_effect=unpause_error) as mock_unpause:+ if unpause_error:+ with pytest.raises(RuntimeError):+ resume_schedule_after_ad_hoc_run(schema.id, schema.team_id, job.id)+ else:+ assert resume_schedule_after_ad_hoc_run(schema.id, schema.team_id, job.id) is expect_resume++ if expect_resume:+ mock_unpause.assert_called_once_with(str(schema.id))+ else:+ mock_unpause.assert_not_called()+ schema.refresh_from_db()+ marker_kept = bool(marker) and (not expect_resume or unpause_error is not None)+ assert ("admin_unpause_schedule_after_run" in schema.sync_type_config) is marker_kept+ assert schema.sync_type_config["reset_pipeline"] is True+++ "previous_marker",+ [+ # Auto-unpausing would undo an operator's own manual pause.+ None,+ # A failed ad-hoc run left the pause, and only the retry can still complete to resume it.+ "wf-failed-ad-hoc",+ ],+ ids=["paused_by_hand", "paused_by_earlier_ad_hoc_run"],+)+def test_a_schedule_already_paused_is_left_paused(schema, previous_marker):+ if previous_marker:+ schema.sync_type_config = {"admin_unpause_schedule_after_run": previous_marker}+ schema.save()+with (patch(f"{MODULE}.is_schedule_paused", return_value=True),patch(f"{MODULE}.pause_external_data_schedule") as mock_pause,@@ -72,11 +127,11 @@ def test_a_schedule_already_paused_is_left_paused(schema):MagicMock(), schema, billable=False, reset_pipeline=False, workflow_id_prefix="test")- # Auto-unpausing would undo an operator's own manual pause.mock_pause.assert_not_called()assert trigger.schedule_paused_now is Falseschema.refresh_from_db()- assert "admin_unpause_schedule_after_run" not in schema.sync_type_config+ expected_marker = trigger.workflow_id if previous_marker else None+ assert schema.sync_type_config.get("admin_unpause_schedule_after_run") == expected_markerdef test_failed_start_restores_staged_reset_state(schema, no_sync_to_wait_for):