Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): resume ad-hoc sync schedule when v3 load completes (#125552)

Daniel RC846f95dfd311
diff --git a/products/warehouse_sources/backend/ad_hoc_sync.py b/products/warehouse_sources/backend/ad_hoc_sync.py
index 48aa620e42f7..803b3c66205a 100644
--- a/products/warehouse_sources/backend/ad_hoc_sync.py
+++ b/products/warehouse_sources/backend/ad_hoc_sync.py
@@ -11,6 +11,7 @@ behavior rather than just their shape.
"""
import time
+import uuid
from typing import Any
from django.conf import settings
@@ -22,7 +23,12 @@ from temporalio.common import WorkflowIDReusePolicy
from posthog.dataclasses import frozen
from posthog.temporal.utils import ExternalDataWorkflowInputs
-from products.data_warehouse.backend.facade.api import pause_external_data_schedule, unpause_external_data_schedule
+from products.data_warehouse.backend.facade.api import (
+ a_unpause_external_data_schedule,
+ pause_external_data_schedule,
+ unpause_external_data_schedule,
+)
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import (
CDC_SNAPSHOT_LANE_KEY,
ExternalDataSchema,
@@ -131,8 +137,11 @@ def trigger_ad_hoc_sync(
updates["cdc_mode"] = "snapshot"
removes.append("cdc_last_log_position")
extra_model_fields["initial_sync_complete"] = False
- if paused_now:
- updates["admin_unpause_schedule_after_run"] = True
+ workflow_id = f"{schema.id}-{workflow_id_prefix}-{int(time.time())}"
+ # A schedule that an earlier ad-hoc run paused carries its marker, so this run takes it over.
+ # Otherwise only the earlier run could resume the schedule, and that run already ended.
+ if paused_now or config_before.get("admin_unpause_schedule_after_run"):
+ updates["admin_unpause_schedule_after_run"] = workflow_id
staged = bool(updates or removes or extra_model_fields)
if staged:
# Merged under the row lock: capture writes this config concurrently, and saving the copy
@@ -153,7 +162,6 @@ def trigger_ad_hoc_sync(
billable=billable,
reset_pipeline=None,
)
- workflow_id = f"{schema.id}-{workflow_id_prefix}-{int(time.time())}"
try:
start_external_data_workflow(client, workflow_id, inputs)
except Exception as e:
@@ -184,3 +192,30 @@ def trigger_ad_hoc_sync(
raise WorkflowStartError(str(e)) from e
return AdHocSyncTrigger(workflow_id=workflow_id, schedule_paused_now=paused_now)
+
+
+def resume_schedule_after_ad_hoc_run(schema_id: str | uuid.UUID, team_id: int, job_id: str | uuid.UUID) -> bool:
+ """Unpause the schedule that an ad-hoc run paused, and return whether it did.
+
+ Call this only after the job commits its Completed status. Any other outcome keeps the schedule
+ paused, so a person looks at the schema before syncs resume. It is safe to repeat: unpausing a
+ running schedule does nothing, and the marker goes only after the unpause succeeds.
+ """
+ schema = ExternalDataSchema.objects.filter(id=schema_id, team_id=team_id).only("sync_type_config").first()
+ marker = (schema.sync_type_config or {}).get("admin_unpause_schedule_after_run") if schema else None
+ if not marker:
+ return False
+ # The marker holds the ad-hoc workflow id, so a run that was already loading when the pause landed
+ # cannot resume the schedule and hand the staged reset to the next billable scheduled run. A
+ # legacy `True` marker has no id and matches any run.
+ if isinstance(marker, str):
+ job_workflow_id = (
+ ExternalDataJob.objects.filter(id=job_id, team_id=team_id).values_list("workflow_id", flat=True).first()
+ )
+ if job_workflow_id != marker:
+ return False
+ # The async variant connects and unpauses in one coroutine, so the Temporal client stays on the
+ # event loop that created it. The sync variant runs those two steps on separate loops.
+ async_to_sync(a_unpause_external_data_schedule)(str(schema_id))
+ update_sync_type_config_keys(schema_id, team_id, removes=["admin_unpause_schedule_after_run"])
+ return True
diff --git a/products/warehouse_sources/backend/admin/external_data_schema_admin.py b/products/warehouse_sources/backend/admin/external_data_schema_admin.py
index 12690cbbbee2..ceeaaf027043 100644
--- a/products/warehouse_sources/backend/admin/external_data_schema_admin.py
+++ b/products/warehouse_sources/backend/admin/external_data_schema_admin.py
@@ -304,7 +304,7 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):
return redirect(_change_url(schema.id))
# Single save: stage the partition update(s), reset_pipeline, and the auto-unpause
- # marker (read by `update_external_data_job_model` at workflow end) in one round-trip.
+ # marker (read by `resume_schedule_after_ad_hoc_run` when the run completes) in one round-trip.
# reset_pipeline goes on sync_type_config rather than the workflow input so the pipeline
# can pop it after the first reset; passing it on inputs makes every activity retry
# re-read True and wipe Delta + cursor, restarting from row 0.
@@ -312,8 +312,10 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):
schema.sync_type_config[key] = value
if reset_pipeline:
schema.sync_type_config["reset_pipeline"] = True
- if admin_paused_now:
- schema.sync_type_config["admin_unpause_schedule_after_run"] = True
+ workflow_id = f"{schema.id}-admin-{workflow_kind}-{int(time.time())}"
+ previous_marker = schema.sync_type_config.get("admin_unpause_schedule_after_run")
+ if admin_paused_now or previous_marker:
+ schema.sync_type_config["admin_unpause_schedule_after_run"] = workflow_id
schema.save(update_fields=["sync_type_config"])
inputs = ExternalDataWorkflowInputs(
@@ -323,7 +325,6 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):
billable=False,
reset_pipeline=None,
)
- workflow_id = f"{schema.id}-admin-{workflow_kind}-{int(time.time())}"
try:
_start_external_data_workflow(client, workflow_id, inputs)
except Exception as e:
@@ -338,6 +339,12 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):
schema.save(update_fields=["sync_type_config"])
except Exception:
pass
+ elif previous_marker:
+ try:
+ schema.sync_type_config["admin_unpause_schedule_after_run"] = previous_marker
+ schema.save(update_fields=["sync_type_config"])
+ except Exception:
+ pass
messages.error(
request,
f"Saved {change_label}, but failed to trigger run: {e}. "
@@ -366,7 +373,7 @@ class ExternalDataSchemaAdmin(admin.ModelAdmin):
# the schedule's stored input cannot override `billable` per-trigger. To avoid
# racing with the regular scheduled run, mirror repartition_view: pause the
# schedule (if not already paused), record the auto-unpause marker, and let
- # `update_external_data_job_model` resume the schedule on COMPLETED.
+ # `resume_schedule_after_ad_hoc_run` resume the schedule on COMPLETED.
if request.method != "POST":
return redirect(_change_url(schema_id))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index 27d2a84c4d58..72a4e7683d12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -30,11 +30,7 @@ from posthog.temporal.utils import CDPProducerWorkflowInputs, ExternalDataWorkfl
from posthog.usage_ingestion.client import UsageRecord, areport_usage
from posthog.utils import get_machine_id
-from products.data_warehouse.backend.facade.api import (
- a_unpause_external_data_schedule,
- create_warehouse_templates_for_source,
- update_external_job_status,
-)
+from products.data_warehouse.backend.facade.api import create_warehouse_templates_for_source, update_external_job_status
from products.managed_warehouse.backend.facade.temporal import (
DataImportsDuckLakeCopyInputs,
DuckLakeCopyDataImportsWorkflow,
@@ -42,6 +38,7 @@ from products.managed_warehouse.backend.facade.temporal import (
DuckLakeRegisterDataImportsWorkflow,
build_register_data_imports_workflow_id,
)
+from products.warehouse_sources.backend.ad_hoc_sync import resume_schedule_after_ad_hoc_run
from products.warehouse_sources.backend.billing import billed_usage_for_job
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import (
@@ -758,29 +755,15 @@ async def _update_job_status(inputs: UpdateExternalDataJobStatusInputs, logger:
# non-COMPLETED outcome (FAILED, BILLING_LIMIT_REACHED, …) the flag stays
# set and the schedule stays paused — a human looks at it before resuming.
if inputs.status == ExternalDataJob.Status.COMPLETED:
- await _maybe_unpause_schedule_after_admin_run(inputs.schema_id, logger)
-
-
-async def _maybe_unpause_schedule_after_admin_run(schema_id: str, logger) -> None:
- try:
- schema = await database_sync_to_async_pool(ExternalDataSchema.objects.get)(id=schema_id)
- except ExternalDataSchema.DoesNotExist:
- return
-
- sync_type_config = schema.sync_type_config or {}
- if not sync_type_config.get("admin_unpause_schedule_after_run"):
- return
-
- try:
- await a_unpause_external_data_schedule(schema_id)
- except Exception:
- logger.exception(f"Failed to auto-unpause schedule for schema {schema_id} after admin run")
- return
-
- sync_type_config.pop("admin_unpause_schedule_after_run", None)
- schema.sync_type_config = sync_type_config
- await database_sync_to_async_pool(schema.save)(update_fields=["sync_type_config"])
- logger.info(f"Auto-unpaused schedule for schema {schema_id} after successful admin-triggered run")
+ try:
+ if await database_sync_to_async_pool(resume_schedule_after_ad_hoc_run)(
+ inputs.schema_id, inputs.team_id, job_id
+ ):
+ logger.info(
+ f"Auto-unpaused schedule for schema {inputs.schema_id} after successful admin-triggered run"
+ )
+ except Exception:
+ logger.exception(f"Failed to auto-unpause schedule for schema {inputs.schema_id} after admin run")
@dataclasses.dataclass
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index 631e49332494..b7112f04080e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -23,6 +23,7 @@ from posthog.exceptions_capture import capture_exception
from posthog.utils import get_machine_id
from products.data_warehouse.backend.facade.api import update_external_job_status
+from products.warehouse_sources.backend.ad_hoc_sync import resume_schedule_after_ad_hoc_run
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.table import DataWarehouseTable
@@ -732,6 +733,9 @@ def _mark_job_completed(export_signal: ExportSignalMessage) -> None:
_release_pipeline_lock_for_job(export_signal)
+ if job_completed:
+ _resume_schedule_after_ad_hoc_run(export_signal)
+
# tonic's timeout layer cancels a call that outruns the client's per-request RPC deadline and
# surfaces it as status CANCELLED with the message "Timeout expired" — the client-side analog of
@@ -761,6 +765,37 @@ def _is_retryable_temporal_rpc_error(exc: BaseException) -> bool:
return isinstance(exc, RuntimeError) and str(exc).startswith("Failed client connect:")
+@retry(
+ retry=retry_if_exception(_is_retryable_temporal_rpc_error),
+ stop=stop_after_attempt(3),
+ wait=wait_exponential_jitter(initial=1, max=5),
+ reraise=True,
+)
+def _resume_schedule_with_retry(export_signal: ExportSignalMessage) -> bool:
+ return resume_schedule_after_ad_hoc_run(export_signal.schema_id, export_signal.team_id, export_signal.job_id)
+
+
+def _resume_schedule_after_ad_hoc_run(export_signal: ExportSignalMessage) -> None:
+ try:
+ if _resume_schedule_with_retry(export_signal):
+ logger.info(
+ "schedule_resumed_after_ad_hoc_run",
+ team_id=export_signal.team_id,
+ external_data_schema_id=export_signal.schema_id,
+ external_data_job_id=export_signal.job_id,
+ )
+ except Exception as e:
+ # The job is already Completed, so a failed unpause must not fail and retry the batch.
+ logger.error(
+ "failed_to_resume_schedule_after_ad_hoc_run",
+ team_id=export_signal.team_id,
+ external_data_schema_id=export_signal.schema_id,
+ external_data_job_id=export_signal.job_id,
+ exc_info=True,
+ )
+ capture_exception(e)
+
+
def _trigger_ducklake_register_data_imports(export_signal: ExportSignalMessage, prepared_queryable_folder: str) -> None:
"""Fire-and-forget start of `ducklake-register.data-imports` after a V3 final batch lands.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 533b57b0f4a7..13f1504d0504 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -464,10 +464,12 @@ class TestMarkJobCompleted:
# If the job was cancelled (absorbing Failed) after the final batch passed
# should_process_batch, the Completed write is rejected and the staged
# incremental cursor must NOT be promoted.
- ("completed_write_lands", "Completed", True),
- ("completed_write_absorbed_by_failed", "Failed", False),
+ ("completed_write_lands", "Completed", True, None),
+ ("completed_write_absorbed_by_failed", "Failed", False, None),
+ ("schedule_resume_fails", "Completed", True, RuntimeError("temporal down")),
]
)
+ @patch(f"{_PROCESSOR}.resume_schedule_after_ad_hoc_run")
@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.transaction")
@patch(
"products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.release_v3_pipeline_lock"
@@ -490,13 +492,16 @@ class TestMarkJobCompleted:
_case: str,
resulting_status: str,
expect_promotion: bool,
+ resume_error: Exception | None,
mock_update: MagicMock,
mock_schema_objects: MagicMock,
mock_finish_row_tracking: AsyncMock,
mock_job_objects: MagicMock,
mock_release: MagicMock,
mock_transaction: MagicMock,
+ mock_resume: MagicMock,
) -> None:
+ mock_resume.side_effect = resume_error
model = MagicMock()
model.status = resulting_status
mock_update.return_value = model
@@ -511,9 +516,11 @@ class TestMarkJobCompleted:
if expect_promotion:
schema.promote_staged_incremental_values.assert_called_once_with("run-abc-a1", merge_source_cursors=ANY)
+ mock_resume.assert_called_once_with("schema-1", 1, "job-1")
else:
schema.promote_staged_incremental_values.assert_not_called()
mock_finish_row_tracking.assert_not_awaited()
+ mock_resume.assert_not_called()
# The pipeline lock must be released either way, or the next sync is blocked.
mock_release.assert_called_once_with(team_id=1, schema_id="schema-1", token="wf-run-1")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
index 9948e39176a9..4f2882b9f627 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
@@ -39,7 +39,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.s
_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSource
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.backend.types import ExternalDataJobStatus, ExternalDataSourceType
class TestCustomerFacingError(SimpleTestCase):
@@ -193,6 +193,50 @@ class TestUpdateExternalDataJobModelActivity(BaseTest):
mock_finish_row_tracking.assert_called_once_with(self.team.id, inputs.schema_id)
mock_update_job_status.assert_called_once()
+ @parameterized.expand(
+ [
+ ("completed", ExternalDataJob.Status.COMPLETED, True),
+ ("failed", ExternalDataJob.Status.FAILED, False),
+ ("billing_limit", ExternalDataJob.Status.BILLING_LIMIT_REACHED, False),
+ ]
+ )
+ def test_resumes_a_schedule_paused_for_an_ad_hoc_run_only_on_completion(
+ self, _name: str, status: ExternalDataJobStatus, expect_resume: bool
+ ) -> None:
+ env = ActivityEnvironment()
+ inputs = UpdateExternalDataJobStatusInputs(
+ team_id=self.team.id,
+ job_id="019fde98-0727-0000-3f05-9991b4c84155",
+ schema_id="019fde98-0727-0000-3f05-9991b4c84156",
+ source_id="019fde98-0727-0000-3f05-9991b4c84157",
+ status=status,
+ internal_error=None,
+ latest_error=None,
+ )
+
+ with (
+ mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.get_rows",
+ return_value=0,
+ ),
+ mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.finish_row_tracking"
+ ),
+ mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.update_external_job_status"
+ ),
+ mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.external_data_job.resume_schedule_after_ad_hoc_run",
+ return_value=True,
+ ) as mock_resume,
+ ):
+ asyncio.run(env.run(update_external_data_job_model, inputs))
+
+ if expect_resume:
+ mock_resume.assert_called_once_with(inputs.schema_id, self.team.id, inputs.job_id)
+ else:
+ mock_resume.assert_not_called()
+
@parameterized.expand(
[
("handed_a_token", "run-token", ["update", "release"]),
diff --git a/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py b/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py
index 9646fd5d77f4..37c83b110c9a 100644
--- a/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py
+++ b/products/warehouse_sources/backend/tests/test_ad_hoc_sync.py
@@ -7,8 +7,10 @@ from posthog.api.test.test_team import create_team
from products.warehouse_sources.backend.ad_hoc_sync import (
SyncStillRunningError,
WorkflowStartError,
+ resume_schedule_after_ad_hoc_run,
trigger_ad_hoc_sync,
)
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import (
ExternalDataSchema,
update_sync_type_config_keys,
@@ -62,7 +64,60 @@ def test_failed_start_rolls_back_the_pause_it_took(schema):
assert "admin_unpause_schedule_after_run" not in schema.sync_type_config
-def test_a_schedule_already_paused_is_left_paused(schema):
+ "marker,unpause_error,expect_resume",
+ [
+ ("wf-ad-hoc", None, True),
+ (True, None, True),
+ (None, None, False),
+ ("wf-other-run", None, False),
+ # The marker is the only record of the pause, so it stays until the unpause succeeds.
+ ("wf-ad-hoc", RuntimeError("temporal down"), True),
+ ],
+ ids=["own_run", "legacy_marker", "no_marker", "other_run", "unpause_fails"],
+)
+def test_resume_schedule_after_ad_hoc_run(schema, marker, unpause_error, expect_resume):
+ schema.sync_type_config = {
+ "reset_pipeline": True,
+ **({"admin_unpause_schedule_after_run": marker} if marker else {}),
+ }
+ schema.save()
+ job = ExternalDataJob.objects.create(
+ team_id=schema.team_id, pipeline=schema.source, schema=schema, status="Completed", workflow_id="wf-ad-hoc"
+ )
+
+ with patch(f"{MODULE}.a_unpause_external_data_schedule", side_effect=unpause_error) as mock_unpause:
+ if unpause_error:
+ with pytest.raises(RuntimeError):
+ resume_schedule_after_ad_hoc_run(schema.id, schema.team_id, job.id)
+ else:
+ assert resume_schedule_after_ad_hoc_run(schema.id, schema.team_id, job.id) is expect_resume
+
+ if expect_resume:
+ mock_unpause.assert_called_once_with(str(schema.id))
+ else:
+ mock_unpause.assert_not_called()
+ schema.refresh_from_db()
+ marker_kept = bool(marker) and (not expect_resume or unpause_error is not None)
+ assert ("admin_unpause_schedule_after_run" in schema.sync_type_config) is marker_kept
+ assert schema.sync_type_config["reset_pipeline"] is True
+
+
+ "previous_marker",
+ [
+ # Auto-unpausing would undo an operator's own manual pause.
+ None,
+ # A failed ad-hoc run left the pause, and only the retry can still complete to resume it.
+ "wf-failed-ad-hoc",
+ ],
+ ids=["paused_by_hand", "paused_by_earlier_ad_hoc_run"],
+)
+def test_a_schedule_already_paused_is_left_paused(schema, previous_marker):
+ if previous_marker:
+ schema.sync_type_config = {"admin_unpause_schedule_after_run": previous_marker}
+ schema.save()
+
with (
patch(f"{MODULE}.is_schedule_paused", return_value=True),
patch(f"{MODULE}.pause_external_data_schedule") as mock_pause,
@@ -72,11 +127,11 @@ def test_a_schedule_already_paused_is_left_paused(schema):
MagicMock(), schema, billable=False, reset_pipeline=False, workflow_id_prefix="test"
)
- # Auto-unpausing would undo an operator's own manual pause.
mock_pause.assert_not_called()
assert trigger.schedule_paused_now is False
schema.refresh_from_db()
- assert "admin_unpause_schedule_after_run" not in schema.sync_type_config
+ expected_marker = trigger.workflow_id if previous_marker else None
+ assert schema.sync_type_config.get("admin_unpause_schedule_after_run") == expected_marker
def test_failed_start_restores_staged_reset_state(schema, no_sync_to_wait_for):