:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): ride out worker shutdowns without a flag (#130091)
Daniel RCad21cc954641
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex f6a52dcf323d..cf39aaa2f88a 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -334,8 +334,7 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env("DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS", 300.0, type_cast=float)# At a worker shutdown an import that has not yet committed a resume cursor or staged a carry-over-# value keeps the worker, up to a deadline below the graceful shutdown timeout. The-# `warehouse-import-ride-out` flag turns this on per schema. This is the longest such an import holds+# value keeps the worker, up to a deadline below the graceful shutdown timeout. This is the longest such an import holds# the worker when it could have handed off sooner; a run that restarts from scratch either way keeps# the grace period minus a margin instead. The gate reads GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS, so set the grace through this# variable rather than the worker's command-line flag.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.pyindex b4a58834c86d..540f21925183 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py@@ -24,8 +24,6 @@ if TYPE_CHECKING:# Covers the gap to the next item or safe point, the result flush, and the SDK's cancellation of# activities once the graceful shutdown timeout passes. A grace period shorter than this gives a# deadline of zero, so the import hands off at once.-# Per-schema rollout flag. Off, every hand-off follows the legacy rule.-IMPORT_RIDE_OUT_FLAG = "warehouse-import-ride-out"RIDE_OUT_GRACE_MARGIN_SECONDS = 900.0# An execution younger than this restarts for nearly free, so it leaves the worker at once and the# pod keeps draining fast. Shorter than `NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE` on purpose: that onediff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 876de3dd9176..e47d8e86d700 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -73,7 +73,6 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.e)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfigfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (- IMPORT_RIDE_OUT_FLAG,DurableProgress,RideOutConfig,)@@ -97,7 +96,6 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline)from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTSfrom products.warehouse_sources.backend.temporal.data_imports.row_tracking import setup_row_tracking-from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabledfrom products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistryfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (FAST_RETURN_PROBE_TIMEOUT,@@ -748,9 +746,7 @@ async def _import_data_with_reporting()# A deadline hand-off on the retry path would cost an attempt, so the same rule as preemption.ride_out: RideOutConfig | None = None- if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(- schema, IMPORT_RIDE_OUT_FLAG- ):+ if inputs.handoffs_are_free:# The ride-out's stopwatch starts when the monitor reports the shutdown, after the shutdown grace.ride_out = RideOutConfig(grace_seconds=_graceful_shutdown_timeout_seconds() - shutdown_grace_seconds,diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex 56eba6e34677..b3cbb2a7ad35 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -45,7 +45,6 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfigfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (- IMPORT_RIDE_OUT_FLAG,DurableProgress,RideOutConfig,)@@ -1282,7 +1281,6 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workflmock.patch.object(module, "bind_job_context"),mock.patch.object(module, "trim_source_job_inputs", new=mock.AsyncMock()),mock.patch.object(module, "database_sync_to_async_pool", new=_passthrough),- mock.patch.object(module, "is_schema_flag_enabled", return_value=False),mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),mock.patch.object(@@ -2226,24 +2224,20 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(@pytest.mark.asyncio@pytest.mark.parametrize(- "flag_enabled,handoffs_are_free,grace,shutdown_grace,expected",+ "handoffs_are_free,grace,shutdown_grace,expected",[- pytest.param(True, True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),+ pytest.param(True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),# The ride-out starts when the shutdown grace ends, so its deadline leaves that time out.pytest.param(- True, True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"+ True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"),# The worker falls back to a five-minute graceful shutdown when the setting is unset.- pytest.param(- True, True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"- ),+ pytest.param(True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),# Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.- pytest.param(True, False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),- pytest.param(False, True, 7200, 0.0, None, id="off_when_the_flag_is_off"),+ pytest.param(False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),],)-async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(- flag_enabled: bool,+async def test_the_pipeline_rides_out_only_with_free_handoffs(handoffs_are_free: bool,grace: int | None,shutdown_grace: float,@@ -2258,15 +2252,10 @@ async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(source.source_for_pipeline.return_value = mock.MagicMock()schema = _incremental_schema(is_incremental=True, lookback_seconds=None)- with (- _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,- mock.patch.object(module, "is_schema_flag_enabled", return_value=flag_enabled) as flag_check,- ):+ with _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock:await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free))assert run_mock.await_args.kwargs["ride_out"] == expected- if handoffs_are_free:- flag_check.assert_called_once_with(schema, IMPORT_RIDE_OUT_FLAG)assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)