Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): ride out worker shutdowns without a flag (#130091)

Daniel RCad21cc954641
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index f6a52dcf323d..cf39aaa2f88a 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -334,8 +334,7 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env(
"DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS", 300.0, type_cast=float
)
# At a worker shutdown an import that has not yet committed a resume cursor or staged a carry-over
-# value keeps the worker, up to a deadline below the graceful shutdown timeout. The
-# `warehouse-import-ride-out` flag turns this on per schema. This is the longest such an import holds
+# value keeps the worker, up to a deadline below the graceful shutdown timeout. This is the longest such an import holds
# the worker when it could have handed off sooner; a run that restarts from scratch either way keeps
# the grace period minus a margin instead. The gate reads GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS, so set the grace through this
# variable rather than the worker's command-line flag.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
index b4a58834c86d..540f21925183 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
@@ -24,8 +24,6 @@ if TYPE_CHECKING:
# Covers the gap to the next item or safe point, the result flush, and the SDK's cancellation of
# activities once the graceful shutdown timeout passes. A grace period shorter than this gives a
# deadline of zero, so the import hands off at once.
-# Per-schema rollout flag. Off, every hand-off follows the legacy rule.
-IMPORT_RIDE_OUT_FLAG = "warehouse-import-ride-out"
RIDE_OUT_GRACE_MARGIN_SECONDS = 900.0
# An execution younger than this restarts for nearly free, so it leaves the worker at once and the
# pod keeps draining fast. Shorter than `NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE` on purpose: that one
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 876de3dd9176..e47d8e86d700 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -73,7 +73,6 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.e
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
- IMPORT_RIDE_OUT_FLAG,
DurableProgress,
RideOutConfig,
)
@@ -97,7 +96,6 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
)
from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTS
from products.warehouse_sources.backend.temporal.data_imports.row_tracking import setup_row_tracking
-from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabled
from products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
FAST_RETURN_PROBE_TIMEOUT,
@@ -748,9 +746,7 @@ async def _import_data_with_reporting(
)
# A deadline hand-off on the retry path would cost an attempt, so the same rule as preemption.
ride_out: RideOutConfig | None = None
- if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(
- schema, IMPORT_RIDE_OUT_FLAG
- ):
+ if inputs.handoffs_are_free:
# The ride-out's stopwatch starts when the monitor reports the shutdown, after the shutdown grace.
ride_out = RideOutConfig(
grace_seconds=_graceful_shutdown_timeout_seconds() - shutdown_grace_seconds,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index 56eba6e34677..b3cbb2a7ad35 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -45,7 +45,6 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
- IMPORT_RIDE_OUT_FLAG,
DurableProgress,
RideOutConfig,
)
@@ -1282,7 +1281,6 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workfl
mock.patch.object(module, "bind_job_context"),
mock.patch.object(module, "trim_source_job_inputs", new=mock.AsyncMock()),
mock.patch.object(module, "database_sync_to_async_pool", new=_passthrough),
- mock.patch.object(module, "is_schema_flag_enabled", return_value=False),
mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),
mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),
mock.patch.object(
@@ -2226,24 +2224,20 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "flag_enabled,handoffs_are_free,grace,shutdown_grace,expected",
+ "handoffs_are_free,grace,shutdown_grace,expected",
[
- pytest.param(True, True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),
+ pytest.param(True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),
# The ride-out starts when the shutdown grace ends, so its deadline leaves that time out.
pytest.param(
- True, True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"
+ True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"
),
# The worker falls back to a five-minute graceful shutdown when the setting is unset.
- pytest.param(
- True, True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"
- ),
+ pytest.param(True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),
# Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.
- pytest.param(True, False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),
- pytest.param(False, True, 7200, 0.0, None, id="off_when_the_flag_is_off"),
+ pytest.param(False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),
],
)
-async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(
- flag_enabled: bool,
+async def test_the_pipeline_rides_out_only_with_free_handoffs(
handoffs_are_free: bool,
grace: int | None,
shutdown_grace: float,
@@ -2258,15 +2252,10 @@ async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(
source.source_for_pipeline.return_value = mock.MagicMock()
schema = _incremental_schema(is_incremental=True, lookback_seconds=None)
- with (
- _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,
- mock.patch.object(module, "is_schema_flag_enabled", return_value=flag_enabled) as flag_check,
- ):
+ with _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock:
await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free))
assert run_mock.await_args.kwargs["ride_out"] == expected
- if handoffs_are_free:
- flag_check.assert_called_once_with(schema, IMPORT_RIDE_OUT_FLAG)
assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)