:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): let import attempts finish briefly before shutdown hand-off (#123322)
Tom Owers45bf2bdb69c3
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 0da3b903822c..105c2e94eef2 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -336,6 +336,12 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env(DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS: float = get_from_env("DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS", 3600.0, type_cast=float)+# At a worker shutdown an import keeps running this long before any hand-off check sees the shutdown.+# Many imports that run at the signal finish in this time, and a hand-off would restart them on+# another worker. 0 hands off at once. The import uses at most half the graceful shutdown timeout.+DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS: float = get_from_env(+ "DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS", 60.0, type_cast=float+)# Restore unfinished append runs only after all extract and load workers have this implementation.DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED: bool = get_from_env("DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED", False, type_cast=str_to_booldiff --git a/posthog/temporal/common/shutdown.py b/posthog/temporal/common/shutdown.pyindex 189a03149bc9..a98f8c7da002 100644--- a/posthog/temporal/common/shutdown.py+++ b/posthog/temporal/common/shutdown.py@@ -5,7 +5,7 @@ import asyncioimport datetimeimport threadingimport contextvars-from collections.abc import Callable+from collections.abc import Awaitable, Callablefrom structlog import get_loggerfrom temporalio import activity@@ -64,9 +64,23 @@ class ShutdownMonitor:All Temporal activities should consider `WorkerShuttingDownError` as aretryable exception, at least if they wish to have new workers pick it up.++ With `grace_seconds`, the monitor reports the shutdown only that long after the+ worker got the signal. Every check, wait and shutdown callback sees the shutdown+ then, so an activity that ends inside the grace never stops early. The worker+ takes no new activities from the signal on, so only activities that already run+ use the grace. Keep the grace well below the worker's graceful shutdown timeout."""- def __init__(self):+ def __init__(+ self,+ grace_seconds: float = 0.0,+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,+ ):+ if grace_seconds < 0:+ raise ValueError(f"grace_seconds must not be negative, got {grace_seconds}")+ self._grace_seconds = grace_seconds+ self._sleep = sleepself._monitor_shutdown_task: asyncio.Task[None] | None = Noneself._monitor_shutdown_thread: threading.Thread | None = Noneself._on_shutdown_callbacks: list[tuple[contextvars.Context, Callable[[], None]]] = []@@ -115,6 +129,13 @@ class ShutdownMonitor:# Not running in an activity context.return+ if self._grace_seconds > 0:+ self.logger.info(+ "Shutdown signal received, the activity keeps running for the grace period.",+ grace_seconds=self._grace_seconds,+ )+ await self._sleep(self._grace_seconds)+self._record_shutdown_time()self.logger.info("Shutdown detected.")# Run the callbacks before the event is set: a caller that sees the event can raise and@@ -129,7 +150,7 @@ class ShutdownMonitor:self._monitor_shutdown_task = asyncio.create_task(monitor())def run_on_shutdown(self, callback: Callable[[], None]) -> None:- """Call `callback` once when the worker starts to shut down.+ """Call `callback` once when the monitor reports the shutdown, which is after the grace.`callback` runs with a copy of the caller's contextvars, taken now, so it sees the structlogcontext the caller has bound (for example the job context an activity binds after it enters@@ -143,6 +164,9 @@ class ShutdownMonitor:Notice we must copy the context to preserve the activity context for themonitoring thread."""+ if self._grace_seconds > 0:+ raise ValueError("A shutdown grace needs the async monitor: use `async with ShutdownMonitor(...)`")+context = contextvars.copy_context()def monitor() -> None:@@ -218,7 +242,7 @@ class ShutdownMonitor:self._shutdown_at = time.monotonic()def seconds_since_shutdown(self) -> float | None:- """How long ago the worker started to shut down, or None while it still runs.+ """How long ago the monitor started to report the shutdown (at the end of the grace), or None before.A caller created after the shutdown cannot learn the moment from a callback, because thecallbacks have already run by then. It reads the elapsed time here instead.diff --git a/posthog/temporal/tests/test_shutdown.py b/posthog/temporal/tests/test_shutdown.pyindex baaccbe79301..415d01017a93 100644--- a/posthog/temporal/tests/test_shutdown.py+++ b/posthog/temporal/tests/test_shutdown.py@@ -14,7 +14,7 @@ from temporalio.common import RetryPolicyfrom temporalio.exceptions import ActivityError, ApplicationErrorfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker-from posthog.temporal.common.shutdown import ShutdownSignalListener+from posthog.temporal.common.shutdown import ShutdownMonitor, ShutdownSignalListener, WorkerShuttingDownErrorfrom posthog.temporal.tests.utils.workflow import Waiter, WaitInputs, WaitMode, WaitWorkflow@@ -207,3 +207,88 @@ async def test_shutdown_signal_listener_keeps_the_first_signal(wakeup_fd_full: bsignal.signal(sig, handler)read_sock.close()write_sock.close()+++class _GraceSleep:+ """Stands in for `asyncio.sleep` in the monitor, so a test decides when the grace ends."""++ def __init__(self) -> None:+ self.requested: list[float] = []+ self.started = asyncio.Event()+ self.release = asyncio.Event()++ async def __call__(self, seconds: float) -> None:+ self.requested.append(seconds)+ self.started.set()+ await self.release.wait()+++async def _settle() -> None:+ for _ in range(5):+ await asyncio.sleep(0)+++ "grace_seconds,finishes_in_grace,expected",+ [+ # Today's behaviour: the first check after the signal hands off.+ pytest.param(0.0, False, "handed_off_at_signal", id="no_grace_hands_off_at_once"),+ pytest.param(60.0, True, "finished", id="finishes_inside_the_grace"),+ pytest.param(60.0, False, "handed_off_after_grace", id="still_running_at_grace_end_hands_off"),+ ],+)+async def test_shutdown_grace_delays_what_every_check_sees(+ activity_environment, grace_seconds: float, finishes_in_grace: bool, expected: str+):+ sleep = _GraceSleep()+ callback_ran: list[bool] = []++ async def activity_body() -> str:+ async with ShutdownMonitor(grace_seconds=grace_seconds, sleep=sleep) as monitor:+ monitor.run_on_shutdown(lambda: callback_ran.append(True))+ wait_for_shutdown = asyncio.ensure_future(monitor.wait_for_worker_shutdown())+ await _settle()++ # Before the signal, no check sees a shutdown.+ monitor.raise_if_is_worker_shutdown()+ assert (monitor.is_worker_shutdown(), monitor.seconds_since_shutdown(), callback_ran) == (False, None, [])++ activity_environment.worker_shutdown()+ await _settle()+ try:+ monitor.raise_if_is_worker_shutdown()+ except WorkerShuttingDownError:+ assert wait_for_shutdown.done() and callback_ran == [True]+ return "handed_off_at_signal"++ assert sleep.requested == [grace_seconds]+ assert (monitor.is_worker_shutdown(), wait_for_shutdown.done(), callback_ran) == (False, False, [])+ if finishes_in_grace:+ return "finished"++ sleep.release.set()+ await asyncio.wait_for(wait_for_shutdown, timeout=5)+ assert callback_ran == [True]+ try:+ monitor.raise_if_is_worker_shutdown()+ except WorkerShuttingDownError:+ return "handed_off_after_grace"+ return "kept_running"++ assert await activity_environment.run(activity_body) == expected+++ "grace_seconds,sync,error",+ [+ pytest.param(-1.0, False, "must not be negative", id="negative_grace"),+ # The thread monitor has no grace timer, so a grace there would never end.+ pytest.param(60.0, True, "needs the async monitor", id="grace_on_the_thread_monitor"),+ ],+)+def test_shutdown_grace_rejects_a_configuration_it_cannot_honour(grace_seconds: float, sync: bool, error: str):+ with pytest.raises(ValueError, match=error):+ monitor = ShutdownMonitor(grace_seconds=grace_seconds)+ if sync:+ monitor.start_sync()diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.pyindex b5fb93238c07..ce563a64dd11 100644--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py@@ -174,12 +174,21 @@ def get_import_not_preempted_metric(source_type: str | None, reason: str) -> Met)-def get_import_finished_during_shutdown_metric(source_type: str | None, rode_out: bool) -> MetricCounter:+def get_import_finished_during_shutdown_metric(+ source_type: str | None, rode_out: bool, *, within_grace: bool+) -> MetricCounter:# The imports a hand-off would have restarted. `rode_out` is True when the gate kept the import- # on the worker past the shutdown at least once.+ # on the worker past the shutdown at least once. `within_grace` is True when the import finished+ # before the shutdown monitor reported the shutdown, so no hand-off check saw it.return (activity.metric_meter()- .with_additional_attributes({"source_type": source_type or "unknown", "rode_out": str(rode_out).lower()})+ .with_additional_attributes(+ {+ "source_type": source_type or "unknown",+ "rode_out": str(rode_out).lower(),+ "within_grace": str(within_grace).lower(),+ }+ ).create_counter("warehouse_import_finished_during_shutdown_total","Imports that finished on a worker that was shutting down instead of handing off.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.pyindex f41ed30c474e..d6d869f7f772 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py@@ -894,12 +894,18 @@ async def update_row_tracking_after_batch(await decrement_rows(team_id, schema_id, row_count)-def is_young_first_attempt() -> bool:+def is_young_first_attempt(shutdown_grace_seconds: float = 0.0) -> bool:+ """`shutdown_grace_seconds` is the time the shutdown monitor let the run continue after the signal.++ It does not count toward the age, so the grace never turns a run that was young at the signal into+ one that keeps the worker until it finishes.+ """if not activity.in_activity():return Falseinfo = activity.info()- return info.attempt == 1 and datetime.now(UTC) - info.started_time < NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE+ age = datetime.now(UTC) - info.started_time - timedelta(seconds=shutdown_grace_seconds)+ return info.attempt == 1 and age < NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGEdef should_check_shutdown(@@ -907,6 +913,7 @@ def should_check_shutdown(resource: SourceResponse,reset_pipeline: bool,source_is_resumable: bool,+ shutdown_grace_seconds: float = 0.0,) -> bool:# A descending run does not raise, because each stop restarts it from the old watermark. An ascending# incremental run keeps its progress after a stop only through the handoff checkpoint.@@ -916,7 +923,9 @@ def should_check_shutdown(incremental_sync_raise_during_shutdown = (schema.should_use_incremental_field and resource.sort_mode != "desc" and not reset_pipeline)- return incremental_sync_raise_during_shutdown or source_is_resumable or is_young_first_attempt()+ return (+ incremental_sync_raise_during_shutdown or source_is_resumable or is_young_first_attempt(shutdown_grace_seconds)+ )async def finalize_desc_sort_incremental_value(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.pyindex f103eb8386db..b4a58834c86d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py@@ -85,10 +85,11 @@ class DurableProgress:self._handoff_reason = reason-def activity_execution_age_seconds() -> float | None:+def activity_execution_age_seconds(shutdown_grace_seconds: float = 0.0) -> float | None:+ """The age leaves out the shutdown grace, for the reason given on `is_young_first_attempt`."""if not activity.in_activity():return None- return (datetime.now(UTC) - activity.info().started_time).total_seconds()+ return (datetime.now(UTC) - activity.info().started_time).total_seconds() - shutdown_grace_secondsclass HandoffGate:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.pyindex bb5455a5e0c7..78b29b720cb4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py@@ -75,6 +75,9 @@ class TestShouldCheckShutdown:("ascending_incremental", True, "asc", False, False, True, 2, 20, True),("unordered_incremental", True, None, False, False, True, 2, 20, True),("descending_incremental", True, "desc", False, False, True, 2, 20, False),+ # The shutdown grace does not count toward the age.+ ("young_at_signal_after_grace", False, "asc", True, False, True, 1, 10, True, 60.0),+ ("old_at_signal_after_grace", False, "asc", True, False, True, 1, 12, False, 60.0),])def test_should_check_shutdown(@@ -88,6 +91,7 @@ class TestShouldCheckShutdown:attempt: int,age_minutes: int,expected: bool,+ shutdown_grace_seconds: float = 0.0,) -> None:schema = MagicMock(should_use_incremental_field=is_incremental)resource = MagicMock(sort_mode=sort_mode)@@ -97,7 +101,12 @@ class TestShouldCheckShutdown:patch(f"{_EXTRACT_MODULE}.activity.in_activity", return_value=in_activity),patch(f"{_EXTRACT_MODULE}.activity.info", return_value=info) as activity_info,):- assert should_check_shutdown(schema, resource, reset_pipeline, source_is_resumable) is expected+ assert (+ should_check_shutdown(+ schema, resource, reset_pipeline, source_is_resumable, shutdown_grace_seconds=shutdown_grace_seconds+ )+ is expected+ )if not in_activity:activity_info.assert_not_called()diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.pyindex ace3fb8de0c4..57df0da793f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py@@ -103,6 +103,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,+ shutdown_grace_seconds: float = 0.0,) -> None:if not source_response.lanes:raise ValueError(f"{source_response.name} declares no lanes; run it on PipelineV3")@@ -126,6 +127,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,+ shutdown_grace_seconds=shutdown_grace_seconds,)# The base built the first lane; it shares the base's batch list so the two never disagree.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 478b7e8aac0b..439d0b3a05f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -2,6 +2,7 @@ import timeimport asyncioimport decimalimport datetime+import functoolsimport contextlibfrom collections.abc import AsyncGenerator, AsyncIteratorfrom typing import TYPE_CHECKING, Any, Generic@@ -66,6 +67,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.rDurableProgress,HandoffGate,RideOutConfig,+ activity_execution_age_seconds,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (PipelineSafePointHandler,@@ -200,6 +202,8 @@ class PipelineV3(Generic[ResumableData]):_ride_out: RideOutConfig | None = None_progress: DurableProgress | None = None_handoff_gate: HandoffGate | None = None+ # The grace of the run's `ShutdownMonitor`. The age checks of a hand-off leave it out.+ _shutdown_grace_seconds: float = 0.0_resumed_incremental_run_uuid: str | None = None_sent_resumed_run_finalization: bool = False_writes_staged_repartition_scheme: bool = False@@ -230,11 +234,13 @@ class PipelineV3(Generic[ResumableData]):continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,+ shutdown_grace_seconds: float = 0.0,) -> None:self._resource = source_responseself._kept_rows_synced = kept_rows_syncedself._ride_out = ride_outself._progress = progress+ self._shutdown_grace_seconds = shutdown_grace_secondsself._source_cursor_manager = source_cursor_managerself._resource_name = source_response.name@@ -686,7 +692,7 @@ class PipelineV3(Generic[ResumableData]):carry_over_enabled = self._preemption is not None and self._preemption.watermark_carry_over_enabledif checkpoint is not None and not checkpoint.is_void and carry_over_enabled:return PreemptionDecision(eligible=True, reason="watermark_carry_over")- if is_young_first_attempt():+ if is_young_first_attempt(self._shutdown_grace_seconds):return PreemptionDecision(eligible=True, reason="young_first_attempt")if checkpoint is not None and not checkpoint.is_void:return PreemptionDecision(@@ -719,10 +725,17 @@ class PipelineV3(Generic[ResumableData]):# A hand-off retires a laned run's companion jobs, so that run keeps the legacy rule.config=None if self._resource.lanes else self._ride_out,legacy_wants_handoff=lambda: should_check_shutdown(- self._schema, self._resource, self._reset_pipeline, self._resumable_source_manager is not None+ self._schema,+ self._resource,+ self._reset_pipeline,+ self._resumable_source_manager is not None,+ shutdown_grace_seconds=self._shutdown_grace_seconds,),restarts_from_scratch=self._restarts_from_scratch,logger=self._logger,+ execution_age_seconds=functools.partial(+ activity_execution_age_seconds, shutdown_grace_seconds=self._shutdown_grace_seconds+ ),)return self._handoff_gate@@ -1078,8 +1091,13 @@ class PipelineV3(Generic[ResumableData]):chunk_index += 1await self._finalize(row_count=row_count)- if activity.in_activity() and self._shutdown_monitor.is_worker_shutdown():- get_import_finished_during_shutdown_metric(source_type, handoff_gate.rode_out).add(1)+ # The signal, not the monitor: an import that finished inside the shutdown grace counts too.+ if activity.in_activity() and activity.is_worker_shutdown():+ get_import_finished_during_shutdown_metric(+ source_type,+ handoff_gate.rode_out,+ within_grace=not self._shutdown_monitor.is_worker_shutdown(),+ ).add(1)# With zero batches, `_finalize` sent no final-batch notification, so the load# consumer will never hear about this run and cannot finalize it — the workflow must.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex eb75a1fc6254..a277eb6f56cb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -17,8 +17,9 @@ from django.test import override_settingsimport pyarrow as paimport deltalakefrom asgiref.sync import async_to_sync+from temporalio.testing import ActivityEnvironment-from posthog.temporal.common.shutdown import WorkerShuttingDownError+from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDownErrorfrom products.warehouse_sources.backend.models import external_data_schema as schema_modelsfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema@@ -2019,6 +2020,67 @@ class TestResumeCursorCommit:# A wrapped resource re-reads its last written page; the merge on the primary key absorbs it.assert sorted(set(written + resumed)) == list(range(1, page_count + 1))+ @pytest.mark.parametrize(+ "grace_seconds,expected_written",+ [+ # Today's behaviour: the first item after the signal hands off.+ pytest.param(0.0, ["a", "b"], id="no_grace_hands_off_at_the_signal"),+ # The run keeps writing and committing through the grace, and hands off at its end.+ pytest.param(60.0, ["a", "b", "c", "d"], id="hands_off_after_the_grace"),+ ],+ )+ @pytest.mark.asyncio+ async def test_a_handoff_after_the_shutdown_grace_resumes_after_its_last_written_batch(+ self, grace_seconds: float, expected_written: list[str]+ ) -> None:+ redis = MagicMock()+ manager = _manager()+ env = ActivityEnvironment()+ loop = asyncio.get_running_loop()+ grace_started, grace_ended = asyncio.Event(), asyncio.Event()+ monitors: list[ShutdownMonitor] = []+ ids = ["a", "b", "c", "d", "e"]++ async def grace_sleep(seconds: float) -> None:+ grace_started.set()+ await grace_ended.wait()++ async def send_the_signal() -> None:+ env.worker_shutdown()+ await (grace_started.wait() if grace_seconds else monitors[0].wait_for_worker_shutdown())++ async def end_the_grace() -> None:+ grace_ended.set()+ await monitors[0].wait_for_worker_shutdown()++ def on_the_event_loop(step: Any) -> None:+ asyncio.run_coroutine_threadsafe(step(), loop).result(timeout=5)++ def items():+ for row_id in ids:+ if row_id == "b":+ on_the_event_loop(send_the_signal)+ if row_id == "d" and grace_seconds:+ on_the_event_loop(end_the_grace)+ manager.save_state(_Cursor(row_id))+ yield pa.table({"id": [row_id]})++ pipeline = _runnable_pipeline(manager, items)++ async def activity_body() -> None:+ async with ShutdownMonitor(grace_seconds=grace_seconds, sleep=grace_sleep) as monitor:+ monitors.append(monitor)+ pipeline._shutdown_monitor = monitor+ await _run_expecting(pipeline, redis, WorkerShuttingDownError)++ await env.run(activity_body)++ committed = [json.loads(call.args[1])["id"] for call in redis.set.call_args_list]+ assert _staged_ids(pipeline) == expected_written+ assert committed[-1] == expected_written[-1]+ # The next attempt reads after the committed cursor, so every row lands exactly once.+ assert expected_written + ids[ids.index(committed[-1]) + 1 :] == ids+@pytest.mark.asyncioasync def test_cursor_not_persisted_for_rows_the_batcher_still_holds(self) -> None:redis = MagicMock()@@ -3143,10 +3205,14 @@ class TestSourcePreemption:if checkpoint == "void":pipeline._handoff_checkpoint.observe(None)- with patch(f"{_PIPELINE}.is_young_first_attempt", return_value=young):+ pipeline._shutdown_grace_seconds = 60.0++ with patch(f"{_PIPELINE}.is_young_first_attempt", return_value=young) as young_check:decision = pipeline._preemption_decision()assert (decision.eligible, decision.reason) == expected+ # The age leaves out the shutdown grace, so the grace never makes a young run old.+ assert all(call.args == (60.0,) for call in young_check.call_args_list)@pytest.mark.parametrize("checkpoints,laned,expected_ids,expected_reason",diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 8f37245b9d67..5e12de6bf350 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -230,6 +230,18 @@ CONTINUABLE_SYNC_TYPES = frozenset(MAX_CONTINUED_JOBS = 20+def _graceful_shutdown_timeout_seconds() -> float:+ grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS+ return float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds()+++def _import_shutdown_grace_seconds() -> float:+ # At most half the SDK's graceful shutdown timeout, so the hand-off after the grace always has+ # time to finish before the SDK cancels the activity.+ grace = min(settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS, _graceful_shutdown_timeout_seconds() / 2)+ return max(grace, 0.0)++@database_sync_to_async_pooldef _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[ExternalDataJob]:"""The jobs before this one that worker restarts stopped, newest first, when this job can continue their cursor.@@ -526,7 +538,8 @@ async def _run_import_attempt(inputs: ImportDataActivityInputs) -> PipelineResulasync def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger: FilteringBoundLogger, progress: DurableProgress) -> PipelineResult:- async with Heartbeater(factor=30), ShutdownMonitor() as shutdown_monitor:+ shutdown_grace_seconds = _import_shutdown_grace_seconds()+ async with Heartbeater(factor=30), ShutdownMonitor(grace_seconds=shutdown_grace_seconds) as shutdown_monitor:await setup_row_tracking(inputs.team_id, inputs.schema_id)try:@@ -692,9 +705,9 @@ async def _import_data_with_reporting(if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(schema, IMPORT_RIDE_OUT_FLAG):- grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS+ # The ride-out's stopwatch starts when the monitor reports the shutdown, after the shutdown grace.ride_out = RideOutConfig(- grace_seconds=float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds(),+ grace_seconds=_graceful_shutdown_timeout_seconds() - shutdown_grace_seconds,max_seconds=settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS,)@@ -914,6 +927,7 @@ async def _import_data_with_reporting(continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,+ shutdown_grace_seconds=shutdown_grace_seconds,)else:raise ValueError(f"Source type {model.pipeline.source_type} not supported")@@ -988,9 +1002,9 @@ POSTHOG_DATABASE_UNAVAILABLE_MESSAGE = (def _log_worker_shutdown_during_import(logger: FilteringBoundLogger) -> None:- # One line per import still running when the worker got SIGTERM. A matching hand-off line from- # _handle_import_error means the import moved to another pod; without one, the import held- # this pod until it finished or the graceful shutdown timeout ran out.+ # One line per import still running when the shutdown grace after SIGTERM ended. A matching+ # hand-off line from _handle_import_error means the import moved to another pod; without one,+ # the import held this pod until it finished or the graceful shutdown timeout ran out.info = activity.info()logger.info("Worker shutdown detected while import is running",@@ -1306,6 +1320,7 @@ async def _run(continued_job_ids: tuple[str, ...] = (),ride_out: RideOutConfig | None = None,progress: DurableProgress | None = None,+ shutdown_grace_seconds: float = 0.0,) -> PipelineResult:try:reset_pipeline = (@@ -1331,6 +1346,7 @@ async def _run(continued_job_ids=continued_job_ids,ride_out=ride_out,progress=progress,+ shutdown_grace_seconds=shutdown_grace_seconds,)result = await pipeline.run()diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex 41a4e86cad37..e7f2a716f069 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -2149,25 +2149,33 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(@pytest.mark.asyncio@pytest.mark.parametrize(- "flag_enabled,handoffs_are_free,grace,expected",+ "flag_enabled,handoffs_are_free,grace,shutdown_grace,expected",[- pytest.param(True, True, 7200, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),+ pytest.param(True, True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),+ # The ride-out starts when the shutdown grace ends, so its deadline leaves that time out.+ pytest.param(+ True, True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"+ ),# The worker falls back to a five-minute graceful shutdown when the setting is unset.- pytest.param(True, True, None, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),+ pytest.param(+ True, True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"+ ),# Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.- pytest.param(True, False, 7200, None, id="off_when_a_handoff_uses_a_retry_attempt"),- pytest.param(False, True, 7200, None, id="off_when_the_flag_is_off"),+ pytest.param(True, False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),+ pytest.param(False, True, 7200, 0.0, None, id="off_when_the_flag_is_off"),],)async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(flag_enabled: bool,handoffs_are_free: bool,grace: int | None,+ shutdown_grace: float,expected: RideOutConfig | None,settings,):settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS = 1800.0settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = grace+ settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS = shutdown_gracesource = mock.MagicMock(spec=SimpleSource)source.parse_config.return_value = {}source.source_for_pipeline.return_value = mock.MagicMock()@@ -2185,6 +2193,37 @@ async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)+ "grace_setting,graceful_timeout,expected",+ [+ pytest.param(60.0, 7200, 60.0, id="default"),+ pytest.param(0.0, 7200, 0.0, id="zero_hands_off_at_the_signal"),+ # The hand-off after the grace keeps at least half the graceful shutdown timeout.+ pytest.param(600.0, None, 150.0, id="capped_at_half_the_default_timeout"),+ pytest.param(-5.0, 7200, 0.0, id="negative_is_no_grace"),+ ],+)+async def test_the_shutdown_grace_reaches_the_monitor_and_the_pipeline(+ grace_setting: float, graceful_timeout: int | None, expected: float, settings+):+ settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS = grace_setting+ settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = graceful_timeout+ source = mock.MagicMock(spec=SimpleSource)+ source.parse_config.return_value = {}+ source.source_for_pipeline.return_value = mock.MagicMock()+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)++ with (+ _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,+ mock.patch.object(module, "ShutdownMonitor", return_value=_FakeAsyncCM()) as monitor_cls,+ ):+ await import_data_activity_sync(_inputs_no_reset())++ monitor_cls.assert_called_once_with(grace_seconds=expected)+ assert run_mock.await_args.kwargs["shutdown_grace_seconds"] == expected++@parameterized.expand([("no_checkpoint_past_the_allowance", True, PROGRESSLESS_RESUMABLE_ATTEMPTS + 1, False, True),