:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): ride out the worker shutdown when a hand-off would lose work (#120595)
Daniel RCf4973727a7ab
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 7533754c28e9..3054c2371eff 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -313,6 +313,15 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_ENABLED: bool = get_from_env(DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env("DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS", 300.0, type_cast=float)+# At a worker shutdown an import that has not yet committed a resume cursor or staged a carry-over+# value keeps the worker, up to a deadline below the graceful shutdown timeout. The+# `warehouse-import-ride-out` flag turns this on per schema. This is the longest such an import holds+# the worker when it could have handed off sooner; a run that restarts from scratch either way keeps+# the grace period minus a margin instead. The gate reads GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS, so set the grace through this+# variable rather than the worker's command-line flag.+DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS: float = get_from_env(+ "DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS", 3600.0, type_cast=float+)# Restore unfinished append runs only after all extract and load workers have this implementation.DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED: bool = get_from_env("DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED", False, type_cast=str_to_booldiff --git a/posthog/temporal/common/shutdown.py b/posthog/temporal/common/shutdown.pyindex bad0a5ffb5d5..189a03149bc9 100644--- a/posthog/temporal/common/shutdown.py+++ b/posthog/temporal/common/shutdown.py@@ -1,6 +1,8 @@+import timeimport signalimport typingimport asyncio+import datetimeimport threadingimport contextvarsfrom collections.abc import Callable@@ -8,6 +10,10 @@ from collections.abc import Callablefrom structlog import get_loggerfrom temporalio import activity+# The SDK cancels activities still running this long after the shutdown started, unless the worker+# was built with another timeout.+DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT = datetime.timedelta(minutes=5)+LOGGER = get_logger(__name__)@@ -67,6 +73,7 @@ class ShutdownMonitor:self._is_shutdown_event = asyncio.Event()self._is_shutdown_event_sync = threading.Event()self._stop_event_sync = threading.Event()+ self._shutdown_at: float | None = Nonedef __str__(self) -> str:"""Return a string representation of this `ShutdownMonitor`."""@@ -108,6 +115,7 @@ class ShutdownMonitor:# Not running in an activity context.return+ self._record_shutdown_time()self.logger.info("Shutdown detected.")# Run the callbacks before the event is set: a caller that sees the event can raise and# leave the monitor at once, and a callback that ran after that would never run at all.@@ -154,6 +162,7 @@ class ShutdownMonitor:# it's a wrapper on `threading.Event.wait`, which does return a `bool`# indicating the reason. So we must also check if the event was set.if activity.is_worker_shutdown():+ self._record_shutdown_time()self.logger.info("Shutdown detected.")self._is_shutdown_event_sync.set()break@@ -204,6 +213,20 @@ class ShutdownMonitor:"""Check if worker is shutting down."""return self._is_shutdown_event.is_set() or self._is_shutdown_event_sync.is_set()+ def _record_shutdown_time(self) -> None:+ if self._shutdown_at is None:+ self._shutdown_at = time.monotonic()++ def seconds_since_shutdown(self) -> float | None:+ """How long ago the worker started to shut down, or None while it still runs.++ A caller created after the shutdown cannot learn the moment from a callback, because the+ callbacks have already run by then. It reads the elapsed time here instead.+ """+ if self._shutdown_at is None:+ return None+ return time.monotonic() - self._shutdown_at+def raise_if_is_worker_shutdown(self):"""Raise an exception if worker is shutting down."""if self.is_worker_shutdown():diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.pyindex 22f327d22371..55fdc62def10 100644--- a/posthog/temporal/common/worker.py+++ b/posthog/temporal/common/worker.py@@ -40,6 +40,7 @@ from posthog.temporal.common.interceptor import is_task_queue_supportedfrom posthog.temporal.common.liveness_tracker import LivenessInterceptorfrom posthog.temporal.common.logger import get_write_only_loggerfrom posthog.temporal.common.posthog_client import PostHogClientInterceptor+from posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUTfrom posthog.temporal.common.slo_interceptor import SloInterceptorfrom posthog.temporal.common.utils import configure_asyncify_executor, shutdown_asyncify_executorfrom posthog.temporal.data_modeling.metrics import (@@ -547,7 +548,7 @@ async def create_worker(workflows=workflows,activities=activities,workflow_runner=UnsandboxedWorkflowRunner(),- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,interceptors=supported_interceptors,activity_executor=ThreadPoolExecutor(max_workers=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS),tuner=WorkerTuner.create_resource_based(@@ -573,7 +574,7 @@ async def create_worker(workflows=workflows,activities=activities,workflow_runner=UnsandboxedWorkflowRunner(),- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,interceptors=supported_interceptors,activity_executor=ThreadPoolExecutor(max_workers=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS),max_concurrent_activities=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS,@@ -591,7 +592,7 @@ async def create_worker(activities=config.activities,max_concurrent_activities=config.max_concurrent_activities,activity_executor=ThreadPoolExecutor(max_workers=config.max_concurrent_activities),- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,interceptors=[interceptor()for interceptor in ALL_INTERCEPTOR_CLASSESdiff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex ecbbfccbbc4b..12c370d6dd5b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -922,7 +922,12 @@ class ExternalDataJobWorkflow(PostHogWorkflow):delay = min(IMPORT_HANDOFF_FIRST_DELAY * 2 ** min(early_handoffs, 10), IMPORT_HANDOFF_MAX_DELAY)workflow.logger.info("Import handed off by a worker that is shutting down, running it again",- extra={"handoffs": handoffs, "delay_seconds": delay.total_seconds()},+ extra={+ "handoffs": handoffs,+ "delay_seconds": delay.total_seconds(),+ "handoff_made_progress": pipeline_result.get("handoff_made_progress"),+ "handoff_reason": pipeline_result.get("handoff_reason"),+ },)await workflow.sleep(delay)finally:diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.pyindex b3ab9e416f72..59940b8c0654 100644--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py@@ -160,12 +160,26 @@ def get_import_not_preempted_metric(source_type: str | None, reason: str) -> Met)-def get_shutdown_handoff_delay_metric(source_type: str | None, mode: str) -> MetricHistogramFloat:+def get_import_finished_during_shutdown_metric(source_type: str | None, rode_out: bool) -> MetricCounter:+ # The imports a hand-off would have restarted. `rode_out` is True when the gate kept the import+ # on the worker past the shutdown at least once.+ return (+ activity.metric_meter()+ .with_additional_attributes({"source_type": source_type or "unknown", "rode_out": str(rode_out).lower()})+ .create_counter(+ "warehouse_import_finished_during_shutdown_total",+ "Imports that finished on a worker that was shutting down instead of handing off.",+ )+ )+++def get_shutdown_handoff_delay_metric(source_type: str | None, mode: str, reason: str) -> MetricHistogramFloat:# `mode` is `preempted` when the pipeline stopped waiting for the source, and `cooperative`- # when the source yielded an item or reached a safe point.+ # when the source yielded an item or reached a safe point. `reason` is why the gate let the+ # import leave the worker (see `HandoffGate`).return (activity.metric_meter()- .with_additional_attributes({"source_type": source_type or "unknown", "mode": mode})+ .with_additional_attributes({"source_type": source_type or "unknown", "mode": mode, "reason": reason}).create_histogram_float("warehouse_import_shutdown_handoff_delay_seconds","Seconds from the worker shutdown to the hand-off of an import.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.pyindex c8c3db8df523..7216e551ebcc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py@@ -10,7 +10,7 @@ it gives the source a quiet period to hand off in the usual way, and then it raiimport timeimport asynciofrom collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable-from typing import Any, TypeVar+from typing import TYPE_CHECKING, Any, TypeVarfrom structlog.types import FilteringBoundLoggerfrom temporalio import activity@@ -27,8 +27,14 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.abaPulledItem,)+if TYPE_CHECKING:+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import HandoffGate+T = TypeVar("T")+# A ride-out can last an hour; progress that lands meanwhile must end the wait within this time.+RIDE_OUT_WAIT_SLICE_SECONDS = 30.0+@frozenclass PreemptionConfig:@@ -42,6 +48,9 @@ class PreemptionDecision:eligible: bool# A metric label: why the run can continue on another worker, or why it cannot.reason: str+ # A run that cannot continue elsewhere and restarts from scratch either way. A hand-off before+ # the grace period ends then only saves the retry attempt the kill at its end would cost.+ restarts_from_scratch: bool = Falseclass SourcePreemptedError(WorkerShuttingDownError):@@ -59,12 +68,17 @@ class ShutdownStopwatch:def __init__(self, shutdown_monitor: ShutdownMonitor, clock: Callable[[], float] = time.monotonic) -> None:self._clock = clock+ self._shutdown_monitor = shutdown_monitorself._started_at: float | None = Noneshutdown_monitor.run_on_shutdown(self.start)def start(self) -> None:- if self._started_at is None:- self._started_at = self._clock()+ if self._started_at is not None:+ return+ # A stopwatch built after the shutdown - a source setup that ran long - never receives the+ # monitor's callback, so it backdates its start to the shutdown itself. The grace period+ # runs from there, and a deadline measured from here instead would outlast it.+ self._started_at = self._clock() - (self._shutdown_monitor.seconds_since_shutdown() or 0.0)def elapsed_seconds(self) -> float | None:return None if self._started_at is None else self._clock() - self._started_at@@ -81,6 +95,7 @@ class SourcePreemptor:fence_source: Callable[[], Awaitable[bool]],source_type: str,logger: FilteringBoundLogger,+ gate: "HandoffGate | None" = None,) -> None:self._config = configself._shutdown_monitor = shutdown_monitor@@ -89,8 +104,13 @@ class SourcePreemptor:self._fence_source = fence_sourceself._source_type = source_typeself._logger = logger+ # A gate that rides out makes the quiet period at least as long as its deadline, and lets+ # a run that could not be preempted leave the worker once the deadline passes.+ self._gate = gateself._decision: PreemptionDecision | None = None- self._quiet_period_extension = 0.0+ # Set after a failed fence: the next attempt to leave waits until this point on the stopwatch.+ self._fence_retry_at = 0.0+ self._held_by_ride_out = Falseasync def iterate(self, items: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]:"""Yield the items of the source. Raises `SourcePreemptedError` when it preempts the source.@@ -128,21 +148,33 @@ class SourcePreemptor:self._stopwatch.start()decision = await self._decision_at_shutdown()- if not decision.eligible:+ if not decision.eligible and not self._preempts_at_deadline(decision):await asyncio.wait({pull})return+ # A run that restarts from scratch gets no quiet period: nothing it could do in that time+ # changes what the next attempt reads. A failed fence still extends the wait.+ quiet_period = self._config.quiet_period_seconds if decision.eligible else 0.0elapsed = self._stopwatch.elapsed_seconds() or 0.0- remaining = self._config.quiet_period_seconds + self._quiet_period_extension - elapsed+ quiet_remaining = max(quiet_period, self._fence_retry_at) - elapsed+ ride_out_remaining = self._gate.seconds_until_deadline() if self._gate is not None else 0.0+ if ride_out_remaining > quiet_remaining:+ self._held_by_ride_out = True+ remaining = max(quiet_remaining, ride_out_remaining)if remaining > 0:- # An item or a safe point in this time hands the run off in the usual way.- await asyncio.wait({pull}, timeout=remaining)+ # An item or a safe point in this time hands the run off in the usual way. Short+ # slices, so progress that lands during a ride-out ends the wait soon after.+ await asyncio.wait({pull}, timeout=min(remaining, RIDE_OUT_WAIT_SLICE_SECONDS))continue+ if self._gate is not None and (self._held_by_ride_out or not decision.eligible):+ preemption_reason = self._gate.leave_reason()+ else:+ preemption_reason = decision.reasonif not await self._fence_source():# The source is in the middle of a write of its resume state. A hand-off now could# let that write land after the next attempt starts.- self._quiet_period_extension += self._config.quiet_period_seconds+ self._fence_retry_at = elapsed + self._config.quiet_period_secondsawait self._logger.awarning("Could not stop the writes of the source, waiting another quiet period before preemption",quiet_period_seconds=self._config.quiet_period_seconds,@@ -151,16 +183,22 @@ class SourcePreemptor:# The source can no longer store resume state, so the run must leave this worker now,# also when an item arrived during the fence. The next attempt reads that item again.+ if self._gate is not None:+ self._gate.record_preemption(preemption_reason)if activity.in_activity():- get_import_preempted_metric(self._source_type, decision.reason).add(1)+ get_import_preempted_metric(self._source_type, preemption_reason).add(1)await self._logger.ainfo("Preempting the import because the source did not hand off in the quiet period",- preemption_reason=decision.reason,+ preemption_reason=preemption_reason,quiet_period_seconds=self._config.quiet_period_seconds,seconds_since_shutdown=round(self._stopwatch.elapsed_seconds() or 0.0, 1),)raise SourcePreemptedError.build()+ def _preempts_at_deadline(self, decision: PreemptionDecision) -> bool:+ """Whether a run that cannot continue elsewhere still leaves the worker before the grace ends."""+ return self._gate is not None and self._gate.ride_out_enabled and decision.restarts_from_scratch+async def _decision_at_shutdown(self) -> PreemptionDecision:# Decided once, at the shutdown. A run that was young then stays eligible for the whole# quiet period.@@ -172,6 +210,11 @@ class SourcePreemptor:preemption_reason=self._decision.reason,quiet_period_seconds=self._config.quiet_period_seconds,)+ elif self._preempts_at_deadline(self._decision):+ await self._logger.ainfo(+ "Worker is shutting down, the source keeps the worker until the ride-out deadline",+ not_preempted_reason=self._decision.reason,+ )else:if activity.in_activity():get_import_not_preempted_metric(self._source_type, self._decision.reason).add(1)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.pynew file mode 100644index 000000000000..f103eb8386db--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py@@ -0,0 +1,215 @@+"""Ride-out: an import that has nothing durable yet keeps its worker through the shutdown grace.++`HandoffGate` is the one place that decides whether an import leaves a shutting-down worker now.+Two invariants hold it up: the activity's `Heartbeater` runs on the event loop while the source runs+on its own thread, so an import that rides out keeps heartbeating and never counts as a zombie; and+the deadline stays below the SDK's graceful shutdown timeout, which cancels activities, and below the+pod's `terminationGracePeriodSeconds`.+"""++import threading+from collections.abc import Callable+from datetime import UTC, datetime+from typing import TYPE_CHECKING++from structlog.types import FilteringBoundLogger+from temporalio import activity++from posthog.dataclasses import frozen+from posthog.temporal.common.shutdown import ShutdownMonitor++if TYPE_CHECKING:+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import ShutdownStopwatch++# Covers the gap to the next item or safe point, the result flush, and the SDK's cancellation of+# activities once the graceful shutdown timeout passes. A grace period shorter than this gives a+# deadline of zero, so the import hands off at once.+# Per-schema rollout flag. Off, every hand-off follows the legacy rule.+IMPORT_RIDE_OUT_FLAG = "warehouse-import-ride-out"+RIDE_OUT_GRACE_MARGIN_SECONDS = 900.0+# An execution younger than this restarts for nearly free, so it leaves the worker at once and the+# pod keeps draining fast. Shorter than `NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE` on purpose: that one+# decides whether a run may hand off at all, this one only whether it is worth holding the worker.+RIDE_OUT_MIN_EXECUTION_AGE_SECONDS = 120.0+++@frozen+class RideOutConfig:+ grace_seconds: float+ # How long a run that could hand off sooner may hold the worker; the grace minus the margin bounds+ # it too.+ max_seconds: float++ def deadline_seconds_after_shutdown(self, *, restarts_from_scratch: bool) -> float:+ before_grace_ends = max(self.grace_seconds - RIDE_OUT_GRACE_MARGIN_SECONDS, 0.0)+ if restarts_from_scratch:+ return before_grace_ends+ return min(before_grace_ends, self.max_seconds)+++class DurableProgress:+ """What this execution has made durable, shared by the event loop and the source thread."""++ def __init__(self, *, carry_over_counts: bool = True) -> None:+ self._lock = threading.Lock()+ self._reasons: list[str] = []+ # A staged carry-over value is progress only when the next execution reads it.+ self._carry_over_counts = carry_over_counts+ self._handoff_reason: str | None = None++ def mark(self, reason: str) -> None:+ if reason == "carry_over_staged" and not self._carry_over_counts:+ return+ with self._lock:+ if reason not in self._reasons:+ self._reasons.append(reason)++ @property+ def has_progress(self) -> bool:+ with self._lock:+ return bool(self._reasons)++ @property+ def reasons(self) -> tuple[str, ...]:+ with self._lock:+ return tuple(self._reasons)++ @property+ def handoff_reason(self) -> str | None:+ return self._handoff_reason++ def record_handoff_reason(self, reason: str) -> None:+ """Keep the first reason: an abandoned source thread can still ask the gate after a preemption."""+ with self._lock:+ if self._handoff_reason is None:+ self._handoff_reason = reason+++def activity_execution_age_seconds() -> float | None:+ if not activity.in_activity():+ return None+ return (datetime.now(UTC) - activity.info().started_time).total_seconds()+++class HandoffGate:+ """Decides whether the import leaves a shutting-down worker now.++ Every observer of the shutdown asks the gate: the item loop, the safe point handler on the+ source thread, the source's own waits, and the preemptor. With `config` None the gate is today's+ behaviour, which `legacy_wants_handoff` provides.+ """++ def __init__(+ self,+ *,+ shutdown_monitor: ShutdownMonitor,+ stopwatch: "ShutdownStopwatch",+ progress: DurableProgress,+ config: RideOutConfig | None,+ legacy_wants_handoff: Callable[[], bool],+ logger: FilteringBoundLogger,+ restarts_from_scratch: Callable[[], bool] = lambda: False,+ execution_age_seconds: Callable[[], float | None] = activity_execution_age_seconds,+ ) -> None:+ self._shutdown_monitor = shutdown_monitor+ self._stopwatch = stopwatch+ self._progress = progress+ self._config = config+ self._legacy_wants_handoff = legacy_wants_handoff+ self._logger = logger+ self._restarts_from_scratch = restarts_from_scratch+ self._execution_age_seconds = execution_age_seconds+ self._lock = threading.Lock()+ self._logged_handoff = False+ self.rode_out = False++ @property+ def ride_out_enabled(self) -> bool:+ return self._config is not None++ def should_hand_off_now(self) -> bool:+ if not self._shutdown_monitor.is_worker_shutdown():+ return False+ # The stopwatch's own callback runs on the event loop; the source thread can see the+ # shutdown before it.+ self._stopwatch.start()+ if self._config is None:+ return self._decide("legacy", self._legacy_wants_handoff())+ early_reason = self._reason_to_leave_before_the_deadline()+ if early_reason is not None:+ return self._decide(early_reason, True)+ return self._decide("deadline", self.seconds_until_deadline() <= 0.0)++ def may_hand_off(self) -> bool:+ """Whether the run may leave the worker here, with the shutdown left to the monitor.++ The item loop and the safe point ask this and then let+ `ShutdownMonitor.raise_if_is_worker_shutdown` have the last word, as the pipeline always+ has. With the ride-out off that is the legacy decision alone: reading the shutdown flag+ here first would be the same in production and a different thing to every caller that+ drives the shutdown through the monitor.+ """+ if self._config is not None:+ return self.should_hand_off_now()+ if not self._legacy_wants_handoff():+ return False+ # The stopwatch for the reason, not the monitor: it is set once the shutdown callback has+ # run, and reading it costs the monitor nothing.+ if self._stopwatch.elapsed_seconds() is not None:+ self._progress.record_handoff_reason("legacy")+ return True++ def seconds_until_deadline(self) -> float:+ """How long the preemptor must still wait for the source before it hands off. Zero when the+ gate would hand off at once."""+ if self._config is None or self._reason_to_leave_before_the_deadline() is not None:+ return 0.0+ elapsed = self._stopwatch.elapsed_seconds() or 0.0+ deadline = self._config.deadline_seconds_after_shutdown(restarts_from_scratch=self._restarts_from_scratch())+ return max(deadline - elapsed, 0.0)++ def leave_reason(self) -> str:+ """Why the gate lets the import leave now: `progress`, `young`, or the `deadline`."""+ return self._reason_to_leave_before_the_deadline() or "deadline"++ def record_preemption(self, reason: str) -> None:+ self._progress.record_handoff_reason(f"preempted_{reason}")++ def raise_if_should_hand_off(self) -> None:+ if self.may_hand_off():+ self._shutdown_monitor.raise_if_is_worker_shutdown()++ def _reason_to_leave_before_the_deadline(self) -> str | None:+ if self._progress.has_progress:+ return "progress"+ age = self._execution_age_seconds()+ if age is not None and age < RIDE_OUT_MIN_EXECUTION_AGE_SECONDS:+ return "young"+ return None++ def _decide(self, reason: str, hand_off: bool) -> bool:+ if not hand_off:+ if self._config is None:+ return False+ with self._lock:+ first_time = not self.rode_out+ self.rode_out = True+ if first_time:+ self._logger.info(+ "Worker is shutting down and the import has no durable progress yet, riding out the grace period",+ deadline_seconds=round(self.seconds_until_deadline(), 1),+ execution_age_seconds=round(self._execution_age_seconds() or 0.0, 1),+ )+ return False+ self._progress.record_handoff_reason(reason)+ with self._lock:+ log_it = self.rode_out and not self._logged_handoff+ self._logged_handoff = True+ if log_it:+ self._logger.info(+ "Handing the import off after riding out the shutdown",+ handoff_reason=reason,+ seconds_since_shutdown=round(self._stopwatch.elapsed_seconds() or 0.0, 1),+ progress=list(self._progress.reasons),+ )+ return Truediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.pyindex e377b02b2759..c12df03f191a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py@@ -2,8 +2,6 @@ import timefrom collections.abc import Callablefrom typing import Any-from posthog.temporal.common.shutdown import ShutdownMonitor-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager@@ -22,13 +20,13 @@ class PipelineSafePointHandler:def __init__(self,*,- shutdown_monitor: ShutdownMonitor,+ raise_if_should_hand_off: Callable[[], None],resumable_source_manager: ResumableSourceManager[Any],has_unwritten_rows: Callable[[], bool],commit_interval_seconds: float | None = None,clock: Callable[[], float] = time.monotonic,):- self._shutdown_monitor = shutdown_monitor+ self._raise_if_should_hand_off = raise_if_should_hand_offself._resumable_source_manager = resumable_source_managerself._has_unwritten_rows = has_unwritten_rowsself._commit_interval_seconds = (@@ -42,7 +40,7 @@ class PipelineSafePointHandler:self._resumable_source_manager.confirm()# The pipeline's `except` path writes the rows it holds and then commits the confirmed cursor,# so a raise here hands the run to another worker without losing progress.- self._shutdown_monitor.raise_if_is_worker_shutdown()+ self._raise_if_should_hand_off()now = self._clock()if now - self._last_commit < self._commit_interval_seconds:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.pynew file mode 100644index 000000000000..cd5305fdb76b--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.py@@ -0,0 +1,226 @@+from typing import Any, cast++import pytest+from unittest.mock import MagicMock++from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import ShutdownStopwatch+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ DurableProgress,+ HandoffGate,+ RideOutConfig,+)+++class _Clock:+ def __init__(self) -> None:+ self.now = 0.0++ def __call__(self) -> float:+ return self.now+++class _Monitor:+ def __init__(self, shutting_down: bool, shutdown_age: float | None = None) -> None:+ self.shutting_down = shutting_down+ self.shutdown_age = shutdown_age++ def run_on_shutdown(self, callback: Any) -> None:+ pass++ def is_worker_shutdown(self) -> bool:+ return self.shutting_down++ def seconds_since_shutdown(self) -> float | None:+ return self.shutdown_age++ def raise_if_is_worker_shutdown(self) -> None:+ if self.shutting_down:+ raise RuntimeError("worker shutting down")+++def _gate(+ *,+ shutting_down: bool = True,+ ride_out: bool = True,+ progress: bool = False,+ age: float | None = 600.0,+ legacy: bool = True,+ restarts_from_scratch: bool = False,+ shutdown_age: float | None = None,+) -> tuple[HandoffGate, _Clock, DurableProgress]:+ clock = _Clock()+ monitor = _Monitor(shutting_down, shutdown_age)+ durable = DurableProgress()+ if progress:+ durable.mark("resume_commit")+ gate = HandoffGate(+ shutdown_monitor=cast(Any, monitor),+ stopwatch=ShutdownStopwatch(cast(Any, monitor), clock=clock),+ progress=durable,+ config=RideOutConfig(grace_seconds=7200.0, max_seconds=3600.0) if ride_out else None,+ legacy_wants_handoff=lambda: legacy,+ logger=MagicMock(),+ restarts_from_scratch=lambda: restarts_from_scratch,+ execution_age_seconds=lambda: age,+ )+ return gate, clock, durable+++ "kwargs,elapsed,expected",+ [+ pytest.param({"shutting_down": False}, 0.0, (False, None), id="worker_running"),+ pytest.param({"ride_out": False, "legacy": True}, 0.0, (True, "legacy"), id="kill_switch_off_legacy_hands_off"),+ pytest.param({"ride_out": False, "legacy": False}, 7200.0, (False, None), id="kill_switch_off_legacy_waits"),+ pytest.param({"progress": True}, 0.0, (True, "progress"), id="durable_progress_hands_off_at_once"),+ pytest.param({"age": 10.0}, 0.0, (True, "young"), id="young_execution_hands_off_at_once"),+ pytest.param({}, 3599.0, (False, None), id="no_progress_rides_out_before_the_deadline"),+ pytest.param({}, 3600.0, (True, "deadline"), id="no_progress_hands_off_at_the_deadline"),+ pytest.param({"age": None}, 0.0, (False, None), id="unknown_age_rides_out"),+ # A run that restarts from scratch either way keeps the whole grace period, not the ride-out cap.+ pytest.param({"restarts_from_scratch": True}, 6299.0, (False, None), id="from_scratch_waits_past_the_cap"),+ pytest.param(+ {"restarts_from_scratch": True}, 6300.0, (True, "deadline"), id="from_scratch_leaves_before_grace_ends"+ ),+ ],+)+def test_the_gate_hands_off_only_when_a_restart_loses_nothing(+ kwargs: dict[str, Any], elapsed: float, expected: tuple[bool, str | None]+) -> None:+ gate, clock, progress = _gate(**kwargs)++ gate.should_hand_off_now()+ clock.now = elapsed+ hands_off = gate.should_hand_off_now()++ assert (hands_off, progress.handoff_reason) == expected+++ "kwargs,expected",+ [+ pytest.param({}, 3500.0, id="no_progress"),+ pytest.param({"progress": True}, 0.0, id="progress"),+ pytest.param({"age": 10.0}, 0.0, id="young"),+ pytest.param({"restarts_from_scratch": True}, 6200.0, id="from_scratch"),+ ],+)+def test_the_preemptor_waits_exactly_as_long_as_the_gate_would(kwargs: dict[str, Any], expected: float) -> None:+ gate, clock, _ = _gate(**kwargs)+ gate.should_hand_off_now()+ clock.now = 100.0++ assert gate.seconds_until_deadline() == expected+++def test_a_gate_opened_after_the_shutdown_counts_its_deadline_from_the_shutdown() -> None:+ # A source setup that runs past the shutdown builds the pipeline, and so the stopwatch, after+ # the monitor's callbacks already ran. The deadline must still land inside the grace period.+ gate, clock, _ = _gate(restarts_from_scratch=True, shutdown_age=1200.0)++ # The first question starts the stopwatch, backdated to the shutdown 1200 s ago.+ assert gate.should_hand_off_now() is False++ clock.now = 6300.0 - 1200.0 - 1.0+ assert gate.should_hand_off_now() is False++ clock.now = 6300.0 - 1200.0+ assert gate.should_hand_off_now() is True+++def test_a_staged_carry_over_value_counts_as_progress_only_when_the_next_execution_reads_it() -> None:+ counted, ignored = DurableProgress(carry_over_counts=True), DurableProgress(carry_over_counts=False)+ counted.mark("carry_over_staged")+ ignored.mark("carry_over_staged")+ assert (counted.has_progress, ignored.has_progress) == (True, False)+++def test_the_first_handoff_reason_sticks() -> None:+ # An abandoned source thread can still ask the gate after a preemption; its answer must not+ # overwrite why the run really left.+ gate, clock, progress = _gate()+ gate.should_hand_off_now()+ gate.record_preemption("deadline")+ clock.now = 7200.0++ assert gate.should_hand_off_now() is True+ assert progress.handoff_reason == "preempted_deadline"+++def test_progress_that_lands_during_a_ride_out_ends_it() -> None:+ gate, clock, progress = _gate()+ assert gate.should_hand_off_now() is False+ clock.now = 100.0++ progress.mark("carry_over_staged")++ assert gate.should_hand_off_now() is True+ assert gate.seconds_until_deadline() == 0.0+ assert progress.handoff_reason == "progress"+ assert gate.rode_out is True+++ "kwargs,expected",+ [+ # With the ride-out off the monitor still owns the shutdown flag, so the hand-off point is+ # reached whether or not the worker is shutting down yet.+ pytest.param({"ride_out": False, "shutting_down": False}, True, id="kill_switch_off_asks_the_monitor"),+ pytest.param(+ {"ride_out": False, "shutting_down": False, "legacy": False}, False, id="kill_switch_off_run_waits"+ ),+ pytest.param({"shutting_down": False}, False, id="ride_out_on_reads_the_flag"),+ pytest.param({"progress": True}, True, id="ride_out_on_hands_off_with_progress"),+ ],+)+def test_the_handoff_point_leaves_the_shutdown_flag_to_the_monitor(kwargs: dict[str, Any], expected: bool) -> None:+ gate, _, _ = _gate(**kwargs)++ assert gate.may_hand_off() is expected+++def test_the_legacy_handoff_check_asks_the_monitor_to_raise() -> None:+ # Today's behaviour, which the kill switch keeps: the check asks the monitor to raise instead of+ # deciding on its flag first, so a monitor whose raise is what reports the shutdown still works.+ monitor = MagicMock()+ monitor.is_worker_shutdown.return_value = False+ monitor.raise_if_is_worker_shutdown.side_effect = RuntimeError("worker shutting down")+ gate = HandoffGate(+ shutdown_monitor=cast(Any, monitor),+ stopwatch=ShutdownStopwatch(cast(Any, monitor), clock=_Clock()),+ progress=DurableProgress(),+ config=None,+ legacy_wants_handoff=lambda: True,+ logger=MagicMock(),+ )++ with pytest.raises(RuntimeError, match="worker shutting down"):+ gate.raise_if_should_hand_off()++ # Not even to decide: a caller that counts the checks must see the same calls it always did.+ assert monitor.is_worker_shutdown.call_count == 0+++def test_raise_if_should_hand_off_raises_through_the_monitor_only_at_a_hand_off() -> None:+ gate, clock, _ = _gate()+ gate.raise_if_should_hand_off()++ clock.now = 3600.0+ with pytest.raises(RuntimeError, match="worker shutting down"):+ gate.raise_if_should_hand_off()+++ "grace,max_seconds,restarts_from_scratch,expected",+ [+ pytest.param(7200.0, 3600.0, False, 3600.0, id="max_below_grace"),+ pytest.param(7200.0, 3600.0, True, 6300.0, id="from_scratch_ignores_max"),+ pytest.param(1500.0, 3600.0, False, 600.0, id="grace_minus_margin_below_max"),+ pytest.param(300.0, 3600.0, False, 0.0, id="grace_shorter_than_the_margin"),+ ],+)+def test_the_deadline_stays_below_the_grace_period(+ grace: float, max_seconds: float, restarts_from_scratch: bool, expected: float+) -> None:+ config = RideOutConfig(grace_seconds=grace, max_seconds=max_seconds)+ assert config.deadline_seconds_after_shutdown(restarts_from_scratch=restarts_from_scratch) == expecteddiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.pyindex afd96a6e6c1a..343fc4233313 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py@@ -40,7 +40,7 @@ def test_handler_commits_only_a_cursor_whose_rows_are_written(manager = MagicMock()manager.has_staged_state.return_value = stagedhandler = PipelineSafePointHandler(- shutdown_monitor=MagicMock(),+ raise_if_should_hand_off=lambda: None,resumable_source_manager=manager,has_unwritten_rows=lambda: unwritten_rows,commit_interval_seconds=30.0,@@ -54,11 +54,9 @@ def test_handler_commits_only_a_cursor_whose_rows_are_written(def test_handler_confirms_the_cursor_and_raises_the_shutdown_before_it_commits() -> None:- monitor = MagicMock()- monitor.raise_if_is_worker_shutdown.side_effect = RuntimeError("worker shutting down")manager = MagicMock()handler = PipelineSafePointHandler(- shutdown_monitor=monitor,+ raise_if_should_hand_off=MagicMock(side_effect=RuntimeError("worker shutting down")),resumable_source_manager=manager,has_unwritten_rows=lambda: False,commit_interval_seconds=0.0,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.pyindex d95fc7aa8bcd..6d7b722593dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py@@ -39,3 +39,7 @@ class PipelineResult(TypedDict):fast_returned: NotRequired[bool]handed_off: NotRequired[bool]handoff_attempts_used: NotRequired[int]+ handoff_made_progress: NotRequired[bool]+ # Why the execution left the worker: `progress`, `young`, `deadline`, `legacy`, `preempted_<reason>`+ # or `unknown`.+ handoff_reason: NotRequired[str]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.pyindex adddbb34ea5e..3279780a3882 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobsretire_companion_job,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ DurableProgress,+ RideOutConfig,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.typings import PipelineResultfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline import PipelineV3from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.producer import (@@ -95,6 +99,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,kept_rows_synced: int | None = None,+ ride_out: RideOutConfig | None = None,+ progress: DurableProgress | None = None,) -> None:if not source_response.lanes:raise ValueError(f"{source_response.name} declares no lanes; run it on PipelineV3")@@ -114,6 +120,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ ride_out=ride_out,+ progress=progress,)# The base built the first lane; it shares the base's batch list so the two never disagree.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex f41eb8de45a9..e50877be504d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -34,7 +34,10 @@ from products.warehouse_sources.backend.temporal.data_imports.import_attempt impcurrent_import_attempt,current_import_attempt_cause,)-from products.warehouse_sources.backend.temporal.data_imports.metrics import get_shutdown_handoff_delay_metric+from products.warehouse_sources.backend.temporal.data_imports.metrics import (+ get_import_finished_during_shutdown_metric,+ get_shutdown_handoff_delay_metric,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import (cleanup_memory,commit_source_cursor,@@ -59,6 +62,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.pSourcePreemptedError,SourcePreemptor,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ DurableProgress,+ HandoffGate,+ RideOutConfig,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (PipelineSafePointHandler,framework_cursor_covers_buffered_rows,@@ -188,6 +196,10 @@ class PipelineV3(Generic[ResumableData]):# None when the pipeline waits for the source for as long as the source takes._preemption: PreemptionConfig | None = None_shutdown_stopwatch: ShutdownStopwatch | None = None+ # None when an import without durable progress hands off at once, as it always did.+ _ride_out: RideOutConfig | None = None+ _progress: DurableProgress | None = None+ _handoff_gate: HandoffGate | None = None_resumed_incremental_run_uuid: str | None = None_sent_resumed_run_finalization: bool = False_writes_staged_repartition_scheme: bool = False@@ -214,9 +226,13 @@ class PipelineV3(Generic[ResumableData]):resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,kept_rows_synced: int | None = None,+ ride_out: RideOutConfig | None = None,+ progress: DurableProgress | None = None,) -> None:self._resource = source_responseself._kept_rows_synced = kept_rows_synced+ self._ride_out = ride_out+ self._progress = progressself._source_cursor_manager = source_cursor_managerself._resource_name = source_response.name@@ -632,17 +648,19 @@ class PipelineV3(Generic[ResumableData]):scope = contextlib.ExitStack()if self._resumable_source_manager is not None:handler = PipelineSafePointHandler(- shutdown_monitor=self._shutdown_monitor,+ raise_if_should_hand_off=self._open_handoff_gate().raise_if_should_hand_off,resumable_source_manager=self._resumable_source_manager,has_unwritten_rows=lambda: (self._batcher.should_yield(include_incomplete_chunk=True) or self._pg_producer.has_held_batch),)+ # The gate, not the raw signal: a source wait that ended early during a ride-out would+ # skip every backoff and Retry-After for as long as the gate keeps the run here.scope.enter_context(activate_safe_point(handler,covers_framework_checkpoints=source_items_are_framework_output(items),- is_shutting_down=self._shutdown_monitor.is_worker_shutdown,+ is_shutting_down=self._open_handoff_gate().should_hand_off_now,))return scope@@ -664,16 +682,46 @@ class PipelineV3(Generic[ResumableData]):if is_young_first_attempt():return PreemptionDecision(eligible=True, reason="young_first_attempt")if checkpoint is not None and not checkpoint.is_void:- return PreemptionDecision(eligible=False, reason="watermark_carry_over_disabled")+ return PreemptionDecision(+ eligible=False, reason="watermark_carry_over_disabled", restarts_from_scratch=True+ )if self._schema.should_use_incremental_field:- return PreemptionDecision(eligible=False, reason="no_watermark_carry_over")- return PreemptionDecision(eligible=False, reason="non_resumable_full_refresh")+ return PreemptionDecision(eligible=False, reason="no_watermark_carry_over", restarts_from_scratch=True)+ return PreemptionDecision(eligible=False, reason="non_resumable_full_refresh", restarts_from_scratch=True)async def _fence_abandoned_source(self) -> bool:if self._source_resume_manager is None:return Truereturn await asyncio.to_thread(self._source_resume_manager.revoke_writes, SOURCE_FENCE_TIMEOUT_SECONDS)+ def _open_handoff_gate(self) -> HandoffGate:+ """Build the gate once per run, from the monitor and stopwatch the run has at that point."""+ if self._handoff_gate is not None:+ return self._handoff_gate+ if self._progress is None:+ self._progress = DurableProgress()+ if self._shutdown_stopwatch is None:+ self._shutdown_stopwatch = ShutdownStopwatch(self._shutdown_monitor)+ progress = self._progress+ if self._resumable_source_manager is not None:+ self._resumable_source_manager.on_commit(lambda: progress.mark("resume_commit"))+ self._handoff_gate = HandoffGate(+ shutdown_monitor=self._shutdown_monitor,+ stopwatch=self._shutdown_stopwatch,+ progress=progress,+ # A hand-off retires a laned run's companion jobs, so that run keeps the legacy rule.+ config=None if self._resource.lanes else self._ride_out,+ legacy_wants_handoff=lambda: should_check_shutdown(+ self._schema, self._resource, self._reset_pipeline, self._resumable_source_manager is not None+ ),+ restarts_from_scratch=self._restarts_from_scratch,+ logger=self._logger,+ )+ return self._handoff_gate++ def _restarts_from_scratch(self) -> bool:+ return self._preemption_decision().restarts_from_scratch+def _source_items(self, items: Any, source_type: str) -> AsyncIterator[Any]:if self._preemption is None or self._shutdown_stopwatch is None:return async_iterate(items)@@ -685,6 +733,7 @@ class PipelineV3(Generic[ResumableData]):fence_source=self._fence_abandoned_source,source_type=source_type,logger=self._logger,+ gate=self._open_handoff_gate(),).iterate(items)def _record_handoff_delay(self, error: WorkerShuttingDownError, source_type: str) -> None:@@ -693,7 +742,8 @@ class PipelineV3(Generic[ResumableData]):delay = self._shutdown_stopwatch.elapsed_seconds()if delay is not None:mode = "preempted" if isinstance(error, SourcePreemptedError) else "cooperative"- get_shutdown_handoff_delay_metric(source_type, mode).record(delay)+ reason = (self._progress.handoff_reason if self._progress is not None else None) or "unknown"+ get_shutdown_handoff_delay_metric(source_type, mode, reason).record(delay)async def _commit_resume_state(self) -> None:if self._resumable_source_manager is None:@@ -741,6 +791,9 @@ class PipelineV3(Generic[ResumableData]):)self._staged_handoff_resume_value = resume_valueself._staged_handoff_resume_owner = owner_run_uuid+ # A value inherited from an earlier attempt, or a void one, is not progress of this execution.+ if resume_value is not None and self._queued_own_batch and self._progress is not None:+ self._progress.mark("carry_over_staged")async def _advance_handoff_checkpoint(self, pa_table: pa.Table) -> None:if self._handoff_checkpoint is None:@@ -762,6 +815,7 @@ class PipelineV3(Generic[ResumableData]):)should_resume = resumes_source_cursor or self._continues_incremental_handoffsource_is_resumable = self._resumable_source_manager is not None+ handoff_gate = self._open_handoff_gate()if resumes_source_cursor:await self._logger.ainfo("V3 Pipeline: Resumable source detected - attempting to resume previous import")@@ -966,7 +1020,7 @@ class PipelineV3(Generic[ResumableData]):if wrote_chunk:await self._commit_resume_state()- if should_check_shutdown(self._schema, self._resource, self._reset_pipeline, source_is_resumable):+ if handoff_gate.may_hand_off():if self._shutdown_monitor.is_worker_shutdown() and (self._handoff_checkpoint is not Noneor (source_is_resumable and framework_cursor_covers_buffered_rows(items))@@ -1017,6 +1071,9 @@ class PipelineV3(Generic[ResumableData]):chunk_index += 1await self._finalize(row_count=row_count)+ if activity.in_activity() and self._shutdown_monitor.is_worker_shutdown():+ get_import_finished_during_shutdown_metric(source_type, handoff_gate.rode_out).add(1)+# With zero batches, `_finalize` sent no final-batch notification, so the load# consumer will never hear about this run and cannot finalize it — the workflow must.# See the PipelineResult docstring for the full ownership contract.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 0a11de69837e..2efb16398243 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -32,6 +32,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.pShutdownStopwatch,SourcePreemptedError,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ DurableProgress,+ RideOutConfig,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.abandonable_iterate import (SourceAbandonedError,)@@ -66,6 +70,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.respaged_config,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import shutdown_signalfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (OutputLane,ReplaceScope,@@ -79,6 +84,7 @@ from products.warehouse_sources.backend.temporal.data_imports.workflow_activitiefrom products.warehouse_sources.backend.types import IncrementalFieldType_PIPELINE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline"+_PREEMPTION = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption"_TABLE_REBUILD = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.table_rebuild"_SAFE_POINT = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point"_LANES = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.lanes"@@ -2862,6 +2868,10 @@ class _ShutdownSwitch:def is_worker_shutdown(self) -> bool:return self._flag.is_set()+ def seconds_since_shutdown(self) -> float | None:+ # The stopwatches of these tests are built before the shutdown, so they start on the callback.+ return None+async def wait_for_worker_shutdown(self) -> None:await self.seen_by_event_loop.wait()@@ -2892,6 +2902,24 @@ def _with_preemption(return switch+def _with_ride_out(+ pipeline: PipelineV3, clock: Any, *, max_seconds: float = 3600.0, quiet_period_seconds: float = 0.0+) -> _ShutdownSwitch:+ switch = _with_preemption(pipeline, quiet_period_seconds=quiet_period_seconds)+ pipeline._shutdown_stopwatch = ShutdownStopwatch(cast(Any, switch), clock=clock)+ pipeline._ride_out = RideOutConfig(grace_seconds=7200.0, max_seconds=max_seconds)+ pipeline._progress = DurableProgress()+ return switch+++class _FakeClock:+ def __init__(self) -> None:+ self.now = 0.0++ def __call__(self) -> float:+ return self.now++def _staged_ids(pipeline: PipelineV3) -> list[str]:return [row_id@@ -3076,6 +3104,107 @@ class TestSourcePreemption:assert (decision.eligible, decision.reason) == expected+ @pytest.mark.parametrize(+ "checkpoints,laned,expected_ids,expected_reason",+ [+ pytest.param(False, False, ["a", "b", "c"], "deadline", id="no_progress_rides_out_to_the_deadline"),+ pytest.param(True, False, ["a", "b"], "progress", id="progress_hands_off_at_once"),+ pytest.param(False, True, ["a", "b"], "legacy", id="laned_run_keeps_the_legacy_rule"),+ ],+ )+ @pytest.mark.asyncio+ async def test_a_run_without_durable_progress_rides_out_the_shutdown(+ self, checkpoints: bool, laned: bool, expected_ids: list[str], expected_reason: str+ ) -> None:+ redis = MagicMock()+ manager = _manager()+ clock = _FakeClock()+ wait_ended_early: list[bool] = []++ def items():+ if checkpoints:+ manager.save_state(_Cursor("a"))+ yield pa.table({"id": ["a"]})+ switch.shut_down()+ yield pa.table({"id": ["b"]})+ # A source backoff asks the same signal: during a ride-out it must run its full length.+ is_shutting_down = shutdown_signal()+ assert is_shutting_down is not None+ wait_ended_early.append(is_shutting_down())+ clock.now = 3600.0+ yield pa.table({"id": ["c"]})+ raise RuntimeError("end of the source")++ pipeline = _runnable_pipeline(manager, items)+ if laned:+ pipeline._resource.lanes = [OutputLane(name="users_cdc")]+ # A long quiet period keeps the preemptor out of it: the item loop makes the decision.+ switch = _with_ride_out(pipeline, clock, quiet_period_seconds=7200.0)++ await _run_expecting(pipeline, redis, WorkerShuttingDownError)++ assert _staged_ids(pipeline) == expected_ids+ assert pipeline._progress is not None+ assert pipeline._progress.handoff_reason == expected_reason+ assert pipeline._handoff_gate is not None+ assert pipeline._handoff_gate.rode_out is (expected_reason == "deadline")+ # Leaving at `b` comes before the source asks; a ride-out ran the wait in full.+ assert wait_ended_early == ([False] if expected_reason == "deadline" else [])++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "resumable,quiet_period_seconds,before_deadline,at_deadline",+ [+ # A resumable source stuck in a call: past the quiet period the gate still holds it to the cap.+ pytest.param(True, 300.0, 600.0, 3600.0, id="resumable_held_past_the_quiet_period"),+ # A run that restarts from scratch either way keeps the whole grace period, then leaves+ # instead of dying when it ends, which used to cost a retry attempt.+ pytest.param(False, 300.0, 3600.0, 6300.0, id="from_scratch_keeps_the_whole_grace"),+ ],+ )+ @pytest.mark.asyncio+ async def test_a_source_blocked_in_a_call_is_preempted_at_the_gate_deadline(+ self, resumable: bool, quiet_period_seconds: float, before_deadline: float, at_deadline: float+ ) -> None:+ redis = MagicMock()+ manager = _manager()+ release = threading.Event()+ clock = _FakeClock()++ def items():+ yield pa.table({"id": ["a"]})+ switch.shut_down()+ release.wait()+ yield pa.table({"id": ["b"]})++ pipeline = _runnable_pipeline(manager, items)+ if not resumable:+ pipeline._resumable_source_manager = None+ switch = _with_ride_out(pipeline, clock, quiet_period_seconds=quiet_period_seconds)++ with patch(f"{_PREEMPTION}.RIDE_OUT_WAIT_SLICE_SECONDS", 0.01):+ run = asyncio.ensure_future(_run_expecting(pipeline, redis, SourcePreemptedError))+ try:+ await switch.seen_by_event_loop.wait()+ clock.now = before_deadline+ for _ in range(50):+ await asyncio.sleep(0.01)+ assert not run.done()+ clock.now = at_deadline+ await asyncio.wait_for(run, timeout=10)+ finally:+ release.set()++ assert _staged_ids(pipeline) == ["a"]+ preempted = [+ call.kwargs["preemption_reason"]+ for call in cast(AsyncMock, pipeline._logger.ainfo).await_args_list+ if call.args[0].startswith("Preempting the import")+ ]+ assert preempted == ["deadline"]+ assert pipeline._progress is not None+ assert pipeline._progress.handoff_reason == "preempted_deadline"+@pytest.mark.parametrize("rows_reach_a_batch", [True, False], ids=["rows_staged", "rows_still_buffered"])@pytest.mark.asyncioasync def test_a_preempted_resumable_run_and_its_next_attempt_stage_every_row_once(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.pyindex d8a06148328f..965d20f57859 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py@@ -225,7 +225,7 @@ class TestShutdownDuringARetryWait:manager = MagicMock()manager.has_staged_state.return_value = Falsehandler = PipelineSafePointHandler(- shutdown_monitor=monitor, # type: ignore[arg-type]+ raise_if_should_hand_off=monitor.raise_if_is_worker_shutdown,resumable_source_manager=manager,has_unwritten_rows=lambda: False,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.pyindex 49ad58a89a64..44addcf2c27b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py@@ -70,8 +70,11 @@ class ResumableSourceManager(Generic[ResumableData]):pending: dict[str, str] | None = None,staged: dict[str, str] | None = None,write_fence: _WriteFence | None = None,+ commit_listeners: list[collections.abc.Callable[[], None]] | None = None,):self._inputs = inputs+ # Shared with the siblings: a commit through any namespace is progress of the same run.+ self._commit_listeners = commit_listeners if commit_listeners is not None else []self._data_class = data_classself._logger = inputs.loggerself._namespace = namespace@@ -99,6 +102,7 @@ class ResumableSourceManager(Generic[ResumableData]):pending=self._pending,staged=self._staged,write_fence=self._write_fence,+ commit_listeners=self._commit_listeners,)def revoke_writes(self, timeout_seconds: float) -> bool:@@ -202,10 +206,15 @@ class ResumableSourceManager(Generic[ResumableData]):breakself._staged[key] = json_data+ def on_commit(self, listener: collections.abc.Callable[[], None]) -> None:+ """Call `listener` after each commit that wrote a cursor, from whichever thread commits."""+ self._commit_listeners.append(listener)+def commit(self) -> None:"""Persist every confirmed cursor, across namespaces."""if not self._staged:return+ wrote = Falsewith self._write_fence.writing(), self._get_redis() as redis_client:for key, json_data in list(self._staged.items()):self._logger.debug(f"Saving resumable source state. key={key}")@@ -214,6 +223,10 @@ class ResumableSourceManager(Generic[ResumableData]):functools.partial(redis_client.set, key, json_data, ex=60 * 60 * 24), # 24 hours expiration)del self._staged[key]+ wrote = True+ if wrote:+ for listener in self._commit_listeners:+ listener()@contextmanagerdef committing(self) -> collections.abc.Iterator[None]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex f4f341a15f35..cffca5cbb481 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -28,7 +28,7 @@ from posthog.temporal.common.activity_context import current_activity_attemptfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeaterfrom posthog.temporal.common.logger import get_logger-from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDownError+from posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, ShutdownMonitor, WorkerShuttingDownErrorfrom posthog.temporal.common.utils import is_stale_connection_read_only_errorfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob@@ -64,6 +64,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.etrim_source_job_inputs,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ IMPORT_RIDE_OUT_FLAG,+ DurableProgress,+ RideOutConfig,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (SchemaColumnTypeChangedException,)@@ -83,6 +88,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline)from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTSfrom products.warehouse_sources.backend.temporal.data_imports.row_tracking import setup_row_tracking+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabledfrom products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistryfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (FAST_RETURN_PROBE_TIMEOUT,@@ -390,8 +396,9 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelin# zombie predecessor stands down (its heartbeat timed out, but it may still be running).attempt=current_import_attempt(),):+ progress = DurableProgress(carry_over_counts=settings.DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED)try:- return await _import_data_with_reporting(inputs, logger)+ return await _import_data_with_reporting(inputs, logger, progress)except WorkerShuttingDownError:if not inputs.handoffs_are_free:raise@@ -401,6 +408,8 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelinshould_trigger_cdp_producer=False,handed_off=True,handoff_attempts_used=current_activity_attempt(),+ handoff_made_progress=progress.has_progress,+ handoff_reason=progress.handoff_reason or "unknown",)except (OperationalError, InterfaceError, InternalError, PostHogInternalDatabaseError) as e:# The setup phase (resolving the job/schema/source rows for this run) reads PostHog's@@ -423,7 +432,9 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelinraise NonReportableError(POSTHOG_DATABASE_UNAVAILABLE_MESSAGE) from e-async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger: FilteringBoundLogger) -> PipelineResult:+async def _import_data_with_reporting(+ inputs: ImportDataActivityInputs, logger: FilteringBoundLogger, progress: DurableProgress+) -> PipelineResult:async with Heartbeater(factor=30), ShutdownMonitor() as shutdown_monitor:await setup_row_tracking(inputs.team_id, inputs.schema_id)@@ -584,6 +595,16 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:quiet_period_seconds=settings.DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS,watermark_carry_over_enabled=settings.DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED,)+ # A deadline hand-off on the retry path would cost an attempt, so the same rule as preemption.+ ride_out: RideOutConfig | None = None+ if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(+ schema, IMPORT_RIDE_OUT_FLAG+ ):+ grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS+ ride_out = RideOutConfig(+ grace_seconds=float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds(),+ max_seconds=settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS,+ )if schema.should_use_incremental_field:await logger.adebug(f"Incremental last value being used is: {processed_incremental_last_value}")@@ -779,6 +800,8 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ ride_out=ride_out,+ progress=progress,)else:raise ValueError(f"Source type {model.pipeline.source_type} not supported")@@ -1167,6 +1190,8 @@ async def _run(preemption: PreemptionConfig | None = None,on_resumable_retry_budget: bool = False,kept_rows_synced: int | None = None,+ ride_out: RideOutConfig | None = None,+ progress: DurableProgress | None = None,) -> PipelineResult:try:reset_pipeline = (@@ -1188,6 +1213,8 @@ async def _run(resumed_incremental_value=resumed_incremental_value,preemption=preemption,kept_rows_synced=kept_rows_synced,+ ride_out=ride_out,+ progress=progress,)result = await pipeline.run()diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex eaff04cfbdfa..eae58a94d940 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -38,6 +38,11 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job_transient_error_message,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (+ IMPORT_RIDE_OUT_FLAG,+ DurableProgress,+ RideOutConfig,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core import repartition_controllerfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (SchemaColumnTypeChangedException,@@ -1120,6 +1125,7 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workflmock.patch.object(module, "bind_job_context"),mock.patch.object(module, "trim_source_job_inputs", new=mock.AsyncMock()),mock.patch.object(module, "database_sync_to_async_pool", new=_passthrough),+ mock.patch.object(module, "is_schema_flag_enabled", return_value=False),mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),mock.patch.object(module, "_run", new=mock.AsyncMock(return_value=mock.sentinel.run_result)) as run_mock,@@ -1857,22 +1863,40 @@ async def test_an_interrupted_incremental_import_continues_after_its_last_queued@pytest.mark.asyncio[email protected]("handoffs_are_free", [True, False])-async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(handoffs_are_free: bool):+ "handoffs_are_free,made_progress",+ [(True, False), (True, True), (False, False)],+ ids=["free_without_progress", "free_with_progress", "retry"],+)+async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(+ handoffs_are_free: bool, made_progress: bool+):error = WorkerShuttingDownError("5", "import_data_activity_sync", "data-warehouse-task-queue", 2, "wf", "wt")inputs = dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free, prior_attempts=4)+ async def import_then_hand_off(_inputs, _logger, progress):+ if made_progress:+ progress.mark("resume_commit")+ progress.record_handoff_reason("progress")+ raise error+with (mock.patch.object(module, "tag_queries"),mock.patch.object(module, "report_heartbeat_timeout"),mock.patch.object(module, "aworkload_reporting", return_value=_FakeAsyncCM()),- mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=error)),+ mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=import_then_hand_off)),mock.patch.object(module, "current_activity_attempt", return_value=2),):if handoffs_are_free:result = await import_data_activity_sync(inputs)# The workflow adds the attempts up, so it needs the attempt number of this execution only.- assert result == {"should_trigger_cdp_producer": False, "handed_off": True, "handoff_attempts_used": 2}+ assert result == {+ "should_trigger_cdp_producer": False,+ "handed_off": True,+ "handoff_attempts_used": 2,+ "handoff_made_progress": made_progress,+ "handoff_reason": "progress" if made_progress else "unknown",+ }else:with pytest.raises(WorkerShuttingDownError):await import_data_activity_sync(inputs)@@ -1922,6 +1946,44 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(assert run_mock.await_args.kwargs["preemption"] == expected+ "flag_enabled,handoffs_are_free,grace,expected",+ [+ pytest.param(True, True, 7200, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),+ # The worker falls back to a five-minute graceful shutdown when the setting is unset.+ pytest.param(True, True, None, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),+ # Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.+ pytest.param(True, False, 7200, None, id="off_when_a_handoff_uses_a_retry_attempt"),+ pytest.param(False, True, 7200, None, id="off_when_the_flag_is_off"),+ ],+)+async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(+ flag_enabled: bool,+ handoffs_are_free: bool,+ grace: int | None,+ expected: RideOutConfig | None,+ settings,+):+ settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS = 1800.0+ settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = grace+ source = mock.MagicMock(spec=SimpleSource)+ source.parse_config.return_value = {}+ source.source_for_pipeline.return_value = mock.MagicMock()+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)++ with (+ _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,+ mock.patch.object(module, "is_schema_flag_enabled", return_value=flag_enabled) as flag_check,+ ):+ await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free))++ assert run_mock.await_args.kwargs["ride_out"] == expected+ if handoffs_are_free:+ flag_check.assert_called_once_with(schema, IMPORT_RIDE_OUT_FLAG)+ assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)++@parameterized.expand([("no_checkpoint_past_the_allowance", True, PROGRESSLESS_RESUMABLE_ATTEMPTS + 1, False, True),