Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): ride out the worker shutdown when a hand-off would lose work (#120595)

Daniel RCf4973727a7ab
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 7533754c28e9..3054c2371eff 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -313,6 +313,15 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_ENABLED: bool = get_from_env(
DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env(
"DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS", 300.0, type_cast=float
)
+# At a worker shutdown an import that has not yet committed a resume cursor or staged a carry-over
+# value keeps the worker, up to a deadline below the graceful shutdown timeout. The
+# `warehouse-import-ride-out` flag turns this on per schema. This is the longest such an import holds
+# the worker when it could have handed off sooner; a run that restarts from scratch either way keeps
+# the grace period minus a margin instead. The gate reads GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS, so set the grace through this
+# variable rather than the worker's command-line flag.
+DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS: float = get_from_env(
+ "DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS", 3600.0, type_cast=float
+)
# Restore unfinished append runs only after all extract and load workers have this implementation.
DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED: bool = get_from_env(
"DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED", False, type_cast=str_to_bool
diff --git a/posthog/temporal/common/shutdown.py b/posthog/temporal/common/shutdown.py
index bad0a5ffb5d5..189a03149bc9 100644
--- a/posthog/temporal/common/shutdown.py
+++ b/posthog/temporal/common/shutdown.py
@@ -1,6 +1,8 @@
+import time
import signal
import typing
import asyncio
+import datetime
import threading
import contextvars
from collections.abc import Callable
@@ -8,6 +10,10 @@ from collections.abc import Callable
from structlog import get_logger
from temporalio import activity
+# The SDK cancels activities still running this long after the shutdown started, unless the worker
+# was built with another timeout.
+DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT = datetime.timedelta(minutes=5)
+
LOGGER = get_logger(__name__)
@@ -67,6 +73,7 @@ class ShutdownMonitor:
self._is_shutdown_event = asyncio.Event()
self._is_shutdown_event_sync = threading.Event()
self._stop_event_sync = threading.Event()
+ self._shutdown_at: float | None = None
def __str__(self) -> str:
"""Return a string representation of this `ShutdownMonitor`."""
@@ -108,6 +115,7 @@ class ShutdownMonitor:
# Not running in an activity context.
return
+ self._record_shutdown_time()
self.logger.info("Shutdown detected.")
# Run the callbacks before the event is set: a caller that sees the event can raise and
# leave the monitor at once, and a callback that ran after that would never run at all.
@@ -154,6 +162,7 @@ class ShutdownMonitor:
# it's a wrapper on `threading.Event.wait`, which does return a `bool`
# indicating the reason. So we must also check if the event was set.
if activity.is_worker_shutdown():
+ self._record_shutdown_time()
self.logger.info("Shutdown detected.")
self._is_shutdown_event_sync.set()
break
@@ -204,6 +213,20 @@ class ShutdownMonitor:
"""Check if worker is shutting down."""
return self._is_shutdown_event.is_set() or self._is_shutdown_event_sync.is_set()
+ def _record_shutdown_time(self) -> None:
+ if self._shutdown_at is None:
+ self._shutdown_at = time.monotonic()
+
+ def seconds_since_shutdown(self) -> float | None:
+ """How long ago the worker started to shut down, or None while it still runs.
+
+ A caller created after the shutdown cannot learn the moment from a callback, because the
+ callbacks have already run by then. It reads the elapsed time here instead.
+ """
+ if self._shutdown_at is None:
+ return None
+ return time.monotonic() - self._shutdown_at
+
def raise_if_is_worker_shutdown(self):
"""Raise an exception if worker is shutting down."""
if self.is_worker_shutdown():
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.py
index 22f327d22371..55fdc62def10 100644
--- a/posthog/temporal/common/worker.py
+++ b/posthog/temporal/common/worker.py
@@ -40,6 +40,7 @@ from posthog.temporal.common.interceptor import is_task_queue_supported
from posthog.temporal.common.liveness_tracker import LivenessInterceptor
from posthog.temporal.common.logger import get_write_only_logger
from posthog.temporal.common.posthog_client import PostHogClientInterceptor
+from posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT
from posthog.temporal.common.slo_interceptor import SloInterceptor
from posthog.temporal.common.utils import configure_asyncify_executor, shutdown_asyncify_executor
from posthog.temporal.data_modeling.metrics import (
@@ -547,7 +548,7 @@ async def create_worker(
workflows=workflows,
activities=activities,
workflow_runner=UnsandboxedWorkflowRunner(),
- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),
+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,
interceptors=supported_interceptors,
activity_executor=ThreadPoolExecutor(max_workers=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS),
tuner=WorkerTuner.create_resource_based(
@@ -573,7 +574,7 @@ async def create_worker(
workflows=workflows,
activities=activities,
workflow_runner=UnsandboxedWorkflowRunner(),
- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),
+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,
interceptors=supported_interceptors,
activity_executor=ThreadPoolExecutor(max_workers=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS),
max_concurrent_activities=max_concurrent_activities or DEFAULT_MAX_CONCURRENT_TASKS,
@@ -591,7 +592,7 @@ async def create_worker(
activities=config.activities,
max_concurrent_activities=config.max_concurrent_activities,
activity_executor=ThreadPoolExecutor(max_workers=config.max_concurrent_activities),
- graceful_shutdown_timeout=graceful_shutdown_timeout or dt.timedelta(minutes=5),
+ graceful_shutdown_timeout=graceful_shutdown_timeout or DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT,
interceptors=[
interceptor()
for interceptor in ALL_INTERCEPTOR_CLASSES
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index ecbbfccbbc4b..12c370d6dd5b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -922,7 +922,12 @@ class ExternalDataJobWorkflow(PostHogWorkflow):
delay = min(IMPORT_HANDOFF_FIRST_DELAY * 2 ** min(early_handoffs, 10), IMPORT_HANDOFF_MAX_DELAY)
workflow.logger.info(
"Import handed off by a worker that is shutting down, running it again",
- extra={"handoffs": handoffs, "delay_seconds": delay.total_seconds()},
+ extra={
+ "handoffs": handoffs,
+ "delay_seconds": delay.total_seconds(),
+ "handoff_made_progress": pipeline_result.get("handoff_made_progress"),
+ "handoff_reason": pipeline_result.get("handoff_reason"),
+ },
)
await workflow.sleep(delay)
finally:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
index b3ab9e416f72..59940b8c0654 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
@@ -160,12 +160,26 @@ def get_import_not_preempted_metric(source_type: str | None, reason: str) -> Met
)
-def get_shutdown_handoff_delay_metric(source_type: str | None, mode: str) -> MetricHistogramFloat:
+def get_import_finished_during_shutdown_metric(source_type: str | None, rode_out: bool) -> MetricCounter:
+ # The imports a hand-off would have restarted. `rode_out` is True when the gate kept the import
+ # on the worker past the shutdown at least once.
+ return (
+ activity.metric_meter()
+ .with_additional_attributes({"source_type": source_type or "unknown", "rode_out": str(rode_out).lower()})
+ .create_counter(
+ "warehouse_import_finished_during_shutdown_total",
+ "Imports that finished on a worker that was shutting down instead of handing off.",
+ )
+ )
+
+
+def get_shutdown_handoff_delay_metric(source_type: str | None, mode: str, reason: str) -> MetricHistogramFloat:
# `mode` is `preempted` when the pipeline stopped waiting for the source, and `cooperative`
- # when the source yielded an item or reached a safe point.
+ # when the source yielded an item or reached a safe point. `reason` is why the gate let the
+ # import leave the worker (see `HandoffGate`).
return (
activity.metric_meter()
- .with_additional_attributes({"source_type": source_type or "unknown", "mode": mode})
+ .with_additional_attributes({"source_type": source_type or "unknown", "mode": mode, "reason": reason})
.create_histogram_float(
"warehouse_import_shutdown_handoff_delay_seconds",
"Seconds from the worker shutdown to the hand-off of an import.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py
index c8c3db8df523..7216e551ebcc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/preemption.py
@@ -10,7 +10,7 @@ it gives the source a quiet period to hand off in the usual way, and then it rai
import time
import asyncio
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
from structlog.types import FilteringBoundLogger
from temporalio import activity
@@ -27,8 +27,14 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.aba
PulledItem,
)
+if TYPE_CHECKING:
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import HandoffGate
+
T = TypeVar("T")
+# A ride-out can last an hour; progress that lands meanwhile must end the wait within this time.
+RIDE_OUT_WAIT_SLICE_SECONDS = 30.0
+
@frozen
class PreemptionConfig:
@@ -42,6 +48,9 @@ class PreemptionDecision:
eligible: bool
# A metric label: why the run can continue on another worker, or why it cannot.
reason: str
+ # A run that cannot continue elsewhere and restarts from scratch either way. A hand-off before
+ # the grace period ends then only saves the retry attempt the kill at its end would cost.
+ restarts_from_scratch: bool = False
class SourcePreemptedError(WorkerShuttingDownError):
@@ -59,12 +68,17 @@ class ShutdownStopwatch:
def __init__(self, shutdown_monitor: ShutdownMonitor, clock: Callable[[], float] = time.monotonic) -> None:
self._clock = clock
+ self._shutdown_monitor = shutdown_monitor
self._started_at: float | None = None
shutdown_monitor.run_on_shutdown(self.start)
def start(self) -> None:
- if self._started_at is None:
- self._started_at = self._clock()
+ if self._started_at is not None:
+ return
+ # A stopwatch built after the shutdown - a source setup that ran long - never receives the
+ # monitor's callback, so it backdates its start to the shutdown itself. The grace period
+ # runs from there, and a deadline measured from here instead would outlast it.
+ self._started_at = self._clock() - (self._shutdown_monitor.seconds_since_shutdown() or 0.0)
def elapsed_seconds(self) -> float | None:
return None if self._started_at is None else self._clock() - self._started_at
@@ -81,6 +95,7 @@ class SourcePreemptor:
fence_source: Callable[[], Awaitable[bool]],
source_type: str,
logger: FilteringBoundLogger,
+ gate: "HandoffGate | None" = None,
) -> None:
self._config = config
self._shutdown_monitor = shutdown_monitor
@@ -89,8 +104,13 @@ class SourcePreemptor:
self._fence_source = fence_source
self._source_type = source_type
self._logger = logger
+ # A gate that rides out makes the quiet period at least as long as its deadline, and lets
+ # a run that could not be preempted leave the worker once the deadline passes.
+ self._gate = gate
self._decision: PreemptionDecision | None = None
- self._quiet_period_extension = 0.0
+ # Set after a failed fence: the next attempt to leave waits until this point on the stopwatch.
+ self._fence_retry_at = 0.0
+ self._held_by_ride_out = False
async def iterate(self, items: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]:
"""Yield the items of the source. Raises `SourcePreemptedError` when it preempts the source.
@@ -128,21 +148,33 @@ class SourcePreemptor:
self._stopwatch.start()
decision = await self._decision_at_shutdown()
- if not decision.eligible:
+ if not decision.eligible and not self._preempts_at_deadline(decision):
await asyncio.wait({pull})
return
+ # A run that restarts from scratch gets no quiet period: nothing it could do in that time
+ # changes what the next attempt reads. A failed fence still extends the wait.
+ quiet_period = self._config.quiet_period_seconds if decision.eligible else 0.0
elapsed = self._stopwatch.elapsed_seconds() or 0.0
- remaining = self._config.quiet_period_seconds + self._quiet_period_extension - elapsed
+ quiet_remaining = max(quiet_period, self._fence_retry_at) - elapsed
+ ride_out_remaining = self._gate.seconds_until_deadline() if self._gate is not None else 0.0
+ if ride_out_remaining > quiet_remaining:
+ self._held_by_ride_out = True
+ remaining = max(quiet_remaining, ride_out_remaining)
if remaining > 0:
- # An item or a safe point in this time hands the run off in the usual way.
- await asyncio.wait({pull}, timeout=remaining)
+ # An item or a safe point in this time hands the run off in the usual way. Short
+ # slices, so progress that lands during a ride-out ends the wait soon after.
+ await asyncio.wait({pull}, timeout=min(remaining, RIDE_OUT_WAIT_SLICE_SECONDS))
continue
+ if self._gate is not None and (self._held_by_ride_out or not decision.eligible):
+ preemption_reason = self._gate.leave_reason()
+ else:
+ preemption_reason = decision.reason
if not await self._fence_source():
# The source is in the middle of a write of its resume state. A hand-off now could
# let that write land after the next attempt starts.
- self._quiet_period_extension += self._config.quiet_period_seconds
+ self._fence_retry_at = elapsed + self._config.quiet_period_seconds
await self._logger.awarning(
"Could not stop the writes of the source, waiting another quiet period before preemption",
quiet_period_seconds=self._config.quiet_period_seconds,
@@ -151,16 +183,22 @@ class SourcePreemptor:
# The source can no longer store resume state, so the run must leave this worker now,
# also when an item arrived during the fence. The next attempt reads that item again.
+ if self._gate is not None:
+ self._gate.record_preemption(preemption_reason)
if activity.in_activity():
- get_import_preempted_metric(self._source_type, decision.reason).add(1)
+ get_import_preempted_metric(self._source_type, preemption_reason).add(1)
await self._logger.ainfo(
"Preempting the import because the source did not hand off in the quiet period",
- preemption_reason=decision.reason,
+ preemption_reason=preemption_reason,
quiet_period_seconds=self._config.quiet_period_seconds,
seconds_since_shutdown=round(self._stopwatch.elapsed_seconds() or 0.0, 1),
)
raise SourcePreemptedError.build()
+ def _preempts_at_deadline(self, decision: PreemptionDecision) -> bool:
+ """Whether a run that cannot continue elsewhere still leaves the worker before the grace ends."""
+ return self._gate is not None and self._gate.ride_out_enabled and decision.restarts_from_scratch
+
async def _decision_at_shutdown(self) -> PreemptionDecision:
# Decided once, at the shutdown. A run that was young then stays eligible for the whole
# quiet period.
@@ -172,6 +210,11 @@ class SourcePreemptor:
preemption_reason=self._decision.reason,
quiet_period_seconds=self._config.quiet_period_seconds,
)
+ elif self._preempts_at_deadline(self._decision):
+ await self._logger.ainfo(
+ "Worker is shutting down, the source keeps the worker until the ride-out deadline",
+ not_preempted_reason=self._decision.reason,
+ )
else:
if activity.in_activity():
get_import_not_preempted_metric(self._source_type, self._decision.reason).add(1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
new file mode 100644
index 000000000000..f103eb8386db
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
@@ -0,0 +1,215 @@
+"""Ride-out: an import that has nothing durable yet keeps its worker through the shutdown grace.
+
+`HandoffGate` is the one place that decides whether an import leaves a shutting-down worker now.
+Two invariants hold it up: the activity's `Heartbeater` runs on the event loop while the source runs
+on its own thread, so an import that rides out keeps heartbeating and never counts as a zombie; and
+the deadline stays below the SDK's graceful shutdown timeout, which cancels activities, and below the
+pod's `terminationGracePeriodSeconds`.
+"""
+
+import threading
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from structlog.types import FilteringBoundLogger
+from temporalio import activity
+
+from posthog.dataclasses import frozen
+from posthog.temporal.common.shutdown import ShutdownMonitor
+
+if TYPE_CHECKING:
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import ShutdownStopwatch
+
+# Covers the gap to the next item or safe point, the result flush, and the SDK's cancellation of
+# activities once the graceful shutdown timeout passes. A grace period shorter than this gives a
+# deadline of zero, so the import hands off at once.
+# Per-schema rollout flag. Off, every hand-off follows the legacy rule.
+IMPORT_RIDE_OUT_FLAG = "warehouse-import-ride-out"
+RIDE_OUT_GRACE_MARGIN_SECONDS = 900.0
+# An execution younger than this restarts for nearly free, so it leaves the worker at once and the
+# pod keeps draining fast. Shorter than `NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE` on purpose: that one
+# decides whether a run may hand off at all, this one only whether it is worth holding the worker.
+RIDE_OUT_MIN_EXECUTION_AGE_SECONDS = 120.0
+
+
+@frozen
+class RideOutConfig:
+ grace_seconds: float
+ # How long a run that could hand off sooner may hold the worker; the grace minus the margin bounds
+ # it too.
+ max_seconds: float
+
+ def deadline_seconds_after_shutdown(self, *, restarts_from_scratch: bool) -> float:
+ before_grace_ends = max(self.grace_seconds - RIDE_OUT_GRACE_MARGIN_SECONDS, 0.0)
+ if restarts_from_scratch:
+ return before_grace_ends
+ return min(before_grace_ends, self.max_seconds)
+
+
+class DurableProgress:
+ """What this execution has made durable, shared by the event loop and the source thread."""
+
+ def __init__(self, *, carry_over_counts: bool = True) -> None:
+ self._lock = threading.Lock()
+ self._reasons: list[str] = []
+ # A staged carry-over value is progress only when the next execution reads it.
+ self._carry_over_counts = carry_over_counts
+ self._handoff_reason: str | None = None
+
+ def mark(self, reason: str) -> None:
+ if reason == "carry_over_staged" and not self._carry_over_counts:
+ return
+ with self._lock:
+ if reason not in self._reasons:
+ self._reasons.append(reason)
+
+ @property
+ def has_progress(self) -> bool:
+ with self._lock:
+ return bool(self._reasons)
+
+ @property
+ def reasons(self) -> tuple[str, ...]:
+ with self._lock:
+ return tuple(self._reasons)
+
+ @property
+ def handoff_reason(self) -> str | None:
+ return self._handoff_reason
+
+ def record_handoff_reason(self, reason: str) -> None:
+ """Keep the first reason: an abandoned source thread can still ask the gate after a preemption."""
+ with self._lock:
+ if self._handoff_reason is None:
+ self._handoff_reason = reason
+
+
+def activity_execution_age_seconds() -> float | None:
+ if not activity.in_activity():
+ return None
+ return (datetime.now(UTC) - activity.info().started_time).total_seconds()
+
+
+class HandoffGate:
+ """Decides whether the import leaves a shutting-down worker now.
+
+ Every observer of the shutdown asks the gate: the item loop, the safe point handler on the
+ source thread, the source's own waits, and the preemptor. With `config` None the gate is today's
+ behaviour, which `legacy_wants_handoff` provides.
+ """
+
+ def __init__(
+ self,
+ *,
+ shutdown_monitor: ShutdownMonitor,
+ stopwatch: "ShutdownStopwatch",
+ progress: DurableProgress,
+ config: RideOutConfig | None,
+ legacy_wants_handoff: Callable[[], bool],
+ logger: FilteringBoundLogger,
+ restarts_from_scratch: Callable[[], bool] = lambda: False,
+ execution_age_seconds: Callable[[], float | None] = activity_execution_age_seconds,
+ ) -> None:
+ self._shutdown_monitor = shutdown_monitor
+ self._stopwatch = stopwatch
+ self._progress = progress
+ self._config = config
+ self._legacy_wants_handoff = legacy_wants_handoff
+ self._logger = logger
+ self._restarts_from_scratch = restarts_from_scratch
+ self._execution_age_seconds = execution_age_seconds
+ self._lock = threading.Lock()
+ self._logged_handoff = False
+ self.rode_out = False
+
+ @property
+ def ride_out_enabled(self) -> bool:
+ return self._config is not None
+
+ def should_hand_off_now(self) -> bool:
+ if not self._shutdown_monitor.is_worker_shutdown():
+ return False
+ # The stopwatch's own callback runs on the event loop; the source thread can see the
+ # shutdown before it.
+ self._stopwatch.start()
+ if self._config is None:
+ return self._decide("legacy", self._legacy_wants_handoff())
+ early_reason = self._reason_to_leave_before_the_deadline()
+ if early_reason is not None:
+ return self._decide(early_reason, True)
+ return self._decide("deadline", self.seconds_until_deadline() <= 0.0)
+
+ def may_hand_off(self) -> bool:
+ """Whether the run may leave the worker here, with the shutdown left to the monitor.
+
+ The item loop and the safe point ask this and then let
+ `ShutdownMonitor.raise_if_is_worker_shutdown` have the last word, as the pipeline always
+ has. With the ride-out off that is the legacy decision alone: reading the shutdown flag
+ here first would be the same in production and a different thing to every caller that
+ drives the shutdown through the monitor.
+ """
+ if self._config is not None:
+ return self.should_hand_off_now()
+ if not self._legacy_wants_handoff():
+ return False
+ # The stopwatch for the reason, not the monitor: it is set once the shutdown callback has
+ # run, and reading it costs the monitor nothing.
+ if self._stopwatch.elapsed_seconds() is not None:
+ self._progress.record_handoff_reason("legacy")
+ return True
+
+ def seconds_until_deadline(self) -> float:
+ """How long the preemptor must still wait for the source before it hands off. Zero when the
+ gate would hand off at once."""
+ if self._config is None or self._reason_to_leave_before_the_deadline() is not None:
+ return 0.0
+ elapsed = self._stopwatch.elapsed_seconds() or 0.0
+ deadline = self._config.deadline_seconds_after_shutdown(restarts_from_scratch=self._restarts_from_scratch())
+ return max(deadline - elapsed, 0.0)
+
+ def leave_reason(self) -> str:
+ """Why the gate lets the import leave now: `progress`, `young`, or the `deadline`."""
+ return self._reason_to_leave_before_the_deadline() or "deadline"
+
+ def record_preemption(self, reason: str) -> None:
+ self._progress.record_handoff_reason(f"preempted_{reason}")
+
+ def raise_if_should_hand_off(self) -> None:
+ if self.may_hand_off():
+ self._shutdown_monitor.raise_if_is_worker_shutdown()
+
+ def _reason_to_leave_before_the_deadline(self) -> str | None:
+ if self._progress.has_progress:
+ return "progress"
+ age = self._execution_age_seconds()
+ if age is not None and age < RIDE_OUT_MIN_EXECUTION_AGE_SECONDS:
+ return "young"
+ return None
+
+ def _decide(self, reason: str, hand_off: bool) -> bool:
+ if not hand_off:
+ if self._config is None:
+ return False
+ with self._lock:
+ first_time = not self.rode_out
+ self.rode_out = True
+ if first_time:
+ self._logger.info(
+ "Worker is shutting down and the import has no durable progress yet, riding out the grace period",
+ deadline_seconds=round(self.seconds_until_deadline(), 1),
+ execution_age_seconds=round(self._execution_age_seconds() or 0.0, 1),
+ )
+ return False
+ self._progress.record_handoff_reason(reason)
+ with self._lock:
+ log_it = self.rode_out and not self._logged_handoff
+ self._logged_handoff = True
+ if log_it:
+ self._logger.info(
+ "Handing the import off after riding out the shutdown",
+ handoff_reason=reason,
+ seconds_since_shutdown=round(self._stopwatch.elapsed_seconds() or 0.0, 1),
+ progress=list(self._progress.reasons),
+ )
+ return True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
index e377b02b2759..c12df03f191a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
@@ -2,8 +2,6 @@ import time
from collections.abc import Callable
from typing import Any
-from posthog.temporal.common.shutdown import ShutdownMonitor
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -22,13 +20,13 @@ class PipelineSafePointHandler:
def __init__(
self,
*,
- shutdown_monitor: ShutdownMonitor,
+ raise_if_should_hand_off: Callable[[], None],
resumable_source_manager: ResumableSourceManager[Any],
has_unwritten_rows: Callable[[], bool],
commit_interval_seconds: float | None = None,
clock: Callable[[], float] = time.monotonic,
):
- self._shutdown_monitor = shutdown_monitor
+ self._raise_if_should_hand_off = raise_if_should_hand_off
self._resumable_source_manager = resumable_source_manager
self._has_unwritten_rows = has_unwritten_rows
self._commit_interval_seconds = (
@@ -42,7 +40,7 @@ class PipelineSafePointHandler:
self._resumable_source_manager.confirm()
# The pipeline's `except` path writes the rows it holds and then commits the confirmed cursor,
# so a raise here hands the run to another worker without losing progress.
- self._shutdown_monitor.raise_if_is_worker_shutdown()
+ self._raise_if_should_hand_off()
now = self._clock()
if now - self._last_commit < self._commit_interval_seconds:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.py
new file mode 100644
index 000000000000..cd5305fdb76b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_ride_out.py
@@ -0,0 +1,226 @@
+from typing import Any, cast
+
+import pytest
+from unittest.mock import MagicMock
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import ShutdownStopwatch
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ DurableProgress,
+ HandoffGate,
+ RideOutConfig,
+)
+
+
+class _Clock:
+ def __init__(self) -> None:
+ self.now = 0.0
+
+ def __call__(self) -> float:
+ return self.now
+
+
+class _Monitor:
+ def __init__(self, shutting_down: bool, shutdown_age: float | None = None) -> None:
+ self.shutting_down = shutting_down
+ self.shutdown_age = shutdown_age
+
+ def run_on_shutdown(self, callback: Any) -> None:
+ pass
+
+ def is_worker_shutdown(self) -> bool:
+ return self.shutting_down
+
+ def seconds_since_shutdown(self) -> float | None:
+ return self.shutdown_age
+
+ def raise_if_is_worker_shutdown(self) -> None:
+ if self.shutting_down:
+ raise RuntimeError("worker shutting down")
+
+
+def _gate(
+ *,
+ shutting_down: bool = True,
+ ride_out: bool = True,
+ progress: bool = False,
+ age: float | None = 600.0,
+ legacy: bool = True,
+ restarts_from_scratch: bool = False,
+ shutdown_age: float | None = None,
+) -> tuple[HandoffGate, _Clock, DurableProgress]:
+ clock = _Clock()
+ monitor = _Monitor(shutting_down, shutdown_age)
+ durable = DurableProgress()
+ if progress:
+ durable.mark("resume_commit")
+ gate = HandoffGate(
+ shutdown_monitor=cast(Any, monitor),
+ stopwatch=ShutdownStopwatch(cast(Any, monitor), clock=clock),
+ progress=durable,
+ config=RideOutConfig(grace_seconds=7200.0, max_seconds=3600.0) if ride_out else None,
+ legacy_wants_handoff=lambda: legacy,
+ logger=MagicMock(),
+ restarts_from_scratch=lambda: restarts_from_scratch,
+ execution_age_seconds=lambda: age,
+ )
+ return gate, clock, durable
+
+
+ "kwargs,elapsed,expected",
+ [
+ pytest.param({"shutting_down": False}, 0.0, (False, None), id="worker_running"),
+ pytest.param({"ride_out": False, "legacy": True}, 0.0, (True, "legacy"), id="kill_switch_off_legacy_hands_off"),
+ pytest.param({"ride_out": False, "legacy": False}, 7200.0, (False, None), id="kill_switch_off_legacy_waits"),
+ pytest.param({"progress": True}, 0.0, (True, "progress"), id="durable_progress_hands_off_at_once"),
+ pytest.param({"age": 10.0}, 0.0, (True, "young"), id="young_execution_hands_off_at_once"),
+ pytest.param({}, 3599.0, (False, None), id="no_progress_rides_out_before_the_deadline"),
+ pytest.param({}, 3600.0, (True, "deadline"), id="no_progress_hands_off_at_the_deadline"),
+ pytest.param({"age": None}, 0.0, (False, None), id="unknown_age_rides_out"),
+ # A run that restarts from scratch either way keeps the whole grace period, not the ride-out cap.
+ pytest.param({"restarts_from_scratch": True}, 6299.0, (False, None), id="from_scratch_waits_past_the_cap"),
+ pytest.param(
+ {"restarts_from_scratch": True}, 6300.0, (True, "deadline"), id="from_scratch_leaves_before_grace_ends"
+ ),
+ ],
+)
+def test_the_gate_hands_off_only_when_a_restart_loses_nothing(
+ kwargs: dict[str, Any], elapsed: float, expected: tuple[bool, str | None]
+) -> None:
+ gate, clock, progress = _gate(**kwargs)
+
+ gate.should_hand_off_now()
+ clock.now = elapsed
+ hands_off = gate.should_hand_off_now()
+
+ assert (hands_off, progress.handoff_reason) == expected
+
+
+ "kwargs,expected",
+ [
+ pytest.param({}, 3500.0, id="no_progress"),
+ pytest.param({"progress": True}, 0.0, id="progress"),
+ pytest.param({"age": 10.0}, 0.0, id="young"),
+ pytest.param({"restarts_from_scratch": True}, 6200.0, id="from_scratch"),
+ ],
+)
+def test_the_preemptor_waits_exactly_as_long_as_the_gate_would(kwargs: dict[str, Any], expected: float) -> None:
+ gate, clock, _ = _gate(**kwargs)
+ gate.should_hand_off_now()
+ clock.now = 100.0
+
+ assert gate.seconds_until_deadline() == expected
+
+
+def test_a_gate_opened_after_the_shutdown_counts_its_deadline_from_the_shutdown() -> None:
+ # A source setup that runs past the shutdown builds the pipeline, and so the stopwatch, after
+ # the monitor's callbacks already ran. The deadline must still land inside the grace period.
+ gate, clock, _ = _gate(restarts_from_scratch=True, shutdown_age=1200.0)
+
+ # The first question starts the stopwatch, backdated to the shutdown 1200 s ago.
+ assert gate.should_hand_off_now() is False
+
+ clock.now = 6300.0 - 1200.0 - 1.0
+ assert gate.should_hand_off_now() is False
+
+ clock.now = 6300.0 - 1200.0
+ assert gate.should_hand_off_now() is True
+
+
+def test_a_staged_carry_over_value_counts_as_progress_only_when_the_next_execution_reads_it() -> None:
+ counted, ignored = DurableProgress(carry_over_counts=True), DurableProgress(carry_over_counts=False)
+ counted.mark("carry_over_staged")
+ ignored.mark("carry_over_staged")
+ assert (counted.has_progress, ignored.has_progress) == (True, False)
+
+
+def test_the_first_handoff_reason_sticks() -> None:
+ # An abandoned source thread can still ask the gate after a preemption; its answer must not
+ # overwrite why the run really left.
+ gate, clock, progress = _gate()
+ gate.should_hand_off_now()
+ gate.record_preemption("deadline")
+ clock.now = 7200.0
+
+ assert gate.should_hand_off_now() is True
+ assert progress.handoff_reason == "preempted_deadline"
+
+
+def test_progress_that_lands_during_a_ride_out_ends_it() -> None:
+ gate, clock, progress = _gate()
+ assert gate.should_hand_off_now() is False
+ clock.now = 100.0
+
+ progress.mark("carry_over_staged")
+
+ assert gate.should_hand_off_now() is True
+ assert gate.seconds_until_deadline() == 0.0
+ assert progress.handoff_reason == "progress"
+ assert gate.rode_out is True
+
+
+ "kwargs,expected",
+ [
+ # With the ride-out off the monitor still owns the shutdown flag, so the hand-off point is
+ # reached whether or not the worker is shutting down yet.
+ pytest.param({"ride_out": False, "shutting_down": False}, True, id="kill_switch_off_asks_the_monitor"),
+ pytest.param(
+ {"ride_out": False, "shutting_down": False, "legacy": False}, False, id="kill_switch_off_run_waits"
+ ),
+ pytest.param({"shutting_down": False}, False, id="ride_out_on_reads_the_flag"),
+ pytest.param({"progress": True}, True, id="ride_out_on_hands_off_with_progress"),
+ ],
+)
+def test_the_handoff_point_leaves_the_shutdown_flag_to_the_monitor(kwargs: dict[str, Any], expected: bool) -> None:
+ gate, _, _ = _gate(**kwargs)
+
+ assert gate.may_hand_off() is expected
+
+
+def test_the_legacy_handoff_check_asks_the_monitor_to_raise() -> None:
+ # Today's behaviour, which the kill switch keeps: the check asks the monitor to raise instead of
+ # deciding on its flag first, so a monitor whose raise is what reports the shutdown still works.
+ monitor = MagicMock()
+ monitor.is_worker_shutdown.return_value = False
+ monitor.raise_if_is_worker_shutdown.side_effect = RuntimeError("worker shutting down")
+ gate = HandoffGate(
+ shutdown_monitor=cast(Any, monitor),
+ stopwatch=ShutdownStopwatch(cast(Any, monitor), clock=_Clock()),
+ progress=DurableProgress(),
+ config=None,
+ legacy_wants_handoff=lambda: True,
+ logger=MagicMock(),
+ )
+
+ with pytest.raises(RuntimeError, match="worker shutting down"):
+ gate.raise_if_should_hand_off()
+
+ # Not even to decide: a caller that counts the checks must see the same calls it always did.
+ assert monitor.is_worker_shutdown.call_count == 0
+
+
+def test_raise_if_should_hand_off_raises_through_the_monitor_only_at_a_hand_off() -> None:
+ gate, clock, _ = _gate()
+ gate.raise_if_should_hand_off()
+
+ clock.now = 3600.0
+ with pytest.raises(RuntimeError, match="worker shutting down"):
+ gate.raise_if_should_hand_off()
+
+
+ "grace,max_seconds,restarts_from_scratch,expected",
+ [
+ pytest.param(7200.0, 3600.0, False, 3600.0, id="max_below_grace"),
+ pytest.param(7200.0, 3600.0, True, 6300.0, id="from_scratch_ignores_max"),
+ pytest.param(1500.0, 3600.0, False, 600.0, id="grace_minus_margin_below_max"),
+ pytest.param(300.0, 3600.0, False, 0.0, id="grace_shorter_than_the_margin"),
+ ],
+)
+def test_the_deadline_stays_below_the_grace_period(
+ grace: float, max_seconds: float, restarts_from_scratch: bool, expected: float
+) -> None:
+ config = RideOutConfig(grace_seconds=grace, max_seconds=max_seconds)
+ assert config.deadline_seconds_after_shutdown(restarts_from_scratch=restarts_from_scratch) == expected
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py
index afd96a6e6c1a..343fc4233313 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_safe_point.py
@@ -40,7 +40,7 @@ def test_handler_commits_only_a_cursor_whose_rows_are_written(
manager = MagicMock()
manager.has_staged_state.return_value = staged
handler = PipelineSafePointHandler(
- shutdown_monitor=MagicMock(),
+ raise_if_should_hand_off=lambda: None,
resumable_source_manager=manager,
has_unwritten_rows=lambda: unwritten_rows,
commit_interval_seconds=30.0,
@@ -54,11 +54,9 @@ def test_handler_commits_only_a_cursor_whose_rows_are_written(
def test_handler_confirms_the_cursor_and_raises_the_shutdown_before_it_commits() -> None:
- monitor = MagicMock()
- monitor.raise_if_is_worker_shutdown.side_effect = RuntimeError("worker shutting down")
manager = MagicMock()
handler = PipelineSafePointHandler(
- shutdown_monitor=monitor,
+ raise_if_should_hand_off=MagicMock(side_effect=RuntimeError("worker shutting down")),
resumable_source_manager=manager,
has_unwritten_rows=lambda: False,
commit_interval_seconds=0.0,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py
index d95fc7aa8bcd..6d7b722593dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/typings.py
@@ -39,3 +39,7 @@ class PipelineResult(TypedDict):
fast_returned: NotRequired[bool]
handed_off: NotRequired[bool]
handoff_attempts_used: NotRequired[int]
+ handoff_made_progress: NotRequired[bool]
+ # Why the execution left the worker: `progress`, `young`, `deadline`, `legacy`, `preempted_<reason>`
+ # or `unknown`.
+ handoff_reason: NotRequired[str]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
index adddbb34ea5e..3279780a3882 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs
retire_companion_job,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ DurableProgress,
+ RideOutConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.typings import PipelineResult
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline import PipelineV3
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.producer import (
@@ -95,6 +99,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
resumed_incremental_value: Any = None,
preemption: PreemptionConfig | None = None,
kept_rows_synced: int | None = None,
+ ride_out: RideOutConfig | None = None,
+ progress: DurableProgress | None = None,
) -> None:
if not source_response.lanes:
raise ValueError(f"{source_response.name} declares no lanes; run it on PipelineV3")
@@ -114,6 +120,8 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ ride_out=ride_out,
+ progress=progress,
)
# The base built the first lane; it shares the base's batch list so the two never disagree.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index f41eb8de45a9..e50877be504d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -34,7 +34,10 @@ from products.warehouse_sources.backend.temporal.data_imports.import_attempt imp
current_import_attempt,
current_import_attempt_cause,
)
-from products.warehouse_sources.backend.temporal.data_imports.metrics import get_shutdown_handoff_delay_metric
+from products.warehouse_sources.backend.temporal.data_imports.metrics import (
+ get_import_finished_during_shutdown_metric,
+ get_shutdown_handoff_delay_metric,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import (
cleanup_memory,
commit_source_cursor,
@@ -59,6 +62,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.p
SourcePreemptedError,
SourcePreemptor,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ DurableProgress,
+ HandoffGate,
+ RideOutConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (
PipelineSafePointHandler,
framework_cursor_covers_buffered_rows,
@@ -188,6 +196,10 @@ class PipelineV3(Generic[ResumableData]):
# None when the pipeline waits for the source for as long as the source takes.
_preemption: PreemptionConfig | None = None
_shutdown_stopwatch: ShutdownStopwatch | None = None
+ # None when an import without durable progress hands off at once, as it always did.
+ _ride_out: RideOutConfig | None = None
+ _progress: DurableProgress | None = None
+ _handoff_gate: HandoffGate | None = None
_resumed_incremental_run_uuid: str | None = None
_sent_resumed_run_finalization: bool = False
_writes_staged_repartition_scheme: bool = False
@@ -214,9 +226,13 @@ class PipelineV3(Generic[ResumableData]):
resumed_incremental_value: Any = None,
preemption: PreemptionConfig | None = None,
kept_rows_synced: int | None = None,
+ ride_out: RideOutConfig | None = None,
+ progress: DurableProgress | None = None,
) -> None:
self._resource = source_response
self._kept_rows_synced = kept_rows_synced
+ self._ride_out = ride_out
+ self._progress = progress
self._source_cursor_manager = source_cursor_manager
self._resource_name = source_response.name
@@ -632,17 +648,19 @@ class PipelineV3(Generic[ResumableData]):
scope = contextlib.ExitStack()
if self._resumable_source_manager is not None:
handler = PipelineSafePointHandler(
- shutdown_monitor=self._shutdown_monitor,
+ raise_if_should_hand_off=self._open_handoff_gate().raise_if_should_hand_off,
resumable_source_manager=self._resumable_source_manager,
has_unwritten_rows=lambda: (
self._batcher.should_yield(include_incomplete_chunk=True) or self._pg_producer.has_held_batch
),
)
+ # The gate, not the raw signal: a source wait that ended early during a ride-out would
+ # skip every backoff and Retry-After for as long as the gate keeps the run here.
scope.enter_context(
activate_safe_point(
handler,
covers_framework_checkpoints=source_items_are_framework_output(items),
- is_shutting_down=self._shutdown_monitor.is_worker_shutdown,
+ is_shutting_down=self._open_handoff_gate().should_hand_off_now,
)
)
return scope
@@ -664,16 +682,46 @@ class PipelineV3(Generic[ResumableData]):
if is_young_first_attempt():
return PreemptionDecision(eligible=True, reason="young_first_attempt")
if checkpoint is not None and not checkpoint.is_void:
- return PreemptionDecision(eligible=False, reason="watermark_carry_over_disabled")
+ return PreemptionDecision(
+ eligible=False, reason="watermark_carry_over_disabled", restarts_from_scratch=True
+ )
if self._schema.should_use_incremental_field:
- return PreemptionDecision(eligible=False, reason="no_watermark_carry_over")
- return PreemptionDecision(eligible=False, reason="non_resumable_full_refresh")
+ return PreemptionDecision(eligible=False, reason="no_watermark_carry_over", restarts_from_scratch=True)
+ return PreemptionDecision(eligible=False, reason="non_resumable_full_refresh", restarts_from_scratch=True)
async def _fence_abandoned_source(self) -> bool:
if self._source_resume_manager is None:
return True
return await asyncio.to_thread(self._source_resume_manager.revoke_writes, SOURCE_FENCE_TIMEOUT_SECONDS)
+ def _open_handoff_gate(self) -> HandoffGate:
+ """Build the gate once per run, from the monitor and stopwatch the run has at that point."""
+ if self._handoff_gate is not None:
+ return self._handoff_gate
+ if self._progress is None:
+ self._progress = DurableProgress()
+ if self._shutdown_stopwatch is None:
+ self._shutdown_stopwatch = ShutdownStopwatch(self._shutdown_monitor)
+ progress = self._progress
+ if self._resumable_source_manager is not None:
+ self._resumable_source_manager.on_commit(lambda: progress.mark("resume_commit"))
+ self._handoff_gate = HandoffGate(
+ shutdown_monitor=self._shutdown_monitor,
+ stopwatch=self._shutdown_stopwatch,
+ progress=progress,
+ # A hand-off retires a laned run's companion jobs, so that run keeps the legacy rule.
+ config=None if self._resource.lanes else self._ride_out,
+ legacy_wants_handoff=lambda: should_check_shutdown(
+ self._schema, self._resource, self._reset_pipeline, self._resumable_source_manager is not None
+ ),
+ restarts_from_scratch=self._restarts_from_scratch,
+ logger=self._logger,
+ )
+ return self._handoff_gate
+
+ def _restarts_from_scratch(self) -> bool:
+ return self._preemption_decision().restarts_from_scratch
+
def _source_items(self, items: Any, source_type: str) -> AsyncIterator[Any]:
if self._preemption is None or self._shutdown_stopwatch is None:
return async_iterate(items)
@@ -685,6 +733,7 @@ class PipelineV3(Generic[ResumableData]):
fence_source=self._fence_abandoned_source,
source_type=source_type,
logger=self._logger,
+ gate=self._open_handoff_gate(),
).iterate(items)
def _record_handoff_delay(self, error: WorkerShuttingDownError, source_type: str) -> None:
@@ -693,7 +742,8 @@ class PipelineV3(Generic[ResumableData]):
delay = self._shutdown_stopwatch.elapsed_seconds()
if delay is not None:
mode = "preempted" if isinstance(error, SourcePreemptedError) else "cooperative"
- get_shutdown_handoff_delay_metric(source_type, mode).record(delay)
+ reason = (self._progress.handoff_reason if self._progress is not None else None) or "unknown"
+ get_shutdown_handoff_delay_metric(source_type, mode, reason).record(delay)
async def _commit_resume_state(self) -> None:
if self._resumable_source_manager is None:
@@ -741,6 +791,9 @@ class PipelineV3(Generic[ResumableData]):
)
self._staged_handoff_resume_value = resume_value
self._staged_handoff_resume_owner = owner_run_uuid
+ # A value inherited from an earlier attempt, or a void one, is not progress of this execution.
+ if resume_value is not None and self._queued_own_batch and self._progress is not None:
+ self._progress.mark("carry_over_staged")
async def _advance_handoff_checkpoint(self, pa_table: pa.Table) -> None:
if self._handoff_checkpoint is None:
@@ -762,6 +815,7 @@ class PipelineV3(Generic[ResumableData]):
)
should_resume = resumes_source_cursor or self._continues_incremental_handoff
source_is_resumable = self._resumable_source_manager is not None
+ handoff_gate = self._open_handoff_gate()
if resumes_source_cursor:
await self._logger.ainfo("V3 Pipeline: Resumable source detected - attempting to resume previous import")
@@ -966,7 +1020,7 @@ class PipelineV3(Generic[ResumableData]):
if wrote_chunk:
await self._commit_resume_state()
- if should_check_shutdown(self._schema, self._resource, self._reset_pipeline, source_is_resumable):
+ if handoff_gate.may_hand_off():
if self._shutdown_monitor.is_worker_shutdown() and (
self._handoff_checkpoint is not None
or (source_is_resumable and framework_cursor_covers_buffered_rows(items))
@@ -1017,6 +1071,9 @@ class PipelineV3(Generic[ResumableData]):
chunk_index += 1
await self._finalize(row_count=row_count)
+ if activity.in_activity() and self._shutdown_monitor.is_worker_shutdown():
+ get_import_finished_during_shutdown_metric(source_type, handoff_gate.rode_out).add(1)
+
# With zero batches, `_finalize` sent no final-batch notification, so the load
# consumer will never hear about this run and cannot finalize it — the workflow must.
# See the PipelineResult docstring for the full ownership contract.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 0a11de69837e..2efb16398243 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -32,6 +32,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.p
ShutdownStopwatch,
SourcePreemptedError,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ DurableProgress,
+ RideOutConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.abandonable_iterate import (
SourceAbandonedError,
)
@@ -66,6 +70,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
paged_config,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import shutdown_signal
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (
OutputLane,
ReplaceScope,
@@ -79,6 +84,7 @@ from products.warehouse_sources.backend.temporal.data_imports.workflow_activitie
from products.warehouse_sources.backend.types import IncrementalFieldType
_PIPELINE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline"
+_PREEMPTION = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption"
_TABLE_REBUILD = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.table_rebuild"
_SAFE_POINT = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point"
_LANES = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.lanes"
@@ -2862,6 +2868,10 @@ class _ShutdownSwitch:
def is_worker_shutdown(self) -> bool:
return self._flag.is_set()
+ def seconds_since_shutdown(self) -> float | None:
+ # The stopwatches of these tests are built before the shutdown, so they start on the callback.
+ return None
+
async def wait_for_worker_shutdown(self) -> None:
await self.seen_by_event_loop.wait()
@@ -2892,6 +2902,24 @@ def _with_preemption(
return switch
+def _with_ride_out(
+ pipeline: PipelineV3, clock: Any, *, max_seconds: float = 3600.0, quiet_period_seconds: float = 0.0
+) -> _ShutdownSwitch:
+ switch = _with_preemption(pipeline, quiet_period_seconds=quiet_period_seconds)
+ pipeline._shutdown_stopwatch = ShutdownStopwatch(cast(Any, switch), clock=clock)
+ pipeline._ride_out = RideOutConfig(grace_seconds=7200.0, max_seconds=max_seconds)
+ pipeline._progress = DurableProgress()
+ return switch
+
+
+class _FakeClock:
+ def __init__(self) -> None:
+ self.now = 0.0
+
+ def __call__(self) -> float:
+ return self.now
+
+
def _staged_ids(pipeline: PipelineV3) -> list[str]:
return [
row_id
@@ -3076,6 +3104,107 @@ class TestSourcePreemption:
assert (decision.eligible, decision.reason) == expected
+ @pytest.mark.parametrize(
+ "checkpoints,laned,expected_ids,expected_reason",
+ [
+ pytest.param(False, False, ["a", "b", "c"], "deadline", id="no_progress_rides_out_to_the_deadline"),
+ pytest.param(True, False, ["a", "b"], "progress", id="progress_hands_off_at_once"),
+ pytest.param(False, True, ["a", "b"], "legacy", id="laned_run_keeps_the_legacy_rule"),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_a_run_without_durable_progress_rides_out_the_shutdown(
+ self, checkpoints: bool, laned: bool, expected_ids: list[str], expected_reason: str
+ ) -> None:
+ redis = MagicMock()
+ manager = _manager()
+ clock = _FakeClock()
+ wait_ended_early: list[bool] = []
+
+ def items():
+ if checkpoints:
+ manager.save_state(_Cursor("a"))
+ yield pa.table({"id": ["a"]})
+ switch.shut_down()
+ yield pa.table({"id": ["b"]})
+ # A source backoff asks the same signal: during a ride-out it must run its full length.
+ is_shutting_down = shutdown_signal()
+ assert is_shutting_down is not None
+ wait_ended_early.append(is_shutting_down())
+ clock.now = 3600.0
+ yield pa.table({"id": ["c"]})
+ raise RuntimeError("end of the source")
+
+ pipeline = _runnable_pipeline(manager, items)
+ if laned:
+ pipeline._resource.lanes = [OutputLane(name="users_cdc")]
+ # A long quiet period keeps the preemptor out of it: the item loop makes the decision.
+ switch = _with_ride_out(pipeline, clock, quiet_period_seconds=7200.0)
+
+ await _run_expecting(pipeline, redis, WorkerShuttingDownError)
+
+ assert _staged_ids(pipeline) == expected_ids
+ assert pipeline._progress is not None
+ assert pipeline._progress.handoff_reason == expected_reason
+ assert pipeline._handoff_gate is not None
+ assert pipeline._handoff_gate.rode_out is (expected_reason == "deadline")
+ # Leaving at `b` comes before the source asks; a ride-out ran the wait in full.
+ assert wait_ended_early == ([False] if expected_reason == "deadline" else [])
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "resumable,quiet_period_seconds,before_deadline,at_deadline",
+ [
+ # A resumable source stuck in a call: past the quiet period the gate still holds it to the cap.
+ pytest.param(True, 300.0, 600.0, 3600.0, id="resumable_held_past_the_quiet_period"),
+ # A run that restarts from scratch either way keeps the whole grace period, then leaves
+ # instead of dying when it ends, which used to cost a retry attempt.
+ pytest.param(False, 300.0, 3600.0, 6300.0, id="from_scratch_keeps_the_whole_grace"),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_a_source_blocked_in_a_call_is_preempted_at_the_gate_deadline(
+ self, resumable: bool, quiet_period_seconds: float, before_deadline: float, at_deadline: float
+ ) -> None:
+ redis = MagicMock()
+ manager = _manager()
+ release = threading.Event()
+ clock = _FakeClock()
+
+ def items():
+ yield pa.table({"id": ["a"]})
+ switch.shut_down()
+ release.wait()
+ yield pa.table({"id": ["b"]})
+
+ pipeline = _runnable_pipeline(manager, items)
+ if not resumable:
+ pipeline._resumable_source_manager = None
+ switch = _with_ride_out(pipeline, clock, quiet_period_seconds=quiet_period_seconds)
+
+ with patch(f"{_PREEMPTION}.RIDE_OUT_WAIT_SLICE_SECONDS", 0.01):
+ run = asyncio.ensure_future(_run_expecting(pipeline, redis, SourcePreemptedError))
+ try:
+ await switch.seen_by_event_loop.wait()
+ clock.now = before_deadline
+ for _ in range(50):
+ await asyncio.sleep(0.01)
+ assert not run.done()
+ clock.now = at_deadline
+ await asyncio.wait_for(run, timeout=10)
+ finally:
+ release.set()
+
+ assert _staged_ids(pipeline) == ["a"]
+ preempted = [
+ call.kwargs["preemption_reason"]
+ for call in cast(AsyncMock, pipeline._logger.ainfo).await_args_list
+ if call.args[0].startswith("Preempting the import")
+ ]
+ assert preempted == ["deadline"]
+ assert pipeline._progress is not None
+ assert pipeline._progress.handoff_reason == "preempted_deadline"
+
@pytest.mark.parametrize("rows_reach_a_batch", [True, False], ids=["rows_staged", "rows_still_buffered"])
@pytest.mark.asyncio
async def test_a_preempted_resumable_run_and_its_next_attempt_stage_every_row_once(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py
index d8a06148328f..965d20f57859 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_retry_budget.py
@@ -225,7 +225,7 @@ class TestShutdownDuringARetryWait:
manager = MagicMock()
manager.has_staged_state.return_value = False
handler = PipelineSafePointHandler(
- shutdown_monitor=monitor, # type: ignore[arg-type]
+ raise_if_should_hand_off=monitor.raise_if_is_worker_shutdown,
resumable_source_manager=manager,
has_unwritten_rows=lambda: False,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
index 49ad58a89a64..44addcf2c27b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
@@ -70,8 +70,11 @@ class ResumableSourceManager(Generic[ResumableData]):
pending: dict[str, str] | None = None,
staged: dict[str, str] | None = None,
write_fence: _WriteFence | None = None,
+ commit_listeners: list[collections.abc.Callable[[], None]] | None = None,
):
self._inputs = inputs
+ # Shared with the siblings: a commit through any namespace is progress of the same run.
+ self._commit_listeners = commit_listeners if commit_listeners is not None else []
self._data_class = data_class
self._logger = inputs.logger
self._namespace = namespace
@@ -99,6 +102,7 @@ class ResumableSourceManager(Generic[ResumableData]):
pending=self._pending,
staged=self._staged,
write_fence=self._write_fence,
+ commit_listeners=self._commit_listeners,
)
def revoke_writes(self, timeout_seconds: float) -> bool:
@@ -202,10 +206,15 @@ class ResumableSourceManager(Generic[ResumableData]):
break
self._staged[key] = json_data
+ def on_commit(self, listener: collections.abc.Callable[[], None]) -> None:
+ """Call `listener` after each commit that wrote a cursor, from whichever thread commits."""
+ self._commit_listeners.append(listener)
+
def commit(self) -> None:
"""Persist every confirmed cursor, across namespaces."""
if not self._staged:
return
+ wrote = False
with self._write_fence.writing(), self._get_redis() as redis_client:
for key, json_data in list(self._staged.items()):
self._logger.debug(f"Saving resumable source state. key={key}")
@@ -214,6 +223,10 @@ class ResumableSourceManager(Generic[ResumableData]):
functools.partial(redis_client.set, key, json_data, ex=60 * 60 * 24), # 24 hours expiration
)
del self._staged[key]
+ wrote = True
+ if wrote:
+ for listener in self._commit_listeners:
+ listener()
@contextmanager
def committing(self) -> collections.abc.Iterator[None]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index f4f341a15f35..cffca5cbb481 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -28,7 +28,7 @@ from posthog.temporal.common.activity_context import current_activity_attempt
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
from posthog.temporal.common.logger import get_logger
-from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDownError
+from posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, ShutdownMonitor, WorkerShuttingDownError
from posthog.temporal.common.utils import is_stale_connection_read_only_error
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
@@ -64,6 +64,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.e
trim_source_job_inputs,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ IMPORT_RIDE_OUT_FLAG,
+ DurableProgress,
+ RideOutConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
SchemaColumnTypeChangedException,
)
@@ -83,6 +88,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
)
from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTS
from products.warehouse_sources.backend.temporal.data_imports.row_tracking import setup_row_tracking
+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabled
from products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
FAST_RETURN_PROBE_TIMEOUT,
@@ -390,8 +396,9 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelin
# zombie predecessor stands down (its heartbeat timed out, but it may still be running).
attempt=current_import_attempt(),
):
+ progress = DurableProgress(carry_over_counts=settings.DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED)
try:
- return await _import_data_with_reporting(inputs, logger)
+ return await _import_data_with_reporting(inputs, logger, progress)
except WorkerShuttingDownError:
if not inputs.handoffs_are_free:
raise
@@ -401,6 +408,8 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelin
should_trigger_cdp_producer=False,
handed_off=True,
handoff_attempts_used=current_activity_attempt(),
+ handoff_made_progress=progress.has_progress,
+ handoff_reason=progress.handoff_reason or "unknown",
)
except (OperationalError, InterfaceError, InternalError, PostHogInternalDatabaseError) as e:
# The setup phase (resolving the job/schema/source rows for this run) reads PostHog's
@@ -423,7 +432,9 @@ async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> Pipelin
raise NonReportableError(POSTHOG_DATABASE_UNAVAILABLE_MESSAGE) from e
-async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger: FilteringBoundLogger) -> PipelineResult:
+async def _import_data_with_reporting(
+ inputs: ImportDataActivityInputs, logger: FilteringBoundLogger, progress: DurableProgress
+) -> PipelineResult:
async with Heartbeater(factor=30), ShutdownMonitor() as shutdown_monitor:
await setup_row_tracking(inputs.team_id, inputs.schema_id)
@@ -584,6 +595,16 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:
quiet_period_seconds=settings.DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS,
watermark_carry_over_enabled=settings.DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED,
)
+ # A deadline hand-off on the retry path would cost an attempt, so the same rule as preemption.
+ ride_out: RideOutConfig | None = None
+ if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(
+ schema, IMPORT_RIDE_OUT_FLAG
+ ):
+ grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS
+ ride_out = RideOutConfig(
+ grace_seconds=float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds(),
+ max_seconds=settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS,
+ )
if schema.should_use_incremental_field:
await logger.adebug(f"Incremental last value being used is: {processed_incremental_last_value}")
@@ -779,6 +800,8 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ ride_out=ride_out,
+ progress=progress,
)
else:
raise ValueError(f"Source type {model.pipeline.source_type} not supported")
@@ -1167,6 +1190,8 @@ async def _run(
preemption: PreemptionConfig | None = None,
on_resumable_retry_budget: bool = False,
kept_rows_synced: int | None = None,
+ ride_out: RideOutConfig | None = None,
+ progress: DurableProgress | None = None,
) -> PipelineResult:
try:
reset_pipeline = (
@@ -1188,6 +1213,8 @@ async def _run(
resumed_incremental_value=resumed_incremental_value,
preemption=preemption,
kept_rows_synced=kept_rows_synced,
+ ride_out=ride_out,
+ progress=progress,
)
result = await pipeline.run()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index eaff04cfbdfa..eae58a94d940 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -38,6 +38,11 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job
_transient_error_message,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.ride_out import (
+ IMPORT_RIDE_OUT_FLAG,
+ DurableProgress,
+ RideOutConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core import repartition_controller
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
SchemaColumnTypeChangedException,
@@ -1120,6 +1125,7 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workfl
mock.patch.object(module, "bind_job_context"),
mock.patch.object(module, "trim_source_job_inputs", new=mock.AsyncMock()),
mock.patch.object(module, "database_sync_to_async_pool", new=_passthrough),
+ mock.patch.object(module, "is_schema_flag_enabled", return_value=False),
mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),
mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),
mock.patch.object(module, "_run", new=mock.AsyncMock(return_value=mock.sentinel.run_result)) as run_mock,
@@ -1857,22 +1863,40 @@ async def test_an_interrupted_incremental_import_continues_after_its_last_queued
@pytest.mark.asyncio
[email protected]("handoffs_are_free", [True, False])
-async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(handoffs_are_free: bool):
+ "handoffs_are_free,made_progress",
+ [(True, False), (True, True), (False, False)],
+ ids=["free_without_progress", "free_with_progress", "retry"],
+)
+async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(
+ handoffs_are_free: bool, made_progress: bool
+):
error = WorkerShuttingDownError("5", "import_data_activity_sync", "data-warehouse-task-queue", 2, "wf", "wt")
inputs = dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free, prior_attempts=4)
+ async def import_then_hand_off(_inputs, _logger, progress):
+ if made_progress:
+ progress.mark("resume_commit")
+ progress.record_handoff_reason("progress")
+ raise error
+
with (
mock.patch.object(module, "tag_queries"),
mock.patch.object(module, "report_heartbeat_timeout"),
mock.patch.object(module, "aworkload_reporting", return_value=_FakeAsyncCM()),
- mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=error)),
+ mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=import_then_hand_off)),
mock.patch.object(module, "current_activity_attempt", return_value=2),
):
if handoffs_are_free:
result = await import_data_activity_sync(inputs)
# The workflow adds the attempts up, so it needs the attempt number of this execution only.
- assert result == {"should_trigger_cdp_producer": False, "handed_off": True, "handoff_attempts_used": 2}
+ assert result == {
+ "should_trigger_cdp_producer": False,
+ "handed_off": True,
+ "handoff_attempts_used": 2,
+ "handoff_made_progress": made_progress,
+ "handoff_reason": "progress" if made_progress else "unknown",
+ }
else:
with pytest.raises(WorkerShuttingDownError):
await import_data_activity_sync(inputs)
@@ -1922,6 +1946,44 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(
assert run_mock.await_args.kwargs["preemption"] == expected
+ "flag_enabled,handoffs_are_free,grace,expected",
+ [
+ pytest.param(True, True, 7200, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),
+ # The worker falls back to a five-minute graceful shutdown when the setting is unset.
+ pytest.param(True, True, None, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),
+ # Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.
+ pytest.param(True, False, 7200, None, id="off_when_a_handoff_uses_a_retry_attempt"),
+ pytest.param(False, True, 7200, None, id="off_when_the_flag_is_off"),
+ ],
+)
+async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(
+ flag_enabled: bool,
+ handoffs_are_free: bool,
+ grace: int | None,
+ expected: RideOutConfig | None,
+ settings,
+):
+ settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS = 1800.0
+ settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = grace
+ source = mock.MagicMock(spec=SimpleSource)
+ source.parse_config.return_value = {}
+ source.source_for_pipeline.return_value = mock.MagicMock()
+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)
+
+ with (
+ _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,
+ mock.patch.object(module, "is_schema_flag_enabled", return_value=flag_enabled) as flag_check,
+ ):
+ await import_data_activity_sync(dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free))
+
+ assert run_mock.await_args.kwargs["ride_out"] == expected
+ if handoffs_are_free:
+ flag_check.assert_called_once_with(schema, IMPORT_RIDE_OUT_FLAG)
+ assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)
+
+
@parameterized.expand(
[
("no_checkpoint_past_the_allowance", True, PROGRESSLESS_RESUMABLE_ATTEMPTS + 1, False, True),