Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): let import attempts finish briefly before shutdown hand-off (#123322)

Tom Owers45bf2bdb69c3
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 0da3b903822c..105c2e94eef2 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -336,6 +336,12 @@ DATA_WAREHOUSE_IMPORT_PREEMPTION_QUIET_PERIOD_SECONDS: float = get_from_env(
DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS: float = get_from_env(
"DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS", 3600.0, type_cast=float
)
+# At a worker shutdown an import keeps running this long before any hand-off check sees the shutdown.
+# Many imports that run at the signal finish in this time, and a hand-off would restart them on
+# another worker. 0 hands off at once. The import uses at most half the graceful shutdown timeout.
+DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS: float = get_from_env(
+ "DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS", 60.0, type_cast=float
+)
# Restore unfinished append runs only after all extract and load workers have this implementation.
DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED: bool = get_from_env(
"DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED", False, type_cast=str_to_bool
diff --git a/posthog/temporal/common/shutdown.py b/posthog/temporal/common/shutdown.py
index 189a03149bc9..a98f8c7da002 100644
--- a/posthog/temporal/common/shutdown.py
+++ b/posthog/temporal/common/shutdown.py
@@ -5,7 +5,7 @@ import asyncio
import datetime
import threading
import contextvars
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from structlog import get_logger
from temporalio import activity
@@ -64,9 +64,23 @@ class ShutdownMonitor:
All Temporal activities should consider `WorkerShuttingDownError` as a
retryable exception, at least if they wish to have new workers pick it up.
+
+ With `grace_seconds`, the monitor reports the shutdown only that long after the
+ worker got the signal. Every check, wait and shutdown callback sees the shutdown
+ then, so an activity that ends inside the grace never stops early. The worker
+ takes no new activities from the signal on, so only activities that already run
+ use the grace. Keep the grace well below the worker's graceful shutdown timeout.
"""
- def __init__(self):
+ def __init__(
+ self,
+ grace_seconds: float = 0.0,
+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+ ):
+ if grace_seconds < 0:
+ raise ValueError(f"grace_seconds must not be negative, got {grace_seconds}")
+ self._grace_seconds = grace_seconds
+ self._sleep = sleep
self._monitor_shutdown_task: asyncio.Task[None] | None = None
self._monitor_shutdown_thread: threading.Thread | None = None
self._on_shutdown_callbacks: list[tuple[contextvars.Context, Callable[[], None]]] = []
@@ -115,6 +129,13 @@ class ShutdownMonitor:
# Not running in an activity context.
return
+ if self._grace_seconds > 0:
+ self.logger.info(
+ "Shutdown signal received, the activity keeps running for the grace period.",
+ grace_seconds=self._grace_seconds,
+ )
+ await self._sleep(self._grace_seconds)
+
self._record_shutdown_time()
self.logger.info("Shutdown detected.")
# Run the callbacks before the event is set: a caller that sees the event can raise and
@@ -129,7 +150,7 @@ class ShutdownMonitor:
self._monitor_shutdown_task = asyncio.create_task(monitor())
def run_on_shutdown(self, callback: Callable[[], None]) -> None:
- """Call `callback` once when the worker starts to shut down.
+ """Call `callback` once when the monitor reports the shutdown, which is after the grace.
`callback` runs with a copy of the caller's contextvars, taken now, so it sees the structlog
context the caller has bound (for example the job context an activity binds after it enters
@@ -143,6 +164,9 @@ class ShutdownMonitor:
Notice we must copy the context to preserve the activity context for the
monitoring thread.
"""
+ if self._grace_seconds > 0:
+ raise ValueError("A shutdown grace needs the async monitor: use `async with ShutdownMonitor(...)`")
+
context = contextvars.copy_context()
def monitor() -> None:
@@ -218,7 +242,7 @@ class ShutdownMonitor:
self._shutdown_at = time.monotonic()
def seconds_since_shutdown(self) -> float | None:
- """How long ago the worker started to shut down, or None while it still runs.
+ """How long ago the monitor started to report the shutdown (at the end of the grace), or None before.
A caller created after the shutdown cannot learn the moment from a callback, because the
callbacks have already run by then. It reads the elapsed time here instead.
diff --git a/posthog/temporal/tests/test_shutdown.py b/posthog/temporal/tests/test_shutdown.py
index baaccbe79301..415d01017a93 100644
--- a/posthog/temporal/tests/test_shutdown.py
+++ b/posthog/temporal/tests/test_shutdown.py
@@ -14,7 +14,7 @@ from temporalio.common import RetryPolicy
from temporalio.exceptions import ActivityError, ApplicationError
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
-from posthog.temporal.common.shutdown import ShutdownSignalListener
+from posthog.temporal.common.shutdown import ShutdownMonitor, ShutdownSignalListener, WorkerShuttingDownError
from posthog.temporal.tests.utils.workflow import Waiter, WaitInputs, WaitMode, WaitWorkflow
@@ -207,3 +207,88 @@ async def test_shutdown_signal_listener_keeps_the_first_signal(wakeup_fd_full: b
signal.signal(sig, handler)
read_sock.close()
write_sock.close()
+
+
+class _GraceSleep:
+ """Stands in for `asyncio.sleep` in the monitor, so a test decides when the grace ends."""
+
+ def __init__(self) -> None:
+ self.requested: list[float] = []
+ self.started = asyncio.Event()
+ self.release = asyncio.Event()
+
+ async def __call__(self, seconds: float) -> None:
+ self.requested.append(seconds)
+ self.started.set()
+ await self.release.wait()
+
+
+async def _settle() -> None:
+ for _ in range(5):
+ await asyncio.sleep(0)
+
+
+ "grace_seconds,finishes_in_grace,expected",
+ [
+ # Today's behaviour: the first check after the signal hands off.
+ pytest.param(0.0, False, "handed_off_at_signal", id="no_grace_hands_off_at_once"),
+ pytest.param(60.0, True, "finished", id="finishes_inside_the_grace"),
+ pytest.param(60.0, False, "handed_off_after_grace", id="still_running_at_grace_end_hands_off"),
+ ],
+)
+async def test_shutdown_grace_delays_what_every_check_sees(
+ activity_environment, grace_seconds: float, finishes_in_grace: bool, expected: str
+):
+ sleep = _GraceSleep()
+ callback_ran: list[bool] = []
+
+ async def activity_body() -> str:
+ async with ShutdownMonitor(grace_seconds=grace_seconds, sleep=sleep) as monitor:
+ monitor.run_on_shutdown(lambda: callback_ran.append(True))
+ wait_for_shutdown = asyncio.ensure_future(monitor.wait_for_worker_shutdown())
+ await _settle()
+
+ # Before the signal, no check sees a shutdown.
+ monitor.raise_if_is_worker_shutdown()
+ assert (monitor.is_worker_shutdown(), monitor.seconds_since_shutdown(), callback_ran) == (False, None, [])
+
+ activity_environment.worker_shutdown()
+ await _settle()
+ try:
+ monitor.raise_if_is_worker_shutdown()
+ except WorkerShuttingDownError:
+ assert wait_for_shutdown.done() and callback_ran == [True]
+ return "handed_off_at_signal"
+
+ assert sleep.requested == [grace_seconds]
+ assert (monitor.is_worker_shutdown(), wait_for_shutdown.done(), callback_ran) == (False, False, [])
+ if finishes_in_grace:
+ return "finished"
+
+ sleep.release.set()
+ await asyncio.wait_for(wait_for_shutdown, timeout=5)
+ assert callback_ran == [True]
+ try:
+ monitor.raise_if_is_worker_shutdown()
+ except WorkerShuttingDownError:
+ return "handed_off_after_grace"
+ return "kept_running"
+
+ assert await activity_environment.run(activity_body) == expected
+
+
+ "grace_seconds,sync,error",
+ [
+ pytest.param(-1.0, False, "must not be negative", id="negative_grace"),
+ # The thread monitor has no grace timer, so a grace there would never end.
+ pytest.param(60.0, True, "needs the async monitor", id="grace_on_the_thread_monitor"),
+ ],
+)
+def test_shutdown_grace_rejects_a_configuration_it_cannot_honour(grace_seconds: float, sync: bool, error: str):
+ with pytest.raises(ValueError, match=error):
+ monitor = ShutdownMonitor(grace_seconds=grace_seconds)
+ if sync:
+ monitor.start_sync()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
index b5fb93238c07..ce563a64dd11 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
@@ -174,12 +174,21 @@ def get_import_not_preempted_metric(source_type: str | None, reason: str) -> Met
)
-def get_import_finished_during_shutdown_metric(source_type: str | None, rode_out: bool) -> MetricCounter:
+def get_import_finished_during_shutdown_metric(
+ source_type: str | None, rode_out: bool, *, within_grace: bool
+) -> MetricCounter:
# The imports a hand-off would have restarted. `rode_out` is True when the gate kept the import
- # on the worker past the shutdown at least once.
+ # on the worker past the shutdown at least once. `within_grace` is True when the import finished
+ # before the shutdown monitor reported the shutdown, so no hand-off check saw it.
return (
activity.metric_meter()
- .with_additional_attributes({"source_type": source_type or "unknown", "rode_out": str(rode_out).lower()})
+ .with_additional_attributes(
+ {
+ "source_type": source_type or "unknown",
+ "rode_out": str(rode_out).lower(),
+ "within_grace": str(within_grace).lower(),
+ }
+ )
.create_counter(
"warehouse_import_finished_during_shutdown_total",
"Imports that finished on a worker that was shutting down instead of handing off.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py
index f41ed30c474e..d6d869f7f772 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py
@@ -894,12 +894,18 @@ async def update_row_tracking_after_batch(
await decrement_rows(team_id, schema_id, row_count)
-def is_young_first_attempt() -> bool:
+def is_young_first_attempt(shutdown_grace_seconds: float = 0.0) -> bool:
+ """`shutdown_grace_seconds` is the time the shutdown monitor let the run continue after the signal.
+
+ It does not count toward the age, so the grace never turns a run that was young at the signal into
+ one that keeps the worker until it finishes.
+ """
if not activity.in_activity():
return False
info = activity.info()
- return info.attempt == 1 and datetime.now(UTC) - info.started_time < NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE
+ age = datetime.now(UTC) - info.started_time - timedelta(seconds=shutdown_grace_seconds)
+ return info.attempt == 1 and age < NON_RESUMABLE_HANDOFF_MAX_ATTEMPT_AGE
def should_check_shutdown(
@@ -907,6 +913,7 @@ def should_check_shutdown(
resource: SourceResponse,
reset_pipeline: bool,
source_is_resumable: bool,
+ shutdown_grace_seconds: float = 0.0,
) -> bool:
# A descending run does not raise, because each stop restarts it from the old watermark. An ascending
# incremental run keeps its progress after a stop only through the handoff checkpoint.
@@ -916,7 +923,9 @@ def should_check_shutdown(
incremental_sync_raise_during_shutdown = (
schema.should_use_incremental_field and resource.sort_mode != "desc" and not reset_pipeline
)
- return incremental_sync_raise_during_shutdown or source_is_resumable or is_young_first_attempt()
+ return (
+ incremental_sync_raise_during_shutdown or source_is_resumable or is_young_first_attempt(shutdown_grace_seconds)
+ )
async def finalize_desc_sort_incremental_value(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
index f103eb8386db..b4a58834c86d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/ride_out.py
@@ -85,10 +85,11 @@ class DurableProgress:
self._handoff_reason = reason
-def activity_execution_age_seconds() -> float | None:
+def activity_execution_age_seconds(shutdown_grace_seconds: float = 0.0) -> float | None:
+ """The age leaves out the shutdown grace, for the reason given on `is_young_first_attempt`."""
if not activity.in_activity():
return None
- return (datetime.now(UTC) - activity.info().started_time).total_seconds()
+ return (datetime.now(UTC) - activity.info().started_time).total_seconds() - shutdown_grace_seconds
class HandoffGate:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py
index bb5455a5e0c7..78b29b720cb4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py
@@ -75,6 +75,9 @@ class TestShouldCheckShutdown:
("ascending_incremental", True, "asc", False, False, True, 2, 20, True),
("unordered_incremental", True, None, False, False, True, 2, 20, True),
("descending_incremental", True, "desc", False, False, True, 2, 20, False),
+ # The shutdown grace does not count toward the age.
+ ("young_at_signal_after_grace", False, "asc", True, False, True, 1, 10, True, 60.0),
+ ("old_at_signal_after_grace", False, "asc", True, False, True, 1, 12, False, 60.0),
]
)
def test_should_check_shutdown(
@@ -88,6 +91,7 @@ class TestShouldCheckShutdown:
attempt: int,
age_minutes: int,
expected: bool,
+ shutdown_grace_seconds: float = 0.0,
) -> None:
schema = MagicMock(should_use_incremental_field=is_incremental)
resource = MagicMock(sort_mode=sort_mode)
@@ -97,7 +101,12 @@ class TestShouldCheckShutdown:
patch(f"{_EXTRACT_MODULE}.activity.in_activity", return_value=in_activity),
patch(f"{_EXTRACT_MODULE}.activity.info", return_value=info) as activity_info,
):
- assert should_check_shutdown(schema, resource, reset_pipeline, source_is_resumable) is expected
+ assert (
+ should_check_shutdown(
+ schema, resource, reset_pipeline, source_is_resumable, shutdown_grace_seconds=shutdown_grace_seconds
+ )
+ is expected
+ )
if not in_activity:
activity_info.assert_not_called()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
index ace3fb8de0c4..57df0da793f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py
@@ -103,6 +103,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
+ shutdown_grace_seconds: float = 0.0,
) -> None:
if not source_response.lanes:
raise ValueError(f"{source_response.name} declares no lanes; run it on PipelineV3")
@@ -126,6 +127,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):
continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
+ shutdown_grace_seconds=shutdown_grace_seconds,
)
# The base built the first lane; it shares the base's batch list so the two never disagree.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 478b7e8aac0b..439d0b3a05f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -2,6 +2,7 @@ import time
import asyncio
import decimal
import datetime
+import functools
import contextlib
from collections.abc import AsyncGenerator, AsyncIterator
from typing import TYPE_CHECKING, Any, Generic
@@ -66,6 +67,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.r
DurableProgress,
HandoffGate,
RideOutConfig,
+ activity_execution_age_seconds,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (
PipelineSafePointHandler,
@@ -200,6 +202,8 @@ class PipelineV3(Generic[ResumableData]):
_ride_out: RideOutConfig | None = None
_progress: DurableProgress | None = None
_handoff_gate: HandoffGate | None = None
+ # The grace of the run's `ShutdownMonitor`. The age checks of a hand-off leave it out.
+ _shutdown_grace_seconds: float = 0.0
_resumed_incremental_run_uuid: str | None = None
_sent_resumed_run_finalization: bool = False
_writes_staged_repartition_scheme: bool = False
@@ -230,11 +234,13 @@ class PipelineV3(Generic[ResumableData]):
continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
+ shutdown_grace_seconds: float = 0.0,
) -> None:
self._resource = source_response
self._kept_rows_synced = kept_rows_synced
self._ride_out = ride_out
self._progress = progress
+ self._shutdown_grace_seconds = shutdown_grace_seconds
self._source_cursor_manager = source_cursor_manager
self._resource_name = source_response.name
@@ -686,7 +692,7 @@ class PipelineV3(Generic[ResumableData]):
carry_over_enabled = self._preemption is not None and self._preemption.watermark_carry_over_enabled
if checkpoint is not None and not checkpoint.is_void and carry_over_enabled:
return PreemptionDecision(eligible=True, reason="watermark_carry_over")
- if is_young_first_attempt():
+ if is_young_first_attempt(self._shutdown_grace_seconds):
return PreemptionDecision(eligible=True, reason="young_first_attempt")
if checkpoint is not None and not checkpoint.is_void:
return PreemptionDecision(
@@ -719,10 +725,17 @@ class PipelineV3(Generic[ResumableData]):
# A hand-off retires a laned run's companion jobs, so that run keeps the legacy rule.
config=None if self._resource.lanes else self._ride_out,
legacy_wants_handoff=lambda: should_check_shutdown(
- self._schema, self._resource, self._reset_pipeline, self._resumable_source_manager is not None
+ self._schema,
+ self._resource,
+ self._reset_pipeline,
+ self._resumable_source_manager is not None,
+ shutdown_grace_seconds=self._shutdown_grace_seconds,
),
restarts_from_scratch=self._restarts_from_scratch,
logger=self._logger,
+ execution_age_seconds=functools.partial(
+ activity_execution_age_seconds, shutdown_grace_seconds=self._shutdown_grace_seconds
+ ),
)
return self._handoff_gate
@@ -1078,8 +1091,13 @@ class PipelineV3(Generic[ResumableData]):
chunk_index += 1
await self._finalize(row_count=row_count)
- if activity.in_activity() and self._shutdown_monitor.is_worker_shutdown():
- get_import_finished_during_shutdown_metric(source_type, handoff_gate.rode_out).add(1)
+ # The signal, not the monitor: an import that finished inside the shutdown grace counts too.
+ if activity.in_activity() and activity.is_worker_shutdown():
+ get_import_finished_during_shutdown_metric(
+ source_type,
+ handoff_gate.rode_out,
+ within_grace=not self._shutdown_monitor.is_worker_shutdown(),
+ ).add(1)
# With zero batches, `_finalize` sent no final-batch notification, so the load
# consumer will never hear about this run and cannot finalize it — the workflow must.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index eb75a1fc6254..a277eb6f56cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -17,8 +17,9 @@ from django.test import override_settings
import pyarrow as pa
import deltalake
from asgiref.sync import async_to_sync
+from temporalio.testing import ActivityEnvironment
-from posthog.temporal.common.shutdown import WorkerShuttingDownError
+from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDownError
from products.warehouse_sources.backend.models import external_data_schema as schema_models
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
@@ -2019,6 +2020,67 @@ class TestResumeCursorCommit:
# A wrapped resource re-reads its last written page; the merge on the primary key absorbs it.
assert sorted(set(written + resumed)) == list(range(1, page_count + 1))
+ @pytest.mark.parametrize(
+ "grace_seconds,expected_written",
+ [
+ # Today's behaviour: the first item after the signal hands off.
+ pytest.param(0.0, ["a", "b"], id="no_grace_hands_off_at_the_signal"),
+ # The run keeps writing and committing through the grace, and hands off at its end.
+ pytest.param(60.0, ["a", "b", "c", "d"], id="hands_off_after_the_grace"),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_a_handoff_after_the_shutdown_grace_resumes_after_its_last_written_batch(
+ self, grace_seconds: float, expected_written: list[str]
+ ) -> None:
+ redis = MagicMock()
+ manager = _manager()
+ env = ActivityEnvironment()
+ loop = asyncio.get_running_loop()
+ grace_started, grace_ended = asyncio.Event(), asyncio.Event()
+ monitors: list[ShutdownMonitor] = []
+ ids = ["a", "b", "c", "d", "e"]
+
+ async def grace_sleep(seconds: float) -> None:
+ grace_started.set()
+ await grace_ended.wait()
+
+ async def send_the_signal() -> None:
+ env.worker_shutdown()
+ await (grace_started.wait() if grace_seconds else monitors[0].wait_for_worker_shutdown())
+
+ async def end_the_grace() -> None:
+ grace_ended.set()
+ await monitors[0].wait_for_worker_shutdown()
+
+ def on_the_event_loop(step: Any) -> None:
+ asyncio.run_coroutine_threadsafe(step(), loop).result(timeout=5)
+
+ def items():
+ for row_id in ids:
+ if row_id == "b":
+ on_the_event_loop(send_the_signal)
+ if row_id == "d" and grace_seconds:
+ on_the_event_loop(end_the_grace)
+ manager.save_state(_Cursor(row_id))
+ yield pa.table({"id": [row_id]})
+
+ pipeline = _runnable_pipeline(manager, items)
+
+ async def activity_body() -> None:
+ async with ShutdownMonitor(grace_seconds=grace_seconds, sleep=grace_sleep) as monitor:
+ monitors.append(monitor)
+ pipeline._shutdown_monitor = monitor
+ await _run_expecting(pipeline, redis, WorkerShuttingDownError)
+
+ await env.run(activity_body)
+
+ committed = [json.loads(call.args[1])["id"] for call in redis.set.call_args_list]
+ assert _staged_ids(pipeline) == expected_written
+ assert committed[-1] == expected_written[-1]
+ # The next attempt reads after the committed cursor, so every row lands exactly once.
+ assert expected_written + ids[ids.index(committed[-1]) + 1 :] == ids
+
@pytest.mark.asyncio
async def test_cursor_not_persisted_for_rows_the_batcher_still_holds(self) -> None:
redis = MagicMock()
@@ -3143,10 +3205,14 @@ class TestSourcePreemption:
if checkpoint == "void":
pipeline._handoff_checkpoint.observe(None)
- with patch(f"{_PIPELINE}.is_young_first_attempt", return_value=young):
+ pipeline._shutdown_grace_seconds = 60.0
+
+ with patch(f"{_PIPELINE}.is_young_first_attempt", return_value=young) as young_check:
decision = pipeline._preemption_decision()
assert (decision.eligible, decision.reason) == expected
+ # The age leaves out the shutdown grace, so the grace never makes a young run old.
+ assert all(call.args == (60.0,) for call in young_check.call_args_list)
@pytest.mark.parametrize(
"checkpoints,laned,expected_ids,expected_reason",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 8f37245b9d67..5e12de6bf350 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -230,6 +230,18 @@ CONTINUABLE_SYNC_TYPES = frozenset(
MAX_CONTINUED_JOBS = 20
+def _graceful_shutdown_timeout_seconds() -> float:
+ grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS
+ return float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds()
+
+
+def _import_shutdown_grace_seconds() -> float:
+ # At most half the SDK's graceful shutdown timeout, so the hand-off after the grace always has
+ # time to finish before the SDK cancels the activity.
+ grace = min(settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS, _graceful_shutdown_timeout_seconds() / 2)
+ return max(grace, 0.0)
+
+
@database_sync_to_async_pool
def _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[ExternalDataJob]:
"""The jobs before this one that worker restarts stopped, newest first, when this job can continue their cursor.
@@ -526,7 +538,8 @@ async def _run_import_attempt(inputs: ImportDataActivityInputs) -> PipelineResul
async def _import_data_with_reporting(
inputs: ImportDataActivityInputs, logger: FilteringBoundLogger, progress: DurableProgress
) -> PipelineResult:
- async with Heartbeater(factor=30), ShutdownMonitor() as shutdown_monitor:
+ shutdown_grace_seconds = _import_shutdown_grace_seconds()
+ async with Heartbeater(factor=30), ShutdownMonitor(grace_seconds=shutdown_grace_seconds) as shutdown_monitor:
await setup_row_tracking(inputs.team_id, inputs.schema_id)
try:
@@ -692,9 +705,9 @@ async def _import_data_with_reporting(
if inputs.handoffs_are_free and await database_sync_to_async_pool(is_schema_flag_enabled)(
schema, IMPORT_RIDE_OUT_FLAG
):
- grace = settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS
+ # The ride-out's stopwatch starts when the monitor reports the shutdown, after the shutdown grace.
ride_out = RideOutConfig(
- grace_seconds=float(grace) if grace is not None else DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT.total_seconds(),
+ grace_seconds=_graceful_shutdown_timeout_seconds() - shutdown_grace_seconds,
max_seconds=settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS,
)
@@ -914,6 +927,7 @@ async def _import_data_with_reporting(
continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
+ shutdown_grace_seconds=shutdown_grace_seconds,
)
else:
raise ValueError(f"Source type {model.pipeline.source_type} not supported")
@@ -988,9 +1002,9 @@ POSTHOG_DATABASE_UNAVAILABLE_MESSAGE = (
def _log_worker_shutdown_during_import(logger: FilteringBoundLogger) -> None:
- # One line per import still running when the worker got SIGTERM. A matching hand-off line from
- # _handle_import_error means the import moved to another pod; without one, the import held
- # this pod until it finished or the graceful shutdown timeout ran out.
+ # One line per import still running when the shutdown grace after SIGTERM ended. A matching
+ # hand-off line from _handle_import_error means the import moved to another pod; without one,
+ # the import held this pod until it finished or the graceful shutdown timeout ran out.
info = activity.info()
logger.info(
"Worker shutdown detected while import is running",
@@ -1306,6 +1320,7 @@ async def _run(
continued_job_ids: tuple[str, ...] = (),
ride_out: RideOutConfig | None = None,
progress: DurableProgress | None = None,
+ shutdown_grace_seconds: float = 0.0,
) -> PipelineResult:
try:
reset_pipeline = (
@@ -1331,6 +1346,7 @@ async def _run(
continued_job_ids=continued_job_ids,
ride_out=ride_out,
progress=progress,
+ shutdown_grace_seconds=shutdown_grace_seconds,
)
result = await pipeline.run()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index 41a4e86cad37..e7f2a716f069 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -2149,25 +2149,33 @@ async def test_the_pipeline_preempts_only_with_the_setting_on_and_free_handoffs(
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "flag_enabled,handoffs_are_free,grace,expected",
+ "flag_enabled,handoffs_are_free,grace,shutdown_grace,expected",
[
- pytest.param(True, True, 7200, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),
+ pytest.param(True, True, 7200, 0.0, RideOutConfig(grace_seconds=7200.0, max_seconds=1800.0), id="on"),
+ # The ride-out starts when the shutdown grace ends, so its deadline leaves that time out.
+ pytest.param(
+ True, True, 7200, 60.0, RideOutConfig(grace_seconds=7140.0, max_seconds=1800.0), id="after_shutdown_grace"
+ ),
# The worker falls back to a five-minute graceful shutdown when the setting is unset.
- pytest.param(True, True, None, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"),
+ pytest.param(
+ True, True, None, 0.0, RideOutConfig(grace_seconds=300.0, max_seconds=1800.0), id="no_grace_period"
+ ),
# Holding the worker past the deadline on the retry path would cost an attempt at the hand-off.
- pytest.param(True, False, 7200, None, id="off_when_a_handoff_uses_a_retry_attempt"),
- pytest.param(False, True, 7200, None, id="off_when_the_flag_is_off"),
+ pytest.param(True, False, 7200, 0.0, None, id="off_when_a_handoff_uses_a_retry_attempt"),
+ pytest.param(False, True, 7200, 0.0, None, id="off_when_the_flag_is_off"),
],
)
async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(
flag_enabled: bool,
handoffs_are_free: bool,
grace: int | None,
+ shutdown_grace: float,
expected: RideOutConfig | None,
settings,
):
settings.DATA_WAREHOUSE_IMPORT_RIDE_OUT_MAX_SECONDS = 1800.0
settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = grace
+ settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS = shutdown_grace
source = mock.MagicMock(spec=SimpleSource)
source.parse_config.return_value = {}
source.source_for_pipeline.return_value = mock.MagicMock()
@@ -2185,6 +2193,37 @@ async def test_the_pipeline_rides_out_only_with_the_flag_on_and_free_handoffs(
assert isinstance(run_mock.await_args.kwargs["progress"], DurableProgress)
+ "grace_setting,graceful_timeout,expected",
+ [
+ pytest.param(60.0, 7200, 60.0, id="default"),
+ pytest.param(0.0, 7200, 0.0, id="zero_hands_off_at_the_signal"),
+ # The hand-off after the grace keeps at least half the graceful shutdown timeout.
+ pytest.param(600.0, None, 150.0, id="capped_at_half_the_default_timeout"),
+ pytest.param(-5.0, 7200, 0.0, id="negative_is_no_grace"),
+ ],
+)
+async def test_the_shutdown_grace_reaches_the_monitor_and_the_pipeline(
+ grace_setting: float, graceful_timeout: int | None, expected: float, settings
+):
+ settings.DATA_WAREHOUSE_IMPORT_SHUTDOWN_GRACE_SECONDS = grace_setting
+ settings.GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = graceful_timeout
+ source = mock.MagicMock(spec=SimpleSource)
+ source.parse_config.return_value = {}
+ source.source_for_pipeline.return_value = mock.MagicMock()
+ schema = _incremental_schema(is_incremental=True, lookback_seconds=None)
+
+ with (
+ _patched_activity_reaching_run(source, schema, workflow_run_id="wfrun-1") as run_mock,
+ mock.patch.object(module, "ShutdownMonitor", return_value=_FakeAsyncCM()) as monitor_cls,
+ ):
+ await import_data_activity_sync(_inputs_no_reset())
+
+ monitor_cls.assert_called_once_with(grace_seconds=expected)
+ assert run_mock.await_args.kwargs["shutdown_grace_seconds"] == expected
+
+
@parameterized.expand(
[
("no_checkpoint_past_the_allowance", True, PROGRESSLESS_RESUMABLE_ATTEMPTS + 1, False, True),