Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): stop loader restarts from using up batch retries (#119747)

Tom Owers94c6b3990304
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
index 1a7a36cdde1d..0a4d5e78b729 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
@@ -76,7 +76,9 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (
TAKEOVER_STALE_THRESHOLD_SECONDS,
BatchQueue,
ClaimCursor,
+ ExecutingAttempt,
FailedRunRef,
+ InterruptionHistory,
PendingBatch,
QueueDepth,
_Unset,
@@ -485,6 +487,25 @@ class DeltaBatchConsumerAdapter:
exclude_sync_types=self._claim_exclude_sync_types,
)
+ async def mark_shutdown_interruptions(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ attempts: list[ExecutingAttempt],
+ ) -> int:
+ with observe_queue_query("mark_shutdown_interruptions"):
+ return await BatchQueue.mark_shutdown_interruptions(conn, attempts=attempts)
+
+ async def get_interruption_history(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ batch_id: str,
+ attempt: int,
+ ) -> InterruptionHistory:
+ with observe_queue_query("get_interruption_history"):
+ return await BatchQueue.get_interruption_history(conn, batch_id=batch_id, attempt=attempt)
+
async def reconcile_failed_runs(
self,
conn: psycopg.AsyncConnection[Any],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index 7e34c12385ad..e7d2a9897d75 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -66,7 +66,9 @@ from products.warehouse_sources_queue.backend.core.batch_consumer import (
from products.warehouse_sources_queue.backend.core.health import HealthState
from products.warehouse_sources_queue.backend.core.jobs_db import (
FRESHNESS_WINDOW_SECONDS,
+ ExecutingAttempt,
FailedRunRef,
+ InterruptionHistory,
OrphanedRunRef,
PendingBatch,
QueueDepth,
@@ -213,6 +215,18 @@ def _gauge_slot_granted():
yield
[email protected](autouse=True)
+def interruption_history():
+ # No earlier interruptions by default; the real SQL can't run against mock connections.
+ with patch.object(
+ consumer_module.BatchQueue,
+ "get_interruption_history",
+ new_callable=AsyncMock,
+ return_value=InterruptionHistory(),
+ ) as mock_history:
+ yield mock_history
+
+
class TestProcessSingle:
@pytest.mark.asyncio
async def test_success_updates_status_to_executing_then_succeeded(self):
@@ -337,33 +351,141 @@ class TestProcessSingle:
assert mock_fail.call_args[1]["reason"] == message # run still fails with the actionable message
assert mock_capture.called is expect_capture
+ @pytest.mark.parametrize(
+ "latest_attempt,recovered_interruptions,expect_fail",
+ [
+ (3, 0, True),
+ # Three shutdowns plus one error: the batch has two retries left.
+ (4, 3, False),
+ (6, 3, True),
+ (13, 10, True),
+ ],
+ ids=["errors_only", "interruptions_do_not_count", "errors_after_interruptions", "at_interruption_cap"],
+ )
@pytest.mark.asyncio
- async def test_max_attempts_exceeded_fails_run(self):
+ async def test_entry_check_counts_only_errors_against_max_attempts(
+ self, interruption_history: AsyncMock, latest_attempt: int, recovered_interruptions: int, expect_fail: bool
+ ):
consumer = _make_consumer(max_attempts=3)
- batch = _make_batch(latest_attempt=3)
+ batch = _make_batch(latest_attempt=latest_attempt)
+ interruption_history.return_value = InterruptionHistory(recovered=recovered_interruptions)
+ consumer._process_batch = AsyncMock()
- with patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail:
- await consumer._process_single(batch)
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ new_callable=AsyncMock,
+ ),
+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),
+ patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,
+ ):
+ succeeded = await consumer._process_single(batch)
- mock_fail.assert_called_once()
- assert "max retries exceeded" in mock_fail.call_args[1]["reason"]
+ assert mock_fail.called is expect_fail
+ assert succeeded is not expect_fail
+ assert consumer._process_batch.called is not expect_fail
+ if expect_fail:
+ assert "max retries exceeded" in mock_fail.call_args[1]["reason"]
+ @pytest.mark.parametrize(
+ "latest_attempt,recovered_interruptions,history_error,expect_fail",
+ [
+ (1, 0, None, True),
+ (2, 1, None, True),
+ # Three shutdowns came before this first error, so one retry is left.
+ (3, 3, None, False),
+ # A failed history read must not fail the run before the hard cap of retries plus interruptions.
+ (3, 3, psycopg.OperationalError("server closed the connection"), False),
+ (11, 10, psycopg.OperationalError("server closed the connection"), True),
+ ],
+ ids=[
+ "errors_only",
+ "errors_after_interruption",
+ "interruptions_do_not_count",
+ "history_read_fails_below_hard_cap",
+ "history_read_fails_at_hard_cap",
+ ],
+ )
@pytest.mark.asyncio
- async def test_error_at_max_attempts_fails_run(self):
+ async def test_error_at_max_attempts_fails_run(
+ self,
+ interruption_history: AsyncMock,
+ latest_attempt: int,
+ recovered_interruptions: int,
+ history_error: Exception | None,
+ expect_fail: bool,
+ ):
consumer = _make_consumer(max_attempts=2)
- batch = _make_batch(latest_attempt=1)
+ batch = _make_batch(latest_attempt=latest_attempt)
consumer._process_batch = AsyncMock(side_effect=RuntimeError("crash"))
+ interruption_history.return_value = InterruptionHistory(recovered=recovered_interruptions)
+ interruption_history.side_effect = history_error
+ states: list[str] = []
+
+ async def track_status(conn, *, batch_id, job_state, attempt, **kwargs):
+ states.append(job_state)
+ return True
with (
patch(
"products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
- new_callable=AsyncMock,
+ side_effect=track_status,
),
+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),
patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,
):
await consumer._process_single(batch)
- mock_fail.assert_called_once()
+ assert mock_fail.called is expect_fail
+ assert (SourceBatchStatus.State.WAITING_RETRY in states) is not expect_fail
+
+ @pytest.mark.parametrize(
+ "shutdown_during_executing_write,expected_states,expected_marked_attempts",
+ [
+ (False, [], []),
+ # The shutdown snapshot can miss an attempt whose executing write is in flight.
+ (True, ["executing"], [2]),
+ ],
+ ids=["before_executing_write", "during_executing_write"],
+ )
+ @pytest.mark.asyncio
+ async def test_shutdown_stops_a_batch_before_its_load(
+ self,
+ shutdown_during_executing_write: bool,
+ expected_states: list[str],
+ expected_marked_attempts: list[int],
+ ):
+ consumer = _make_consumer()
+ batch = _make_batch(latest_attempt=1)
+ consumer._process_batch = AsyncMock()
+ states: list[str] = []
+
+ async def track_status(conn, *, batch_id, job_state, attempt, **kwargs):
+ states.append(job_state)
+ if shutdown_during_executing_write:
+ consumer._shutdown.set()
+ return True
+
+ if not shutdown_during_executing_write:
+ consumer._shutdown.set()
+
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ side_effect=track_status,
+ ),
+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),
+ patch.object(
+ consumer_module.BatchQueue, "mark_shutdown_interruptions", new_callable=AsyncMock, return_value=1
+ ) as mock_mark,
+ ):
+ succeeded = await consumer._process_single(batch)
+
+ assert succeeded is False
+ consumer._process_batch.assert_not_called()
+ assert states == expected_states
+ assert [a.attempt for c in mock_mark.call_args_list for a in c.kwargs["attempts"]] == expected_marked_attempts
+ assert consumer._executing == {}
@pytest.mark.asyncio
async def test_non_reportable_error_skips_error_tracking_even_at_max_attempts(self):
@@ -676,10 +798,59 @@ class TestRecoverySweep:
consumer._recovery_conn, batches=[stale_batch], owner_token=consumer._owner_token, skip_locked=False
)
+ @pytest.mark.parametrize(
+ "latest_attempt,history,expected_error,expected_fail_reason,expected_metric",
+ [
+ (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None),
+ (
+ 3,
+ InterruptionHistory(shutdown_marked=True),
+ {"error": "executing interrupted by worker shutdown", "interrupted": True},
+ None,
+ "recovered",
+ ),
+ # Earlier shutdowns do not use the retry budget of a later unmarked loss.
+ (5, InterruptionHistory(recovered=3), {"error": "executing timed out - pod restart or OOM"}, None, None),
+ (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None),
+ (
+ 10,
+ InterruptionHistory(recovered=9, shutdown_marked=True),
+ {"error": "executing interrupted by worker shutdown", "interrupted": True},
+ None,
+ "recovered",
+ ),
+ (
+ 11,
+ InterruptionHistory(recovered=10, shutdown_marked=True),
+ None,
+ "max interruptions exceeded (loading stopped 11 times by a worker shutdown)",
+ "cap_exceeded",
+ ),
+ ],
+ ids=[
+ "unmarked_at_max_retries",
+ "shutdown_at_max_retries",
+ "unmarked_after_interruptions",
+ "unmarked_errors_after_interruptions",
+ "shutdown_at_interruption_cap",
+ "shutdown_past_interruption_cap",
+ ],
+ )
@pytest.mark.asyncio
- async def test_fails_exhausted_stale_batch(self):
+ async def test_shutdown_interruptions_use_their_own_cap(
+ self,
+ interruption_history: AsyncMock,
+ latest_attempt: int,
+ history: InterruptionHistory,
+ expected_error: dict[str, Any] | None,
+ expected_fail_reason: str | None,
+ expected_metric: str | None,
+ ):
consumer = _make_consumer(max_attempts=3)
- stale_batch = _make_batch(latest_attempt=3)
+ stale_batch = _make_batch(latest_attempt=latest_attempt, created_at=datetime(2026, 1, 1, tzinfo=UTC))
+ interruption_history.return_value = history
+ outcomes = ("recovered", "cap_exceeded")
+ before = {o: consumer._metrics.batch_interruptions_total.labels(outcome=o)._value.get() for o in outcomes}
with (
patch(
@@ -687,6 +858,10 @@ class TestRecoverySweep:
new_callable=AsyncMock,
return_value=[stale_batch],
),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ new_callable=AsyncMock,
+ ) as mock_status,
patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,
patch(
"products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",
@@ -695,10 +870,64 @@ class TestRecoverySweep:
):
await consumer._recovery_sweep()
- mock_fail.assert_called_once()
- assert "max retries exceeded" in mock_fail.call_args[1]["reason"]
+ interruption_history.assert_awaited_once_with(
+ consumer._recovery_conn,
+ batch_id=stale_batch.id,
+ attempt=latest_attempt,
+ )
+ if expected_fail_reason is None:
+ mock_fail.assert_not_called()
+ assert mock_status.await_count == 1
+ # The re-queue keeps the attempt, so the next delivery still runs the redelivery checks.
+ assert mock_status.call_args.kwargs["attempt"] == latest_attempt
+ assert mock_status.call_args.kwargs["error_response"] == expected_error
+ else:
+ mock_status.assert_not_called()
+ mock_fail.assert_awaited_once()
+ assert mock_fail.call_args.kwargs["reason"] == expected_fail_reason
+ for outcome in outcomes:
+ delta = consumer._metrics.batch_interruptions_total.labels(outcome=outcome)._value.get() - before[outcome]
+ assert delta == (1 if outcome == expected_metric else 0)
mock_unlock.assert_called_once()
+ @pytest.mark.asyncio
+ async def test_each_stale_batch_is_judged_on_its_own_history(self, interruption_history: AsyncMock):
+ consumer = _make_consumer(max_attempts=3)
+ interrupted = _make_batch(id="00000000-0000-0000-0000-00000000000a", latest_attempt=3)
+ crashed = _make_batch(
+ id="00000000-0000-0000-0000-00000000000b", schema_id="schema-2", run_uuid="run-2", latest_attempt=3
+ )
+ histories = {
+ interrupted.id: InterruptionHistory(shutdown_marked=True),
+ crashed.id: InterruptionHistory(),
+ }
+
+ async def history_for(conn, *, batch_id, **kwargs):
+ return histories[batch_id]
+
+ interruption_history.side_effect = history_for
+
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_stale_executing",
+ new_callable=AsyncMock,
+ return_value=[interrupted, crashed],
+ ),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ new_callable=AsyncMock,
+ ) as mock_status,
+ patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",
+ new_callable=AsyncMock,
+ ),
+ ):
+ await consumer._recovery_sweep()
+
+ assert [c.kwargs["batch_id"] for c in mock_status.call_args_list] == [interrupted.id]
+ assert [c.args[0].run_uuid for c in mock_fail.call_args_list] == ["run-2"]
+
@pytest.mark.asyncio
async def test_sweep_continues_past_terminally_retired_batch(self):
# A batch failed between the stale scan and the requeue must be skipped,
@@ -4119,6 +4348,97 @@ class TestShutdown:
assert drained.is_set()
assert len(consumer._in_flight) == 0
+ @pytest.mark.asyncio
+ async def test_close_marks_the_attempt_of_a_batch_still_loading(self):
+ consumer = _make_consumer()
+ batch = _make_batch(latest_attempt=1, created_at=datetime(2026, 1, 1, tzinfo=UTC))
+ loading = asyncio.Event()
+
+ async def load_forever(*args: Any) -> None:
+ loading.set()
+ await asyncio.Event().wait()
+
+ consumer._process_batch = load_forever
+ mark_conn = _make_healthy_conn()
+ before = consumer._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown")._value.get()
+
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ new_callable=AsyncMock,
+ return_value=True,
+ ),
+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),
+ patch.object(consumer._adapter, "release_all_owned", new_callable=AsyncMock),
+ patch.object(consumer, "_connect", new_callable=AsyncMock, return_value=mark_conn),
+ patch.object(
+ consumer_module.BatchQueue, "mark_shutdown_interruptions", new_callable=AsyncMock, return_value=1
+ ) as mock_mark,
+ patch.object(batch_consumer_module, "SHUTDOWN_DRAIN_TIMEOUT_SECONDS", 0.01),
+ ):
+
+ async def process_single() -> None:
+ await consumer._process_single(batch)
+
+ consumer._in_flight[(1, "schema-1")] = asyncio.create_task(process_single())
+ consumer._metrics.active_groups.inc()
+ await asyncio.wait_for(loading.wait(), timeout=5.0)
+ await consumer._close()
+
+ mock_mark.assert_awaited_once_with(
+ mark_conn,
+ attempts=[ExecutingAttempt(batch_id=batch.id, batch_created_at=batch.created_at, attempt=2)],
+ )
+ mark_conn.close.assert_awaited_once()
+ assert consumer._executing == {}
+ after = consumer._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown")._value.get()
+ assert after - before == 1
+
+ @pytest.mark.parametrize(
+ "executing,connect_error,mark_error,expect_mark",
+ [
+ (False, None, None, False),
+ (True, None, None, True),
+ (True, psycopg.OperationalError("connection refused"), None, False),
+ (True, None, psycopg.OperationalError("server closed the connection"), True),
+ ],
+ ids=["nothing_executing", "marks", "connect_fails", "mark_fails"],
+ )
+ @pytest.mark.asyncio
+ async def test_close_marking_is_best_effort(
+ self,
+ executing: bool,
+ connect_error: Exception | None,
+ mark_error: Exception | None,
+ expect_mark: bool,
+ ):
+ consumer = _make_consumer()
+ if executing:
+ consumer._executing["batch-a"] = ExecutingAttempt(batch_id="batch-a", batch_created_at=None, attempt=1)
+
+ with (
+ patch.object(consumer._adapter, "release_all_owned", new_callable=AsyncMock) as mock_release_all,
+ patch.object(
+ consumer,
+ "_connect",
+ new_callable=AsyncMock,
+ return_value=_make_healthy_conn(),
+ side_effect=connect_error,
+ ) as mock_connect,
+ patch.object(
+ consumer_module.BatchQueue,
+ "mark_shutdown_interruptions",
+ new_callable=AsyncMock,
+ return_value=1,
+ side_effect=mark_error,
+ ) as mock_mark,
+ ):
+ await consumer._close()
+
+ assert mock_connect.called is executing
+ assert mock_mark.called is expect_mark
+ mock_release_all.assert_awaited_once()
+
class TestDispatchGroups:
def test_fetch_limit_caps_to_free_slots_and_poll_limit(self):
@@ -4731,6 +5051,25 @@ class TestProcessGroupCoalescing:
process_batch.assert_not_called()
assert statuses == [*[(b.id, "executing", 1) for b in batches], *[(b.id, "succeeded", 1) for b in batches]]
+ @pytest.mark.asyncio
+ async def test_every_member_of_a_loading_set_is_marked_on_shutdown(self):
+ # A shutdown cuts off every member of the set, so each one must carry the marker.
+ executing_while_loading: list[ExecutingAttempt] = []
+ process_batch = AsyncMock()
+
+ async def process_batches(batches, verify_ownership=None):
+ executing_while_loading.extend(consumer._executing.values())
+
+ consumer = self._consumer(process_batches, process_batch)
+ batches = _run_batches(3)
+
+ await self._run_group(consumer, batches)
+
+ assert executing_while_loading == [
+ ExecutingAttempt(batch_id=b.id, batch_created_at=b.created_at, attempt=1) for b in batches
+ ]
+ assert consumer._executing == {}
+
@pytest.mark.parametrize(
"error,expected_latest_attempt,expected_status_attempt",
[
@@ -4770,6 +5109,29 @@ class TestProcessGroupCoalescing:
(batches[1].id, "succeeded", expected_status_attempt),
]
+ @pytest.mark.parametrize(
+ "error",
+ [CoalescingDeclined("schemas differ"), RuntimeError("merge blew up")],
+ ids=["declined", "failed"],
+ )
+ @pytest.mark.asyncio
+ async def test_a_set_that_cannot_load_during_shutdown_starts_no_new_attempts(self, error: Exception):
+ # The shutdown marker covers only the set's own attempts, so a fallback delivery that
+ # started in the drain would use a retry when the drain cancels it.
+ process_batch = AsyncMock()
+
+ async def process_batches(batches, verify_ownership=None):
+ consumer._shutdown.set()
+ raise error
+
+ consumer = self._consumer(process_batches, process_batch)
+ batches = _run_batches(2)
+
+ statuses = await self._run_group(consumer, batches)
+
+ process_batch.assert_not_called()
+ assert statuses == [(b.id, "executing", 1) for b in batches]
+
@pytest.mark.asyncio
async def test_a_set_whose_post_write_status_update_fails_falls_back_to_its_members(self):
# The set's write (process_batches) already landed; only the bookkeeping status update
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
index 89996073d59b..310164036f68 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
@@ -28,6 +28,8 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (
TAKEOVER_STALE_THRESHOLD_SECONDS,
BatchQueue,
ClaimCursor,
+ ExecutingAttempt,
+ InterruptionHistory,
PendingBatch,
QueueDepth,
_claim_window_sql,
@@ -2751,6 +2753,134 @@ class TestRecoverySweepVsLiveOwner:
assert row is not None and row[0] == "succeeded", "the sweep must not re-queue a batch its owner finished"
+async def _backdate_batch_log(conn: psycopg.AsyncConnection[Any], *, batch_id: str, age_seconds: int) -> None:
+ await conn.execute(
+ f"UPDATE {STATUS_TABLE} SET created_at = created_at - make_interval(secs => %s) WHERE batch_id = %s",
+ [age_seconds, batch_id],
+ )
+ await conn.execute(
+ f"UPDATE {BATCH_TABLE} SET state_changed_at = state_changed_at - make_interval(secs => %s) WHERE id = %s",
+ [age_seconds, batch_id],
+ )
+
+
+async def _batch_created_at(conn: psycopg.AsyncConnection[Any], batch_id: str) -> Any:
+ cur = await conn.execute(f"SELECT created_at FROM {BATCH_TABLE} WHERE id = %s", (batch_id,))
+ row = await cur.fetchone()
+ assert row is not None
+ return row[0]
+
+
[email protected]_db(transaction=True)
+class TestShutdownInterruptions:
+ @pytest.mark.parametrize(
+ "writes,marked_attempt,with_created_at,expected_marked",
+ [
+ ([("executing", 1)], 1, True, 1),
+ ([("executing", 1)], 1, False, 1),
+ # The batch finished during the drain: the marker must not make it executing again.
+ ([("executing", 1), ("succeeded", 1)], 1, True, 0),
+ ([("executing", 1), ("waiting_retry", 1)], 1, True, 0),
+ # Another consumer took the batch over at a later attempt.
+ ([("executing", 1), ("waiting_retry", 1), ("executing", 2)], 1, True, 0),
+ ],
+ ids=["executing", "executing_without_created_at", "succeeded", "waiting_retry", "taken_over"],
+ )
+ @pytest.mark.asyncio
+ async def test_marks_only_an_attempt_that_is_still_executing(
+ self,
+ conn,
+ writes: list[tuple[str, int]],
+ marked_attempt: int,
+ with_created_at: bool,
+ expected_marked: int,
+ ):
+ bid = await _insert_batch(conn)
+ for job_state, attempt in writes:
+ await BatchQueue.update_status(conn, batch_id=bid, job_state=job_state, attempt=attempt)
+ state_before = await _batch_state(conn, bid)
+ created_at = await _batch_created_at(conn, bid) if with_created_at else None
+
+ marked = await BatchQueue.mark_shutdown_interruptions(
+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=created_at, attempt=marked_attempt)]
+ )
+
+ assert marked == expected_marked
+ assert await _batch_state(conn, bid) == state_before
+ history = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=marked_attempt)
+ assert history.shutdown_marked is (expected_marked == 1)
+
+ @pytest.mark.asyncio
+ async def test_marker_keeps_the_attempt_start_as_its_time(self, conn):
+ # A marker dated now() would sort above an owner write that committed after it, and it
+ # would also restart the recovery grace clock of a batch that nothing runs any more.
+ bid = await _insert_batch(conn)
+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=1)
+ await _backdate_batch_log(conn, batch_id=bid, age_seconds=120)
+
+ marked = await BatchQueue.mark_shutdown_interruptions(
+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=None, attempt=1)]
+ )
+
+ assert marked == 1
+ stale = await BatchQueue.get_stale_executing(conn, grace_seconds=60)
+ assert [str(batch.id) for batch in stale] == [bid]
+
+ @pytest.mark.asyncio
+ async def test_history_counts_recovered_interruptions_of_one_batch_only(self, conn):
+ bid = await _insert_batch(conn)
+ other = await _insert_batch(conn, batch_index=1)
+ interrupted = {"error": "executing interrupted by worker shutdown", "interrupted": True}
+ for batch_id in (bid, other):
+ await BatchQueue.update_status(conn, batch_id=batch_id, job_state="executing", attempt=1)
+ await BatchQueue.mark_shutdown_interruptions(
+ conn, attempts=[ExecutingAttempt(batch_id=batch_id, batch_created_at=None, attempt=1)]
+ )
+ await BatchQueue.update_status(
+ conn, batch_id=batch_id, job_state="waiting_retry", attempt=1, error_response=interrupted
+ )
+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=2)
+ await BatchQueue.update_status(
+ conn, batch_id=bid, job_state="waiting_retry", attempt=2, error_response={"error": "boom"}
+ )
+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=3)
+
+ latest = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=3)
+ first = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=1)
+
+ assert latest == InterruptionHistory(recovered=1, shutdown_marked=False)
+ assert first == InterruptionHistory(recovered=1, shutdown_marked=True)
+
+ @pytest.mark.parametrize(
+ "shutdown_marked,expected_state",
+ [(True, "waiting_retry"), (False, "failed")],
+ ids=["shutdown_is_an_interruption", "unannounced_loss_uses_a_retry"],
+ )
+ @pytest.mark.asyncio
+ async def test_sweep_requeues_a_batch_at_max_attempts_only_after_a_shutdown(
+ self, conn, _db_url, shutdown_marked: bool, expected_state: str
+ ):
+ bid = await _insert_batch(conn)
+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=3)
+ if shutdown_marked:
+ await BatchQueue.mark_shutdown_interruptions(
+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=None, attempt=3)]
+ )
+ await _backdate_batch_log(conn, batch_id=bid, age_seconds=120)
+
+ consumer = BatchConsumer(
+ config=ConsumerConfig(database_url=_db_url, recovery_grace_seconds=60, max_attempts=3),
+ process_batch=AsyncMock(),
+ )
+ consumer._recovery_conn = conn
+ await consumer._recovery_sweep()
+
+ state = await _batch_state(conn, bid)
+ assert (state.state, state.attempt) == (expected_state, 3 if shutdown_marked else 0)
+ history = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=4)
+ assert history.recovered == (1 if shutdown_marked else 0)
+
+
@pytest.mark.django_db(transaction=True)
class TestGetFailedRuns:
"""The reconcile sweep judges candidacy from the denormalized batch columns
diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.md
index 206b817eb88c..c73d2418c771 100644
--- a/products/warehouse_sources_queue/backend/README.md
+++ b/products/warehouse_sources_queue/backend/README.md
@@ -53,6 +53,13 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/b
- **Three sweeps**, not one:
- **Recovery sweep** (every 30s, `get_stale_executing`): batches whose latest status is `executing`, older than the 300s grace window, with no live lease over their group.
The sweep deletes the expired lease first so a resurrecting owner cannot renew it, then re-queues the batch as `waiting_retry` (with a compare-and-swap on the observed `state_changed_at`) or fails the run at max attempts.
+ - **Interruptions are not retries.** A consumer that gets SIGTERM marks each attempt it is still running with an `executing` status row that carries `{"shutdown": true}` (`mark_shutdown_interruptions`), before it drains.
+ The sweep re-queues a marked attempt with `{"interrupted": true}`, and that attempt does not count against `max_attempts`. `max_interruptions` (10) caps it separately.
+ An attempt without a marker lost its consumer without notice, most often to an OOM kill, so it still uses a retry.
+ After SIGTERM, a group task starts no new load. An attempt that wrote `executing` while the shutdown began marks itself.
+ The marker row has the time of the attempt start, so it does not restart the grace clock.
+ If the consumer cannot read the interruption history, only `max_attempts + max_interruptions` deliveries fail the run.
+ The attempt number still goes up on each delivery, because the loader's redelivery checks read it.
- **Reconcile sweep** (every 300s, `reconcile_failed_runs` + `get_failed_runs`): finds runs with a `failed` queue batch whose `ExternalDataJob` was left non-terminal and marks the job failed.
It is single-flighted fleet-wide through a sentinel lease row (`try_acquire_reconcile_sweep_slot`, team_id 0 / `__reconcile-sweep__`, 240s slot TTL): the sweeps reconcile global state, so N pods running them concurrently is pure duplicated load, exactly when the queue DB can least afford it.
- **Stranded-run sweep** (same cadence and connection, `get_stale_stranded_runs`): runs the loader abandoned, meaning non-terminal batches, no live lease, and no loader progress for 6 hours.
diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.py
index f895119cccc1..0f474d7e6194 100644
--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py
+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py
@@ -25,7 +25,12 @@ from products.warehouse_sources_queue.backend.core.batch_phase import (
publish_phase_gauges,
track_batch_phases,
)
-from products.warehouse_sources_queue.backend.core.jobs_db import PendingBatch
+from products.warehouse_sources_queue.backend.core.jobs_db import (
+ INTERRUPTED_KEY,
+ ExecutingAttempt,
+ InterruptionHistory,
+ PendingBatch,
+)
from products.warehouse_sources_queue.backend.core.metrics import DELTA_CONSUMER_METRICS, ConsumerMetrics
logger = structlog.get_logger(__name__)
@@ -33,6 +38,10 @@ logger = structlog.get_logger(__name__)
T = TypeVar("T")
MAX_ATTEMPTS = 3
+# Recovered shutdowns a batch may take on top of its retries. A deploy or a node drain stops
+# every consumer pod in turn, so a slow batch can be cut off several times on a busy day.
+# The cap still ends a batch that makes its consumer stop every time it runs.
+MAX_INTERRUPTIONS = 10
POLL_INTERVAL_SECONDS = 2.0
RECOVERY_INTERVAL_SECONDS = 30.0
RETRY_BACKOFF_BASE_SECONDS = 15
@@ -48,6 +57,7 @@ GAUGE_SLOT_WARMUP_SECONDS = RECONCILE_INTERVAL_SECONDS
RECONCILE_LOOKBACK_SECONDS = 24 * 60 * 60 # wide enough to catch jobs orphaned by consumer outages
SHUTDOWN_DRAIN_TIMEOUT_SECONDS = 30.0
+SHUTDOWN_MARK_TIMEOUT_SECONDS = 5.0
GAUGE_SLOT_RELEASE_TIMEOUT_SECONDS = 3.0
# Cap on the jitter window between failed polls — flat retries make the whole
@@ -293,6 +303,7 @@ class BatchConsumerConfig:
database_url: str
max_concurrency: int = 16
max_attempts: int = MAX_ATTEMPTS
+ max_interruptions: int = MAX_INTERRUPTIONS
poll_interval_seconds: float = POLL_INTERVAL_SECONDS
poll_limit: int = 50
health_port: int = 8080
@@ -448,6 +459,26 @@ class BatchConsumerAdapter(Protocol):
keep_locks: bool = False,
) -> list[PendingBatch]: ...
+ async def mark_shutdown_interruptions(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ attempts: list[ExecutingAttempt],
+ ) -> int:
+ """Record that this consumer shut down while these attempts ran; returns the count marked.
+
+ Must mark only an attempt that is still executing. A sink that cannot record this returns 0,
+ and the recovery sweep then counts the attempt against the retry budget."""
+ ...
+
+ async def get_interruption_history(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ batch_id: str,
+ attempt: int,
+ ) -> InterruptionHistory: ...
+
async def reconcile_failed_runs(
self,
conn: psycopg.AsyncConnection[Any],
@@ -539,6 +570,8 @@ class BatchConsumer:
# batch_id -> monotonic start, for the stuck-batch watchdog.
self._inflight_started: dict[str, float] = {}
self._inflight_progress: dict[str, BatchPhaseProgress] = {}
+ # batch_id -> the attempt this consumer wrote as 'executing', for the shutdown marker.
+ self._executing: dict[str, ExecutingAttempt] = {}
self._last_stuck_log_monotonic = 0.0
# Consecutive failed polls, for the poll-failure liveness trip.
self._consecutive_poll_failures = 0
@@ -1424,18 +1457,22 @@ class BatchConsumer:
is_final_batch=batches[-1].is_final_batch,
resource_name=head.resource_name,
)
- for batch in batches:
- await self._adapter.update_status(
- status_conn,
- batch_id=batch.id,
- job_state=self._adapter.executing_state,
- attempt=attempts[batch.id],
- batch_created_at=batch.created_at,
- )
-
+ if await self._stop_for_shutdown(status_conn, batches, {}):
+ return False
heartbeat_task: asyncio.Task[None] | None = None
start = time.monotonic()
try:
+ for batch in batches:
+ await self._adapter.update_status(
+ status_conn,
+ batch_id=batch.id,
+ job_state=self._adapter.executing_state,
+ attempt=attempts[batch.id],
+ batch_created_at=batch.created_at,
+ )
+ self._track_executing(batch, attempts[batch.id])
+ if await self._stop_for_shutdown(status_conn, batches, attempts):
+ return False
if lock_conn is not None:
heartbeat_task = asyncio.create_task(self._set_heartbeat(lock_conn, batches, attempts))
try:
@@ -1519,6 +1556,8 @@ class BatchConsumer:
return await self._process_singly(batches, lock_conn, spent_attempt=True)
finally:
await self._stop_heartbeat(heartbeat_task)
+ for batch in batches:
+ self._executing.pop(batch.id, None)
async def _process_singly(
self, batches: list[PendingBatch], lock_conn: psycopg.AsyncConnection[Any] | None, *, spent_attempt: bool
@@ -1549,18 +1588,23 @@ class BatchConsumer:
attempt: int,
lock_conn: psycopg.AsyncConnection[Any] | None = None,
) -> bool:
+ status_conn = await self._get_status_conn(lock_conn)
+
if attempt > self._config.max_attempts:
- logger.error(
- self._event("batch_max_retries_exceeded"),
- batch_id=batch.id,
- run_uuid=batch.run_uuid,
- attempt=attempt,
+ spent, retries_used = await self._retry_budget_spent(
+ status_conn, batch, attempt, limit=self._config.max_attempts + 1
)
- await self._verify_ownership(lock_conn, batch)
- await self._fail_run(batch, reason=f"max retries exceeded (attempt {attempt})", conn=lock_conn)
- return False
-
- status_conn = await self._get_status_conn(lock_conn)
+ if spent:
+ logger.error(
+ self._event("batch_max_retries_exceeded"),
+ batch_id=batch.id,
+ run_uuid=batch.run_uuid,
+ attempt=attempt,
+ retries_used=retries_used,
+ )
+ await self._verify_ownership(lock_conn, batch)
+ await self._fail_run(batch, reason=f"max retries exceeded (attempt {attempt})", conn=lock_conn)
+ return False
logger.info(
self._event("batch_picked_up"),
@@ -1591,6 +1635,9 @@ class BatchConsumer:
self._metrics.batches_processed_total.labels(status="skipped").inc()
return False
+ if should_process and await self._stop_for_shutdown(status_conn, [batch], {}):
+ return False
+
# Pre-increment: if we OOM during processing, recovery sees attempt=N+1
# and knows this attempt was consumed.
await self._adapter.update_status(
@@ -1600,6 +1647,10 @@ class BatchConsumer:
attempt=attempt,
batch_created_at=batch.created_at,
)
+ self._track_executing(batch, attempt)
+
+ if should_process and await self._stop_for_shutdown(status_conn, [batch], {batch.id: attempt}):
+ return False
if should_process:
if lock_conn is not None:
@@ -1655,6 +1706,7 @@ class BatchConsumer:
await self._handle_batch_failure(batch, attempt, err, lock_conn=lock_conn, status_conn=status_conn)
return False
finally:
+ self._executing.pop(batch.id, None)
if heartbeat_task is not None:
heartbeat_task.cancel()
try:
@@ -1662,6 +1714,34 @@ class BatchConsumer:
except asyncio.CancelledError:
pass
+ def _track_executing(self, batch: PendingBatch, attempt: int) -> None:
+ self._executing[batch.id] = ExecutingAttempt(
+ batch_id=batch.id, batch_created_at=batch.created_at, attempt=attempt
+ )
+
+ async def _interruption_history(
+ self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch, attempt: int
+ ) -> InterruptionHistory:
+ return await self._adapter.get_interruption_history(conn, batch_id=batch.id, attempt=attempt)
+
+ async def _retry_budget_spent(
+ self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch, attempt: int, *, limit: int
+ ) -> tuple[bool, int | None]:
+ """Whether ``attempt`` used up ``limit`` retries, with the retries used (None when unknown).
+
+ The retries used are the attempt number minus the earlier deliveries that a consumer
+ shutdown cut off. When the history read fails, only the hard cap of ``limit`` plus
+ ``max_interruptions`` deliveries counts as spent. Thus a transient read error does not
+ fail the run, and a batch still cannot retry forever.
+ """
+ try:
+ history = await self._interruption_history(conn, batch, attempt)
+ except Exception as e:
+ logger.warning(self._event("interruption_history_read_failed"), batch_id=batch.id, error=str(e))
+ return attempt >= limit + self._config.max_interruptions, None
+ retries_used = max(0, attempt - history.recovered)
+ return retries_used >= limit, retries_used
+
async def _handle_batch_failure(
self,
batch: PendingBatch,
@@ -1711,7 +1791,10 @@ class BatchConsumer:
capture_exception(err)
await self._verify_ownership(lock_conn, batch)
await self._fail_run(batch, reason=reason, conn=lock_conn)
- elif attempt >= self._config.max_attempts:
+ elif (
+ attempt >= self._config.max_attempts
+ and (await self._retry_budget_spent(status_conn, batch, attempt, limit=self._config.max_attempts))[0]
+ ):
reason = f"max retries exceeded: {err}"
if isinstance(err, NonReportableError):
# A sustained blip (e.g. TransientObjectStoreError) stays out of error tracking
@@ -2008,30 +2091,7 @@ class BatchConsumer:
# Claim the corpse first: once the expired lease is gone, a resurrecting
# owner's renew matches nothing, so it can't finish over the re-queue.
await self._adapter.delete_expired_lease(conn, team_id=batch.team_id, schema_id=batch.schema_id)
- if batch.latest_attempt >= self._config.max_attempts:
- logger.warning(
- self._event("batch_recovered_max_retries_exceeded"), attempt=batch.latest_attempt
- )
- # fail_run's bulk write re-reads each batch's latest status in-statement and
- # targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.
- await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)
- else:
- try:
- await self._adapter.update_status(
- conn,
- batch_id=batch.id,
- job_state=self._adapter.waiting_retry_state,
- attempt=batch.latest_attempt,
- error_response={"error": "executing timed out - pod restart or OOM"},
- batch_created_at=batch.created_at,
- expected_state_changed_at=batch.state_changed_at,
- )
- except OwnershipLostError:
- # The state moved after the stale scan (owner finished, or the batch
- # went terminal): leave it, without aborting the rest of the sweep.
- logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)
- else:
- logger.warning(self._event("batch_recovered_for_retry"), attempt=batch.latest_attempt)
+ await self._recover_stale_batch(conn, batch)
finally:
structlog.contextvars.unbind_contextvars(*recovery_bound_keys)
finally:
@@ -2043,6 +2103,138 @@ class BatchConsumer:
except Exception:
logger.exception(self._event("recovery_sweep_unlock_failed"))
+ async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> None:
+ """Re-queue or fail one executing batch whose consumer stopped.
+
+ A consumer that shut down gracefully marked the attempt first. That attempt is an
+ interruption: it uses no retry, and only ``max_interruptions`` limits it. An unmarked
+ attempt lost its consumer without notice, which is most often an OOM kill caused by the
+ batch itself, so it uses a retry as a processing error does.
+ """
+ history = await self._interruption_history(conn, batch, batch.latest_attempt)
+ if history.shutdown_marked:
+ interruptions = history.recovered + 1
+ if interruptions > self._config.max_interruptions:
+ logger.warning(
+ self._event("batch_recovered_max_interruptions_exceeded"),
+ attempt=batch.latest_attempt,
+ interruptions=interruptions,
+ )
+ self._metrics.batch_interruptions_total.labels(outcome="cap_exceeded").inc()
+ await self._fail_run(
+ batch,
+ reason=f"max interruptions exceeded (loading stopped {interruptions} times by a worker shutdown)",
+ conn=conn,
+ )
+ return
+ error_response: dict[str, Any] = {
+ "error": "executing interrupted by worker shutdown",
+ INTERRUPTED_KEY: True,
+ }
+ recovered_event = "batch_recovered_after_interruption"
+ else:
+ retries_used = batch.latest_attempt - history.recovered
+ if retries_used >= self._config.max_attempts:
+ logger.warning(
+ self._event("batch_recovered_max_retries_exceeded"),
+ attempt=batch.latest_attempt,
+ retries_used=retries_used,
+ )
+ # fail_run's bulk write re-reads each batch's latest status in-statement and
+ # targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.
+ await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)
+ return
+ error_response = {"error": "executing timed out - pod restart or OOM"}
+ recovered_event = "batch_recovered_for_retry"
+ try:
+ await self._adapter.update_status(
+ conn,
+ batch_id=batch.id,
+ job_state=self._adapter.waiting_retry_state,
+ attempt=batch.latest_attempt,
+ error_response=error_response,
+ batch_created_at=batch.created_at,
+ expected_state_changed_at=batch.state_changed_at,
+ )
+ except OwnershipLostError:
+ # The state moved after the stale scan (owner finished, or the batch
+ # went terminal): leave it, without aborting the rest of the sweep.
+ logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)
+ return
+ if history.shutdown_marked:
+ self._metrics.batch_interruptions_total.labels(outcome="recovered").inc()
+ logger.warning(
+ self._event(recovered_event), attempt=batch.latest_attempt, interruptions=history.recovered + 1
+ )
+ else:
+ logger.warning(self._event(recovered_event), attempt=batch.latest_attempt)
+
+ async def _mark_shutdown_interruptions(self) -> None:
+ """Mark the attempts still running at shutdown, so the recovery sweep spends no retry on them.
+
+ Best-effort on a fresh connection, because a group task may be using its own connection
+ or the poll connection at this moment. An attempt that is not marked uses a retry.
+ """
+ attempts = list(self._executing.values())
+ if not attempts:
+ return
+ try:
+ async with asyncio.timeout(SHUTDOWN_MARK_TIMEOUT_SECONDS):
+ conn = await self._connect()
+ except Exception as e:
+ self._log_mark_failure(attempts, e)
+ return
+ try:
+ await self._write_shutdown_markers(conn, attempts)
+ finally:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(conn.close(), timeout=SHUTDOWN_MARK_TIMEOUT_SECONDS)
+
+ async def _stop_for_shutdown(
+ self, conn: psycopg.AsyncConnection[Any], batches: list[PendingBatch], attempts: dict[str, int]
+ ) -> bool:
+ """Return True when the consumer shuts down, so that the caller does not start a load.
+
+ ``attempts`` holds the attempts that the caller already wrote as 'executing'. This method
+ marks them, because the shutdown snapshot in ``_close`` can come before the caller tracks
+ them. Without the marker, the recovery sweep would count those attempts as retries.
+ """
+ if not self._shutdown.is_set():
+ return False
+ logger.info(
+ self._event("shutdown_before_load"),
+ batch_ids=[batch.id for batch in batches],
+ executing=len(attempts),
+ )
+ executing = [
+ ExecutingAttempt(batch_id=batch.id, batch_created_at=batch.created_at, attempt=attempts[batch.id])
+ for batch in batches
+ if batch.id in attempts
+ ]
+ if executing:
+ await self._write_shutdown_markers(conn, executing)
+ return True
+
+ async def _write_shutdown_markers(
+ self, conn: psycopg.AsyncConnection[Any], attempts: list[ExecutingAttempt]
+ ) -> None:
+ """Best-effort: an attempt that is not marked uses a retry, as before."""
+ try:
+ async with asyncio.timeout(SHUTDOWN_MARK_TIMEOUT_SECONDS):
+ marked = await self._adapter.mark_shutdown_interruptions(conn, attempts=attempts)
+ except Exception as e:
+ self._log_mark_failure(attempts, e)
+ return
+ self._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown").inc(marked)
+ logger.info(self._event("shutdown_interruptions_marked"), count=marked, executing=len(attempts))
+
+ def _log_mark_failure(self, attempts: list[ExecutingAttempt], error: Exception) -> None:
+ logger.warning(
+ self._event("shutdown_interruptions_mark_failed"),
+ count=len(attempts),
+ error=str(error) or type(error).__name__,
+ )
+
async def _release_queue_gauges_slot(self) -> None:
"""Best-effort: a slow or failing queue DB must never delay or fail the shutdown."""
conn = self._recovery_conn
@@ -2070,6 +2262,10 @@ class BatchConsumer:
except asyncio.CancelledError:
pass
+ # Before the drain: the marker has to land before the pod stops, and the drain can use
+ # up the termination grace period.
+ await self._mark_shutdown_interruptions()
+
# Free the gauge slot before the drain: no pod samples while the old holder drains.
await self._release_queue_gauges_slot()
diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.py
index a8a06b1582ad..5dd855cfa2bf 100644
--- a/products/warehouse_sources_queue/backend/core/jobs_db.py
+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py
@@ -28,6 +28,8 @@ from typing import Any
import psycopg
from psycopg.rows import dict_row
+from posthog.dataclasses import frozen
+
BATCH_TABLE = "sourcebatch"
STATUS_TABLE = "sourcebatchstatus"
STATUS_VIEW = "v_latest_source_batch_status"
@@ -139,6 +141,13 @@ FRESHNESS_WINDOW = f"{FRESHNESS_WINDOW_SECONDS} seconds"
# of tenants own the queue, large enough that one bursty tenant does not.
DEPTH_TOP_GROUPS = 5
+# Keys in a status row's error_response that keep interruptions apart from processing errors.
+# A consumer that shuts down while a batch runs writes SHUTDOWN_MARKER_KEY on an 'executing'
+# row of that attempt. The recovery sweep writes INTERRUPTED_KEY on the 'waiting_retry' row
+# that re-queues the batch, and each such row is one interruption that uses no retry.
+SHUTDOWN_MARKER_KEY = "shutdown"
+INTERRUPTED_KEY = "interrupted"
+
class _Unset:
"""Sentinel for ``update_status_unless_failed(expected_state_changed_at=...)``.
@@ -956,6 +965,75 @@ def _stale_executing_sql(scope_sql: str = "") -> str:
"""
+# FOR UPDATE OF b serializes the marker with the owner's own status writes, which lock the
+# same row. A batch that wrote 'succeeded' or 'waiting_retry' first fails the state check
+# when the lock is released, so the marker can never make a finished batch 'executing' again.
+# The marker keeps the batch's state and attempt, so the denormalized columns stay as they are.
+# The marker row is dated at the start of the attempt, not now(). An owner write that started
+# before the marker but committed after it would otherwise sort below the marker, and the latest
+# status would read 'executing' for a batch whose state column says 'succeeded'.
+MARK_SHUTDOWN_INTERRUPTIONS_SQL = f"""
+ WITH marks AS (
+ SELECT *
+ FROM unnest(%(batch_ids)s::uuid[], %(batch_created_ats)s::timestamptz[], %(attempts)s::int[])
+ AS m(batch_id, batch_created_at, attempt)
+ ),
+ target AS (
+ SELECT b.id, m.attempt, COALESCE(b.state_changed_at, now()) AS attempt_started_at
+ FROM marks m
+ JOIN {BATCH_TABLE} b ON b.id = m.batch_id
+ WHERE b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+ AND (m.batch_created_at IS NULL OR b.created_at = m.batch_created_at)
+ AND b.latest_state = 'executing'
+ AND b.latest_attempt = m.attempt
+ ORDER BY b.id
+ FOR UPDATE OF b
+ ),
+ ins AS (
+ INSERT INTO {STATUS_TABLE} (batch_id, job_state, attempt, exec_time, error_response, created_at)
+ SELECT t.id, 'executing', t.attempt, now(), %(error_response)s, t.attempt_started_at
+ FROM target t
+ RETURNING batch_id
+ )
+ SELECT count(*) FROM ins
+"""
+
+
+INTERRUPTION_HISTORY_SQL = f"""
+ SELECT
+ count(*) FILTER (
+ WHERE job_state = 'waiting_retry' AND error_response->>'{INTERRUPTED_KEY}' = 'true'
+ ) AS recovered,
+ COALESCE(bool_or(
+ job_state = 'executing'
+ AND attempt = %(attempt)s
+ AND error_response->>'{SHUTDOWN_MARKER_KEY}' = 'true'
+ ), false) AS shutdown_marked
+ FROM {STATUS_TABLE}
+ WHERE batch_id = %(batch_id)s
+ AND created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+"""
+
+
+@frozen
+class InterruptionHistory:
+ """What a batch's status log says about deliveries that a worker shutdown cut off."""
+
+ # Earlier deliveries that the recovery sweep re-queued as interruptions.
+ recovered: int = 0
+ # The consumer shut down gracefully while the asked-for attempt was running.
+ shutdown_marked: bool = False
+
+
+@frozen
+class ExecutingAttempt:
+ """A batch attempt that a consumer is running, as it was written to the status log."""
+
+ batch_id: str
+ batch_created_at: datetime | None
+ attempt: int
+
+
@dataclass(frozen=False, slots=True)
class ClaimCursor:
"""Where one consumer's next claim starts its team window.
@@ -1913,6 +1991,45 @@ class BatchQueue:
return [PendingBatch(**row) for row in rows]
+ @staticmethod
+ async def mark_shutdown_interruptions(
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ attempts: list[ExecutingAttempt],
+ ) -> int:
+ """Record that the consumer shut down while these attempts ran. Returns the count marked.
+
+ An attempt is marked only while its batch is still 'executing' at that attempt, so a
+ batch that finished, or that another consumer took over, is left alone.
+ """
+ if not attempts:
+ return 0
+ cursor = await conn.execute(
+ MARK_SHUTDOWN_INTERRUPTIONS_SQL,
+ {
+ "batch_ids": [a.batch_id for a in attempts],
+ "batch_created_ats": [a.batch_created_at for a in attempts],
+ "attempts": [a.attempt for a in attempts],
+ "error_response": json.dumps({SHUTDOWN_MARKER_KEY: True}),
+ },
+ )
+ row = await cursor.fetchone()
+ return int(row[0]) if row else 0
+
+ @staticmethod
+ async def get_interruption_history(
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ batch_id: str,
+ attempt: int,
+ ) -> InterruptionHistory:
+ """Count the batch's recovered interruptions, and whether a shutdown cut off ``attempt``."""
+ cursor = await conn.execute(INTERRUPTION_HISTORY_SQL, {"batch_id": batch_id, "attempt": attempt})
+ row = await cursor.fetchone()
+ if row is None:
+ return InterruptionHistory()
+ return InterruptionHistory(recovered=int(row[0]), shutdown_marked=bool(row[1]))
+
@staticmethod
async def fail_run(
conn: psycopg.AsyncConnection[Any],
diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.py
index 1b5bb9605683..503e2067c660 100644
--- a/products/warehouse_sources_queue/backend/core/metrics.py
+++ b/products/warehouse_sources_queue/backend/core/metrics.py
@@ -35,6 +35,15 @@ BATCH_RETRY_TOTAL = Counter(
labelnames=["attempt", "error_type"],
)
+# Batches a consumer shutdown cut off mid-run. "marked_on_shutdown" counts the attempts a
+# stopping consumer marked, "recovered" the sweep's re-queues that used no retry, and
+# "cap_exceeded" the runs failed because one batch was cut off too many times.
+BATCH_INTERRUPTIONS_TOTAL = Counter(
+ "warehouse_pg_consumer_batch_interruptions_total",
+ "Batch attempts cut off by a consumer shutdown, by outcome",
+ labelnames=["outcome"],
+)
+
RUNS_FAILED_TOTAL = Counter(
"warehouse_pg_consumer_runs_failed_total",
"Total runs fully failed via fail_run()",
@@ -302,6 +311,7 @@ class ConsumerMetrics:
batches_processed_total: Counter
batch_processing_duration_seconds: Histogram
batch_retry_total: Counter
+ batch_interruptions_total: Counter
runs_failed_total: Counter
poll_duration_seconds: Histogram
poll_batches_fetched: Histogram
@@ -354,6 +364,7 @@ DELTA_CONSUMER_METRICS = ConsumerMetrics(
batches_processed_total=BATCHES_PROCESSED_TOTAL,
batch_processing_duration_seconds=BATCH_PROCESSING_DURATION_SECONDS,
batch_retry_total=BATCH_RETRY_TOTAL,
+ batch_interruptions_total=BATCH_INTERRUPTIONS_TOTAL,
runs_failed_total=RUNS_FAILED_TOTAL,
poll_duration_seconds=POLL_DURATION_SECONDS,
poll_batches_fetched=POLL_BATCHES_FETCHED,
@@ -394,6 +405,11 @@ def make_consumer_metrics(prefix: str) -> ConsumerMetrics:
"Total batch retry attempts",
labelnames=["attempt", "error_type"],
),
+ batch_interruptions_total=Counter(
+ f"{p}_batch_interruptions_total",
+ "Batch attempts cut off by a consumer shutdown, by outcome",
+ labelnames=["outcome"],
+ ),
runs_failed_total=Counter(
f"{p}_runs_failed_total",
"Total runs fully failed via fail_run()",
diff --git a/products/warehouse_sources_queue/backend/sdk/jobs.py b/products/warehouse_sources_queue/backend/sdk/jobs.py
index 84e7806ea1b0..e06ed499aa25 100644
--- a/products/warehouse_sources_queue/backend/sdk/jobs.py
+++ b/products/warehouse_sources_queue/backend/sdk/jobs.py
@@ -32,6 +32,7 @@ from products.warehouse_sources_queue.backend.core.generic_jobs import (
Job,
JobsTable,
)
+from products.warehouse_sources_queue.backend.core.jobs_db import ExecutingAttempt, InterruptionHistory
from products.warehouse_sources_queue.backend.core.metrics import ConsumerMetrics, make_consumer_metrics
if TYPE_CHECKING:
@@ -332,6 +333,24 @@ class GenericJobAdapter:
limit=self._recovery_sweep_limit,
)
+ async def mark_shutdown_interruptions(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ attempts: list[ExecutingAttempt],
+ ) -> int:
+ # Generic jobs do not record shutdown markers yet, so every lost attempt uses a retry.
+ return 0
+
+ async def get_interruption_history(
+ self,
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ batch_id: str,
+ attempt: int,
+ ) -> InterruptionHistory:
+ return InterruptionHistory()
+
async def reconcile_failed_runs(
self,
conn: psycopg.AsyncConnection[Any],