:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): stop loader restarts from using up batch retries (#119747)
Tom Owers94c6b3990304
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.pyindex 1a7a36cdde1d..0a4d5e78b729 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py@@ -76,7 +76,9 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (TAKEOVER_STALE_THRESHOLD_SECONDS,BatchQueue,ClaimCursor,+ ExecutingAttempt,FailedRunRef,+ InterruptionHistory,PendingBatch,QueueDepth,_Unset,@@ -485,6 +487,25 @@ class DeltaBatchConsumerAdapter:exclude_sync_types=self._claim_exclude_sync_types,)+ async def mark_shutdown_interruptions(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ attempts: list[ExecutingAttempt],+ ) -> int:+ with observe_queue_query("mark_shutdown_interruptions"):+ return await BatchQueue.mark_shutdown_interruptions(conn, attempts=attempts)++ async def get_interruption_history(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ batch_id: str,+ attempt: int,+ ) -> InterruptionHistory:+ with observe_queue_query("get_interruption_history"):+ return await BatchQueue.get_interruption_history(conn, batch_id=batch_id, attempt=attempt)+async def reconcile_failed_runs(self,conn: psycopg.AsyncConnection[Any],diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex 7e34c12385ad..e7d2a9897d75 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -66,7 +66,9 @@ from products.warehouse_sources_queue.backend.core.batch_consumer import (from products.warehouse_sources_queue.backend.core.health import HealthStatefrom products.warehouse_sources_queue.backend.core.jobs_db import (FRESHNESS_WINDOW_SECONDS,+ ExecutingAttempt,FailedRunRef,+ InterruptionHistory,OrphanedRunRef,PendingBatch,QueueDepth,@@ -213,6 +215,18 @@ def _gauge_slot_granted():yield[email protected](autouse=True)+def interruption_history():+ # No earlier interruptions by default; the real SQL can't run against mock connections.+ with patch.object(+ consumer_module.BatchQueue,+ "get_interruption_history",+ new_callable=AsyncMock,+ return_value=InterruptionHistory(),+ ) as mock_history:+ yield mock_history++class TestProcessSingle:@pytest.mark.asyncioasync def test_success_updates_status_to_executing_then_succeeded(self):@@ -337,33 +351,141 @@ class TestProcessSingle:assert mock_fail.call_args[1]["reason"] == message # run still fails with the actionable messageassert mock_capture.called is expect_capture+ @pytest.mark.parametrize(+ "latest_attempt,recovered_interruptions,expect_fail",+ [+ (3, 0, True),+ # Three shutdowns plus one error: the batch has two retries left.+ (4, 3, False),+ (6, 3, True),+ (13, 10, True),+ ],+ ids=["errors_only", "interruptions_do_not_count", "errors_after_interruptions", "at_interruption_cap"],+ )@pytest.mark.asyncio- async def test_max_attempts_exceeded_fails_run(self):+ async def test_entry_check_counts_only_errors_against_max_attempts(+ self, interruption_history: AsyncMock, latest_attempt: int, recovered_interruptions: int, expect_fail: bool+ ):consumer = _make_consumer(max_attempts=3)- batch = _make_batch(latest_attempt=3)+ batch = _make_batch(latest_attempt=latest_attempt)+ interruption_history.return_value = InterruptionHistory(recovered=recovered_interruptions)+ consumer._process_batch = AsyncMock()- with patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail:- await consumer._process_single(batch)+ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ new_callable=AsyncMock,+ ),+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),+ patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,+ ):+ succeeded = await consumer._process_single(batch)- mock_fail.assert_called_once()- assert "max retries exceeded" in mock_fail.call_args[1]["reason"]+ assert mock_fail.called is expect_fail+ assert succeeded is not expect_fail+ assert consumer._process_batch.called is not expect_fail+ if expect_fail:+ assert "max retries exceeded" in mock_fail.call_args[1]["reason"]+ @pytest.mark.parametrize(+ "latest_attempt,recovered_interruptions,history_error,expect_fail",+ [+ (1, 0, None, True),+ (2, 1, None, True),+ # Three shutdowns came before this first error, so one retry is left.+ (3, 3, None, False),+ # A failed history read must not fail the run before the hard cap of retries plus interruptions.+ (3, 3, psycopg.OperationalError("server closed the connection"), False),+ (11, 10, psycopg.OperationalError("server closed the connection"), True),+ ],+ ids=[+ "errors_only",+ "errors_after_interruption",+ "interruptions_do_not_count",+ "history_read_fails_below_hard_cap",+ "history_read_fails_at_hard_cap",+ ],+ )@pytest.mark.asyncio- async def test_error_at_max_attempts_fails_run(self):+ async def test_error_at_max_attempts_fails_run(+ self,+ interruption_history: AsyncMock,+ latest_attempt: int,+ recovered_interruptions: int,+ history_error: Exception | None,+ expect_fail: bool,+ ):consumer = _make_consumer(max_attempts=2)- batch = _make_batch(latest_attempt=1)+ batch = _make_batch(latest_attempt=latest_attempt)consumer._process_batch = AsyncMock(side_effect=RuntimeError("crash"))+ interruption_history.return_value = InterruptionHistory(recovered=recovered_interruptions)+ interruption_history.side_effect = history_error+ states: list[str] = []++ async def track_status(conn, *, batch_id, job_state, attempt, **kwargs):+ states.append(job_state)+ return Truewith (patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",- new_callable=AsyncMock,+ side_effect=track_status,),+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,):await consumer._process_single(batch)- mock_fail.assert_called_once()+ assert mock_fail.called is expect_fail+ assert (SourceBatchStatus.State.WAITING_RETRY in states) is not expect_fail++ @pytest.mark.parametrize(+ "shutdown_during_executing_write,expected_states,expected_marked_attempts",+ [+ (False, [], []),+ # The shutdown snapshot can miss an attempt whose executing write is in flight.+ (True, ["executing"], [2]),+ ],+ ids=["before_executing_write", "during_executing_write"],+ )+ @pytest.mark.asyncio+ async def test_shutdown_stops_a_batch_before_its_load(+ self,+ shutdown_during_executing_write: bool,+ expected_states: list[str],+ expected_marked_attempts: list[int],+ ):+ consumer = _make_consumer()+ batch = _make_batch(latest_attempt=1)+ consumer._process_batch = AsyncMock()+ states: list[str] = []++ async def track_status(conn, *, batch_id, job_state, attempt, **kwargs):+ states.append(job_state)+ if shutdown_during_executing_write:+ consumer._shutdown.set()+ return True++ if not shutdown_during_executing_write:+ consumer._shutdown.set()++ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ side_effect=track_status,+ ),+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),+ patch.object(+ consumer_module.BatchQueue, "mark_shutdown_interruptions", new_callable=AsyncMock, return_value=1+ ) as mock_mark,+ ):+ succeeded = await consumer._process_single(batch)++ assert succeeded is False+ consumer._process_batch.assert_not_called()+ assert states == expected_states+ assert [a.attempt for c in mock_mark.call_args_list for a in c.kwargs["attempts"]] == expected_marked_attempts+ assert consumer._executing == {}@pytest.mark.asyncioasync def test_non_reportable_error_skips_error_tracking_even_at_max_attempts(self):@@ -676,10 +798,59 @@ class TestRecoverySweep:consumer._recovery_conn, batches=[stale_batch], owner_token=consumer._owner_token, skip_locked=False)+ @pytest.mark.parametrize(+ "latest_attempt,history,expected_error,expected_fail_reason,expected_metric",+ [+ (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None),+ (+ 3,+ InterruptionHistory(shutdown_marked=True),+ {"error": "executing interrupted by worker shutdown", "interrupted": True},+ None,+ "recovered",+ ),+ # Earlier shutdowns do not use the retry budget of a later unmarked loss.+ (5, InterruptionHistory(recovered=3), {"error": "executing timed out - pod restart or OOM"}, None, None),+ (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None),+ (+ 10,+ InterruptionHistory(recovered=9, shutdown_marked=True),+ {"error": "executing interrupted by worker shutdown", "interrupted": True},+ None,+ "recovered",+ ),+ (+ 11,+ InterruptionHistory(recovered=10, shutdown_marked=True),+ None,+ "max interruptions exceeded (loading stopped 11 times by a worker shutdown)",+ "cap_exceeded",+ ),+ ],+ ids=[+ "unmarked_at_max_retries",+ "shutdown_at_max_retries",+ "unmarked_after_interruptions",+ "unmarked_errors_after_interruptions",+ "shutdown_at_interruption_cap",+ "shutdown_past_interruption_cap",+ ],+ )@pytest.mark.asyncio- async def test_fails_exhausted_stale_batch(self):+ async def test_shutdown_interruptions_use_their_own_cap(+ self,+ interruption_history: AsyncMock,+ latest_attempt: int,+ history: InterruptionHistory,+ expected_error: dict[str, Any] | None,+ expected_fail_reason: str | None,+ expected_metric: str | None,+ ):consumer = _make_consumer(max_attempts=3)- stale_batch = _make_batch(latest_attempt=3)+ stale_batch = _make_batch(latest_attempt=latest_attempt, created_at=datetime(2026, 1, 1, tzinfo=UTC))+ interruption_history.return_value = history+ outcomes = ("recovered", "cap_exceeded")+ before = {o: consumer._metrics.batch_interruptions_total.labels(outcome=o)._value.get() for o in outcomes}with (patch(@@ -687,6 +858,10 @@ class TestRecoverySweep:new_callable=AsyncMock,return_value=[stale_batch],),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ new_callable=AsyncMock,+ ) as mock_status,patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",@@ -695,10 +870,64 @@ class TestRecoverySweep:):await consumer._recovery_sweep()- mock_fail.assert_called_once()- assert "max retries exceeded" in mock_fail.call_args[1]["reason"]+ interruption_history.assert_awaited_once_with(+ consumer._recovery_conn,+ batch_id=stale_batch.id,+ attempt=latest_attempt,+ )+ if expected_fail_reason is None:+ mock_fail.assert_not_called()+ assert mock_status.await_count == 1+ # The re-queue keeps the attempt, so the next delivery still runs the redelivery checks.+ assert mock_status.call_args.kwargs["attempt"] == latest_attempt+ assert mock_status.call_args.kwargs["error_response"] == expected_error+ else:+ mock_status.assert_not_called()+ mock_fail.assert_awaited_once()+ assert mock_fail.call_args.kwargs["reason"] == expected_fail_reason+ for outcome in outcomes:+ delta = consumer._metrics.batch_interruptions_total.labels(outcome=outcome)._value.get() - before[outcome]+ assert delta == (1 if outcome == expected_metric else 0)mock_unlock.assert_called_once()+ @pytest.mark.asyncio+ async def test_each_stale_batch_is_judged_on_its_own_history(self, interruption_history: AsyncMock):+ consumer = _make_consumer(max_attempts=3)+ interrupted = _make_batch(id="00000000-0000-0000-0000-00000000000a", latest_attempt=3)+ crashed = _make_batch(+ id="00000000-0000-0000-0000-00000000000b", schema_id="schema-2", run_uuid="run-2", latest_attempt=3+ )+ histories = {+ interrupted.id: InterruptionHistory(shutdown_marked=True),+ crashed.id: InterruptionHistory(),+ }++ async def history_for(conn, *, batch_id, **kwargs):+ return histories[batch_id]++ interruption_history.side_effect = history_for++ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_stale_executing",+ new_callable=AsyncMock,+ return_value=[interrupted, crashed],+ ),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ new_callable=AsyncMock,+ ) as mock_status,+ patch.object(consumer, "_fail_run", new_callable=AsyncMock) as mock_fail,+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",+ new_callable=AsyncMock,+ ),+ ):+ await consumer._recovery_sweep()++ assert [c.kwargs["batch_id"] for c in mock_status.call_args_list] == [interrupted.id]+ assert [c.args[0].run_uuid for c in mock_fail.call_args_list] == ["run-2"]+@pytest.mark.asyncioasync def test_sweep_continues_past_terminally_retired_batch(self):# A batch failed between the stale scan and the requeue must be skipped,@@ -4119,6 +4348,97 @@ class TestShutdown:assert drained.is_set()assert len(consumer._in_flight) == 0+ @pytest.mark.asyncio+ async def test_close_marks_the_attempt_of_a_batch_still_loading(self):+ consumer = _make_consumer()+ batch = _make_batch(latest_attempt=1, created_at=datetime(2026, 1, 1, tzinfo=UTC))+ loading = asyncio.Event()++ async def load_forever(*args: Any) -> None:+ loading.set()+ await asyncio.Event().wait()++ consumer._process_batch = load_forever+ mark_conn = _make_healthy_conn()+ before = consumer._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown")._value.get()++ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ new_callable=AsyncMock,+ return_value=True,+ ),+ patch.object(DeltaBatchConsumerAdapter, "should_process_batch", new_callable=AsyncMock, return_value=True),+ patch.object(consumer._adapter, "release_all_owned", new_callable=AsyncMock),+ patch.object(consumer, "_connect", new_callable=AsyncMock, return_value=mark_conn),+ patch.object(+ consumer_module.BatchQueue, "mark_shutdown_interruptions", new_callable=AsyncMock, return_value=1+ ) as mock_mark,+ patch.object(batch_consumer_module, "SHUTDOWN_DRAIN_TIMEOUT_SECONDS", 0.01),+ ):++ async def process_single() -> None:+ await consumer._process_single(batch)++ consumer._in_flight[(1, "schema-1")] = asyncio.create_task(process_single())+ consumer._metrics.active_groups.inc()+ await asyncio.wait_for(loading.wait(), timeout=5.0)+ await consumer._close()++ mock_mark.assert_awaited_once_with(+ mark_conn,+ attempts=[ExecutingAttempt(batch_id=batch.id, batch_created_at=batch.created_at, attempt=2)],+ )+ mark_conn.close.assert_awaited_once()+ assert consumer._executing == {}+ after = consumer._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown")._value.get()+ assert after - before == 1++ @pytest.mark.parametrize(+ "executing,connect_error,mark_error,expect_mark",+ [+ (False, None, None, False),+ (True, None, None, True),+ (True, psycopg.OperationalError("connection refused"), None, False),+ (True, None, psycopg.OperationalError("server closed the connection"), True),+ ],+ ids=["nothing_executing", "marks", "connect_fails", "mark_fails"],+ )+ @pytest.mark.asyncio+ async def test_close_marking_is_best_effort(+ self,+ executing: bool,+ connect_error: Exception | None,+ mark_error: Exception | None,+ expect_mark: bool,+ ):+ consumer = _make_consumer()+ if executing:+ consumer._executing["batch-a"] = ExecutingAttempt(batch_id="batch-a", batch_created_at=None, attempt=1)++ with (+ patch.object(consumer._adapter, "release_all_owned", new_callable=AsyncMock) as mock_release_all,+ patch.object(+ consumer,+ "_connect",+ new_callable=AsyncMock,+ return_value=_make_healthy_conn(),+ side_effect=connect_error,+ ) as mock_connect,+ patch.object(+ consumer_module.BatchQueue,+ "mark_shutdown_interruptions",+ new_callable=AsyncMock,+ return_value=1,+ side_effect=mark_error,+ ) as mock_mark,+ ):+ await consumer._close()++ assert mock_connect.called is executing+ assert mock_mark.called is expect_mark+ mock_release_all.assert_awaited_once()+class TestDispatchGroups:def test_fetch_limit_caps_to_free_slots_and_poll_limit(self):@@ -4731,6 +5051,25 @@ class TestProcessGroupCoalescing:process_batch.assert_not_called()assert statuses == [*[(b.id, "executing", 1) for b in batches], *[(b.id, "succeeded", 1) for b in batches]]+ @pytest.mark.asyncio+ async def test_every_member_of_a_loading_set_is_marked_on_shutdown(self):+ # A shutdown cuts off every member of the set, so each one must carry the marker.+ executing_while_loading: list[ExecutingAttempt] = []+ process_batch = AsyncMock()++ async def process_batches(batches, verify_ownership=None):+ executing_while_loading.extend(consumer._executing.values())++ consumer = self._consumer(process_batches, process_batch)+ batches = _run_batches(3)++ await self._run_group(consumer, batches)++ assert executing_while_loading == [+ ExecutingAttempt(batch_id=b.id, batch_created_at=b.created_at, attempt=1) for b in batches+ ]+ assert consumer._executing == {}+@pytest.mark.parametrize("error,expected_latest_attempt,expected_status_attempt",[@@ -4770,6 +5109,29 @@ class TestProcessGroupCoalescing:(batches[1].id, "succeeded", expected_status_attempt),]+ @pytest.mark.parametrize(+ "error",+ [CoalescingDeclined("schemas differ"), RuntimeError("merge blew up")],+ ids=["declined", "failed"],+ )+ @pytest.mark.asyncio+ async def test_a_set_that_cannot_load_during_shutdown_starts_no_new_attempts(self, error: Exception):+ # The shutdown marker covers only the set's own attempts, so a fallback delivery that+ # started in the drain would use a retry when the drain cancels it.+ process_batch = AsyncMock()++ async def process_batches(batches, verify_ownership=None):+ consumer._shutdown.set()+ raise error++ consumer = self._consumer(process_batches, process_batch)+ batches = _run_batches(2)++ statuses = await self._run_group(consumer, batches)++ process_batch.assert_not_called()+ assert statuses == [(b.id, "executing", 1) for b in batches]+@pytest.mark.asyncioasync def test_a_set_whose_post_write_status_update_fails_falls_back_to_its_members(self):# The set's write (process_batches) already landed; only the bookkeeping status updatediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.pyindex 89996073d59b..310164036f68 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py@@ -28,6 +28,8 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (TAKEOVER_STALE_THRESHOLD_SECONDS,BatchQueue,ClaimCursor,+ ExecutingAttempt,+ InterruptionHistory,PendingBatch,QueueDepth,_claim_window_sql,@@ -2751,6 +2753,134 @@ class TestRecoverySweepVsLiveOwner:assert row is not None and row[0] == "succeeded", "the sweep must not re-queue a batch its owner finished"+async def _backdate_batch_log(conn: psycopg.AsyncConnection[Any], *, batch_id: str, age_seconds: int) -> None:+ await conn.execute(+ f"UPDATE {STATUS_TABLE} SET created_at = created_at - make_interval(secs => %s) WHERE batch_id = %s",+ [age_seconds, batch_id],+ )+ await conn.execute(+ f"UPDATE {BATCH_TABLE} SET state_changed_at = state_changed_at - make_interval(secs => %s) WHERE id = %s",+ [age_seconds, batch_id],+ )+++async def _batch_created_at(conn: psycopg.AsyncConnection[Any], batch_id: str) -> Any:+ cur = await conn.execute(f"SELECT created_at FROM {BATCH_TABLE} WHERE id = %s", (batch_id,))+ row = await cur.fetchone()+ assert row is not None+ return row[0]++[email protected]_db(transaction=True)+class TestShutdownInterruptions:+ @pytest.mark.parametrize(+ "writes,marked_attempt,with_created_at,expected_marked",+ [+ ([("executing", 1)], 1, True, 1),+ ([("executing", 1)], 1, False, 1),+ # The batch finished during the drain: the marker must not make it executing again.+ ([("executing", 1), ("succeeded", 1)], 1, True, 0),+ ([("executing", 1), ("waiting_retry", 1)], 1, True, 0),+ # Another consumer took the batch over at a later attempt.+ ([("executing", 1), ("waiting_retry", 1), ("executing", 2)], 1, True, 0),+ ],+ ids=["executing", "executing_without_created_at", "succeeded", "waiting_retry", "taken_over"],+ )+ @pytest.mark.asyncio+ async def test_marks_only_an_attempt_that_is_still_executing(+ self,+ conn,+ writes: list[tuple[str, int]],+ marked_attempt: int,+ with_created_at: bool,+ expected_marked: int,+ ):+ bid = await _insert_batch(conn)+ for job_state, attempt in writes:+ await BatchQueue.update_status(conn, batch_id=bid, job_state=job_state, attempt=attempt)+ state_before = await _batch_state(conn, bid)+ created_at = await _batch_created_at(conn, bid) if with_created_at else None++ marked = await BatchQueue.mark_shutdown_interruptions(+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=created_at, attempt=marked_attempt)]+ )++ assert marked == expected_marked+ assert await _batch_state(conn, bid) == state_before+ history = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=marked_attempt)+ assert history.shutdown_marked is (expected_marked == 1)++ @pytest.mark.asyncio+ async def test_marker_keeps_the_attempt_start_as_its_time(self, conn):+ # A marker dated now() would sort above an owner write that committed after it, and it+ # would also restart the recovery grace clock of a batch that nothing runs any more.+ bid = await _insert_batch(conn)+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=1)+ await _backdate_batch_log(conn, batch_id=bid, age_seconds=120)++ marked = await BatchQueue.mark_shutdown_interruptions(+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=None, attempt=1)]+ )++ assert marked == 1+ stale = await BatchQueue.get_stale_executing(conn, grace_seconds=60)+ assert [str(batch.id) for batch in stale] == [bid]++ @pytest.mark.asyncio+ async def test_history_counts_recovered_interruptions_of_one_batch_only(self, conn):+ bid = await _insert_batch(conn)+ other = await _insert_batch(conn, batch_index=1)+ interrupted = {"error": "executing interrupted by worker shutdown", "interrupted": True}+ for batch_id in (bid, other):+ await BatchQueue.update_status(conn, batch_id=batch_id, job_state="executing", attempt=1)+ await BatchQueue.mark_shutdown_interruptions(+ conn, attempts=[ExecutingAttempt(batch_id=batch_id, batch_created_at=None, attempt=1)]+ )+ await BatchQueue.update_status(+ conn, batch_id=batch_id, job_state="waiting_retry", attempt=1, error_response=interrupted+ )+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=2)+ await BatchQueue.update_status(+ conn, batch_id=bid, job_state="waiting_retry", attempt=2, error_response={"error": "boom"}+ )+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=3)++ latest = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=3)+ first = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=1)++ assert latest == InterruptionHistory(recovered=1, shutdown_marked=False)+ assert first == InterruptionHistory(recovered=1, shutdown_marked=True)++ @pytest.mark.parametrize(+ "shutdown_marked,expected_state",+ [(True, "waiting_retry"), (False, "failed")],+ ids=["shutdown_is_an_interruption", "unannounced_loss_uses_a_retry"],+ )+ @pytest.mark.asyncio+ async def test_sweep_requeues_a_batch_at_max_attempts_only_after_a_shutdown(+ self, conn, _db_url, shutdown_marked: bool, expected_state: str+ ):+ bid = await _insert_batch(conn)+ await BatchQueue.update_status(conn, batch_id=bid, job_state="executing", attempt=3)+ if shutdown_marked:+ await BatchQueue.mark_shutdown_interruptions(+ conn, attempts=[ExecutingAttempt(batch_id=bid, batch_created_at=None, attempt=3)]+ )+ await _backdate_batch_log(conn, batch_id=bid, age_seconds=120)++ consumer = BatchConsumer(+ config=ConsumerConfig(database_url=_db_url, recovery_grace_seconds=60, max_attempts=3),+ process_batch=AsyncMock(),+ )+ consumer._recovery_conn = conn+ await consumer._recovery_sweep()++ state = await _batch_state(conn, bid)+ assert (state.state, state.attempt) == (expected_state, 3 if shutdown_marked else 0)+ history = await BatchQueue.get_interruption_history(conn, batch_id=bid, attempt=4)+ assert history.recovered == (1 if shutdown_marked else 0)++@pytest.mark.django_db(transaction=True)class TestGetFailedRuns:"""The reconcile sweep judges candidacy from the denormalized batch columnsdiff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.mdindex 206b817eb88c..c73d2418c771 100644--- a/products/warehouse_sources_queue/backend/README.md+++ b/products/warehouse_sources_queue/backend/README.md@@ -53,6 +53,13 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/b- **Three sweeps**, not one:- **Recovery sweep** (every 30s, `get_stale_executing`): batches whose latest status is `executing`, older than the 300s grace window, with no live lease over their group.The sweep deletes the expired lease first so a resurrecting owner cannot renew it, then re-queues the batch as `waiting_retry` (with a compare-and-swap on the observed `state_changed_at`) or fails the run at max attempts.+ - **Interruptions are not retries.** A consumer that gets SIGTERM marks each attempt it is still running with an `executing` status row that carries `{"shutdown": true}` (`mark_shutdown_interruptions`), before it drains.+ The sweep re-queues a marked attempt with `{"interrupted": true}`, and that attempt does not count against `max_attempts`. `max_interruptions` (10) caps it separately.+ An attempt without a marker lost its consumer without notice, most often to an OOM kill, so it still uses a retry.+ After SIGTERM, a group task starts no new load. An attempt that wrote `executing` while the shutdown began marks itself.+ The marker row has the time of the attempt start, so it does not restart the grace clock.+ If the consumer cannot read the interruption history, only `max_attempts + max_interruptions` deliveries fail the run.+ The attempt number still goes up on each delivery, because the loader's redelivery checks read it.- **Reconcile sweep** (every 300s, `reconcile_failed_runs` + `get_failed_runs`): finds runs with a `failed` queue batch whose `ExternalDataJob` was left non-terminal and marks the job failed.It is single-flighted fleet-wide through a sentinel lease row (`try_acquire_reconcile_sweep_slot`, team_id 0 / `__reconcile-sweep__`, 240s slot TTL): the sweeps reconcile global state, so N pods running them concurrently is pure duplicated load, exactly when the queue DB can least afford it.- **Stranded-run sweep** (same cadence and connection, `get_stale_stranded_runs`): runs the loader abandoned, meaning non-terminal batches, no live lease, and no loader progress for 6 hours.diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.pyindex f895119cccc1..0f474d7e6194 100644--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py@@ -25,7 +25,12 @@ from products.warehouse_sources_queue.backend.core.batch_phase import (publish_phase_gauges,track_batch_phases,)-from products.warehouse_sources_queue.backend.core.jobs_db import PendingBatch+from products.warehouse_sources_queue.backend.core.jobs_db import (+ INTERRUPTED_KEY,+ ExecutingAttempt,+ InterruptionHistory,+ PendingBatch,+)from products.warehouse_sources_queue.backend.core.metrics import DELTA_CONSUMER_METRICS, ConsumerMetricslogger = structlog.get_logger(__name__)@@ -33,6 +38,10 @@ logger = structlog.get_logger(__name__)T = TypeVar("T")MAX_ATTEMPTS = 3+# Recovered shutdowns a batch may take on top of its retries. A deploy or a node drain stops+# every consumer pod in turn, so a slow batch can be cut off several times on a busy day.+# The cap still ends a batch that makes its consumer stop every time it runs.+MAX_INTERRUPTIONS = 10POLL_INTERVAL_SECONDS = 2.0RECOVERY_INTERVAL_SECONDS = 30.0RETRY_BACKOFF_BASE_SECONDS = 15@@ -48,6 +57,7 @@ GAUGE_SLOT_WARMUP_SECONDS = RECONCILE_INTERVAL_SECONDSRECONCILE_LOOKBACK_SECONDS = 24 * 60 * 60 # wide enough to catch jobs orphaned by consumer outagesSHUTDOWN_DRAIN_TIMEOUT_SECONDS = 30.0+SHUTDOWN_MARK_TIMEOUT_SECONDS = 5.0GAUGE_SLOT_RELEASE_TIMEOUT_SECONDS = 3.0# Cap on the jitter window between failed polls — flat retries make the whole@@ -293,6 +303,7 @@ class BatchConsumerConfig:database_url: strmax_concurrency: int = 16max_attempts: int = MAX_ATTEMPTS+ max_interruptions: int = MAX_INTERRUPTIONSpoll_interval_seconds: float = POLL_INTERVAL_SECONDSpoll_limit: int = 50health_port: int = 8080@@ -448,6 +459,26 @@ class BatchConsumerAdapter(Protocol):keep_locks: bool = False,) -> list[PendingBatch]: ...+ async def mark_shutdown_interruptions(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ attempts: list[ExecutingAttempt],+ ) -> int:+ """Record that this consumer shut down while these attempts ran; returns the count marked.++ Must mark only an attempt that is still executing. A sink that cannot record this returns 0,+ and the recovery sweep then counts the attempt against the retry budget."""+ ...++ async def get_interruption_history(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ batch_id: str,+ attempt: int,+ ) -> InterruptionHistory: ...+async def reconcile_failed_runs(self,conn: psycopg.AsyncConnection[Any],@@ -539,6 +570,8 @@ class BatchConsumer:# batch_id -> monotonic start, for the stuck-batch watchdog.self._inflight_started: dict[str, float] = {}self._inflight_progress: dict[str, BatchPhaseProgress] = {}+ # batch_id -> the attempt this consumer wrote as 'executing', for the shutdown marker.+ self._executing: dict[str, ExecutingAttempt] = {}self._last_stuck_log_monotonic = 0.0# Consecutive failed polls, for the poll-failure liveness trip.self._consecutive_poll_failures = 0@@ -1424,18 +1457,22 @@ class BatchConsumer:is_final_batch=batches[-1].is_final_batch,resource_name=head.resource_name,)- for batch in batches:- await self._adapter.update_status(- status_conn,- batch_id=batch.id,- job_state=self._adapter.executing_state,- attempt=attempts[batch.id],- batch_created_at=batch.created_at,- )-+ if await self._stop_for_shutdown(status_conn, batches, {}):+ return Falseheartbeat_task: asyncio.Task[None] | None = Nonestart = time.monotonic()try:+ for batch in batches:+ await self._adapter.update_status(+ status_conn,+ batch_id=batch.id,+ job_state=self._adapter.executing_state,+ attempt=attempts[batch.id],+ batch_created_at=batch.created_at,+ )+ self._track_executing(batch, attempts[batch.id])+ if await self._stop_for_shutdown(status_conn, batches, attempts):+ return Falseif lock_conn is not None:heartbeat_task = asyncio.create_task(self._set_heartbeat(lock_conn, batches, attempts))try:@@ -1519,6 +1556,8 @@ class BatchConsumer:return await self._process_singly(batches, lock_conn, spent_attempt=True)finally:await self._stop_heartbeat(heartbeat_task)+ for batch in batches:+ self._executing.pop(batch.id, None)async def _process_singly(self, batches: list[PendingBatch], lock_conn: psycopg.AsyncConnection[Any] | None, *, spent_attempt: bool@@ -1549,18 +1588,23 @@ class BatchConsumer:attempt: int,lock_conn: psycopg.AsyncConnection[Any] | None = None,) -> bool:+ status_conn = await self._get_status_conn(lock_conn)+if attempt > self._config.max_attempts:- logger.error(- self._event("batch_max_retries_exceeded"),- batch_id=batch.id,- run_uuid=batch.run_uuid,- attempt=attempt,+ spent, retries_used = await self._retry_budget_spent(+ status_conn, batch, attempt, limit=self._config.max_attempts + 1)- await self._verify_ownership(lock_conn, batch)- await self._fail_run(batch, reason=f"max retries exceeded (attempt {attempt})", conn=lock_conn)- return False-- status_conn = await self._get_status_conn(lock_conn)+ if spent:+ logger.error(+ self._event("batch_max_retries_exceeded"),+ batch_id=batch.id,+ run_uuid=batch.run_uuid,+ attempt=attempt,+ retries_used=retries_used,+ )+ await self._verify_ownership(lock_conn, batch)+ await self._fail_run(batch, reason=f"max retries exceeded (attempt {attempt})", conn=lock_conn)+ return Falselogger.info(self._event("batch_picked_up"),@@ -1591,6 +1635,9 @@ class BatchConsumer:self._metrics.batches_processed_total.labels(status="skipped").inc()return False+ if should_process and await self._stop_for_shutdown(status_conn, [batch], {}):+ return False+# Pre-increment: if we OOM during processing, recovery sees attempt=N+1# and knows this attempt was consumed.await self._adapter.update_status(@@ -1600,6 +1647,10 @@ class BatchConsumer:attempt=attempt,batch_created_at=batch.created_at,)+ self._track_executing(batch, attempt)++ if should_process and await self._stop_for_shutdown(status_conn, [batch], {batch.id: attempt}):+ return Falseif should_process:if lock_conn is not None:@@ -1655,6 +1706,7 @@ class BatchConsumer:await self._handle_batch_failure(batch, attempt, err, lock_conn=lock_conn, status_conn=status_conn)return Falsefinally:+ self._executing.pop(batch.id, None)if heartbeat_task is not None:heartbeat_task.cancel()try:@@ -1662,6 +1714,34 @@ class BatchConsumer:except asyncio.CancelledError:pass+ def _track_executing(self, batch: PendingBatch, attempt: int) -> None:+ self._executing[batch.id] = ExecutingAttempt(+ batch_id=batch.id, batch_created_at=batch.created_at, attempt=attempt+ )++ async def _interruption_history(+ self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch, attempt: int+ ) -> InterruptionHistory:+ return await self._adapter.get_interruption_history(conn, batch_id=batch.id, attempt=attempt)++ async def _retry_budget_spent(+ self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch, attempt: int, *, limit: int+ ) -> tuple[bool, int | None]:+ """Whether ``attempt`` used up ``limit`` retries, with the retries used (None when unknown).++ The retries used are the attempt number minus the earlier deliveries that a consumer+ shutdown cut off. When the history read fails, only the hard cap of ``limit`` plus+ ``max_interruptions`` deliveries counts as spent. Thus a transient read error does not+ fail the run, and a batch still cannot retry forever.+ """+ try:+ history = await self._interruption_history(conn, batch, attempt)+ except Exception as e:+ logger.warning(self._event("interruption_history_read_failed"), batch_id=batch.id, error=str(e))+ return attempt >= limit + self._config.max_interruptions, None+ retries_used = max(0, attempt - history.recovered)+ return retries_used >= limit, retries_used+async def _handle_batch_failure(self,batch: PendingBatch,@@ -1711,7 +1791,10 @@ class BatchConsumer:capture_exception(err)await self._verify_ownership(lock_conn, batch)await self._fail_run(batch, reason=reason, conn=lock_conn)- elif attempt >= self._config.max_attempts:+ elif (+ attempt >= self._config.max_attempts+ and (await self._retry_budget_spent(status_conn, batch, attempt, limit=self._config.max_attempts))[0]+ ):reason = f"max retries exceeded: {err}"if isinstance(err, NonReportableError):# A sustained blip (e.g. TransientObjectStoreError) stays out of error tracking@@ -2008,30 +2091,7 @@ class BatchConsumer:# Claim the corpse first: once the expired lease is gone, a resurrecting# owner's renew matches nothing, so it can't finish over the re-queue.await self._adapter.delete_expired_lease(conn, team_id=batch.team_id, schema_id=batch.schema_id)- if batch.latest_attempt >= self._config.max_attempts:- logger.warning(- self._event("batch_recovered_max_retries_exceeded"), attempt=batch.latest_attempt- )- # fail_run's bulk write re-reads each batch's latest status in-statement and- # targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.- await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)- else:- try:- await self._adapter.update_status(- conn,- batch_id=batch.id,- job_state=self._adapter.waiting_retry_state,- attempt=batch.latest_attempt,- error_response={"error": "executing timed out - pod restart or OOM"},- batch_created_at=batch.created_at,- expected_state_changed_at=batch.state_changed_at,- )- except OwnershipLostError:- # The state moved after the stale scan (owner finished, or the batch- # went terminal): leave it, without aborting the rest of the sweep.- logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)- else:- logger.warning(self._event("batch_recovered_for_retry"), attempt=batch.latest_attempt)+ await self._recover_stale_batch(conn, batch)finally:structlog.contextvars.unbind_contextvars(*recovery_bound_keys)finally:@@ -2043,6 +2103,138 @@ class BatchConsumer:except Exception:logger.exception(self._event("recovery_sweep_unlock_failed"))+ async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> None:+ """Re-queue or fail one executing batch whose consumer stopped.++ A consumer that shut down gracefully marked the attempt first. That attempt is an+ interruption: it uses no retry, and only ``max_interruptions`` limits it. An unmarked+ attempt lost its consumer without notice, which is most often an OOM kill caused by the+ batch itself, so it uses a retry as a processing error does.+ """+ history = await self._interruption_history(conn, batch, batch.latest_attempt)+ if history.shutdown_marked:+ interruptions = history.recovered + 1+ if interruptions > self._config.max_interruptions:+ logger.warning(+ self._event("batch_recovered_max_interruptions_exceeded"),+ attempt=batch.latest_attempt,+ interruptions=interruptions,+ )+ self._metrics.batch_interruptions_total.labels(outcome="cap_exceeded").inc()+ await self._fail_run(+ batch,+ reason=f"max interruptions exceeded (loading stopped {interruptions} times by a worker shutdown)",+ conn=conn,+ )+ return+ error_response: dict[str, Any] = {+ "error": "executing interrupted by worker shutdown",+ INTERRUPTED_KEY: True,+ }+ recovered_event = "batch_recovered_after_interruption"+ else:+ retries_used = batch.latest_attempt - history.recovered+ if retries_used >= self._config.max_attempts:+ logger.warning(+ self._event("batch_recovered_max_retries_exceeded"),+ attempt=batch.latest_attempt,+ retries_used=retries_used,+ )+ # fail_run's bulk write re-reads each batch's latest status in-statement and+ # targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.+ await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)+ return+ error_response = {"error": "executing timed out - pod restart or OOM"}+ recovered_event = "batch_recovered_for_retry"+ try:+ await self._adapter.update_status(+ conn,+ batch_id=batch.id,+ job_state=self._adapter.waiting_retry_state,+ attempt=batch.latest_attempt,+ error_response=error_response,+ batch_created_at=batch.created_at,+ expected_state_changed_at=batch.state_changed_at,+ )+ except OwnershipLostError:+ # The state moved after the stale scan (owner finished, or the batch+ # went terminal): leave it, without aborting the rest of the sweep.+ logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)+ return+ if history.shutdown_marked:+ self._metrics.batch_interruptions_total.labels(outcome="recovered").inc()+ logger.warning(+ self._event(recovered_event), attempt=batch.latest_attempt, interruptions=history.recovered + 1+ )+ else:+ logger.warning(self._event(recovered_event), attempt=batch.latest_attempt)++ async def _mark_shutdown_interruptions(self) -> None:+ """Mark the attempts still running at shutdown, so the recovery sweep spends no retry on them.++ Best-effort on a fresh connection, because a group task may be using its own connection+ or the poll connection at this moment. An attempt that is not marked uses a retry.+ """+ attempts = list(self._executing.values())+ if not attempts:+ return+ try:+ async with asyncio.timeout(SHUTDOWN_MARK_TIMEOUT_SECONDS):+ conn = await self._connect()+ except Exception as e:+ self._log_mark_failure(attempts, e)+ return+ try:+ await self._write_shutdown_markers(conn, attempts)+ finally:+ with contextlib.suppress(Exception):+ await asyncio.wait_for(conn.close(), timeout=SHUTDOWN_MARK_TIMEOUT_SECONDS)++ async def _stop_for_shutdown(+ self, conn: psycopg.AsyncConnection[Any], batches: list[PendingBatch], attempts: dict[str, int]+ ) -> bool:+ """Return True when the consumer shuts down, so that the caller does not start a load.++ ``attempts`` holds the attempts that the caller already wrote as 'executing'. This method+ marks them, because the shutdown snapshot in ``_close`` can come before the caller tracks+ them. Without the marker, the recovery sweep would count those attempts as retries.+ """+ if not self._shutdown.is_set():+ return False+ logger.info(+ self._event("shutdown_before_load"),+ batch_ids=[batch.id for batch in batches],+ executing=len(attempts),+ )+ executing = [+ ExecutingAttempt(batch_id=batch.id, batch_created_at=batch.created_at, attempt=attempts[batch.id])+ for batch in batches+ if batch.id in attempts+ ]+ if executing:+ await self._write_shutdown_markers(conn, executing)+ return True++ async def _write_shutdown_markers(+ self, conn: psycopg.AsyncConnection[Any], attempts: list[ExecutingAttempt]+ ) -> None:+ """Best-effort: an attempt that is not marked uses a retry, as before."""+ try:+ async with asyncio.timeout(SHUTDOWN_MARK_TIMEOUT_SECONDS):+ marked = await self._adapter.mark_shutdown_interruptions(conn, attempts=attempts)+ except Exception as e:+ self._log_mark_failure(attempts, e)+ return+ self._metrics.batch_interruptions_total.labels(outcome="marked_on_shutdown").inc(marked)+ logger.info(self._event("shutdown_interruptions_marked"), count=marked, executing=len(attempts))++ def _log_mark_failure(self, attempts: list[ExecutingAttempt], error: Exception) -> None:+ logger.warning(+ self._event("shutdown_interruptions_mark_failed"),+ count=len(attempts),+ error=str(error) or type(error).__name__,+ )+async def _release_queue_gauges_slot(self) -> None:"""Best-effort: a slow or failing queue DB must never delay or fail the shutdown."""conn = self._recovery_conn@@ -2070,6 +2262,10 @@ class BatchConsumer:except asyncio.CancelledError:pass+ # Before the drain: the marker has to land before the pod stops, and the drain can use+ # up the termination grace period.+ await self._mark_shutdown_interruptions()+# Free the gauge slot before the drain: no pod samples while the old holder drains.await self._release_queue_gauges_slot()diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.pyindex a8a06b1582ad..5dd855cfa2bf 100644--- a/products/warehouse_sources_queue/backend/core/jobs_db.py+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py@@ -28,6 +28,8 @@ from typing import Anyimport psycopgfrom psycopg.rows import dict_row+from posthog.dataclasses import frozen+BATCH_TABLE = "sourcebatch"STATUS_TABLE = "sourcebatchstatus"STATUS_VIEW = "v_latest_source_batch_status"@@ -139,6 +141,13 @@ FRESHNESS_WINDOW = f"{FRESHNESS_WINDOW_SECONDS} seconds"# of tenants own the queue, large enough that one bursty tenant does not.DEPTH_TOP_GROUPS = 5+# Keys in a status row's error_response that keep interruptions apart from processing errors.+# A consumer that shuts down while a batch runs writes SHUTDOWN_MARKER_KEY on an 'executing'+# row of that attempt. The recovery sweep writes INTERRUPTED_KEY on the 'waiting_retry' row+# that re-queues the batch, and each such row is one interruption that uses no retry.+SHUTDOWN_MARKER_KEY = "shutdown"+INTERRUPTED_KEY = "interrupted"+class _Unset:"""Sentinel for ``update_status_unless_failed(expected_state_changed_at=...)``.@@ -956,6 +965,75 @@ def _stale_executing_sql(scope_sql: str = "") -> str:"""+# FOR UPDATE OF b serializes the marker with the owner's own status writes, which lock the+# same row. A batch that wrote 'succeeded' or 'waiting_retry' first fails the state check+# when the lock is released, so the marker can never make a finished batch 'executing' again.+# The marker keeps the batch's state and attempt, so the denormalized columns stay as they are.+# The marker row is dated at the start of the attempt, not now(). An owner write that started+# before the marker but committed after it would otherwise sort below the marker, and the latest+# status would read 'executing' for a batch whose state column says 'succeeded'.+MARK_SHUTDOWN_INTERRUPTIONS_SQL = f"""+ WITH marks AS (+ SELECT *+ FROM unnest(%(batch_ids)s::uuid[], %(batch_created_ats)s::timestamptz[], %(attempts)s::int[])+ AS m(batch_id, batch_created_at, attempt)+ ),+ target AS (+ SELECT b.id, m.attempt, COALESCE(b.state_changed_at, now()) AS attempt_started_at+ FROM marks m+ JOIN {BATCH_TABLE} b ON b.id = m.batch_id+ WHERE b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+ AND (m.batch_created_at IS NULL OR b.created_at = m.batch_created_at)+ AND b.latest_state = 'executing'+ AND b.latest_attempt = m.attempt+ ORDER BY b.id+ FOR UPDATE OF b+ ),+ ins AS (+ INSERT INTO {STATUS_TABLE} (batch_id, job_state, attempt, exec_time, error_response, created_at)+ SELECT t.id, 'executing', t.attempt, now(), %(error_response)s, t.attempt_started_at+ FROM target t+ RETURNING batch_id+ )+ SELECT count(*) FROM ins+"""+++INTERRUPTION_HISTORY_SQL = f"""+ SELECT+ count(*) FILTER (+ WHERE job_state = 'waiting_retry' AND error_response->>'{INTERRUPTED_KEY}' = 'true'+ ) AS recovered,+ COALESCE(bool_or(+ job_state = 'executing'+ AND attempt = %(attempt)s+ AND error_response->>'{SHUTDOWN_MARKER_KEY}' = 'true'+ ), false) AS shutdown_marked+ FROM {STATUS_TABLE}+ WHERE batch_id = %(batch_id)s+ AND created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+"""+++@frozen+class InterruptionHistory:+ """What a batch's status log says about deliveries that a worker shutdown cut off."""++ # Earlier deliveries that the recovery sweep re-queued as interruptions.+ recovered: int = 0+ # The consumer shut down gracefully while the asked-for attempt was running.+ shutdown_marked: bool = False+++@frozen+class ExecutingAttempt:+ """A batch attempt that a consumer is running, as it was written to the status log."""++ batch_id: str+ batch_created_at: datetime | None+ attempt: int++@dataclass(frozen=False, slots=True)class ClaimCursor:"""Where one consumer's next claim starts its team window.@@ -1913,6 +1991,45 @@ class BatchQueue:return [PendingBatch(**row) for row in rows]+ @staticmethod+ async def mark_shutdown_interruptions(+ conn: psycopg.AsyncConnection[Any],+ *,+ attempts: list[ExecutingAttempt],+ ) -> int:+ """Record that the consumer shut down while these attempts ran. Returns the count marked.++ An attempt is marked only while its batch is still 'executing' at that attempt, so a+ batch that finished, or that another consumer took over, is left alone.+ """+ if not attempts:+ return 0+ cursor = await conn.execute(+ MARK_SHUTDOWN_INTERRUPTIONS_SQL,+ {+ "batch_ids": [a.batch_id for a in attempts],+ "batch_created_ats": [a.batch_created_at for a in attempts],+ "attempts": [a.attempt for a in attempts],+ "error_response": json.dumps({SHUTDOWN_MARKER_KEY: True}),+ },+ )+ row = await cursor.fetchone()+ return int(row[0]) if row else 0++ @staticmethod+ async def get_interruption_history(+ conn: psycopg.AsyncConnection[Any],+ *,+ batch_id: str,+ attempt: int,+ ) -> InterruptionHistory:+ """Count the batch's recovered interruptions, and whether a shutdown cut off ``attempt``."""+ cursor = await conn.execute(INTERRUPTION_HISTORY_SQL, {"batch_id": batch_id, "attempt": attempt})+ row = await cursor.fetchone()+ if row is None:+ return InterruptionHistory()+ return InterruptionHistory(recovered=int(row[0]), shutdown_marked=bool(row[1]))+@staticmethodasync def fail_run(conn: psycopg.AsyncConnection[Any],diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.pyindex 1b5bb9605683..503e2067c660 100644--- a/products/warehouse_sources_queue/backend/core/metrics.py+++ b/products/warehouse_sources_queue/backend/core/metrics.py@@ -35,6 +35,15 @@ BATCH_RETRY_TOTAL = Counter(labelnames=["attempt", "error_type"],)+# Batches a consumer shutdown cut off mid-run. "marked_on_shutdown" counts the attempts a+# stopping consumer marked, "recovered" the sweep's re-queues that used no retry, and+# "cap_exceeded" the runs failed because one batch was cut off too many times.+BATCH_INTERRUPTIONS_TOTAL = Counter(+ "warehouse_pg_consumer_batch_interruptions_total",+ "Batch attempts cut off by a consumer shutdown, by outcome",+ labelnames=["outcome"],+)+RUNS_FAILED_TOTAL = Counter("warehouse_pg_consumer_runs_failed_total","Total runs fully failed via fail_run()",@@ -302,6 +311,7 @@ class ConsumerMetrics:batches_processed_total: Counterbatch_processing_duration_seconds: Histogrambatch_retry_total: Counter+ batch_interruptions_total: Counterruns_failed_total: Counterpoll_duration_seconds: Histogrampoll_batches_fetched: Histogram@@ -354,6 +364,7 @@ DELTA_CONSUMER_METRICS = ConsumerMetrics(batches_processed_total=BATCHES_PROCESSED_TOTAL,batch_processing_duration_seconds=BATCH_PROCESSING_DURATION_SECONDS,batch_retry_total=BATCH_RETRY_TOTAL,+ batch_interruptions_total=BATCH_INTERRUPTIONS_TOTAL,runs_failed_total=RUNS_FAILED_TOTAL,poll_duration_seconds=POLL_DURATION_SECONDS,poll_batches_fetched=POLL_BATCHES_FETCHED,@@ -394,6 +405,11 @@ def make_consumer_metrics(prefix: str) -> ConsumerMetrics:"Total batch retry attempts",labelnames=["attempt", "error_type"],),+ batch_interruptions_total=Counter(+ f"{p}_batch_interruptions_total",+ "Batch attempts cut off by a consumer shutdown, by outcome",+ labelnames=["outcome"],+ ),runs_failed_total=Counter(f"{p}_runs_failed_total","Total runs fully failed via fail_run()",diff --git a/products/warehouse_sources_queue/backend/sdk/jobs.py b/products/warehouse_sources_queue/backend/sdk/jobs.pyindex 84e7806ea1b0..e06ed499aa25 100644--- a/products/warehouse_sources_queue/backend/sdk/jobs.py+++ b/products/warehouse_sources_queue/backend/sdk/jobs.py@@ -32,6 +32,7 @@ from products.warehouse_sources_queue.backend.core.generic_jobs import (Job,JobsTable,)+from products.warehouse_sources_queue.backend.core.jobs_db import ExecutingAttempt, InterruptionHistoryfrom products.warehouse_sources_queue.backend.core.metrics import ConsumerMetrics, make_consumer_metricsif TYPE_CHECKING:@@ -332,6 +333,24 @@ class GenericJobAdapter:limit=self._recovery_sweep_limit,)+ async def mark_shutdown_interruptions(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ attempts: list[ExecutingAttempt],+ ) -> int:+ # Generic jobs do not record shutdown markers yet, so every lost attempt uses a retry.+ return 0++ async def get_interruption_history(+ self,+ conn: psycopg.AsyncConnection[Any],+ *,+ batch_id: str,+ attempt: int,+ ) -> InterruptionHistory:+ return InterruptionHistory()+async def reconcile_failed_runs(self,conn: psycopg.AsyncConnection[Any],