:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): split recovery sweeps by shutdown marker (#120472)
Tom Owerseef1dab50db1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex 4b7b90eac27f..ee27058a656e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -87,10 +87,12 @@ from products.warehouse_sources_queue.backend.core.metrics import (OLDEST_UNCLAIMED_BATCH_SECONDS,ORPHANED_BATCHES_DRAINED_TOTAL,QUEUE_SAMPLE_GAUGES,+ RECOVERY_SWEEP_OUTCOMES,RUNS_RECONCILED_TOTAL,SERIALIZED_BATCHES,SLOT_WAITING_BATCHES,TOP_GROUPS_CLAIMABLE_SHARE,+ make_consumer_metrics,)from products.warehouse_sources_queue.backend.models import SourceBatchStatus@@ -194,6 +196,21 @@ def _make_healthy_conn(closed: bool = False, broken: bool = False) -> AsyncMock:return conn+def _sweep_outcome_counts(consumer: BatchConsumer) -> dict[str, float]:+ return {+ outcome: consumer._metrics.recovery_sweeps_total.labels(outcome=outcome)._value.get()+ for outcome in RECOVERY_SWEEP_OUTCOMES+ }+++def _sweep_outcome_delta(before: dict[str, float], after: dict[str, float]) -> dict[str, float]:+ return {+ outcome: after[outcome] - before[outcome]+ for outcome in RECOVERY_SWEEP_OUTCOMES+ if after[outcome] != before[outcome]+ }++@pytest.fixture(autouse=True)def _lease_renewal_succeeds():# Group dispatch renews the lease before processing; the real SQL can't run@@ -799,25 +816,34 @@ class TestRecoverySweep:)@pytest.mark.parametrize(- "latest_attempt,history,expected_error,expected_fail_reason,expected_metric",+ "latest_attempt,history,expected_error,expected_fail_reason,expected_metric,expected_sweep_outcome",[- (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None),+ (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None, "orphans_found"),(3,InterruptionHistory(shutdown_marked=True),{"error": "executing interrupted by worker shutdown", "interrupted": True},None,"recovered",+ "orphans_found_after_shutdown",),# Earlier shutdowns do not use the retry budget of a later unmarked loss.- (5, InterruptionHistory(recovered=3), {"error": "executing timed out - pod restart or OOM"}, None, None),- (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None),+ (+ 5,+ InterruptionHistory(recovered=3),+ {"error": "executing timed out - pod restart or OOM"},+ None,+ None,+ "orphans_found",+ ),+ (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None, "orphans_found"),(10,InterruptionHistory(recovered=9, shutdown_marked=True),{"error": "executing interrupted by worker shutdown", "interrupted": True},None,"recovered",+ "orphans_found_after_shutdown",),(11,@@ -825,6 +851,7 @@ class TestRecoverySweep:None,"max interruptions exceeded (loading stopped 11 times by a worker shutdown)","cap_exceeded",+ "orphans_found_after_shutdown",),],ids=[@@ -845,12 +872,14 @@ class TestRecoverySweep:expected_error: dict[str, Any] | None,expected_fail_reason: str | None,expected_metric: str | None,+ expected_sweep_outcome: str,):consumer = _make_consumer(max_attempts=3)stale_batch = _make_batch(latest_attempt=latest_attempt, created_at=datetime(2026, 1, 1, tzinfo=UTC))interruption_history.return_value = historyoutcomes = ("recovered", "cap_exceeded")before = {o: consumer._metrics.batch_interruptions_total.labels(outcome=o)._value.get() for o in outcomes}+ sweeps_before = _sweep_outcome_counts(consumer)with (patch(@@ -888,6 +917,7 @@ class TestRecoverySweep:for outcome in outcomes:delta = consumer._metrics.batch_interruptions_total.labels(outcome=outcome)._value.get() - before[outcome]assert delta == (1 if outcome == expected_metric else 0)+ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {expected_sweep_outcome: 1}mock_unlock.assert_called_once()@pytest.mark.asyncio@@ -906,6 +936,7 @@ class TestRecoverySweep:return histories[batch_id]interruption_history.side_effect = history_for+ sweeps_before = _sweep_outcome_counts(consumer)with (patch(@@ -927,6 +958,61 @@ class TestRecoverySweep:assert [c.kwargs["batch_id"] for c in mock_status.call_args_list] == [interrupted.id]assert [c.args[0].run_uuid for c in mock_fail.call_args_list] == ["run-2"]+ # One unmarked orphan makes the whole sweep unexplained, so a crash next to a deploy still alerts.+ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {"orphans_found": 1}++ @pytest.mark.parametrize(+ "failing_call",+ ["history", "requeue"],+ )+ @pytest.mark.asyncio+ async def test_sweep_that_stops_early_counts_as_unexplained(+ self, interruption_history: AsyncMock, failing_call: str+ ):+ consumer = _make_consumer(max_attempts=3)+ stale_batch = _make_batch(latest_attempt=1)+ interruption_history.return_value = InterruptionHistory(shutdown_marked=True)+ if failing_call == "history":+ interruption_history.side_effect = psycopg.OperationalError("connection lost")+ sweeps_before = _sweep_outcome_counts(consumer)++ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_stale_executing",+ new_callable=AsyncMock,+ return_value=[stale_batch],+ ),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",+ new_callable=AsyncMock,+ side_effect=psycopg.OperationalError("connection lost") if failing_call == "requeue" else None,+ ),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",+ new_callable=AsyncMock,+ ),+ pytest.raises(psycopg.OperationalError),+ ):+ await consumer._recovery_sweep()++ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {"orphans_found": 1}++ def test_every_sweep_outcome_is_exported_before_the_first_sweep(self):+ metrics = make_consumer_metrics("test_sweep_outcome_export")+ batch_consumer_module.BatchConsumer(+ config=ConsumerConfig(database_url="postgres://unused:unused@localhost/unused"),+ process_batch=AsyncMock(),+ adapter=MagicMock(),+ metrics=metrics,+ )++ exported = {+ sample.labels["outcome"]: sample.value+ for family in metrics.recovery_sweeps_total.collect()+ for sample in family.samples+ if sample.name.endswith("_total")+ }+ assert exported == dict.fromkeys(RECOVERY_SWEEP_OUTCOMES, 0.0)@pytest.mark.asyncioasync def test_sweep_continues_past_terminally_retired_batch(self):diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.mdindex 45731c04badc..67b3e28cc136 100644--- a/products/warehouse_sources_queue/backend/README.md+++ b/products/warehouse_sources_queue/backend/README.md@@ -134,6 +134,7 @@ The overall topology (Temporal producer activity, two tables, a fleet of consumeThe consumer engine emits nine `warehouse_pg_consumer_*` metrics (`metrics.py`): `batches_processed_total`, `batch_processing_duration_seconds`, `batch_retry_total`, `runs_failed_total`, `poll_duration_seconds`, `poll_batches_fetched`, `poll_failures_total`, `active_groups`, `recovery_sweeps_total`.Other sinks get that nine-metric engine set under their own `{prefix}_pg_consumer_*` names via `make_consumer_metrics`, so dashboards and KEDA queries never conflate two consumers' series.Further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total`, `runs_terminalized_stale_total` and `runs_failed_aged_out_total`.+`recovery_sweeps_total` has three outcomes. `clean` means the sweep found no orphaned batch. `orphans_found_after_shutdown` means every orphan carried the shutdown marker of a graceful stop, such as a deploy. `orphans_found` means at least one orphan had no marker, so a consumer died without notice (an OOM kill or a crash) or lost its lease. Alerts read `orphans_found`.The headline health signal is `warehouse_pg_queue_oldest_unclaimed_batch_seconds`: the age of the oldest batch no consumer has picked up yet.It is the loader's data-freshness signal and rises whenever loading stalls, regardless of cause, so its alert fires even when every other signal looks green.diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.pyindex 0f474d7e6194..329554b197f7 100644--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py@@ -31,7 +31,11 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (InterruptionHistory,PendingBatch,)-from products.warehouse_sources_queue.backend.core.metrics import DELTA_CONSUMER_METRICS, ConsumerMetrics+from products.warehouse_sources_queue.backend.core.metrics import (+ DELTA_CONSUMER_METRICS,+ RECOVERY_SWEEP_OUTCOMES,+ ConsumerMetrics,+)logger = structlog.get_logger(__name__)@@ -552,6 +556,9 @@ class BatchConsumer:self._owner_token = str(uuid4())self._health_reporter = health_reporterself._metrics = metrics or DELTA_CONSUMER_METRICS+ # Export every sweep outcome from pod start. Alerts read the after-shutdown series to know this build runs.+ for outcome in RECOVERY_SWEEP_OUTCOMES:+ self._metrics.recovery_sweeps_total.labels(outcome=outcome)self._shutdown = asyncio.Event()self._poll_conn: psycopg.AsyncConnection[Any] | None = Noneself._recovery_conn: psycopg.AsyncConnection[Any] | None = None@@ -2060,8 +2067,8 @@ class BatchConsumer:conn = await self._ensure_recovery_conn()- self._metrics.recovery_sweeps_total.labels(outcome="orphans_found").inc()logger.info(self._event("recovery_sweep_found_stale_batches"), count=len(stale))+ shutdown_marked_count = 0recovery_bound_keys = ("team_id",@@ -2091,10 +2098,14 @@ class BatchConsumer:# Claim the corpse first: once the expired lease is gone, a resurrecting# owner's renew matches nothing, so it can't finish over the re-queue.await self._adapter.delete_expired_lease(conn, team_id=batch.team_id, schema_id=batch.schema_id)- await self._recover_stale_batch(conn, batch)+ if await self._recover_stale_batch(conn, batch):+ shutdown_marked_count += 1finally:structlog.contextvars.unbind_contextvars(*recovery_bound_keys)finally:+ # A sweep that stops early counts as unexplained, so an error can never hide a crash.+ outcome = "orphans_found_after_shutdown" if shutdown_marked_count == len(stale) else "orphans_found"+ self._metrics.recovery_sweeps_total.labels(outcome=outcome).inc()# Release probe locks held by keep_locks=True (advisory sinks). For the# lease sink this only deletes leases this pod owns, so it's a no-op for# the abandoned groups recovered above.@@ -2103,13 +2114,15 @@ class BatchConsumer:except Exception:logger.exception(self._event("recovery_sweep_unlock_failed"))- async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> None:+ async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> bool:"""Re-queue or fail one executing batch whose consumer stopped.A consumer that shut down gracefully marked the attempt first. That attempt is aninterruption: it uses no retry, and only ``max_interruptions`` limits it. An unmarkedattempt lost its consumer without notice, which is most often an OOM kill caused by thebatch itself, so it uses a retry as a processing error does.++ Returns True when the attempt carries a shutdown marker."""history = await self._interruption_history(conn, batch, batch.latest_attempt)if history.shutdown_marked:@@ -2126,7 +2139,7 @@ class BatchConsumer:reason=f"max interruptions exceeded (loading stopped {interruptions} times by a worker shutdown)",conn=conn,)- return+ return Trueerror_response: dict[str, Any] = {"error": "executing interrupted by worker shutdown",INTERRUPTED_KEY: True,@@ -2143,7 +2156,7 @@ class BatchConsumer:# fail_run's bulk write re-reads each batch's latest status in-statement and# targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)- return+ return Falseerror_response = {"error": "executing timed out - pod restart or OOM"}recovered_event = "batch_recovered_for_retry"try:@@ -2160,7 +2173,7 @@ class BatchConsumer:# The state moved after the stale scan (owner finished, or the batch# went terminal): leave it, without aborting the rest of the sweep.logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)- return+ return history.shutdown_markedif history.shutdown_marked:self._metrics.batch_interruptions_total.labels(outcome="recovered").inc()logger.warning(@@ -2168,6 +2181,7 @@ class BatchConsumer:)else:logger.warning(self._event(recovered_event), attempt=batch.latest_attempt)+ return history.shutdown_markedasync def _mark_shutdown_interruptions(self) -> None:"""Mark the attempts still running at shutdown, so the recovery sweep spends no retry on them.diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.pyindex 05bdcaca3087..62ab90033c13 100644--- a/products/warehouse_sources_queue/backend/core/metrics.py+++ b/products/warehouse_sources_queue/backend/core/metrics.py@@ -75,6 +75,12 @@ ACTIVE_GROUPS = Gauge(multiprocess_mode="livesum",)+# "orphans_found" counts the sweeps that found at least one executing batch without a shutdown+# marker, which means a consumer died without notice (an OOM kill or a crash) or lost its lease.+# "orphans_found_after_shutdown" counts the sweeps whose orphans all came from a graceful shutdown,+# such as a deploy, a scale-down or a liveness restart. Alerts read "orphans_found".+RECOVERY_SWEEP_OUTCOMES = ("clean", "orphans_found", "orphans_found_after_shutdown")+RECOVERY_SWEEPS_TOTAL = Counter("warehouse_pg_consumer_recovery_sweeps_total","Total recovery sweeps executed",