Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): split recovery sweeps by shutdown marker (#120472)

Tom Owerseef1dab50db1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index 4b7b90eac27f..ee27058a656e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -87,10 +87,12 @@ from products.warehouse_sources_queue.backend.core.metrics import (
OLDEST_UNCLAIMED_BATCH_SECONDS,
ORPHANED_BATCHES_DRAINED_TOTAL,
QUEUE_SAMPLE_GAUGES,
+ RECOVERY_SWEEP_OUTCOMES,
RUNS_RECONCILED_TOTAL,
SERIALIZED_BATCHES,
SLOT_WAITING_BATCHES,
TOP_GROUPS_CLAIMABLE_SHARE,
+ make_consumer_metrics,
)
from products.warehouse_sources_queue.backend.models import SourceBatchStatus
@@ -194,6 +196,21 @@ def _make_healthy_conn(closed: bool = False, broken: bool = False) -> AsyncMock:
return conn
+def _sweep_outcome_counts(consumer: BatchConsumer) -> dict[str, float]:
+ return {
+ outcome: consumer._metrics.recovery_sweeps_total.labels(outcome=outcome)._value.get()
+ for outcome in RECOVERY_SWEEP_OUTCOMES
+ }
+
+
+def _sweep_outcome_delta(before: dict[str, float], after: dict[str, float]) -> dict[str, float]:
+ return {
+ outcome: after[outcome] - before[outcome]
+ for outcome in RECOVERY_SWEEP_OUTCOMES
+ if after[outcome] != before[outcome]
+ }
+
+
@pytest.fixture(autouse=True)
def _lease_renewal_succeeds():
# Group dispatch renews the lease before processing; the real SQL can't run
@@ -799,25 +816,34 @@ class TestRecoverySweep:
)
@pytest.mark.parametrize(
- "latest_attempt,history,expected_error,expected_fail_reason,expected_metric",
+ "latest_attempt,history,expected_error,expected_fail_reason,expected_metric,expected_sweep_outcome",
[
- (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None),
+ (3, InterruptionHistory(), None, "max retries exceeded (likely OOM)", None, "orphans_found"),
(
3,
InterruptionHistory(shutdown_marked=True),
{"error": "executing interrupted by worker shutdown", "interrupted": True},
None,
"recovered",
+ "orphans_found_after_shutdown",
),
# Earlier shutdowns do not use the retry budget of a later unmarked loss.
- (5, InterruptionHistory(recovered=3), {"error": "executing timed out - pod restart or OOM"}, None, None),
- (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None),
+ (
+ 5,
+ InterruptionHistory(recovered=3),
+ {"error": "executing timed out - pod restart or OOM"},
+ None,
+ None,
+ "orphans_found",
+ ),
+ (6, InterruptionHistory(recovered=3), None, "max retries exceeded (likely OOM)", None, "orphans_found"),
(
10,
InterruptionHistory(recovered=9, shutdown_marked=True),
{"error": "executing interrupted by worker shutdown", "interrupted": True},
None,
"recovered",
+ "orphans_found_after_shutdown",
),
(
11,
@@ -825,6 +851,7 @@ class TestRecoverySweep:
None,
"max interruptions exceeded (loading stopped 11 times by a worker shutdown)",
"cap_exceeded",
+ "orphans_found_after_shutdown",
),
],
ids=[
@@ -845,12 +872,14 @@ class TestRecoverySweep:
expected_error: dict[str, Any] | None,
expected_fail_reason: str | None,
expected_metric: str | None,
+ expected_sweep_outcome: str,
):
consumer = _make_consumer(max_attempts=3)
stale_batch = _make_batch(latest_attempt=latest_attempt, created_at=datetime(2026, 1, 1, tzinfo=UTC))
interruption_history.return_value = history
outcomes = ("recovered", "cap_exceeded")
before = {o: consumer._metrics.batch_interruptions_total.labels(outcome=o)._value.get() for o in outcomes}
+ sweeps_before = _sweep_outcome_counts(consumer)
with (
patch(
@@ -888,6 +917,7 @@ class TestRecoverySweep:
for outcome in outcomes:
delta = consumer._metrics.batch_interruptions_total.labels(outcome=outcome)._value.get() - before[outcome]
assert delta == (1 if outcome == expected_metric else 0)
+ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {expected_sweep_outcome: 1}
mock_unlock.assert_called_once()
@pytest.mark.asyncio
@@ -906,6 +936,7 @@ class TestRecoverySweep:
return histories[batch_id]
interruption_history.side_effect = history_for
+ sweeps_before = _sweep_outcome_counts(consumer)
with (
patch(
@@ -927,6 +958,61 @@ class TestRecoverySweep:
assert [c.kwargs["batch_id"] for c in mock_status.call_args_list] == [interrupted.id]
assert [c.args[0].run_uuid for c in mock_fail.call_args_list] == ["run-2"]
+ # One unmarked orphan makes the whole sweep unexplained, so a crash next to a deploy still alerts.
+ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {"orphans_found": 1}
+
+ @pytest.mark.parametrize(
+ "failing_call",
+ ["history", "requeue"],
+ )
+ @pytest.mark.asyncio
+ async def test_sweep_that_stops_early_counts_as_unexplained(
+ self, interruption_history: AsyncMock, failing_call: str
+ ):
+ consumer = _make_consumer(max_attempts=3)
+ stale_batch = _make_batch(latest_attempt=1)
+ interruption_history.return_value = InterruptionHistory(shutdown_marked=True)
+ if failing_call == "history":
+ interruption_history.side_effect = psycopg.OperationalError("connection lost")
+ sweeps_before = _sweep_outcome_counts(consumer)
+
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_stale_executing",
+ new_callable=AsyncMock,
+ return_value=[stale_batch],
+ ),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.update_status_unless_failed",
+ new_callable=AsyncMock,
+ side_effect=psycopg.OperationalError("connection lost") if failing_call == "requeue" else None,
+ ),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.unlock_for_batches",
+ new_callable=AsyncMock,
+ ),
+ pytest.raises(psycopg.OperationalError),
+ ):
+ await consumer._recovery_sweep()
+
+ assert _sweep_outcome_delta(sweeps_before, _sweep_outcome_counts(consumer)) == {"orphans_found": 1}
+
+ def test_every_sweep_outcome_is_exported_before_the_first_sweep(self):
+ metrics = make_consumer_metrics("test_sweep_outcome_export")
+ batch_consumer_module.BatchConsumer(
+ config=ConsumerConfig(database_url="postgres://unused:unused@localhost/unused"),
+ process_batch=AsyncMock(),
+ adapter=MagicMock(),
+ metrics=metrics,
+ )
+
+ exported = {
+ sample.labels["outcome"]: sample.value
+ for family in metrics.recovery_sweeps_total.collect()
+ for sample in family.samples
+ if sample.name.endswith("_total")
+ }
+ assert exported == dict.fromkeys(RECOVERY_SWEEP_OUTCOMES, 0.0)
@pytest.mark.asyncio
async def test_sweep_continues_past_terminally_retired_batch(self):
diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.md
index 45731c04badc..67b3e28cc136 100644
--- a/products/warehouse_sources_queue/backend/README.md
+++ b/products/warehouse_sources_queue/backend/README.md
@@ -134,6 +134,7 @@ The overall topology (Temporal producer activity, two tables, a fleet of consume
The consumer engine emits nine `warehouse_pg_consumer_*` metrics (`metrics.py`): `batches_processed_total`, `batch_processing_duration_seconds`, `batch_retry_total`, `runs_failed_total`, `poll_duration_seconds`, `poll_batches_fetched`, `poll_failures_total`, `active_groups`, `recovery_sweeps_total`.
Other sinks get that nine-metric engine set under their own `{prefix}_pg_consumer_*` names via `make_consumer_metrics`, so dashboards and KEDA queries never conflate two consumers' series.
Further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total`, `runs_terminalized_stale_total` and `runs_failed_aged_out_total`.
+`recovery_sweeps_total` has three outcomes. `clean` means the sweep found no orphaned batch. `orphans_found_after_shutdown` means every orphan carried the shutdown marker of a graceful stop, such as a deploy. `orphans_found` means at least one orphan had no marker, so a consumer died without notice (an OOM kill or a crash) or lost its lease. Alerts read `orphans_found`.
The headline health signal is `warehouse_pg_queue_oldest_unclaimed_batch_seconds`: the age of the oldest batch no consumer has picked up yet.
It is the loader's data-freshness signal and rises whenever loading stalls, regardless of cause, so its alert fires even when every other signal looks green.
diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.py
index 0f474d7e6194..329554b197f7 100644
--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py
+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py
@@ -31,7 +31,11 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (
InterruptionHistory,
PendingBatch,
)
-from products.warehouse_sources_queue.backend.core.metrics import DELTA_CONSUMER_METRICS, ConsumerMetrics
+from products.warehouse_sources_queue.backend.core.metrics import (
+ DELTA_CONSUMER_METRICS,
+ RECOVERY_SWEEP_OUTCOMES,
+ ConsumerMetrics,
+)
logger = structlog.get_logger(__name__)
@@ -552,6 +556,9 @@ class BatchConsumer:
self._owner_token = str(uuid4())
self._health_reporter = health_reporter
self._metrics = metrics or DELTA_CONSUMER_METRICS
+ # Export every sweep outcome from pod start. Alerts read the after-shutdown series to know this build runs.
+ for outcome in RECOVERY_SWEEP_OUTCOMES:
+ self._metrics.recovery_sweeps_total.labels(outcome=outcome)
self._shutdown = asyncio.Event()
self._poll_conn: psycopg.AsyncConnection[Any] | None = None
self._recovery_conn: psycopg.AsyncConnection[Any] | None = None
@@ -2060,8 +2067,8 @@ class BatchConsumer:
conn = await self._ensure_recovery_conn()
- self._metrics.recovery_sweeps_total.labels(outcome="orphans_found").inc()
logger.info(self._event("recovery_sweep_found_stale_batches"), count=len(stale))
+ shutdown_marked_count = 0
recovery_bound_keys = (
"team_id",
@@ -2091,10 +2098,14 @@ class BatchConsumer:
# Claim the corpse first: once the expired lease is gone, a resurrecting
# owner's renew matches nothing, so it can't finish over the re-queue.
await self._adapter.delete_expired_lease(conn, team_id=batch.team_id, schema_id=batch.schema_id)
- await self._recover_stale_batch(conn, batch)
+ if await self._recover_stale_batch(conn, batch):
+ shutdown_marked_count += 1
finally:
structlog.contextvars.unbind_contextvars(*recovery_bound_keys)
finally:
+ # A sweep that stops early counts as unexplained, so an error can never hide a crash.
+ outcome = "orphans_found_after_shutdown" if shutdown_marked_count == len(stale) else "orphans_found"
+ self._metrics.recovery_sweeps_total.labels(outcome=outcome).inc()
# Release probe locks held by keep_locks=True (advisory sinks). For the
# lease sink this only deletes leases this pod owns, so it's a no-op for
# the abandoned groups recovered above.
@@ -2103,13 +2114,15 @@ class BatchConsumer:
except Exception:
logger.exception(self._event("recovery_sweep_unlock_failed"))
- async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> None:
+ async def _recover_stale_batch(self, conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> bool:
"""Re-queue or fail one executing batch whose consumer stopped.
A consumer that shut down gracefully marked the attempt first. That attempt is an
interruption: it uses no retry, and only ``max_interruptions`` limits it. An unmarked
attempt lost its consumer without notice, which is most often an OOM kill caused by the
batch itself, so it uses a retry as a processing error does.
+
+ Returns True when the attempt carries a shutdown marker.
"""
history = await self._interruption_history(conn, batch, batch.latest_attempt)
if history.shutdown_marked:
@@ -2126,7 +2139,7 @@ class BatchConsumer:
reason=f"max interruptions exceeded (loading stopped {interruptions} times by a worker shutdown)",
conn=conn,
)
- return
+ return True
error_response: dict[str, Any] = {
"error": "executing interrupted by worker shutdown",
INTERRUPTED_KEY: True,
@@ -2143,7 +2156,7 @@ class BatchConsumer:
# fail_run's bulk write re-reads each batch's latest status in-statement and
# targets only non-terminal ones, so it can't flip a meanwhile-succeeded batch.
await self._fail_run(batch, reason="max retries exceeded (likely OOM)", conn=conn)
- return
+ return False
error_response = {"error": "executing timed out - pod restart or OOM"}
recovered_event = "batch_recovered_for_retry"
try:
@@ -2160,7 +2173,7 @@ class BatchConsumer:
# The state moved after the stale scan (owner finished, or the batch
# went terminal): leave it, without aborting the rest of the sweep.
logger.info(self._event("batch_recovery_skipped_state_moved"), batch_id=batch.id)
- return
+ return history.shutdown_marked
if history.shutdown_marked:
self._metrics.batch_interruptions_total.labels(outcome="recovered").inc()
logger.warning(
@@ -2168,6 +2181,7 @@ class BatchConsumer:
)
else:
logger.warning(self._event(recovered_event), attempt=batch.latest_attempt)
+ return history.shutdown_marked
async def _mark_shutdown_interruptions(self) -> None:
"""Mark the attempts still running at shutdown, so the recovery sweep spends no retry on them.
diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.py
index 05bdcaca3087..62ab90033c13 100644
--- a/products/warehouse_sources_queue/backend/core/metrics.py
+++ b/products/warehouse_sources_queue/backend/core/metrics.py
@@ -75,6 +75,12 @@ ACTIVE_GROUPS = Gauge(
multiprocess_mode="livesum",
)
+# "orphans_found" counts the sweeps that found at least one executing batch without a shutdown
+# marker, which means a consumer died without notice (an OOM kill or a crash) or lost its lease.
+# "orphans_found_after_shutdown" counts the sweeps whose orphans all came from a graceful shutdown,
+# such as a deploy, a scale-down or a liveness restart. Alerts read "orphans_found".
+RECOVERY_SWEEP_OUTCOMES = ("clean", "orphans_found", "orphans_found_after_shutdown")
+
RECOVERY_SWEEPS_TOTAL = Counter(
"warehouse_pg_consumer_recovery_sweeps_total",
"Total recovery sweeps executed",