Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): reconnect the loader heartbeat after a closed connection (#120167)

Tom Owersa61bd1d20a91
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index 94407679fdaf..7e34c12385ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -80,6 +80,8 @@ from products.warehouse_sources_queue.backend.core.metrics import (
CLAIMABLE_GROUPS,
DEPTH_PROBE_TIMEOUTS_TOTAL,
DEPTH_SAMPLE_AGE_SECONDS,
+ HEARTBEAT_RECONNECT_FAILURES_TOTAL,
+ HEARTBEAT_RECONNECTS_TOTAL,
OLDEST_UNCLAIMED_BATCH_SECONDS,
ORPHANED_BATCHES_DRAINED_TOTAL,
QUEUE_SAMPLE_GAUGES,
@@ -4156,36 +4158,234 @@ class TestDispatchGroups:
await task
+_CONSUMER_MODULE = (
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer"
+)
+
+
+class _ReconnectCounts(NamedTuple):
+ reconnects: float
+ failures: float
+
+
+def _reconnect_counts() -> _ReconnectCounts:
+ return _ReconnectCounts(
+ reconnects=HEARTBEAT_RECONNECTS_TOTAL._value.get(),
+ failures=HEARTBEAT_RECONNECT_FAILURES_TOTAL._value.get(),
+ )
+
+
+def _bounded_sleep(limit: int = 10) -> Any:
+ calls = 0
+
+ async def fast_sleep(_seconds: float) -> None:
+ nonlocal calls
+ calls += 1
+ if calls > limit:
+ raise AssertionError("heartbeat loop did not exit after lease loss")
+
+ return fast_sleep
+
+
class TestBatchHeartbeat:
+ @pytest.mark.parametrize(
+ "error,closed,broken,per_group_connections,expect_reconnect",
+ [
+ pytest.param(
+ psycopg.OperationalError("the connection is closed"), True, False, True, True, id="closed_connection"
+ ),
+ pytest.param(
+ psycopg.OperationalError("server closed the connection unexpectedly"),
+ False,
+ False,
+ True,
+ True,
+ id="dropped_mid_query",
+ ),
+ pytest.param(
+ psycopg.InterfaceError("connection already closed"), False, False, True, True, id="interface_error"
+ ),
+ pytest.param(psycopg.OperationalError("broken pipe"), False, True, True, True, id="broken_connection"),
+ pytest.param(psycopg.errors.AdminShutdown("terminating"), False, False, True, True, id="admin_shutdown"),
+ pytest.param(ConnectionError("blip"), False, False, True, False, id="non_psycopg_blip"),
+ pytest.param(psycopg.errors.DeadlockDetected("deadlock"), False, False, True, False, id="deadlock"),
+ pytest.param(
+ psycopg.errors.QueryCanceled("canceling statement due to statement timeout"),
+ False,
+ False,
+ True,
+ False,
+ id="statement_timeout",
+ ),
+ pytest.param(
+ psycopg.OperationalError("the connection is closed"),
+ True,
+ False,
+ False,
+ False,
+ id="advisory_lock_adapter_keeps_its_session",
+ ),
+ ],
+ )
@pytest.mark.asyncio
- async def test_transient_renewal_failure_does_not_end_heartbeat(self):
- # A single pgbouncer/network blip previously killed the heartbeat
- # silently for the rest of the batch: renewals stopped, the lease
- # expired under a healthy owner, and another pod could re-claim the
- # batch mid-apply. The loop must survive transient errors and only
- # stop when the lease is genuinely lost.
+ async def test_renewal_failure_reconnects_only_for_a_dead_connection(
+ self, error, closed, broken, per_group_connections, expect_reconnect
+ ):
consumer = _make_consumer()
- renew = AsyncMock(side_effect=[ConnectionError("blip"), True, False])
+ old_conn = _make_healthy_conn(closed=closed, broken=broken)
+ new_conn = _make_healthy_conn()
+ renew = AsyncMock(side_effect=[error, True, False])
refresh = AsyncMock()
+ before = _reconnect_counts()
- sleep_count = 0
+ with (
+ patch.object(consumer._adapter, "renew_lease", renew),
+ patch.object(consumer._adapter, "update_status", refresh),
+ patch.object(consumer._adapter, "per_group_connections", per_group_connections),
+ patch.object(consumer, "_connect", new_callable=AsyncMock, return_value=new_conn) as mock_connect,
+ patch.object(batch_consumer_module.asyncio, "sleep", _bounded_sleep()),
+ ):
+ await consumer._batch_heartbeat(old_conn, _make_batch(), attempt=1)
- async def fast_sleep(_seconds):
- nonlocal sleep_count
- sleep_count += 1
- if sleep_count > 10:
- raise AssertionError("heartbeat loop did not exit after lease loss")
+ after = _reconnect_counts()
+ renew_conns = [call.args[0] for call in renew.await_args_list]
+ refresh.assert_awaited_once()
+ if expect_reconnect:
+ # The failed tick renews once on the new connection, and every later tick stays on it.
+ assert renew_conns == [old_conn, new_conn, new_conn]
+ assert refresh.await_args_list[-1].args[0] is new_conn
+ mock_connect.assert_awaited_once()
+ old_conn.close.assert_awaited_once()
+ assert consumer._group_conns[(1, "schema-1")] is new_conn
+ assert after == _ReconnectCounts(before.reconnects + 1, before.failures)
+ else:
+ assert renew_conns == [old_conn, old_conn, old_conn]
+ assert refresh.await_args_list[-1].args[0] is old_conn
+ mock_connect.assert_not_awaited()
+ old_conn.close.assert_not_awaited()
+ assert consumer._group_conns == {}
+ assert after == before
+
+ @pytest.mark.parametrize(
+ "connect_side_effect,renew_side_effect,expected_renew_conns",
+ [
+ pytest.param(
+ [psycopg.OperationalError("connection refused"), "new"],
+ ["dropped", "dropped", True, False],
+ ["old", "old", "new", "new"],
+ id="dial_fails_then_succeeds_next_tick",
+ ),
+ pytest.param(
+ ["new"],
+ ["dropped", psycopg.OperationalError("server closed the connection unexpectedly"), True, False],
+ ["old", "new", "new", "new"],
+ id="renew_on_new_connection_fails_then_succeeds_next_tick",
+ ),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_failed_reconnect_counts_a_failure_and_retries_next_tick(
+ self, connect_side_effect, renew_side_effect, expected_renew_conns
+ ):
+ consumer = _make_consumer()
+ conns = {"old": _make_healthy_conn(closed=True), "new": _make_healthy_conn()}
+ dropped = psycopg.OperationalError("the connection is closed")
+ renew = AsyncMock(side_effect=[dropped if item == "dropped" else item for item in renew_side_effect])
+ refresh = AsyncMock()
+ before = _reconnect_counts()
with (
patch.object(consumer._adapter, "renew_lease", renew),
patch.object(consumer._adapter, "update_status", refresh),
- patch.object(batch_consumer_module.asyncio, "sleep", fast_sleep),
+ patch.object(
+ consumer,
+ "_connect",
+ new_callable=AsyncMock,
+ side_effect=[conns[item] if isinstance(item, str) else item for item in connect_side_effect],
+ ),
+ patch.object(batch_consumer_module.asyncio, "sleep", _bounded_sleep()),
):
- await consumer._batch_heartbeat(AsyncMock(), _make_batch(), attempt=1)
+ await consumer._batch_heartbeat(conns["old"], _make_batch(), attempt=1)
- # blip -> continue; renewed -> status refresh; lease lost -> stop
- assert renew.await_count == 3
+ assert [call.args[0] for call in renew.await_args_list] == [conns[name] for name in expected_renew_conns]
refresh.assert_awaited_once()
+ assert refresh.await_args_list[-1].args[0] is conns["new"]
+ assert _reconnect_counts() == _ReconnectCounts(before.reconnects + 1, before.failures + 1)
+
+
+class TestHeartbeatReconnectInGroup:
+ @staticmethod
+ def _make_consumer_with_sink(process_batch: Any) -> BatchConsumer:
+ config = ConsumerConfig(database_url="postgres://unused:unused@localhost/unused", recovery_grace_seconds=30)
+ consumer = BatchConsumer(config=config, process_batch=process_batch)
+ consumer._poll_conn = _make_healthy_conn()
+ consumer._recovery_conn = _make_healthy_conn()
+ return consumer
+
+ @pytest.mark.parametrize("lease_kept", [True, False], ids=["lease_kept", "lease_lost_during_outage"])
+ @pytest.mark.asyncio
+ async def test_connection_drop_mid_batch(self, lease_kept):
+ old_conn = _make_healthy_conn()
+ new_conn = _make_healthy_conn()
+ renewed_on_new_conn = asyncio.Event()
+ sink_writes: list[str] = []
+ lease = {"held": True}
+ real_sleep = asyncio.sleep
+
+ async def fake_renew(conn, *, team_id, schema_id, owner_token, lease_ttl_seconds):
+ if conn is old_conn and old_conn.closed:
+ raise psycopg.OperationalError("the connection is closed")
+ if conn is new_conn:
+ renewed_on_new_conn.set()
+ return lease["held"]
+
+ async def fake_process(batch, verify_ownership):
+ # A pooler restart kills the group connection while the sink write runs. When the
+ # lease is lost, another worker reclaimed the batch while the connection was down.
+ old_conn.closed = True
+ lease["held"] = lease_kept
+ await asyncio.wait_for(renewed_on_new_conn.wait(), timeout=5)
+ verify_ownership()
+ sink_writes.append(batch.id)
+
+ async def fast_sleep(_seconds: float) -> None:
+ await real_sleep(0)
+
+ consumer = self._make_consumer_with_sink(fake_process)
+
+ with (
+ patch(f"{_CONSUMER_MODULE}.BatchQueue.renew_lease", side_effect=fake_renew) as mock_renew,
+ patch(f"{_CONSUMER_MODULE}.BatchQueue.verify_group_lease_sync", side_effect=lambda *a, **k: lease["held"]),
+ patch(
+ f"{_CONSUMER_MODULE}.BatchQueue.verify_advisory_lock",
+ new_callable=AsyncMock,
+ side_effect=lambda conn, **k: lease["held"],
+ ) as mock_verify,
+ patch(f"{_CONSUMER_MODULE}.BatchQueue.update_status_unless_failed", new_callable=AsyncMock) as mock_status,
+ patch(f"{_CONSUMER_MODULE}.BatchQueue.unlock_for_batches", new_callable=AsyncMock) as mock_unlock,
+ patch.object(consumer._adapter, "should_process_batch", new_callable=AsyncMock, return_value=True),
+ patch.object(consumer, "_connect", new_callable=AsyncMock, side_effect=[old_conn, new_conn]),
+ patch.object(batch_consumer_module.asyncio, "sleep", fast_sleep),
+ ):
+ await consumer._process_group((1, "schema-1"), [_make_batch()])
+
+ assert any(call.args[0] is new_conn for call in mock_renew.await_args_list)
+ written_states = [
+ (call.args[0], call.kwargs["job_state"])
+ for call in mock_status.await_args_list
+ if call.kwargs["job_state"] != SourceBatchStatus.State.EXECUTING.value
+ ]
+ if lease_kept:
+ assert sink_writes == [_make_batch().id]
+ assert written_states == [(new_conn, SourceBatchStatus.State.SUCCEEDED.value)]
+ assert mock_verify.await_args_list[-1].args[0] is new_conn
+ else:
+ # The pre-commit check fences the write, so only the worker that reclaimed the batch writes it.
+ assert sink_writes == []
+ assert written_states == []
+ assert mock_unlock.await_args_list[-1].args[0] is new_conn
+ new_conn.close.assert_awaited()
+ assert consumer._group_conns == {}
class TestIsRetryableError:
diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.py
index 7a693bb14a07..f895119cccc1 100644
--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py
+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py
@@ -151,6 +151,22 @@ def _is_transient_connection_error(error: BaseException, conn: psycopg.AsyncConn
return isinstance(error, psycopg.OperationalError) and (conn.closed or conn.broken)
+def _is_dead_group_connection_error(error: BaseException, conn: psycopg.AsyncConnection[Any]) -> bool:
+ """Whether a heartbeat error means ``conn`` can never run another query.
+
+ A pooler restart closes the socket, and psycopg then fails every later query on that
+ connection with "the connection is closed", so a retry on it cannot succeed. A deadlock
+ or a statement timeout leaves the connection usable, so those errors keep it.
+ """
+ if conn.closed or conn.broken:
+ return True
+ return (
+ isinstance(error, psycopg.InterfaceError)
+ or _is_connection_dropped_error(error)
+ or _is_admin_shutdown_error(error)
+ )
+
+
# How libpq and psycopg word a connection that died under an in-flight query. The queue DB
# sits behind a pooler, so a pool recycle, a failover, or a server-side backend cull leaves
# the socket gone mid-statement. Nothing committed, so redialing and re-running is enough.
@@ -513,6 +529,9 @@ class BatchConsumer:
self._recovery_task: asyncio.Task[None] | None = None
self._heartbeat_task: asyncio.Task[None] | None = None
self._in_flight: dict[tuple[int, str], asyncio.Task[None]] = {}
+ # The connection each group task uses for its lease and status writes. The batch
+ # heartbeat replaces an entry when the old connection drops.
+ self._group_conns: dict[tuple[int, str], psycopg.AsyncConnection[Any]] = {}
# Monotonic stamp of the last reconcile sweep; runs inside the recovery loop so both share one connection.
self._last_reconcile_monotonic = 0.0
self._holds_gauge_slot = False
@@ -955,6 +974,7 @@ class BatchConsumer:
try:
if owns_conn:
group_conn = await self._connect()
+ self._group_conns[key] = group_conn
else:
group_conn = self._poll_conn
assert group_conn is not None
@@ -1023,6 +1043,8 @@ class BatchConsumer:
)
break
finally:
+ if owns_conn:
+ group_conn = self._group_conns.pop(key, group_conn)
await self._unlock_group(group_conn, batches, team_id=team_id, schema_id=schema_id)
if owns_conn and group_conn is not None:
try:
@@ -1143,6 +1165,52 @@ class BatchConsumer:
if not renewed:
raise OwnershipLostError(f"group lease lost for ({batch.team_id}, {batch.schema_id})")
+ def _current_lock_conn(
+ self, batch: PendingBatch, lock_conn: psycopg.AsyncConnection[Any] | None
+ ) -> psycopg.AsyncConnection[Any] | None:
+ """Return the group's live connection, which the batch heartbeat can swap in for ``lock_conn``."""
+ if lock_conn is None:
+ return None
+ return self._group_conns.get(_group_key(batch), lock_conn)
+
+ async def _reconnect_group_conn(
+ self, batch: PendingBatch, dead_conn: psycopg.AsyncConnection[Any], error: Exception
+ ) -> psycopg.AsyncConnection[Any] | None:
+ """Open a new group connection in place of ``dead_conn``. Returns None when the dial fails.
+
+ The group lease is a row that holds the owner token, not a session lock, so the new
+ connection does not lose ownership. The caller renews the lease on it, and
+ ``renew_lease`` extends only a lease that is still live and still ours. A lease that
+ expired while the connection was down stays lost, so a reconnect cannot take back a
+ batch that the recovery sweep has queued again.
+ """
+ try:
+ new_conn = await self._connect()
+ except Exception as connect_error:
+ self._metrics.heartbeat_reconnect_failures_total.inc()
+ logger.warning(
+ self._event("batch_heartbeat_reconnect_failed"),
+ batch_id=batch.id,
+ team_id=batch.team_id,
+ external_data_schema_id=batch.schema_id,
+ error=str(connect_error),
+ )
+ return None
+ self._group_conns[_group_key(batch)] = new_conn
+ self._metrics.heartbeat_reconnects_total.inc()
+ logger.warning(
+ self._event("batch_heartbeat_reconnected"),
+ batch_id=batch.id,
+ team_id=batch.team_id,
+ external_data_schema_id=batch.schema_id,
+ error=str(error),
+ )
+ try:
+ await asyncio.wait_for(dead_conn.close(), timeout=5.0)
+ except Exception:
+ pass
+ return new_conn
+
def _coalesce(self, batches: list[PendingBatch]) -> list[list[PendingBatch]]:
if self._process_batches is None:
return [[batch] for batch in batches]
@@ -1173,35 +1241,53 @@ class BatchConsumer:
abandons the in-flight batch instead of writing ``succeeded``.
A coalesced set shares one lease and refreshes every constituent's row.
+
+ When the group connection drops, the heartbeat opens a new one, renews the
+ lease on it once, and publishes it in ``_group_conns``. The batch loop then
+ does its post-process writes on the new connection, not on the dead one.
"""
batch = batches[0]
interval = max((self._config.recovery_grace_seconds or RECOVERY_GRACE_SECONDS) / 3, 10.0)
while True:
await asyncio.sleep(interval)
try:
- renewed = await self._adapter.renew_lease(
- lock_conn,
- team_id=batch.team_id,
- schema_id=batch.schema_id,
- owner_token=self._owner_token,
- lease_ttl_seconds=self._lease_ttl_seconds,
- )
+ renewed = await self._renew_heartbeat_lease(lock_conn, batch)
except Exception as e:
- # A transient blip (pgbouncer bounce, network hiccup) must not
- # end the heartbeat for the rest of a long batch: with renewals
- # stopped, the lease expires and the executing row ages past
- # grace while the owner is still healthily applying — inviting
- # a concurrent re-claim of a batch that is mid-write. Log and
- # try again next tick; a genuinely dead connection keeps
- # failing and the lease-expiry backstop still applies.
- logger.warning(
- self._event("batch_heartbeat_renewal_failed"),
- batch_id=batch.id,
- team_id=batch.team_id,
- external_data_schema_id=batch.schema_id,
- error=str(e),
- )
- continue
+ if not (self._adapter.per_group_connections and _is_dead_group_connection_error(e, lock_conn)):
+ # A transient blip (pgbouncer bounce, network hiccup) must not
+ # end the heartbeat for the rest of a long batch: with renewals
+ # stopped, the lease expires and the executing row ages past
+ # grace while the owner is still healthily applying — inviting
+ # a concurrent re-claim of a batch that is mid-write. Log and
+ # try again next tick on the same connection.
+ logger.warning(
+ self._event("batch_heartbeat_renewal_failed"),
+ batch_id=batch.id,
+ team_id=batch.team_id,
+ external_data_schema_id=batch.schema_id,
+ error=str(e),
+ )
+ continue
+ # A dropped connection fails every later query, so the heartbeat needs a new
+ # one. Advisory-lock adapters skip this branch because their lock belongs to
+ # the old session, and a new session would not hold it.
+ replacement = await self._reconnect_group_conn(batch, lock_conn, e)
+ if replacement is None:
+ continue
+ lock_conn = replacement
+ try:
+ renewed = await self._renew_heartbeat_lease(lock_conn, batch)
+ except Exception as retry_error:
+ self._metrics.heartbeat_reconnect_failures_total.inc()
+ logger.warning(
+ self._event("batch_heartbeat_renewal_failed"),
+ batch_id=batch.id,
+ team_id=batch.team_id,
+ external_data_schema_id=batch.schema_id,
+ error=str(retry_error),
+ after_reconnect=True,
+ )
+ continue
if not renewed:
# Ownership is gone (reclaimed or expired) — never transient: stop
# heartbeating so the per-batch ownership checks fence the apply.
@@ -1230,6 +1316,15 @@ class BatchConsumer:
error=str(e),
)
+ async def _renew_heartbeat_lease(self, lock_conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> bool:
+ return await self._adapter.renew_lease(
+ lock_conn,
+ team_id=batch.team_id,
+ schema_id=batch.schema_id,
+ owner_token=self._owner_token,
+ lease_ttl_seconds=self._lease_ttl_seconds,
+ )
+
@contextlib.contextmanager
def _batch_log_context(self, batch: PendingBatch, attempt: int) -> Iterator[None]:
"""Bind structlog contextvars so every downstream log line (including loader calls)
@@ -1279,6 +1374,7 @@ class BatchConsumer:
async def _process_single(self, batch: PendingBatch, lock_conn: psycopg.AsyncConnection[Any] | None = None) -> bool:
"""Bind per-batch log context, then process. Returns True only on success."""
attempt = batch.latest_attempt + 1
+ lock_conn = self._current_lock_conn(batch, lock_conn)
with self._batch_log_context(batch, attempt):
# Renew-or-abandon at entry (not a pure verify): re-ups the lease on
# every batch so a long run of short batches can't expire it at
@@ -1300,6 +1396,7 @@ class BatchConsumer:
"""
assert self._process_batches is not None
head = batches[0]
+ lock_conn = self._current_lock_conn(head, lock_conn)
attempts = {batch.id: batch.latest_attempt + 1 for batch in batches}
run_uuids = list(dict.fromkeys(batch.run_uuid for batch in batches))
row_count = sum(batch.row_count for batch in batches)
@@ -1366,6 +1463,9 @@ class BatchConsumer:
await self._stop_heartbeat(heartbeat_task)
heartbeat_task = None
+ lock_conn = self._current_lock_conn(head, lock_conn)
+ if lock_conn is not None:
+ status_conn = lock_conn
try:
for batch in batches:
@@ -1516,6 +1616,9 @@ class BatchConsumer:
except asyncio.CancelledError:
pass
heartbeat_task = None
+ lock_conn = self._current_lock_conn(batch, lock_conn)
+ if lock_conn is not None:
+ status_conn = lock_conn
await self._adapter.after_batch_processed(status_conn, batch=batch)
@@ -1546,6 +1649,9 @@ class BatchConsumer:
self._metrics.batches_processed_total.labels(status="error").inc()
self._metrics.batch_retry_total.labels(attempt=str(attempt), error_type=type(err).__name__).inc()
+ # The failure path reads and writes on the group connection, which the heartbeat shares.
+ await self._stop_heartbeat(heartbeat_task)
+ heartbeat_task = None
await self._handle_batch_failure(batch, attempt, err, lock_conn=lock_conn, status_conn=status_conn)
return False
finally:
@@ -1566,6 +1672,10 @@ class BatchConsumer:
status_conn: psycopg.AsyncConnection[Any],
) -> None:
"""Write the retry/terminal state after a processing error."""
+ # After a drop, the failure writes must use the connection the heartbeat swapped in.
+ lock_conn = self._current_lock_conn(batch, lock_conn)
+ if lock_conn is not None:
+ status_conn = lock_conn
if not self._adapter.is_retryable_error(err):
# Deterministic failures do not benefit from retrying. Preserve their
# messages, except for explicitly classified permanent apply errors.
@@ -2011,9 +2121,12 @@ ProcessBatchFn = Callable[[PendingBatch], Coroutine[Any, Any, None]]
ProcessBatchesFn = Callable[[list[PendingBatch]], Coroutine[Any, Any, None]]
+def _group_key(batch: PendingBatch) -> tuple[int, str]:
+ return getattr(batch, "consumer_group_key", (batch.team_id, batch.schema_id))
+
+
def _group_by_key(batches: list[PendingBatch]) -> dict[tuple[int, str], list[PendingBatch]]:
groups: dict[tuple[int, str], list[PendingBatch]] = defaultdict(list)
for batch in batches:
- key = getattr(batch, "consumer_group_key", (batch.team_id, batch.schema_id))
- groups[key].append(batch)
+ groups[_group_key(batch)].append(batch)
return groups
diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.py
index 24f25e64863a..1b5bb9605683 100644
--- a/products/warehouse_sources_queue/backend/core/metrics.py
+++ b/products/warehouse_sources_queue/backend/core/metrics.py
@@ -312,6 +312,8 @@ class ConsumerMetrics:
coalesced_set_batches: Histogram
coalesced_set_runs: Histogram
coalesced_set_rows: Histogram
+ heartbeat_reconnects_total: Counter
+ heartbeat_reconnect_failures_total: Counter
COALESCED_SET_BATCHES_BUCKETS = (1, 2, 4, 8, 16, 32, 64, 128, 256)
@@ -339,6 +341,15 @@ COALESCED_SET_ROWS = Histogram(
buckets=COALESCED_SET_ROWS_BUCKETS,
)
+HEARTBEAT_RECONNECTS_TOTAL = Counter(
+ "warehouse_pg_consumer_heartbeat_reconnects_total",
+ "Batch heartbeats that replaced a dropped group connection with a new one",
+)
+HEARTBEAT_RECONNECT_FAILURES_TOTAL = Counter(
+ "warehouse_pg_consumer_heartbeat_reconnect_failures_total",
+ "Batch heartbeats that could not open a new group connection or renew the lease on it",
+)
+
DELTA_CONSUMER_METRICS = ConsumerMetrics(
batches_processed_total=BATCHES_PROCESSED_TOTAL,
batch_processing_duration_seconds=BATCH_PROCESSING_DURATION_SECONDS,
@@ -353,6 +364,8 @@ DELTA_CONSUMER_METRICS = ConsumerMetrics(
coalesced_set_batches=COALESCED_SET_BATCHES,
coalesced_set_runs=COALESCED_SET_RUNS,
coalesced_set_rows=COALESCED_SET_ROWS,
+ heartbeat_reconnects_total=HEARTBEAT_RECONNECTS_TOTAL,
+ heartbeat_reconnect_failures_total=HEARTBEAT_RECONNECT_FAILURES_TOTAL,
)
_metrics_by_prefix: dict[str, ConsumerMetrics] = {}
@@ -430,6 +443,14 @@ def make_consumer_metrics(prefix: str) -> ConsumerMetrics:
"Rows per set loaded as a single write",
buckets=COALESCED_SET_ROWS_BUCKETS,
),
+ heartbeat_reconnects_total=Counter(
+ f"{p}_heartbeat_reconnects_total",
+ "Batch heartbeats that replaced a dropped group connection with a new one",
+ ),
+ heartbeat_reconnect_failures_total=Counter(
+ f"{p}_heartbeat_reconnect_failures_total",
+ "Batch heartbeats that could not open a new group connection or renew the lease on it",
+ ),
)
_metrics_by_prefix[prefix] = metrics
return metrics