:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): reconnect the loader heartbeat after a closed connection (#120167)
Tom Owersa61bd1d20a91
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex 94407679fdaf..7e34c12385ad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -80,6 +80,8 @@ from products.warehouse_sources_queue.backend.core.metrics import (CLAIMABLE_GROUPS,DEPTH_PROBE_TIMEOUTS_TOTAL,DEPTH_SAMPLE_AGE_SECONDS,+ HEARTBEAT_RECONNECT_FAILURES_TOTAL,+ HEARTBEAT_RECONNECTS_TOTAL,OLDEST_UNCLAIMED_BATCH_SECONDS,ORPHANED_BATCHES_DRAINED_TOTAL,QUEUE_SAMPLE_GAUGES,@@ -4156,36 +4158,234 @@ class TestDispatchGroups:await task+_CONSUMER_MODULE = (+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer"+)+++class _ReconnectCounts(NamedTuple):+ reconnects: float+ failures: float+++def _reconnect_counts() -> _ReconnectCounts:+ return _ReconnectCounts(+ reconnects=HEARTBEAT_RECONNECTS_TOTAL._value.get(),+ failures=HEARTBEAT_RECONNECT_FAILURES_TOTAL._value.get(),+ )+++def _bounded_sleep(limit: int = 10) -> Any:+ calls = 0++ async def fast_sleep(_seconds: float) -> None:+ nonlocal calls+ calls += 1+ if calls > limit:+ raise AssertionError("heartbeat loop did not exit after lease loss")++ return fast_sleep++class TestBatchHeartbeat:+ @pytest.mark.parametrize(+ "error,closed,broken,per_group_connections,expect_reconnect",+ [+ pytest.param(+ psycopg.OperationalError("the connection is closed"), True, False, True, True, id="closed_connection"+ ),+ pytest.param(+ psycopg.OperationalError("server closed the connection unexpectedly"),+ False,+ False,+ True,+ True,+ id="dropped_mid_query",+ ),+ pytest.param(+ psycopg.InterfaceError("connection already closed"), False, False, True, True, id="interface_error"+ ),+ pytest.param(psycopg.OperationalError("broken pipe"), False, True, True, True, id="broken_connection"),+ pytest.param(psycopg.errors.AdminShutdown("terminating"), False, False, True, True, id="admin_shutdown"),+ pytest.param(ConnectionError("blip"), False, False, True, False, id="non_psycopg_blip"),+ pytest.param(psycopg.errors.DeadlockDetected("deadlock"), False, False, True, False, id="deadlock"),+ pytest.param(+ psycopg.errors.QueryCanceled("canceling statement due to statement timeout"),+ False,+ False,+ True,+ False,+ id="statement_timeout",+ ),+ pytest.param(+ psycopg.OperationalError("the connection is closed"),+ True,+ False,+ False,+ False,+ id="advisory_lock_adapter_keeps_its_session",+ ),+ ],+ )@pytest.mark.asyncio- async def test_transient_renewal_failure_does_not_end_heartbeat(self):- # A single pgbouncer/network blip previously killed the heartbeat- # silently for the rest of the batch: renewals stopped, the lease- # expired under a healthy owner, and another pod could re-claim the- # batch mid-apply. The loop must survive transient errors and only- # stop when the lease is genuinely lost.+ async def test_renewal_failure_reconnects_only_for_a_dead_connection(+ self, error, closed, broken, per_group_connections, expect_reconnect+ ):consumer = _make_consumer()- renew = AsyncMock(side_effect=[ConnectionError("blip"), True, False])+ old_conn = _make_healthy_conn(closed=closed, broken=broken)+ new_conn = _make_healthy_conn()+ renew = AsyncMock(side_effect=[error, True, False])refresh = AsyncMock()+ before = _reconnect_counts()- sleep_count = 0+ with (+ patch.object(consumer._adapter, "renew_lease", renew),+ patch.object(consumer._adapter, "update_status", refresh),+ patch.object(consumer._adapter, "per_group_connections", per_group_connections),+ patch.object(consumer, "_connect", new_callable=AsyncMock, return_value=new_conn) as mock_connect,+ patch.object(batch_consumer_module.asyncio, "sleep", _bounded_sleep()),+ ):+ await consumer._batch_heartbeat(old_conn, _make_batch(), attempt=1)- async def fast_sleep(_seconds):- nonlocal sleep_count- sleep_count += 1- if sleep_count > 10:- raise AssertionError("heartbeat loop did not exit after lease loss")+ after = _reconnect_counts()+ renew_conns = [call.args[0] for call in renew.await_args_list]+ refresh.assert_awaited_once()+ if expect_reconnect:+ # The failed tick renews once on the new connection, and every later tick stays on it.+ assert renew_conns == [old_conn, new_conn, new_conn]+ assert refresh.await_args_list[-1].args[0] is new_conn+ mock_connect.assert_awaited_once()+ old_conn.close.assert_awaited_once()+ assert consumer._group_conns[(1, "schema-1")] is new_conn+ assert after == _ReconnectCounts(before.reconnects + 1, before.failures)+ else:+ assert renew_conns == [old_conn, old_conn, old_conn]+ assert refresh.await_args_list[-1].args[0] is old_conn+ mock_connect.assert_not_awaited()+ old_conn.close.assert_not_awaited()+ assert consumer._group_conns == {}+ assert after == before++ @pytest.mark.parametrize(+ "connect_side_effect,renew_side_effect,expected_renew_conns",+ [+ pytest.param(+ [psycopg.OperationalError("connection refused"), "new"],+ ["dropped", "dropped", True, False],+ ["old", "old", "new", "new"],+ id="dial_fails_then_succeeds_next_tick",+ ),+ pytest.param(+ ["new"],+ ["dropped", psycopg.OperationalError("server closed the connection unexpectedly"), True, False],+ ["old", "new", "new", "new"],+ id="renew_on_new_connection_fails_then_succeeds_next_tick",+ ),+ ],+ )+ @pytest.mark.asyncio+ async def test_failed_reconnect_counts_a_failure_and_retries_next_tick(+ self, connect_side_effect, renew_side_effect, expected_renew_conns+ ):+ consumer = _make_consumer()+ conns = {"old": _make_healthy_conn(closed=True), "new": _make_healthy_conn()}+ dropped = psycopg.OperationalError("the connection is closed")+ renew = AsyncMock(side_effect=[dropped if item == "dropped" else item for item in renew_side_effect])+ refresh = AsyncMock()+ before = _reconnect_counts()with (patch.object(consumer._adapter, "renew_lease", renew),patch.object(consumer._adapter, "update_status", refresh),- patch.object(batch_consumer_module.asyncio, "sleep", fast_sleep),+ patch.object(+ consumer,+ "_connect",+ new_callable=AsyncMock,+ side_effect=[conns[item] if isinstance(item, str) else item for item in connect_side_effect],+ ),+ patch.object(batch_consumer_module.asyncio, "sleep", _bounded_sleep()),):- await consumer._batch_heartbeat(AsyncMock(), _make_batch(), attempt=1)+ await consumer._batch_heartbeat(conns["old"], _make_batch(), attempt=1)- # blip -> continue; renewed -> status refresh; lease lost -> stop- assert renew.await_count == 3+ assert [call.args[0] for call in renew.await_args_list] == [conns[name] for name in expected_renew_conns]refresh.assert_awaited_once()+ assert refresh.await_args_list[-1].args[0] is conns["new"]+ assert _reconnect_counts() == _ReconnectCounts(before.reconnects + 1, before.failures + 1)+++class TestHeartbeatReconnectInGroup:+ @staticmethod+ def _make_consumer_with_sink(process_batch: Any) -> BatchConsumer:+ config = ConsumerConfig(database_url="postgres://unused:unused@localhost/unused", recovery_grace_seconds=30)+ consumer = BatchConsumer(config=config, process_batch=process_batch)+ consumer._poll_conn = _make_healthy_conn()+ consumer._recovery_conn = _make_healthy_conn()+ return consumer++ @pytest.mark.parametrize("lease_kept", [True, False], ids=["lease_kept", "lease_lost_during_outage"])+ @pytest.mark.asyncio+ async def test_connection_drop_mid_batch(self, lease_kept):+ old_conn = _make_healthy_conn()+ new_conn = _make_healthy_conn()+ renewed_on_new_conn = asyncio.Event()+ sink_writes: list[str] = []+ lease = {"held": True}+ real_sleep = asyncio.sleep++ async def fake_renew(conn, *, team_id, schema_id, owner_token, lease_ttl_seconds):+ if conn is old_conn and old_conn.closed:+ raise psycopg.OperationalError("the connection is closed")+ if conn is new_conn:+ renewed_on_new_conn.set()+ return lease["held"]++ async def fake_process(batch, verify_ownership):+ # A pooler restart kills the group connection while the sink write runs. When the+ # lease is lost, another worker reclaimed the batch while the connection was down.+ old_conn.closed = True+ lease["held"] = lease_kept+ await asyncio.wait_for(renewed_on_new_conn.wait(), timeout=5)+ verify_ownership()+ sink_writes.append(batch.id)++ async def fast_sleep(_seconds: float) -> None:+ await real_sleep(0)++ consumer = self._make_consumer_with_sink(fake_process)++ with (+ patch(f"{_CONSUMER_MODULE}.BatchQueue.renew_lease", side_effect=fake_renew) as mock_renew,+ patch(f"{_CONSUMER_MODULE}.BatchQueue.verify_group_lease_sync", side_effect=lambda *a, **k: lease["held"]),+ patch(+ f"{_CONSUMER_MODULE}.BatchQueue.verify_advisory_lock",+ new_callable=AsyncMock,+ side_effect=lambda conn, **k: lease["held"],+ ) as mock_verify,+ patch(f"{_CONSUMER_MODULE}.BatchQueue.update_status_unless_failed", new_callable=AsyncMock) as mock_status,+ patch(f"{_CONSUMER_MODULE}.BatchQueue.unlock_for_batches", new_callable=AsyncMock) as mock_unlock,+ patch.object(consumer._adapter, "should_process_batch", new_callable=AsyncMock, return_value=True),+ patch.object(consumer, "_connect", new_callable=AsyncMock, side_effect=[old_conn, new_conn]),+ patch.object(batch_consumer_module.asyncio, "sleep", fast_sleep),+ ):+ await consumer._process_group((1, "schema-1"), [_make_batch()])++ assert any(call.args[0] is new_conn for call in mock_renew.await_args_list)+ written_states = [+ (call.args[0], call.kwargs["job_state"])+ for call in mock_status.await_args_list+ if call.kwargs["job_state"] != SourceBatchStatus.State.EXECUTING.value+ ]+ if lease_kept:+ assert sink_writes == [_make_batch().id]+ assert written_states == [(new_conn, SourceBatchStatus.State.SUCCEEDED.value)]+ assert mock_verify.await_args_list[-1].args[0] is new_conn+ else:+ # The pre-commit check fences the write, so only the worker that reclaimed the batch writes it.+ assert sink_writes == []+ assert written_states == []+ assert mock_unlock.await_args_list[-1].args[0] is new_conn+ new_conn.close.assert_awaited()+ assert consumer._group_conns == {}class TestIsRetryableError:diff --git a/products/warehouse_sources_queue/backend/core/batch_consumer.py b/products/warehouse_sources_queue/backend/core/batch_consumer.pyindex 7a693bb14a07..f895119cccc1 100644--- a/products/warehouse_sources_queue/backend/core/batch_consumer.py+++ b/products/warehouse_sources_queue/backend/core/batch_consumer.py@@ -151,6 +151,22 @@ def _is_transient_connection_error(error: BaseException, conn: psycopg.AsyncConnreturn isinstance(error, psycopg.OperationalError) and (conn.closed or conn.broken)+def _is_dead_group_connection_error(error: BaseException, conn: psycopg.AsyncConnection[Any]) -> bool:+ """Whether a heartbeat error means ``conn`` can never run another query.++ A pooler restart closes the socket, and psycopg then fails every later query on that+ connection with "the connection is closed", so a retry on it cannot succeed. A deadlock+ or a statement timeout leaves the connection usable, so those errors keep it.+ """+ if conn.closed or conn.broken:+ return True+ return (+ isinstance(error, psycopg.InterfaceError)+ or _is_connection_dropped_error(error)+ or _is_admin_shutdown_error(error)+ )++# How libpq and psycopg word a connection that died under an in-flight query. The queue DB# sits behind a pooler, so a pool recycle, a failover, or a server-side backend cull leaves# the socket gone mid-statement. Nothing committed, so redialing and re-running is enough.@@ -513,6 +529,9 @@ class BatchConsumer:self._recovery_task: asyncio.Task[None] | None = Noneself._heartbeat_task: asyncio.Task[None] | None = Noneself._in_flight: dict[tuple[int, str], asyncio.Task[None]] = {}+ # The connection each group task uses for its lease and status writes. The batch+ # heartbeat replaces an entry when the old connection drops.+ self._group_conns: dict[tuple[int, str], psycopg.AsyncConnection[Any]] = {}# Monotonic stamp of the last reconcile sweep; runs inside the recovery loop so both share one connection.self._last_reconcile_monotonic = 0.0self._holds_gauge_slot = False@@ -955,6 +974,7 @@ class BatchConsumer:try:if owns_conn:group_conn = await self._connect()+ self._group_conns[key] = group_connelse:group_conn = self._poll_connassert group_conn is not None@@ -1023,6 +1043,8 @@ class BatchConsumer:)breakfinally:+ if owns_conn:+ group_conn = self._group_conns.pop(key, group_conn)await self._unlock_group(group_conn, batches, team_id=team_id, schema_id=schema_id)if owns_conn and group_conn is not None:try:@@ -1143,6 +1165,52 @@ class BatchConsumer:if not renewed:raise OwnershipLostError(f"group lease lost for ({batch.team_id}, {batch.schema_id})")+ def _current_lock_conn(+ self, batch: PendingBatch, lock_conn: psycopg.AsyncConnection[Any] | None+ ) -> psycopg.AsyncConnection[Any] | None:+ """Return the group's live connection, which the batch heartbeat can swap in for ``lock_conn``."""+ if lock_conn is None:+ return None+ return self._group_conns.get(_group_key(batch), lock_conn)++ async def _reconnect_group_conn(+ self, batch: PendingBatch, dead_conn: psycopg.AsyncConnection[Any], error: Exception+ ) -> psycopg.AsyncConnection[Any] | None:+ """Open a new group connection in place of ``dead_conn``. Returns None when the dial fails.++ The group lease is a row that holds the owner token, not a session lock, so the new+ connection does not lose ownership. The caller renews the lease on it, and+ ``renew_lease`` extends only a lease that is still live and still ours. A lease that+ expired while the connection was down stays lost, so a reconnect cannot take back a+ batch that the recovery sweep has queued again.+ """+ try:+ new_conn = await self._connect()+ except Exception as connect_error:+ self._metrics.heartbeat_reconnect_failures_total.inc()+ logger.warning(+ self._event("batch_heartbeat_reconnect_failed"),+ batch_id=batch.id,+ team_id=batch.team_id,+ external_data_schema_id=batch.schema_id,+ error=str(connect_error),+ )+ return None+ self._group_conns[_group_key(batch)] = new_conn+ self._metrics.heartbeat_reconnects_total.inc()+ logger.warning(+ self._event("batch_heartbeat_reconnected"),+ batch_id=batch.id,+ team_id=batch.team_id,+ external_data_schema_id=batch.schema_id,+ error=str(error),+ )+ try:+ await asyncio.wait_for(dead_conn.close(), timeout=5.0)+ except Exception:+ pass+ return new_conn+def _coalesce(self, batches: list[PendingBatch]) -> list[list[PendingBatch]]:if self._process_batches is None:return [[batch] for batch in batches]@@ -1173,35 +1241,53 @@ class BatchConsumer:abandons the in-flight batch instead of writing ``succeeded``.A coalesced set shares one lease and refreshes every constituent's row.++ When the group connection drops, the heartbeat opens a new one, renews the+ lease on it once, and publishes it in ``_group_conns``. The batch loop then+ does its post-process writes on the new connection, not on the dead one."""batch = batches[0]interval = max((self._config.recovery_grace_seconds or RECOVERY_GRACE_SECONDS) / 3, 10.0)while True:await asyncio.sleep(interval)try:- renewed = await self._adapter.renew_lease(- lock_conn,- team_id=batch.team_id,- schema_id=batch.schema_id,- owner_token=self._owner_token,- lease_ttl_seconds=self._lease_ttl_seconds,- )+ renewed = await self._renew_heartbeat_lease(lock_conn, batch)except Exception as e:- # A transient blip (pgbouncer bounce, network hiccup) must not- # end the heartbeat for the rest of a long batch: with renewals- # stopped, the lease expires and the executing row ages past- # grace while the owner is still healthily applying — inviting- # a concurrent re-claim of a batch that is mid-write. Log and- # try again next tick; a genuinely dead connection keeps- # failing and the lease-expiry backstop still applies.- logger.warning(- self._event("batch_heartbeat_renewal_failed"),- batch_id=batch.id,- team_id=batch.team_id,- external_data_schema_id=batch.schema_id,- error=str(e),- )- continue+ if not (self._adapter.per_group_connections and _is_dead_group_connection_error(e, lock_conn)):+ # A transient blip (pgbouncer bounce, network hiccup) must not+ # end the heartbeat for the rest of a long batch: with renewals+ # stopped, the lease expires and the executing row ages past+ # grace while the owner is still healthily applying — inviting+ # a concurrent re-claim of a batch that is mid-write. Log and+ # try again next tick on the same connection.+ logger.warning(+ self._event("batch_heartbeat_renewal_failed"),+ batch_id=batch.id,+ team_id=batch.team_id,+ external_data_schema_id=batch.schema_id,+ error=str(e),+ )+ continue+ # A dropped connection fails every later query, so the heartbeat needs a new+ # one. Advisory-lock adapters skip this branch because their lock belongs to+ # the old session, and a new session would not hold it.+ replacement = await self._reconnect_group_conn(batch, lock_conn, e)+ if replacement is None:+ continue+ lock_conn = replacement+ try:+ renewed = await self._renew_heartbeat_lease(lock_conn, batch)+ except Exception as retry_error:+ self._metrics.heartbeat_reconnect_failures_total.inc()+ logger.warning(+ self._event("batch_heartbeat_renewal_failed"),+ batch_id=batch.id,+ team_id=batch.team_id,+ external_data_schema_id=batch.schema_id,+ error=str(retry_error),+ after_reconnect=True,+ )+ continueif not renewed:# Ownership is gone (reclaimed or expired) — never transient: stop# heartbeating so the per-batch ownership checks fence the apply.@@ -1230,6 +1316,15 @@ class BatchConsumer:error=str(e),)+ async def _renew_heartbeat_lease(self, lock_conn: psycopg.AsyncConnection[Any], batch: PendingBatch) -> bool:+ return await self._adapter.renew_lease(+ lock_conn,+ team_id=batch.team_id,+ schema_id=batch.schema_id,+ owner_token=self._owner_token,+ lease_ttl_seconds=self._lease_ttl_seconds,+ )+@contextlib.contextmanagerdef _batch_log_context(self, batch: PendingBatch, attempt: int) -> Iterator[None]:"""Bind structlog contextvars so every downstream log line (including loader calls)@@ -1279,6 +1374,7 @@ class BatchConsumer:async def _process_single(self, batch: PendingBatch, lock_conn: psycopg.AsyncConnection[Any] | None = None) -> bool:"""Bind per-batch log context, then process. Returns True only on success."""attempt = batch.latest_attempt + 1+ lock_conn = self._current_lock_conn(batch, lock_conn)with self._batch_log_context(batch, attempt):# Renew-or-abandon at entry (not a pure verify): re-ups the lease on# every batch so a long run of short batches can't expire it at@@ -1300,6 +1396,7 @@ class BatchConsumer:"""assert self._process_batches is not Nonehead = batches[0]+ lock_conn = self._current_lock_conn(head, lock_conn)attempts = {batch.id: batch.latest_attempt + 1 for batch in batches}run_uuids = list(dict.fromkeys(batch.run_uuid for batch in batches))row_count = sum(batch.row_count for batch in batches)@@ -1366,6 +1463,9 @@ class BatchConsumer:await self._stop_heartbeat(heartbeat_task)heartbeat_task = None+ lock_conn = self._current_lock_conn(head, lock_conn)+ if lock_conn is not None:+ status_conn = lock_conntry:for batch in batches:@@ -1516,6 +1616,9 @@ class BatchConsumer:except asyncio.CancelledError:passheartbeat_task = None+ lock_conn = self._current_lock_conn(batch, lock_conn)+ if lock_conn is not None:+ status_conn = lock_connawait self._adapter.after_batch_processed(status_conn, batch=batch)@@ -1546,6 +1649,9 @@ class BatchConsumer:self._metrics.batches_processed_total.labels(status="error").inc()self._metrics.batch_retry_total.labels(attempt=str(attempt), error_type=type(err).__name__).inc()+ # The failure path reads and writes on the group connection, which the heartbeat shares.+ await self._stop_heartbeat(heartbeat_task)+ heartbeat_task = Noneawait self._handle_batch_failure(batch, attempt, err, lock_conn=lock_conn, status_conn=status_conn)return Falsefinally:@@ -1566,6 +1672,10 @@ class BatchConsumer:status_conn: psycopg.AsyncConnection[Any],) -> None:"""Write the retry/terminal state after a processing error."""+ # After a drop, the failure writes must use the connection the heartbeat swapped in.+ lock_conn = self._current_lock_conn(batch, lock_conn)+ if lock_conn is not None:+ status_conn = lock_connif not self._adapter.is_retryable_error(err):# Deterministic failures do not benefit from retrying. Preserve their# messages, except for explicitly classified permanent apply errors.@@ -2011,9 +2121,12 @@ ProcessBatchFn = Callable[[PendingBatch], Coroutine[Any, Any, None]]ProcessBatchesFn = Callable[[list[PendingBatch]], Coroutine[Any, Any, None]]+def _group_key(batch: PendingBatch) -> tuple[int, str]:+ return getattr(batch, "consumer_group_key", (batch.team_id, batch.schema_id))++def _group_by_key(batches: list[PendingBatch]) -> dict[tuple[int, str], list[PendingBatch]]:groups: dict[tuple[int, str], list[PendingBatch]] = defaultdict(list)for batch in batches:- key = getattr(batch, "consumer_group_key", (batch.team_id, batch.schema_id))- groups[key].append(batch)+ groups[_group_key(batch)].append(batch)return groupsdiff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.pyindex 24f25e64863a..1b5bb9605683 100644--- a/products/warehouse_sources_queue/backend/core/metrics.py+++ b/products/warehouse_sources_queue/backend/core/metrics.py@@ -312,6 +312,8 @@ class ConsumerMetrics:coalesced_set_batches: Histogramcoalesced_set_runs: Histogramcoalesced_set_rows: Histogram+ heartbeat_reconnects_total: Counter+ heartbeat_reconnect_failures_total: CounterCOALESCED_SET_BATCHES_BUCKETS = (1, 2, 4, 8, 16, 32, 64, 128, 256)@@ -339,6 +341,15 @@ COALESCED_SET_ROWS = Histogram(buckets=COALESCED_SET_ROWS_BUCKETS,)+HEARTBEAT_RECONNECTS_TOTAL = Counter(+ "warehouse_pg_consumer_heartbeat_reconnects_total",+ "Batch heartbeats that replaced a dropped group connection with a new one",+)+HEARTBEAT_RECONNECT_FAILURES_TOTAL = Counter(+ "warehouse_pg_consumer_heartbeat_reconnect_failures_total",+ "Batch heartbeats that could not open a new group connection or renew the lease on it",+)+DELTA_CONSUMER_METRICS = ConsumerMetrics(batches_processed_total=BATCHES_PROCESSED_TOTAL,batch_processing_duration_seconds=BATCH_PROCESSING_DURATION_SECONDS,@@ -353,6 +364,8 @@ DELTA_CONSUMER_METRICS = ConsumerMetrics(coalesced_set_batches=COALESCED_SET_BATCHES,coalesced_set_runs=COALESCED_SET_RUNS,coalesced_set_rows=COALESCED_SET_ROWS,+ heartbeat_reconnects_total=HEARTBEAT_RECONNECTS_TOTAL,+ heartbeat_reconnect_failures_total=HEARTBEAT_RECONNECT_FAILURES_TOTAL,)_metrics_by_prefix: dict[str, ConsumerMetrics] = {}@@ -430,6 +443,14 @@ def make_consumer_metrics(prefix: str) -> ConsumerMetrics:"Rows per set loaded as a single write",buckets=COALESCED_SET_ROWS_BUCKETS,),+ heartbeat_reconnects_total=Counter(+ f"{p}_heartbeat_reconnects_total",+ "Batch heartbeats that replaced a dropped group connection with a new one",+ ),+ heartbeat_reconnect_failures_total=Counter(+ f"{p}_heartbeat_reconnect_failures_total",+ "Batch heartbeats that could not open a new group connection or renew the lease on it",+ ),)_metrics_by_prefix[prefix] = metricsreturn metrics