Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse): reconnect person-property consumer after db errors (#127094)

Javier Bahamondesaa0bee7aafdb
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py
index 2296155f3214..81081f8a79c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py
@@ -20,6 +20,7 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Any
from django.conf import settings
+from django.db import close_old_connections
import structlog
from prometheus_client import Counter
@@ -282,11 +283,14 @@ class PersonPropertyUpdateConsumer:
try:
granted = self._grant()
except Exception:
- # The limiter degrades a Redis outage to its in-memory fallback internally; anything
- # that still escapes here is unexpected. Treat it like a transient failure — back off
- # and retry the acquire rather than letting it escape and crash-loop the pod. The
- # message stays uncommitted meanwhile, so nothing is sent or dropped.
+ # The limiter degrades a Redis outage to its in-memory fallback internally. The rate
+ # read hits Postgres and can fail when pgbouncer drops the connection. Treat both like
+ # a transient failure — back off and retry the acquire rather than letting it escape
+ # and crash-loop the pod. The message stays uncommitted meanwhile, so nothing is sent
+ # or dropped.
logger.exception("person_property_update.grant_error")
+ # This loop runs outside a request, so Django never drops a dead connection on its own.
+ close_old_connections()
self._health_reporter()
self._sleep(_RETRY_BACKOFF_SECONDS)
continue
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py
index 1524796a3e23..ccb69a584e7a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py
@@ -5,6 +5,7 @@ from types import SimpleNamespace
import pytest
from unittest.mock import MagicMock, patch
+from django.db import OperationalError
from django.test import override_settings
from parameterized import parameterized
@@ -354,6 +355,29 @@ class TestRateGate:
assert capture.call_count == 1
assert len(sleeps) == 2 # backed off once per grant error before the grant
+ def test_grant_error_from_a_dead_db_connection_recovers_after_reset(self):
+ connection = {"alive": False}
+
+ def grant() -> bool:
+ if not connection["alive"]:
+ raise OperationalError("the connection is closed")
+ return True
+
+ capture = MagicMock(return_value=_capture_result(succeeded=True))
+ c = PersonPropertyUpdateConsumer(capture_fn=capture, grant_fn=grant, dlq_producer=MagicMock())
+ retries: list[float] = []
+
+ def sleep(seconds: float) -> None:
+ retries.append(seconds)
+ if len(retries) >= 5:
+ c._shutdown = True
+
+ c._sleep = sleep
+
+ with patch(f"{_MODULE}.close_old_connections", side_effect=lambda: connection.update(alive=True)):
+ assert c.process_record(self._VALUE) == SENT
+ assert capture.call_count == 1
+
def test_reports_liveness_while_throttled(self):
# A pod that's merely rate-limited must keep heartbeating, or its liveness probe kills it
# mid-wait. The reporter fires on each throttled poll.