:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse): reconnect person-property consumer after db errors (#127094)
Javier Bahamondesaa0bee7aafdb
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.pyindex 2296155f3214..81081f8a79c9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py@@ -20,6 +20,7 @@ from concurrent.futures import ThreadPoolExecutorfrom typing import Anyfrom django.conf import settings+from django.db import close_old_connectionsimport structlogfrom prometheus_client import Counter@@ -282,11 +283,14 @@ class PersonPropertyUpdateConsumer:try:granted = self._grant()except Exception:- # The limiter degrades a Redis outage to its in-memory fallback internally; anything- # that still escapes here is unexpected. Treat it like a transient failure — back off- # and retry the acquire rather than letting it escape and crash-loop the pod. The- # message stays uncommitted meanwhile, so nothing is sent or dropped.+ # The limiter degrades a Redis outage to its in-memory fallback internally. The rate+ # read hits Postgres and can fail when pgbouncer drops the connection. Treat both like+ # a transient failure — back off and retry the acquire rather than letting it escape+ # and crash-loop the pod. The message stays uncommitted meanwhile, so nothing is sent+ # or dropped.logger.exception("person_property_update.grant_error")+ # This loop runs outside a request, so Django never drops a dead connection on its own.+ close_old_connections()self._health_reporter()self._sleep(_RETRY_BACKOFF_SECONDS)continuediff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.pyindex 1524796a3e23..ccb69a584e7a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py@@ -5,6 +5,7 @@ from types import SimpleNamespaceimport pytestfrom unittest.mock import MagicMock, patch+from django.db import OperationalErrorfrom django.test import override_settingsfrom parameterized import parameterized@@ -354,6 +355,29 @@ class TestRateGate:assert capture.call_count == 1assert len(sleeps) == 2 # backed off once per grant error before the grant+ def test_grant_error_from_a_dead_db_connection_recovers_after_reset(self):+ connection = {"alive": False}++ def grant() -> bool:+ if not connection["alive"]:+ raise OperationalError("the connection is closed")+ return True++ capture = MagicMock(return_value=_capture_result(succeeded=True))+ c = PersonPropertyUpdateConsumer(capture_fn=capture, grant_fn=grant, dlq_producer=MagicMock())+ retries: list[float] = []++ def sleep(seconds: float) -> None:+ retries.append(seconds)+ if len(retries) >= 5:+ c._shutdown = True++ c._sleep = sleep++ with patch(f"{_MODULE}.close_old_connections", side_effect=lambda: connection.update(alive=True)):+ assert c.process_record(self._VALUE) == SENT+ assert capture.call_count == 1+def test_reports_liveness_while_throttled(self):# A pod that's merely rate-limited must keep heartbeating, or its liveness probe kills it# mid-wait. The reporter fires on each throttled poll.