:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop reporting transient pool timeouts from the v3 lock takeover (#126729)
Tom Owersfe291faa4a68
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.pyindex bae3944a14cf..8879e4c100c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py@@ -15,6 +15,7 @@ from temporalio.client import Client, WorkflowExecutionStatusfrom posthog.exceptions_capture import capture_exceptionfrom posthog.settings import WAREHOUSE_SOURCES_DATABASE_URLfrom posthog.temporal.common.client import sync_connect+from posthog.temporal.common.db_errors import is_transient_db_errorfrom posthog.temporal.common.logger import get_loggerfrom products.data_warehouse.backend.facade.api import update_external_job_status@@ -334,7 +335,8 @@ def _describe_holder_workflow(holder_run_id=holder_run_id,error=str(e),)- capture_exception(e)+ if not is_transient_db_error(e):+ capture_exception(e)return HolderWorkflowDescription(status=None, job=None, status_is_assumed=False)@@ -356,7 +358,8 @@ def _take_over_stale_running_job(reason="queue_db_connect_failed",error=str(e),)- capture_exception(e)+ if not is_transient_db_error(e):+ capture_exception(e)return Falsetry:@@ -374,7 +377,8 @@ def _take_over_stale_running_job(reason="queue_db_query_failed",error=str(e),)- capture_exception(e)+ if not is_transient_db_error(e):+ capture_exception(e)return Falsehold_seconds = (datetime.now(UTC) - holder_job.created_at).total_seconds()@@ -414,7 +418,8 @@ def _take_over_stale_running_job(reason="queue_batch_fail_failed",error=str(e),)- capture_exception(e)+ if not is_transient_db_error(e):+ capture_exception(e)return Falsefinally:conn.close()@@ -437,7 +442,8 @@ def _take_over_stale_running_job(holder_token=holder,error=str(e),)- capture_exception(e)+ if not is_transient_db_error(e):+ capture_exception(e)return Falselogger.warning(diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.pyindex d5a035ba67ce..9b859dc64732 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py@@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom django.db import OperationalError+import psycopgfrom temporalio.client import WorkflowExecutionStatusfrom posthog.models import Organization, Team@@ -471,6 +472,30 @@ class TestTakeOverStaleRunningJob:mock_conn_cls.connect.side_effect = RuntimeError("connection refused")assert self._run() is False+ # A PgBouncer pool-saturation blip (query_wait_timeout) clears on its own, so it must not be+ # reported to error tracking as a bug - unlike a genuine failure from the queue DB query.+ @pytest.mark.parametrize(+ "error,expect_captured",+ [+ pytest.param(psycopg.errors.ProtocolViolation("query_wait_timeout"), False, id="transient_pool_timeout"),+ pytest.param(ValueError("unexpected row shape"), True, id="genuine_bug"),+ ],+ )+ @patch(f"{MODULE}.capture_exception")+ @patch(f"{MODULE}.BatchQueue")+ @patch(f"{MODULE}.psycopg.Connection")+ def test_capture_exception_skipped_for_transient_queue_db_errors(+ self,+ mock_conn_cls: MagicMock,+ mock_queue: MagicMock,+ mock_capture: MagicMock,+ error: Exception,+ expect_captured: bool,+ ) -> None:+ mock_queue.get_run_activity_summary.side_effect = error+ assert self._run() is False+ assert mock_capture.called is expect_captured+@patch(f"{MODULE}._release_and_acquire", return_value=True)@patch(f"{MODULE}.update_external_job_status")@patch(f"{MODULE}.BatchQueue")