Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): stop reporting transient pool timeouts from the v3 lock takeover (#126729)

Tom Owersfe291faa4a68
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py
index bae3944a14cf..8879e4c100c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/acquire_v3_lock.py
@@ -15,6 +15,7 @@ from temporalio.client import Client, WorkflowExecutionStatus
from posthog.exceptions_capture import capture_exception
from posthog.settings import WAREHOUSE_SOURCES_DATABASE_URL
from posthog.temporal.common.client import sync_connect
+from posthog.temporal.common.db_errors import is_transient_db_error
from posthog.temporal.common.logger import get_logger
from products.data_warehouse.backend.facade.api import update_external_job_status
@@ -334,7 +335,8 @@ def _describe_holder_workflow(
holder_run_id=holder_run_id,
error=str(e),
)
- capture_exception(e)
+ if not is_transient_db_error(e):
+ capture_exception(e)
return HolderWorkflowDescription(status=None, job=None, status_is_assumed=False)
@@ -356,7 +358,8 @@ def _take_over_stale_running_job(
reason="queue_db_connect_failed",
error=str(e),
)
- capture_exception(e)
+ if not is_transient_db_error(e):
+ capture_exception(e)
return False
try:
@@ -374,7 +377,8 @@ def _take_over_stale_running_job(
reason="queue_db_query_failed",
error=str(e),
)
- capture_exception(e)
+ if not is_transient_db_error(e):
+ capture_exception(e)
return False
hold_seconds = (datetime.now(UTC) - holder_job.created_at).total_seconds()
@@ -414,7 +418,8 @@ def _take_over_stale_running_job(
reason="queue_batch_fail_failed",
error=str(e),
)
- capture_exception(e)
+ if not is_transient_db_error(e):
+ capture_exception(e)
return False
finally:
conn.close()
@@ -437,7 +442,8 @@ def _take_over_stale_running_job(
holder_token=holder,
error=str(e),
)
- capture_exception(e)
+ if not is_transient_db_error(e):
+ capture_exception(e)
return False
logger.warning(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py
index d5a035ba67ce..9b859dc64732 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/test_acquire_v3_lock.py
@@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
from django.db import OperationalError
+import psycopg
from temporalio.client import WorkflowExecutionStatus
from posthog.models import Organization, Team
@@ -471,6 +472,30 @@ class TestTakeOverStaleRunningJob:
mock_conn_cls.connect.side_effect = RuntimeError("connection refused")
assert self._run() is False
+ # A PgBouncer pool-saturation blip (query_wait_timeout) clears on its own, so it must not be
+ # reported to error tracking as a bug - unlike a genuine failure from the queue DB query.
+ @pytest.mark.parametrize(
+ "error,expect_captured",
+ [
+ pytest.param(psycopg.errors.ProtocolViolation("query_wait_timeout"), False, id="transient_pool_timeout"),
+ pytest.param(ValueError("unexpected row shape"), True, id="genuine_bug"),
+ ],
+ )
+ @patch(f"{MODULE}.capture_exception")
+ @patch(f"{MODULE}.BatchQueue")
+ @patch(f"{MODULE}.psycopg.Connection")
+ def test_capture_exception_skipped_for_transient_queue_db_errors(
+ self,
+ mock_conn_cls: MagicMock,
+ mock_queue: MagicMock,
+ mock_capture: MagicMock,
+ error: Exception,
+ expect_captured: bool,
+ ) -> None:
+ mock_queue.get_run_activity_summary.side_effect = error
+ assert self._run() is False
+ assert mock_capture.called is expect_captured
+
@patch(f"{MODULE}._release_and_acquire", return_value=True)
@patch(f"{MODULE}.update_external_job_status")
@patch(f"{MODULE}.BatchQueue")