Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry a transient db blip in table semantics enrichment (#118235)

Tom Owers55ebe4056df5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/enrich_table_semantics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/enrich_table_semantics.py
index db5c11fd3c58..5709b1a202ec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/enrich_table_semantics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/enrich_table_semantics.py
@@ -56,6 +56,7 @@ from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
get_canonical_descriptions_for_source,
)
+from products.warehouse_sources.backend.temporal.data_imports.util import retry_internal_db_operation
# Write-only (no Kafka `log_entries` production): this is an internal background activity, not a
# user-facing sync, and its workflow type isn't mapped in `resolve_log_source`. The temporal worker's
@@ -294,10 +295,14 @@ def enrich_table_semantics_sync(team_id: int, schema_id: uuid.UUID) -> dict[str,
"""Generate and persist semantic annotations for one warehouse table. Safe to re-run."""
log = logger.bind(team_id=team_id, schema_id=str(schema_id))
- team = (
- Team.objects.select_related("organization")
- .only("id", "uuid", "organization_id", "organization__is_ai_data_processing_approved")
- .get(id=team_id)
+ # Retried individually, not via `with_internal_db_retries` on the whole function, because the
+ # LLM call further down is expensive and must not be repeated for a blip reaching our own database.
+ team = retry_internal_db_operation(
+ lambda: (
+ Team.objects.select_related("organization")
+ .only("id", "uuid", "organization_id", "organization__is_ai_data_processing_approved")
+ .get(id=team_id)
+ )
)
# Context accumulated as we learn it; shared by the "started" and "completed" analytics events.
@@ -313,10 +318,12 @@ def enrich_table_semantics_sync(team_id: int, schema_id: uuid.UUID) -> dict[str,
emit_completed("skipped", reason="ai_data_processing_not_approved")
return {"status": "skipped", "reason": "ai_data_processing_not_approved"}
- schema = (
- ExternalDataSchema.objects.select_related("source", "table")
- .filter(team_id=team_id, deleted=False)
- .get(id=schema_id)
+ schema = retry_internal_db_operation(
+ lambda: (
+ ExternalDataSchema.objects.select_related("source", "table")
+ .filter(team_id=team_id, deleted=False)
+ .get(id=schema_id)
+ )
)
table = schema.table
event_props["source_type"] = schema.source.source_type
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_enrich_table_semantics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_enrich_table_semantics.py
index ed42c502eb0d..65eb238c3f94 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_enrich_table_semantics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_enrich_table_semantics.py
@@ -5,6 +5,8 @@ from typing import Any
import pytest
from unittest.mock import MagicMock, patch
+from django.db.models.query import QuerySet
+from django.db.utils import OperationalError as DjangoOperationalError
from django.test import override_settings
from temporalio.testing import ActivityEnvironment
@@ -39,6 +41,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
pytestmark = pytest.mark.django_db
_USAGE = {"model": "claude-haiku-4-5", "prompt_tokens": 120, "completion_tokens": 40, "total_tokens": 160}
+_UTIL_MODULE = "products.warehouse_sources.backend.temporal.data_imports.util"
@pytest.fixture(autouse=True)
@@ -477,6 +480,42 @@ class TestCanonicalDescriptionsResolver:
class TestEnrichTableSemanticsSync:
+ def test_retries_transient_internal_db_error_on_initial_team_lookup(self):
+ # A DNS blip resolving PostHog's own database must be retried, not raised straight through -
+ # it would otherwise reach error tracking as an unactionable OperationalError on every sync.
+ team = _team()
+ schema, table = _make_schema(
+ team,
+ columns=[{"name": "amount", "data_type": "Int64", "is_nullable": False}],
+ description="Stripe charges",
+ )
+ canonical = {"Charge": {"columns": {"amount": "charge amount in cents"}}}
+
+ failures = 0
+ original_get = QuerySet.get
+
+ def flaky_team_get(self, *args, **kwargs):
+ nonlocal failures
+ if self.model is Team and failures == 0:
+ failures += 1
+ raise DjangoOperationalError("[Errno -2] Name or service not known")
+ return original_get(self, *args, **kwargs)
+
+ with (
+ patch.object(QuerySet, "get", flaky_team_get),
+ patch(f"{_UTIL_MODULE}.close_stale_db_connections") as mock_close,
+ patch("time.sleep"),
+ patch.object(enrich, "get_canonical_descriptions_for_source", return_value=canonical),
+ patch.object(enrich, "_generate_descriptions") as mock_llm,
+ ):
+ result = enrich_table_semantics_sync(team.pk, schema.id)
+
+ mock_llm.assert_not_called()
+ # The helper evicts the connection once per retry, so exactly one retry happened.
+ assert failures == 1
+ mock_close.assert_called_once()
+ assert result == {"status": "done", "canonical_annotations": 1, "ai_annotations": 0}
+
def test_skipped_when_ai_data_processing_not_approved(self):
team = _team()
team.organization.is_ai_data_processing_approved = False