:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(customer-analytics): retry google transport errors in calendar sync quietly (#119353)
posthog[bot]428c7c576fff
diff --git a/products/customer_analytics/backend/temporal/calendar_sync.py b/products/customer_analytics/backend/temporal/calendar_sync.pyindex 44b6dc97582d..61a7ff60666e 100644--- a/products/customer_analytics/backend/temporal/calendar_sync.py+++ b/products/customer_analytics/backend/temporal/calendar_sync.py@@ -24,6 +24,7 @@ with workflow.unsafe.imports_passed_through():import structlogfrom posthog.sync import database_sync_to_async+ from posthog.temporal.common.errors import NonReportableApplicationErrorfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule@@ -35,6 +36,7 @@ COORDINATOR_INTERVAL_MINUTES = 5MAX_SYNCS_PER_RUN = 200BACKFILL_PAGES_PER_RUN = 100GOOGLE_WORKSPACE_RETRY_DELAY = timedelta(hours=1)+GOOGLE_TRANSPORT_RETRY_DELAY = timedelta(seconds=30)@dataclass@@ -171,8 +173,19 @@ def _create_google_workspace_budget_error(integration_id: int, team_id: int, errreturn ApplicationError(str(error), next_retry_delay=GOOGLE_WORKSPACE_RETRY_DELAY)+def _create_google_transport_error(error: Exception) -> ApplicationError:+ # A proxy or network blip usually clears on the next attempt, so report it only when the retries run out.+ info = activity.info()+ max_attempts = info.retry_policy.maximum_attempts if info.retry_policy else 0+ if max_attempts and info.attempt >= max_attempts:+ return ApplicationError(str(error), next_retry_delay=GOOGLE_TRANSPORT_RETRY_DELAY)+ return NonReportableApplicationError(str(error), next_retry_delay=GOOGLE_TRANSPORT_RETRY_DELAY)++def _run_calendar_sync(input: CalendarSyncInput) -> CalendarSyncOutput:# Deferred: the sync logic pulls requests/HogQL layers that don't belong in the sandbox.+ import requests # noqa: PLC0415+from posthog.egress.google_workspace.transport import GoogleWorkspaceEgressBudgetExhausted # noqa: PLC0415from products.conversations.backend.facade import api as conversations # noqa: PLC0415@@ -189,6 +202,8 @@ def _run_calendar_sync(input: CalendarSyncInput) -> CalendarSyncOutput:raise ApplicationError(str(e), non_retryable="refresh failed" in str(e).lower()) from eexcept GoogleWorkspaceEgressBudgetExhausted as error:raise _create_google_workspace_budget_error(input.integration_id, input.team_id, error) from error+ except (requests.ConnectionError, requests.Timeout) as error:+ raise _create_google_transport_error(error) from errorreturn CalendarSyncOutput(fetched=counts.fetched,upserted=counts.upserted,diff --git a/products/customer_analytics/backend/test/test_calendar_sync.py b/products/customer_analytics/backend/test/test_calendar_sync.pyindex 3a07b27bf609..69839adbf6ef 100644--- a/products/customer_analytics/backend/test/test_calendar_sync.py+++ b/products/customer_analytics/backend/test/test_calendar_sync.py@@ -4,11 +4,15 @@ import time_machinefrom posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patch+from django.test import SimpleTestCasefrom django.utils import timezone+import requestsfrom parameterized import parameterized+from temporalio.common import RetryPolicyfrom posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration+from posthog.temporal.common.posthog_client import is_expected_activity_failurefrom products.customer_analytics.backend.logic import calendar_syncfrom products.customer_analytics.backend.models import Account, Meeting, MeetingParticipant, MeetingStatus@@ -365,3 +369,26 @@ class TestCalendarSync(BaseTest):meeting.refresh_from_db()assert updated == 1assert meeting.account_id == account.id+++class TestCalendarSyncTransportErrors(SimpleTestCase):+ @parameterized.expand(+ [+ ("proxy_error_first_attempt", requests.exceptions.ProxyError("504"), 1, True),+ ("read_timeout_second_attempt", requests.exceptions.ReadTimeout("timed out"), 2, True),+ ("proxy_error_final_attempt", requests.exceptions.ProxyError("504"), 3, False),+ ]+ )+ def test_transport_error_is_reported_only_on_final_attempt(+ self, _name: str, error: Exception, attempt: int, expected: bool+ ) -> None:+ info = MagicMock(attempt=attempt, retry_policy=RetryPolicy(maximum_attempts=3))+ with (+ patch.object(temporal_calendar_sync.activity, "info", return_value=info),+ patch.object(calendar_sync, "sync_calendar_integration", side_effect=error),+ self.assertRaises(temporal_calendar_sync.ApplicationError) as raised,+ ):+ temporal_calendar_sync._run_calendar_sync(temporal_calendar_sync.CalendarSyncInput(1, 1))++ assert not raised.exception.non_retryable+ assert is_expected_activity_failure(raised.exception) is expected