Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(customer-analytics): retry google transport errors in calendar sync quietly (#119353)

posthog[bot]428c7c576fff
diff --git a/products/customer_analytics/backend/temporal/calendar_sync.py b/products/customer_analytics/backend/temporal/calendar_sync.py
index 44b6dc97582d..61a7ff60666e 100644
--- a/products/customer_analytics/backend/temporal/calendar_sync.py
+++ b/products/customer_analytics/backend/temporal/calendar_sync.py
@@ -24,6 +24,7 @@ with workflow.unsafe.imports_passed_through():
import structlog
from posthog.sync import database_sync_to_async
+ from posthog.temporal.common.errors import NonReportableApplicationError
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule
@@ -35,6 +36,7 @@ COORDINATOR_INTERVAL_MINUTES = 5
MAX_SYNCS_PER_RUN = 200
BACKFILL_PAGES_PER_RUN = 100
GOOGLE_WORKSPACE_RETRY_DELAY = timedelta(hours=1)
+GOOGLE_TRANSPORT_RETRY_DELAY = timedelta(seconds=30)
@dataclass
@@ -171,8 +173,19 @@ def _create_google_workspace_budget_error(integration_id: int, team_id: int, err
return ApplicationError(str(error), next_retry_delay=GOOGLE_WORKSPACE_RETRY_DELAY)
+def _create_google_transport_error(error: Exception) -> ApplicationError:
+ # A proxy or network blip usually clears on the next attempt, so report it only when the retries run out.
+ info = activity.info()
+ max_attempts = info.retry_policy.maximum_attempts if info.retry_policy else 0
+ if max_attempts and info.attempt >= max_attempts:
+ return ApplicationError(str(error), next_retry_delay=GOOGLE_TRANSPORT_RETRY_DELAY)
+ return NonReportableApplicationError(str(error), next_retry_delay=GOOGLE_TRANSPORT_RETRY_DELAY)
+
+
def _run_calendar_sync(input: CalendarSyncInput) -> CalendarSyncOutput:
# Deferred: the sync logic pulls requests/HogQL layers that don't belong in the sandbox.
+ import requests # noqa: PLC0415
+
from posthog.egress.google_workspace.transport import GoogleWorkspaceEgressBudgetExhausted # noqa: PLC0415
from products.conversations.backend.facade import api as conversations # noqa: PLC0415
@@ -189,6 +202,8 @@ def _run_calendar_sync(input: CalendarSyncInput) -> CalendarSyncOutput:
raise ApplicationError(str(e), non_retryable="refresh failed" in str(e).lower()) from e
except GoogleWorkspaceEgressBudgetExhausted as error:
raise _create_google_workspace_budget_error(input.integration_id, input.team_id, error) from error
+ except (requests.ConnectionError, requests.Timeout) as error:
+ raise _create_google_transport_error(error) from error
return CalendarSyncOutput(
fetched=counts.fetched,
upserted=counts.upserted,
diff --git a/products/customer_analytics/backend/test/test_calendar_sync.py b/products/customer_analytics/backend/test/test_calendar_sync.py
index 3a07b27bf609..69839adbf6ef 100644
--- a/products/customer_analytics/backend/test/test_calendar_sync.py
+++ b/products/customer_analytics/backend/test/test_calendar_sync.py
@@ -4,11 +4,15 @@ import time_machine
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
+from django.test import SimpleTestCase
from django.utils import timezone
+import requests
from parameterized import parameterized
+from temporalio.common import RetryPolicy
from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration
+from posthog.temporal.common.posthog_client import is_expected_activity_failure
from products.customer_analytics.backend.logic import calendar_sync
from products.customer_analytics.backend.models import Account, Meeting, MeetingParticipant, MeetingStatus
@@ -365,3 +369,26 @@ class TestCalendarSync(BaseTest):
meeting.refresh_from_db()
assert updated == 1
assert meeting.account_id == account.id
+
+
+class TestCalendarSyncTransportErrors(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("proxy_error_first_attempt", requests.exceptions.ProxyError("504"), 1, True),
+ ("read_timeout_second_attempt", requests.exceptions.ReadTimeout("timed out"), 2, True),
+ ("proxy_error_final_attempt", requests.exceptions.ProxyError("504"), 3, False),
+ ]
+ )
+ def test_transport_error_is_reported_only_on_final_attempt(
+ self, _name: str, error: Exception, attempt: int, expected: bool
+ ) -> None:
+ info = MagicMock(attempt=attempt, retry_policy=RetryPolicy(maximum_attempts=3))
+ with (
+ patch.object(temporal_calendar_sync.activity, "info", return_value=info),
+ patch.object(calendar_sync, "sync_calendar_integration", side_effect=error),
+ self.assertRaises(temporal_calendar_sync.ApplicationError) as raised,
+ ):
+ temporal_calendar_sync._run_calendar_sync(temporal_calendar_sync.CalendarSyncInput(1, 1))
+
+ assert not raised.exception.non_retryable
+ assert is_expected_activity_failure(raised.exception) is expected