:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry Skio read timeouts instead of failing the sync (#126102)
Tom Owers6caa06a8c182
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.pyindex 23181d58b838..89a21c539e02 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py@@ -4,10 +4,14 @@ from typing import Any, Optionalimport requestsfrom structlog.types import FilteringBoundLogger+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom posthog.dataclasses import frozen-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (+ make_tracked_session,+ retry_with_deadline,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.skio.settings import (@@ -18,6 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.skio.setti)REQUEST_TIMEOUT_SECONDS = 60+MAX_RETRY_ATTEMPTS = 5# Skio's Hasura instance answers an invalid or revoked token with HTTP 200 and this message in the# GraphQL errors body (verified against the live API), so credential failures are classified on the@@ -55,6 +60,15 @@ def _format_timestamp(value: Any) -> str:return str(value)+@retry_with_deadline(+ # Skio's GraphQL endpoint is queried over POST, which the tracked session's transport-level+ # retry does not cover (it only retries idempotent methods), so a read timeout or dropped+ # connection would otherwise fail the whole import attempt on the first blip.+ retry=retry_if_exception_type((requests.ReadTimeout, requests.ConnectionError)),+ stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),+ wait=wait_exponential_jitter(initial=2, max=60),+ reraise=True,+)def _post_query(session: requests.Session,api_token: str,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.pyindex 6319df8d14b0..bdeec0a4ac98 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py@@ -86,6 +86,13 @@ You can generate an API token in your Skio dashboard: click **API** in the leftFIELD_NOT_FOUND_ERROR: "Your Skio API token does not have access to this table. Generate a new token under API in your Skio dashboard, then reconnect.",}+ def get_retryable_errors(self) -> set[str]:+ # `_post_query` (skio.py) already retries a read timeout or dropped connection with+ # backoff, since the transport's own retry only covers idempotent methods and this+ # GraphQL endpoint is queried over POST. If that budget still exhausts, Temporal retries+ # the whole activity, so the failure is transient and self-recovering.+ return {"HTTPSConnectionPool(host='graphql.skio.com', port=443)"}+def get_schemas(self,config: SkioSourceConfig,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.pyindex 84c1af7b8771..2dd13f3e4a1f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py@@ -6,7 +6,9 @@ import pytestfrom unittest.mock import MagicMock, patchimport requests+from parameterized import parameterized+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (DriveResult,Script,@@ -26,6 +28,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.skio.skio)from products.warehouse_sources.backend.temporal.data_imports.sources.skio.source import SkioSource+# Verbatim message shape `requests`/urllib3 raise for a read timeout or a dropped connection+# against the live host; both are the transient failures `_post_query` retries.+READ_TIMEOUT = requests.ReadTimeout(+ "HTTPSConnectionPool(host='graphql.skio.com', port=443): Read timed out. (read timeout=60.0)"+)+CONNECTION_ERROR = requests.ConnectionError(+ "HTTPSConnectionPool(host='graphql.skio.com', port=443): Max retries exceeded with url: /v1/graphql"+)+TOKEN = "test-token"# Verbatim message from the live API for an invalid or revoked token (Hasura returns it with@@ -145,6 +156,26 @@ class TestSkioTransport:assert isinstance(result.raised, requests.HTTPError)+ @parameterized.expand([("read_timeout", READ_TIMEOUT), ("connection_error", CONNECTION_ERROR)])+ def test_transient_network_error_is_retried_then_succeeds(self, _name: str, error: BaseException) -> None:+ # The GraphQL endpoint is queried over POST, which the tracked session's transport-level+ # retry does not cover (it only retries idempotent methods), so a read timeout or dropped+ # connection must be retried by the source itself or it fails the whole import attempt on+ # the first blip.+ result = _run([error, _response(_page(_rows(0, 1)))])++ assert result.raised is None+ assert result.rows == _rows(0, 1)++ @parameterized.expand([("read_timeout", READ_TIMEOUT), ("connection_error", CONNECTION_ERROR)])+ def test_transient_network_error_that_keeps_failing_matches_retryable_errors(+ self, _name: str, error: BaseException+ ) -> None:+ result = _run(always(error))++ assert isinstance(result.raised, type(error))+ assert error_message_matches(str(result.raised), SkioSource().get_retryable_errors())+class TestValidateCredentials:def test_valid_token(self) -> None: