Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry Skio read timeouts instead of failing the sync (#126102)

Tom Owers6caa06a8c182
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py
index 23181d58b838..89a21c539e02 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/skio.py
@@ -4,10 +4,14 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.skio.settings import (
@@ -18,6 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.skio.setti
)
REQUEST_TIMEOUT_SECONDS = 60
+MAX_RETRY_ATTEMPTS = 5
# Skio's Hasura instance answers an invalid or revoked token with HTTP 200 and this message in the
# GraphQL errors body (verified against the live API), so credential failures are classified on the
@@ -55,6 +60,15 @@ def _format_timestamp(value: Any) -> str:
return str(value)
+@retry_with_deadline(
+ # Skio's GraphQL endpoint is queried over POST, which the tracked session's transport-level
+ # retry does not cover (it only retries idempotent methods), so a read timeout or dropped
+ # connection would otherwise fail the whole import attempt on the first blip.
+ retry=retry_if_exception_type((requests.ReadTimeout, requests.ConnectionError)),
+ stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
+ wait=wait_exponential_jitter(initial=2, max=60),
+ reraise=True,
+)
def _post_query(
session: requests.Session,
api_token: str,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py
index 6319df8d14b0..bdeec0a4ac98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/source.py
@@ -86,6 +86,13 @@ You can generate an API token in your Skio dashboard: click **API** in the left
FIELD_NOT_FOUND_ERROR: "Your Skio API token does not have access to this table. Generate a new token under API in your Skio dashboard, then reconnect.",
}
+ def get_retryable_errors(self) -> set[str]:
+ # `_post_query` (skio.py) already retries a read timeout or dropped connection with
+ # backoff, since the transport's own retry only covers idempotent methods and this
+ # GraphQL endpoint is queried over POST. If that budget still exhausts, Temporal retries
+ # the whole activity, so the failure is transient and self-recovering.
+ return {"HTTPSConnectionPool(host='graphql.skio.com', port=443)"}
+
def get_schemas(
self,
config: SkioSourceConfig,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py
index 84c1af7b8771..2dd13f3e4a1f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/skio/tests/test_skio.py
@@ -6,7 +6,9 @@ import pytest
from unittest.mock import MagicMock, patch
import requests
+from parameterized import parameterized
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
Script,
@@ -26,6 +28,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.skio.skio
)
from products.warehouse_sources.backend.temporal.data_imports.sources.skio.source import SkioSource
+# Verbatim message shape `requests`/urllib3 raise for a read timeout or a dropped connection
+# against the live host; both are the transient failures `_post_query` retries.
+READ_TIMEOUT = requests.ReadTimeout(
+ "HTTPSConnectionPool(host='graphql.skio.com', port=443): Read timed out. (read timeout=60.0)"
+)
+CONNECTION_ERROR = requests.ConnectionError(
+ "HTTPSConnectionPool(host='graphql.skio.com', port=443): Max retries exceeded with url: /v1/graphql"
+)
+
TOKEN = "test-token"
# Verbatim message from the live API for an invalid or revoked token (Hasura returns it with
@@ -145,6 +156,26 @@ class TestSkioTransport:
assert isinstance(result.raised, requests.HTTPError)
+ @parameterized.expand([("read_timeout", READ_TIMEOUT), ("connection_error", CONNECTION_ERROR)])
+ def test_transient_network_error_is_retried_then_succeeds(self, _name: str, error: BaseException) -> None:
+ # The GraphQL endpoint is queried over POST, which the tracked session's transport-level
+ # retry does not cover (it only retries idempotent methods), so a read timeout or dropped
+ # connection must be retried by the source itself or it fails the whole import attempt on
+ # the first blip.
+ result = _run([error, _response(_page(_rows(0, 1)))])
+
+ assert result.raised is None
+ assert result.rows == _rows(0, 1)
+
+ @parameterized.expand([("read_timeout", READ_TIMEOUT), ("connection_error", CONNECTION_ERROR)])
+ def test_transient_network_error_that_keeps_failing_matches_retryable_errors(
+ self, _name: str, error: BaseException
+ ) -> None:
+ result = _run(always(error))
+
+ assert isinstance(result.raised, type(error))
+ assert error_message_matches(str(result.raised), SkioSource().get_retryable_errors())
+
class TestValidateCredentials:
def test_valid_token(self) -> None: