Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): move postgres and mysql plugin vendors to the plugin folder (#128258)

Tom Owers0f53017e8cdc
diff --git a/products/warehouse_sources/backend/management/commands/generate_source_configs.py b/products/warehouse_sources/backend/management/commands/generate_source_configs.py
index 9d015590e996..19b653027a9b 100644
--- a/products/warehouse_sources/backend/management/commands/generate_source_configs.py
+++ b/products/warehouse_sources/backend/management/commands/generate_source_configs.py
@@ -62,6 +62,7 @@ class SourceConfigGenerator:
def __init__(self):
self.generated_classes: dict[str, str] = {}
self.imports: set[str] = set()
+ self.is_plugin_vendor = False
self.typing_import: set[str] = set()
self.nested_configs: dict[str, str] = {}
@@ -122,6 +123,7 @@ class SourceConfigGenerator:
def generate_source_config(self, source_type: SourceKey, source_config: SourceConfig) -> None:
self.imports.add(self._config_import(source_type))
+ self.is_plugin_vendor = self._config_import(source_type).startswith(f"from {PLUGIN_SOURCES_PACKAGE}.sdk")
class_name = self._get_config_class_name(source_type)
fields = []
@@ -433,7 +435,10 @@ class SourceConfigGenerator:
def _process_ssh_tunnel_field(self, field: SourceFieldSSHTunnelConfig) -> str:
"""Process a SSH tunnel field by referencing the existing SSHTunnelConfig."""
- self.imports.add("from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig")
+ if self.is_plugin_vendor:
+ self.imports.add(f"from {PLUGIN_SOURCES_PACKAGE}.sdk import SSHTunnelConfig")
+ else:
+ self.imports.add("from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig")
python_field_name, should_alias = self._make_python_identifier(field.name)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
index ac31f696f72a..99471453674a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
@@ -77,9 +77,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.sour
_INVALID_CREDENTIALS_ERROR,
MySQLSource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source import (
- PlanetScaleMySQLSource,
-)
from products.warehouse_sources.backend.types import IncrementalFieldType
# ---------------------------------------------------------------------------
@@ -1164,9 +1161,8 @@ class TestStreamingConnectionTimeouts:
],
ids=["read_timeout", "write_timeout"],
)
- @pytest.mark.parametrize("source_class", [MySQLSource, PlanetScaleMySQLSource])
- def test_driver_timeout_is_retryable(self, source_class, error, kept_out_of_error_tracking):
- source = source_class()
+ def test_driver_timeout_is_retryable(self, error, kept_out_of_error_tracking):
+ source = MySQLSource()
assert error_message_matches(str(error), source.get_retryable_errors()) is kept_out_of_error_tracking
assert not error_message_matches(str(error), {**Any_Source_Errors, **source.get_non_retryable_errors()})
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py
index 77da927398a9..9384dfd6029b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py
@@ -8,10 +8,6 @@ import psycopg
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errors
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
-from products.warehouse_sources.backend.temporal.data_imports.sources.neon.source import NeonSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_postgres.source import (
- PlanetScalePostgresSource,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres import client_deadline
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (
ClientDeadlineExceededError,
@@ -19,7 +15,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.c
deadline_server_cursor_factory,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.supabase.source import SupabaseSource
def test_a_statement_that_ends_in_time_is_not_cancelled() -> None:
@@ -180,9 +175,8 @@ def test_a_silent_server_cursor_round_trip_ends_at_the_deadline(round_trip, expe
round_trip(cursor)
[email protected]("source_class", [PostgresSource, NeonSource, SupabaseSource, PlanetScalePostgresSource])
-def test_the_deadline_error_is_retryable_for_every_postgres_source(source_class: type[PostgresSource]) -> None:
- source = source_class()
+def test_the_deadline_error_is_retryable_for_the_postgres_source() -> None:
+ source = PostgresSource()
message = str(ClientDeadlineExceededError(660))
assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py
index 82430c366bd8..4ea135731fcc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py
@@ -32,9 +32,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
SnowflakeSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.stripe import StripeSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.supabase import (
- SupabaseSourceConfig,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.temporalio import (
TemporalIOSourceConfig,
)
@@ -50,6 +47,7 @@ from products.warehouse_sources.sources.doit._config import DoItSourceConfig
from products.warehouse_sources.sources.gitlab._config import GitLabSourceConfig
from products.warehouse_sources.sources.gorgias._config import GorgiasSourceConfig
from products.warehouse_sources.sources.shopify._config import ShopifySourceConfig
+from products.warehouse_sources.sources.supabase._config import SupabaseSourceConfig
# The form submits "temporary-dataset", but dataclasses.asdict() persists "temporary_dataset",
diff --git a/products/warehouse_sources/sources/neon/__init__.py b/products/warehouse_sources/sources/neon/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py b/products/warehouse_sources/sources/neon/_config.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py
rename to products/warehouse_sources/sources/neon/_config.py
index f8a099c08b16..25c34b5ce2ed 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py
+++ b/products/warehouse_sources/sources/neon/_config.py
@@ -1,8 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import SSHTunnelConfig, config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py b/products/warehouse_sources/sources/neon/source.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py
rename to products/warehouse_sources/sources/neon/source.py
index 123352d2e90a..017f478c0106 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py
+++ b/products/warehouse_sources/sources/neon/source.py
@@ -1,23 +1,21 @@
from collections.abc import Mapping
from typing import Any, Optional
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
SourceConfig,
SourceFieldInputConfig,
+ SourceRegistry,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
- PostgresSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
+from products.warehouse_sources.sources.sdk.internals import (
_DATABASE_NOT_FOUND_VALIDATION_ERROR,
PostgresSource,
+ PostgresSourceConfig,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
_NEON_POOLED_HOST_CDC_ERROR = (
"This is Neon's pooled endpoint, which doesn't support logical replication. For CDC, use the "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py b/products/warehouse_sources/sources/neon/test_neon_source.py
similarity index 77%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py
rename to products/warehouse_sources/sources/neon/test_neon_source.py
index 97d5d80b4674..b42a340b0694 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py
+++ b/products/warehouse_sources/sources/neon/test_neon_source.py
@@ -1,15 +1,15 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.neon.source import (
- _NEON_DATABASE_NOT_FOUND_ERROR,
- NeonSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
+from products.warehouse_sources.sources.neon.source import _NEON_DATABASE_NOT_FOUND_ERROR, NeonSource
+from products.warehouse_sources.sources.sdk.internals import (
_DATABASE_NOT_FOUND_VALIDATION_ERROR,
_INVALID_CREDENTIALS_VALIDATION_ERROR,
+ Any_Source_Errors,
+ ClientDeadlineExceededError,
PostgresSource,
)
+from products.warehouse_sources.sources.sdk.testing import error_message_matches
@pytest.mark.parametrize(
@@ -66,3 +66,12 @@ def test_validate_credentials_points_a_missing_database_at_the_neon_connect_dial
with mock.patch.object(PostgresSource, "validate_credentials", return_value=postgres_result):
assert NeonSource().validate_credentials(config, team_id=1) == expected
+
+
+def test_the_deadline_error_is_retryable() -> None:
+ source = NeonSource()
+ message = str(ClientDeadlineExceededError(660))
+
+ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})
+ assert error_message_matches(message, source.get_retryable_errors())
+ assert error_message_matches(message, source.get_retry_exhausted_errors())
diff --git a/products/warehouse_sources/sources/planetscale_mysql/__init__.py b/products/warehouse_sources/sources/planetscale_mysql/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.py b/products/warehouse_sources/sources/planetscale_mysql/_config.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.py
rename to products/warehouse_sources/sources/planetscale_mysql/_config.py
index fb576543bdf3..56b8015e0095 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.py
+++ b/products/warehouse_sources/sources/planetscale_mysql/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.py b/products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.py
rename to products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.py
index f518de907671..6b122a1dfe54 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.py
+++ b/products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.py
@@ -6,11 +6,11 @@ from django.conf import settings
import pymysql
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (
+from products.warehouse_sources.sources.sdk.internals import (
_MYSQL_SAFE_CONVERSIONS,
METADATA_READ_TIMEOUT_SECONDS,
MySQLImplementation,
+ MySQLSourceConfig,
_connect_with_transient_retry,
connection_timeouts,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.py b/products/warehouse_sources/sources/planetscale_mysql/source.py
similarity index 91%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.py
rename to products/warehouse_sources/sources/planetscale_mysql/source.py
index 325b2fda8834..ab736aac15a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.py
+++ b/products/warehouse_sources/sources/planetscale_mysql/source.py
@@ -1,24 +1,20 @@
import re
from typing import Optional, cast
-from posthog.exceptions_capture import capture_exception
-
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation
+from products.warehouse_sources.sources.sdk import (
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
+ SourceRegistry,
+ capture_exception,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.source import MySQLSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (
- PlanetScaleMySQLImplementation,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk.internals import MySQLSource, MySQLSourceConfig
_PLANETSCALE_MYSQL_IMPLEMENTATION = PlanetScaleMySQLImplementation()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.py b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.py
rename to products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.py
index 4feb44bfafc3..4b71d30b8dc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.py
+++ b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.py
@@ -4,23 +4,16 @@ from typing import Any, cast
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalemysql import (
- PlanetScaleMySQLSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (
+from products.warehouse_sources.sources.planetscale_mysql._config import PlanetScaleMySQLSourceConfig
+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation
+from products.warehouse_sources.sources.sdk.internals import (
CONNECT_TIMEOUT_SECONDS,
METADATA_READ_TIMEOUT_SECONDS,
WRITE_TIMEOUT_SECONDS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (
- PlanetScaleMySQLImplementation,
+ MySQLSourceConfig,
)
-_CONNECT_TARGET = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql."
- "_connect_with_transient_retry"
-)
+_CONNECT_TARGET = "products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql._connect_with_transient_retry"
def _config(port: int | str = 3306) -> MySQLSourceConfig:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.py b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.py
rename to products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.py
index f79af7739de1..146ea148b2eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.py
+++ b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.py
@@ -4,18 +4,14 @@ from typing import cast
import pytest
from unittest import mock
-from products.warehouse_sources.backend.facade.source_config import SourceFieldInputConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalemysql import (
- PlanetScaleMySQLSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (
- PlanetScaleMySQLImplementation,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source import (
- PlanetScaleMySQLSource,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+import pymysql
+
+from products.warehouse_sources.sources.planetscale_mysql._config import PlanetScaleMySQLSourceConfig
+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation
+from products.warehouse_sources.sources.planetscale_mysql.source import PlanetScaleMySQLSource
+from products.warehouse_sources.sources.sdk import ExternalDataSourceType, SourceFieldInputConfig
+from products.warehouse_sources.sources.sdk.internals import Any_Source_Errors, MySQLSourceConfig
+from products.warehouse_sources.sources.sdk.testing import error_message_matches
def _config(host: str = "aws.connect.psdb.cloud") -> MySQLSourceConfig:
@@ -173,9 +169,7 @@ def test_connection_failures_map_to_actionable_messages(raw_error, expected_frag
with (
mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),
mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception(raw_error)),
- mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"
- ) as capture,
+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,
):
success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)
@@ -200,9 +194,7 @@ def test_retryable_sync_errors_are_not_captured_during_validation(raw_error):
with (
mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),
mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception(raw_error)),
- mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"
- ) as capture,
+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,
):
success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)
@@ -218,12 +210,24 @@ def test_unrecognized_failure_is_captured_and_reported_generically():
with (
mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),
mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception("something unexpected")),
- mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"
- ) as capture,
+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,
):
success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)
capture.assert_called_once()
assert success is False
assert error == "Could not connect to PlanetScale MySQL. Please check all connection details are valid."
+
+
+ "error",
+ [
+ pymysql.err.OperationalError(2013, "Lost connection to MySQL server during query (timed out)"),
+ pymysql.err.OperationalError(2006, "MySQL server has gone away (TimeoutError('timed out'))"),
+ ],
+ ids=["read_timeout", "write_timeout"],
+)
+def test_driver_timeout_is_not_non_retryable(error: pymysql.err.OperationalError) -> None:
+ source = PlanetScaleMySQLSource()
+
+ assert not error_message_matches(str(error), {**Any_Source_Errors, **source.get_non_retryable_errors()})
diff --git a/products/warehouse_sources/sources/planetscale_postgres/__init__.py b/products/warehouse_sources/sources/planetscale_postgres/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py b/products/warehouse_sources/sources/planetscale_postgres/_config.py
similarity index 87%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py
rename to products/warehouse_sources/sources/planetscale_postgres/_config.py
index d04aa946055a..c3dc64a89f42 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py
+++ b/products/warehouse_sources/sources/planetscale_postgres/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py b/products/warehouse_sources/sources/planetscale_postgres/source.py
similarity index 90%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py
rename to products/warehouse_sources/sources/planetscale_postgres/source.py
index ff3da3ffda51..d4287cbbd286 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py
+++ b/products/warehouse_sources/sources/planetscale_postgres/source.py
@@ -2,21 +2,18 @@ import re
from collections.abc import Mapping
from typing import Any
-from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ NOT_DETECTABLE,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
SourceConfig,
SourceFieldInputConfig,
SourceFieldSSHTunnelConfig,
+ SourceRegistry,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
- PostgresSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk.internals import PostgresSource, PostgresSourceConfig
# planetscale.com is the dashboard, never a database endpoint — Postgres hosts live on
# psdb.cloud. Pasting the dashboard address otherwise fails as an opaque connection timeout.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py b/products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py
rename to products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.py
index 682f2a7e6b12..8cbfbf1679b5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py
+++ b/products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.py
@@ -1,18 +1,16 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.facade.source_config import SourceFieldInputConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalepostgres import (
- PlanetScalePostgresSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
+from products.warehouse_sources.sources.planetscale_postgres._config import PlanetScalePostgresSourceConfig
+from products.warehouse_sources.sources.planetscale_postgres.source import PlanetScalePostgresSource, _is_psbouncer_port
+from products.warehouse_sources.sources.sdk import SourceFieldInputConfig
+from products.warehouse_sources.sources.sdk.internals import (
+ Any_Source_Errors,
+ ClientDeadlineExceededError,
+ PostgresSource,
PostgresSourceConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_postgres.source import (
- PlanetScalePostgresSource,
- _is_psbouncer_port,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource
+from products.warehouse_sources.sources.sdk.testing import error_message_matches
_DIRECT_HOST = "xxxxxxxxxx-useast1-1.horizon.psdb.cloud"
@@ -116,3 +114,12 @@ def test_cdc_on_the_direct_port_defers_to_postgres():
super_check.assert_called_once()
assert super_check.call_args.kwargs["team_id"] == 7
assert errors == []
+
+
+def test_the_deadline_error_is_retryable() -> None:
+ source = PlanetScalePostgresSource()
+ message = str(ClientDeadlineExceededError(660))
+
+ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})
+ assert error_message_matches(message, source.get_retryable_errors())
+ assert error_message_matches(message, source.get_retry_exhausted_errors())
diff --git a/products/warehouse_sources/sources/sdk/__init__.py b/products/warehouse_sources/sources/sdk/__init__.py
index b1edfaa8f28b..66e85f639162 100644
--- a/products/warehouse_sources/sources/sdk/__init__.py
+++ b/products/warehouse_sources/sources/sdk/__init__.py
@@ -22,6 +22,7 @@ from posthog.models.integration import (
)
from posthog.models.integration.databricks import NOT_A_DATABRICKS_HOST_MESSAGE, is_databricks_workspace_host
from posthog.schema_enums import ProductKey
+from posthog.security.postgres_hosts import SUPABASE_DIRECT_HOST_IPV4_HINT, SUPABASE_DIRECT_HOST_RE
from posthog.security.url_validation import is_url_allowed
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.shutdown import WorkerShuttingDownError
@@ -40,6 +41,7 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldOauthConfig,
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
+ SourceFieldSSHTunnelConfig,
SourceFieldSwitchGroupConfig,
SuggestedTable,
)
@@ -53,7 +55,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
ExternalDataSchema,
update_sync_type_config_keys,
)
-from products.warehouse_sources.backend.models.ssh_tunnel import from_private_key
+from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig, from_private_key
from products.warehouse_sources.backend.temporal.data_imports.sources.common import (
config,
credentials_service,
@@ -390,6 +392,9 @@ __all__ = [
"RowFilterColumn",
"SQLSource",
"SQLSourceImplementation",
+ "SSHTunnelConfig",
+ "SUPABASE_DIRECT_HOST_IPV4_HINT",
+ "SUPABASE_DIRECT_HOST_RE",
"SafeSQL",
"SelectQueryBuilder",
"SimpleSource",
@@ -403,6 +408,7 @@ __all__ = [
"SourceFieldInputConfigType",
"SourceFieldOauthAccountSelectConfig",
"SourceFieldOauthConfig",
+ "SourceFieldSSHTunnelConfig",
"SourceFieldSelectConfig",
"SourceFieldSelectConfigOption",
"SourceFieldSwitchGroupConfig",
diff --git a/products/warehouse_sources/sources/sdk/internals.py b/products/warehouse_sources/sources/sdk/internals.py
index 25738dcb0774..4b48d7d248a3 100644
--- a/products/warehouse_sources/sources/sdk/internals.py
+++ b/products/warehouse_sources/sources/sdk/internals.py
@@ -39,7 +39,30 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_
WORKSPACE_FIELDS,
LogAnalyticsWorkspaceSource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
+ PostgresSourceConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.mongodb.source import MongoDBSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (
+ _MYSQL_SAFE_CONVERSIONS,
+ CONNECT_TIMEOUT_SECONDS,
+ METADATA_READ_TIMEOUT_SECONDS,
+ WRITE_TIMEOUT_SECONDS,
+ MySQLImplementation,
+ _connect_with_transient_retry,
+ connection_timeouts,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.source import MySQLSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (
+ ClientDeadlineExceededError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
+ _DATABASE_NOT_FOUND_VALIDATION_ERROR,
+ _HOST_UNREACHABLE_ERROR,
+ _INVALID_CREDENTIALS_VALIDATION_ERROR,
+ PostgresSource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.tests.contract_harness import (
check_extraction,
start_source,
@@ -55,16 +78,31 @@ __all__ = [
"AzureLogAnalyticsResumeConfig",
"Batcher",
"BinaryColumnReporter",
+ "CONNECT_TIMEOUT_SECONDS",
+ "ClientDeadlineExceededError",
"DISCOVER_SCHEMAS_ACTIVITY_TIMEOUT",
"LogAnalyticsWorkspaceSource",
+ "METADATA_READ_TIMEOUT_SECONDS",
"MongoDBSource",
+ "MySQLImplementation",
+ "MySQLSource",
+ "MySQLSourceConfig",
"NamingConvention",
"PARTITION_KEY",
+ "PostgresSource",
+ "PostgresSourceConfig",
"Transient_Error_Messages",
"WORKSPACE_FIELDS",
+ "WRITE_TIMEOUT_SECONDS",
"WorkspaceCredentials",
+ "_DATABASE_NOT_FOUND_VALIDATION_ERROR",
+ "_HOST_UNREACHABLE_ERROR",
+ "_INVALID_CREDENTIALS_VALIDATION_ERROR",
+ "_MYSQL_SAFE_CONVERSIONS",
+ "_connect_with_transient_retry",
"append_partition_key_to_table",
"check_extraction",
+ "connection_timeouts",
"filter_trino_incremental_fields",
"incremental_type_to_initial_value",
"incremental_value_for_trino_column",
diff --git a/products/warehouse_sources/sources/supabase/__init__.py b/products/warehouse_sources/sources/supabase/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py b/products/warehouse_sources/sources/supabase/_config.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py
rename to products/warehouse_sources/sources/supabase/_config.py
index 0ae5f66964db..0d7fc865db53 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py
+++ b/products/warehouse_sources/sources/supabase/_config.py
@@ -1,8 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import SSHTunnelConfig, config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.py b/products/warehouse_sources/sources/supabase/source.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.py
rename to products/warehouse_sources/sources/supabase/source.py
index 4bcf4d6de126..b12bf22d0ad6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.py
+++ b/products/warehouse_sources/sources/supabase/source.py
@@ -1,33 +1,26 @@
import re
from typing import Optional
-from posthog.security.postgres_hosts import (
+from products.warehouse_sources.sources.sdk import (
SUPABASE_DIRECT_HOST_IPV4_HINT as _SUPABASE_DIRECT_HOST_IPV4_HINT,
SUPABASE_DIRECT_HOST_RE as _SUPABASE_DIRECT_HOST_RE,
-)
-
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
SourceConfig,
SourceFieldInputConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
+ SourceRegistry,
SourceSchema,
+ detect,
rank_incremental_fields,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
- PostgresSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
+from products.warehouse_sources.sources.sdk.internals import (
_HOST_UNREACHABLE_ERROR,
_INVALID_CREDENTIALS_VALIDATION_ERROR,
PostgresSource,
+ PostgresSourceConfig,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
# The Supabase dashboard shows `https://<ref>.supabase.co` as the "Project URL" — that's the
# REST/API endpoint, not a Postgres host. Pasting it (often with the scheme) into the host field
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.py b/products/warehouse_sources/sources/supabase/test_supabase_source.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.py
rename to products/warehouse_sources/sources/supabase/test_supabase_source.py
index 1bd76c5f5961..cef6cae291d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.py
+++ b/products/warehouse_sources/sources/supabase/test_supabase_source.py
@@ -1,16 +1,22 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.facade.source_config import ReleaseStatus, SourceFieldInputConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
+from products.warehouse_sources.sources.sdk import (
+ IncrementalField,
+ IncrementalFieldType,
+ ReleaseStatus,
+ SourceFieldInputConfig,
+ SourceSchema,
+)
+from products.warehouse_sources.sources.sdk.internals import (
_HOST_UNREACHABLE_ERROR,
_INVALID_CREDENTIALS_VALIDATION_ERROR,
+ Any_Source_Errors,
+ ClientDeadlineExceededError,
PostgresSource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.supabase.source import SupabaseSource
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk.testing import error_message_matches
+from products.warehouse_sources.sources.supabase.source import SupabaseSource
def _field(name: str) -> SourceFieldInputConfig:
@@ -325,3 +331,12 @@ def test_expired_realtime_partition_gets_actionable_message(raw_error, expect_re
else:
assert "realtime.messages" not in friendly
assert "no longer exists" in friendly.lower()
+
+
+def test_the_deadline_error_is_retryable() -> None:
+ source = SupabaseSource()
+ message = str(ClientDeadlineExceededError(660))
+
+ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})
+ assert error_message_matches(message, source.get_retryable_errors())
+ assert error_message_matches(message, source.get_retry_exhausted_errors())