:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-warehouse): move postgres and mysql plugin vendors to the plugin folder (#128258)
Tom Owers0f53017e8cdc
diff --git a/products/warehouse_sources/backend/management/commands/generate_source_configs.py b/products/warehouse_sources/backend/management/commands/generate_source_configs.pyindex 9d015590e996..19b653027a9b 100644--- a/products/warehouse_sources/backend/management/commands/generate_source_configs.py+++ b/products/warehouse_sources/backend/management/commands/generate_source_configs.py@@ -62,6 +62,7 @@ class SourceConfigGenerator:def __init__(self):self.generated_classes: dict[str, str] = {}self.imports: set[str] = set()+ self.is_plugin_vendor = Falseself.typing_import: set[str] = set()self.nested_configs: dict[str, str] = {}@@ -122,6 +123,7 @@ class SourceConfigGenerator:def generate_source_config(self, source_type: SourceKey, source_config: SourceConfig) -> None:self.imports.add(self._config_import(source_type))+ self.is_plugin_vendor = self._config_import(source_type).startswith(f"from {PLUGIN_SOURCES_PACKAGE}.sdk")class_name = self._get_config_class_name(source_type)fields = []@@ -433,7 +435,10 @@ class SourceConfigGenerator:def _process_ssh_tunnel_field(self, field: SourceFieldSSHTunnelConfig) -> str:"""Process a SSH tunnel field by referencing the existing SSHTunnelConfig."""- self.imports.add("from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig")+ if self.is_plugin_vendor:+ self.imports.add(f"from {PLUGIN_SOURCES_PACKAGE}.sdk import SSHTunnelConfig")+ else:+ self.imports.add("from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig")python_field_name, should_alias = self._make_python_identifier(field.name)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.pyindex ac31f696f72a..99471453674a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py@@ -77,9 +77,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.sour_INVALID_CREDENTIALS_ERROR,MySQLSource,)-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source import (- PlanetScaleMySQLSource,-)from products.warehouse_sources.backend.types import IncrementalFieldType# ---------------------------------------------------------------------------@@ -1164,9 +1161,8 @@ class TestStreamingConnectionTimeouts:],ids=["read_timeout", "write_timeout"],)- @pytest.mark.parametrize("source_class", [MySQLSource, PlanetScaleMySQLSource])- def test_driver_timeout_is_retryable(self, source_class, error, kept_out_of_error_tracking):- source = source_class()+ def test_driver_timeout_is_retryable(self, error, kept_out_of_error_tracking):+ source = MySQLSource()assert error_message_matches(str(error), source.get_retryable_errors()) is kept_out_of_error_trackingassert not error_message_matches(str(error), {**Any_Source_Errors, **source.get_non_retryable_errors()})diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.pyindex 77da927398a9..9384dfd6029b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test/test_client_deadline.py@@ -8,10 +8,6 @@ import psycopgfrom products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errorsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches-from products.warehouse_sources.backend.temporal.data_imports.sources.neon.source import NeonSource-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_postgres.source import (- PlanetScalePostgresSource,-)from products.warehouse_sources.backend.temporal.data_imports.sources.postgres import client_deadlinefrom products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (ClientDeadlineExceededError,@@ -19,7 +15,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdeadline_server_cursor_factory,)from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource-from products.warehouse_sources.backend.temporal.data_imports.sources.supabase.source import SupabaseSourcedef test_a_statement_that_ends_in_time_is_not_cancelled() -> None:@@ -180,9 +175,8 @@ def test_a_silent_server_cursor_round_trip_ends_at_the_deadline(round_trip, experound_trip(cursor)[email protected]("source_class", [PostgresSource, NeonSource, SupabaseSource, PlanetScalePostgresSource])-def test_the_deadline_error_is_retryable_for_every_postgres_source(source_class: type[PostgresSource]) -> None:- source = source_class()+def test_the_deadline_error_is_retryable_for_the_postgres_source() -> None:+ source = PostgresSource()message = str(ClientDeadlineExceededError(660))assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.pyindex 82430c366bd8..4ea135731fcc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py@@ -32,9 +32,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_SnowflakeSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.stripe import StripeSourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.supabase import (- SupabaseSourceConfig,-)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.temporalio import (TemporalIOSourceConfig,)@@ -50,6 +47,7 @@ from products.warehouse_sources.sources.doit._config import DoItSourceConfigfrom products.warehouse_sources.sources.gitlab._config import GitLabSourceConfigfrom products.warehouse_sources.sources.gorgias._config import GorgiasSourceConfigfrom products.warehouse_sources.sources.shopify._config import ShopifySourceConfig+from products.warehouse_sources.sources.supabase._config import SupabaseSourceConfig# The form submits "temporary-dataset", but dataclasses.asdict() persists "temporary_dataset",diff --git a/products/warehouse_sources/sources/neon/__init__.py b/products/warehouse_sources/sources/neon/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py b/products/warehouse_sources/sources/neon/_config.pysimilarity index 79%rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.pyrename to products/warehouse_sources/sources/neon/_config.pyindex f8a099c08b16..25c34b5ce2ed 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py+++ b/products/warehouse_sources/sources/neon/_config.py@@ -1,8 +1,7 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.-from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config+from products.warehouse_sources.sources.sdk import SSHTunnelConfig, config@config.configdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py b/products/warehouse_sources/sources/neon/source.pysimilarity index 89%rename from products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.pyrename to products/warehouse_sources/sources/neon/source.pyindex 123352d2e90a..017f478c0106 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py+++ b/products/warehouse_sources/sources/neon/source.py@@ -1,23 +1,21 @@from collections.abc import Mappingfrom typing import Any, Optional-from products.warehouse_sources.backend.facade.codebase_detection import detect-from products.warehouse_sources.backend.facade.source_config import (+from products.warehouse_sources.sources.sdk import (DataWarehouseSourceCategory,+ ExternalDataSourceType,+ FieldType,ReleaseStatus,SourceConfig,SourceFieldInputConfig,+ SourceRegistry,+ detect,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (- PostgresSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (+from products.warehouse_sources.sources.sdk.internals import (_DATABASE_NOT_FOUND_VALIDATION_ERROR,PostgresSource,+ PostgresSourceConfig,)-from products.warehouse_sources.backend.types import ExternalDataSourceType_NEON_POOLED_HOST_CDC_ERROR = ("This is Neon's pooled endpoint, which doesn't support logical replication. For CDC, use the "diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py b/products/warehouse_sources/sources/neon/test_neon_source.pysimilarity index 77%rename from products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.pyrename to products/warehouse_sources/sources/neon/test_neon_source.pyindex 97d5d80b4674..b42a340b0694 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py+++ b/products/warehouse_sources/sources/neon/test_neon_source.py@@ -1,15 +1,15 @@import pytestfrom unittest import mock-from products.warehouse_sources.backend.temporal.data_imports.sources.neon.source import (- _NEON_DATABASE_NOT_FOUND_ERROR,- NeonSource,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (+from products.warehouse_sources.sources.neon.source import _NEON_DATABASE_NOT_FOUND_ERROR, NeonSource+from products.warehouse_sources.sources.sdk.internals import (_DATABASE_NOT_FOUND_VALIDATION_ERROR,_INVALID_CREDENTIALS_VALIDATION_ERROR,+ Any_Source_Errors,+ ClientDeadlineExceededError,PostgresSource,)+from products.warehouse_sources.sources.sdk.testing import error_message_matches@pytest.mark.parametrize(@@ -66,3 +66,12 @@ def test_validate_credentials_points_a_missing_database_at_the_neon_connect_dialwith mock.patch.object(PostgresSource, "validate_credentials", return_value=postgres_result):assert NeonSource().validate_credentials(config, team_id=1) == expected+++def test_the_deadline_error_is_retryable() -> None:+ source = NeonSource()+ message = str(ClientDeadlineExceededError(660))++ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})+ assert error_message_matches(message, source.get_retryable_errors())+ assert error_message_matches(message, source.get_retry_exhausted_errors())diff --git a/products/warehouse_sources/sources/planetscale_mysql/__init__.py b/products/warehouse_sources/sources/planetscale_mysql/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.py b/products/warehouse_sources/sources/planetscale_mysql/_config.pysimilarity index 79%rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.pyrename to products/warehouse_sources/sources/planetscale_mysql/_config.pyindex fb576543bdf3..56b8015e0095 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalemysql.py+++ b/products/warehouse_sources/sources/planetscale_mysql/_config.py@@ -1,7 +1,7 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config+from products.warehouse_sources.sources.sdk import config@config.configdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.py b/products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.pysimilarity index 93%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.pyrename to products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.pyindex f518de907671..6b122a1dfe54 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/planetscale_mysql.py+++ b/products/warehouse_sources/sources/planetscale_mysql/planetscale_mysql.py@@ -6,11 +6,11 @@ from django.conf import settingsimport pymysql-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (+from products.warehouse_sources.sources.sdk.internals import (_MYSQL_SAFE_CONVERSIONS,METADATA_READ_TIMEOUT_SECONDS,MySQLImplementation,+ MySQLSourceConfig,_connect_with_transient_retry,connection_timeouts,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.py b/products/warehouse_sources/sources/planetscale_mysql/source.pysimilarity index 91%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.pyrename to products/warehouse_sources/sources/planetscale_mysql/source.pyindex 325b2fda8834..ab736aac15a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/source.py+++ b/products/warehouse_sources/sources/planetscale_mysql/source.py@@ -1,24 +1,20 @@import refrom typing import Optional, cast-from posthog.exceptions_capture import capture_exception--from products.warehouse_sources.backend.facade.codebase_detection import detect-from products.warehouse_sources.backend.facade.source_config import (+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation+from products.warehouse_sources.sources.sdk import (DataWarehouseSourceCategory,+ ExternalDataSourceType,+ FieldType,ReleaseStatus,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,+ SourceRegistry,+ capture_exception,+ detect,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.source import MySQLSource-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (- PlanetScaleMySQLImplementation,-)-from products.warehouse_sources.backend.types import ExternalDataSourceType+from products.warehouse_sources.sources.sdk.internals import MySQLSource, MySQLSourceConfig_PLANETSCALE_MYSQL_IMPLEMENTATION = PlanetScaleMySQLImplementation()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.py b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.pysimilarity index 83%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.pyrename to products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.pyindex 4feb44bfafc3..4b71d30b8dc9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql.py+++ b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql.py@@ -4,23 +4,16 @@ from typing import Any, castimport pytestfrom unittest import mock-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalemysql import (- PlanetScaleMySQLSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (+from products.warehouse_sources.sources.planetscale_mysql._config import PlanetScaleMySQLSourceConfig+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation+from products.warehouse_sources.sources.sdk.internals import (CONNECT_TIMEOUT_SECONDS,METADATA_READ_TIMEOUT_SECONDS,WRITE_TIMEOUT_SECONDS,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (- PlanetScaleMySQLImplementation,+ MySQLSourceConfig,)-_CONNECT_TARGET = (- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql."- "_connect_with_transient_retry"-)+_CONNECT_TARGET = "products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql._connect_with_transient_retry"def _config(port: int | str = 3306) -> MySQLSourceConfig:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.py b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.pysimilarity index 84%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.pyrename to products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.pyindex f79af7739de1..146ea148b2eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_mysql/test_planetscale_mysql_source.py+++ b/products/warehouse_sources/sources/planetscale_mysql/test_planetscale_mysql_source.py@@ -4,18 +4,14 @@ from typing import castimport pytestfrom unittest import mock-from products.warehouse_sources.backend.facade.source_config import SourceFieldInputConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalemysql import (- PlanetScaleMySQLSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.planetscale_mysql import (- PlanetScaleMySQLImplementation,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source import (- PlanetScaleMySQLSource,-)-from products.warehouse_sources.backend.types import ExternalDataSourceType+import pymysql++from products.warehouse_sources.sources.planetscale_mysql._config import PlanetScaleMySQLSourceConfig+from products.warehouse_sources.sources.planetscale_mysql.planetscale_mysql import PlanetScaleMySQLImplementation+from products.warehouse_sources.sources.planetscale_mysql.source import PlanetScaleMySQLSource+from products.warehouse_sources.sources.sdk import ExternalDataSourceType, SourceFieldInputConfig+from products.warehouse_sources.sources.sdk.internals import Any_Source_Errors, MySQLSourceConfig+from products.warehouse_sources.sources.sdk.testing import error_message_matchesdef _config(host: str = "aws.connect.psdb.cloud") -> MySQLSourceConfig:@@ -173,9 +169,7 @@ def test_connection_failures_map_to_actionable_messages(raw_error, expected_fragwith (mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception(raw_error)),- mock.patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"- ) as capture,+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,):success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)@@ -200,9 +194,7 @@ def test_retryable_sync_errors_are_not_captured_during_validation(raw_error):with (mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception(raw_error)),- mock.patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"- ) as capture,+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,):success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)@@ -218,12 +210,24 @@ def test_unrecognized_failure_is_captured_and_reported_generically():with (mock.patch.object(PlanetScaleMySQLSource, "is_database_host_valid", return_value=(True, None)),mock.patch.object(PlanetScaleMySQLSource, "get_schemas", side_effect=Exception("something unexpected")),- mock.patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_mysql.source.capture_exception"- ) as capture,+ mock.patch("products.warehouse_sources.sources.planetscale_mysql.source.capture_exception") as capture,):success, error = PlanetScaleMySQLSource().validate_credentials(_config(), team_id=1)capture.assert_called_once()assert success is Falseassert error == "Could not connect to PlanetScale MySQL. Please check all connection details are valid."+++ "error",+ [+ pymysql.err.OperationalError(2013, "Lost connection to MySQL server during query (timed out)"),+ pymysql.err.OperationalError(2006, "MySQL server has gone away (TimeoutError('timed out'))"),+ ],+ ids=["read_timeout", "write_timeout"],+)+def test_driver_timeout_is_not_non_retryable(error: pymysql.err.OperationalError) -> None:+ source = PlanetScaleMySQLSource()++ assert not error_message_matches(str(error), {**Any_Source_Errors, **source.get_non_retryable_errors()})diff --git a/products/warehouse_sources/sources/planetscale_postgres/__init__.py b/products/warehouse_sources/sources/planetscale_postgres/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py b/products/warehouse_sources/sources/planetscale_postgres/_config.pysimilarity index 87%rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.pyrename to products/warehouse_sources/sources/planetscale_postgres/_config.pyindex d04aa946055a..c3dc64a89f42 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py+++ b/products/warehouse_sources/sources/planetscale_postgres/_config.py@@ -1,7 +1,7 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config+from products.warehouse_sources.sources.sdk import config@config.configdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py b/products/warehouse_sources/sources/planetscale_postgres/source.pysimilarity index 90%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.pyrename to products/warehouse_sources/sources/planetscale_postgres/source.pyindex ff3da3ffda51..d4287cbbd286 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py+++ b/products/warehouse_sources/sources/planetscale_postgres/source.py@@ -2,21 +2,18 @@ import refrom collections.abc import Mappingfrom typing import Any-from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE-from products.warehouse_sources.backend.facade.source_config import (+from products.warehouse_sources.sources.sdk import (+ NOT_DETECTABLE,DataWarehouseSourceCategory,+ ExternalDataSourceType,+ FieldType,ReleaseStatus,SourceConfig,SourceFieldInputConfig,SourceFieldSSHTunnelConfig,+ SourceRegistry,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (- PostgresSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource-from products.warehouse_sources.backend.types import ExternalDataSourceType+from products.warehouse_sources.sources.sdk.internals import PostgresSource, PostgresSourceConfig# planetscale.com is the dashboard, never a database endpoint — Postgres hosts live on# psdb.cloud. Pasting the dashboard address otherwise fails as an opaque connection timeout.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py b/products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.pysimilarity index 82%rename from products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.pyrename to products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.pyindex 682f2a7e6b12..8cbfbf1679b5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py+++ b/products/warehouse_sources/sources/planetscale_postgres/test_planetscale_postgres_source.py@@ -1,18 +1,16 @@import pytestfrom unittest import mock-from products.warehouse_sources.backend.facade.source_config import SourceFieldInputConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.planetscalepostgres import (- PlanetScalePostgresSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (+from products.warehouse_sources.sources.planetscale_postgres._config import PlanetScalePostgresSourceConfig+from products.warehouse_sources.sources.planetscale_postgres.source import PlanetScalePostgresSource, _is_psbouncer_port+from products.warehouse_sources.sources.sdk import SourceFieldInputConfig+from products.warehouse_sources.sources.sdk.internals import (+ Any_Source_Errors,+ ClientDeadlineExceededError,+ PostgresSource,PostgresSourceConfig,)-from products.warehouse_sources.backend.temporal.data_imports.sources.planetscale_postgres.source import (- PlanetScalePostgresSource,- _is_psbouncer_port,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource+from products.warehouse_sources.sources.sdk.testing import error_message_matches_DIRECT_HOST = "xxxxxxxxxx-useast1-1.horizon.psdb.cloud"@@ -116,3 +114,12 @@ def test_cdc_on_the_direct_port_defers_to_postgres():super_check.assert_called_once()assert super_check.call_args.kwargs["team_id"] == 7assert errors == []+++def test_the_deadline_error_is_retryable() -> None:+ source = PlanetScalePostgresSource()+ message = str(ClientDeadlineExceededError(660))++ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})+ assert error_message_matches(message, source.get_retryable_errors())+ assert error_message_matches(message, source.get_retry_exhausted_errors())diff --git a/products/warehouse_sources/sources/sdk/__init__.py b/products/warehouse_sources/sources/sdk/__init__.pyindex b1edfaa8f28b..66e85f639162 100644--- a/products/warehouse_sources/sources/sdk/__init__.py+++ b/products/warehouse_sources/sources/sdk/__init__.py@@ -22,6 +22,7 @@ from posthog.models.integration import ()from posthog.models.integration.databricks import NOT_A_DATABRICKS_HOST_MESSAGE, is_databricks_workspace_hostfrom posthog.schema_enums import ProductKey+from posthog.security.postgres_hosts import SUPABASE_DIRECT_HOST_IPV4_HINT, SUPABASE_DIRECT_HOST_REfrom posthog.security.url_validation import is_url_allowedfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.common.shutdown import WorkerShuttingDownError@@ -40,6 +41,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldOauthConfig,SourceFieldSelectConfig,SourceFieldSelectConfigOption,+ SourceFieldSSHTunnelConfig,SourceFieldSwitchGroupConfig,SuggestedTable,)@@ -53,7 +55,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (ExternalDataSchema,update_sync_type_config_keys,)-from products.warehouse_sources.backend.models.ssh_tunnel import from_private_key+from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig, from_private_keyfrom products.warehouse_sources.backend.temporal.data_imports.sources.common import (config,credentials_service,@@ -390,6 +392,9 @@ __all__ = ["RowFilterColumn","SQLSource","SQLSourceImplementation",+ "SSHTunnelConfig",+ "SUPABASE_DIRECT_HOST_IPV4_HINT",+ "SUPABASE_DIRECT_HOST_RE","SafeSQL","SelectQueryBuilder","SimpleSource",@@ -403,6 +408,7 @@ __all__ = ["SourceFieldInputConfigType","SourceFieldOauthAccountSelectConfig","SourceFieldOauthConfig",+ "SourceFieldSSHTunnelConfig","SourceFieldSelectConfig","SourceFieldSelectConfigOption","SourceFieldSwitchGroupConfig",diff --git a/products/warehouse_sources/sources/sdk/internals.py b/products/warehouse_sources/sources/sdk/internals.pyindex 25738dcb0774..4b48d7d248a3 100644--- a/products/warehouse_sources/sources/sdk/internals.py+++ b/products/warehouse_sources/sources/sdk/internals.py@@ -39,7 +39,30 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_WORKSPACE_FIELDS,LogAnalyticsWorkspaceSource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (+ PostgresSourceConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.mongodb.source import MongoDBSource+from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (+ _MYSQL_SAFE_CONVERSIONS,+ CONNECT_TIMEOUT_SECONDS,+ METADATA_READ_TIMEOUT_SECONDS,+ WRITE_TIMEOUT_SECONDS,+ MySQLImplementation,+ _connect_with_transient_retry,+ connection_timeouts,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.source import MySQLSource+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (+ ClientDeadlineExceededError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (+ _DATABASE_NOT_FOUND_VALIDATION_ERROR,+ _HOST_UNREACHABLE_ERROR,+ _INVALID_CREDENTIALS_VALIDATION_ERROR,+ PostgresSource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.tests.contract_harness import (check_extraction,start_source,@@ -55,16 +78,31 @@ __all__ = ["AzureLogAnalyticsResumeConfig","Batcher","BinaryColumnReporter",+ "CONNECT_TIMEOUT_SECONDS",+ "ClientDeadlineExceededError","DISCOVER_SCHEMAS_ACTIVITY_TIMEOUT","LogAnalyticsWorkspaceSource",+ "METADATA_READ_TIMEOUT_SECONDS","MongoDBSource",+ "MySQLImplementation",+ "MySQLSource",+ "MySQLSourceConfig","NamingConvention","PARTITION_KEY",+ "PostgresSource",+ "PostgresSourceConfig","Transient_Error_Messages","WORKSPACE_FIELDS",+ "WRITE_TIMEOUT_SECONDS","WorkspaceCredentials",+ "_DATABASE_NOT_FOUND_VALIDATION_ERROR",+ "_HOST_UNREACHABLE_ERROR",+ "_INVALID_CREDENTIALS_VALIDATION_ERROR",+ "_MYSQL_SAFE_CONVERSIONS",+ "_connect_with_transient_retry","append_partition_key_to_table","check_extraction",+ "connection_timeouts","filter_trino_incremental_fields","incremental_type_to_initial_value","incremental_value_for_trino_column",diff --git a/products/warehouse_sources/sources/supabase/__init__.py b/products/warehouse_sources/sources/supabase/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py b/products/warehouse_sources/sources/supabase/_config.pysimilarity index 79%rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.pyrename to products/warehouse_sources/sources/supabase/_config.pyindex 0ae5f66964db..0d7fc865db53 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py+++ b/products/warehouse_sources/sources/supabase/_config.py@@ -1,8 +1,7 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.-from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnelConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config+from products.warehouse_sources.sources.sdk import SSHTunnelConfig, config@config.configdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.py b/products/warehouse_sources/sources/supabase/source.pysimilarity index 95%rename from products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.pyrename to products/warehouse_sources/sources/supabase/source.pyindex 4bcf4d6de126..b12bf22d0ad6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/source.py+++ b/products/warehouse_sources/sources/supabase/source.py@@ -1,33 +1,26 @@import refrom typing import Optional-from posthog.security.postgres_hosts import (+from products.warehouse_sources.sources.sdk import (SUPABASE_DIRECT_HOST_IPV4_HINT as _SUPABASE_DIRECT_HOST_IPV4_HINT,SUPABASE_DIRECT_HOST_RE as _SUPABASE_DIRECT_HOST_RE,-)--from products.warehouse_sources.backend.facade.codebase_detection import detect-from products.warehouse_sources.backend.facade.source_config import (DataWarehouseSourceCategory,+ ExternalDataSourceType,+ FieldType,ReleaseStatus,SourceConfig,SourceFieldInputConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (+ SourceRegistry,SourceSchema,+ detect,rank_incremental_fields,)-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (- PostgresSourceConfig,-)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (+from products.warehouse_sources.sources.sdk.internals import (_HOST_UNREACHABLE_ERROR,_INVALID_CREDENTIALS_VALIDATION_ERROR,PostgresSource,+ PostgresSourceConfig,)-from products.warehouse_sources.backend.types import ExternalDataSourceType# The Supabase dashboard shows `https://<ref>.supabase.co` as the "Project URL" — that's the# REST/API endpoint, not a Postgres host. Pasting it (often with the scheme) into the host fielddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.py b/products/warehouse_sources/sources/supabase/test_supabase_source.pysimilarity index 93%rename from products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.pyrename to products/warehouse_sources/sources/supabase/test_supabase_source.pyindex 1bd76c5f5961..cef6cae291d3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/supabase/test_supabase_source.py+++ b/products/warehouse_sources/sources/supabase/test_supabase_source.py@@ -1,16 +1,22 @@import pytestfrom unittest import mock-from products.warehouse_sources.backend.facade.source_config import ReleaseStatus, SourceFieldInputConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (+from products.warehouse_sources.sources.sdk import (+ IncrementalField,+ IncrementalFieldType,+ ReleaseStatus,+ SourceFieldInputConfig,+ SourceSchema,+)+from products.warehouse_sources.sources.sdk.internals import (_HOST_UNREACHABLE_ERROR,_INVALID_CREDENTIALS_VALIDATION_ERROR,+ Any_Source_Errors,+ ClientDeadlineExceededError,PostgresSource,)-from products.warehouse_sources.backend.temporal.data_imports.sources.supabase.source import SupabaseSource-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+from products.warehouse_sources.sources.sdk.testing import error_message_matches+from products.warehouse_sources.sources.supabase.source import SupabaseSourcedef _field(name: str) -> SourceFieldInputConfig:@@ -325,3 +331,12 @@ def test_expired_realtime_partition_gets_actionable_message(raw_error, expect_reelse:assert "realtime.messages" not in friendlyassert "no longer exists" in friendly.lower()+++def test_the_deadline_error_is_retryable() -> None:+ source = SupabaseSource()+ message = str(ClientDeadlineExceededError(660))++ assert not error_message_matches(message, {**Any_Source_Errors, **source.get_non_retryable_errors()})+ assert error_message_matches(message, source.get_retryable_errors())+ assert error_message_matches(message, source.get_retry_exhausted_errors())