Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): test a destination's credentials before creating it (#115558)

Tom Owersa647a97c8bae
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_check.py b/products/warehouse_sources/backend/presentation/destination_connection_check.py
index d8951ca1b906..43d15e4f0ddc 100644
--- a/products/warehouse_sources/backend/presentation/destination_connection_check.py
+++ b/products/warehouse_sources/backend/presentation/destination_connection_check.py
@@ -42,6 +42,10 @@ FAILURE_MESSAGES: dict[CheckFailure, str] = {
}
+# Failures caused by the destination's database or schema. The rest come from the integration.
+CONFIG_FAILURES = frozenset({CheckFailure.UNKNOWN_DATABASE, CheckFailure.MISSING_PRIVILEGE})
+
+
class DestinationConnectionCheckError(Exception):
def __init__(self, failure: CheckFailure) -> None:
super().__init__(FAILURE_MESSAGES[failure])
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_testing.py b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
new file mode 100644
index 000000000000..c3ce79dfa493
--- /dev/null
+++ b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
@@ -0,0 +1,211 @@
+from collections.abc import Callable, Mapping
+from dataclasses import replace
+from typing import Any, Literal
+from uuid import uuid4
+
+from django.db import models
+
+from posthog.dataclasses import frozen
+from posthog.models.integration import Integration
+
+from products.batch_exports.backend.facade import api as batch_exports_api
+from products.batch_exports.backend.facade.contracts import (
+ DestinationTestStep,
+ DestinationTestStepResult,
+ UnsupportedDestinationTestError,
+)
+from products.warehouse_sources.backend.facade.models import ExternalDataDestination
+from products.warehouse_sources.backend.presentation.destination_connection_check import (
+ CONFIG_FAILURES,
+ DestinationConnectionCheckError,
+ check_postgres_destination,
+)
+
+# The BigQuery and Databricks steps create and then drop scratch objects named `<table>_test` in the
+# customer's warehouse. Each run gets its own random table name, so the cleanup can never drop an
+# object the customer already has.
+CONNECTION_TEST_TABLE_PREFIX = "posthog_destination_check"
+
+
+def new_connection_test_table() -> str:
+ return f"{CONNECTION_TEST_TABLE_PREFIX}_{uuid4().hex[:12]}"
+
+
+class DestinationConnectionTestStepStatus(models.TextChoices):
+ PASSED = "Passed"
+ FAILED = "Failed"
+ SKIPPED = "Skipped"
+
+
+@frozen
+class ConnectionTestFailure:
+ step_name: str
+ message: str
+ field: Literal["integration", "config"]
+
+ def __str__(self) -> str:
+ return f"{self.step_name}: {self.message}"
+
+
+@frozen
+class ConnectionTestRun:
+ steps: tuple[DestinationTestStep, ...]
+ failure: ConnectionTestFailure | None
+
+
+def _strip_slashes(value: Any) -> str | None:
+ if not value:
+ return None
+ return str(value).strip("/") or None
+
+
+def _bigquery_config(config: Mapping[str, Any], test_table: str) -> dict[str, object]:
+ # The facade merges the integration's config over this, so `project_id` is always the
+ # integration's project. The writer also uses that project when the destination names none.
+ return {
+ "project_id": config.get("project") or config.get("project_id") or None,
+ "dataset_id": config.get("dataset") or config.get("dataset_id") or None,
+ "table_id": test_table,
+ }
+
+
+def _s3_config(config: Mapping[str, Any], _test_table: str) -> dict[str, object]:
+ return {
+ "bucket_name": _strip_slashes(config.get("bucket") or config.get("bucket_name")),
+ "region": (config.get("region") or "").strip() or None,
+ }
+
+
+def _azure_blob_config(config: Mapping[str, Any], _test_table: str) -> dict[str, object]:
+ return {"container_name": _strip_slashes(config.get("container_name") or config.get("container"))}
+
+
+def _snowflake_config(config: Mapping[str, Any], _test_table: str) -> dict[str, object]:
+ return {
+ "database": config.get("database") or None,
+ "schema": config.get("schema") or "PUBLIC",
+ "warehouse": config.get("warehouse") or None,
+ "role": config.get("role") or None,
+ }
+
+
+def _databricks_config(config: Mapping[str, Any], test_table: str) -> dict[str, object]:
+ return {
+ "catalog": config.get("catalog") or "main",
+ "schema": config.get("schema") or "default",
+ "http_path": config.get("http_path") or None,
+ "table_name": test_table,
+ }
+
+
+# Defaults and key aliases mirror each warehouse writer, so the test checks the same target the
+# sync writes to.
+_CONFIG_ADAPTERS: dict[str, Callable[[Mapping[str, Any], str], dict[str, object]]] = {
+ str(ExternalDataDestination.Type.BIGQUERY): _bigquery_config,
+ str(ExternalDataDestination.Type.S3): _s3_config,
+ str(ExternalDataDestination.Type.AZURE_BLOB): _azure_blob_config,
+ str(ExternalDataDestination.Type.SNOWFLAKE): _snowflake_config,
+ str(ExternalDataDestination.Type.DATABRICKS): _databricks_config,
+}
+
+_TEST_TYPES: dict[str, str] = {
+ # The writer creates a missing dataset, so BigQuery uses the test that accepts one it can create.
+ str(ExternalDataDestination.Type.BIGQUERY): "BigQueryCreateDataset",
+ str(ExternalDataDestination.Type.AZURE_BLOB): "AzureBlob",
+ str(ExternalDataDestination.Type.SNOWFLAKE): "Snowflake",
+ str(ExternalDataDestination.Type.DATABRICKS): "Databricks",
+}
+
+# The first steps of each test check the credentials on the integration. The steps after them
+# check the target named in the destination's config.
+_CREDENTIAL_STEP_COUNT: dict[str, int] = {
+ "BigQueryCreateDataset": 3,
+ "AwsS3": 1,
+ "S3Compatible": 0,
+ "AzureBlob": 0,
+ "Snowflake": 1,
+ "Databricks": 1,
+}
+
+
+def connection_test_type(destination_type: str, integration: Integration) -> str | None:
+ """The batch exports test that covers this destination, or None when there is none."""
+ if destination_type == ExternalDataDestination.Type.S3:
+ return "AwsS3" if integration.kind == Integration.IntegrationKind.AWS_S3 else "S3Compatible"
+ return _TEST_TYPES.get(destination_type)
+
+
+def run_connection_test(
+ destination_type: str, integration: Integration, config: Mapping[str, Any] | None
+) -> ConnectionTestRun | None:
+ """Run each step of the destination's connection test, and stop at the first failed step.
+
+ Returns None when no test covers this destination type. Steps after a failure stay in the
+ result with no result of their own.
+ """
+ test_type = connection_test_type(destination_type, integration)
+ if test_type is None:
+ return None
+
+ try:
+ described = batch_exports_api.describe_destination_test(test_type)
+ except UnsupportedDestinationTestError:
+ return None
+
+ test_config = _CONFIG_ADAPTERS[destination_type](
+ config if isinstance(config, Mapping) else {}, new_connection_test_table()
+ )
+ steps = list(described.steps)
+ for index in range(len(steps)):
+ step = batch_exports_api.run_destination_test_step(
+ integration.team_id,
+ destination_type=test_type,
+ config=test_config,
+ integration_id=integration.id,
+ step=index,
+ )
+ steps[index] = step
+ if step.result is not None and step.result.status == DestinationConnectionTestStepStatus.FAILED:
+ field: Literal["integration", "config"] = (
+ "integration" if index < _CREDENTIAL_STEP_COUNT.get(test_type, 0) else "config"
+ )
+ failure = ConnectionTestFailure(
+ step_name=step.name,
+ message=step.result.message or "The step failed. Check the destination settings and try again.",
+ field=field,
+ )
+ return ConnectionTestRun(steps=tuple(steps), failure=failure)
+
+ return ConnectionTestRun(steps=tuple(steps), failure=None)
+
+
+_POSTGRES_STEP = DestinationTestStep(
+ name="Connect to Postgres",
+ description="Connect to the server and check that the user can create tables in the schema.",
+ result=None,
+)
+
+
+def run_postgres_connection_test(integration: Integration, config: Mapping[str, Any] | None) -> ConnectionTestRun:
+ """Run the Postgres check as a single step, so it reports like the other connection tests."""
+ try:
+ check_postgres_destination(integration, dict(config) if isinstance(config, Mapping) else None)
+ except DestinationConnectionCheckError as error:
+ step = replace(
+ _POSTGRES_STEP,
+ result=DestinationTestStepResult(
+ status=DestinationConnectionTestStepStatus.FAILED.value, message=str(error)
+ ),
+ )
+ failure = ConnectionTestFailure(
+ step_name=step.name,
+ message=str(error),
+ field="config" if error.failure in CONFIG_FAILURES else "integration",
+ )
+ return ConnectionTestRun(steps=(step,), failure=failure)
+
+ step = replace(
+ _POSTGRES_STEP,
+ result=DestinationTestStepResult(status=DestinationConnectionTestStepStatus.PASSED.value, message=None),
+ )
+ return ConnectionTestRun(steps=(step,), failure=None)
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
index 085c4bb2cdaf..3511cf9ae589 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
@@ -25,10 +25,16 @@ from products.warehouse_sources.backend.facade.models import (
ExternalDataSourceDestination,
)
from products.warehouse_sources.backend.presentation.destination_connection_check import (
- CheckFailure,
+ CONFIG_FAILURES,
DestinationConnectionCheckError,
check_postgres_destination,
)
+from products.warehouse_sources.backend.presentation.destination_connection_testing import (
+ ConnectionTestRun,
+ DestinationConnectionTestStepStatus,
+ run_connection_test,
+ run_postgres_connection_test,
+)
from products.warehouse_sources.backend.presentation.views.destination_links import MAX_DESTINATIONS_PER_LINK
from products.warehouse_sources.backend.presentation.views.external_data_schema import resync_schema
@@ -99,6 +105,41 @@ class AddSourcesResponseSerializer(serializers.Serializer):
)
+class DestinationConnectionTestStepSerializer(serializers.Serializer):
+ name = serializers.CharField(help_text="What this step checks.")
+ description = serializers.CharField(help_text="A longer explanation of what this step checks.")
+ status = serializers.ChoiceField(
+ choices=DestinationConnectionTestStepStatus.choices,
+ allow_null=True,
+ help_text="The result of the step. Null when the step did not run because an earlier step failed.",
+ )
+ message = serializers.CharField(
+ allow_null=True,
+ help_text="Why the step failed or was skipped. Null when the step passed or did not run.",
+ )
+
+
+class DestinationConnectionTestResponseSerializer(serializers.Serializer):
+ steps = DestinationConnectionTestStepSerializer(
+ many=True,
+ help_text="The steps of the connection test, in order. The test stops at the first failed step.",
+ )
+
+
+def _connection_test_response(run: ConnectionTestRun) -> dict[str, Any]:
+ return {
+ "steps": [
+ {
+ "name": step.name,
+ "description": step.description,
+ "status": step.result.status if step.result else None,
+ "message": step.result.message if step.result else None,
+ }
+ for step in run.steps
+ ]
+ }
+
+
def _source_summary(source: ExternalDataSource, *, via_table_override: bool = False) -> dict[str, Any]:
# Prefix is part of how a person recognizes a source, since one connector can be added twice.
return {
@@ -256,6 +297,8 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
self._reject_retargeting(attrs)
elif destination_type == ExternalDataDestination.Type.POSTGRES:
self._check_postgres_connection(integration, attrs.get("config"))
+ else:
+ self._run_connection_test(destination_type, integration, attrs.get("config"))
return attrs
@@ -265,13 +308,14 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
try:
check_postgres_destination(integration, config)
except DestinationConnectionCheckError as error:
- field = (
- "config"
- if error.failure in (CheckFailure.UNKNOWN_DATABASE, CheckFailure.MISSING_PRIVILEGE)
- else "integration"
- )
+ field = "config" if error.failure in CONFIG_FAILURES else "integration"
raise ValidationError({field: str(error)})
+ def _run_connection_test(self, destination_type: str, integration: Integration, config: Any) -> None:
+ run = run_connection_test(destination_type, integration, config)
+ if run is not None and run.failure is not None:
+ raise ValidationError({run.failure.field: str(run.failure)})
+
# Where a destination points is fixed once it exists. Everything already synced sits at the
# current server and schema, so repointing one strands that data and needs a full resync of
# every table that syncs there. A second destination is the supported way to write elsewhere.
@@ -543,6 +587,26 @@ class ExternalDataDestinationViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewS
).data
)
+ @extend_schema(request=None, responses={200: DestinationConnectionTestResponseSerializer})
+ @action(methods=["POST"], detail=True)
+ def test_connection(self, request: Request, *args: Any, **kwargs: Any) -> Response:
+ """Test the credentials and target of an existing destination, and return the result of each step."""
+ destination: ExternalDataDestination = self.get_object()
+ integration = destination.integration
+ if destination.is_posthog_warehouse or integration is None:
+ raise ValidationError(
+ {"type": "The PostHog warehouse is managed for you, so it has no connection to test."}
+ )
+
+ if destination.type == ExternalDataDestination.Type.POSTGRES:
+ run: ConnectionTestRun | None = run_postgres_connection_test(integration, destination.config)
+ else:
+ run = run_connection_test(destination.type, integration, destination.config)
+ if run is None:
+ raise ValidationError({"type": f"Connection tests are not available for {destination.type} destinations."})
+
+ return Response(DestinationConnectionTestResponseSerializer(_connection_test_response(run)).data)
+
def perform_update(self, serializer: serializers.BaseSerializer) -> None:
# `.instance` is `Any | None` on the base serializer type, but `update`/`partial_update`
# always construct this viewset's serializer with the existing instance to update.
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
index 3093568d43e5..835cc74c606c 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
@@ -18,6 +18,11 @@ from posthog.models.user import User
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.access_control.backend.models.access_control import AccessControl
+from products.batch_exports.backend.facade.contracts import (
+ DestinationTest,
+ DestinationTestStep,
+ DestinationTestStepResult,
+)
from products.warehouse_sources.backend.models.external_data_destination import (
ExternalDataDestination,
ExternalDataSchemaDestination,
@@ -32,12 +37,38 @@ from products.warehouse_sources.backend.presentation.destination_connection_chec
CheckFailure,
DestinationConnectionCheckError,
)
+from products.warehouse_sources.backend.presentation.destination_connection_testing import (
+ CONNECTION_TEST_TABLE_PREFIX,
+ new_connection_test_table,
+)
from products.warehouse_sources.backend.presentation.views.external_data_destination import (
ExternalDataDestinationViewSet,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+def _destination_test(step_count: int) -> DestinationTest:
+ return DestinationTest(
+ steps=tuple(
+ DestinationTestStep(name=f"Step {index}", description=f"Checks {index}", result=None)
+ for index in range(step_count)
+ )
+ )
+
+
+def _step_results(*statuses: str):
+ def run_step(team_id: int, *, destination_type: str, config, integration_id: int | None, step: int):
+ status = statuses[step] if step < len(statuses) else "Passed"
+ message = None if status == "Passed" else f"Step {step} said no"
+ return DestinationTestStep(
+ name=f"Step {step}",
+ description=f"Checks {step}",
+ result=DestinationTestStepResult(status=status, message=message),
+ )
+
+ return run_step
+
+
class DestinationAPITestBase(APIBaseTest):
def setUp(self) -> None:
super().setUp()
@@ -46,6 +77,19 @@ class DestinationAPITestBase(APIBaseTest):
)
self.check_connection = check_patcher.start()
self.addCleanup(check_patcher.stop)
+ facade = "products.warehouse_sources.backend.presentation.destination_connection_testing.batch_exports_api"
+ describe_patcher = patch(f"{facade}.describe_destination_test", return_value=_destination_test(1))
+ self.describe_test = describe_patcher.start()
+ self.addCleanup(describe_patcher.stop)
+ run_patcher = patch(f"{facade}.run_destination_test_step", side_effect=_step_results("Passed"))
+ self.run_test_step = run_patcher.start()
+ self.addCleanup(run_patcher.stop)
+ table_patcher = patch(
+ "products.warehouse_sources.backend.presentation.destination_connection_testing.new_connection_test_table",
+ return_value=f"{CONNECTION_TEST_TABLE_PREFIX}_abc123",
+ )
+ table_patcher.start()
+ self.addCleanup(table_patcher.stop)
self.base = f"/api/projects/{self.team.pk}/external_data_destinations"
self.source = ExternalDataSource.objects.create(
team=self.team,
@@ -150,6 +194,189 @@ class TestExternalDataDestinationAPI(DestinationAPITestBase):
assert response.status_code == status.HTTP_201_CREATED, response.json()
self.check_connection.assert_called_once_with(integration, {"database": "analytics", "schema": "posthog"})
+ @parameterized.expand(
+ [
+ (
+ "bigquery",
+ ExternalDataDestination.Type.BIGQUERY,
+ Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT,
+ {"dataset": "analytics", "project": "acme-project"},
+ "BigQueryCreateDataset",
+ {
+ "project_id": "acme-project",
+ "dataset_id": "analytics",
+ "table_id": f"{CONNECTION_TEST_TABLE_PREFIX}_abc123",
+ },
+ ),
+ (
+ "bigquery_aliases_without_project",
+ ExternalDataDestination.Type.BIGQUERY,
+ Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT,
+ {"dataset_id": "analytics"},
+ "BigQueryCreateDataset",
+ {
+ "project_id": None,
+ "dataset_id": "analytics",
+ "table_id": f"{CONNECTION_TEST_TABLE_PREFIX}_abc123",
+ },
+ ),
+ (
+ "aws_s3",
+ ExternalDataDestination.Type.S3,
+ Integration.IntegrationKind.AWS_S3,
+ {"bucket": "/my-bucket/", "region": " us-east-1 ", "prefix": "posthog"},
+ "AwsS3",
+ {"bucket_name": "my-bucket", "region": "us-east-1"},
+ ),
+ (
+ "s3_compatible",
+ ExternalDataDestination.Type.S3,
+ Integration.IntegrationKind.S3_COMPATIBLE,
+ {"bucket_name": "my-bucket"},
+ "S3Compatible",
+ {"bucket_name": "my-bucket", "region": None},
+ ),
+ (
+ "azure_blob",
+ ExternalDataDestination.Type.AZURE_BLOB,
+ Integration.IntegrationKind.AZURE_BLOB,
+ {"container": "exports"},
+ "AzureBlob",
+ {"container_name": "exports"},
+ ),
+ (
+ "snowflake",
+ ExternalDataDestination.Type.SNOWFLAKE,
+ Integration.IntegrationKind.SNOWFLAKE,
+ {"database": "ANALYTICS", "warehouse": "COMPUTE_WH"},
+ "Snowflake",
+ {"database": "ANALYTICS", "schema": "PUBLIC", "warehouse": "COMPUTE_WH", "role": None},
+ ),
+ (
+ "databricks",
+ ExternalDataDestination.Type.DATABRICKS,
+ Integration.IntegrationKind.DATABRICKS,
+ {"http_path": "/sql/1.0/warehouses/abc"},
+ "Databricks",
+ {
+ "catalog": "main",
+ "schema": "default",
+ "http_path": "/sql/1.0/warehouses/abc",
+ "table_name": f"{CONNECTION_TEST_TABLE_PREFIX}_abc123",
+ },
+ ),
+ ]
+ )
+ def test_creating_a_destination_runs_its_connection_test(
+ self,
+ _name: str,
+ destination_type: str,
+ kind: str,
+ config: dict,
+ expected_test: str,
+ expected_config: dict,
+ ) -> None:
+ integration = self._integration(kind)
+ self.describe_test.return_value = _destination_test(2)
+
+ response = self.client.post(
+ self.base,
+ {"type": destination_type, "name": "tested", "integration": integration.pk, "config": config},
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ self.describe_test.assert_called_once_with(expected_test)
+ assert [call.kwargs for call in self.run_test_step.call_args_list] == [
+ {
+ "destination_type": expected_test,
+ "config": expected_config,
+ "integration_id": integration.pk,
+ "step": step,
+ }
+ for step in range(2)
+ ]
+ self.check_connection.assert_not_called()
+
+ def test_each_connection_test_run_uses_its_own_scratch_table(self) -> None:
+ names = {new_connection_test_table() for _ in range(5)}
+
+ assert len(names) == 5
+ assert all(name.startswith(f"{CONNECTION_TEST_TABLE_PREFIX}_") for name in names)
+
+ @parameterized.expand(
+ [
+ # BigQuery's first three steps check the service account and project on the integration.
+ (
+ "credential_step",
+ ExternalDataDestination.Type.BIGQUERY,
+ Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT,
+ ("Skipped", "Failed"),
+ 1,
+ "integration",
+ ),
+ (
+ "target_step",
+ ExternalDataDestination.Type.BIGQUERY,
+ Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT,
+ ("Skipped", "Skipped", "Passed", "Failed"),
+ 3,
+ "config",
+ ),
+ (
+ "assume_role",
+ ExternalDataDestination.Type.S3,
+ Integration.IntegrationKind.AWS_S3,
+ ("Failed",),
+ 0,
+ "integration",
+ ),
+ (
+ "bucket",
+ ExternalDataDestination.Type.S3,
+ Integration.IntegrationKind.S3_COMPATIBLE,
+ ("Failed",),
+ 0,
+ "config",
+ ),
+ ]
+ )
+ def test_a_destination_that_fails_a_connection_test_step_is_not_created(
+ self,
+ _name: str,
+ destination_type: str,
+ kind: str,
+ statuses: tuple[str, ...],
+ failed_step: int,
+ expected_field: str,
+ ) -> None:
+ self.describe_test.return_value = _destination_test(5)
+ self.run_test_step.side_effect = _step_results(*statuses)
+
+ response = self.client.post(
+ self.base,
+ {"type": destination_type, "name": "unusable", "integration": self._integration(kind).pk},
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert response.json()["attr"] == expected_field
+ assert response.json()["detail"] == f"Step {failed_step}: Step {failed_step} said no"
+ assert self.run_test_step.call_count == failed_step + 1
+ assert not ExternalDataDestination.objects.for_team(self.team.pk).filter(name="unusable").exists()
+
+ @parameterized.expand(
+ [
+ (ExternalDataDestination.Type.POSTGRES, Integration.IntegrationKind.POSTGRESQL),
+ (ExternalDataDestination.Type.REDSHIFT, Integration.IntegrationKind.AWS_REDSHIFT),
+ ]
+ )
+ def test_types_without_a_batch_export_test_skip_it(self, destination_type: str, kind: str) -> None:
+ self._create_destination(type=destination_type, integration=self._integration(kind).pk)
+
+ self.describe_test.assert_not_called()
+ self.run_test_step.assert_not_called()
+
def test_updating_a_destination_does_not_run_the_connection_test(self) -> None:
destination = self._create_destination()
self.check_connection.reset_mock()
@@ -159,6 +386,66 @@ class TestExternalDataDestinationAPI(DestinationAPITestBase):
assert response.status_code == status.HTTP_200_OK
self.check_connection.assert_not_called()
+ def test_test_connection_returns_each_step_and_stops_at_the_first_failure(self) -> None:
+ destination = self._create_destination(
+ type=ExternalDataDestination.Type.BIGQUERY,
+ integration=self._integration(Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT).pk,
+ config={"dataset": "analytics"},
+ )
+ self.describe_test.return_value = _destination_test(3)
+ self.run_test_step.side_effect = _step_results("Skipped", "Failed")
+
+ response = self.client.post(f"{self.base}/{destination.id}/test_connection")
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ assert response.json() == {
+ "steps": [
+ {"name": "Step 0", "description": "Checks 0", "status": "Skipped", "message": "Step 0 said no"},
+ {"name": "Step 1", "description": "Checks 1", "status": "Failed", "message": "Step 1 said no"},
+ {"name": "Step 2", "description": "Checks 2", "status": None, "message": None},
+ ]
+ }
+
+ @parameterized.expand(
+ [
+ (None, "Passed", None),
+ (
+ DestinationConnectionCheckError(CheckFailure.AUTHENTICATION),
+ "Failed",
+ FAILURE_MESSAGES[CheckFailure.AUTHENTICATION],
+ ),
+ ]
+ )
+ def test_test_connection_runs_the_postgres_check_as_one_step(
+ self, error: Exception | None, expected_status: str, expected_message: str | None
+ ) -> None:
+ destination = self._create_destination(config={"database": "analytics"})
+
+ with patch(
+ "products.warehouse_sources.backend.presentation.destination_connection_testing.check_postgres_destination",
+ side_effect=error,
+ ) as check:
+ response = self.client.post(f"{self.base}/{destination.id}/test_connection")
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ check.assert_called_once_with(destination.integration, {"database": "analytics"})
+ assert [(step["status"], step["message"]) for step in response.json()["steps"]] == [
+ (expected_status, expected_message)
+ ]
+
+ def test_test_connection_rejects_destinations_without_a_test(self) -> None:
+ redshift = self._create_destination(
+ type=ExternalDataDestination.Type.REDSHIFT,
+ integration=self._integration(Integration.IntegrationKind.AWS_REDSHIFT).pk,
+ )
+ warehouse = get_or_create_warehouse_destination(self.team.pk)
+
+ for destination in (redshift, warehouse):
+ response = self.client.post(f"{self.base}/{destination.id}/test_connection")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, destination.type
+ assert response.json()["attr"] == "type"
+
def test_the_posthog_warehouse_is_not_user_managed(self) -> None:
response = self.client.post(self.base, {"type": ExternalDataDestination.Type.POSTHOG_WAREHOUSE, "name": "mine"})
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 008beb0ca315..d96655ffae40 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -209,6 +209,43 @@ export interface AddSourcesResponseApi {
resync_failures: ResyncFailureApi[]
}
+/**
+ * * `Passed` - Passed
+ * * `Failed` - Failed
+ * * `Skipped` - Skipped
+ */
+export type DestinationConnectionTestStepStatusEnumApi =
+ (typeof DestinationConnectionTestStepStatusEnumApi)[keyof typeof DestinationConnectionTestStepStatusEnumApi]
+
+export const DestinationConnectionTestStepStatusEnumApi = {
+ Passed: 'Passed',
+ Failed: 'Failed',
+ Skipped: 'Skipped',
+} as const
+
+export interface DestinationConnectionTestStepApi {
+ /** What this step checks. */
+ name: string
+ /** A longer explanation of what this step checks. */
+ description: string
+ /** The result of the step. Null when the step did not run because an earlier step failed.
+ *
+ * * `Passed` - Passed
+ * * `Failed` - Failed
+ * * `Skipped` - Skipped */
+ status: DestinationConnectionTestStepStatusEnumApi | null
+ /**
+ * Why the step failed or was skipped. Null when the step passed or did not run.
+ * @nullable
+ */
+ message: string | null
+}
+
+export interface DestinationConnectionTestResponseApi {
+ /** The steps of the connection test, in order. The test stops at the first failed step. */
+ steps: DestinationConnectionTestStepApi[]
+}
+
/**
* * `full_refresh` - full_refresh
* * `incremental` - incremental
diff --git a/products/warehouse_sources/frontend/generated/api.ts b/products/warehouse_sources/frontend/generated/api.ts
index b597b093ec05..d497013e0086 100644
--- a/products/warehouse_sources/frontend/generated/api.ts
+++ b/products/warehouse_sources/frontend/generated/api.ts
@@ -18,6 +18,7 @@ import type {
CredentialAccountsRequestApi,
DatabaseSchemaRequestApi,
DeleteWebhookResponseApi,
+ DestinationConnectionTestResponseApi,
DirectConnectionSourceOptionApi,
DraftCustomManifestRequestApi,
DraftCustomManifestResponseApi,
@@ -254,6 +255,27 @@ export const externalDataDestinationsAddSourcesCreate = async (
})
}
+export const getExternalDataDestinationsTestConnectionCreateUrl = (projectId: string, id: string) => {
+ return `/api/projects/${projectId}/external_data_destinations/${id}/test_connection/`
+}
+
+/**
+ * Test the credentials and target of an existing destination, and return the result of each step.
+ */
+export const externalDataDestinationsTestConnectionCreate = async (
+ projectId: string,
+ id: string,
+ options?: RequestInit
+): Promise<DestinationConnectionTestResponseApi> => {
+ return apiMutator<DestinationConnectionTestResponseApi>(
+ getExternalDataDestinationsTestConnectionCreateUrl(projectId, id),
+ {
+ ...options,
+ method: 'POST',
+ }
+ )
+}
+
export const getExternalDataSchemasListUrl = (projectId: string, params?: ExternalDataSchemasListParams) => {
const normalizedParams = new URLSearchParams()
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 5a385f8c9b3b..679e97aaedff 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -36405,6 +36405,43 @@ export namespace Schemas {
response_id: string;
}
+ /**
+ * * `Passed` - Passed
+ * * `Failed` - Failed
+ * * `Skipped` - Skipped
+ */
+ export type DestinationConnectionTestStepStatusEnum = typeof DestinationConnectionTestStepStatusEnum[keyof typeof DestinationConnectionTestStepStatusEnum];
+
+
+ export const DestinationConnectionTestStepStatusEnum = {
+ Passed: 'Passed',
+ Failed: 'Failed',
+ Skipped: 'Skipped',
+ } as const;
+
+ export interface DestinationConnectionTestStep {
+ /** What this step checks. */
+ name: string;
+ /** A longer explanation of what this step checks. */
+ description: string;
+ /** The result of the step. Null when the step did not run because an earlier step failed.
+ *
+ * * `Passed` - Passed
+ * * `Failed` - Failed
+ * * `Skipped` - Skipped */
+ status: DestinationConnectionTestStepStatusEnum | null;
+ /**
+ * Why the step failed or was skipped. Null when the step passed or did not run.
+ * @nullable
+ */
+ message: string | null;
+ }
+
+ export interface DestinationConnectionTestResponse {
+ /** The steps of the connection test, in order. The test stops at the first failed step. */
+ steps: DestinationConnectionTestStep[];
+ }
+
/**
* * `Desktop` - Desktop
* * `Mobile` - Mobile