:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add destination connect link and token scopes (#122827)
Daniel RC333f13503647
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_testing.py b/products/warehouse_sources/backend/presentation/destination_connection_testing.pyindex c3ce79dfa493..7c5b950f10a0 100644--- a/products/warehouse_sources/backend/presentation/destination_connection_testing.py+++ b/products/warehouse_sources/backend/presentation/destination_connection_testing.py@@ -128,6 +128,13 @@ _CREDENTIAL_STEP_COUNT: dict[str, int] = {}+def has_connection_test(destination_type: str) -> bool:+ """Whether creating a destination of this type tests its connection first."""+ return destination_type in (ExternalDataDestination.Type.POSTGRES, ExternalDataDestination.Type.S3) or (+ destination_type in _TEST_TYPES+ )++def connection_test_type(destination_type: str, integration: Integration) -> str | None:"""The batch exports test that covers this destination, or None when there is none."""if destination_type == ExternalDataDestination.Type.S3:diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.pyindex 3511cf9ae589..811443ec1247 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py@@ -1,6 +1,8 @@from typing import Any, Optional, cast+from urllib.parse import quote-from django.db import transaction+from django.conf import settings+from django.db import models, transactionfrom django.db.models import Qfrom drf_spectacular.utils import extend_schema, extend_schema_field@@ -9,6 +11,7 @@ from rest_framework.exceptions import PermissionDenied, ValidationErrorfrom rest_framework.request import Requestfrom rest_framework.response import Response+from posthog.api.mixins import ValidatedRequest, validated_requestfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedFieldfrom posthog.api.utils import action@@ -32,6 +35,7 @@ from products.warehouse_sources.backend.presentation.destination_connection_checfrom products.warehouse_sources.backend.presentation.destination_connection_testing import (ConnectionTestRun,DestinationConnectionTestStepStatus,+ has_connection_test,run_connection_test,run_postgres_connection_test,)@@ -61,6 +65,55 @@ DESTINATION_INTEGRATION_KINDS: dict[str, tuple[str, ...]] = {# time a schema resolves to it, and every team has exactly one, so it is not user-managed.USER_CREATABLE_TYPES = frozenset(DESTINATION_INTEGRATION_KINDS)+# Built from `Type`, so the values cannot drift from the model. django-stubs only types the+# literal form of the functional API.+UserCreatableDestinationType = models.TextChoices( # type: ignore[call-overload]+ "UserCreatableDestinationType",+ [+ (choice.name, (choice.value, choice.label))+ for choice in ExternalDataDestination.Type+ if choice in USER_CREATABLE_TYPES+ ],+)+++class DestinationConnectLinkQuerySerializer(serializers.Serializer):+ destination_type = serializers.ChoiceField(+ choices=UserCreatableDestinationType.choices,+ help_text="The destination type to connect, e.g. 'Snowflake' or 'S3'.",+ )+ integration_kind = serializers.CharField(+ required=False,+ help_text=(+ "Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' "+ "(the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar."+ ),+ )++ def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:+ destination_type = attrs["destination_type"]+ allowed_kinds = DESTINATION_INTEGRATION_KINDS[destination_type]+ kind = attrs.get("integration_kind") or allowed_kinds[0]+ if kind not in allowed_kinds:+ expected = " or ".join(f"'{allowed}'" for allowed in allowed_kinds)+ raise ValidationError(+ {"integration_kind": f"A {destination_type} destination needs a {expected} integration."}+ )+ attrs["integration_kind"] = kind+ return attrs+++class DestinationConnectLinkSerializer(serializers.Serializer):+ destination_type = serializers.CharField(help_text="The destination type the link is for.")+ integration_kind = serializers.CharField(help_text="The integration kind the page creates.")+ connect_url = serializers.CharField(+ help_text=(+ "Full URL to share with the user. It opens the connection form for this integration kind in PostHog, "+ "so credentials never pass through the agent or the chat."+ )+ )+ instructions = serializers.CharField(help_text="Next steps for the agent to relay to the user.")+class SyncedSourceSerializer(serializers.Serializer):"""One source that writes to a destination. Shape only — never used to deserialize."""@@ -418,6 +471,17 @@ class ExternalDataDestinationViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewS# there is no single object to check specific access against here; require resource-level# access to `external_data_source` instead.requires_resource_level_access = True+ # Token auth rejects any action missing from these lists.+ scope_object_read_actions = ["list", "retrieve", "connect_link"]+ scope_object_write_actions = [+ "create",+ "update",+ "partial_update",+ "patch",+ "destroy",+ "add_sources",+ "test_connection",+ ]# `.unscoped()` is import-safe (the fail-closed manager raises on `.all()` without team# context); the mixin scopes every request by team_id.queryset = ExternalDataDestination.objects.unscoped().all()@@ -493,6 +557,44 @@ class ExternalDataDestinationViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSif level is None or not access_level_satisfied_for_resource("warehouse_table", level, "editor"):raise PermissionDenied("You do not have editor access to every table wired to this destination.")+ @validated_request(+ query_serializer=DestinationConnectLinkQuerySerializer,+ responses={200: DestinationConnectLinkSerializer},+ )+ @action(methods=["GET"], detail=False, pagination_class=None)+ def connect_link(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:+ """Return a secure browser link where the user connects the credentials for a destination.++ The page saves an integration and does not create the destination, so secrets never pass+ through the agent. The agent then creates the destination with the new integration's id.+ """+ destination_type = request.validated_query_data["destination_type"]+ kind = request.validated_query_data["integration_kind"]+ target_check = (+ "Creating the destination tests the connection, so a wrong target fails at that step."+ if has_connection_test(destination_type)+ else f"Nothing tests a {destination_type} target before the first sync, so confirm it with the user "+ "before creating the destination."+ )+ data = {+ "destination_type": destination_type,+ "integration_kind": kind,+ "connect_url": (+ f"{settings.SITE_URL}/project/{self.team_id}/data-warehouse/connect-destination?kind={quote(kind)}"+ ),+ "instructions": (+ f"Share this link with the user. They enter their {destination_type} credentials directly in "+ "PostHog. Never ask them to paste passwords, keys or tokens into the chat. The page saves the "+ "connection as an integration and shows its id. It does not create the destination. When the user "+ "says they are done, ask them for that id. If they cannot give it, call integrations-list with "+ f"kind='{kind}' and take the entry with the latest created_at (the list is oldest first), then "+ "confirm its display_name with the user. Then call external-data-destinations-create with "+ f"type='{destination_type}', a name, that integration id, and the target config that tool lists "+ f"for {destination_type}. {target_check}"+ ),+ }+ return Response(DestinationConnectLinkSerializer(data).data)+@extend_schema(request=AddSourcesRequestSerializer, responses={200: AddSourcesResponseSerializer})@action(methods=["POST"], detail=True)def add_sources(self, request: Request, *args: Any, **kwargs: Any) -> Response:diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py b/products/warehouse_sources/backend/tests/api/test_external_data_destination.pyindex 835cc74c606c..a1777c401283 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py@@ -1,3 +1,4 @@+import jsonfrom uuid import uuid4import pytest@@ -5,6 +6,7 @@ from posthog.test.base import APIBaseTestfrom unittest.mock import patchfrom django.db import connection+from django.test import SimpleTestCasefrom django.test.utils import CaptureQueriesContextfrom parameterized import parameterized@@ -14,7 +16,9 @@ from rest_framework.response import Responsefrom posthog.constants import AvailableFeaturefrom posthog.models.integration import Integration+from posthog.models.personal_api_key import PersonalAPIKey, hash_key_valuefrom posthog.models.user import User+from posthog.models.utils import generate_random_token_personalfrom products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.access_control.backend.models.access_control import AccessControl@@ -42,6 +46,7 @@ from products.warehouse_sources.backend.presentation.destination_connection_testnew_connection_test_table,)from products.warehouse_sources.backend.presentation.views.external_data_destination import (+ DestinationConnectLinkQuerySerializer,ExternalDataDestinationViewSet,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -1245,3 +1250,81 @@ class TestDestinationResumeAccessControl(DestinationAPITestBase):self.hidden_link.refresh_from_db()assert self.destination.status == ExternalDataDestination.Status.PAUSEDassert self.hidden_link.enabled is False+++class TestDestinationConnectLinkQuery(SimpleTestCase):+ @parameterized.expand(+ [+ ("posthog_warehouse", {"destination_type": "PostHogWarehouse"}, "destination_type"),+ ("unknown_type", {"destination_type": "Oracle"}, "destination_type"),+ ("kind_of_another_type", {"destination_type": "S3", "integration_kind": "snowflake"}, "integration_kind"),+ (+ "second_kind_on_single_kind_type",+ {"destination_type": "Snowflake", "integration_kind": "aws-s3"},+ "integration_kind",+ ),+ ]+ )+ def test_rejects(self, _name: str, query: dict[str, str], field: str) -> None:+ serializer = DestinationConnectLinkQuerySerializer(data=query)++ assert not serializer.is_valid()+ assert field in serializer.errors+++class TestDestinationConnectLink(DestinationAPITestBase):+ @parameterized.expand(+ [+ ("single_kind", "destination_type=Snowflake", "Snowflake", "snowflake", True),+ ("s3_defaults_to_aws", "destination_type=S3", "S3", "aws-s3", True),+ ("s3_compatible", "destination_type=S3&integration_kind=s3-compatible", "S3", "s3-compatible", True),+ ("untested_type", "destination_type=Redshift", "Redshift", "aws-redshift", False),+ ]+ )+ def test_points_at_the_connect_page_for_the_integration_kind(+ self, _name: str, query: str, destination_type: str, kind: str, tested_on_create: bool+ ) -> None:+ response = self.client.get(f"{self.base}/connect_link?{query}")++ assert response.status_code == status.HTTP_200_OK, response.json()+ data = response.json()+ assert data["destination_type"] == destination_type+ assert data["integration_kind"] == kind+ assert data["connect_url"].endswith(f"/project/{self.team.pk}/data-warehouse/connect-destination?kind={kind}")+ assert f"kind='{kind}'" in data["instructions"]+ assert f"type='{destination_type}'" in data["instructions"]+ assert ("tests the connection" in data["instructions"]) == tested_on_create+++class TestDestinationAPIKeyScopes(DestinationAPITestBase):+ def setUp(self) -> None:+ super().setUp()+ self.destination = self._create_destination(+ type=ExternalDataDestination.Type.BIGQUERY,+ integration=self._integration(Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT).pk,+ config={"dataset": "analytics"},+ )+ self.client.force_authenticate(None)++ @parameterized.expand(+ [+ ("external_data_source:read", "GET", "connect_link?destination_type=Postgres", status.HTTP_200_OK),+ ("external_data_source:read", "POST", "{destination}/add_sources", status.HTTP_403_FORBIDDEN),+ ("external_data_source:write", "POST", "{destination}/add_sources", status.HTTP_200_OK),+ ("external_data_source:read", "POST", "{destination}/test_connection", status.HTTP_403_FORBIDDEN),+ ("external_data_source:write", "POST", "{destination}/test_connection", status.HTTP_200_OK),+ ]+ )+ def test_custom_actions_map_to_scopes(self, scope: str, method: str, path: str, expected_status: int) -> None:+ value = generate_random_token_personal()+ PersonalAPIKey.objects.create(user=self.user, label="test", secure_value=hash_key_value(value), scopes=[scope])++ response = self.client.generic(+ method,+ f"{self.base}/{path.format(destination=self.destination.id)}",+ json.dumps({"source_ids": [str(self.source.id)]}),+ content_type="application/json",+ headers={"authorization": f"Bearer {value}"},+ )++ assert response.status_code == expected_status, response.json()diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex ebc2b2c6131c..ef3a77f6779c 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -246,6 +246,17 @@ export interface DestinationConnectionTestResponseApi {steps: DestinationConnectionTestStepApi[]}+export interface DestinationConnectLinkApi {+ /** The destination type the link is for. */+ destination_type: string+ /** The integration kind the page creates. */+ integration_kind: string+ /** Full URL to share with the user. It opens the connection form for this integration kind in PostHog, so credentials never pass through the agent or the chat. */+ connect_url: string+ /** Next steps for the agent to relay to the user. */+ instructions: string+}+/*** * `full_refresh` - full_refresh* * `incremental` - incremental@@ -14864,6 +14875,40 @@ export type ExternalDataDestinationsListParams = {offset?: number}+export type ExternalDataDestinationsConnectLinkRetrieveParams = {+ /**+ * The destination type to connect, e.g. 'Snowflake' or 'S3'.+ *+ * * `Redshift` - Redshift+ * * `Snowflake` - Snowflake+ * * `BigQuery` - BigQuery+ * * `Postgres` - Postgres+ * * `Databricks` - Databricks+ * * `AzureBlob` - Azure Blob+ * * `S3` - S3+ * @minLength 1+ */+ destination_type: ExternalDataDestinationsConnectLinkRetrieveDestinationType+ /**+ * Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' (the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar.+ * @minLength 1+ */+ integration_kind?: string+}++export type ExternalDataDestinationsConnectLinkRetrieveDestinationType =+ (typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType)[keyof typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType]++export const ExternalDataDestinationsConnectLinkRetrieveDestinationType = {+ Redshift: 'Redshift',+ Snowflake: 'Snowflake',+ BigQuery: 'BigQuery',+ Postgres: 'Postgres',+ Databricks: 'Databricks',+ AzureBlob: 'AzureBlob',+ S3: 'S3',+} as const+export type ExternalDataSchemasListParams = {/*** Number of results to return per page.diff --git a/products/warehouse_sources/frontend/generated/api.ts b/products/warehouse_sources/frontend/generated/api.tsindex d74be9b45cf9..6bbe25e02533 100644--- a/products/warehouse_sources/frontend/generated/api.ts+++ b/products/warehouse_sources/frontend/generated/api.ts@@ -18,11 +18,13 @@ import type {CredentialAccountsRequestApi,DatabaseSchemaRequestApi,DeleteWebhookResponseApi,+ DestinationConnectLinkApi,DestinationConnectionTestResponseApi,DirectConnectionSourceOptionApi,DraftCustomManifestRequestApi,DraftCustomManifestResponseApi,ExternalDataDestinationApi,+ ExternalDataDestinationsConnectLinkRetrieveParams,ExternalDataDestinationsListParams,ExternalDataJobSerializersApi,ExternalDataSchemaApi,@@ -281,6 +283,42 @@ export const externalDataDestinationsTestConnectionCreate = async ()}+export const getExternalDataDestinationsConnectLinkRetrieveUrl = (+ projectId: string,+ params: ExternalDataDestinationsConnectLinkRetrieveParams+) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/external_data_destinations/connect_link/?${stringifiedParams}`+ : `/api/projects/${projectId}/external_data_destinations/connect_link/`+}++/**+ * Return a secure browser link where the user connects the credentials for a destination.+ *+ * The page saves an integration and does not create the destination, so secrets never pass+ * through the agent. The agent then creates the destination with the new integration's id.+ */+export const externalDataDestinationsConnectLinkRetrieve = async (+ projectId: string,+ params: ExternalDataDestinationsConnectLinkRetrieveParams,+ options?: RequestInit+): Promise<DestinationConnectLinkApi> => {+ return apiMutator<DestinationConnectLinkApi>(getExternalDataDestinationsConnectLinkRetrieveUrl(projectId, params), {+ ...options,+ method: 'GET',+ })+}+export const getExternalDataSchemasListUrl = (projectId: string, params?: ExternalDataSchemasListParams) => {const normalizedParams = new URLSearchParams()diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 2e31c0ec41b6..dc557db4b84b 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -37938,6 +37938,17 @@ export namespace Schemas {response_id: string;}+ export interface DestinationConnectLink {+ /** The destination type the link is for. */+ destination_type: string;+ /** The integration kind the page creates. */+ integration_kind: string;+ /** Full URL to share with the user. It opens the connection form for this integration kind in PostHog, so credentials never pass through the agent or the chat. */+ connect_url: string;+ /** Next steps for the agent to relay to the user. */+ instructions: string;+ }+/*** * `Passed` - Passed* * `Failed` - Failed@@ -134152,6 +134163,40 @@ export namespace Schemas {offset?: number;};+ export type ExternalDataDestinationsConnectLinkRetrieveParams = {+ /**+ * The destination type to connect, e.g. 'Snowflake' or 'S3'.+ *+ * * `Redshift` - Redshift+ * * `Snowflake` - Snowflake+ * * `BigQuery` - BigQuery+ * * `Postgres` - Postgres+ * * `Databricks` - Databricks+ * * `AzureBlob` - Azure Blob+ * * `S3` - S3+ * @minLength 1+ */+ destination_type: ExternalDataDestinationsConnectLinkRetrieveDestinationType;+ /**+ * Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' (the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar.+ * @minLength 1+ */+ integration_kind?: string;+ };++ export type ExternalDataDestinationsConnectLinkRetrieveDestinationType = typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType[keyof typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType];+++ export const ExternalDataDestinationsConnectLinkRetrieveDestinationType = {+ Redshift: 'Redshift',+ Snowflake: 'Snowflake',+ BigQuery: 'BigQuery',+ Postgres: 'Postgres',+ Databricks: 'Databricks',+ AzureBlob: 'AzureBlob',+ S3: 'S3',+ } as const;+export type ExternalDataSchemasListParams = {/*** Number of results to return per page.