Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add destination connect link and token scopes (#122827)

Daniel RC333f13503647
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_testing.py b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
index c3ce79dfa493..7c5b950f10a0 100644
--- a/products/warehouse_sources/backend/presentation/destination_connection_testing.py
+++ b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
@@ -128,6 +128,13 @@ _CREDENTIAL_STEP_COUNT: dict[str, int] = {
}
+def has_connection_test(destination_type: str) -> bool:
+ """Whether creating a destination of this type tests its connection first."""
+ return destination_type in (ExternalDataDestination.Type.POSTGRES, ExternalDataDestination.Type.S3) or (
+ destination_type in _TEST_TYPES
+ )
+
+
def connection_test_type(destination_type: str, integration: Integration) -> str | None:
"""The batch exports test that covers this destination, or None when there is none."""
if destination_type == ExternalDataDestination.Type.S3:
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
index 3511cf9ae589..811443ec1247 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
@@ -1,6 +1,8 @@
from typing import Any, Optional, cast
+from urllib.parse import quote
-from django.db import transaction
+from django.conf import settings
+from django.db import models, transaction
from django.db.models import Q
from drf_spectacular.utils import extend_schema, extend_schema_field
@@ -9,6 +11,7 @@ from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.request import Request
from rest_framework.response import Response
+from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
from posthog.api.utils import action
@@ -32,6 +35,7 @@ from products.warehouse_sources.backend.presentation.destination_connection_chec
from products.warehouse_sources.backend.presentation.destination_connection_testing import (
ConnectionTestRun,
DestinationConnectionTestStepStatus,
+ has_connection_test,
run_connection_test,
run_postgres_connection_test,
)
@@ -61,6 +65,55 @@ DESTINATION_INTEGRATION_KINDS: dict[str, tuple[str, ...]] = {
# time a schema resolves to it, and every team has exactly one, so it is not user-managed.
USER_CREATABLE_TYPES = frozenset(DESTINATION_INTEGRATION_KINDS)
+# Built from `Type`, so the values cannot drift from the model. django-stubs only types the
+# literal form of the functional API.
+UserCreatableDestinationType = models.TextChoices( # type: ignore[call-overload]
+ "UserCreatableDestinationType",
+ [
+ (choice.name, (choice.value, choice.label))
+ for choice in ExternalDataDestination.Type
+ if choice in USER_CREATABLE_TYPES
+ ],
+)
+
+
+class DestinationConnectLinkQuerySerializer(serializers.Serializer):
+ destination_type = serializers.ChoiceField(
+ choices=UserCreatableDestinationType.choices,
+ help_text="The destination type to connect, e.g. 'Snowflake' or 'S3'.",
+ )
+ integration_kind = serializers.CharField(
+ required=False,
+ help_text=(
+ "Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' "
+ "(the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar."
+ ),
+ )
+
+ def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
+ destination_type = attrs["destination_type"]
+ allowed_kinds = DESTINATION_INTEGRATION_KINDS[destination_type]
+ kind = attrs.get("integration_kind") or allowed_kinds[0]
+ if kind not in allowed_kinds:
+ expected = " or ".join(f"'{allowed}'" for allowed in allowed_kinds)
+ raise ValidationError(
+ {"integration_kind": f"A {destination_type} destination needs a {expected} integration."}
+ )
+ attrs["integration_kind"] = kind
+ return attrs
+
+
+class DestinationConnectLinkSerializer(serializers.Serializer):
+ destination_type = serializers.CharField(help_text="The destination type the link is for.")
+ integration_kind = serializers.CharField(help_text="The integration kind the page creates.")
+ connect_url = serializers.CharField(
+ help_text=(
+ "Full URL to share with the user. It opens the connection form for this integration kind in PostHog, "
+ "so credentials never pass through the agent or the chat."
+ )
+ )
+ instructions = serializers.CharField(help_text="Next steps for the agent to relay to the user.")
+
class SyncedSourceSerializer(serializers.Serializer):
"""One source that writes to a destination. Shape only — never used to deserialize."""
@@ -418,6 +471,17 @@ class ExternalDataDestinationViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewS
# there is no single object to check specific access against here; require resource-level
# access to `external_data_source` instead.
requires_resource_level_access = True
+ # Token auth rejects any action missing from these lists.
+ scope_object_read_actions = ["list", "retrieve", "connect_link"]
+ scope_object_write_actions = [
+ "create",
+ "update",
+ "partial_update",
+ "patch",
+ "destroy",
+ "add_sources",
+ "test_connection",
+ ]
# `.unscoped()` is import-safe (the fail-closed manager raises on `.all()` without team
# context); the mixin scopes every request by team_id.
queryset = ExternalDataDestination.objects.unscoped().all()
@@ -493,6 +557,44 @@ class ExternalDataDestinationViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewS
if level is None or not access_level_satisfied_for_resource("warehouse_table", level, "editor"):
raise PermissionDenied("You do not have editor access to every table wired to this destination.")
+ @validated_request(
+ query_serializer=DestinationConnectLinkQuerySerializer,
+ responses={200: DestinationConnectLinkSerializer},
+ )
+ @action(methods=["GET"], detail=False, pagination_class=None)
+ def connect_link(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:
+ """Return a secure browser link where the user connects the credentials for a destination.
+
+ The page saves an integration and does not create the destination, so secrets never pass
+ through the agent. The agent then creates the destination with the new integration's id.
+ """
+ destination_type = request.validated_query_data["destination_type"]
+ kind = request.validated_query_data["integration_kind"]
+ target_check = (
+ "Creating the destination tests the connection, so a wrong target fails at that step."
+ if has_connection_test(destination_type)
+ else f"Nothing tests a {destination_type} target before the first sync, so confirm it with the user "
+ "before creating the destination."
+ )
+ data = {
+ "destination_type": destination_type,
+ "integration_kind": kind,
+ "connect_url": (
+ f"{settings.SITE_URL}/project/{self.team_id}/data-warehouse/connect-destination?kind={quote(kind)}"
+ ),
+ "instructions": (
+ f"Share this link with the user. They enter their {destination_type} credentials directly in "
+ "PostHog. Never ask them to paste passwords, keys or tokens into the chat. The page saves the "
+ "connection as an integration and shows its id. It does not create the destination. When the user "
+ "says they are done, ask them for that id. If they cannot give it, call integrations-list with "
+ f"kind='{kind}' and take the entry with the latest created_at (the list is oldest first), then "
+ "confirm its display_name with the user. Then call external-data-destinations-create with "
+ f"type='{destination_type}', a name, that integration id, and the target config that tool lists "
+ f"for {destination_type}. {target_check}"
+ ),
+ }
+ return Response(DestinationConnectLinkSerializer(data).data)
+
@extend_schema(request=AddSourcesRequestSerializer, responses={200: AddSourcesResponseSerializer})
@action(methods=["POST"], detail=True)
def add_sources(self, request: Request, *args: Any, **kwargs: Any) -> Response:
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
index 835cc74c606c..a1777c401283 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
@@ -1,3 +1,4 @@
+import json
from uuid import uuid4
import pytest
@@ -5,6 +6,7 @@ from posthog.test.base import APIBaseTest
from unittest.mock import patch
from django.db import connection
+from django.test import SimpleTestCase
from django.test.utils import CaptureQueriesContext
from parameterized import parameterized
@@ -14,7 +16,9 @@ from rest_framework.response import Response
from posthog.constants import AvailableFeature
from posthog.models.integration import Integration
+from posthog.models.personal_api_key import PersonalAPIKey, hash_key_value
from posthog.models.user import User
+from posthog.models.utils import generate_random_token_personal
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.access_control.backend.models.access_control import AccessControl
@@ -42,6 +46,7 @@ from products.warehouse_sources.backend.presentation.destination_connection_test
new_connection_test_table,
)
from products.warehouse_sources.backend.presentation.views.external_data_destination import (
+ DestinationConnectLinkQuerySerializer,
ExternalDataDestinationViewSet,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -1245,3 +1250,81 @@ class TestDestinationResumeAccessControl(DestinationAPITestBase):
self.hidden_link.refresh_from_db()
assert self.destination.status == ExternalDataDestination.Status.PAUSED
assert self.hidden_link.enabled is False
+
+
+class TestDestinationConnectLinkQuery(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("posthog_warehouse", {"destination_type": "PostHogWarehouse"}, "destination_type"),
+ ("unknown_type", {"destination_type": "Oracle"}, "destination_type"),
+ ("kind_of_another_type", {"destination_type": "S3", "integration_kind": "snowflake"}, "integration_kind"),
+ (
+ "second_kind_on_single_kind_type",
+ {"destination_type": "Snowflake", "integration_kind": "aws-s3"},
+ "integration_kind",
+ ),
+ ]
+ )
+ def test_rejects(self, _name: str, query: dict[str, str], field: str) -> None:
+ serializer = DestinationConnectLinkQuerySerializer(data=query)
+
+ assert not serializer.is_valid()
+ assert field in serializer.errors
+
+
+class TestDestinationConnectLink(DestinationAPITestBase):
+ @parameterized.expand(
+ [
+ ("single_kind", "destination_type=Snowflake", "Snowflake", "snowflake", True),
+ ("s3_defaults_to_aws", "destination_type=S3", "S3", "aws-s3", True),
+ ("s3_compatible", "destination_type=S3&integration_kind=s3-compatible", "S3", "s3-compatible", True),
+ ("untested_type", "destination_type=Redshift", "Redshift", "aws-redshift", False),
+ ]
+ )
+ def test_points_at_the_connect_page_for_the_integration_kind(
+ self, _name: str, query: str, destination_type: str, kind: str, tested_on_create: bool
+ ) -> None:
+ response = self.client.get(f"{self.base}/connect_link?{query}")
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ data = response.json()
+ assert data["destination_type"] == destination_type
+ assert data["integration_kind"] == kind
+ assert data["connect_url"].endswith(f"/project/{self.team.pk}/data-warehouse/connect-destination?kind={kind}")
+ assert f"kind='{kind}'" in data["instructions"]
+ assert f"type='{destination_type}'" in data["instructions"]
+ assert ("tests the connection" in data["instructions"]) == tested_on_create
+
+
+class TestDestinationAPIKeyScopes(DestinationAPITestBase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.destination = self._create_destination(
+ type=ExternalDataDestination.Type.BIGQUERY,
+ integration=self._integration(Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT).pk,
+ config={"dataset": "analytics"},
+ )
+ self.client.force_authenticate(None)
+
+ @parameterized.expand(
+ [
+ ("external_data_source:read", "GET", "connect_link?destination_type=Postgres", status.HTTP_200_OK),
+ ("external_data_source:read", "POST", "{destination}/add_sources", status.HTTP_403_FORBIDDEN),
+ ("external_data_source:write", "POST", "{destination}/add_sources", status.HTTP_200_OK),
+ ("external_data_source:read", "POST", "{destination}/test_connection", status.HTTP_403_FORBIDDEN),
+ ("external_data_source:write", "POST", "{destination}/test_connection", status.HTTP_200_OK),
+ ]
+ )
+ def test_custom_actions_map_to_scopes(self, scope: str, method: str, path: str, expected_status: int) -> None:
+ value = generate_random_token_personal()
+ PersonalAPIKey.objects.create(user=self.user, label="test", secure_value=hash_key_value(value), scopes=[scope])
+
+ response = self.client.generic(
+ method,
+ f"{self.base}/{path.format(destination=self.destination.id)}",
+ json.dumps({"source_ids": [str(self.source.id)]}),
+ content_type="application/json",
+ headers={"authorization": f"Bearer {value}"},
+ )
+
+ assert response.status_code == expected_status, response.json()
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index ebc2b2c6131c..ef3a77f6779c 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -246,6 +246,17 @@ export interface DestinationConnectionTestResponseApi {
steps: DestinationConnectionTestStepApi[]
}
+export interface DestinationConnectLinkApi {
+ /** The destination type the link is for. */
+ destination_type: string
+ /** The integration kind the page creates. */
+ integration_kind: string
+ /** Full URL to share with the user. It opens the connection form for this integration kind in PostHog, so credentials never pass through the agent or the chat. */
+ connect_url: string
+ /** Next steps for the agent to relay to the user. */
+ instructions: string
+}
+
/**
* * `full_refresh` - full_refresh
* * `incremental` - incremental
@@ -14864,6 +14875,40 @@ export type ExternalDataDestinationsListParams = {
offset?: number
}
+export type ExternalDataDestinationsConnectLinkRetrieveParams = {
+ /**
+ * The destination type to connect, e.g. 'Snowflake' or 'S3'.
+ *
+ * * `Redshift` - Redshift
+ * * `Snowflake` - Snowflake
+ * * `BigQuery` - BigQuery
+ * * `Postgres` - Postgres
+ * * `Databricks` - Databricks
+ * * `AzureBlob` - Azure Blob
+ * * `S3` - S3
+ * @minLength 1
+ */
+ destination_type: ExternalDataDestinationsConnectLinkRetrieveDestinationType
+ /**
+ * Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' (the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar.
+ * @minLength 1
+ */
+ integration_kind?: string
+}
+
+export type ExternalDataDestinationsConnectLinkRetrieveDestinationType =
+ (typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType)[keyof typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType]
+
+export const ExternalDataDestinationsConnectLinkRetrieveDestinationType = {
+ Redshift: 'Redshift',
+ Snowflake: 'Snowflake',
+ BigQuery: 'BigQuery',
+ Postgres: 'Postgres',
+ Databricks: 'Databricks',
+ AzureBlob: 'AzureBlob',
+ S3: 'S3',
+} as const
+
export type ExternalDataSchemasListParams = {
/**
* Number of results to return per page.
diff --git a/products/warehouse_sources/frontend/generated/api.ts b/products/warehouse_sources/frontend/generated/api.ts
index d74be9b45cf9..6bbe25e02533 100644
--- a/products/warehouse_sources/frontend/generated/api.ts
+++ b/products/warehouse_sources/frontend/generated/api.ts
@@ -18,11 +18,13 @@ import type {
CredentialAccountsRequestApi,
DatabaseSchemaRequestApi,
DeleteWebhookResponseApi,
+ DestinationConnectLinkApi,
DestinationConnectionTestResponseApi,
DirectConnectionSourceOptionApi,
DraftCustomManifestRequestApi,
DraftCustomManifestResponseApi,
ExternalDataDestinationApi,
+ ExternalDataDestinationsConnectLinkRetrieveParams,
ExternalDataDestinationsListParams,
ExternalDataJobSerializersApi,
ExternalDataSchemaApi,
@@ -281,6 +283,42 @@ export const externalDataDestinationsTestConnectionCreate = async (
)
}
+export const getExternalDataDestinationsConnectLinkRetrieveUrl = (
+ projectId: string,
+ params: ExternalDataDestinationsConnectLinkRetrieveParams
+) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/external_data_destinations/connect_link/?${stringifiedParams}`
+ : `/api/projects/${projectId}/external_data_destinations/connect_link/`
+}
+
+/**
+ * Return a secure browser link where the user connects the credentials for a destination.
+ *
+ * The page saves an integration and does not create the destination, so secrets never pass
+ * through the agent. The agent then creates the destination with the new integration's id.
+ */
+export const externalDataDestinationsConnectLinkRetrieve = async (
+ projectId: string,
+ params: ExternalDataDestinationsConnectLinkRetrieveParams,
+ options?: RequestInit
+): Promise<DestinationConnectLinkApi> => {
+ return apiMutator<DestinationConnectLinkApi>(getExternalDataDestinationsConnectLinkRetrieveUrl(projectId, params), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getExternalDataSchemasListUrl = (projectId: string, params?: ExternalDataSchemasListParams) => {
const normalizedParams = new URLSearchParams()
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2e31c0ec41b6..dc557db4b84b 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -37938,6 +37938,17 @@ export namespace Schemas {
response_id: string;
}
+ export interface DestinationConnectLink {
+ /** The destination type the link is for. */
+ destination_type: string;
+ /** The integration kind the page creates. */
+ integration_kind: string;
+ /** Full URL to share with the user. It opens the connection form for this integration kind in PostHog, so credentials never pass through the agent or the chat. */
+ connect_url: string;
+ /** Next steps for the agent to relay to the user. */
+ instructions: string;
+ }
+
/**
* * `Passed` - Passed
* * `Failed` - Failed
@@ -134152,6 +134163,40 @@ export namespace Schemas {
offset?: number;
};
+ export type ExternalDataDestinationsConnectLinkRetrieveParams = {
+ /**
+ * The destination type to connect, e.g. 'Snowflake' or 'S3'.
+ *
+ * * `Redshift` - Redshift
+ * * `Snowflake` - Snowflake
+ * * `BigQuery` - BigQuery
+ * * `Postgres` - Postgres
+ * * `Databricks` - Databricks
+ * * `AzureBlob` - Azure Blob
+ * * `S3` - S3
+ * @minLength 1
+ */
+ destination_type: ExternalDataDestinationsConnectLinkRetrieveDestinationType;
+ /**
+ * Which integration kind to connect when the type accepts more than one. Only S3 does: 'aws-s3' (the default) or 's3-compatible' for MinIO, Wasabi, R2, GCS and similar.
+ * @minLength 1
+ */
+ integration_kind?: string;
+ };
+
+ export type ExternalDataDestinationsConnectLinkRetrieveDestinationType = typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType[keyof typeof ExternalDataDestinationsConnectLinkRetrieveDestinationType];
+
+
+ export const ExternalDataDestinationsConnectLinkRetrieveDestinationType = {
+ Redshift: 'Redshift',
+ Snowflake: 'Snowflake',
+ BigQuery: 'BigQuery',
+ Postgres: 'Postgres',
+ Databricks: 'Databricks',
+ AzureBlob: 'AzureBlob',
+ S3: 'S3',
+ } as const;
+
export type ExternalDataSchemasListParams = {
/**
* Number of results to return per page.