:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-destinations): allow redshift iam role and access keys (#128397)
Tom Owersd3c0edc78f41
diff --git a/posthog/models/integration/__init__.py b/posthog/models/integration/__init__.pyindex 19c89fc54ad0..f7d2dd7a17da 100644--- a/posthog/models/integration/__init__.py+++ b/posthog/models/integration/__init__.py@@ -26,6 +26,7 @@ from .auth_types import (register_kind_auth_types,)from .aws import (+ AnyRedshiftIntegration,AWSCredentialsIntegration,AWSRedshiftIntegration,AWSRedshiftRoleBasedIntegration,@@ -33,8 +34,12 @@ from .aws import (AWSS3Integration,AWSS3RoleBasedIntegration,DuplicateNameError,+ RedshiftProvisionedCluster,+ RedshiftServerlessWorkgroup,S3CompatibleIntegration,is_unique_aws_role_by_organization_id,+ parse_redshift_host,+ redshift_integration_from_record,validate_aws_credentials,)from .azure_blob import AzureBlobIntegration, AzureBlobIntegrationError@@ -286,6 +291,11 @@ __all__ = ["AWSCredentialsIntegration","AWSS3Integration","AWSRedshiftIntegration",+ "AnyRedshiftIntegration",+ "RedshiftProvisionedCluster",+ "RedshiftServerlessWorkgroup",+ "parse_redshift_host",+ "redshift_integration_from_record","S3CompatibleIntegration","STRIPE_POSTHOG_SECRET_NAMES","StripeIntegration",diff --git a/posthog/models/integration/aws.py b/posthog/models/integration/aws.pyindex baf0a3f89d14..b4c3fcaed8f1 100644--- a/posthog/models/integration/aws.py+++ b/posthog/models/integration/aws.py@@ -14,12 +14,14 @@ from rest_framework.exceptions import ValidationErrorfrom posthog.credentials import AWSKeyPairfrom posthog.credentials_service.callers import CredentialsCallerfrom posthog.credentials_service.settings_fallback import get_or_settings+from posthog.dataclasses import frozenfrom posthog.integrations.contracts import IntegrationRecordfrom posthog.models.user import Userfrom posthog.models.utils import IntegrityErrorfrom posthog.security.url_validation import is_url_allowedfrom . import common, model+from .postgres import RedshiftIntegration_AWSKindType = Literal[model.Integration.IntegrationKind.AWS_REDSHIFT, model.Integration.IntegrationKind.AWS_S3]@@ -347,6 +349,42 @@ AWS_REDSHIFT_SERVERLESS_WORKGROUP_ARN_RE = re.compile()+@frozen+class RedshiftProvisionedCluster:+ """A provisioned Redshift cluster parsed from its endpoint hostname."""++ cluster_identifier: str+ region: str+++@frozen+class RedshiftServerlessWorkgroup:+ """A Redshift Serverless workgroup parsed from its endpoint hostname."""++ workgroup: str+ account_id: str+ region: str+++def parse_redshift_host(host: str) -> RedshiftProvisionedCluster | RedshiftServerlessWorkgroup:+ """Parse a Redshift endpoint hostname into its cluster or workgroup location.++ The IAM role and access key integrations need this to ask AWS for temporary database+ credentials, because the AWS APIs take the cluster or workgroup name, not the host.+ """+ match host.split(".", maxsplit=3):+ case [cluster_identifier, _, region, "redshift.amazonaws.com"]:+ return RedshiftProvisionedCluster(cluster_identifier=cluster_identifier, region=region)+ case [workgroup, account_id, region, "redshift-serverless.amazonaws.com"]:+ return RedshiftServerlessWorkgroup(workgroup=workgroup, account_id=account_id, region=region)+ case _:+ raise common.IntegrationError(+ f"Redshift host '{host}' is not a recognized AWS Redshift endpoint. Expected "+ "'<cluster>.<id>.<region>.redshift.amazonaws.com' or "+ "'<workgroup>.<account>.<region>.redshift-serverless.amazonaws.com'"+ )++def _return_redshift_workgroup_arn(**config) -> str | None:if (workgroup_arn := config.get("workgroup_arn")) is not None:if not isinstance(workgroup_arn, str) or not AWS_REDSHIFT_SERVERLESS_WORKGROUP_ARN_RE.match(workgroup_arn):@@ -397,6 +435,26 @@ class AWSRedshiftRoleBasedIntegration(AWSRoleBasedIntegration):"""ARN of a Redshift Serverless workgroup, required when the cluster is Serverless."""return self.integration.config.get("workgroup_arn")+ def workgroup_arn_for(self, server: RedshiftServerlessWorkgroup) -> str:+ """Return the workgroup ARN, checked against the workgroup the host points at.++ Workgroup ARNs contain a UUID that cannot be derived from the endpoint hostname,+ so Serverless requires the ARN on the integration to scope the assumed role exactly.+ """+ workgroup_arn = self.workgroup_arn+ if workgroup_arn is None:+ raise common.IntegrationError(+ "A Redshift Serverless workgroup requires 'workgroup_arn' to be set on the integration"+ )++ _, _, _, arn_region, arn_account_id, _ = workgroup_arn.split(":", maxsplit=5)+ if arn_region != server.region or arn_account_id != server.account_id:+ raise common.IntegrationError(+ f"The integration's workgroup ARN '{workgroup_arn}' does not match the region or account "+ f"of the Redshift Serverless host (region '{server.region}', account '{server.account_id}')"+ )+ return workgroup_arn+@staticmethoddef read_extra_configuration_parameters(**config) -> dict[str, Any]:return _return_extra_redshift_configuration(**config)@@ -597,3 +655,18 @@ class S3CompatibleIntegration:},created_by=created_by,)+++AnyRedshiftIntegration = AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration+++def redshift_integration_from_record(integration: IntegrationRecord) -> AnyRedshiftIntegration:+ """Wrap an `aws-redshift` integration in the class its stored config belongs to.++ The kind covers three classes, told apart only by which keys the integration stores.+ """+ if "aws_role_arn" in (integration.config or {}):+ return AWSRedshiftRoleBasedIntegration(integration)+ if "aws_access_key_id" in (integration.sensitive_config or {}):+ return AWSRedshiftIntegration(integration)+ return RedshiftIntegration(integration)diff --git a/products/batch_exports/backend/facade/destinations/redshift.py b/products/batch_exports/backend/facade/destinations/redshift.pyindex 63724c10e8e1..be7308a61d2f 100644--- a/products/batch_exports/backend/facade/destinations/redshift.py+++ b/products/batch_exports/backend/facade/destinations/redshift.py@@ -1,7 +1,8 @@"""Redshift destination wiring for batch_exports.-Re-exports the client that the warehouse_sources Redshift writer reuses.+Re-exports the client and the connection resolver that the warehouse_sources Redshift+writer reuses.Importing this module loads the destination's vendor SDK, so keep it off the``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``.@@ -10,9 +11,11 @@ Importing this module loads the destination's vendor SDK, so keep it off thefrom products.batch_exports.backend.temporal.destinations.redshift_batch_export import (RedshiftClient,StringLimitExceededError,+ resolve_redshift_client_inputs,)__all__ = ["RedshiftClient","StringLimitExceededError",+ "resolve_redshift_client_inputs",]diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.pyindex 7cbfc280c38d..022104a7098c 100644--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py@@ -25,6 +25,7 @@ from posthog.integrations import IntegrationManager, IntegrationNotFoundfrom posthog.models import Teamfrom posthog.models.integration import (TLS,+ AnyRedshiftIntegration,Authority,AWSRedshiftIntegration,AWSRedshiftRoleBasedIntegration,@@ -33,6 +34,10 @@ from posthog.models.integration import (Credentials,IntegrationError,RedshiftIntegration,+ RedshiftProvisionedCluster,+ RedshiftServerlessWorkgroup,+ parse_redshift_host,+ redshift_integration_from_record,)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeater@@ -899,21 +904,14 @@ def _get_merge_settings(return NotRequiredMergeSettings(requires_merge=False)-async def _get_redshift_integration(- integration_id: int, team_id: int-) -> AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration:+async def _get_redshift_integration(integration_id: int, team_id: int) -> AnyRedshiftIntegration:"""Fetch a Redshift integration from the database."""try:integration = await IntegrationManager.aws_redshift.credentials.aget(team_id, integration_id)except IntegrationNotFound:raise IntegrationError(f"Redshift integration with ID '{integration_id}' not found for team '{team_id}'")- if "aws_role_arn" in integration.config:- return AWSRedshiftRoleBasedIntegration(integration)- elif "aws_access_key_id" in integration.sensitive_config:- return AWSRedshiftIntegration(integration)- else:- return RedshiftIntegration(integration)+ return redshift_integration_from_record(integration)async def _get_aws_s3_integration(integration_id: int, team_id: int) -> AWSS3RoleBasedIntegration | AWSS3Integration:@@ -935,51 +933,10 @@ class RedshiftCredentials:password: str = dataclasses.field(repr=False)-@frozen-class ProvisionedCluster:- """A provisioned Redshift cluster parsed from its endpoint hostname."""-- cluster_identifier: str- region: str---@frozen-class ServerlessWorkgroup:- """A Redshift Serverless workgroup parsed from its endpoint hostname."""-- workgroup: str- account_id: str- region: str---def _parse_redshift_host(host: str) -> ProvisionedCluster | ServerlessWorkgroup:- """Parse a Redshift endpoint hostname into its cluster or workgroup location."""- match host.split(".", maxsplit=3):- case [cluster_identifier, _, region, "redshift.amazonaws.com"]:- return ProvisionedCluster(cluster_identifier=cluster_identifier, region=region)- case [workgroup, account_id, region, "redshift-serverless.amazonaws.com"]:- return ServerlessWorkgroup(workgroup=workgroup, account_id=account_id, region=region)- case _:- raise IntegrationError(- f"Redshift host '{host}' is not a recognized AWS Redshift endpoint. Expected "- "'<cluster>.<id>.<region>.redshift.amazonaws.com' or "- "'<workgroup>.<account>.<region>.redshift-serverless.amazonaws.com'"- )---def _get_host_from_connection(connection: ConnectionParameters | ServerConnectionParameters) -> str:- if connection.host is None:- raise IntegrationError(- "An AWS Redshift integration requires the cluster endpoint to be set as 'host' in the "- "batch export configuration"- )- return connection.host--async def _get_temporary_redshift_credentials(aws_credentials: AWSCredentials,*,- server: ProvisionedCluster | ServerlessWorkgroup,+ server: RedshiftProvisionedCluster | RedshiftServerlessWorkgroup,user: str,database: str,auto_create: bool | None = None,@@ -1017,7 +974,7 @@ async def _get_temporary_redshift_credentials()match server:- case ProvisionedCluster():+ case RedshiftProvisionedCluster():optional: dict[str, typing.Any] = {}if auto_create is not None:optional["AutoCreate"] = auto_create@@ -1036,7 +993,7 @@ async def _get_temporary_redshift_credentials(return RedshiftCredentials(user=response["DbUser"], password=response["DbPassword"])- case ServerlessWorkgroup():+ case RedshiftServerlessWorkgroup():async with session.client("redshift-serverless", region_name=server.region) as redshift_serverless:response = await redshift_serverless.get_credentials(workgroupName=server.workgroup,@@ -1074,12 +1031,12 @@ async def _get_redshift_client_inputs(def _get_redshift_credentials_policy_statements(integration: AWSRedshiftRoleBasedIntegration,- server: ProvisionedCluster | ServerlessWorkgroup,+ server: RedshiftProvisionedCluster | RedshiftServerlessWorkgroup,database: str,) -> list[PolicyStatement]:"""Build policy statements scoping an assumed role to obtaining Redshift credentials."""match server:- case ProvisionedCluster():+ case RedshiftProvisionedCluster():account_id = integration.aws_account_idget_cluster_credentials_policy = PolicyStatement(Effect="Allow",@@ -1106,22 +1063,8 @@ def _get_redshift_credentials_policy_statements()return policy_statements- case ServerlessWorkgroup():- # Workgroup ARNs contain a UUID that cannot be derived from the endpoint hostname,- # so Serverless requires the ARN on the integration to scope the policy exactly.- workgroup_arn = integration.workgroup_arn- if workgroup_arn is None:- raise IntegrationError(- "A Redshift Serverless workgroup requires 'workgroup_arn' to be set on the integration"- )-- _, _, _, arn_region, arn_account_id, _ = workgroup_arn.split(":", maxsplit=5)- if arn_region != server.region or arn_account_id != server.account_id:- raise IntegrationError(- f"The integration's workgroup ARN '{workgroup_arn}' does not match the region or account "- f"of the Redshift Serverless host (region '{server.region}', account '{server.account_id}')"- )-+ case RedshiftServerlessWorkgroup():+ workgroup_arn = integration.workgroup_arn_for(server)return [PolicyStatement(Effect="Allow",@@ -1135,10 +1078,37 @@ def _get_redshift_credentials_policy_statements(async def _get_redshift_client_inputs_from_integration(- integration: AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration,+ integration: AnyRedshiftIntegration,inputs: RedshiftInsertInputs | RedshiftCopyActivityInputs,) -> _PostgreSQLClientInputsProtocol:- """Return PostgreSQL client inputs to establish a connection from Redshift.+ """Return PostgreSQL client inputs for a batch export backed by an integration."""+ host = inputs.connection.host+ if host is None and not isinstance(integration, RedshiftIntegration):+ raise IntegrationError(+ "An AWS Redshift integration requires the cluster endpoint to be set as 'host' in the "+ "batch export configuration"+ )++ return await resolve_redshift_client_inputs(+ integration,+ host=host,+ port=inputs.connection.port,+ database=inputs.connection.database,+ team_id=inputs.batch_export.team_id,+ session_name=f"PostHog-batch-exports-{inputs.batch_export.batch_export_id}",+ )+++async def resolve_redshift_client_inputs(+ integration: AnyRedshiftIntegration,+ *,+ host: str | None,+ port: int,+ database: str,+ team_id: int,+ session_name: str,+) -> _PostgreSQLClientInputsProtocol:+ """Return PostgreSQL client inputs to connect to Redshift with an integration.These inputs can be obtained from an integration by:* Generating temporary credentials using a short-lived session via assuming@@ -1147,26 +1117,26 @@ async def _get_redshift_client_inputs_from_integration(in the integration.* Directly reading inputs stored in the integration.- Depending on the type of integration, is which one we'll use.+ Depending on the type of integration, is which one we'll use. `host` and `port` are+ only read for the two AWS integrations, because a plain Redshift integration stores+ its own."""client_inputs: _PostgreSQLClientInputsProtocolmatch integration:case AWSRedshiftRoleBasedIntegration():- host = _get_host_from_connection(inputs.connection)- server = _parse_redshift_host(host)+ host = _require_aws_host(host)+ server = parse_redshift_host(host)- team = await Team.objects.aget(id=inputs.batch_export.team_id)+ team = await Team.objects.aget(id=team_id)external_id = f"posthog-{team.organization_id}"- policy_statements = _get_redshift_credentials_policy_statements(- integration, server, inputs.connection.database- )+ policy_statements = _get_redshift_credentials_policy_statements(integration, server, database)credentials = await get_credentials_using_user_aws_role(integration.aws_role_arn,external_id,- session_name=f"PostHog-batch-exports-{inputs.batch_export.batch_export_id}",+ session_name=session_name,policy_statements=policy_statements,)@@ -1174,7 +1144,7 @@ async def _get_redshift_client_inputs_from_integration(credentials,server=server,user=integration.user,- database=inputs.connection.database,+ database=database,groups=integration.groups,auto_create=integration.auto_create,)@@ -1182,13 +1152,13 @@ async def _get_redshift_client_inputs_from_integration(user=redshift_credentials.user,password=redshift_credentials.password,host=host,- port=inputs.connection.port,- database=inputs.connection.database,+ port=port,+ database=database,)case AWSRedshiftIntegration():- host = _get_host_from_connection(inputs.connection)- server = _parse_redshift_host(host)+ host = _require_aws_host(host)+ server = parse_redshift_host(host)redshift_credentials = await _get_temporary_redshift_credentials(AWSCredentials(@@ -1197,7 +1167,7 @@ async def _get_redshift_client_inputs_from_integration(),server=server,user=integration.user,- database=inputs.connection.database,+ database=database,groups=integration.groups,auto_create=integration.auto_create,)@@ -1205,8 +1175,8 @@ async def _get_redshift_client_inputs_from_integration(user=redshift_credentials.user,password=redshift_credentials.password,host=host,- port=inputs.connection.port,- database=inputs.connection.database,+ port=port,+ database=database,)case RedshiftIntegration():@@ -1218,6 +1188,12 @@ async def _get_redshift_client_inputs_from_integration(return client_inputs+def _require_aws_host(host: str | None) -> str:+ if host is None:+ raise IntegrationError("An AWS Redshift integration requires the cluster endpoint as 'host'")+ return host++@activity.defn@handle_non_retryable_errors(NON_RETRYABLE_ERROR_TYPES)async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs) -> BatchExportResult:diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.pyindex 24fb129c4ea8..54e439495b55 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py@@ -13,18 +13,22 @@ import aioboto3import pytest_asyncioimport botocore.exceptions-from posthog.models.integration import AWSRedshiftRoleBasedIntegration, Integration, IntegrationError+from posthog.models.integration import (+ AWSRedshiftRoleBasedIntegration,+ Integration,+ IntegrationError,+ RedshiftProvisionedCluster,+ RedshiftServerlessWorkgroup,+ parse_redshift_host,+)from products.batch_exports.backend.service import AWSCredentials, BatchExportEventsModelfrom products.batch_exports.backend.temporal.destinations.redshift_batch_export import (ClientErrorGroup,InsufficientS3PermissionsError,- ProvisionedCluster,RedshiftS3CopyError,- ServerlessWorkgroup,_get_redshift_credentials_policy_statements,_get_table_schemas,- _parse_redshift_host,check_and_raise_redshift_copy_error,is_s3_read_access_denied,upload_manifest_file,@@ -327,22 +331,22 @@ async def test_check_and_raise_redshift_copy_error_iam_role(error, expected_erro[("examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com",- ProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2"),+ RedshiftProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2"),),("default-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com",- ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1"),+ RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1"),),],)def test_parse_redshift_host(host, expected):- assert _parse_redshift_host(host) == expected+ assert parse_redshift_host(host) == expected@pytest.mark.parametrize("host", ["localhost", "my-redshift.example.com", "8.8.8.8", ""])def test_parse_redshift_host_rejects_unrecognized_endpoints(host):with pytest.raises(IntegrationError):- _parse_redshift_host(host)+ parse_redshift_host(host)def _aws_redshift_role_integration(**extra_config) -> AWSRedshiftRoleBasedIntegration:@@ -359,7 +363,7 @@ def _aws_redshift_role_integration(**extra_config) -> AWSRedshiftRoleBasedIntegrdef test_provisioned_cluster_credentials_policy_scopes_to_exact_resources():integration = _aws_redshift_role_integration(groups=["exporters"], auto_create=True)- server = ProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2")+ server = RedshiftProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2")statements = _get_redshift_credentials_policy_statements(integration, server, "posthog")@@ -378,7 +382,7 @@ def test_provisioned_cluster_credentials_policy_scopes_to_exact_resources():def test_serverless_workgroup_credentials_policy_scopes_to_workgroup_arn():workgroup_arn = "arn:aws:redshift-serverless:us-east-1:123456789012:workgroup/abc-123"integration = _aws_redshift_role_integration(workgroup_arn=workgroup_arn)- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")statements = _get_redshift_credentials_policy_statements(integration, server, "posthog")@@ -395,7 +399,7 @@ def test_serverless_workgroup_credentials_policy_requires_workgroup_arn():# There must never be a wildcard fallback here: without the exact workgroup ARN# the policy cannot be scoped, so credentials must not be requested at all.integration = _aws_redshift_role_integration()- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")with pytest.raises(IntegrationError):_get_redshift_credentials_policy_statements(integration, server, "posthog")@@ -412,7 +416,7 @@ def test_serverless_workgroup_credentials_policy_requires_workgroup_arn():)def test_serverless_workgroup_credentials_policy_rejects_mismatched_arn(workgroup_arn):integration = _aws_redshift_role_integration(workgroup_arn=workgroup_arn)- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")with pytest.raises(IntegrationError):_get_redshift_credentials_policy_statements(integration, server, "posthog")diff --git a/products/data_warehouse/frontend/shared/components/DestinationModal.tsx b/products/data_warehouse/frontend/shared/components/DestinationModal.tsxindex 9b930511833a..9ef7e528ca5c 100644--- a/products/data_warehouse/frontend/shared/components/DestinationModal.tsx+++ b/products/data_warehouse/frontend/shared/components/DestinationModal.tsx@@ -19,8 +19,15 @@ import { DestinationIcon, destinationTypeLabel } from './DestinationIcon'export function DestinationModal(props: DestinationModalLogicProps): JSX.Element {const logic = destinationModalLogic(props)- const { isOpen, editing, editingIntegration, isDestinationFormSubmitting, destinationForm, definition } =- useValues(logic)+ const {+ isOpen,+ editing,+ editingIntegration,+ selectedIntegration,+ isDestinationFormSubmitting,+ destinationForm,+ definition,+ } = useValues(logic)const { closeModal, submitDestinationForm, deleteDestination, setDestinationType, setIntegrationKind } =useActions(logic)@@ -136,7 +143,7 @@ export function DestinationModal(props: DestinationModalLogicProps): JSX.Element}</LemonField>- <Fields isNew={!editing} formValues={destinationForm} selectedIntegration={editingIntegration} />+ <Fields isNew={!editing} formValues={destinationForm} selectedIntegration={selectedIntegration} />{editing ? (<LemonBanner type="info">diff --git a/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts b/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.tsindex 73d96f022f5f..9a6af4d5b3c7 100644--- a/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts+++ b/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts@@ -1,3 +1,5 @@+import type { IntegrationType } from '~/types'+import {CREATABLE_DESTINATION_TYPES,WAREHOUSE_DESTINATIONS,@@ -102,6 +104,27 @@ describe('warehouse destination definitions', () => {expect(validateDestinationFields(definition, { isNew: true, formValues })).toEqual({})})+ it.each([+ ['a username and password connection', { host: 'my-cluster.example.com', user: 'posthog' }, false],+ [+ 'an IAM role connection',+ { aws_role_arn: 'arn:aws:iam::123456789012:role/posthog', user: 'posthog' },+ true,+ ],+ ['an access key connection', { name: 'keys', user: 'posthog' }, true],+ ['no connection picked yet', null, false],+ ])('Redshift asks for a host only when it needs one, with %s', (_label, config, needsHost) => {+ const selectedIntegration = config ? ({ id: 1, kind: 'aws-redshift', config } as IntegrationType) : null++ const errors = validateDestinationFields(WAREHOUSE_DESTINATIONS.Redshift, {+ isNew: true,+ formValues: { database: 'dev', schema: 'public' },+ selectedIntegration,+ })++ expect(errors.host).toEqual(needsHost ? 'This field is required' : undefined)+ })+it('treats a whitespace-only value as missing', () => {const errors = validateDestinationFields(WAREHOUSE_DESTINATIONS.BigQuery, {isNew: true,diff --git a/products/data_warehouse/frontend/shared/destinations/redshift.tsx b/products/data_warehouse/frontend/shared/destinations/redshift.tsxindex d50b99a574d4..ff616bd693c3 100644--- a/products/data_warehouse/frontend/shared/destinations/redshift.tsx+++ b/products/data_warehouse/frontend/shared/destinations/redshift.tsx@@ -2,28 +2,60 @@ import { LemonInput } from '@posthog/lemon-ui'import { LemonField } from 'lib/lemon-ui/LemonField'+import type { IntegrationType } from '~/types'+import type { WarehouseDestinationDefinition } from './types'+/**+ * A username and password connection stores the cluster's host. An IAM role or access key+ * connection stores only the AWS identity, so the destination has to name the cluster.+ */+function needsClusterEndpoint(integration: IntegrationType | null | undefined): boolean {+ return !!integration && !integration.config?.host+}+// Redshift speaks the Postgres wire protocol and its writer subclasses the Postgres one, so it-// takes the same two config fields. Kept as its own definition rather than sharing Postgres's,+// takes the same database and schema. Kept as its own definition rather than sharing Postgres's,// because the two pick credentials from different integration kinds.export const redshiftDefinition: WarehouseDestinationDefinition = {type: 'Redshift',integrationKinds: ['aws-redshift'],defaults: () => ({ database: 'dev', schema: 'public' }),- requiredFields: () => ['database', 'schema'],- configKeys: ['database', 'schema'],- retargetingKeys: ['database', 'schema'],- Fields: function RedshiftFields({ isNew }) {+ requiredFields: ({ selectedIntegration }) =>+ needsClusterEndpoint(selectedIntegration) ? ['host', 'database', 'schema'] : ['database', 'schema'],+ configKeys: ['host', 'port', 'database', 'schema'],+ retargetingKeys: ['host', 'database', 'schema'],+ Fields: function RedshiftFields({ isNew, selectedIntegration }) {return (- <div className="flex gap-2">- <LemonField name="database" label="Database" className="flex-1">- <LemonInput placeholder="dev" disabled={!isNew} data-attr="warehouse-destination-database" />- </LemonField>- <LemonField name="schema" label="Schema" className="flex-1">- <LemonInput placeholder="public" disabled={!isNew} data-attr="warehouse-destination-schema" />- </LemonField>- </div>+ <>+ {needsClusterEndpoint(selectedIntegration) ? (+ <div className="flex flex-wrap gap-2">+ <LemonField+ name="host"+ label="Host"+ className="flex-1 min-w-60"+ info="The cluster or workgroup endpoint. This connection signs in with AWS, so it has no host of its own."+ >+ <LemonInput+ placeholder="my-cluster.abc123.us-east-1.redshift.amazonaws.com"+ disabled={!isNew}+ data-attr="warehouse-destination-host"+ />+ </LemonField>+ <LemonField name="port" label="Port" showOptional className="w-28">+ <LemonInput type="number" placeholder="5439" data-attr="warehouse-destination-port" />+ </LemonField>+ </div>+ ) : null}+ <div className="flex flex-wrap gap-2">+ <LemonField name="database" label="Database" className="flex-1 min-w-40">+ <LemonInput placeholder="dev" disabled={!isNew} data-attr="warehouse-destination-database" />+ </LemonField>+ <LemonField name="schema" label="Schema" className="flex-1 min-w-40">+ <LemonInput placeholder="public" disabled={!isNew} data-attr="warehouse-destination-schema" />+ </LemonField>+ </div>+ </>)},}diff --git a/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts b/products/data_warehouse/frontend/shared/logics/destinationModalLogic.tsindex 77805b7206db..f0910d18fdd4 100644--- a/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts+++ b/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts@@ -47,6 +47,10 @@ export interface DestinationFormValues extends Record<string, any> {const DEFAULT_TYPE: CreatableDestinationType = 'Postgres'+function findIntegration(integrations: IntegrationType[] | null, id: number | null): IntegrationType | null {+ return (id && integrations?.find((one) => one.id === id)) || null+}+export function newDestinationForm(type: CreatableDestinationType = DEFAULT_TYPE): DestinationFormValues {const definition = WAREHOUSE_DESTINATIONS[type]return {@@ -78,6 +82,7 @@ export interface destinationModalLogicValues {isDestinationFormSubmitting: booleanisDestinationFormValid: booleanisOpen: boolean+ selectedIntegration: IntegrationType | nullshowDestinationFormErrors: boolean}@@ -192,6 +197,7 @@ export interface destinationModalLogicMeta {definition: (destinationForm: any) => WarehouseDestinationDefinitionavailableIntegrations: (integrations: any, destinationForm: any) => IntegrationType[]editingIntegration: (editing: any, integrations: any) => IntegrationType | null+ selectedIntegration: (integrations: any, destinationForm: any) => IntegrationType | null}}@@ -247,6 +253,7 @@ export const destinationModalLogic = kea<destinationModalLogicType>([{isNew: !values.editing,formValues,+ selectedIntegration: findIntegration(values.integrations, formValues.integrationId),}),}),@@ -294,8 +301,13 @@ export const destinationModalLogic = kea<destinationModalLogicType>([(editing: ExternalDataDestinationApi | null,integrations: IntegrationType[] | null- ): IntegrationType | null =>- (editing?.integration && integrations?.find((one) => one.id === editing.integration)) || null,+ ): IntegrationType | null => findIntegration(integrations, editing?.integration ?? null),+ ],+ // The connection picked in the form, so a type's fields can depend on how it signs in.+ selectedIntegration: [+ (s: any) => [s.integrations, s.destinationForm],+ (integrations: IntegrationType[] | null, destinationForm: DestinationFormValues): IntegrationType | null =>+ findIntegration(integrations, destinationForm.integrationId),],}),listeners(({ actions, values, props }: any) => ({diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.pyindex 26fc074c4a5f..ebe7254027f7 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py@@ -17,6 +17,14 @@ from posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api.utils import actionfrom posthog.integrations import IntegrationKind, IntegrationRecordfrom posthog.integrations.fields import IntegrationReferenceField+from posthog.models.integration import (+ AWSRedshiftRoleBasedIntegration,+ IntegrationError,+ RedshiftIntegration,+ RedshiftServerlessWorkgroup,+ parse_redshift_host,+ redshift_integration_from_record,+)from posthog.permissions import is_service_authfrom products.access_control.backend.facade.user_access_control import access_level_satisfied_for_resource@@ -390,6 +398,12 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):"latest_error_at",]+ # A JSONField takes any JSON value, and every check and `update()` read config as an object.+ def validate_config(self, value: Any) -> dict[str, Any]:+ if not isinstance(value, dict):+ raise ValidationError('Config must be an object of settings, such as {"database": "analytics"}.')+ return value+def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:destination_type = attrs.get("type", getattr(self.instance, "type", None))@@ -419,8 +433,13 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):if self.instance is not None:self._reject_retargeting(attrs)+ if destination_type == ExternalDataDestination.Type.REDSHIFT and "config" in attrs:+ # `update()` merges the PATCH onto the saved config, so check the port the next sync uses.+ self._check_redshift_port({**(self.instance.config or {}), **attrs["config"]})elif destination_type == ExternalDataDestination.Type.POSTGRES:self._check_postgres_connection(integration, attrs.get("config"))+ elif destination_type == ExternalDataDestination.Type.REDSHIFT:+ self._check_redshift_target(integration, attrs.get("config"))else:self._run_connection_test(destination_type, integration, attrs.get("config"))@@ -435,6 +454,62 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):field = "config" if error.failure in CONFIG_FAILURES else "integration"raise ValidationError({field: str(error)})+ # A username and password connection stores its own host. An IAM role or access key+ # connection stores only the AWS identity, so the destination has to name the cluster.+ def _check_redshift_target(self, integration: IntegrationRecord, config: dict[str, Any] | None) -> None:+ try:+ redshift = redshift_integration_from_record(integration)+ except IntegrationError as error:+ raise ValidationError({"integration": str(error)})+ if isinstance(redshift, RedshiftIntegration):+ self._require_complete_password_connection(integration)+ return++ config = config or {}+ host = config.get("host")+ if not isinstance(host, str) or not host:+ raise ValidationError(+ {"config": "Add the cluster endpoint as host. This connection uses AWS credentials, so it has no host."}+ )+ try:+ server = parse_redshift_host(host)+ if isinstance(redshift, AWSRedshiftRoleBasedIntegration) and isinstance(+ server, RedshiftServerlessWorkgroup+ ):+ redshift.workgroup_arn_for(server)+ except IntegrationError as error:+ raise ValidationError({"config": str(error)})++ self._check_redshift_port(config)++ def _check_redshift_port(self, config: dict[str, Any]) -> None:+ port = config.get("port")+ if port is not None and (isinstance(port, bool) or not isinstance(port, int) or not 0 < port <= 65535):+ raise ValidationError({"config": "Port must be a number from 1 to 65535."})++ # A connection with no AWS identity signs in with a password, and the writer reads all of these+ # fields on the first sync. Without this, a broken connection saves and only fails then.+ REDSHIFT_PASSWORD_FIELDS: tuple[tuple[str, str, bool], ...] = (+ ("host", "host", False),+ ("port", "port", False),+ ("user", "username", False),+ ("password", "password", True),+ )++ def _require_complete_password_connection(self, integration: IntegrationRecord) -> None:+ config = integration.config or {}+ sensitive_config = integration.sensitive_config or {}+ missing = [+ label+ for key, label, sensitive in self.REDSHIFT_PASSWORD_FIELDS+ if not str((sensitive_config if sensitive else config).get(key) or "").strip()+ ]+ if missing:+ listed = missing[0] if len(missing) == 1 else f"{', '.join(missing[:-1])} or {missing[-1]}"+ raise ValidationError(+ {"integration": f"This Redshift connection has no {listed}. Reconnect Redshift, then try again."}+ )+def _run_connection_test(self, destination_type: str, integration: IntegrationRecord, config: Any) -> None:run = run_connection_test(destination_type, integration, config)if run is not None and run.failure is not None:@@ -453,7 +528,7 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):# spelling would let the other through.RETARGETING_FIELDS_BY_TYPE: dict[str, tuple[str, ...]] = {str(ExternalDataDestination.Type.POSTGRES): ("database", "schema"),- str(ExternalDataDestination.Type.REDSHIFT): ("database", "schema"),+ str(ExternalDataDestination.Type.REDSHIFT): ("host", "database", "schema"),str(ExternalDataDestination.Type.SNOWFLAKE): ("database", "schema"),str(ExternalDataDestination.Type.DATABRICKS): ("catalog", "schema"),str(ExternalDataDestination.Type.BIGQUERY): ("dataset", "dataset_id", "project", "project_id"),@@ -465,6 +540,7 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):# often has a stray space at one end, and the store then rejects the name on every run. A# prefix is not here: a prefix with a space is a valid key, and rows may already sit under it.TRIMMED_FIELDS_BY_TYPE: dict[str, tuple[str, ...]] = {+ str(ExternalDataDestination.Type.REDSHIFT): ("host",),str(ExternalDataDestination.Type.S3): ("bucket", "bucket_name", "region"),str(ExternalDataDestination.Type.AZURE_BLOB): ("container_name", "container"),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.pyindex 28e675aa8983..5f38c20b5d4d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py@@ -1,14 +1,22 @@from collections.abc import AsyncIteratorfrom contextlib import asynccontextmanager+from typing import Anyimport pytestfrom unittest import mockimport psycopgimport pyarrow as pa+import botocore.exceptions++from posthog.models.integration import IntegrationErrorfrom products.batch_exports.backend.facade.destinations.postgres import PostgreSQLClientfrom products.batch_exports.backend.facade.destinations.redshift import StringLimitExceededError+from products.batch_exports.backend.facade.destinations.s3 import (+ ExternalRoleArnNotConfiguredError,+ InvalidCredentialsError,+)from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (DestinationBatchContext,DestinationRunContext,@@ -24,6 +32,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline)+def _aws_error(code: str, operation: str = "GetClusterCredentials", status: int = 400) -> Exception:+ response: Any = {"Error": {"Code": code, "Message": "fake-secret"}, "ResponseMetadata": {"HTTPStatusCode": status}}+ return botocore.exceptions.ClientError(response, operation)++def _ctx() -> DestinationRunContext:return DestinationRunContext(team_id=1,@@ -58,13 +71,22 @@ class LocalRedshiftWriter(RedshiftDestinationWriter):class TestConfigurationErrors:@pytest.mark.parametrize("finalize", [False, True], ids=["write_batch", "finalize_run"])@pytest.mark.parametrize(- "error",+ "error,expected",[- psycopg.errors.InsufficientPrivilege("permission denied for schema fake-secret"),- psycopg.errors.InsufficientPrivilege("permission denied for table fake-secret"),+ (psycopg.errors.InsufficientPrivilege("permission denied for schema fake-secret"), "USAGE and CREATE"),+ (psycopg.errors.InsufficientPrivilege("permission denied for table fake-secret"), "USAGE and CREATE"),+ (_aws_error("AccessDenied", "AssumeRole"), "cannot assume the role"),+ (InvalidCredentialsError("fake-secret"), "cannot assume the role"),+ (_aws_error("AccessDenied"), "redshift:GetClusterCredentials"),+ (_aws_error("AccessDeniedException", "GetCredentials"), "redshift-serverless:GetCredentials"),+ (_aws_error("InvalidClientTokenId"), "rejected the access key"),+ (_aws_error("ClusterNotFound"), "cannot find the Redshift cluster"),+ (_aws_error("ResourceNotFoundException", "GetCredentials"), "cannot find the Redshift cluster"),],)- async def test_permanent_errors_are_configuration_errors(self, error: Exception, finalize: bool) -> None:+ async def test_permanent_errors_are_configuration_errors(+ self, error: Exception, expected: str, finalize: bool+ ) -> None:ctx = _ctx()client = mock.AsyncMock(spec=PostgreSQLClient)writer = LocalRedshiftWriter(ctx, client)@@ -73,7 +95,7 @@ class TestConfigurationErrors:with (mock.patch.object(writer, "_table_exists", return_value=False),mock.patch.object(writer, "_ensure_table", side_effect=error),- pytest.raises(DestinationConfigurationError, match="USAGE and CREATE") as caught,+ pytest.raises(DestinationConfigurationError, match=expected) as caught,):if finalize:await writer.finalize_run(ctx)@@ -100,6 +122,11 @@ class TestConfigurationErrors:StringLimitExceededError("fake-secret", "charges", "public"),UnrelatedTableExistsError("fake-secret"),TimeoutError("fake-secret"),+ _aws_error("Throttling", "AssumeRole"),+ _aws_error("AccessDenied", status=500),+ _aws_error("InternalFailure", status=503),+ botocore.exceptions.EndpointConnectionError(endpoint_url="https://redshift.us-east-1.amazonaws.com"),+ ExternalRoleArnNotConfiguredError(),],)async def test_transient_errors_propagate_unchanged(self, error: Exception, finalize: bool) -> None:@@ -122,3 +149,17 @@ class TestConfigurationErrors:assert caught.value is errorassert not isinstance(caught.value, DestinationConfigurationError)++ async def test_an_unusable_host_reports_what_to_fix(self) -> None:+ ctx = _ctx()+ writer = LocalRedshiftWriter(ctx, mock.AsyncMock(spec=PostgreSQLClient))+ error = IntegrationError("Redshift host 'db.example.com' is not a recognized AWS Redshift endpoint")++ with (+ mock.patch.object(writer, "_table_exists", return_value=False),+ mock.patch.object(writer, "_ensure_table", side_effect=error),+ pytest.raises(DestinationConfigurationError) as caught,+ ):+ await writer.write_batch(_batches(), DestinationBatchContext(run=ctx, batch_index=0, is_final_batch=False))++ assert caught.value.detail == str(error)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.pynew file mode 100644index 000000000000..8287811bc9f5--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.py@@ -0,0 +1,142 @@+from typing import Any++import pytest+from unittest import mock++from posthog.integrations import IntegrationRecord+from posthog.models.integration import Integration++from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (+ DestinationBatchContext,+ DestinationRunContext,+)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (+ DestinationConfigurationError,+)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.writers.redshift import (+ RedshiftDestinationWriter,+)++CLUSTER_HOST = "my-cluster.abc123.us-east-1.redshift.amazonaws.com"+WORKGROUP_HOST = "my-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com"+++def _ctx(**config: Any) -> DestinationRunContext:+ return DestinationRunContext(+ team_id=1,+ schema_id="schema",+ source_id="source",+ job_id="job",+ run_uuid="run-a1",+ destination_id="destination",+ destination_type="Redshift",+ destination_name="test redshift",+ table_name="charges",+ sync_type="incremental",+ config={"database": "analytics", "schema": "public", **config},+ integration_id=7,+ )+++def _password_integration() -> Integration:+ return Integration(+ kind=Integration.IntegrationKind.AWS_REDSHIFT,+ config={"host": "stored.example.com", "port": 5555, "user": "stored_user", "ssl_mode": "require"},+ sensitive_config={"password": "stored-password"},+ )+++def _access_key_integration() -> Integration:+ return Integration(+ kind=Integration.IntegrationKind.AWS_REDSHIFT,+ config={"name": "keys", "user": "posthog", "groups": ["loaders"]},+ sensitive_config={"aws_access_key_id": "AKIAEXAMPLE", "aws_secret_access_key": "fake-secret"},+ )+++class LocalRedshiftWriter(RedshiftDestinationWriter):+ def __init__(self, ctx: DestinationRunContext, integration: Integration) -> None:+ super().__init__(ctx)+ self._integration = integration++ async def _load_integration(self, integration_id: int) -> IntegrationRecord:+ return self._integration+++def _aws_session(aws: mock.MagicMock) -> mock.MagicMock:+ client = mock.MagicMock()+ client.__aenter__.return_value = aws+ client.__aexit__.return_value = None+ session = mock.MagicMock()+ session.client.return_value = client+ return session+++class TestRedshiftConnection:+ async def test_a_password_connection_uses_its_own_host_and_ignores_the_config(self) -> None:+ writer = LocalRedshiftWriter(_ctx(host=CLUSTER_HOST, port=5439), _password_integration())++ with mock.patch("aioboto3.Session") as session:+ client = await writer._make_client()++ session.assert_not_called()+ assert (client.host, client.port, client.user, client.password, client.database) == (+ "stored.example.com",+ 5555,+ "stored_user",+ "stored-password",+ "analytics",+ )++ @pytest.mark.parametrize(+ "config,port,service",+ [+ ({"host": CLUSTER_HOST}, 5439, "redshift"),+ ({"host": CLUSTER_HOST, "port": 5440}, 5440, "redshift"),+ ({"host": WORKGROUP_HOST}, 5439, "redshift-serverless"),+ ],+ ids=["cluster_default_port", "cluster_custom_port", "serverless_workgroup"],+ )+ async def test_an_access_key_connection_signs_in_with_temporary_credentials_on_the_config_host(+ self, config: dict[str, Any], port: int, service: str+ ) -> None:+ writer = LocalRedshiftWriter(_ctx(**config), _access_key_integration())+ aws = mock.MagicMock()+ aws.get_cluster_credentials = mock.AsyncMock(+ return_value={"DbUser": "IAM:posthog", "DbPassword": "temporary-password"}+ )+ aws.get_credentials = mock.AsyncMock(return_value={"dbUser": "IAM:posthog", "dbPassword": "temporary-password"})++ with mock.patch("aioboto3.Session", return_value=_aws_session(aws)) as session:+ client = await writer._make_client()++ assert session.call_args.kwargs["aws_access_key_id"] == "AKIAEXAMPLE"+ assert session.return_value.client.call_args.args == (service,)+ assert (client.host, client.port, client.user, client.password, client.database) == (+ config["host"],+ port,+ "IAM:posthog",+ "temporary-password",+ "analytics",+ )+ if service == "redshift":+ assert aws.get_cluster_credentials.call_args.kwargs == {+ "DbUser": "posthog",+ "DbName": "analytics",+ "ClusterIdentifier": "my-cluster",+ "DurationSeconds": 3600,+ "DbGroups": ["loaders"],+ }++ async def test_an_aws_connection_without_a_host_is_a_configuration_error(self) -> None:+ ctx = _ctx()+ writer = LocalRedshiftWriter(ctx, _access_key_integration())++ with mock.patch("aioboto3.Session") as session, pytest.raises(DestinationConfigurationError) as caught:+ await writer.write_batch(+ mock.MagicMock(), DestinationBatchContext(run=ctx, batch_index=0, is_final_batch=False)+ )++ session.assert_not_called()+ assert "host" in caught.value.detaildiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.pyindex 70b6213812b8..1983ad40ae28 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py@@ -260,7 +260,7 @@ class PostgresDestinationWriter:# --- connection -------------------------------------------------------------------- def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:+ async def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:# `from_inputs` accepts anything exposing credentials/authority/tls, which the# integration wrapper does, so the batch export path and this one resolve credentials# identically.@@ -284,7 +284,7 @@ class PostgresDestinationWriter:integration = await self._load_integration(self._ctx.integration_id)except PostgreSQLIntegrationNotFoundError as err:raise DestinationConfigurationError(self._ctx.destination_name, MISSING_INTEGRATION_DETAIL) from err- return self._client_from_integration(integration)+ return await self._client_from_integration(integration)@asynccontextmanagerasync def _client(self) -> AsyncIterator[PostgreSQLClient]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.pyindex 908dea22ad11..4d144911d8aa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py@@ -2,8 +2,13 @@Redshift speaks the Postgres wire protocol, so this builds on the Postgres writer and on batchexports' `RedshiftClient`, which is itself a `PostgreSQLClient`. A Redshift destination is-backed by an `aws-redshift` integration, which is a `PostgreSQLServerIntegration` and so-carries the cluster's host, user and password.+backed by an `aws-redshift` integration in one of three shapes:++- Username and password. The integration carries the cluster's host, user and password.+- An IAM role, or an AWS access key pair. The integration carries only the AWS identity and the+ database user, so the destination's `config` carries the cluster's `host` and `port`. Each+ connection asks AWS for temporary database credentials, through the same+ `resolve_redshift_client_inputs` batch exports connects with.Three things genuinely differ and are overridden here:@@ -27,14 +32,16 @@ from contextlib import contextmanagerfrom typing import Any, ClassVar, castimport pyarrow as pa+import botocore.exceptionsfrom psycopg import sqlfrom psycopg.errors import InsufficientPrivilegefrom posthog.integrations import IntegrationRecord-from posthog.models.integration import RedshiftIntegration+from posthog.models.integration import IntegrationError, RedshiftIntegration, redshift_integration_from_recordfrom products.batch_exports.backend.facade.destinations.postgres import Fields, PostgreSQLClient-from products.batch_exports.backend.facade.destinations.redshift import RedshiftClient+from products.batch_exports.backend.facade.destinations.redshift import RedshiftClient, resolve_redshift_client_inputs+from products.batch_exports.backend.facade.destinations.s3 import InvalidCredentialsErrorfrom products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (BatchWriteOutcome,DestinationBatchContext,@@ -55,15 +62,49 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline# reports the same overflow as `StringLimitExceededError` and never retries it.VARCHAR_MAX = "VARCHAR(MAX)"+DEFAULT_PORT = 5439+PERMISSIONS_DETAIL = ("The Redshift user lacks permissions. Grant the user USAGE and CREATE on the schema. ""Grant rights on the tables it syncs, then run the sync again.")+ROLE_DETAIL = (+ "PostHog cannot assume the role. Check the role ARN and the trust policy with the external ID, "+ "then run the sync again."+)+AWS_CREDENTIALS_DETAIL = "AWS rejected the access key. Check the access key ID and secret, then run the sync again."+AWS_PERMISSIONS_DETAIL = (+ "AWS denied temporary database credentials. Allow redshift:GetClusterCredentials for the database user, "+ "or redshift-serverless:GetCredentials for the workgroup, then run the sync again."+)+CLUSTER_NOT_FOUND_DETAIL = (+ "AWS cannot find the Redshift cluster or workgroup. Check the host on the destination, then run the sync again."+)def _configuration_error_detail(err: Exception) -> str | None:if isinstance(err, InsufficientPrivilege):return PERMISSIONS_DETAIL+ if isinstance(err, IntegrationError):+ # Written by us for a host or workgroup ARN that cannot work, and holds no secret.+ return str(err)+ if isinstance(err, InvalidCredentialsError):+ return ROLE_DETAIL+ if isinstance(err, botocore.exceptions.ClientError):+ return _aws_error_detail(err)+ return None+++def _aws_error_detail(err: botocore.exceptions.ClientError) -> str | None:+ if err.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) >= 500:+ return None+ code = err.response.get("Error", {}).get("Code")+ if code in ("InvalidClientTokenId", "SignatureDoesNotMatch", "UnrecognizedClientException"):+ return AWS_CREDENTIALS_DETAIL+ if code in ("AccessDenied", "AccessDeniedException"):+ return ROLE_DETAIL if err.operation_name == "AssumeRole" else AWS_PERMISSIONS_DETAIL+ if code in ("ClusterNotFound", "ClusterNotFoundFault", "ResourceNotFoundException"):+ return CLUSTER_NOT_FOUND_DETAILreturn None@@ -136,8 +177,18 @@ class RedshiftDestinationWriter(PostgresDestinationWriter):with self._configuration_errors():await super().abort_run(ctx)- def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:- return RedshiftClient.from_inputs(RedshiftIntegration(integration), database=self._database)+ async def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:+ config = self._ctx.config or {}+ inputs = await resolve_redshift_client_inputs(+ redshift_integration_from_record(integration),+ host=config.get("host") or None,+ port=int(config.get("port") or DEFAULT_PORT),+ database=self._database,+ team_id=self._ctx.team_id,+ # The same session name the S3 writer uses, so one role's CloudTrail shows one sync.+ session_name=f"PostHog-warehouse-sync-{self._ctx.schema_id}",+ )+ return RedshiftClient.from_inputs(inputs, database=self._database)def _column_type(self, arrow_type: pa.DataType) -> str:return redshift_type_for(arrow_type)diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py b/products/warehouse_sources/backend/tests/api/test_external_data_destination.pyindex d570d28e984a..e733175071ad 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py@@ -62,6 +62,11 @@ from products.warehouse_sources.backend.presentation.views.external_data_destina)from products.warehouse_sources.backend.types import ExternalDataSourceType+REDSHIFT_ROLE_ARN = "arn:aws:iam::123456789012:role/posthog-redshift"+REDSHIFT_CLUSTER_HOST = "my-cluster.abc123.us-east-1.redshift.amazonaws.com"+REDSHIFT_WORKGROUP_HOST = "my-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com"+REDSHIFT_WORKGROUP_ARN = "arn:aws:redshift-serverless:us-east-1:123456789012:workgroup/1a2b3c4d"+def _destination_test(step_count: int) -> DestinationTest:return DestinationTest(@@ -117,11 +122,32 @@ class DestinationAPITestBase(APIBaseTest):self.schema = ExternalDataSchema.objects.create(team=self.team, source=self.source, name="charges")def _integration(self, kind: str = Integration.IntegrationKind.POSTGRESQL) -> Integration:+ if kind == Integration.IntegrationKind.AWS_REDSHIFT:+ return self._redshift_integration("password")self._integration_seq = getattr(self, "_integration_seq", 0) + 1return Integration.objects.create(team=self.team, kind=kind, integration_id=f"{kind}-{self._integration_seq}", config={})+ def _redshift_integration(self, mode: str, **config) -> Integration:+ self._integration_seq = getattr(self, "_integration_seq", 0) + 1+ sensitive_config: dict[str, str] = {}+ if mode == "role":+ config = {"aws_role_arn": REDSHIFT_ROLE_ARN, "user": "posthog", **config}+ elif mode == "access_key":+ config = {"name": "keys", "user": "posthog", **config}+ sensitive_config = {"aws_access_key_id": "AKIAEXAMPLE", "aws_secret_access_key": "fake-secret"}+ else:+ config = {"host": "my-cluster.example.com", "port": 5439, "user": "posthog", **config}+ sensitive_config = {"password": "fake-secret"}+ return Integration.objects.create(+ team=self.team,+ kind=Integration.IntegrationKind.AWS_REDSHIFT,+ integration_id=f"redshift-{self._integration_seq}",+ config=config,+ sensitive_config=sensitive_config,+ )+def _create_destination(self, **overrides) -> ExternalDataDestination:payload = {"type": ExternalDataDestination.Type.POSTGRES,@@ -393,6 +419,180 @@ class TestExternalDataDestinationAPI(DestinationAPITestBase):self.describe_test.assert_not_called()self.run_test_step.assert_not_called()+ @parameterized.expand(+ [+ ("password_needs_no_host", "password", {}, {}, None),+ ("role_with_cluster_host", "role", {}, {"host": REDSHIFT_CLUSTER_HOST}, None),+ ("role_with_cluster_host_and_port", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 5440}, None),+ ("access_key_with_cluster_host", "access_key", {}, {"host": REDSHIFT_CLUSTER_HOST}, None),+ ("host_with_stray_space", "access_key", {}, {"host": f" {REDSHIFT_CLUSTER_HOST} "}, None),+ (+ "role_with_workgroup_host_and_arn",+ "role",+ {"workgroup_arn": REDSHIFT_WORKGROUP_ARN},+ {"host": REDSHIFT_WORKGROUP_HOST},+ None,+ ),+ ("access_key_with_workgroup_host", "access_key", {}, {"host": REDSHIFT_WORKGROUP_HOST}, None),+ ("role_without_host", "role", {}, {}, "Add the cluster endpoint"),+ ("access_key_without_host", "access_key", {}, {}, "Add the cluster endpoint"),+ ("role_with_blank_host", "role", {}, {"host": " "}, "Add the cluster endpoint"),+ ("host_not_a_redshift_endpoint", "role", {}, {"host": "db.example.com"}, "not a recognized"),+ ("role_with_workgroup_host_no_arn", "role", {}, {"host": REDSHIFT_WORKGROUP_HOST}, "workgroup_arn"),+ (+ "role_with_workgroup_arn_in_another_account",+ "role",+ {"workgroup_arn": REDSHIFT_WORKGROUP_ARN.replace("123456789012", "210987654321")},+ {"host": REDSHIFT_WORKGROUP_HOST},+ "does not match",+ ),+ ("port_zero", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 0}, "Port must be"),+ ("port_too_large", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 70000}, "Port must be"),+ ("port_as_text", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": "5439"}, "Port must be"),+ ]+ )+ def test_a_redshift_destination_names_the_cluster_when_its_connection_has_none(+ self, _name: str, mode: str, integration_config: dict, target: dict, error: str | None+ ) -> None:+ response = self.client.post(+ self.base,+ {+ "type": ExternalDataDestination.Type.REDSHIFT,+ "name": "analytics redshift",+ "integration": self._redshift_integration(mode, **integration_config).pk,+ "config": {"database": "dev", "schema": "public", **target},+ },+ format="json",+ )++ if error is None:+ assert response.status_code == status.HTTP_201_CREATED, response.json()+ if "host" in target:+ assert response.json()["config"]["host"] == target["host"].strip()+ else:+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ assert response.json()["attr"] == "config"+ assert error in response.json()["detail"]++ @parameterized.expand(+ [+ ("empty", {}, {}, "host, port, username or password"),+ ("no_password", {"host": "my-cluster.example.com", "port": 5439, "user": "posthog"}, {}, "password"),+ (+ "blank_password",+ {"host": "my-cluster.example.com", "port": 5439, "user": "posthog"},+ {"password": ""},+ "password",+ ),+ ("no_host", {"port": 5439, "user": "posthog"}, {"password": "fake-secret"}, "host"),+ ("blank_host", {"host": " ", "port": 5439, "user": "posthog"}, {"password": "fake-secret"}, "host"),+ ("no_port", {"host": "my-cluster.example.com", "user": "posthog"}, {"password": "fake-secret"}, "port"),+ (+ "no_user",+ {"host": "my-cluster.example.com", "port": 5439},+ {"password": "fake-secret"},+ "username",+ ),+ ]+ )+ def test_a_redshift_destination_rejects_an_incomplete_password_connection(+ self, _name: str, config: dict, sensitive_config: dict, missing: str+ ) -> None:+ integration = Integration.objects.create(+ team=self.team,+ kind=Integration.IntegrationKind.AWS_REDSHIFT,+ integration_id=f"redshift-incomplete-{_name}",+ config=config,+ sensitive_config=sensitive_config,+ )++ response = self.client.post(+ self.base,+ {+ "type": ExternalDataDestination.Type.REDSHIFT,+ "name": "broken redshift",+ "integration": integration.pk,+ "config": {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST},+ },+ format="json",+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ assert response.json()["attr"] == "integration"+ assert f"This Redshift connection has no {missing}." in response.json()["detail"]+ assert "fake-secret" not in response.content.decode()+ assert not ExternalDataDestination.objects.for_team(self.team.pk).filter(name="broken redshift").exists()++ @parameterized.expand(+ [+ ("another_host", {"host": "other.abc123.us-east-1.redshift.amazonaws.com"}, "keeps the host"),+ ("port_negative", {"port": -1}, "Port must be"),+ ("port_too_large", {"port": 70000}, "Port must be"),+ ("port_as_text", {"port": "5440"}, "Port must be"),+ ("port_as_bool", {"port": True}, "Port must be"),+ ("valid_port", {"port": 5440}, None),+ ]+ )+ def test_a_redshift_destination_update_keeps_the_host_and_checks_the_port(+ self, _name: str, patch: dict, error: str | None+ ) -> None:+ created = {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST, "port": 5439}+ destination = self._create_destination(+ type=ExternalDataDestination.Type.REDSHIFT,+ integration=self._redshift_integration("access_key").pk,+ config=created,+ )++ response = self.client.patch(f"{self.base}/{destination.id}", {"config": patch}, format="json")++ destination.refresh_from_db()+ if error is None:+ assert response.status_code == status.HTTP_200_OK, response.json()+ assert destination.config == {**created, **patch}+ else:+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ assert response.json()["attr"] == "config"+ assert error in response.json()["detail"]+ assert destination.config == created++ @parameterized.expand(+ [+ ("list_on_create", "create", ["invalid"]),+ ("text_on_create", "create", "invalid"),+ ("number_on_create", "create", 5439),+ ("list_on_update", "update", ["invalid"]),+ ("text_on_update", "update", "invalid"),+ ]+ )+ def test_a_destination_config_must_be_an_object(self, _name: str, action: str, config: object) -> None:+ integration = self._redshift_integration("role")+ if action == "create":+ response = self.client.post(+ self.base,+ {+ "type": ExternalDataDestination.Type.REDSHIFT,+ "name": "malformed redshift",+ "integration": integration.pk,+ "config": config,+ },+ format="json",+ )+ else:+ destination = self._create_destination(+ type=ExternalDataDestination.Type.REDSHIFT,+ name="malformed redshift",+ integration=integration.pk,+ config={"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST},+ )+ response = self.client.patch(f"{self.base}/{destination.id}", {"config": config}, format="json")++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ assert response.json()["attr"] == "config"+ assert "Config must be an object" in response.json()["detail"]+ if action == "update":+ destination.refresh_from_db()+ assert destination.config == {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST}+def test_updating_a_destination_does_not_run_the_connection_test(self) -> None:destination = self._create_destination()self.check_connection.reset_mock()diff --git a/products/warehouse_sources/mcp/warehouse_destinations.yaml b/products/warehouse_sources/mcp/warehouse_destinations.yamlindex 29c74453804c..38efbb1b074a 100644--- a/products/warehouse_sources/mcp/warehouse_destinations.yaml+++ b/products/warehouse_sources/mcp/warehouse_destinations.yaml@@ -92,14 +92,15 @@ tools:description: >Create a place where warehouse sources write synced rows, from an integration that holds the credentials.Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the- integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake- takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks- takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression;- Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the- connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so- a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm- it with the user first. A new destination receives nothing until a source is attached with- external-data-destinations-add-sources.+ integration id and the target config for the type: Postgres takes database and schema; Redshift takes+ database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or+ access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional+ role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume;+ S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional+ prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns+ a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync+ fails. The target cannot change after creation, so confirm it with the user first. A new destination+ receives nothing until a source is attached with external-data-destinations-add-sources.feature_flag: warehouse-multi-destinationexternal-data-destinations-list:operation: external_data_destinations_listdiff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex a645a20ea141..99d8d6e589a5 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -6363,7 +6363,7 @@"feature_flag": "warehouse-multi-destination"},"external-data-destinations-create": {- "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",+ "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres takes database and schema; Redshift takes database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.","category": "Warehouse destinations","feature": "warehouse_sources","summary": "Create a warehouse destination",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex b5ad174b0eca..66c6fe898196 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -6496,7 +6496,7 @@"feature_flag": "warehouse-multi-destination"},"external-data-destinations-create": {- "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",+ "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres takes database and schema; Redshift takes database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.","category": "Warehouse destinations","feature": "warehouse_sources","summary": "Create a warehouse destination",