Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-destinations): allow redshift iam role and access keys (#128397)

Tom Owersd3c0edc78f41
diff --git a/posthog/models/integration/__init__.py b/posthog/models/integration/__init__.py
index 19c89fc54ad0..f7d2dd7a17da 100644
--- a/posthog/models/integration/__init__.py
+++ b/posthog/models/integration/__init__.py
@@ -26,6 +26,7 @@ from .auth_types import (
register_kind_auth_types,
)
from .aws import (
+ AnyRedshiftIntegration,
AWSCredentialsIntegration,
AWSRedshiftIntegration,
AWSRedshiftRoleBasedIntegration,
@@ -33,8 +34,12 @@ from .aws import (
AWSS3Integration,
AWSS3RoleBasedIntegration,
DuplicateNameError,
+ RedshiftProvisionedCluster,
+ RedshiftServerlessWorkgroup,
S3CompatibleIntegration,
is_unique_aws_role_by_organization_id,
+ parse_redshift_host,
+ redshift_integration_from_record,
validate_aws_credentials,
)
from .azure_blob import AzureBlobIntegration, AzureBlobIntegrationError
@@ -286,6 +291,11 @@ __all__ = [
"AWSCredentialsIntegration",
"AWSS3Integration",
"AWSRedshiftIntegration",
+ "AnyRedshiftIntegration",
+ "RedshiftProvisionedCluster",
+ "RedshiftServerlessWorkgroup",
+ "parse_redshift_host",
+ "redshift_integration_from_record",
"S3CompatibleIntegration",
"STRIPE_POSTHOG_SECRET_NAMES",
"StripeIntegration",
diff --git a/posthog/models/integration/aws.py b/posthog/models/integration/aws.py
index baf0a3f89d14..b4c3fcaed8f1 100644
--- a/posthog/models/integration/aws.py
+++ b/posthog/models/integration/aws.py
@@ -14,12 +14,14 @@ from rest_framework.exceptions import ValidationError
from posthog.credentials import AWSKeyPair
from posthog.credentials_service.callers import CredentialsCaller
from posthog.credentials_service.settings_fallback import get_or_settings
+from posthog.dataclasses import frozen
from posthog.integrations.contracts import IntegrationRecord
from posthog.models.user import User
from posthog.models.utils import IntegrityError
from posthog.security.url_validation import is_url_allowed
from . import common, model
+from .postgres import RedshiftIntegration
_AWSKindType = Literal[model.Integration.IntegrationKind.AWS_REDSHIFT, model.Integration.IntegrationKind.AWS_S3]
@@ -347,6 +349,42 @@ AWS_REDSHIFT_SERVERLESS_WORKGROUP_ARN_RE = re.compile(
)
+@frozen
+class RedshiftProvisionedCluster:
+ """A provisioned Redshift cluster parsed from its endpoint hostname."""
+
+ cluster_identifier: str
+ region: str
+
+
+@frozen
+class RedshiftServerlessWorkgroup:
+ """A Redshift Serverless workgroup parsed from its endpoint hostname."""
+
+ workgroup: str
+ account_id: str
+ region: str
+
+
+def parse_redshift_host(host: str) -> RedshiftProvisionedCluster | RedshiftServerlessWorkgroup:
+ """Parse a Redshift endpoint hostname into its cluster or workgroup location.
+
+ The IAM role and access key integrations need this to ask AWS for temporary database
+ credentials, because the AWS APIs take the cluster or workgroup name, not the host.
+ """
+ match host.split(".", maxsplit=3):
+ case [cluster_identifier, _, region, "redshift.amazonaws.com"]:
+ return RedshiftProvisionedCluster(cluster_identifier=cluster_identifier, region=region)
+ case [workgroup, account_id, region, "redshift-serverless.amazonaws.com"]:
+ return RedshiftServerlessWorkgroup(workgroup=workgroup, account_id=account_id, region=region)
+ case _:
+ raise common.IntegrationError(
+ f"Redshift host '{host}' is not a recognized AWS Redshift endpoint. Expected "
+ "'<cluster>.<id>.<region>.redshift.amazonaws.com' or "
+ "'<workgroup>.<account>.<region>.redshift-serverless.amazonaws.com'"
+ )
+
+
def _return_redshift_workgroup_arn(**config) -> str | None:
if (workgroup_arn := config.get("workgroup_arn")) is not None:
if not isinstance(workgroup_arn, str) or not AWS_REDSHIFT_SERVERLESS_WORKGROUP_ARN_RE.match(workgroup_arn):
@@ -397,6 +435,26 @@ class AWSRedshiftRoleBasedIntegration(AWSRoleBasedIntegration):
"""ARN of a Redshift Serverless workgroup, required when the cluster is Serverless."""
return self.integration.config.get("workgroup_arn")
+ def workgroup_arn_for(self, server: RedshiftServerlessWorkgroup) -> str:
+ """Return the workgroup ARN, checked against the workgroup the host points at.
+
+ Workgroup ARNs contain a UUID that cannot be derived from the endpoint hostname,
+ so Serverless requires the ARN on the integration to scope the assumed role exactly.
+ """
+ workgroup_arn = self.workgroup_arn
+ if workgroup_arn is None:
+ raise common.IntegrationError(
+ "A Redshift Serverless workgroup requires 'workgroup_arn' to be set on the integration"
+ )
+
+ _, _, _, arn_region, arn_account_id, _ = workgroup_arn.split(":", maxsplit=5)
+ if arn_region != server.region or arn_account_id != server.account_id:
+ raise common.IntegrationError(
+ f"The integration's workgroup ARN '{workgroup_arn}' does not match the region or account "
+ f"of the Redshift Serverless host (region '{server.region}', account '{server.account_id}')"
+ )
+ return workgroup_arn
+
@staticmethod
def read_extra_configuration_parameters(**config) -> dict[str, Any]:
return _return_extra_redshift_configuration(**config)
@@ -597,3 +655,18 @@ class S3CompatibleIntegration:
},
created_by=created_by,
)
+
+
+AnyRedshiftIntegration = AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration
+
+
+def redshift_integration_from_record(integration: IntegrationRecord) -> AnyRedshiftIntegration:
+ """Wrap an `aws-redshift` integration in the class its stored config belongs to.
+
+ The kind covers three classes, told apart only by which keys the integration stores.
+ """
+ if "aws_role_arn" in (integration.config or {}):
+ return AWSRedshiftRoleBasedIntegration(integration)
+ if "aws_access_key_id" in (integration.sensitive_config or {}):
+ return AWSRedshiftIntegration(integration)
+ return RedshiftIntegration(integration)
diff --git a/products/batch_exports/backend/facade/destinations/redshift.py b/products/batch_exports/backend/facade/destinations/redshift.py
index 63724c10e8e1..be7308a61d2f 100644
--- a/products/batch_exports/backend/facade/destinations/redshift.py
+++ b/products/batch_exports/backend/facade/destinations/redshift.py
@@ -1,7 +1,8 @@
"""
Redshift destination wiring for batch_exports.
-Re-exports the client that the warehouse_sources Redshift writer reuses.
+Re-exports the client and the connection resolver that the warehouse_sources Redshift
+writer reuses.
Importing this module loads the destination's vendor SDK, so keep it off the
``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``.
@@ -10,9 +11,11 @@ Importing this module loads the destination's vendor SDK, so keep it off the
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
RedshiftClient,
StringLimitExceededError,
+ resolve_redshift_client_inputs,
)
__all__ = [
"RedshiftClient",
"StringLimitExceededError",
+ "resolve_redshift_client_inputs",
]
diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
index 7cbfc280c38d..022104a7098c 100644
--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
@@ -25,6 +25,7 @@ from posthog.integrations import IntegrationManager, IntegrationNotFound
from posthog.models import Team
from posthog.models.integration import (
TLS,
+ AnyRedshiftIntegration,
Authority,
AWSRedshiftIntegration,
AWSRedshiftRoleBasedIntegration,
@@ -33,6 +34,10 @@ from posthog.models.integration import (
Credentials,
IntegrationError,
RedshiftIntegration,
+ RedshiftProvisionedCluster,
+ RedshiftServerlessWorkgroup,
+ parse_redshift_host,
+ redshift_integration_from_record,
)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
@@ -899,21 +904,14 @@ def _get_merge_settings(
return NotRequiredMergeSettings(requires_merge=False)
-async def _get_redshift_integration(
- integration_id: int, team_id: int
-) -> AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration:
+async def _get_redshift_integration(integration_id: int, team_id: int) -> AnyRedshiftIntegration:
"""Fetch a Redshift integration from the database."""
try:
integration = await IntegrationManager.aws_redshift.credentials.aget(team_id, integration_id)
except IntegrationNotFound:
raise IntegrationError(f"Redshift integration with ID '{integration_id}' not found for team '{team_id}'")
- if "aws_role_arn" in integration.config:
- return AWSRedshiftRoleBasedIntegration(integration)
- elif "aws_access_key_id" in integration.sensitive_config:
- return AWSRedshiftIntegration(integration)
- else:
- return RedshiftIntegration(integration)
+ return redshift_integration_from_record(integration)
async def _get_aws_s3_integration(integration_id: int, team_id: int) -> AWSS3RoleBasedIntegration | AWSS3Integration:
@@ -935,51 +933,10 @@ class RedshiftCredentials:
password: str = dataclasses.field(repr=False)
-@frozen
-class ProvisionedCluster:
- """A provisioned Redshift cluster parsed from its endpoint hostname."""
-
- cluster_identifier: str
- region: str
-
-
-@frozen
-class ServerlessWorkgroup:
- """A Redshift Serverless workgroup parsed from its endpoint hostname."""
-
- workgroup: str
- account_id: str
- region: str
-
-
-def _parse_redshift_host(host: str) -> ProvisionedCluster | ServerlessWorkgroup:
- """Parse a Redshift endpoint hostname into its cluster or workgroup location."""
- match host.split(".", maxsplit=3):
- case [cluster_identifier, _, region, "redshift.amazonaws.com"]:
- return ProvisionedCluster(cluster_identifier=cluster_identifier, region=region)
- case [workgroup, account_id, region, "redshift-serverless.amazonaws.com"]:
- return ServerlessWorkgroup(workgroup=workgroup, account_id=account_id, region=region)
- case _:
- raise IntegrationError(
- f"Redshift host '{host}' is not a recognized AWS Redshift endpoint. Expected "
- "'<cluster>.<id>.<region>.redshift.amazonaws.com' or "
- "'<workgroup>.<account>.<region>.redshift-serverless.amazonaws.com'"
- )
-
-
-def _get_host_from_connection(connection: ConnectionParameters | ServerConnectionParameters) -> str:
- if connection.host is None:
- raise IntegrationError(
- "An AWS Redshift integration requires the cluster endpoint to be set as 'host' in the "
- "batch export configuration"
- )
- return connection.host
-
-
async def _get_temporary_redshift_credentials(
aws_credentials: AWSCredentials,
*,
- server: ProvisionedCluster | ServerlessWorkgroup,
+ server: RedshiftProvisionedCluster | RedshiftServerlessWorkgroup,
user: str,
database: str,
auto_create: bool | None = None,
@@ -1017,7 +974,7 @@ async def _get_temporary_redshift_credentials(
)
match server:
- case ProvisionedCluster():
+ case RedshiftProvisionedCluster():
optional: dict[str, typing.Any] = {}
if auto_create is not None:
optional["AutoCreate"] = auto_create
@@ -1036,7 +993,7 @@ async def _get_temporary_redshift_credentials(
return RedshiftCredentials(user=response["DbUser"], password=response["DbPassword"])
- case ServerlessWorkgroup():
+ case RedshiftServerlessWorkgroup():
async with session.client("redshift-serverless", region_name=server.region) as redshift_serverless:
response = await redshift_serverless.get_credentials(
workgroupName=server.workgroup,
@@ -1074,12 +1031,12 @@ async def _get_redshift_client_inputs(
def _get_redshift_credentials_policy_statements(
integration: AWSRedshiftRoleBasedIntegration,
- server: ProvisionedCluster | ServerlessWorkgroup,
+ server: RedshiftProvisionedCluster | RedshiftServerlessWorkgroup,
database: str,
) -> list[PolicyStatement]:
"""Build policy statements scoping an assumed role to obtaining Redshift credentials."""
match server:
- case ProvisionedCluster():
+ case RedshiftProvisionedCluster():
account_id = integration.aws_account_id
get_cluster_credentials_policy = PolicyStatement(
Effect="Allow",
@@ -1106,22 +1063,8 @@ def _get_redshift_credentials_policy_statements(
)
return policy_statements
- case ServerlessWorkgroup():
- # Workgroup ARNs contain a UUID that cannot be derived from the endpoint hostname,
- # so Serverless requires the ARN on the integration to scope the policy exactly.
- workgroup_arn = integration.workgroup_arn
- if workgroup_arn is None:
- raise IntegrationError(
- "A Redshift Serverless workgroup requires 'workgroup_arn' to be set on the integration"
- )
-
- _, _, _, arn_region, arn_account_id, _ = workgroup_arn.split(":", maxsplit=5)
- if arn_region != server.region or arn_account_id != server.account_id:
- raise IntegrationError(
- f"The integration's workgroup ARN '{workgroup_arn}' does not match the region or account "
- f"of the Redshift Serverless host (region '{server.region}', account '{server.account_id}')"
- )
-
+ case RedshiftServerlessWorkgroup():
+ workgroup_arn = integration.workgroup_arn_for(server)
return [
PolicyStatement(
Effect="Allow",
@@ -1135,10 +1078,37 @@ def _get_redshift_credentials_policy_statements(
async def _get_redshift_client_inputs_from_integration(
- integration: AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration,
+ integration: AnyRedshiftIntegration,
inputs: RedshiftInsertInputs | RedshiftCopyActivityInputs,
) -> _PostgreSQLClientInputsProtocol:
- """Return PostgreSQL client inputs to establish a connection from Redshift.
+ """Return PostgreSQL client inputs for a batch export backed by an integration."""
+ host = inputs.connection.host
+ if host is None and not isinstance(integration, RedshiftIntegration):
+ raise IntegrationError(
+ "An AWS Redshift integration requires the cluster endpoint to be set as 'host' in the "
+ "batch export configuration"
+ )
+
+ return await resolve_redshift_client_inputs(
+ integration,
+ host=host,
+ port=inputs.connection.port,
+ database=inputs.connection.database,
+ team_id=inputs.batch_export.team_id,
+ session_name=f"PostHog-batch-exports-{inputs.batch_export.batch_export_id}",
+ )
+
+
+async def resolve_redshift_client_inputs(
+ integration: AnyRedshiftIntegration,
+ *,
+ host: str | None,
+ port: int,
+ database: str,
+ team_id: int,
+ session_name: str,
+) -> _PostgreSQLClientInputsProtocol:
+ """Return PostgreSQL client inputs to connect to Redshift with an integration.
These inputs can be obtained from an integration by:
* Generating temporary credentials using a short-lived session via assuming
@@ -1147,26 +1117,26 @@ async def _get_redshift_client_inputs_from_integration(
in the integration.
* Directly reading inputs stored in the integration.
- Depending on the type of integration, is which one we'll use.
+ Depending on the type of integration, is which one we'll use. `host` and `port` are
+ only read for the two AWS integrations, because a plain Redshift integration stores
+ its own.
"""
client_inputs: _PostgreSQLClientInputsProtocol
match integration:
case AWSRedshiftRoleBasedIntegration():
- host = _get_host_from_connection(inputs.connection)
- server = _parse_redshift_host(host)
+ host = _require_aws_host(host)
+ server = parse_redshift_host(host)
- team = await Team.objects.aget(id=inputs.batch_export.team_id)
+ team = await Team.objects.aget(id=team_id)
external_id = f"posthog-{team.organization_id}"
- policy_statements = _get_redshift_credentials_policy_statements(
- integration, server, inputs.connection.database
- )
+ policy_statements = _get_redshift_credentials_policy_statements(integration, server, database)
credentials = await get_credentials_using_user_aws_role(
integration.aws_role_arn,
external_id,
- session_name=f"PostHog-batch-exports-{inputs.batch_export.batch_export_id}",
+ session_name=session_name,
policy_statements=policy_statements,
)
@@ -1174,7 +1144,7 @@ async def _get_redshift_client_inputs_from_integration(
credentials,
server=server,
user=integration.user,
- database=inputs.connection.database,
+ database=database,
groups=integration.groups,
auto_create=integration.auto_create,
)
@@ -1182,13 +1152,13 @@ async def _get_redshift_client_inputs_from_integration(
user=redshift_credentials.user,
password=redshift_credentials.password,
host=host,
- port=inputs.connection.port,
- database=inputs.connection.database,
+ port=port,
+ database=database,
)
case AWSRedshiftIntegration():
- host = _get_host_from_connection(inputs.connection)
- server = _parse_redshift_host(host)
+ host = _require_aws_host(host)
+ server = parse_redshift_host(host)
redshift_credentials = await _get_temporary_redshift_credentials(
AWSCredentials(
@@ -1197,7 +1167,7 @@ async def _get_redshift_client_inputs_from_integration(
),
server=server,
user=integration.user,
- database=inputs.connection.database,
+ database=database,
groups=integration.groups,
auto_create=integration.auto_create,
)
@@ -1205,8 +1175,8 @@ async def _get_redshift_client_inputs_from_integration(
user=redshift_credentials.user,
password=redshift_credentials.password,
host=host,
- port=inputs.connection.port,
- database=inputs.connection.database,
+ port=port,
+ database=database,
)
case RedshiftIntegration():
@@ -1218,6 +1188,12 @@ async def _get_redshift_client_inputs_from_integration(
return client_inputs
+def _require_aws_host(host: str | None) -> str:
+ if host is None:
+ raise IntegrationError("An AWS Redshift integration requires the cluster endpoint as 'host'")
+ return host
+
+
@activity.defn
@handle_non_retryable_errors(NON_RETRYABLE_ERROR_TYPES)
async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs) -> BatchExportResult:
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
index 24fb129c4ea8..54e439495b55 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
@@ -13,18 +13,22 @@ import aioboto3
import pytest_asyncio
import botocore.exceptions
-from posthog.models.integration import AWSRedshiftRoleBasedIntegration, Integration, IntegrationError
+from posthog.models.integration import (
+ AWSRedshiftRoleBasedIntegration,
+ Integration,
+ IntegrationError,
+ RedshiftProvisionedCluster,
+ RedshiftServerlessWorkgroup,
+ parse_redshift_host,
+)
from products.batch_exports.backend.service import AWSCredentials, BatchExportEventsModel
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
ClientErrorGroup,
InsufficientS3PermissionsError,
- ProvisionedCluster,
RedshiftS3CopyError,
- ServerlessWorkgroup,
_get_redshift_credentials_policy_statements,
_get_table_schemas,
- _parse_redshift_host,
check_and_raise_redshift_copy_error,
is_s3_read_access_denied,
upload_manifest_file,
@@ -327,22 +331,22 @@ async def test_check_and_raise_redshift_copy_error_iam_role(error, expected_erro
[
(
"examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com",
- ProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2"),
+ RedshiftProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2"),
),
(
"default-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com",
- ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1"),
+ RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1"),
),
],
)
def test_parse_redshift_host(host, expected):
- assert _parse_redshift_host(host) == expected
+ assert parse_redshift_host(host) == expected
@pytest.mark.parametrize("host", ["localhost", "my-redshift.example.com", "8.8.8.8", ""])
def test_parse_redshift_host_rejects_unrecognized_endpoints(host):
with pytest.raises(IntegrationError):
- _parse_redshift_host(host)
+ parse_redshift_host(host)
def _aws_redshift_role_integration(**extra_config) -> AWSRedshiftRoleBasedIntegration:
@@ -359,7 +363,7 @@ def _aws_redshift_role_integration(**extra_config) -> AWSRedshiftRoleBasedIntegr
def test_provisioned_cluster_credentials_policy_scopes_to_exact_resources():
integration = _aws_redshift_role_integration(groups=["exporters"], auto_create=True)
- server = ProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2")
+ server = RedshiftProvisionedCluster(cluster_identifier="examplecluster", region="us-west-2")
statements = _get_redshift_credentials_policy_statements(integration, server, "posthog")
@@ -378,7 +382,7 @@ def test_provisioned_cluster_credentials_policy_scopes_to_exact_resources():
def test_serverless_workgroup_credentials_policy_scopes_to_workgroup_arn():
workgroup_arn = "arn:aws:redshift-serverless:us-east-1:123456789012:workgroup/abc-123"
integration = _aws_redshift_role_integration(workgroup_arn=workgroup_arn)
- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
statements = _get_redshift_credentials_policy_statements(integration, server, "posthog")
@@ -395,7 +399,7 @@ def test_serverless_workgroup_credentials_policy_requires_workgroup_arn():
# There must never be a wildcard fallback here: without the exact workgroup ARN
# the policy cannot be scoped, so credentials must not be requested at all.
integration = _aws_redshift_role_integration()
- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
with pytest.raises(IntegrationError):
_get_redshift_credentials_policy_statements(integration, server, "posthog")
@@ -412,7 +416,7 @@ def test_serverless_workgroup_credentials_policy_requires_workgroup_arn():
)
def test_serverless_workgroup_credentials_policy_rejects_mismatched_arn(workgroup_arn):
integration = _aws_redshift_role_integration(workgroup_arn=workgroup_arn)
- server = ServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
+ server = RedshiftServerlessWorkgroup(workgroup="default-wg", account_id="123456789012", region="us-east-1")
with pytest.raises(IntegrationError):
_get_redshift_credentials_policy_statements(integration, server, "posthog")
diff --git a/products/data_warehouse/frontend/shared/components/DestinationModal.tsx b/products/data_warehouse/frontend/shared/components/DestinationModal.tsx
index 9b930511833a..9ef7e528ca5c 100644
--- a/products/data_warehouse/frontend/shared/components/DestinationModal.tsx
+++ b/products/data_warehouse/frontend/shared/components/DestinationModal.tsx
@@ -19,8 +19,15 @@ import { DestinationIcon, destinationTypeLabel } from './DestinationIcon'
export function DestinationModal(props: DestinationModalLogicProps): JSX.Element {
const logic = destinationModalLogic(props)
- const { isOpen, editing, editingIntegration, isDestinationFormSubmitting, destinationForm, definition } =
- useValues(logic)
+ const {
+ isOpen,
+ editing,
+ editingIntegration,
+ selectedIntegration,
+ isDestinationFormSubmitting,
+ destinationForm,
+ definition,
+ } = useValues(logic)
const { closeModal, submitDestinationForm, deleteDestination, setDestinationType, setIntegrationKind } =
useActions(logic)
@@ -136,7 +143,7 @@ export function DestinationModal(props: DestinationModalLogicProps): JSX.Element
}
</LemonField>
- <Fields isNew={!editing} formValues={destinationForm} selectedIntegration={editingIntegration} />
+ <Fields isNew={!editing} formValues={destinationForm} selectedIntegration={selectedIntegration} />
{editing ? (
<LemonBanner type="info">
diff --git a/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts b/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts
index 73d96f022f5f..9a6af4d5b3c7 100644
--- a/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts
+++ b/products/data_warehouse/frontend/shared/destinations/destinationDefinitions.test.ts
@@ -1,3 +1,5 @@
+import type { IntegrationType } from '~/types'
+
import {
CREATABLE_DESTINATION_TYPES,
WAREHOUSE_DESTINATIONS,
@@ -102,6 +104,27 @@ describe('warehouse destination definitions', () => {
expect(validateDestinationFields(definition, { isNew: true, formValues })).toEqual({})
})
+ it.each([
+ ['a username and password connection', { host: 'my-cluster.example.com', user: 'posthog' }, false],
+ [
+ 'an IAM role connection',
+ { aws_role_arn: 'arn:aws:iam::123456789012:role/posthog', user: 'posthog' },
+ true,
+ ],
+ ['an access key connection', { name: 'keys', user: 'posthog' }, true],
+ ['no connection picked yet', null, false],
+ ])('Redshift asks for a host only when it needs one, with %s', (_label, config, needsHost) => {
+ const selectedIntegration = config ? ({ id: 1, kind: 'aws-redshift', config } as IntegrationType) : null
+
+ const errors = validateDestinationFields(WAREHOUSE_DESTINATIONS.Redshift, {
+ isNew: true,
+ formValues: { database: 'dev', schema: 'public' },
+ selectedIntegration,
+ })
+
+ expect(errors.host).toEqual(needsHost ? 'This field is required' : undefined)
+ })
+
it('treats a whitespace-only value as missing', () => {
const errors = validateDestinationFields(WAREHOUSE_DESTINATIONS.BigQuery, {
isNew: true,
diff --git a/products/data_warehouse/frontend/shared/destinations/redshift.tsx b/products/data_warehouse/frontend/shared/destinations/redshift.tsx
index d50b99a574d4..ff616bd693c3 100644
--- a/products/data_warehouse/frontend/shared/destinations/redshift.tsx
+++ b/products/data_warehouse/frontend/shared/destinations/redshift.tsx
@@ -2,28 +2,60 @@ import { LemonInput } from '@posthog/lemon-ui'
import { LemonField } from 'lib/lemon-ui/LemonField'
+import type { IntegrationType } from '~/types'
+
import type { WarehouseDestinationDefinition } from './types'
+/**
+ * A username and password connection stores the cluster's host. An IAM role or access key
+ * connection stores only the AWS identity, so the destination has to name the cluster.
+ */
+function needsClusterEndpoint(integration: IntegrationType | null | undefined): boolean {
+ return !!integration && !integration.config?.host
+}
+
// Redshift speaks the Postgres wire protocol and its writer subclasses the Postgres one, so it
-// takes the same two config fields. Kept as its own definition rather than sharing Postgres's,
+// takes the same database and schema. Kept as its own definition rather than sharing Postgres's,
// because the two pick credentials from different integration kinds.
export const redshiftDefinition: WarehouseDestinationDefinition = {
type: 'Redshift',
integrationKinds: ['aws-redshift'],
defaults: () => ({ database: 'dev', schema: 'public' }),
- requiredFields: () => ['database', 'schema'],
- configKeys: ['database', 'schema'],
- retargetingKeys: ['database', 'schema'],
- Fields: function RedshiftFields({ isNew }) {
+ requiredFields: ({ selectedIntegration }) =>
+ needsClusterEndpoint(selectedIntegration) ? ['host', 'database', 'schema'] : ['database', 'schema'],
+ configKeys: ['host', 'port', 'database', 'schema'],
+ retargetingKeys: ['host', 'database', 'schema'],
+ Fields: function RedshiftFields({ isNew, selectedIntegration }) {
return (
- <div className="flex gap-2">
- <LemonField name="database" label="Database" className="flex-1">
- <LemonInput placeholder="dev" disabled={!isNew} data-attr="warehouse-destination-database" />
- </LemonField>
- <LemonField name="schema" label="Schema" className="flex-1">
- <LemonInput placeholder="public" disabled={!isNew} data-attr="warehouse-destination-schema" />
- </LemonField>
- </div>
+ <>
+ {needsClusterEndpoint(selectedIntegration) ? (
+ <div className="flex flex-wrap gap-2">
+ <LemonField
+ name="host"
+ label="Host"
+ className="flex-1 min-w-60"
+ info="The cluster or workgroup endpoint. This connection signs in with AWS, so it has no host of its own."
+ >
+ <LemonInput
+ placeholder="my-cluster.abc123.us-east-1.redshift.amazonaws.com"
+ disabled={!isNew}
+ data-attr="warehouse-destination-host"
+ />
+ </LemonField>
+ <LemonField name="port" label="Port" showOptional className="w-28">
+ <LemonInput type="number" placeholder="5439" data-attr="warehouse-destination-port" />
+ </LemonField>
+ </div>
+ ) : null}
+ <div className="flex flex-wrap gap-2">
+ <LemonField name="database" label="Database" className="flex-1 min-w-40">
+ <LemonInput placeholder="dev" disabled={!isNew} data-attr="warehouse-destination-database" />
+ </LemonField>
+ <LemonField name="schema" label="Schema" className="flex-1 min-w-40">
+ <LemonInput placeholder="public" disabled={!isNew} data-attr="warehouse-destination-schema" />
+ </LemonField>
+ </div>
+ </>
)
},
}
diff --git a/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts b/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts
index 77805b7206db..f0910d18fdd4 100644
--- a/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts
+++ b/products/data_warehouse/frontend/shared/logics/destinationModalLogic.ts
@@ -47,6 +47,10 @@ export interface DestinationFormValues extends Record<string, any> {
const DEFAULT_TYPE: CreatableDestinationType = 'Postgres'
+function findIntegration(integrations: IntegrationType[] | null, id: number | null): IntegrationType | null {
+ return (id && integrations?.find((one) => one.id === id)) || null
+}
+
export function newDestinationForm(type: CreatableDestinationType = DEFAULT_TYPE): DestinationFormValues {
const definition = WAREHOUSE_DESTINATIONS[type]
return {
@@ -78,6 +82,7 @@ export interface destinationModalLogicValues {
isDestinationFormSubmitting: boolean
isDestinationFormValid: boolean
isOpen: boolean
+ selectedIntegration: IntegrationType | null
showDestinationFormErrors: boolean
}
@@ -192,6 +197,7 @@ export interface destinationModalLogicMeta {
definition: (destinationForm: any) => WarehouseDestinationDefinition
availableIntegrations: (integrations: any, destinationForm: any) => IntegrationType[]
editingIntegration: (editing: any, integrations: any) => IntegrationType | null
+ selectedIntegration: (integrations: any, destinationForm: any) => IntegrationType | null
}
}
@@ -247,6 +253,7 @@ export const destinationModalLogic = kea<destinationModalLogicType>([
{
isNew: !values.editing,
formValues,
+ selectedIntegration: findIntegration(values.integrations, formValues.integrationId),
}
),
}),
@@ -294,8 +301,13 @@ export const destinationModalLogic = kea<destinationModalLogicType>([
(
editing: ExternalDataDestinationApi | null,
integrations: IntegrationType[] | null
- ): IntegrationType | null =>
- (editing?.integration && integrations?.find((one) => one.id === editing.integration)) || null,
+ ): IntegrationType | null => findIntegration(integrations, editing?.integration ?? null),
+ ],
+ // The connection picked in the form, so a type's fields can depend on how it signs in.
+ selectedIntegration: [
+ (s: any) => [s.integrations, s.destinationForm],
+ (integrations: IntegrationType[] | null, destinationForm: DestinationFormValues): IntegrationType | null =>
+ findIntegration(integrations, destinationForm.integrationId),
],
}),
listeners(({ actions, values, props }: any) => ({
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
index 26fc074c4a5f..ebe7254027f7 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
@@ -17,6 +17,14 @@ from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.integrations import IntegrationKind, IntegrationRecord
from posthog.integrations.fields import IntegrationReferenceField
+from posthog.models.integration import (
+ AWSRedshiftRoleBasedIntegration,
+ IntegrationError,
+ RedshiftIntegration,
+ RedshiftServerlessWorkgroup,
+ parse_redshift_host,
+ redshift_integration_from_record,
+)
from posthog.permissions import is_service_auth
from products.access_control.backend.facade.user_access_control import access_level_satisfied_for_resource
@@ -390,6 +398,12 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
"latest_error_at",
]
+ # A JSONField takes any JSON value, and every check and `update()` read config as an object.
+ def validate_config(self, value: Any) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise ValidationError('Config must be an object of settings, such as {"database": "analytics"}.')
+ return value
+
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
destination_type = attrs.get("type", getattr(self.instance, "type", None))
@@ -419,8 +433,13 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
if self.instance is not None:
self._reject_retargeting(attrs)
+ if destination_type == ExternalDataDestination.Type.REDSHIFT and "config" in attrs:
+ # `update()` merges the PATCH onto the saved config, so check the port the next sync uses.
+ self._check_redshift_port({**(self.instance.config or {}), **attrs["config"]})
elif destination_type == ExternalDataDestination.Type.POSTGRES:
self._check_postgres_connection(integration, attrs.get("config"))
+ elif destination_type == ExternalDataDestination.Type.REDSHIFT:
+ self._check_redshift_target(integration, attrs.get("config"))
else:
self._run_connection_test(destination_type, integration, attrs.get("config"))
@@ -435,6 +454,62 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
field = "config" if error.failure in CONFIG_FAILURES else "integration"
raise ValidationError({field: str(error)})
+ # A username and password connection stores its own host. An IAM role or access key
+ # connection stores only the AWS identity, so the destination has to name the cluster.
+ def _check_redshift_target(self, integration: IntegrationRecord, config: dict[str, Any] | None) -> None:
+ try:
+ redshift = redshift_integration_from_record(integration)
+ except IntegrationError as error:
+ raise ValidationError({"integration": str(error)})
+ if isinstance(redshift, RedshiftIntegration):
+ self._require_complete_password_connection(integration)
+ return
+
+ config = config or {}
+ host = config.get("host")
+ if not isinstance(host, str) or not host:
+ raise ValidationError(
+ {"config": "Add the cluster endpoint as host. This connection uses AWS credentials, so it has no host."}
+ )
+ try:
+ server = parse_redshift_host(host)
+ if isinstance(redshift, AWSRedshiftRoleBasedIntegration) and isinstance(
+ server, RedshiftServerlessWorkgroup
+ ):
+ redshift.workgroup_arn_for(server)
+ except IntegrationError as error:
+ raise ValidationError({"config": str(error)})
+
+ self._check_redshift_port(config)
+
+ def _check_redshift_port(self, config: dict[str, Any]) -> None:
+ port = config.get("port")
+ if port is not None and (isinstance(port, bool) or not isinstance(port, int) or not 0 < port <= 65535):
+ raise ValidationError({"config": "Port must be a number from 1 to 65535."})
+
+ # A connection with no AWS identity signs in with a password, and the writer reads all of these
+ # fields on the first sync. Without this, a broken connection saves and only fails then.
+ REDSHIFT_PASSWORD_FIELDS: tuple[tuple[str, str, bool], ...] = (
+ ("host", "host", False),
+ ("port", "port", False),
+ ("user", "username", False),
+ ("password", "password", True),
+ )
+
+ def _require_complete_password_connection(self, integration: IntegrationRecord) -> None:
+ config = integration.config or {}
+ sensitive_config = integration.sensitive_config or {}
+ missing = [
+ label
+ for key, label, sensitive in self.REDSHIFT_PASSWORD_FIELDS
+ if not str((sensitive_config if sensitive else config).get(key) or "").strip()
+ ]
+ if missing:
+ listed = missing[0] if len(missing) == 1 else f"{', '.join(missing[:-1])} or {missing[-1]}"
+ raise ValidationError(
+ {"integration": f"This Redshift connection has no {listed}. Reconnect Redshift, then try again."}
+ )
+
def _run_connection_test(self, destination_type: str, integration: IntegrationRecord, config: Any) -> None:
run = run_connection_test(destination_type, integration, config)
if run is not None and run.failure is not None:
@@ -453,7 +528,7 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
# spelling would let the other through.
RETARGETING_FIELDS_BY_TYPE: dict[str, tuple[str, ...]] = {
str(ExternalDataDestination.Type.POSTGRES): ("database", "schema"),
- str(ExternalDataDestination.Type.REDSHIFT): ("database", "schema"),
+ str(ExternalDataDestination.Type.REDSHIFT): ("host", "database", "schema"),
str(ExternalDataDestination.Type.SNOWFLAKE): ("database", "schema"),
str(ExternalDataDestination.Type.DATABRICKS): ("catalog", "schema"),
str(ExternalDataDestination.Type.BIGQUERY): ("dataset", "dataset_id", "project", "project_id"),
@@ -465,6 +540,7 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
# often has a stray space at one end, and the store then rejects the name on every run. A
# prefix is not here: a prefix with a space is a valid key, and rows may already sit under it.
TRIMMED_FIELDS_BY_TYPE: dict[str, tuple[str, ...]] = {
+ str(ExternalDataDestination.Type.REDSHIFT): ("host",),
str(ExternalDataDestination.Type.S3): ("bucket", "bucket_name", "region"),
str(ExternalDataDestination.Type.AZURE_BLOB): ("container_name", "container"),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py
index 28e675aa8983..5f38c20b5d4d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_configuration_errors.py
@@ -1,14 +1,22 @@
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
+from typing import Any
import pytest
from unittest import mock
import psycopg
import pyarrow as pa
+import botocore.exceptions
+
+from posthog.models.integration import IntegrationError
from products.batch_exports.backend.facade.destinations.postgres import PostgreSQLClient
from products.batch_exports.backend.facade.destinations.redshift import StringLimitExceededError
+from products.batch_exports.backend.facade.destinations.s3 import (
+ ExternalRoleArnNotConfiguredError,
+ InvalidCredentialsError,
+)
from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (
DestinationBatchContext,
DestinationRunContext,
@@ -24,6 +32,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
)
+def _aws_error(code: str, operation: str = "GetClusterCredentials", status: int = 400) -> Exception:
+ response: Any = {"Error": {"Code": code, "Message": "fake-secret"}, "ResponseMetadata": {"HTTPStatusCode": status}}
+ return botocore.exceptions.ClientError(response, operation)
+
+
def _ctx() -> DestinationRunContext:
return DestinationRunContext(
team_id=1,
@@ -58,13 +71,22 @@ class LocalRedshiftWriter(RedshiftDestinationWriter):
class TestConfigurationErrors:
@pytest.mark.parametrize("finalize", [False, True], ids=["write_batch", "finalize_run"])
@pytest.mark.parametrize(
- "error",
+ "error,expected",
[
- psycopg.errors.InsufficientPrivilege("permission denied for schema fake-secret"),
- psycopg.errors.InsufficientPrivilege("permission denied for table fake-secret"),
+ (psycopg.errors.InsufficientPrivilege("permission denied for schema fake-secret"), "USAGE and CREATE"),
+ (psycopg.errors.InsufficientPrivilege("permission denied for table fake-secret"), "USAGE and CREATE"),
+ (_aws_error("AccessDenied", "AssumeRole"), "cannot assume the role"),
+ (InvalidCredentialsError("fake-secret"), "cannot assume the role"),
+ (_aws_error("AccessDenied"), "redshift:GetClusterCredentials"),
+ (_aws_error("AccessDeniedException", "GetCredentials"), "redshift-serverless:GetCredentials"),
+ (_aws_error("InvalidClientTokenId"), "rejected the access key"),
+ (_aws_error("ClusterNotFound"), "cannot find the Redshift cluster"),
+ (_aws_error("ResourceNotFoundException", "GetCredentials"), "cannot find the Redshift cluster"),
],
)
- async def test_permanent_errors_are_configuration_errors(self, error: Exception, finalize: bool) -> None:
+ async def test_permanent_errors_are_configuration_errors(
+ self, error: Exception, expected: str, finalize: bool
+ ) -> None:
ctx = _ctx()
client = mock.AsyncMock(spec=PostgreSQLClient)
writer = LocalRedshiftWriter(ctx, client)
@@ -73,7 +95,7 @@ class TestConfigurationErrors:
with (
mock.patch.object(writer, "_table_exists", return_value=False),
mock.patch.object(writer, "_ensure_table", side_effect=error),
- pytest.raises(DestinationConfigurationError, match="USAGE and CREATE") as caught,
+ pytest.raises(DestinationConfigurationError, match=expected) as caught,
):
if finalize:
await writer.finalize_run(ctx)
@@ -100,6 +122,11 @@ class TestConfigurationErrors:
StringLimitExceededError("fake-secret", "charges", "public"),
UnrelatedTableExistsError("fake-secret"),
TimeoutError("fake-secret"),
+ _aws_error("Throttling", "AssumeRole"),
+ _aws_error("AccessDenied", status=500),
+ _aws_error("InternalFailure", status=503),
+ botocore.exceptions.EndpointConnectionError(endpoint_url="https://redshift.us-east-1.amazonaws.com"),
+ ExternalRoleArnNotConfiguredError(),
],
)
async def test_transient_errors_propagate_unchanged(self, error: Exception, finalize: bool) -> None:
@@ -122,3 +149,17 @@ class TestConfigurationErrors:
assert caught.value is error
assert not isinstance(caught.value, DestinationConfigurationError)
+
+ async def test_an_unusable_host_reports_what_to_fix(self) -> None:
+ ctx = _ctx()
+ writer = LocalRedshiftWriter(ctx, mock.AsyncMock(spec=PostgreSQLClient))
+ error = IntegrationError("Redshift host 'db.example.com' is not a recognized AWS Redshift endpoint")
+
+ with (
+ mock.patch.object(writer, "_table_exists", return_value=False),
+ mock.patch.object(writer, "_ensure_table", side_effect=error),
+ pytest.raises(DestinationConfigurationError) as caught,
+ ):
+ await writer.write_batch(_batches(), DestinationBatchContext(run=ctx, batch_index=0, is_final_batch=False))
+
+ assert caught.value.detail == str(error)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.py
new file mode 100644
index 000000000000..8287811bc9f5
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_redshift_connection.py
@@ -0,0 +1,142 @@
+from typing import Any
+
+import pytest
+from unittest import mock
+
+from posthog.integrations import IntegrationRecord
+from posthog.models.integration import Integration
+
+from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (
+ DestinationBatchContext,
+ DestinationRunContext,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (
+ DestinationConfigurationError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.writers.redshift import (
+ RedshiftDestinationWriter,
+)
+
+CLUSTER_HOST = "my-cluster.abc123.us-east-1.redshift.amazonaws.com"
+WORKGROUP_HOST = "my-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com"
+
+
+def _ctx(**config: Any) -> DestinationRunContext:
+ return DestinationRunContext(
+ team_id=1,
+ schema_id="schema",
+ source_id="source",
+ job_id="job",
+ run_uuid="run-a1",
+ destination_id="destination",
+ destination_type="Redshift",
+ destination_name="test redshift",
+ table_name="charges",
+ sync_type="incremental",
+ config={"database": "analytics", "schema": "public", **config},
+ integration_id=7,
+ )
+
+
+def _password_integration() -> Integration:
+ return Integration(
+ kind=Integration.IntegrationKind.AWS_REDSHIFT,
+ config={"host": "stored.example.com", "port": 5555, "user": "stored_user", "ssl_mode": "require"},
+ sensitive_config={"password": "stored-password"},
+ )
+
+
+def _access_key_integration() -> Integration:
+ return Integration(
+ kind=Integration.IntegrationKind.AWS_REDSHIFT,
+ config={"name": "keys", "user": "posthog", "groups": ["loaders"]},
+ sensitive_config={"aws_access_key_id": "AKIAEXAMPLE", "aws_secret_access_key": "fake-secret"},
+ )
+
+
+class LocalRedshiftWriter(RedshiftDestinationWriter):
+ def __init__(self, ctx: DestinationRunContext, integration: Integration) -> None:
+ super().__init__(ctx)
+ self._integration = integration
+
+ async def _load_integration(self, integration_id: int) -> IntegrationRecord:
+ return self._integration
+
+
+def _aws_session(aws: mock.MagicMock) -> mock.MagicMock:
+ client = mock.MagicMock()
+ client.__aenter__.return_value = aws
+ client.__aexit__.return_value = None
+ session = mock.MagicMock()
+ session.client.return_value = client
+ return session
+
+
+class TestRedshiftConnection:
+ async def test_a_password_connection_uses_its_own_host_and_ignores_the_config(self) -> None:
+ writer = LocalRedshiftWriter(_ctx(host=CLUSTER_HOST, port=5439), _password_integration())
+
+ with mock.patch("aioboto3.Session") as session:
+ client = await writer._make_client()
+
+ session.assert_not_called()
+ assert (client.host, client.port, client.user, client.password, client.database) == (
+ "stored.example.com",
+ 5555,
+ "stored_user",
+ "stored-password",
+ "analytics",
+ )
+
+ @pytest.mark.parametrize(
+ "config,port,service",
+ [
+ ({"host": CLUSTER_HOST}, 5439, "redshift"),
+ ({"host": CLUSTER_HOST, "port": 5440}, 5440, "redshift"),
+ ({"host": WORKGROUP_HOST}, 5439, "redshift-serverless"),
+ ],
+ ids=["cluster_default_port", "cluster_custom_port", "serverless_workgroup"],
+ )
+ async def test_an_access_key_connection_signs_in_with_temporary_credentials_on_the_config_host(
+ self, config: dict[str, Any], port: int, service: str
+ ) -> None:
+ writer = LocalRedshiftWriter(_ctx(**config), _access_key_integration())
+ aws = mock.MagicMock()
+ aws.get_cluster_credentials = mock.AsyncMock(
+ return_value={"DbUser": "IAM:posthog", "DbPassword": "temporary-password"}
+ )
+ aws.get_credentials = mock.AsyncMock(return_value={"dbUser": "IAM:posthog", "dbPassword": "temporary-password"})
+
+ with mock.patch("aioboto3.Session", return_value=_aws_session(aws)) as session:
+ client = await writer._make_client()
+
+ assert session.call_args.kwargs["aws_access_key_id"] == "AKIAEXAMPLE"
+ assert session.return_value.client.call_args.args == (service,)
+ assert (client.host, client.port, client.user, client.password, client.database) == (
+ config["host"],
+ port,
+ "IAM:posthog",
+ "temporary-password",
+ "analytics",
+ )
+ if service == "redshift":
+ assert aws.get_cluster_credentials.call_args.kwargs == {
+ "DbUser": "posthog",
+ "DbName": "analytics",
+ "ClusterIdentifier": "my-cluster",
+ "DurationSeconds": 3600,
+ "DbGroups": ["loaders"],
+ }
+
+ async def test_an_aws_connection_without_a_host_is_a_configuration_error(self) -> None:
+ ctx = _ctx()
+ writer = LocalRedshiftWriter(ctx, _access_key_integration())
+
+ with mock.patch("aioboto3.Session") as session, pytest.raises(DestinationConfigurationError) as caught:
+ await writer.write_batch(
+ mock.MagicMock(), DestinationBatchContext(run=ctx, batch_index=0, is_final_batch=False)
+ )
+
+ session.assert_not_called()
+ assert "host" in caught.value.detail
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
index 70b6213812b8..1983ad40ae28 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
@@ -260,7 +260,7 @@ class PostgresDestinationWriter:
# --- connection -------------------------------------------------------------------
- def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
+ async def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
# `from_inputs` accepts anything exposing credentials/authority/tls, which the
# integration wrapper does, so the batch export path and this one resolve credentials
# identically.
@@ -284,7 +284,7 @@ class PostgresDestinationWriter:
integration = await self._load_integration(self._ctx.integration_id)
except PostgreSQLIntegrationNotFoundError as err:
raise DestinationConfigurationError(self._ctx.destination_name, MISSING_INTEGRATION_DETAIL) from err
- return self._client_from_integration(integration)
+ return await self._client_from_integration(integration)
@asynccontextmanager
async def _client(self) -> AsyncIterator[PostgreSQLClient]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
index 908dea22ad11..4d144911d8aa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
@@ -2,8 +2,13 @@
Redshift speaks the Postgres wire protocol, so this builds on the Postgres writer and on batch
exports' `RedshiftClient`, which is itself a `PostgreSQLClient`. A Redshift destination is
-backed by an `aws-redshift` integration, which is a `PostgreSQLServerIntegration` and so
-carries the cluster's host, user and password.
+backed by an `aws-redshift` integration in one of three shapes:
+
+- Username and password. The integration carries the cluster's host, user and password.
+- An IAM role, or an AWS access key pair. The integration carries only the AWS identity and the
+ database user, so the destination's `config` carries the cluster's `host` and `port`. Each
+ connection asks AWS for temporary database credentials, through the same
+ `resolve_redshift_client_inputs` batch exports connects with.
Three things genuinely differ and are overridden here:
@@ -27,14 +32,16 @@ from contextlib import contextmanager
from typing import Any, ClassVar, cast
import pyarrow as pa
+import botocore.exceptions
from psycopg import sql
from psycopg.errors import InsufficientPrivilege
from posthog.integrations import IntegrationRecord
-from posthog.models.integration import RedshiftIntegration
+from posthog.models.integration import IntegrationError, RedshiftIntegration, redshift_integration_from_record
from products.batch_exports.backend.facade.destinations.postgres import Fields, PostgreSQLClient
-from products.batch_exports.backend.facade.destinations.redshift import RedshiftClient
+from products.batch_exports.backend.facade.destinations.redshift import RedshiftClient, resolve_redshift_client_inputs
+from products.batch_exports.backend.facade.destinations.s3 import InvalidCredentialsError
from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (
BatchWriteOutcome,
DestinationBatchContext,
@@ -55,15 +62,49 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
# reports the same overflow as `StringLimitExceededError` and never retries it.
VARCHAR_MAX = "VARCHAR(MAX)"
+DEFAULT_PORT = 5439
+
PERMISSIONS_DETAIL = (
"The Redshift user lacks permissions. Grant the user USAGE and CREATE on the schema. "
"Grant rights on the tables it syncs, then run the sync again."
)
+ROLE_DETAIL = (
+ "PostHog cannot assume the role. Check the role ARN and the trust policy with the external ID, "
+ "then run the sync again."
+)
+AWS_CREDENTIALS_DETAIL = "AWS rejected the access key. Check the access key ID and secret, then run the sync again."
+AWS_PERMISSIONS_DETAIL = (
+ "AWS denied temporary database credentials. Allow redshift:GetClusterCredentials for the database user, "
+ "or redshift-serverless:GetCredentials for the workgroup, then run the sync again."
+)
+CLUSTER_NOT_FOUND_DETAIL = (
+ "AWS cannot find the Redshift cluster or workgroup. Check the host on the destination, then run the sync again."
+)
def _configuration_error_detail(err: Exception) -> str | None:
if isinstance(err, InsufficientPrivilege):
return PERMISSIONS_DETAIL
+ if isinstance(err, IntegrationError):
+ # Written by us for a host or workgroup ARN that cannot work, and holds no secret.
+ return str(err)
+ if isinstance(err, InvalidCredentialsError):
+ return ROLE_DETAIL
+ if isinstance(err, botocore.exceptions.ClientError):
+ return _aws_error_detail(err)
+ return None
+
+
+def _aws_error_detail(err: botocore.exceptions.ClientError) -> str | None:
+ if err.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) >= 500:
+ return None
+ code = err.response.get("Error", {}).get("Code")
+ if code in ("InvalidClientTokenId", "SignatureDoesNotMatch", "UnrecognizedClientException"):
+ return AWS_CREDENTIALS_DETAIL
+ if code in ("AccessDenied", "AccessDeniedException"):
+ return ROLE_DETAIL if err.operation_name == "AssumeRole" else AWS_PERMISSIONS_DETAIL
+ if code in ("ClusterNotFound", "ClusterNotFoundFault", "ResourceNotFoundException"):
+ return CLUSTER_NOT_FOUND_DETAIL
return None
@@ -136,8 +177,18 @@ class RedshiftDestinationWriter(PostgresDestinationWriter):
with self._configuration_errors():
await super().abort_run(ctx)
- def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
- return RedshiftClient.from_inputs(RedshiftIntegration(integration), database=self._database)
+ async def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
+ config = self._ctx.config or {}
+ inputs = await resolve_redshift_client_inputs(
+ redshift_integration_from_record(integration),
+ host=config.get("host") or None,
+ port=int(config.get("port") or DEFAULT_PORT),
+ database=self._database,
+ team_id=self._ctx.team_id,
+ # The same session name the S3 writer uses, so one role's CloudTrail shows one sync.
+ session_name=f"PostHog-warehouse-sync-{self._ctx.schema_id}",
+ )
+ return RedshiftClient.from_inputs(inputs, database=self._database)
def _column_type(self, arrow_type: pa.DataType) -> str:
return redshift_type_for(arrow_type)
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
index d570d28e984a..e733175071ad 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_destination.py
@@ -62,6 +62,11 @@ from products.warehouse_sources.backend.presentation.views.external_data_destina
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+REDSHIFT_ROLE_ARN = "arn:aws:iam::123456789012:role/posthog-redshift"
+REDSHIFT_CLUSTER_HOST = "my-cluster.abc123.us-east-1.redshift.amazonaws.com"
+REDSHIFT_WORKGROUP_HOST = "my-wg.123456789012.us-east-1.redshift-serverless.amazonaws.com"
+REDSHIFT_WORKGROUP_ARN = "arn:aws:redshift-serverless:us-east-1:123456789012:workgroup/1a2b3c4d"
+
def _destination_test(step_count: int) -> DestinationTest:
return DestinationTest(
@@ -117,11 +122,32 @@ class DestinationAPITestBase(APIBaseTest):
self.schema = ExternalDataSchema.objects.create(team=self.team, source=self.source, name="charges")
def _integration(self, kind: str = Integration.IntegrationKind.POSTGRESQL) -> Integration:
+ if kind == Integration.IntegrationKind.AWS_REDSHIFT:
+ return self._redshift_integration("password")
self._integration_seq = getattr(self, "_integration_seq", 0) + 1
return Integration.objects.create(
team=self.team, kind=kind, integration_id=f"{kind}-{self._integration_seq}", config={}
)
+ def _redshift_integration(self, mode: str, **config) -> Integration:
+ self._integration_seq = getattr(self, "_integration_seq", 0) + 1
+ sensitive_config: dict[str, str] = {}
+ if mode == "role":
+ config = {"aws_role_arn": REDSHIFT_ROLE_ARN, "user": "posthog", **config}
+ elif mode == "access_key":
+ config = {"name": "keys", "user": "posthog", **config}
+ sensitive_config = {"aws_access_key_id": "AKIAEXAMPLE", "aws_secret_access_key": "fake-secret"}
+ else:
+ config = {"host": "my-cluster.example.com", "port": 5439, "user": "posthog", **config}
+ sensitive_config = {"password": "fake-secret"}
+ return Integration.objects.create(
+ team=self.team,
+ kind=Integration.IntegrationKind.AWS_REDSHIFT,
+ integration_id=f"redshift-{self._integration_seq}",
+ config=config,
+ sensitive_config=sensitive_config,
+ )
+
def _create_destination(self, **overrides) -> ExternalDataDestination:
payload = {
"type": ExternalDataDestination.Type.POSTGRES,
@@ -393,6 +419,180 @@ class TestExternalDataDestinationAPI(DestinationAPITestBase):
self.describe_test.assert_not_called()
self.run_test_step.assert_not_called()
+ @parameterized.expand(
+ [
+ ("password_needs_no_host", "password", {}, {}, None),
+ ("role_with_cluster_host", "role", {}, {"host": REDSHIFT_CLUSTER_HOST}, None),
+ ("role_with_cluster_host_and_port", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 5440}, None),
+ ("access_key_with_cluster_host", "access_key", {}, {"host": REDSHIFT_CLUSTER_HOST}, None),
+ ("host_with_stray_space", "access_key", {}, {"host": f" {REDSHIFT_CLUSTER_HOST} "}, None),
+ (
+ "role_with_workgroup_host_and_arn",
+ "role",
+ {"workgroup_arn": REDSHIFT_WORKGROUP_ARN},
+ {"host": REDSHIFT_WORKGROUP_HOST},
+ None,
+ ),
+ ("access_key_with_workgroup_host", "access_key", {}, {"host": REDSHIFT_WORKGROUP_HOST}, None),
+ ("role_without_host", "role", {}, {}, "Add the cluster endpoint"),
+ ("access_key_without_host", "access_key", {}, {}, "Add the cluster endpoint"),
+ ("role_with_blank_host", "role", {}, {"host": " "}, "Add the cluster endpoint"),
+ ("host_not_a_redshift_endpoint", "role", {}, {"host": "db.example.com"}, "not a recognized"),
+ ("role_with_workgroup_host_no_arn", "role", {}, {"host": REDSHIFT_WORKGROUP_HOST}, "workgroup_arn"),
+ (
+ "role_with_workgroup_arn_in_another_account",
+ "role",
+ {"workgroup_arn": REDSHIFT_WORKGROUP_ARN.replace("123456789012", "210987654321")},
+ {"host": REDSHIFT_WORKGROUP_HOST},
+ "does not match",
+ ),
+ ("port_zero", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 0}, "Port must be"),
+ ("port_too_large", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": 70000}, "Port must be"),
+ ("port_as_text", "role", {}, {"host": REDSHIFT_CLUSTER_HOST, "port": "5439"}, "Port must be"),
+ ]
+ )
+ def test_a_redshift_destination_names_the_cluster_when_its_connection_has_none(
+ self, _name: str, mode: str, integration_config: dict, target: dict, error: str | None
+ ) -> None:
+ response = self.client.post(
+ self.base,
+ {
+ "type": ExternalDataDestination.Type.REDSHIFT,
+ "name": "analytics redshift",
+ "integration": self._redshift_integration(mode, **integration_config).pk,
+ "config": {"database": "dev", "schema": "public", **target},
+ },
+ format="json",
+ )
+
+ if error is None:
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ if "host" in target:
+ assert response.json()["config"]["host"] == target["host"].strip()
+ else:
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert response.json()["attr"] == "config"
+ assert error in response.json()["detail"]
+
+ @parameterized.expand(
+ [
+ ("empty", {}, {}, "host, port, username or password"),
+ ("no_password", {"host": "my-cluster.example.com", "port": 5439, "user": "posthog"}, {}, "password"),
+ (
+ "blank_password",
+ {"host": "my-cluster.example.com", "port": 5439, "user": "posthog"},
+ {"password": ""},
+ "password",
+ ),
+ ("no_host", {"port": 5439, "user": "posthog"}, {"password": "fake-secret"}, "host"),
+ ("blank_host", {"host": " ", "port": 5439, "user": "posthog"}, {"password": "fake-secret"}, "host"),
+ ("no_port", {"host": "my-cluster.example.com", "user": "posthog"}, {"password": "fake-secret"}, "port"),
+ (
+ "no_user",
+ {"host": "my-cluster.example.com", "port": 5439},
+ {"password": "fake-secret"},
+ "username",
+ ),
+ ]
+ )
+ def test_a_redshift_destination_rejects_an_incomplete_password_connection(
+ self, _name: str, config: dict, sensitive_config: dict, missing: str
+ ) -> None:
+ integration = Integration.objects.create(
+ team=self.team,
+ kind=Integration.IntegrationKind.AWS_REDSHIFT,
+ integration_id=f"redshift-incomplete-{_name}",
+ config=config,
+ sensitive_config=sensitive_config,
+ )
+
+ response = self.client.post(
+ self.base,
+ {
+ "type": ExternalDataDestination.Type.REDSHIFT,
+ "name": "broken redshift",
+ "integration": integration.pk,
+ "config": {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST},
+ },
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert response.json()["attr"] == "integration"
+ assert f"This Redshift connection has no {missing}." in response.json()["detail"]
+ assert "fake-secret" not in response.content.decode()
+ assert not ExternalDataDestination.objects.for_team(self.team.pk).filter(name="broken redshift").exists()
+
+ @parameterized.expand(
+ [
+ ("another_host", {"host": "other.abc123.us-east-1.redshift.amazonaws.com"}, "keeps the host"),
+ ("port_negative", {"port": -1}, "Port must be"),
+ ("port_too_large", {"port": 70000}, "Port must be"),
+ ("port_as_text", {"port": "5440"}, "Port must be"),
+ ("port_as_bool", {"port": True}, "Port must be"),
+ ("valid_port", {"port": 5440}, None),
+ ]
+ )
+ def test_a_redshift_destination_update_keeps_the_host_and_checks_the_port(
+ self, _name: str, patch: dict, error: str | None
+ ) -> None:
+ created = {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST, "port": 5439}
+ destination = self._create_destination(
+ type=ExternalDataDestination.Type.REDSHIFT,
+ integration=self._redshift_integration("access_key").pk,
+ config=created,
+ )
+
+ response = self.client.patch(f"{self.base}/{destination.id}", {"config": patch}, format="json")
+
+ destination.refresh_from_db()
+ if error is None:
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ assert destination.config == {**created, **patch}
+ else:
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert response.json()["attr"] == "config"
+ assert error in response.json()["detail"]
+ assert destination.config == created
+
+ @parameterized.expand(
+ [
+ ("list_on_create", "create", ["invalid"]),
+ ("text_on_create", "create", "invalid"),
+ ("number_on_create", "create", 5439),
+ ("list_on_update", "update", ["invalid"]),
+ ("text_on_update", "update", "invalid"),
+ ]
+ )
+ def test_a_destination_config_must_be_an_object(self, _name: str, action: str, config: object) -> None:
+ integration = self._redshift_integration("role")
+ if action == "create":
+ response = self.client.post(
+ self.base,
+ {
+ "type": ExternalDataDestination.Type.REDSHIFT,
+ "name": "malformed redshift",
+ "integration": integration.pk,
+ "config": config,
+ },
+ format="json",
+ )
+ else:
+ destination = self._create_destination(
+ type=ExternalDataDestination.Type.REDSHIFT,
+ name="malformed redshift",
+ integration=integration.pk,
+ config={"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST},
+ )
+ response = self.client.patch(f"{self.base}/{destination.id}", {"config": config}, format="json")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert response.json()["attr"] == "config"
+ assert "Config must be an object" in response.json()["detail"]
+ if action == "update":
+ destination.refresh_from_db()
+ assert destination.config == {"database": "dev", "schema": "public", "host": REDSHIFT_CLUSTER_HOST}
+
def test_updating_a_destination_does_not_run_the_connection_test(self) -> None:
destination = self._create_destination()
self.check_connection.reset_mock()
diff --git a/products/warehouse_sources/mcp/warehouse_destinations.yaml b/products/warehouse_sources/mcp/warehouse_destinations.yaml
index 29c74453804c..38efbb1b074a 100644
--- a/products/warehouse_sources/mcp/warehouse_destinations.yaml
+++ b/products/warehouse_sources/mcp/warehouse_destinations.yaml
@@ -92,14 +92,15 @@ tools:
description: >
Create a place where warehouse sources write synced rows, from an integration that holds the credentials.
Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the
- integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake
- takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks
- takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression;
- Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the
- connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so
- a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm
- it with the user first. A new destination receives nothing until a source is attached with
- external-data-destinations-add-sources.
+ integration id and the target config for the type: Postgres takes database and schema; Redshift takes
+ database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or
+ access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional
+ role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume;
+ S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional
+ prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns
+ a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync
+ fails. The target cannot change after creation, so confirm it with the user first. A new destination
+ receives nothing until a source is attached with external-data-destinations-add-sources.
feature_flag: warehouse-multi-destination
external-data-destinations-list:
operation: external_data_destinations_list
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index a645a20ea141..99d8d6e589a5 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -6363,7 +6363,7 @@
"feature_flag": "warehouse-multi-destination"
},
"external-data-destinations-create": {
- "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",
+ "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres takes database and schema; Redshift takes database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",
"category": "Warehouse destinations",
"feature": "warehouse_sources",
"summary": "Create a warehouse destination",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index b5ad174b0eca..66c6fe898196 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -6496,7 +6496,7 @@
"feature_flag": "warehouse-multi-destination"
},
"external-data-destinations-create": {
- "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres and Redshift take database and schema; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",
+ "description": "Create a place where warehouse sources write synced rows, from an integration that holds the credentials. Get the integration id with external-data-destinations-connect-link first. Pass type, a name, the integration id and the target config for the type: Postgres takes database and schema; Redshift takes database and schema, plus host and optional port (default 5439) when the integration uses an IAM role or access keys instead of a username and password; Snowflake takes database, schema, warehouse and optional role; BigQuery takes dataset and optional project; Databricks takes http_path, catalog, schema and volume; S3 takes bucket, region and optional prefix and compression; Azure Blob takes container_name and optional prefix and compression. Every type except Redshift tests the connection on create, and a failed test returns a 400 that names the field to fix. Redshift has no test, so a wrong target only shows up when the first sync fails. The target cannot change after creation, so confirm it with the user first. A new destination receives nothing until a source is attached with external-data-destinations-add-sources.",
"category": "Warehouse destinations",
"feature": "warehouse_sources",
"summary": "Create a warehouse destination",