Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): accept only integrations for redshift copy credentials (#129502)

Tom Owers2fe52f869e92
diff --git a/frontend/src/scenes/data-pipelines/batch-exports/batchExportConfigFormLogic.test.ts b/frontend/src/scenes/data-pipelines/batch-exports/batchExportConfigFormLogic.test.ts
index b200557ac1a7..0ef6b308087b 100644
--- a/frontend/src/scenes/data-pipelines/batch-exports/batchExportConfigFormLogic.test.ts
+++ b/frontend/src/scenes/data-pipelines/batch-exports/batchExportConfigFormLogic.test.ts
@@ -235,50 +235,13 @@ const REDSHIFT_COPY_IAM_BATCH_EXPORT = fixture('fixture-redshift-copy-iam', 'Red
s3_key_prefix: 'rs/copy/',
region_name: 'us-east-1',
authorization: 'arn:aws:iam::123:role/rs-copy',
- bucket_credentials: {
- aws_access_key_id: 'AKIA-bucket',
- aws_secret_access_key: 'bucket-secret',
- },
+ bucket_credentials: 31,
},
exclude_events: [],
include_events: [],
} as any,
})
-const REDSHIFT_COPY_CREDENTIALS_BATCH_EXPORT = fixture(
- 'fixture-redshift-copy-creds',
- 'Redshift COPY Credentials Export',
- {
- type: 'Redshift',
- config: {
- user: 'rs-user',
- password: 'rs-pass',
- host: 'rs-host',
- port: 5439,
- database: 'rs-db',
- schema: 'public',
- table_name: 'events',
- properties_data_type: 'SUPER' as any,
- mode: 'COPY',
- copy_inputs: {
- s3_bucket: 'rs-staging',
- s3_key_prefix: 'rs/copy/',
- region_name: 'us-east-1',
- authorization: {
- aws_access_key_id: 'AKIA-auth',
- aws_secret_access_key: 'auth-secret',
- },
- bucket_credentials: {
- aws_access_key_id: 'AKIA-bucket',
- aws_secret_access_key: 'bucket-secret',
- },
- },
- exclude_events: [],
- include_events: [],
- } as any,
- }
-)
-
const HTTP_BATCH_EXPORT = fixture('fixture-http', 'HTTP Export', {
type: 'HTTP',
config: {
@@ -381,6 +344,31 @@ const REDSHIFT_COPY_INTEGRATIONS_BATCH_EXPORT = fixture(
}
)
+const REDSHIFT_COPY_LEGACY_BATCH_EXPORTS = [
+ { name: 'both', bucketCredentials: {}, authorization: {} },
+ { name: 'bucket', bucketCredentials: {}, authorization: 'arn:aws:iam::000000000000:role/test-copy' },
+ { name: 'authorization', bucketCredentials: S3_STAGING_INTEGRATION.id, authorization: {} },
+ {
+ name: 'unstripped',
+ bucketCredentials: { aws_access_key_id: 'test-bucket-key', aws_secret_access_key: 'test-bucket-secret' },
+ authorization: { aws_access_key_id: 'test-reader-key', aws_secret_access_key: 'test-reader-secret' },
+ },
+].map(({ name, bucketCredentials, authorization }) =>
+ fixture(`fixture-redshift-legacy-${name}`, `Legacy Redshift ${name}`, {
+ ...REDSHIFT_COPY_INTEGRATIONS_BATCH_EXPORT.destination,
+ config: {
+ ...REDSHIFT_COPY_INTEGRATIONS_BATCH_EXPORT.destination.config,
+ copy_inputs: {
+ s3_bucket: 'rs-staging',
+ s3_key_prefix: 'rs/copy/',
+ region_name: 'us-east-1',
+ bucket_credentials: bucketCredentials,
+ authorization,
+ },
+ },
+ } as any)
+)
+
const HOGQL_BATCH_EXPORT: BatchExportConfiguration = {
...fixture('test-hogql-id', 'HogQL Export', { type: 'AwsS3', integration: 31, config: AWS_S3_CONFIG }),
model: 'hogql',
@@ -403,7 +391,7 @@ const ALL_BATCH_EXPORTS: BatchExportConfiguration[] = [
REDSHIFT_INSERT_BATCH_EXPORT,
REDSHIFT_NO_MODE_BATCH_EXPORT,
REDSHIFT_COPY_IAM_BATCH_EXPORT,
- REDSHIFT_COPY_CREDENTIALS_BATCH_EXPORT,
+ ...REDSHIFT_COPY_LEGACY_BATCH_EXPORTS,
REDSHIFT_COPY_INTEGRATIONS_BATCH_EXPORT,
HTTP_BATCH_EXPORT,
DATABRICKS_BATCH_EXPORT,
@@ -844,6 +832,76 @@ describe('batchExportConfigFormLogic', () => {
})
})
+ describe('legacy Redshift COPY credentials', () => {
+ it.each(REDSHIFT_COPY_LEGACY_BATCH_EXPORTS)('omits stored keys when saving $name', async (fixture) => {
+ await initLogic({ service: null, id: fixture.id })
+ logic.actions.setConfigurationValue('table_name', 'updated_events')
+
+ await expectLogic(logic, () => {
+ logic.actions.submitConfiguration()
+ })
+ .toDispatchActions(['submitConfiguration', 'updateBatchExportConfigSuccess'])
+ .toFinishAllListeners()
+
+ const config = patchBodiesById[fixture.id].destination.config
+ expect(config.table_name).toBe('updated_events')
+ const storedCopyInputs = (fixture.destination.config as any).copy_inputs
+ const expectedCopyInputs = { ...storedCopyInputs }
+ for (const field of ['bucket_credentials', 'authorization']) {
+ if (typeof expectedCopyInputs[field] === 'object') {
+ delete expectedCopyInputs[field]
+ }
+ }
+ expect(config.copy_inputs).toEqual(expectedCopyInputs)
+ expect(JSON.stringify(config)).not.toContain('aws_access_key_id')
+ expect(JSON.stringify(config)).not.toContain('aws_secret_access_key')
+ expect(Object.keys(config).filter((key) => key.startsWith('redshift_'))).toEqual([])
+ })
+
+ it.each([
+ { mode: undefined, expectedAuthorization: undefined },
+ { mode: 'SameConnection', expectedAuthorization: S3_STAGING_INTEGRATION.id },
+ { mode: 'DifferentConnection', expectedAuthorization: S3_READER_INTEGRATION.id },
+ { mode: 'IAMRole', expectedAuthorization: 'arn:aws:iam::000000000000:role/test-copy' },
+ ])('replaces selected legacy credentials with $mode', async ({ mode, expectedAuthorization }) => {
+ const fixture = REDSHIFT_COPY_LEGACY_BATCH_EXPORTS[0]
+ await initLogic({ service: null, id: fixture.id })
+ logic.actions.setConfigurationValues({
+ redshift_s3_integration_id: S3_STAGING_INTEGRATION.id,
+ authorization_mode: mode,
+ redshift_authorization_integration_id: S3_READER_INTEGRATION.id,
+ redshift_iam_role: 'arn:aws:iam::000000000000:role/test-copy',
+ })
+
+ await expectLogic(logic, () => {
+ logic.actions.submitConfiguration()
+ })
+ .toDispatchActions(['submitConfiguration', 'updateBatchExportConfigSuccess'])
+ .toFinishAllListeners()
+
+ const copyInputs = patchBodiesById[fixture.id].destination.config.copy_inputs
+ expect(copyInputs).toEqual({
+ s3_bucket: 'rs-staging',
+ s3_key_prefix: 'rs/copy/',
+ region_name: 'us-east-1',
+ bucket_credentials: S3_STAGING_INTEGRATION.id,
+ ...(expectedAuthorization === undefined ? {} : { authorization: expectedAuthorization }),
+ })
+ })
+
+ it('requires a connection when legacy authorization changes to the same connection', async () => {
+ const fixture = REDSHIFT_COPY_LEGACY_BATCH_EXPORTS[0]
+ await initLogic({ service: null, id: fixture.id })
+ logic.actions.setConfigurationValue('authorization_mode', 'SameConnection')
+
+ logic.actions.submitConfiguration()
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(logic.values.configurationErrors.redshift_s3_integration_id).toBe('This field is required')
+ expect(patchBodiesById[fixture.id]).toBeUndefined()
+ })
+ })
+
describe('Redshift export with no stored mode', () => {
// Exports predating the COPY option store no `mode`. Without a default the Command select
// renders blank, so the user has to guess which value reproduces what the export already
@@ -1359,7 +1417,6 @@ describe('batchExportConfigFormLogic', () => {
{ name: 'Snowflake', fixture: SNOWFLAKE_BATCH_EXPORT },
{ name: 'Redshift (INSERT)', fixture: REDSHIFT_INSERT_BATCH_EXPORT },
{ name: 'Redshift (COPY + IAM)', fixture: REDSHIFT_COPY_IAM_BATCH_EXPORT },
- { name: 'Redshift (COPY + Credentials)', fixture: REDSHIFT_COPY_CREDENTIALS_BATCH_EXPORT },
{ name: 'Redshift (COPY + integrations)', fixture: REDSHIFT_COPY_INTEGRATIONS_BATCH_EXPORT },
{ name: 'HTTP', fixture: HTTP_BATCH_EXPORT },
{ name: 'Databricks', fixture: DATABRICKS_BATCH_EXPORT },
diff --git a/frontend/src/scenes/data-pipelines/batch-exports/destinations/redshift.tsx b/frontend/src/scenes/data-pipelines/batch-exports/destinations/redshift.tsx
index e3fffd1224f2..d8af1827b455 100644
--- a/frontend/src/scenes/data-pipelines/batch-exports/destinations/redshift.tsx
+++ b/frontend/src/scenes/data-pipelines/batch-exports/destinations/redshift.tsx
@@ -17,22 +17,9 @@ import type { DestinationDefinition } from './types'
// The two COPY credentials do different jobs: bucket_credentials is what PostHog writes the staged
// files with, authorization is what Redshift reads them back with. Either can be an aws-s3
// Integration id; authorization can also be the ARN of a role attached to the cluster.
-type AuthorizationMode = 'SameConnection' | 'DifferentConnection' | 'IAMRole' | 'Credentials'
+type AuthorizationMode = 'SameConnection' | 'DifferentConnection' | 'IAMRole'
-function buildBucketCredentials(formValues: Record<string, any>): number | Record<string, string> | undefined {
- if (typeof formValues.redshift_s3_integration_id === 'number') {
- return formValues.redshift_s3_integration_id
- }
- if (formValues.redshift_s3_bucket_aws_access_key_id && formValues.redshift_s3_bucket_aws_secret_access_key) {
- return {
- aws_access_key_id: formValues.redshift_s3_bucket_aws_access_key_id,
- aws_secret_access_key: formValues.redshift_s3_bucket_aws_secret_access_key,
- }
- }
- return undefined
-}
-
-function buildAuthorization(formValues: Record<string, any>): number | string | Record<string, string> | undefined {
+function buildAuthorization(formValues: Record<string, any>): number | string | undefined {
switch (formValues.authorization_mode as AuthorizationMode) {
case 'SameConnection':
return typeof formValues.redshift_s3_integration_id === 'number'
@@ -44,13 +31,8 @@ function buildAuthorization(formValues: Record<string, any>): number | string |
: undefined
case 'IAMRole':
return formValues.redshift_iam_role || undefined
- case 'Credentials':
- return formValues.redshift_aws_access_key_id && formValues.redshift_aws_secret_access_key
- ? {
- aws_access_key_id: formValues.redshift_aws_access_key_id,
- aws_secret_access_key: formValues.redshift_aws_secret_access_key,
- }
- : undefined
+ default:
+ return undefined
}
}
@@ -64,9 +46,8 @@ function buildCopyInputs(formValues: Record<string, any>): Record<string, any> {
region_name: formValues.redshift_s3_bucket_region_name,
}
- const bucketCredentials = buildBucketCredentials(formValues)
- if (bucketCredentials !== undefined) {
- copyInputs.bucket_credentials = bucketCredentials
+ if (typeof formValues.redshift_s3_integration_id === 'number') {
+ copyInputs.bucket_credentials = formValues.redshift_s3_integration_id
}
const authorization = buildAuthorization(formValues)
@@ -88,12 +69,9 @@ const REDSHIFT_FORM_ONLY_FIELDS = [
'redshift_s3_bucket_region_name',
'redshift_s3_integration_id',
'redshift_s3_credentials_inline',
- 'redshift_s3_bucket_aws_access_key_id',
- 'redshift_s3_bucket_aws_secret_access_key',
+ 'redshift_authorization_inline',
'redshift_authorization_integration_id',
'redshift_iam_role',
- 'redshift_aws_access_key_id',
- 'redshift_aws_secret_access_key',
] as const
// New exports must store their credentials in Integrations. Exports created before integrations
@@ -102,10 +80,6 @@ function usesConnection(isNew: boolean, formValues: Record<string, any>): boolea
return isNew || !!formValues.integration_id
}
-function usesS3Connection(isNew: boolean, formValues: Record<string, any>): boolean {
- return isNew || !formValues.redshift_s3_credentials_inline
-}
-
// Only a plain Redshift connection stores its own endpoint. AWS ones mint temporary credentials for
// a cluster endpoint that stays on the export, so the export has to carry the host itself.
function connectionCarriesHost(connection: IntegrationType | null | undefined): boolean {
@@ -141,7 +115,11 @@ export const redshiftDefinition: DestinationDefinition = {
// the bucket root, so leaving it blank is a valid choice, not an error.
fields.push('redshift_s3_bucket', 'redshift_s3_bucket_region_name')
- if (usesS3Connection(isNew, formValues)) {
+ if (
+ isNew ||
+ !formValues.redshift_s3_credentials_inline ||
+ formValues.authorization_mode === 'SameConnection'
+ ) {
fields.push('redshift_s3_integration_id')
}
@@ -205,12 +183,8 @@ export const redshiftDefinition: DestinationDefinition = {
result.redshift_s3_key_prefix = copyInputs.s3_key_prefix
result.redshift_s3_bucket_region_name = copyInputs.region_name
result.redshift_s3_integration_id = undefined
- result.redshift_s3_bucket_aws_access_key_id = undefined
- result.redshift_s3_bucket_aws_secret_access_key = undefined
result.redshift_authorization_integration_id = undefined
result.redshift_iam_role = undefined
- result.redshift_aws_access_key_id = undefined
- result.redshift_aws_secret_access_key = undefined
const bucketCredentials = copyInputs.bucket_credentials
// Inline credentials come back as an empty object once the API strips them, so the shape of
@@ -218,22 +192,18 @@ export const redshiftDefinition: DestinationDefinition = {
result.redshift_s3_credentials_inline = bucketCredentials !== undefined && typeof bucketCredentials !== 'number'
if (typeof bucketCredentials === 'number') {
result.redshift_s3_integration_id = bucketCredentials
- } else if (bucketCredentials) {
- result.redshift_s3_bucket_aws_access_key_id = bucketCredentials.aws_access_key_id
- result.redshift_s3_bucket_aws_secret_access_key = bucketCredentials.aws_secret_access_key
}
const authorization = copyInputs.authorization
+ result.redshift_authorization_inline = !!authorization && typeof authorization === 'object'
if (typeof authorization === 'number') {
result.redshift_authorization_integration_id = authorization
result.authorization_mode = authorization === bucketCredentials ? 'SameConnection' : 'DifferentConnection'
} else if (typeof authorization === 'string') {
result.authorization_mode = 'IAMRole'
result.redshift_iam_role = authorization
- } else if (authorization) {
- result.authorization_mode = 'Credentials'
- result.redshift_aws_access_key_id = authorization.aws_access_key_id
- result.redshift_aws_secret_access_key = authorization.aws_secret_access_key
+ } else if (result.redshift_authorization_inline) {
+ result.authorization_mode = undefined
}
return result
@@ -242,21 +212,16 @@ export const redshiftDefinition: DestinationDefinition = {
eventTableExtraFields: { ...PERSON_EVENT_FIELDS },
Fields: function RedshiftFields({ isNew, formValues, selectedIntegration }) {
const useConnection = usesConnection(isNew, formValues)
- const useS3Connection = usesS3Connection(isNew, formValues)
// Until a connection is picked we cannot tell which kind it is, so we keep asking.
const showHost = !useConnection || !connectionCarriesHost(selectedIntegration)
- // "Credentials" is never offered to a new export. It stays reachable while an export still
- // stores inline keys, so an accidental switch away from it can be undone.
- const allowInlineAuthorizationKeys = !useS3Connection || formValues.authorization_mode === 'Credentials'
const authorizationOptions = [
- ...(useS3Connection ? [{ value: 'SameConnection', label: 'Use the S3 connection above' }] : []),
+ { value: 'SameConnection', label: 'Use the S3 connection above' },
{
value: 'DifferentConnection',
- label: useS3Connection ? 'Use a different S3 connection' : 'Use an S3 connection',
+ label: 'Use a different S3 connection',
},
{ value: 'IAMRole', label: 'Use an IAM role attached to the cluster' },
- ...(allowInlineAuthorizationKeys ? [{ value: 'Credentials', label: 'Use AWS access keys' }] : []),
]
return (
@@ -372,54 +337,29 @@ export const redshiftDefinition: DestinationDefinition = {
<LemonInput placeholder="e.g. /posthog-copy-files" />
</LemonField>
- {useS3Connection ? (
- <LemonField
- name="redshift_s3_integration_id"
- label="S3 connection"
- info="The credentials PostHog uses to stage export files in the bucket."
- >
- {({ value, onChange }) => (
- <IntegrationChoice integration="aws-s3" value={value} onChange={onChange} />
- )}
- </LemonField>
- ) : (
- <>
- <LemonBanner type="warning">
- PostHog is moving the S3 staging credentials to saved connections. This export will
- be migrated automatically.
- </LemonBanner>
-
- <div className="flex gap-4">
- <LemonField
- name="redshift_s3_bucket_aws_access_key_id"
- label="AWS Access Key ID"
- className="flex-1"
- >
- <LemonInput placeholder="Leave unchanged" autoComplete="off" />
- </LemonField>
-
- <LemonField
- name="redshift_s3_bucket_aws_secret_access_key"
- label="AWS Secret Access Key"
- className="flex-1"
- >
- <LemonInput
- placeholder="Leave unchanged"
- type="password"
- autoComplete="new-password"
- />
- </LemonField>
- </div>
- </>
+ {(formValues.redshift_s3_credentials_inline || formValues.redshift_authorization_inline) && (
+ <LemonBanner type="info">
+ This export stores AWS access keys. Choose an S3 connection to replace the stored keys.
+ </LemonBanner>
)}
+ <LemonField
+ name="redshift_s3_integration_id"
+ label="S3 connection"
+ info="The credentials PostHog uses to stage export files in the bucket."
+ >
+ {({ value, onChange }) => (
+ <IntegrationChoice integration="aws-s3" value={value} onChange={onChange} />
+ )}
+ </LemonField>
+
<LemonField
name="authorization_mode"
label="How Redshift reads staged files"
className="flex-1"
info="PostHog writes the staged files. Redshift then needs its own access to read them back."
>
- <LemonSelect options={authorizationOptions} />
+ <LemonSelect options={authorizationOptions} placeholder="Keep stored access keys" />
</LemonField>
{formValues.authorization_mode === 'DifferentConnection' && (
@@ -431,34 +371,10 @@ export const redshiftDefinition: DestinationDefinition = {
)}
{formValues.authorization_mode === 'IAMRole' && (
- <LemonField name="redshift_iam_role" label="IAM Role ARN" className="flex-1">
+ <LemonField name="redshift_iam_role" label="IAM role ARN" className="flex-1">
<LemonInput placeholder="e.g. arn:aws:iam::<aws-account-id>:role/<role-name>" />
</LemonField>
)}
-
- {formValues.authorization_mode === 'Credentials' && (
- <div className="flex gap-4">
- <LemonField
- name="redshift_aws_access_key_id"
- label="AWS Access Key ID"
- className="flex-1"
- >
- <LemonInput placeholder="Leave unchanged" autoComplete="off" />
- </LemonField>
-
- <LemonField
- name="redshift_aws_secret_access_key"
- label="AWS Secret Access Key"
- className="flex-1"
- >
- <LemonInput
- placeholder="Leave unchanged"
- type="password"
- autoComplete="new-password"
- />
- </LemonField>
- </div>
- )}
</>
)}
</>
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 83aedbe9db6c..860f9e832728 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6895,7 +6895,7 @@ export type BatchExportServiceRedshift = {
properties_data_type: 'varchar' | 'super'
mode: 'COPY' | 'INSERT'
// Form-only: derived from the shape of copy_inputs.authorization, never sent to the API.
- authorization_mode: 'SameConnection' | 'DifferentConnection' | 'IAMRole' | 'Credentials'
+ authorization_mode: 'SameConnection' | 'DifferentConnection' | 'IAMRole'
copy_inputs: BatchExportServiceRedshiftCopyInputs | null
exclude_events: string[]
include_events: string[]
diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.py
index 31d05f42e379..bf3d55391406 100644
--- a/products/batch_exports/backend/logic/exports.py
+++ b/products/batch_exports/backend/logic/exports.py
@@ -23,6 +23,7 @@ from posthog.models.integration import (
SnowflakeIntegration,
SnowflakeIntegrationError,
)
+from posthog.models.integration.aws import AWS_ROLE_ARN_RE
from posthog.paging import Page, PageRequest
from posthog.queryset_paging import select_page
from posthog.security.url_validation import (
@@ -187,22 +188,63 @@ def _check_file_format(merged_config: dict[str, typing.Any], supported_compressi
)
+_AWS_CREDENTIAL_KEYS = frozenset({"aws_access_key_id", "aws_secret_access_key"})
+
+
+def _reject_new_inline_copy_keys(
+ instance: BatchExport | None,
+ merged_config: dict[str, typing.Any],
+ existing_config: dict[str, typing.Any],
+) -> None:
+ """Reject new or changed inline AWS keys in Redshift `copy_inputs`, whatever the mode.
+
+ Runs for every mode so keys cannot be stored under `INSERT` and later switched to `COPY`
+ as "unchanged" legacy credentials.
+ """
+ copy_inputs = merged_config.get("copy_inputs")
+ if copy_inputs is None:
+ return
+ if not isinstance(copy_inputs, dict):
+ # The workflow JSON-decodes string `copy_inputs`, so a string could carry inline keys.
+ if instance is None or copy_inputs != existing_config.get("copy_inputs"):
+ raise contracts.InvalidBatchExportDestinationError("'copy_inputs' must be an object.")
+ return
+
+ existing_copy_inputs = existing_config.get("copy_inputs") or {}
+ if not isinstance(existing_copy_inputs, dict):
+ existing_copy_inputs = {}
+
+ # TODO: remove legacy comparisons after all stored COPY keys move to integrations.
+ for field_name in ("bucket_credentials", "authorization"):
+ value = copy_inputs.get(field_name)
+ if (
+ isinstance(value, dict)
+ and bool(_AWS_CREDENTIAL_KEYS & value.keys())
+ and (instance is None or value != existing_copy_inputs.get(field_name))
+ ):
+ allowed_source = "an AWS S3 integration"
+ if field_name == "authorization":
+ allowed_source += " or an IAM role ARN"
+ raise contracts.InvalidBatchExportDestinationError(
+ f"'{field_name}' must be {allowed_source}. Inline AWS access keys are no longer accepted."
+ )
+
+
def _check_redshift_copy_inputs(
team_id: int,
- instance: BatchExport | None,
merged_config: dict[str, typing.Any],
existing_config: dict[str, typing.Any],
) -> None:
copy_inputs = merged_config.get("copy_inputs", {})
if not copy_inputs:
raise contracts.InvalidBatchExportDestinationError("Missing required inputs for 'COPY'")
+ if not isinstance(copy_inputs, dict):
+ raise contracts.InvalidBatchExportDestinationError("'copy_inputs' must be an object.")
required_inputs = {"s3_bucket", "region_name", "authorization", "bucket_credentials"}
if required_inputs - copy_inputs.keys():
raise contracts.InvalidBatchExportDestinationError("Missing required input for 'COPY'")
- credential_keys = {"aws_access_key_id", "aws_secret_access_key"}
-
bucket_credentials = copy_inputs["bucket_credentials"]
bucket_integration_id = _coerce_integration_id(bucket_credentials)
authorization = copy_inputs.get("authorization")
@@ -214,19 +256,6 @@ def _check_redshift_copy_inputs(
("bucket_credentials", bucket_integration_id),
("authorization", authorization_integration_id),
):
- # Sticky like the top-level integration: COPY credentials that reference an
- # integration cannot drop back to inline values.
- # TODO: remove this guard once integrations are mandatory for Redshift and
- # inline credentials are gone.
- if (
- instance is not None
- and copy_integration_id is None
- and _coerce_integration_id(existing_copy_inputs.get(field_name)) is not None
- ):
- raise contracts.InvalidBatchExportDestinationError(
- f"Cannot switch '{field_name}' from an integration to inline credentials. "
- "Send a different integration ID to swap, or omit the field to keep the current one."
- )
if copy_integration_id is None:
continue
# These ids live inside `config` rather than the team-scoped `integration`
@@ -237,20 +266,31 @@ def _check_redshift_copy_inputs(
)
if bucket_integration_id is None and (
- not isinstance(bucket_credentials, dict) or credential_keys - bucket_credentials.keys()
+ not isinstance(bucket_credentials, dict) or _AWS_CREDENTIAL_KEYS - bucket_credentials.keys()
):
raise contracts.InvalidBatchExportDestinationError("Missing required bucket credentials for 'COPY'")
if authorization_integration_id is None:
if isinstance(authorization, dict):
- if credential_keys - authorization.keys():
+ if _AWS_CREDENTIAL_KEYS - authorization.keys():
raise contracts.InvalidBatchExportDestinationError("Missing required credentials for 'COPY'")
elif isinstance(authorization, str):
if not authorization.strip():
raise contracts.InvalidBatchExportDestinationError("Missing required IAM role for 'COPY'")
+ # Only changed values are checked, so unrelated edits to existing exports keep saving.
+ # `default` is valid: the COPY statement runs `IAM_ROLE default`.
+ if (
+ authorization != existing_copy_inputs.get("authorization")
+ and authorization != "default"
+ and not AWS_ROLE_ARN_RE.match(authorization)
+ ):
+ raise contracts.InvalidBatchExportDestinationError(
+ "Authorization for 'COPY' must be an IAM role ARN (arn:aws:iam::<account-id>:role/<name>), "
+ "'default', or an AWS S3 integration ID."
+ )
else:
raise contracts.InvalidBatchExportDestinationError(
- "Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."
+ "Authorization for 'COPY' must be an IAM role ARN or an AWS S3 integration ID."
)
@@ -456,7 +496,8 @@ def validate_destination(
)
if merged_config.get("mode") == "COPY":
- _check_redshift_copy_inputs(team_id, instance, merged_config, existing_config)
+ _check_redshift_copy_inputs(team_id, merged_config, existing_config)
+ _reject_new_inline_copy_keys(instance, merged_config, existing_config)
if destination_type in (
BatchExportDestinationType.POSTGRES,
diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
index 518ad584e930..4136958834fd 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
@@ -241,6 +241,7 @@ class SnowflakeDestinationConfigSerializer(serializers.Serializer):
_AWS_CREDENTIALS_SCHEMA = {
"type": "object",
+ "description": "Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.",
"properties": {
"aws_access_key_id": {"type": "string"},
"aws_secret_access_key": {"type": "string"},
@@ -258,7 +259,7 @@ _AWS_CREDENTIALS_SCHEMA = {
}
)
class RedshiftCopyBucketCredentialsField(serializers.JSONField):
- """Inline AWS credentials or the id of an aws-s3-kind Integration."""
+ """AWS S3 integration ID, or unchanged legacy AWS access keys."""
pass
@@ -273,7 +274,7 @@ class RedshiftCopyBucketCredentialsField(serializers.JSONField):
}
)
class RedshiftCopyAuthorizationField(serializers.JSONField):
- """IAM role ARN, inline AWS credentials, or the id of an aws-s3-kind Integration."""
+ """IAM role ARN, AWS S3 integration ID, or unchanged legacy AWS access keys."""
pass
@@ -286,14 +287,14 @@ class RedshiftCopyInputsSerializer(serializers.Serializer):
s3_key_prefix = serializers.CharField(help_text="Key prefix for staged files in the S3 bucket.")
authorization = RedshiftCopyAuthorizationField(
help_text=(
- "Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached "
- "to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration."
+ "Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. "
+ "Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
),
)
bucket_credentials = RedshiftCopyBucketCredentialsField(
help_text=(
- "Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an "
- "aws-s3-kind Integration."
+ "AWS S3 integration ID used to stage files in the S3 bucket. "
+ "Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
),
)
diff --git a/products/batch_exports/backend/tests/api/test_create_redshift.py b/products/batch_exports/backend/tests/api/test_create_redshift.py
index f0eb06786ec7..3867deb19316 100644
--- a/products/batch_exports/backend/tests/api/test_create_redshift.py
+++ b/products/batch_exports/backend/tests/api/test_create_redshift.py
@@ -1,3 +1,6 @@
+import json
+from typing import Any
+
import pytest
from django.test import override_settings
@@ -5,7 +8,8 @@ from django.test.client import Client as HttpClient
from rest_framework import status
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationRecord
+from posthog.models import Team, User
from posthog.security.url_validation import INVALID_HOST_MESSAGE, UNREACHABLE_HOST_MESSAGE
from products.batch_exports.backend.tests.api.fixtures import create_organization, create_team, create_user
@@ -36,7 +40,7 @@ pytestmark = [
"s3_bucket": "my-production-s3-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
"authorization": "default",
},
status.HTTP_201_CREATED,
@@ -47,8 +51,8 @@ pytestmark = [
"s3_bucket": "my-production-s3-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
- "authorization": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
+ "authorization": "integration",
},
status.HTTP_201_CREATED,
),
@@ -58,7 +62,7 @@ pytestmark = [
{
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
"authorization": "default",
},
status.HTTP_400_BAD_REQUEST,
@@ -69,7 +73,7 @@ pytestmark = [
{
"s3_bucket": "my-production-s3-bucket",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
"authorization": "default",
},
status.HTTP_400_BAD_REQUEST,
@@ -105,7 +109,7 @@ pytestmark = [
"s3_bucket": "my-production-s3-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
"authorization": {},
},
status.HTTP_400_BAD_REQUEST,
@@ -117,7 +121,7 @@ pytestmark = [
"s3_bucket": "my-production-s3-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": "integration",
"authorization": "",
},
status.HTTP_400_BAD_REQUEST,
@@ -125,11 +129,21 @@ pytestmark = [
],
)
def test_create_redshift_batch_export_validates_copy_inputs(
- client: HttpClient, mode, copy_inputs, expected_status, temporal, organization, team, user, aws_redshift_integration
-):
- """Test creating a BatchExport with Redshift destination validates inputs for 'COPY'."""
-
- destination_data = {
+ client: HttpClient,
+ mode: str,
+ copy_inputs: dict[str, Any] | None,
+ expected_status: int,
+ team: Team,
+ user: User,
+ aws_redshift_integration: IntegrationRecord,
+ aws_s3_integration: IntegrationRecord,
+) -> None:
+ if copy_inputs:
+ copy_inputs = {
+ key: aws_s3_integration.id if value == "integration" else value for key, value in copy_inputs.items()
+ }
+
+ destination_data: dict[str, Any] = {
"type": "Redshift",
"config": {
"database": "my-db",
@@ -139,7 +153,7 @@ def test_create_redshift_batch_export_validates_copy_inputs(
"mode": mode,
"copy_inputs": copy_inputs,
},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
}
batch_export_data = {
@@ -162,15 +176,18 @@ def test_create_redshift_batch_export_validates_copy_inputs(
assert "Missing required" in response.json()["detail"]
[email protected]("authorization", [["arn:aws:iam::123456789012:role/my-role"], True, 1.5])
+ "authorization", [["arn:aws:iam::123456789012:role/my-role"], True, 1.5, "not-an-arn", "my-role"]
+)
def test_create_redshift_batch_export_rejects_invalid_authorization_type(
- client: HttpClient, authorization, temporal, organization, team, user, aws_redshift_integration
-):
- """Test 'authorization' values that are neither credentials, a role, nor an id are a 400.
-
- These used to pass validation and crash with a TypeError when the export synced.
- """
- destination_data = {
+ client: HttpClient,
+ authorization: object,
+ team: Team,
+ user: User,
+ aws_redshift_integration: IntegrationRecord,
+ aws_s3_integration: IntegrationRecord,
+) -> None:
+ destination_data: dict[str, Any] = {
"type": "Redshift",
"config": {
"database": "my-db",
@@ -180,11 +197,11 @@ def test_create_redshift_batch_export_rejects_invalid_authorization_type(
"s3_bucket": "my-production-s3-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
+ "bucket_credentials": aws_s3_integration.id,
"authorization": authorization,
},
},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
}
batch_export_data = {
@@ -225,7 +242,7 @@ def test_create_redshift_batch_export_fails_with_invalid_host(
comes from the linked Integration rather than from inline config.
"""
- destination_data = {
+ destination_data: dict[str, Any] = {
"type": "Redshift",
"config": {
"database": "my-db",
@@ -233,7 +250,7 @@ def test_create_redshift_batch_export_fails_with_invalid_host(
"schema": "public",
"table_name": "my_events",
},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
}
batch_export_data = {
@@ -268,7 +285,7 @@ def test_create_redshift_batch_export_with_aws_integration(
# No user/password inline: credentials come from the integration. Only the
# cluster endpoint stays in the config.
"config": {"database": "my-db", "host": "8.8.8.8"},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
},
}
@@ -279,13 +296,12 @@ def test_create_redshift_batch_export_with_aws_integration(
def test_create_redshift_batch_export_with_plain_integration(client: HttpClient, temporal, organization, team, user):
- integration = Integration.objects.create(
- team=team,
- kind=Integration.IntegrationKind.AWS_REDSHIFT,
- integration_id="prod-redshift-plain",
+ integration = IntegrationManager.aws_redshift.credentials.create(
+ team_id=team.id,
+ external_id="prod-redshift-plain",
config={"host": "8.8.8.8", "port": 5439, "user": "posthog", "ssl_mode": "require"},
sensitive_config={"password": "very-secret"},
- created_by=user,
+ created_by_id=user.id,
)
batch_export_data = {
@@ -295,7 +311,7 @@ def test_create_redshift_batch_export_with_plain_integration(client: HttpClient,
"type": "Redshift",
# Plain Redshift integrations store the host themselves, so none is needed here.
"config": {"database": "my-db"},
- "integration": integration.pk,
+ "integration": integration.id,
},
}
@@ -328,9 +344,9 @@ def test_create_redshift_batch_export_validates_integration(
integration_pk = None
if failure == "wrong_kind":
- integration_pk = aws_s3_integration.pk
+ integration_pk = aws_s3_integration.id
elif failure == "aws_integration_without_host":
- integration_pk = aws_redshift_integration.pk
+ integration_pk = aws_redshift_integration.id
destination_data: dict = {"type": "Redshift", "config": config}
if integration_pk is not None:
@@ -351,62 +367,79 @@ def test_create_redshift_batch_export_validates_integration(
@pytest.mark.parametrize(
"copy_credential_source",
- ["own_integration", "other_team_integration", "wrong_kind_integration", "nonexistent_integration"],
+ [
+ "own_integration",
+ "iam_role",
+ "inline_bucket_credentials",
+ "inline_authorization",
+ "other_team_integration",
+ "wrong_kind_integration",
+ "nonexistent_integration",
+ ],
)
def test_create_redshift_batch_export_validates_copy_integration_ids(
client: HttpClient,
- copy_credential_source,
- temporal,
- organization,
- team,
- user,
- aws_s3_integration,
- aws_redshift_integration,
-):
+ copy_credential_source: str,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+) -> None:
# These ids live inside `config` rather than the team-scoped `integration` field, so
# tenant scoping is enforced by validation: a foreign or wrong-kind id must 400, not 500.
expected_status: int
- if copy_credential_source == "own_integration":
- integration_id = aws_s3_integration.pk
+ if copy_credential_source in {"own_integration", "iam_role"}:
+ integration_id = aws_s3_integration.id
expected_status = status.HTTP_201_CREATED
+ elif copy_credential_source.startswith("inline_"):
+ integration_id = aws_s3_integration.id
+ expected_status = status.HTTP_400_BAD_REQUEST
elif copy_credential_source == "other_team_integration":
other_organization = create_organization("Other Org")
other_team = create_team(other_organization)
- other_user = create_user("[email protected]", "Other User", other_organization)
- other_integration = Integration.objects.create(
- team=other_team,
- kind=Integration.IntegrationKind.AWS_S3,
- integration_id="other-prod-aws",
- config={"name": "other-prod-aws", "aws_account_id": "999999999999"},
- sensitive_config={"aws_access_key_id": "key", "aws_secret_access_key": "secret"},
- created_by=other_user,
+ other_user = create_user("[email protected]", "Other User", other_organization)
+ other_integration = IntegrationManager.aws_s3.credentials.create(
+ team_id=other_team.id,
+ external_id="other-test-aws",
+ config={"name": "other-test-aws"},
+ sensitive_config={"aws_access_key_id": "test-key", "aws_secret_access_key": "test-secret"},
+ created_by_id=other_user.id,
)
- integration_id = other_integration.pk
+ integration_id = other_integration.id
expected_status = status.HTTP_400_BAD_REQUEST
elif copy_credential_source == "wrong_kind_integration":
- integration_id = aws_redshift_integration.pk
+ integration_id = aws_redshift_integration.id
expected_status = status.HTTP_400_BAD_REQUEST
else:
integration_id = 999999
expected_status = status.HTTP_400_BAD_REQUEST
- destination_data = {
+ copy_inputs: dict[str, Any] = {
+ "s3_bucket": "my-production-s3-bucket",
+ "region_name": "us-east-1",
+ "s3_key_prefix": "posthog-events/",
+ "bucket_credentials": integration_id,
+ "authorization": integration_id,
+ }
+ destination_data: dict[str, Any] = {
"type": "Redshift",
"config": {
"database": "my-db",
"host": "localhost",
"mode": "COPY",
- "copy_inputs": {
- "s3_bucket": "my-production-s3-bucket",
- "region_name": "us-east-1",
- "s3_key_prefix": "posthog-events/",
- "bucket_credentials": integration_id,
- "authorization": integration_id,
- },
+ "copy_inputs": copy_inputs,
},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
}
+ if copy_credential_source == "iam_role":
+ copy_inputs["authorization"] = "arn:aws:iam::000000000000:role/test-copy"
+ elif copy_credential_source.startswith("inline_"):
+ copy_inputs[copy_credential_source.removeprefix("inline_")] = {
+ "aws_access_key_id": "test-access-key",
+ "aws_secret_access_key": "test-secret-key",
+ }
+
batch_export_data = {
"name": "my-production-redshift-destination",
"destination": destination_data,
@@ -418,11 +451,96 @@ def test_create_redshift_batch_export_validates_copy_integration_ids(
assert response.status_code == expected_status, response.json()
- if expected_status == status.HTTP_400_BAD_REQUEST:
+ if copy_credential_source.startswith("inline_"):
+ assert "must be an AWS S3 integration" in response.json()["detail"]
+ assert "Inline AWS access keys are no longer accepted" in response.json()["detail"]
+ assert "test-access-key" not in response.json()["detail"]
+ assert "test-secret-key" not in response.json()["detail"]
+ elif expected_status == status.HTTP_400_BAD_REQUEST:
assert "does not reference an AWS S3 integration" in response.json()["detail"]
else:
# EncryptedJSONField stringifies the stored ids; responses must restore them to ints
# to honor the generated types.
batch_export = get_batch_export_ok(client, team.pk, response.json()["id"])
assert batch_export["destination"]["config"]["copy_inputs"]["bucket_credentials"] == integration_id
- assert batch_export["destination"]["config"]["copy_inputs"]["authorization"] == integration_id
+ assert (
+ batch_export["destination"]["config"]["copy_inputs"]["authorization"]
+ == destination_data["config"]["copy_inputs"]["authorization"]
+ )
+
+
[email protected]("field_name", ["bucket_credentials", "authorization"])
+ "credentials",
+ [
+ {"aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key"},
+ {"aws_access_key_id": "test-access-key"},
+ ],
+ ids=["full_pair", "partial"],
+)
+def test_create_redshift_batch_export_rejects_inline_copy_keys_in_insert_mode(
+ client: HttpClient,
+ field_name: str,
+ credentials: dict[str, str],
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+) -> None:
+ copy_inputs: dict[str, Any] = {
+ "s3_bucket": "my-production-s3-bucket",
+ "region_name": "us-east-1",
+ "s3_key_prefix": "posthog-events/",
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": aws_s3_integration.id,
+ }
+ copy_inputs[field_name] = credentials
+ batch_export_data = {
+ "name": "my-production-redshift-destination",
+ "destination": {
+ "type": "Redshift",
+ "config": {"database": "my-db", "host": "localhost", "mode": "INSERT", "copy_inputs": copy_inputs},
+ "integration": aws_redshift_integration.id,
+ },
+ "interval": "hour",
+ }
+
+ client.force_login(user)
+ response = create_batch_export(client, team.pk, batch_export_data)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert "Inline AWS access keys are no longer accepted" in response.json()["detail"]
+
+
[email protected]("mode", ["INSERT", "COPY"])
+def test_create_redshift_batch_export_rejects_json_string_copy_inputs(
+ client: HttpClient,
+ mode: str,
+ team: Team,
+ user: User,
+ aws_redshift_integration: IntegrationRecord,
+) -> None:
+ copy_inputs = json.dumps(
+ {
+ "s3_bucket": "my-production-s3-bucket",
+ "region_name": "us-east-1",
+ "bucket_credentials": {"aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key"},
+ "authorization": "default",
+ }
+ )
+ batch_export_data = {
+ "name": "my-production-redshift-destination",
+ "destination": {
+ "type": "Redshift",
+ "config": {"database": "my-db", "host": "localhost", "mode": mode, "copy_inputs": copy_inputs},
+ "integration": aws_redshift_integration.id,
+ },
+ "interval": "hour",
+ }
+
+ client.force_login(user)
+ response = create_batch_export(client, team.pk, batch_export_data)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert "'copy_inputs' must be an object." in response.json()["detail"]
+ assert "test-secret-key" not in response.json()["detail"]
diff --git a/products/batch_exports/backend/tests/api/test_update_redshift.py b/products/batch_exports/backend/tests/api/test_update_redshift.py
index 5b356ee7aacb..6cfc092fe06d 100644
--- a/products/batch_exports/backend/tests/api/test_update_redshift.py
+++ b/products/batch_exports/backend/tests/api/test_update_redshift.py
@@ -1,11 +1,15 @@
+from typing import Any
+
import pytest
from django.test.client import Client as HttpClient
from rest_framework import status
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationRecord
+from posthog.models import Team, User
+from products.batch_exports.backend.models.batch_export import BatchExport
from products.batch_exports.backend.tests.api.operations import (
create_batch_export_ok,
get_batch_export_ok,
@@ -18,7 +22,9 @@ pytestmark = [
]
-def create_copy_batch_export(client: HttpClient, team, user, copy_inputs: dict, integration_pk: int) -> dict:
+def create_copy_batch_export(
+ client: HttpClient, team: Team, user: User, copy_inputs: dict[str, Any], integration_pk: int
+) -> dict[str, Any]:
destination_data = {
"type": "Redshift",
"config": {
@@ -32,7 +38,7 @@ def create_copy_batch_export(client: HttpClient, team, user, copy_inputs: dict,
"integration": integration_pk,
}
batch_export_data = {
- "name": "my-production-redshift-destination",
+ "name": "test-redshift-export",
"destination": destination_data,
"interval": "hour",
}
@@ -40,168 +46,273 @@ def create_copy_batch_export(client: HttpClient, team, user, copy_inputs: dict,
return create_batch_export_ok(client, team.pk, batch_export_data)
-def test_can_patch_redshift_batch_export(
- client: HttpClient, temporal, organization, team, user, aws_redshift_integration
-):
- """Test we can patch a Redshift batch export preserving credentials."""
- destination_data = {
- "type": "Redshift",
- "config": {
- "database": "my-db",
- "host": "localhost",
- "schema": "public",
- "table_name": "my_events",
- "mode": "COPY",
- "copy_inputs": {
- "s3_bucket": "my-production-s3-bucket",
- "region_name": "us-east-1",
- "s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
- "authorization": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
- },
[email protected]("field_name", ["authorization", "bucket_credentials"])
+ "patch_kind",
+ ["omit", "bucket", "empty_copy_inputs", "empty_credentials", "unchanged", "integration"],
+)
+def test_can_patch_redshift_batch_export_with_legacy_copy_credentials(
+ client: HttpClient,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+ field_name: str,
+ patch_kind: str,
+) -> None:
+ batch_export = create_copy_batch_export(
+ client,
+ team,
+ user,
+ {
+ "s3_bucket": "test-staging-bucket",
+ "region_name": "us-east-1",
+ "s3_key_prefix": "posthog-events/",
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": aws_s3_integration.id,
},
- "integration": aws_redshift_integration.pk,
- }
-
- batch_export_data = {
- "name": "my-production-redshiftn-destination",
- "destination": destination_data,
- "interval": "hour",
- }
+ aws_redshift_integration.id,
+ )
+ destination = (
+ BatchExport.objects.select_related("destination").get(id=batch_export["id"], team_id=team.id).destination
+ )
+ legacy_credentials = {"aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key"}
+ destination.config["copy_inputs"][field_name] = legacy_credentials
+ destination.save()
- client.force_login(user)
+ config: dict[str, Any] = {"table_name": "updated_events"}
+ if patch_kind == "bucket":
+ config["copy_inputs"] = {"s3_bucket": "updated-test-bucket"}
+ elif patch_kind == "empty_copy_inputs":
+ config["copy_inputs"] = {}
+ elif patch_kind == "empty_credentials":
+ config["copy_inputs"] = {field_name: {}}
+ elif patch_kind == "unchanged":
+ config["copy_inputs"] = {field_name: legacy_credentials}
+ elif patch_kind == "integration":
+ config["copy_inputs"] = {field_name: aws_s3_integration.id}
- batch_export = create_batch_export_ok(
+ response = patch_batch_export(
client,
team.pk,
- batch_export_data,
+ batch_export["id"],
+ {"destination": {"type": "Redshift", "config": config, "integration": destination.integration_id}},
)
+ assert response.status_code == status.HTTP_200_OK, response.json()
- # Updates bucket name, leaves everything else untouched. The integration is re-sent because an
- # export that has one cannot drop it, matching what the edit form submits.
- new_destination_data = {
- "type": "Redshift",
- "config": {
- "copy_inputs": {
- "s3_bucket": "my-new-production-s3-bucket",
- },
- },
- "integration": aws_redshift_integration.pk,
- }
+ destination.refresh_from_db()
+ assert destination.config["table_name"] == "updated_events"
+ if patch_kind == "bucket":
+ assert destination.config["copy_inputs"]["s3_bucket"] == "updated-test-bucket"
+ if patch_kind == "integration":
+ assert int(destination.config["copy_inputs"][field_name]) == aws_s3_integration.id
+ batch_export = get_batch_export_ok(client, team.pk, batch_export["id"])
+ assert batch_export["destination"]["config"]["copy_inputs"][field_name] == aws_s3_integration.id
+ else:
+ assert destination.config["copy_inputs"][field_name] == legacy_credentials
- new_batch_export_data = {
- "destination": new_destination_data,
- }
- response = patch_batch_export(client, team.pk, batch_export["id"], new_batch_export_data)
- assert response.status_code == status.HTTP_200_OK, response.json()
-
- # Verify the bucket name update worked
- batch_export = get_batch_export_ok(client, team.pk, batch_export["id"])
- assert batch_export["destination"]["type"] == "Redshift"
- assert batch_export["destination"]["config"]["copy_inputs"]["s3_bucket"] == "my-new-production-s3-bucket"
+ "field_name,previous_source",
+ [
+ ("bucket_credentials", "integration"),
+ ("authorization", "integration"),
+ ("authorization", "iam_role"),
+ ("bucket_credentials", "legacy"),
+ ("authorization", "legacy"),
+ ],
+)
[email protected]("changed_key", ["aws_access_key_id", "aws_secret_access_key", "both"])
+def test_patching_redshift_batch_export_rejects_changed_inline_copy_credentials(
+ client: HttpClient,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+ field_name: str,
+ previous_source: str,
+ changed_key: str,
+) -> None:
+ batch_export = create_copy_batch_export(
+ client,
+ team,
+ user,
+ {
+ "s3_bucket": "test-staging-bucket",
+ "region_name": "us-east-1",
+ "s3_key_prefix": "posthog-events/",
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": (
+ "arn:aws:iam::000000000000:role/test-copy" if previous_source == "iam_role" else aws_s3_integration.id
+ ),
+ },
+ aws_redshift_integration.id,
+ )
+ destination = (
+ BatchExport.objects.select_related("destination").get(id=batch_export["id"], team_id=team.id).destination
+ )
+ legacy_credentials = {"aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key"}
+ if previous_source == "legacy":
+ destination.config["copy_inputs"][field_name] = legacy_credentials
+ destination.save()
+ destination.refresh_from_db()
+ stored_value = destination.config["copy_inputs"][field_name]
+ credentials = {
+ key: f"changed-{value}" if changed_key in {key, "both"} else value for key, value in legacy_credentials.items()
+ }
+ if previous_source == "legacy" and changed_key != "both":
+ credentials = {changed_key: credentials[changed_key]}
+ response = patch_batch_export(
+ client,
+ team.pk,
+ batch_export["id"],
+ {
+ "destination": {
+ "type": "Redshift",
+ "config": {"copy_inputs": {field_name: credentials}},
+ "integration": destination.integration_id,
+ }
+ },
+ )
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ detail = response.json()["detail"]
+ assert "AWS S3 integration" in detail
+ assert "Inline AWS access keys are no longer accepted" in detail
+ assert all(value not in detail for value in credentials.values())
+ destination.refresh_from_db()
+ assert destination.config["copy_inputs"][field_name] == stored_value
-def test_can_patch_redshift_batch_export_switching_authorization_to_credentials(
- client: HttpClient, temporal, organization, team, user, aws_redshift_integration
-):
- """Test an update can switch 'authorization' from an IAM role to inline credentials.
- The config merge used to raise a TypeError on this transition and return a 500.
- """
+ "new_authorization,expected_status",
+ [
+ ("arn:aws:iam::123456789012:role/new-copy-role", status.HTTP_200_OK),
+ ("default", status.HTTP_200_OK),
+ ("not-an-arn", status.HTTP_400_BAD_REQUEST),
+ ("arn:aws:iam::123:role/too-short-account", status.HTTP_400_BAD_REQUEST),
+ ],
+)
+def test_patching_redshift_batch_export_authorization_from_integration_to_iam_role(
+ client: HttpClient,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+ new_authorization: str,
+ expected_status: int,
+) -> None:
batch_export = create_copy_batch_export(
client,
team,
user,
{
- "s3_bucket": "my-production-s3-bucket",
+ "s3_bucket": "test-staging-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
- "authorization": "arn:aws:iam::123456789012:role/my-role",
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": aws_s3_integration.id,
},
- aws_redshift_integration.pk,
+ aws_redshift_integration.id,
+ )
+ destination = (
+ BatchExport.objects.select_related("destination").get(id=batch_export["id"], team_id=team.id).destination
)
- new_batch_export_data = {
- "destination": {
- "type": "Redshift",
- "config": {
- "copy_inputs": {
- "authorization": {"aws_access_key_id": "new-key", "aws_secret_access_key": "new-secret"},
- },
- },
- "integration": aws_redshift_integration.pk,
+ response = patch_batch_export(
+ client,
+ team.pk,
+ batch_export["id"],
+ {
+ "destination": {
+ "type": "Redshift",
+ "config": {"copy_inputs": {"authorization": new_authorization}},
+ "integration": destination.integration_id,
+ }
},
- }
-
- response = patch_batch_export(client, team.pk, batch_export["id"], new_batch_export_data)
- assert response.status_code == status.HTTP_200_OK, response.json()
+ )
+ assert response.status_code == expected_status, response.json()
- batch_export = get_batch_export_ok(client, team.pk, batch_export["id"])
- assert batch_export["destination"]["config"]["copy_inputs"]["s3_bucket"] == "my-production-s3-bucket"
+ destination.refresh_from_db()
+ if expected_status == status.HTTP_200_OK:
+ assert destination.config["copy_inputs"]["authorization"] == new_authorization
+ else:
+ assert "must be an IAM role ARN" in response.json()["detail"]
+ assert int(destination.config["copy_inputs"]["authorization"]) == aws_s3_integration.id
[email protected]("field_name", ["authorization", "bucket_credentials"])
-def test_patching_redshift_batch_export_rejects_switching_copy_integration_to_inline(
- client: HttpClient, temporal, organization, team, user, aws_s3_integration, aws_redshift_integration, field_name
-):
- """Test COPY credentials backed by an integration cannot switch back to inline values."""
+def test_can_patch_redshift_batch_export_with_unchanged_non_arn_iam_role(
+ client: HttpClient,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+) -> None:
batch_export = create_copy_batch_export(
client,
team,
user,
{
- "s3_bucket": "my-production-s3-bucket",
+ "s3_bucket": "test-staging-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": aws_s3_integration.pk,
- "authorization": aws_s3_integration.pk,
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": "default",
},
- aws_redshift_integration.pk,
+ aws_redshift_integration.id,
+ )
+ destination = (
+ BatchExport.objects.select_related("destination").get(id=batch_export["id"], team_id=team.id).destination
)
+ # Exports saved before ARN validation may hold any non-empty string.
+ destination.config["copy_inputs"]["authorization"] = "legacy-role-value"
+ destination.save()
- new_batch_export_data = {
- "destination": {
- "type": "Redshift",
- "config": {
- "copy_inputs": {
- field_name: {"aws_access_key_id": "abc123", "aws_secret_access_key": "secret"},
- },
- },
- "integration": aws_redshift_integration.pk,
+ response = patch_batch_export(
+ client,
+ team.pk,
+ batch_export["id"],
+ {
+ "destination": {
+ "type": "Redshift",
+ "config": {"table_name": "updated_events"},
+ "integration": destination.integration_id,
+ }
},
- }
+ )
+ assert response.status_code == status.HTTP_200_OK, response.json()
- response = patch_batch_export(client, team.pk, batch_export["id"], new_batch_export_data)
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- assert f"Cannot switch '{field_name}' from an integration to inline credentials" in response.json()["detail"]
+ destination.refresh_from_db()
+ assert destination.config["table_name"] == "updated_events"
+ assert destination.config["copy_inputs"]["authorization"] == "legacy-role-value"
def test_can_patch_redshift_batch_export_swapping_copy_integration(
- client: HttpClient, temporal, organization, team, user, aws_s3_integration, aws_redshift_integration
-):
- """Test COPY credentials backed by an integration can swap to a different integration."""
- other_integration = Integration.objects.create(
- team=team,
- kind=Integration.IntegrationKind.AWS_S3,
- integration_id="prod-aws-2",
- config={"name": "prod-aws-2", "aws_account_id": "123456789012"},
- sensitive_config={"aws_access_key_id": "key-2", "aws_secret_access_key": "secret-2"},
- created_by=user,
+ client: HttpClient,
+ team: Team,
+ user: User,
+ aws_s3_integration: IntegrationRecord,
+ aws_redshift_integration: IntegrationRecord,
+) -> None:
+ other_integration = IntegrationManager.aws_s3.credentials.create(
+ team_id=team.id,
+ external_id="test-aws-2",
+ config={"name": "test-aws-2"},
+ sensitive_config={"aws_access_key_id": "test-key-2", "aws_secret_access_key": "test-secret-2"},
+ created_by_id=user.id,
)
batch_export = create_copy_batch_export(
client,
team,
user,
{
- "s3_bucket": "my-production-s3-bucket",
+ "s3_bucket": "test-staging-bucket",
"region_name": "us-east-1",
"s3_key_prefix": "posthog-events/",
- "bucket_credentials": aws_s3_integration.pk,
- "authorization": aws_s3_integration.pk,
+ "bucket_credentials": aws_s3_integration.id,
+ "authorization": aws_s3_integration.id,
},
- aws_redshift_integration.pk,
+ aws_redshift_integration.id,
)
new_batch_export_data = {
@@ -209,11 +320,11 @@ def test_can_patch_redshift_batch_export_swapping_copy_integration(
"type": "Redshift",
"config": {
"copy_inputs": {
- "bucket_credentials": other_integration.pk,
- "authorization": other_integration.pk,
+ "bucket_credentials": other_integration.id,
+ "authorization": other_integration.id,
},
},
- "integration": aws_redshift_integration.pk,
+ "integration": aws_redshift_integration.id,
},
}
@@ -221,5 +332,5 @@ def test_can_patch_redshift_batch_export_swapping_copy_integration(
assert response.status_code == status.HTTP_200_OK, response.json()
batch_export = get_batch_export_ok(client, team.pk, batch_export["id"])
- assert batch_export["destination"]["config"]["copy_inputs"]["bucket_credentials"] == other_integration.pk
- assert batch_export["destination"]["config"]["copy_inputs"]["authorization"] == other_integration.pk
+ assert batch_export["destination"]["config"]["copy_inputs"]["bucket_credentials"] == other_integration.id
+ assert batch_export["destination"]["config"]["copy_inputs"]["authorization"] == other_integration.id
diff --git a/products/batch_exports/frontend/generated/api.schemas.ts b/products/batch_exports/frontend/generated/api.schemas.ts
index cabc72dc69e6..e3c97a1f5e28 100644
--- a/products/batch_exports/frontend/generated/api.schemas.ts
+++ b/products/batch_exports/frontend/generated/api.schemas.ts
@@ -356,7 +356,7 @@ export const RedshiftExportModeEnumApi = {
} as const
/**
- * Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.
+ * Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.
*/
export type RedshiftCopyInputsApiAuthorization =
| number
@@ -367,7 +367,7 @@ export type RedshiftCopyInputsApiAuthorization =
}
/**
- * Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.
+ * AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.
*/
export type RedshiftCopyInputsApiBucketCredentials =
| number
@@ -386,9 +386,9 @@ export interface RedshiftCopyInputsApi {
region_name: string
/** Key prefix for staged files in the S3 bucket. */
s3_key_prefix: string
- /** Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration. */
+ /** Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected. */
authorization: RedshiftCopyInputsApiAuthorization
- /** Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration. */
+ /** AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected. */
bucket_credentials: RedshiftCopyInputsApiBucketCredentials
}
diff --git a/products/batch_exports/frontend/generated/api.zod.ts b/products/batch_exports/frontend/generated/api.zod.ts
index 300757a2b7b6..face0ff5f1ce 100644
--- a/products/batch_exports/frontend/generated/api.zod.ts
+++ b/products/batch_exports/frontend/generated/api.zod.ts
@@ -400,24 +400,32 @@ export const BatchExportsCreateBody = /* @__PURE__ */ zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
@@ -1386,24 +1394,32 @@ export const BatchExportsUpdateBody = /* @__PURE__ */ zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
@@ -2129,24 +2145,32 @@ export const BatchExportsPartialUpdateBody = /* @__PURE__ */ zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
@@ -2837,24 +2861,32 @@ export const BatchExportsRunTestStepCreateBody = /* @__PURE__ */ zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
@@ -3586,24 +3618,32 @@ export const BatchExportsRunTestStepNewCreateBody = /* @__PURE__ */ zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 4aedcc697727..6a61a2ef99d4 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -17748,7 +17748,7 @@ export namespace Schemas {
} as const;
/**
- * Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.
+ * Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.
*/
export type RedshiftCopyInputsAuthorization = number | string | {
aws_access_key_id: string;
@@ -17756,7 +17756,7 @@ export namespace Schemas {
};
/**
- * Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.
+ * AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.
*/
export type RedshiftCopyInputsBucketCredentials = number | {
aws_access_key_id: string;
@@ -17773,9 +17773,9 @@ export namespace Schemas {
region_name: string;
/** Key prefix for staged files in the S3 bucket. */
s3_key_prefix: string;
- /** Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration. */
+ /** Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected. */
authorization: RedshiftCopyInputsAuthorization;
- /** Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration. */
+ /** AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected. */
bucket_credentials: RedshiftCopyInputsBucketCredentials;
}
diff --git a/services/mcp/src/generated/batch_exports/api.ts b/services/mcp/src/generated/batch_exports/api.ts
index 7fcd79741785..91c87147ffd1 100644
--- a/services/mcp/src/generated/batch_exports/api.ts
+++ b/services/mcp/src/generated/batch_exports/api.ts
@@ -415,24 +415,32 @@ export const BatchExportsCreateBody = () => zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
@@ -1162,24 +1170,32 @@ export const BatchExportsPartialUpdateBody = () => zod
zod
.string()
.describe('ARN of an IAM role attached to the Redshift cluster.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration.'
+ 'Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
bucket_credentials: zod
.union([
zod.number().describe('ID of an aws-s3-kind Integration.'),
- zod.object({
- aws_access_key_id: zod.string(),
- aws_secret_access_key: zod.string(),
- }),
+ zod
+ .object({
+ aws_access_key_id: zod.string(),
+ aws_secret_access_key: zod.string(),
+ })
+ .describe(
+ 'Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.'
+ ),
])
.describe(
- 'Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration.'
+ 'AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected.'
),
})
.describe(
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
index e085400de2e2..46e1b4764c64 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
@@ -427,6 +427,7 @@
"type": "string"
},
{
+ "description": "Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.",
"properties": {
"aws_access_key_id": {
"type": "string"
@@ -439,7 +440,7 @@
"type": "object"
}
],
- "description": "Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration."
+ "description": "Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
},
"bucket_credentials": {
"anyOf": [
@@ -448,6 +449,7 @@
"type": "number"
},
{
+ "description": "Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.",
"properties": {
"aws_access_key_id": {
"type": "string"
@@ -460,7 +462,7 @@
"type": "object"
}
],
- "description": "Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration."
+ "description": "AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
},
"region_name": {
"description": "AWS region of the staging S3 bucket.",
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
index a9e08f6b4925..6c18ef3e318b 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
@@ -426,6 +426,7 @@
"type": "string"
},
{
+ "description": "Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.",
"properties": {
"aws_access_key_id": {
"type": "string"
@@ -438,7 +439,7 @@
"type": "object"
}
],
- "description": "Authorization for Redshift to read staged files during COPY: the ARN of an IAM role attached to the cluster, inline AWS credentials, or the id of an aws-s3-kind Integration."
+ "description": "Authorization for Redshift to read staged files during COPY: an IAM role ARN or an AWS S3 integration ID. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
},
"bucket_credentials": {
"anyOf": [
@@ -447,6 +448,7 @@
"type": "number"
},
{
+ "description": "Legacy AWS access keys. Updates can keep this block unchanged. New or changed keys are rejected.",
"properties": {
"aws_access_key_id": {
"type": "string"
@@ -459,7 +461,7 @@
"type": "object"
}
],
- "description": "Credentials used to stage files in the S3 bucket: inline AWS credentials or the id of an aws-s3-kind Integration."
+ "description": "AWS S3 integration ID used to stage files in the S3 bucket. Updates can keep legacy AWS access keys unchanged. New or changed keys are rejected."
},
"region_name": {
"description": "AWS region of the staging S3 bucket.",