Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

27.2 KB
Raw Blame
import {
    MakeLogicType,
    actions,
    afterMount,
    beforeUnmount,
    kea,
    listeners,
    path,
    props,
    reducers,
    selectors,
} from 'kea'
import { forms } from 'kea-forms'
import type { DeepPartial, DeepPartialMap, FieldName, ValidationErrorType } from 'kea-forms'
import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'

import api, { ApiConfig } from 'lib/api'
import { dayjs } from 'lib/dayjs'
import { LemonDialog } from 'lib/lemon-ui/LemonDialog'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { urls } from 'scenes/urls'

import { FileSystemIconType } from '~/queries/schema/schema-general'
import { Breadcrumb, ProjectTreeRef } from '~/types'

import {
    managedMigrationsAwsIamSetupRetrieve,
    managedMigrationsPromoteCreate,
    managedMigrationsTrialRecordsRetrieve,
} from './generated/api'
import type { BatchImportAWSIAMSetupApi, TrialRecordsResponseApi } from './generated/api.schemas'
import { ManagedMigration } from './types'

export const S3_ROLE_ARN_REGEX = /^arn:aws:iam::\d{12}:role\/.+$/

export interface ManagedMigrationForm {
    source_type: 's3' | 's3_gzip' | 'mixpanel' | 'amplitude'
    access_key?: string
    secret_key?: string
    content_type: 'captured' | 'mixpanel' | 'amplitude'
    // s3 specific fields
    s3_auth_method?: 'iam_role' | 'access_keys'
    role_arn?: string
    s3_region?: string
    s3_bucket?: string
    s3_prefix?: string
    endpoint_url?: string
    // date range specific fields
    start_date?: string
    end_date?: string
    // EU region support for amplitude/mixpanel
    is_eu_region?: boolean
    // Mixpanel-specific auth: a service account (username, secret and project ID) or a project secret
    mixpanel_auth_method?: 'service_account' | 'project_secret'
    mixpanel_service_account_username?: string
    mixpanel_service_account_secret?: string
    mixpanel_project_id?: number
    mixpanel_project_secret?: string
    // Amplitude-specific options
    import_events?: boolean
    generate_identify_events?: boolean
    generate_group_identify_events?: boolean
    // Trial run options
    is_trial?: boolean
    trial_record_limit?: number
}

export const TRIAL_RECORD_LIMIT_DEFAULT = 1000
export const TRIAL_RECORD_LIMIT_MAX = 50000

// The API takes the Mixpanel credentials in its generic access_key / secret_key fields
function mixpanelCredentials(
    values: ManagedMigrationForm
): Pick<ManagedMigrationForm, 'access_key' | 'secret_key' | 'mixpanel_project_id'> {
    if (values.mixpanel_auth_method === 'service_account') {
        return {
            access_key: values.mixpanel_service_account_username,
            secret_key: values.mixpanel_service_account_secret,
            mixpanel_project_id: values.mixpanel_project_id,
        }
    }
    return { access_key: '', secret_key: values.mixpanel_project_secret }
}

const NEW_MANAGED_MIGRATION: ManagedMigrationForm = {
    source_type: 's3',
    access_key: '',
    secret_key: '',
    // Promoted to 'iam_role' once aws_iam_setup confirms role auth is available, so the
    // validator (which only sees form values) can never demand a role_arn the UI is hiding
    s3_auth_method: 'access_keys',
    role_arn: '',
    s3_region: '',
    s3_bucket: '',
    s3_prefix: '',
    endpoint_url: '',
    content_type: 'captured',
    start_date: '',
    end_date: '',
    is_eu_region: false,
    mixpanel_auth_method: 'service_account',
    mixpanel_service_account_username: '',
    mixpanel_service_account_secret: '',
    mixpanel_project_id: undefined,
    mixpanel_project_secret: '',
    import_events: true,
    generate_identify_events: true,
    generate_group_identify_events: true,
    is_trial: false,
    trial_record_limit: TRIAL_RECORD_LIMIT_DEFAULT,
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicValues {
    awsIamSetup: BatchImportAWSIAMSetupApi | null
    awsIamSetupLoading: boolean
    breadcrumbs: Breadcrumb[]
    isManagedMigrationSubmitting: boolean
    isManagedMigrationValid: boolean
    isPolling: boolean
    managedMigration: ManagedMigrationForm
    managedMigrationAllErrors: Record<string, any>
    managedMigrationChanged: boolean
    managedMigrationErrors: DeepPartialMap<ManagedMigrationForm, ValidationErrorType>
    managedMigrationHasErrors: boolean
    managedMigrationId: string | null
    managedMigrationManualErrors: Record<string, any>
    managedMigrationTouched: boolean
    managedMigrationTouches: Record<string, boolean>
    managedMigrationValidationErrors: DeepPartialMap<ManagedMigrationForm, ValidationErrorType>
    migrations: ManagedMigration[]
    migrationsLoading: boolean
    projectTreeRef: ProjectTreeRef
    promotingMigrationId: string | null
    showManagedMigrationErrors: boolean
    trialRecords: TrialRecordsResponseApi | null
    trialRecordsLoading: boolean
    trialResultsId: string | null
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicActions {
    closeTrialResults: () => {
        value: true
    }
    closeTrialResultsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    closeTrialResultsSuccess: (
        trialRecords: null,
        payload?: {
            value: true
        }
    ) => {
        trialRecords: null
        payload?: {
            value: true
        }
    }
    confirmPromoteTrial: (id: string) => {
        id: string
    }
    editManagedMigration: (id: string | null) => {
        id: string | null
    }
    loadAwsIamSetup: () => any
    loadAwsIamSetupFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadAwsIamSetupSuccess: (
        awsIamSetup: BatchImportAWSIAMSetupApi,
        payload?: any
    ) => {
        awsIamSetup: BatchImportAWSIAMSetupApi
        payload?: any
    }
    loadMigrations: () => any
    loadMigrationsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadMigrationsSuccess: (
        migrations: ManagedMigration[],
        payload?: any
    ) => {
        migrations: ManagedMigration[]
        payload?: any
    }
    loadTrialRecordsPage: ({ page }: { page: number }) => {
        page: number
    }
    loadTrialRecordsPageFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadTrialRecordsPageSuccess: (
        trialRecords: TrialRecordsResponseApi | null,
        payload?: {
            page: number
        }
    ) => {
        trialRecords: TrialRecordsResponseApi | null
        payload?: {
            page: number
        }
    }
    pauseMigration: (id: string) => {
        id: string
    }
    promoteTrial: (id: string) => {
        id: string
    }
    promoteTrialFinished: () => {
        value: true
    }
    resetManagedMigration: (values?: ManagedMigrationForm) => {
        values?: ManagedMigrationForm
    }
    resumeMigration: (id: string) => {
        id: string
    }
    setManagedMigrationManualErrors: (errors: Record<string, any>) => {
        errors: Record<string, any>
    }
    setManagedMigrationValue: (
        key: FieldName,
        value: any
    ) => {
        name: FieldName
        value: any
    }
    setManagedMigrationValues: (values: DeepPartial<ManagedMigrationForm>) => {
        values: DeepPartial<ManagedMigrationForm>
    }
    startPolling: () => {
        value: true
    }
    stopPolling: () => {
        value: true
    }
    submitManagedMigration: () => {
        value: boolean
    }
    submitManagedMigrationFailure: (
        error: Error,
        errors: Record<string, any>
    ) => {
        error: Error
        errors: Record<string, any>
    }
    submitManagedMigrationRequest: (managedMigration: ManagedMigrationForm) => {
        managedMigration: ManagedMigrationForm
    }
    submitManagedMigrationSuccess: (managedMigration: ManagedMigrationForm) => {
        managedMigration: ManagedMigrationForm
    }
    touchManagedMigrationField: (key: string) => {
        key: string
    }
    viewTrialResults: (id: string) => {
        id: string
    }
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicProps {
    managedMigrationId: null
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicMeta {
    __keaTypeGenInternalSelectorTypes: {
        breadcrumbs: (managedMigrationId: null) => Breadcrumb[]
        projectTreeRef: (managedMigrationId: null) => ProjectTreeRef
    }
}

export type managedMigrationLogicType = MakeLogicType<
    managedMigrationLogicValues,
    managedMigrationLogicActions,
    managedMigrationLogicProps,
    managedMigrationLogicMeta
>

export const managedMigrationLogic = kea<managedMigrationLogicType>([
    path(['products', 'managed_migrations', 'frontend', 'managedMigrationLogic']),
    props({
        managedMigrationId: null,
    }),
    actions({
        editManagedMigration: (id: string | null) => ({ id }),
        pauseMigration: (id: string) => ({ id }),
        resumeMigration: (id: string) => ({ id }),
        promoteTrial: (id: string) => ({ id }),
        confirmPromoteTrial: (id: string) => ({ id }),
        promoteTrialFinished: true,
        viewTrialResults: (id: string) => ({ id }),
        closeTrialResults: true,
        startPolling: true,
        stopPolling: true,
    }),
    reducers({
        managedMigrationId: [null as string | null, { editManagedMigration: (_, { id }) => id }],
        isPolling: [false, { startPolling: () => true, stopPolling: () => false }],
        trialResultsId: [null as string | null, { viewTrialResults: (_, { id }) => id, closeTrialResults: () => null }],
        promotingMigrationId: [
            null as string | null,
            { confirmPromoteTrial: (_, { id }) => id, promoteTrialFinished: () => null },
        ],
    }),
    loaders(({ values }) => ({
        migrations: [
            [] as ManagedMigration[],
            {
                loadMigrations: async () => {
                    const projectId = ApiConfig.getCurrentProjectId()
                    // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use managedMigrationsList() from 'products/managed_migrations/frontend/generated/api' instead.
                    const response = await api.get(`api/projects/${projectId}/managed_migrations`)
                    return response.results
                },
            },
        ],
        trialRecords: [
            null as TrialRecordsResponseApi | null,
            {
                loadTrialRecordsPage: async ({ page }: { page: number }, breakpoint) => {
                    if (!values.trialResultsId) {
                        return null
                    }
                    await breakpoint(100)
                    const projectId = String(ApiConfig.getCurrentProjectId())
                    const response = await managedMigrationsTrialRecordsRetrieve(projectId, values.trialResultsId, {
                        page,
                    })
                    // Drop out-of-order responses so stale pages can't overwrite newer state.
                    breakpoint()
                    return response
                },
                closeTrialResults: () => null,
            },
        ],
        awsIamSetup: [
            null as BatchImportAWSIAMSetupApi | null,
            {
                loadAwsIamSetup: async () => {
                    const projectId = ApiConfig.getCurrentProjectId()
                    return await managedMigrationsAwsIamSetupRetrieve(String(projectId))
                },
            },
        ],
    })),
    forms({
        managedMigration: {
            defaults: NEW_MANAGED_MIGRATION,
            errors: ({
                source_type,
                content_type,
                access_key,
                secret_key,
                s3_auth_method,
                role_arn,
                s3_region,
                s3_bucket,
                start_date,
                end_date,
                mixpanel_auth_method,
                mixpanel_service_account_username,
                mixpanel_service_account_secret,
                mixpanel_project_id,
                mixpanel_project_secret,
                import_events,
                generate_identify_events,
                generate_group_identify_events,
                is_trial,
                trial_record_limit,
            }: ManagedMigrationForm) => {
                const usesIamRole = (source_type === 's3' || source_type === 's3_gzip') && s3_auth_method === 'iam_role'

                const errors: Record<string, string | null> = {}

                if (source_type === 'mixpanel') {
                    if (mixpanel_auth_method === 'service_account') {
                        errors.mixpanel_service_account_username = !mixpanel_service_account_username
                            ? 'Service account username is required'
                            : null
                        errors.mixpanel_service_account_secret = !mixpanel_service_account_secret
                            ? 'Service account secret is required'
                            : null
                        errors.mixpanel_project_id = !mixpanel_project_id ? 'Project ID is required' : null
                    } else {
                        errors.mixpanel_project_secret = !mixpanel_project_secret ? 'Project secret is required' : null
                    }
                } else if (!usesIamRole) {
                    errors.access_key = !access_key ? 'Access key is required' : null
                    errors.secret_key = !secret_key ? 'Secret key is required' : null
                }

                if (is_trial) {
                    errors.trial_record_limit =
                        !trial_record_limit || trial_record_limit < 1 || trial_record_limit > TRIAL_RECORD_LIMIT_MAX
                            ? `Number of records must be between 1 and ${TRIAL_RECORD_LIMIT_MAX}`
                            : null
                }

                if (usesIamRole) {
                    errors.role_arn = !role_arn
                        ? 'IAM role ARN is required'
                        : !S3_ROLE_ARN_REGEX.test(role_arn)
                          ? 'Enter a valid IAM role ARN, e.g. arn:aws:iam::123456789012:role/posthog-import'
                          : null
                }

                if (source_type === 's3' || source_type === 's3_gzip') {
                    errors.s3_region = !s3_region ? 'S3 region is required' : null
                    errors.s3_bucket = !s3_bucket ? 'S3 bucket is required' : null

                    if (content_type === 'amplitude') {
                        if (!import_events && !generate_identify_events && !generate_group_identify_events) {
                            errors.import_events =
                                'At least one of "Import events", "Generate identify events", or "Generate group identify events" must be enabled'
                        }
                    }
                } else if (source_type === 'mixpanel' || source_type === 'amplitude') {
                    errors.start_date = !start_date ? 'Start date is required' : null
                    errors.end_date = !end_date ? 'End date is required' : null

                    if (start_date && end_date) {
                        const startDateParsed = dayjs(start_date)
                        const endDateParsed = dayjs(end_date)

                        if (endDateParsed.isBefore(startDateParsed)) {
                            errors.end_date = 'End date must be after start date'
                        } else if (endDateParsed.diff(startDateParsed, 'year', true) > 1) {
                            errors.end_date =
                                'Date range cannot exceed 1 year. Please create multiple migration jobs for longer periods.'
                        } else if (
                            source_type === 'amplitude' &&
                            endDateParsed.diff(startDateParsed, 'hour', true) < 1
                        ) {
                            errors.end_date = 'Date range must be at least 1 hour for Amplitude migrations.'
                        }
                    }

                    // For Amplitude, ensure at least one of import_events, generate_identify_events, or generate_group_identify_events is enabled
                    if (source_type === 'amplitude') {
                        if (!import_events && !generate_identify_events && !generate_group_identify_events) {
                            errors.import_events =
                                'At least one of "Import events", "Generate identify events", or "Generate group identify events" must be enabled'
                        }
                    }
                }
                return errors
            },
            submit: async (values: ManagedMigrationForm) => {
                const projectId = ApiConfig.getCurrentProjectId()
                const usesIamRole =
                    (values.source_type === 's3' || values.source_type === 's3_gzip') &&
                    values.s3_auth_method === 'iam_role'
                let payload: ManagedMigrationForm = {
                    source_type: values.source_type,
                    content_type: values.content_type,
                    ...(values.is_trial ? { is_trial: true, trial_record_limit: values.trial_record_limit } : {}),
                    ...(usesIamRole
                        ? { role_arn: values.role_arn }
                        : values.source_type === 'mixpanel'
                          ? mixpanelCredentials(values)
                          : { access_key: values.access_key, secret_key: values.secret_key }),
                }
                if (values.source_type === 's3' || values.source_type === 's3_gzip') {
                    payload = {
                        ...payload,
                        s3_region: values.s3_region,
                        s3_bucket: values.s3_bucket,
                        s3_prefix: values.s3_prefix,
                        ...(values.endpoint_url && !usesIamRole ? { endpoint_url: values.endpoint_url } : {}),
                    }

                    if (values.content_type === 'amplitude') {
                        payload.import_events = values.import_events
                        payload.generate_identify_events = values.generate_identify_events
                        payload.generate_group_identify_events = values.generate_group_identify_events
                    }
                } else if (values.source_type === 'mixpanel' || values.source_type === 'amplitude') {
                    payload = {
                        ...payload,
                        start_date: values.start_date,
                        end_date: values.end_date,
                        is_eu_region: values.is_eu_region,
                    }

                    // Only include Amplitude-specific options for Amplitude migrations
                    if (values.source_type === 'amplitude') {
                        payload.import_events = values.import_events
                        payload.generate_identify_events = values.generate_identify_events
                        payload.generate_group_identify_events = values.generate_group_identify_events
                    }
                }
                try {
                    // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use managedMigrationsCreate() from 'products/managed_migrations/frontend/generated/api' instead.
                    const response = await api.create(`api/projects/${projectId}/managed_migrations`, payload)
                    return response
                } catch (error: any) {
                    if (error.status === 400 && error.data?.error) {
                        throw new Error(error.data.error)
                    }
                    throw error
                }
            },
        },
    }),
    listeners(({ actions, values, cache }) => ({
        loadAwsIamSetupSuccess: ({ awsIamSetup }) => {
            // Default to the recommended role auth once we know it's available, unless the
            // user already started entering access keys while the setup call was in flight
            if (awsIamSetup?.available && !values.managedMigration.access_key && !values.managedMigration.secret_key) {
                actions.setManagedMigrationValue('s3_auth_method', 'iam_role')
            }
        },
        submitManagedMigrationSuccess: async ({ managedMigration }) => {
            if (managedMigration?.is_trial) {
                lemonToast.success('Trial run started — results will appear here when it completes')
            }
            actions.loadMigrations()
            router.actions.push(urls.managedMigration())
        },
        submitManagedMigrationFailure: async ({ error }) => {
            if (error?.message) {
                lemonToast.error(error.message)
            } else {
                lemonToast.error('Failed to create migration. Please try again.')
            }
        },
        pauseMigration: async ({ id }) => {
            try {
                const projectId = ApiConfig.getCurrentProjectId()
                // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. managedMigrationsPauseCreate() from 'products/managed_migrations/frontend/generated/api' serves this route, but its generated types do not describe this call yet, so fix the endpoint's OpenAPI schema first.
                await api.create(`api/projects/${projectId}/managed_migrations/${id}/pause/`)
                lemonToast.success('Migration paused successfully')
                actions.loadMigrations()
            } catch (error: any) {
                lemonToast.error(error?.message || 'Failed to pause migration')
            }
        },
        resumeMigration: async ({ id }) => {
            try {
                const projectId = ApiConfig.getCurrentProjectId()
                // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. managedMigrationsResumeCreate() from 'products/managed_migrations/frontend/generated/api' serves this route, but its generated types do not describe this call yet, so fix the endpoint's OpenAPI schema first.
                await api.create(`api/projects/${projectId}/managed_migrations/${id}/resume/`)
                lemonToast.success('Migration resumed successfully')
                actions.loadMigrations()
            } catch (error: any) {
                lemonToast.error(error?.message || 'Failed to resume migration')
            }
        },
        viewTrialResults: () => {
            actions.loadTrialRecordsPage({ page: 0 })
        },
        loadTrialRecordsPageFailure: async ({ errorObject }) => {
            if (errorObject?.status === 410) {
                lemonToast.error('Trial results have expired and are no longer available')
            } else {
                lemonToast.error('Failed to load trial results')
            }
            actions.closeTrialResults()
        },
        promoteTrial: ({ id }) => {
            if (values.promotingMigrationId) {
                return
            }
            const trial = values.migrations.find((migration: ManagedMigration) => migration.id === id)
            const failedRecords = trial?.state?.trial?.summary?.dropped_records ?? 0
            if (failedRecords === 0) {
                actions.confirmPromoteTrial(id)
                return
            }
            LemonDialog.open({
                title: 'Start the full import?',
                description: `This trial found ${failedRecords} record${
                    failedRecords === 1 ? '' : 's'
                } that can't be imported. These errors are not retriable: the import will pause when it reaches the first failing record, and stay paused until the source data is fixed.`,
                primaryButton: {
                    children: 'Start import anyway',
                    type: 'primary',
                    onClick: () => actions.confirmPromoteTrial(id),
                },
                secondaryButton: { children: 'Cancel' },
            })
        },
        confirmPromoteTrial: async ({ id }) => {
            try {
                const projectId = String(ApiConfig.getCurrentProjectId())
                await managedMigrationsPromoteCreate(projectId, id)
                lemonToast.success('Import started from trial run')
                actions.loadMigrations()
            } catch (error: any) {
                lemonToast.error(error?.data?.error || error?.message || 'Failed to start import from trial')
            } finally {
                actions.promoteTrialFinished()
            }
        },
        loadMigrationsSuccess: () => {
            const hasActiveMigrations = values.migrations.some(
                (migration: ManagedMigration) =>
                    migration.display_status === 'running' || migration.display_status === 'waiting_to_start'
            )
            if (hasActiveMigrations && !values.isPolling) {
                actions.startPolling()
            } else if (!hasActiveMigrations && values.isPolling) {
                actions.stopPolling()
            }
        },
        startPolling: () => {
            cache.disposables.add(() => {
                const intervalId = setInterval(() => {
                    actions.loadMigrations()
                }, 5000)
                return () => clearInterval(intervalId)
            }, 'pollMigrations')
        },
        stopPolling: () => {
            cache.disposables.dispose('pollMigrations')
        },
    })),
    selectors({
        breadcrumbs: [
            (_, p) => [p.managedMigrationId],
            (managedMigrationId: string | null): Breadcrumb[] => [
                {
                    key: 'managed-migrations',
                    name: 'Managed Migrations',
                    path: urls.managedMigration(),
                    iconType: 'data_pipeline_metadata',
                },
                ...(managedMigrationId
                    ? [
                          {
                              key: 'edit-migration',
                              name: 'Manage migration',
                              path: urls.managedMigration(),
                              iconType: 'data_pipeline_metadata' as FileSystemIconType,
                          },
                      ]
                    : []),
            ],
        ],
        projectTreeRef: [
            (_, p) => [p.managedMigrationId],
            (managedMigrationId: string | null): ProjectTreeRef => ({
                type: 'managed-migration',
                ref: managedMigrationId,
            }),
        ],
    }),
    urlToAction(({ actions }) => ({
        [urls.managedMigrationNew()]: () => {
            actions.editManagedMigration('new')
        },
        [`${urls.managedMigration()}/:id`]: ({ id }) => {
            actions.editManagedMigration(id ?? null)
        },
        [urls.managedMigration()]: () => {
            actions.editManagedMigration(null)
        },
    })),
    afterMount(({ actions }) => {
        actions.loadMigrations()
        actions.loadAwsIamSetup()
    }),
    beforeUnmount(({ actions }) => {
        actions.stopPolling()
    }),
])