Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

47.7 KB
Raw Blame
import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import { forms } from 'kea-forms'
import type { DeepPartial, DeepPartialMap, FieldName, ValidationErrorType } from 'kea-forms'
import { loaders } from 'kea-loaders'
import { beforeUnload, router } from 'kea-router'

import { LemonDialog, lemonToast } from '@posthog/lemon-ui'

import api from 'lib/api'
import { integrationsLogic } from 'lib/integrations/integrationsLogic'
import { splitQueries } from 'lib/monaco/multiQueryUtils'
import { addProductIntent } from 'lib/utils/product-intents'
import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'

import { DatabaseSchemaBatchExportTable, ProductIntentContext, ProductKey } from '~/queries/schema/schema-general'
import {
    BatchExportConfiguration,
    BatchExportConfigurationTest,
    BatchExportConfigurationTestStep,
    BatchExportService,
    IntegrationType,
} from '~/types'

import { BatchExportModelEnumApi } from 'products/batch_exports/frontend/generated/api.schemas'

import { batchExportDataLogic } from './batchExportDataLogic'
import { DESTINATIONS } from './destinations'
import { genericPersonEventFields, isSelectedCompressionOptionValid } from './destinations/common'
import { humanizeBatchExportName } from './utils'

export interface BatchExportConfigFormLogicProps {
    service: BatchExportService['type'] | null
    id: string | null
}

// Fields that exist on the form but are not part of destination.config
const TOP_LEVEL_FORM_FIELDS = new Set([
    'name',
    'destination',
    'interval',
    'timezone',
    'offset_day',
    'offset_hour',
    'paused',
    'created_at',
    'start_at',
    'end_at',
    'model',
    'hogql_query',
    'hogql_modifiers',
    'filters',
    'integration_id',
])

const ALLOWED_BASE_CONFIG_KEYS = new Set(['exclude_events', 'include_events'])

function buildDestinationPayload(formValues: Record<string, any>): {
    type: string
    config: Record<string, any>
    integration?: any
} {
    const destinationType = formValues.destination as BatchExportService['type']
    const definition = DESTINATIONS[destinationType]
    // Apply destination-specific transform first (e.g. Redshift's COPY copy_inputs assembly).
    // Destinations without a custom serialize get the raw form values passed through.
    const intermediate = definition?.serialize ? definition.serialize(formValues) : formValues
    // When a destination declares configKeys, drop any config key outside it (plus the base-export
    // keys). Mirrors the backend's allowed = destination_fields ∪ base_field_names check, so stale
    // fields don't survive a deserialize → serialize round-trip and get rejected on save.
    const allowed = definition?.configKeys ? new Set([...definition.configKeys, ...ALLOWED_BASE_CONFIG_KEYS]) : null
    // Strip top-level form fields that don't belong in destination.config
    const config: Record<string, any> = {}
    for (const [key, value] of Object.entries(intermediate)) {
        if (TOP_LEVEL_FORM_FIELDS.has(key)) {
            continue
        }
        if (allowed && !allowed.has(key)) {
            continue
        }
        config[key] = value
    }
    // A persisted compression value can be invalid for the selected file_format (e.g. an
    // externally-created JSONLines export still carrying a Parquet-only codec). Drop it on save so
    // editing an unrelated field doesn't resubmit a combination the backend rejects.
    if ('compression' in config && !isSelectedCompressionOptionValid(config.file_format, config.compression)) {
        config.compression = null
    }
    const result: { type: string; config: Record<string, any>; integration?: any } = {
        type: destinationType,
        config,
    }
    if (definition?.usesIntegration) {
        result.integration = formValues.integration_id
    }
    return result
}

function buildBatchExportPayload(formValues: Record<string, any>): Partial<BatchExportConfiguration> {
    const interval = formValues.interval
    return {
        paused: formValues.paused,
        name: formValues.name,
        interval,
        timezone: interval === 'day' || interval === 'week' ? formValues.timezone : null,
        offset_day: interval === 'week' ? formValues.offset_day : null,
        offset_hour: interval === 'day' || interval === 'week' ? formValues.offset_hour : null,
        model: formValues.model,
        // Only the 'hogql' model edits the query and its modifiers. The events model would save a query as its
        // export schema, and the API rejects modifiers for every other model.
        hogql_query: formValues.model === BatchExportModelEnumApi.Hogql ? formValues.hogql_query : undefined,
        hogql_modifiers: formValues.model === BatchExportModelEnumApi.Hogql ? formValues.hogql_modifiers : undefined,
        // Filters only apply to the events model: the API rejects them for 'hogql' and runs ignore them otherwise
        filters: formValues.model === BatchExportModelEnumApi.Events ? formValues.filters : undefined,
        destination: buildDestinationPayload(formValues) as any,
    }
}

function getConfigurationFromBatchExportConfig(batchExportConfig: BatchExportConfiguration): Record<string, any> {
    const destinationType = batchExportConfig.destination.type
    const definition = DESTINATIONS[destinationType]

    const flatConfig = definition?.deserialize
        ? definition.deserialize(batchExportConfig.destination.config)
        : { ...batchExportConfig.destination.config }

    const config: Record<string, any> = {
        name: batchExportConfig.name,
        destination: destinationType,
        paused: batchExportConfig.paused,
        interval: batchExportConfig.interval,
        timezone: batchExportConfig.timezone,
        offset_day: (batchExportConfig as any).offset_day ?? null,
        offset_hour: (batchExportConfig as any).offset_hour ?? null,
        model: batchExportConfig.model,
        hogql_query: batchExportConfig.hogql_query ?? null,
        hogql_modifiers: batchExportConfig.hogql_modifiers ?? null,
        filters: batchExportConfig.filters,
        ...flatConfig,
    }

    if (definition?.usesIntegration) {
        config.integration_id = (batchExportConfig.destination as { integration?: number }).integration
    }

    return config
}

// $feature_flag_called is high volume and is being deprecated from the events table, so new exports
// leave it out by default. It shows up prefilled under "Exclude events" and can be removed there.
export const DEFAULT_EXCLUDE_EVENTS = ['$feature_flag_called']

export function getDefaultConfiguration(service: string): Record<string, any> {
    const definition = DESTINATIONS[service as BatchExportService['type']]
    return {
        name: humanizeBatchExportName(service as BatchExportService['type']),
        destination: service,
        model: BatchExportModelEnumApi.Events,
        paused: true,
        exclude_events: [...DEFAULT_EXCLUDE_EVENTS],
        ...(definition ? definition.defaults() : {}),
    }
}

const BASE_EVENT_FIELDS = {
    uuid: {
        name: 'uuid',
        hogql_value: 'toString(uuid)',
        type: 'string',
        schema_valid: true,
    },
    timestamp: {
        name: 'timestamp',
        hogql_value: 'timestamp',
        type: 'datetime',
        schema_valid: true,
    },
    event: {
        name: 'event',
        hogql_value: 'event',
        type: 'string',
        schema_valid: true,
    },
    distinct_id: {
        name: 'distinct_id',
        hogql_value: 'toString(distinct_id)',
        type: 'string',
        schema_valid: true,
    },
    properties: {
        name: 'properties',
        hogql_value: 'properties',
        type: 'json',
        schema_valid: true,
    },
} as const

function getEventTable(service: BatchExportService['type']): DatabaseSchemaBatchExportTable {
    const definition = DESTINATIONS[service]
    const overrides = definition?.eventTableOverrides ?? {}
    const includeGeneric = overrides.includeGenericPersonFields !== false

    return {
        type: 'batch_export',
        id: 'Events',
        name: 'events',
        fields: {
            ...BASE_EVENT_FIELDS,
            ...(includeGeneric
                ? genericPersonEventFields({
                      teamIdHogql: overrides.teamIdHogql ?? 'team_id',
                      setName: overrides.setName ?? 'set',
                      setOnceName: overrides.setOnceName ?? 'set_once',
                  })
                : {}),
            ...definition?.eventTableExtraFields,
        },
    }
}

const personsTable: DatabaseSchemaBatchExportTable = {
    type: 'batch_export',
    id: 'Persons',
    name: 'persons',
    fields: {
        team_id: {
            name: 'team_id',
            hogql_value: 'team_id',
            type: 'integer',
            schema_valid: true,
        },
        distinct_id: {
            name: 'distinct_id',
            hogql_value: 'distinct_id',
            type: 'string',
            schema_valid: true,
        },
        person_id: {
            name: 'person_id',
            hogql_value: 'person_id',
            type: 'string',
            schema_valid: true,
        },
        properties: {
            name: 'properties',
            hogql_value: 'properties',
            type: 'json',
            schema_valid: true,
        },
        person_version: {
            name: 'person_version',
            hogql_value: 'person_version',
            type: 'integer',
            schema_valid: true,
        },
        person_distinct_id_version: {
            name: 'person_distinct_id_version',
            hogql_value: 'person_distinct_id_version',
            type: 'integer',
            schema_valid: true,
        },
        created_at: {
            name: 'created_at',
            hogql_value: 'created_at',
            type: 'datetime',
            schema_valid: true,
        },
        is_deleted: {
            name: 'is_deleted',
            hogql_value: 'is_deleted',
            type: 'boolean',
            schema_valid: true,
        },
    },
}

const sessionsTable: DatabaseSchemaBatchExportTable = {
    type: 'batch_export',
    id: 'Sessions',
    name: 'sessions',
    fields: {
        team_id: {
            name: 'team_id',
            hogql_value: 'team_id',
            type: 'integer',
            schema_valid: true,
        },
        session_id: {
            name: 'session_id',
            type: 'string',
            hogql_value: 'session_id',
            schema_valid: true,
        },
        session_id_v7: {
            name: 'session_id_v7',
            type: 'string',
            hogql_value: 'session_id_v7',
            schema_valid: true,
        },
        distinct_id: {
            name: 'distinct_id',
            type: 'string',
            hogql_value: 'distinct_id',
            schema_valid: true,
        },
        start_timestamp: {
            name: 'start_timestamp',
            type: 'datetime',
            hogql_value: 'start_timestamp',
            schema_valid: true,
        },
        end_timestamp: {
            name: 'end_timestamp',
            type: 'datetime',
            hogql_value: 'end_timestamp',
            schema_valid: true,
        },
        urls: {
            name: 'urls',
            type: 'array',
            hogql_value: 'urls',
            schema_valid: true,
        },
        num_uniq_urls: {
            name: 'num_uniq_urls',
            type: 'integer',
            hogql_value: 'num_uniq_urls',
            schema_valid: true,
        },
        entry_current_url: {
            name: 'entry_current_url',
            type: 'string',
            hogql_value: 'entry_current_url',
            schema_valid: true,
        },
        entry_pathname: {
            name: 'entry_pathname',
            type: 'string',
            hogql_value: 'entry_pathname',
            schema_valid: true,
        },
        entry_hostname: {
            name: 'entry_hostname',
            type: 'string',
            hogql_value: 'entry_hostname',
            schema_valid: true,
        },
        end_current_url: {
            name: 'end_current_url',
            type: 'string',
            hogql_value: 'end_current_url',
            schema_valid: true,
        },
        end_pathname: {
            name: 'end_pathname',
            type: 'string',
            hogql_value: 'end_pathname',
            schema_valid: true,
        },
        end_hostname: {
            name: 'end_hostname',
            type: 'string',
            hogql_value: 'end_hostname',
            schema_valid: true,
        },
        entry_utm_source: {
            name: 'entry_utm_source',
            type: 'string',
            hogql_value: 'entry_utm_source',
            schema_valid: true,
        },
        entry_utm_campaign: {
            name: 'entry_utm_campaign',
            type: 'string',
            hogql_value: 'entry_utm_campaign',
            schema_valid: true,
        },
        entry_utm_medium: {
            name: 'entry_utm_medium',
            type: 'string',
            hogql_value: 'entry_utm_medium',
            schema_valid: true,
        },
        entry_utm_term: {
            name: 'entry_utm_term',
            type: 'string',
            hogql_value: 'entry_utm_term',
            schema_valid: true,
        },
        entry_utm_content: {
            name: 'entry_utm_content',
            type: 'string',
            hogql_value: 'entry_utm_content',
            schema_valid: true,
        },
        entry_referring_domain: {
            name: 'entry_referring_domain',
            type: 'string',
            hogql_value: 'entry_referring_domain',
            schema_valid: true,
        },
        entry_gclid: {
            name: 'entry_gclid',
            type: 'string',
            hogql_value: 'entry_gclid',
            schema_valid: true,
        },
        entry_fbclid: {
            name: 'entry_fbclid',
            type: 'string',
            hogql_value: 'entry_fbclid',
            schema_valid: true,
        },
        entry_gad_source: {
            name: 'entry_gad_source',
            type: 'string',
            hogql_value: 'entry_gad_source',
            schema_valid: true,
        },
        pageview_count: {
            name: 'pageview_count',
            type: 'integer',
            hogql_value: 'pageview_count',
            schema_valid: true,
        },
        autocapture_count: {
            name: 'autocapture_count',
            type: 'integer',
            hogql_value: 'autocapture_count',
            schema_valid: true,
        },
        screen_count: {
            name: 'screen_count',
            type: 'integer',
            hogql_value: 'screen_count',
            schema_valid: true,
        },
        channel_type: {
            name: 'channel_type',
            type: 'string',
            hogql_value: 'channel_type',
            schema_valid: true,
        },
        session_duration: {
            name: 'session_duration',
            type: 'integer',
            hogql_value: 'session_duration',
            schema_valid: true,
        },
        duration: {
            name: 'duration',
            type: 'integer',
            hogql_value: 'duration',
            schema_valid: true,
        },
        is_bounce: {
            name: 'is_bounce',
            type: 'boolean',
            hogql_value: 'is_bounce',
            schema_valid: true,
        },
        last_external_click_url: {
            name: 'last_external_click_url',
            type: 'string',
            hogql_value: 'last_external_click_url',
            schema_valid: true,
        },
        page_screen_autocapture_count_up_to: {
            name: 'page_screen_autocapture_count_up_to',
            type: 'string',
            hogql_value: 'page_screen_autocapture_count_up_to',
            schema_valid: true,
        },
        exit_current_url: {
            name: 'exit_current_url',
            type: 'string',
            hogql_value: 'exit_current_url',
            schema_valid: true,
        },
        exit_pathname: {
            name: 'exit_pathname',
            type: 'string',
            hogql_value: 'exit_pathname',
            schema_valid: true,
        },
        vitals_lcp: {
            name: 'vitals_lcp',
            type: 'float',
            hogql_value: 'vitals_lcp',
            schema_valid: true,
        },
        entry_gclsrc: {
            name: 'entry_gclsrc',
            type: 'string',
            hogql_value: 'entry_gclsrc',
            schema_valid: true,
        },
        entry_dclid: {
            name: 'entry_dclid',
            type: 'string',
            hogql_value: 'entry_dclid',
            schema_valid: true,
        },
        entry_gbraid: {
            name: 'entry_gbraid',
            type: 'string',
            hogql_value: 'entry_gbraid',
            schema_valid: true,
        },
        entry_wbraid: {
            name: 'entry_wbraid',
            type: 'string',
            hogql_value: 'entry_wbraid',
            schema_valid: true,
        },
        entry_msclkid: {
            name: 'entry_msclkid',
            type: 'string',
            hogql_value: 'entry_msclkid',
            schema_valid: true,
        },
        entry_twclid: {
            name: 'entry_twclid',
            type: 'string',
            hogql_value: 'entry_twclid',
            schema_valid: true,
        },
        entry_li_fat_id: {
            name: 'entry_li_fat_id',
            type: 'string',
            hogql_value: 'entry_li_fat_id',
            schema_valid: true,
        },
        entry_mc_cid: {
            name: 'entry_mc_cid',
            type: 'string',
            hogql_value: 'entry_mc_cid',
            schema_valid: true,
        },
        entry_igshid: {
            name: 'entry_igshid',
            type: 'string',
            hogql_value: 'entry_igshid',
            schema_valid: true,
        },
        entry_ttclid: {
            name: 'entry_ttclid',
            type: 'string',
            hogql_value: 'entry_ttclid',
            schema_valid: true,
        },
        entry__kx: {
            name: 'entry__kx',
            type: 'string',
            hogql_value: 'entry__kx',
            schema_valid: true,
        },
        entry_irclid: {
            name: 'entry_irclid',
            type: 'string',
            hogql_value: 'entry_irclid',
            schema_valid: true,
        },
    },
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicValues {
    batchExportConfig: BatchExportConfiguration | null // batchExportDataLogic
    batchExportConfigLoading: boolean // batchExportDataLogic
    integrations: IntegrationType[] | null // integrationsLogic
    teamTimezone: string // teamLogic
    teamWeekStartDay: number // teamLogic
    batchExportConfigTest: BatchExportConfigurationTest | null
    batchExportConfigTestLoading: boolean
    batchExportConfigTestStep: BatchExportConfigurationTestStep | null
    batchExportConfigTestStepLoading: boolean
    configuration: Record<string, any>
    configurationAllErrors: Record<string, any>
    configurationChanged: boolean
    configurationErrors: DeepPartialMap<Record<string, any>, ValidationErrorType>
    configurationHasErrors: boolean
    configurationManualErrors: Record<string, any>
    configurationTouched: boolean
    configurationTouches: Record<string, boolean>
    configurationValidationErrors: DeepPartialMap<Record<string, any>, ValidationErrorType>
    isConfigurationSubmitting: boolean
    isConfigurationValid: boolean
    isDatabaseDestination: boolean
    isNew: boolean
    loading: boolean
    logicProps: any
    requiredFields: string[]
    runningStep: number | null
    savedConfiguration: Record<string, any>
    selectedIntegration: IntegrationType | null
    selectedModel: string
    service:
        | 'AwsS3'
        | 'AzureBlob'
        | 'BigQuery'
        | 'Databricks'
        | 'HTTP'
        | 'Postgres'
        | 'Redshift'
        | 'S3Compatible'
        | 'Snowflake'
        | null
    showConfigurationErrors: boolean
    tables: DatabaseSchemaBatchExportTable[]
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicActions {
    loadBatchExportConfig: () => any // batchExportDataLogic
    loadBatchExportConfigSuccess: (
        batchExportConfig: BatchExportConfiguration | null,
        payload?: any
    ) => {
        batchExportConfig: BatchExportConfiguration | null
        payload?: any
    } // batchExportDataLogic
    setBatchExportConfig: (config: BatchExportConfiguration) => BatchExportConfiguration // batchExportDataLogic
    deleteBatchExport: () => boolean
    loadBatchExportConfigTest: () => any
    loadBatchExportConfigTestFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadBatchExportConfigTestSuccess: (
        batchExportConfigTest: BatchExportConfigurationTest | null,
        payload?: any
    ) => {
        batchExportConfigTest: BatchExportConfigurationTest | null
        payload?: any
    }
    resetConfiguration: (values?: Record<string, any>) => {
        values?: Record<string, any>
    }
    runBatchExportConfigTestStep: (step: any) => any
    runBatchExportConfigTestStepFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    runBatchExportConfigTestStepSuccess: (
        batchExportConfigTestStep: BatchExportConfigurationTestStep | null,
        payload?: any
    ) => {
        batchExportConfigTestStep: BatchExportConfigurationTestStep | null
        payload?: any
    }
    setConfigurationManualErrors: (errors: Record<string, any>) => {
        errors: Record<string, any>
    }
    setConfigurationValue: (
        key: FieldName,
        value: any
    ) => {
        name: FieldName
        value: any
    }
    setConfigurationValues: (values: DeepPartial<Record<string, any>>) => {
        values: DeepPartial<Record<string, any>>
    }
    setRunningStep: (step: number | null) => {
        step: number | null
    }
    setSavedConfiguration: (configuration: Record<string, any>) => {
        configuration: Record<string, any>
    }
    submitConfiguration: () => {
        value: boolean
    }
    submitConfigurationFailure: (
        error: Error,
        errors: Record<string, any>
    ) => {
        error: Error
        errors: Record<string, any>
    }
    submitConfigurationRequest: (configuration: Record<string, any>) => {
        configuration: Record<string, any>
    }
    submitConfigurationSuccess: (configuration: Record<string, any>) => {
        configuration: Record<string, any>
    }
    touchConfigurationField: (key: string) => {
        key: string
    }
    updateBatchExportConfigSuccess: (batchExportConfig: BatchExportConfiguration) => {
        batchExportConfig: BatchExportConfiguration
    }
    updateBatchExportConfigTest: (service: any) => any
    updateBatchExportConfigTestFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    updateBatchExportConfigTestSuccess: (
        batchExportConfigTest: BatchExportConfigurationTest | null,
        payload?: any
    ) => {
        batchExportConfigTest: BatchExportConfigurationTest | null
        payload?: any
    }
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicMeta {
    key: string
    __keaTypeGenInternalSelectorTypes: {
        logicProps: (arg: any) => any
        selectedModel: (configuration: Record<string, any>) => string
        service: (
            batchExportConfig: BatchExportConfiguration | null,
            service:
                | 'AwsS3'
                | 'AzureBlob'
                | 'BigQuery'
                | 'Databricks'
                | 'HTTP'
                | 'Postgres'
                | 'Redshift'
                | 'S3Compatible'
                | 'Snowflake'
                | null
        ) =>
            | 'AwsS3'
            | 'AzureBlob'
            | 'BigQuery'
            | 'Databricks'
            | 'HTTP'
            | 'Postgres'
            | 'Redshift'
            | 'S3Compatible'
            | 'Snowflake'
            | null
        isNew: (id: string | null) => boolean
        loading: (batchExportConfigLoading: boolean, batchExportConfigTestLoading: boolean) => boolean
        isDatabaseDestination: (
            service:
                | 'AwsS3'
                | 'AzureBlob'
                | 'BigQuery'
                | 'Databricks'
                | 'HTTP'
                | 'Postgres'
                | 'Redshift'
                | 'S3Compatible'
                | 'Snowflake'
                | null
        ) => boolean
        selectedIntegration: (
            integrations: IntegrationType[] | null,
            configuration: Record<string, any>
        ) => IntegrationType | null
        requiredFields: (
            service:
                | 'AwsS3'
                | 'AzureBlob'
                | 'BigQuery'
                | 'Databricks'
                | 'HTTP'
                | 'Postgres'
                | 'Redshift'
                | 'S3Compatible'
                | 'Snowflake'
                | null,
            isNew: boolean,
            configuration: Record<string, any>,
            selectedIntegration: IntegrationType | null
        ) => string[]
    }
}

export type batchExportConfigFormLogicType = MakeLogicType<
    batchExportConfigFormLogicValues,
    batchExportConfigFormLogicActions,
    BatchExportConfigFormLogicProps,
    batchExportConfigFormLogicMeta
>

// Form logic for creating and editing batch export configurations.
// Owns form state, validation, dirty-checking, test steps, and save/delete actions.
// Reads the underlying config data from batchExportDataLogic.
// Per-destination behaviour (defaults, required fields, payload assembly, validation, JSX) lives
// in the registry under ./destinations/. This file is destination-agnostic.
export const batchExportConfigFormLogic = kea<batchExportConfigFormLogicType>([
    props({ id: null, service: null } as BatchExportConfigFormLogicProps),
    key(({ service, id }: BatchExportConfigFormLogicProps) => {
        if (id) {
            return `ID:${id}`
        }
        return `NEW:${service}`
    }),
    path((key) => ['scenes', 'data-pipelines', 'batch-exports', 'batchExportConfigFormLogic', key]),
    connect((props: BatchExportConfigFormLogicProps) => ({
        values: [
            teamLogic,
            ['timezone as teamTimezone', 'weekStartDay as teamWeekStartDay'],
            batchExportDataLogic({ id: props.id }),
            ['batchExportConfig', 'batchExportConfigLoading'],
            integrationsLogic,
            ['integrations'],
        ],
        actions: [
            batchExportDataLogic({ id: props.id }),
            ['loadBatchExportConfig', 'loadBatchExportConfigSuccess', 'setBatchExportConfig'],
        ],
    })),
    actions({
        setSavedConfiguration: (configuration: Record<string, any>) => ({ configuration }),
        setRunningStep: (step: number | null) => ({ step }),
        deleteBatchExport: () => true,
        updateBatchExportConfigSuccess: (batchExportConfig: BatchExportConfiguration) => ({ batchExportConfig }),
    }),
    loaders(({ props, values, actions }) => ({
        batchExportConfigTest: [
            null as BatchExportConfigurationTest | null,
            {
                loadBatchExportConfigTest: async () => {
                    if (props.service) {
                        try {
                            return await api.batchExports.test(props.service)
                        } catch {
                            return null
                        }
                    }
                    return null
                },
                updateBatchExportConfigTest: async (service) => {
                    if (service) {
                        try {
                            return await api.batchExports.test(service)
                        } catch {
                            return null
                        }
                    }
                    return null
                },
            },
        ],
        batchExportConfigTestStep: [
            null as BatchExportConfigurationTestStep | null,
            {
                runBatchExportConfigTestStep: async (step) => {
                    if (!values.batchExportConfigTest) {
                        return null
                    }

                    actions.setRunningStep(step)
                    if (step === 0) {
                        // TODO: Allow re-running steps that failed
                        values.batchExportConfigTest.steps.forEach((step) => {
                            step.result = null
                        })
                    }

                    const data = buildBatchExportPayload(values.configuration)

                    if (props.id) {
                        return await api.batchExports.runTestStep(props.id, step, data)
                    }
                    return await api.batchExports.runTestStepNew(step, data)
                },
            },
        ],
    })),
    reducers(({ props }) => ({
        tables: [
            props.service
                ? [getEventTable(props.service), personsTable, sessionsTable]
                : ([] as DatabaseSchemaBatchExportTable[]),
            {
                loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return [getEventTable(batchExportConfig.destination.type), personsTable, sessionsTable]
                },
                updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return [getEventTable(batchExportConfig.destination.type), personsTable, sessionsTable]
                },
            },
        ],
        configuration: [
            props.service ? getDefaultConfiguration(props.service) : ({} as Record<string, any>),
            {
                loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return getConfigurationFromBatchExportConfig(batchExportConfig)
                },
                updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return getConfigurationFromBatchExportConfig(batchExportConfig)
                },
            },
        ],
        savedConfiguration: [
            {} as Record<string, any>,
            {
                loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return getConfigurationFromBatchExportConfig(batchExportConfig)
                },
                updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
                    if (!batchExportConfig) {
                        return state
                    }

                    return getConfigurationFromBatchExportConfig(batchExportConfig)
                },
            },
        ],
        runningStep: [
            null as number | null,
            {
                setRunningStep: (_, { step }) => step,
            },
        ],
    })),
    selectors(() => ({
        logicProps: [() => [(_, props) => props], (props) => props],
        selectedModel: [
            (s) => [s.configuration],
            (configuration: Record<string, any>): string => configuration.model ?? BatchExportModelEnumApi.Events,
        ],
        service: [
            (s, p) => [s.batchExportConfig, p.service],
            (
                config: BatchExportConfiguration | null,
                service:
                    | 'AwsS3'
                    | 'AzureBlob'
                    | 'BigQuery'
                    | 'Databricks'
                    | 'HTTP'
                    | 'Postgres'
                    | 'Redshift'
                    | 'S3Compatible'
                    | 'Snowflake'
                    | null
            ) => config?.destination.type || service,
        ],
        isNew: [(_, p) => [p.id], (id: string | null): boolean => !id],
        loading: [
            (s) => [s.batchExportConfigLoading, s.batchExportConfigTestLoading],
            (batchExportConfigLoading: boolean, batchExportConfigTestLoading: boolean) =>
                batchExportConfigLoading || batchExportConfigTestLoading,
        ],
        isDatabaseDestination: [
            (s) => [s.service],
            (
                service:
                    | 'AwsS3'
                    | 'AzureBlob'
                    | 'BigQuery'
                    | 'Databricks'
                    | 'HTTP'
                    | 'Postgres'
                    | 'Redshift'
                    | 'S3Compatible'
                    | 'Snowflake'
                    | null
            ): boolean =>
                !!service && ['Postgres', 'Redshift', 'Snowflake', 'Databricks', 'BigQuery'].includes(service),
        ],
        // The Integration picked in the form, for destinations whose required fields depend on what
        // the Integration itself stores (a plain Redshift connection carries its own host, an AWS
        // one does not).
        selectedIntegration: [
            (s) => [s.integrations, s.configuration],
            (integrations: IntegrationType[] | null, config: Record<string, any>): IntegrationType | null =>
                integrations?.find((integration) => integration.id === config.integration_id) ?? null,
        ],
        requiredFields: [
            (s) => [s.service, s.isNew, s.configuration, s.selectedIntegration],
            (
                service:
                    | 'AwsS3'
                    | 'AzureBlob'
                    | 'BigQuery'
                    | 'Databricks'
                    | 'HTTP'
                    | 'Postgres'
                    | 'Redshift'
                    | 'S3Compatible'
                    | 'Snowflake'
                    | null,
                isNew: boolean,
                config: Record<string, any>,
                selectedIntegration: IntegrationType | null
            ): string[] => {
                const generalRequiredFields = [
                    'interval',
                    'name',
                    'model',
                    ...(config.model === BatchExportModelEnumApi.Hogql ? ['hogql_query'] : []),
                ]
                if (!service) {
                    return generalRequiredFields
                }
                const definition = DESTINATIONS[service as BatchExportService['type']]
                if (!definition) {
                    return generalRequiredFields
                }
                return [
                    ...generalRequiredFields,
                    ...definition.requiredFields({ isNew, formValues: config, selectedIntegration }),
                ]
            },
        ],
    })),
    listeners(({ values, actions }) => ({
        updateBatchExportConfigSuccess: ({ batchExportConfig }) => {
            if (!batchExportConfig) {
                return
            }

            // Reset so that form doesn't think there are unsaved changes.
            actions.resetConfiguration(getConfigurationFromBatchExportConfig(batchExportConfig))
        },
        loadBatchExportConfigSuccess: ({ batchExportConfig }) => {
            if (!batchExportConfig) {
                return
            }

            actions.updateBatchExportConfigTest(batchExportConfig.destination.type)

            // Set timezone to team's timezone if interval is day/week but timezone is not set
            // Check values.configuration since the reducer has already updated it
            if (
                (values.configuration.interval === 'day' || values.configuration.interval === 'week') &&
                !values.configuration.timezone
            ) {
                const teamTz = values.teamTimezone || 'UTC'
                actions.setConfigurationValue('timezone', teamTz)
            }
        },
        runBatchExportConfigTestStepSuccess: ({ batchExportConfigTestStep }) => {
            if (!values.batchExportConfigTest) {
                return
            }

            const step = batchExportConfigTestStep || values.batchExportConfigTestStep
            if (!step) {
                return
            }

            const index = values.batchExportConfigTest.steps.findIndex((item) => item.name === step.name)

            if (index > -1) {
                values.batchExportConfigTest.steps[index] = step

                if (
                    step.result &&
                    (step.result.status === 'Passed' || step.result.status === 'Skipped') &&
                    index < values.batchExportConfigTest.steps.length - 1
                ) {
                    actions.runBatchExportConfigTestStep(index + 1)
                } else {
                    actions.setRunningStep(null)
                }
            }
        },
        runBatchExportConfigTestStepFailure: () => {
            if (!values.batchExportConfigTest || !values.runningStep) {
                return
            }

            values.batchExportConfigTest.steps[values.runningStep].result = {
                status: 'Failed',
                message: `The batch export configuration could not be correctly serialized. Required fields may be missing or have invalid values`,
            }
            actions.setRunningStep(null)
        },
        setConfigurationValue: ({ name, value }) => {
            const fieldName = Array.isArray(name) ? name[0] : name

            // kea-forms keeps a manual error until the field is touched, and a manual error blocks submitting.
            // A save error describes the old value, so it must not outlive an edit.
            if (fieldName in values.configurationManualErrors) {
                actions.touchConfigurationField(String(fieldName))
            }
            // Other models do not send the query, so its save error must not block them while the field is hidden
            if (fieldName === 'model' && 'hogql_query' in values.configurationManualErrors) {
                actions.touchConfigurationField('hogql_query')
            }

            if (fieldName === 'file_format') {
                // Pick a compression that's valid for the newly-selected format, in priority order:
                //   1. keep the current codec if it still fits (e.g. gzip works for both formats);
                //   2. otherwise, when returning to the format the export was saved with, restore the
                //      persisted codec — so a Parquet→JSONLines→Parquet round-trip recovers the saved
                //      compression rather than stranding the export on a default;
                //   3. otherwise fall back to the format's default (zstd for Parquet, none for JSONLines).
                const current = values.configuration.compression
                const saved = values.savedConfiguration
                let next: string | null
                if (current !== null && isSelectedCompressionOptionValid(value, current)) {
                    next = current
                } else if (value === saved.file_format && isSelectedCompressionOptionValid(value, saved.compression)) {
                    next = saved.compression
                } else {
                    next = value === 'Parquet' ? 'zstd' : null
                }
                if (next !== current) {
                    actions.setConfigurationValue('compression', next)
                }
            }

            if (fieldName === 'interval') {
                // if changing to day or week, set the timezone to the team's timezone if not already set
                if (value === 'day' || value === 'week') {
                    // if we didn't have a timezone set before, set it to the team's timezone
                    if (values.savedConfiguration.interval !== 'day' && values.savedConfiguration.interval !== 'week') {
                        const teamTz = values.teamTimezone || 'UTC'
                        actions.setConfigurationValue('timezone', teamTz)
                    }
                    // if changing to week, set the day of the week to the team's week start day
                    if (value === 'week') {
                        const weekStartDay = values.teamWeekStartDay || 0
                        actions.setConfigurationValue('offset_day', weekStartDay)
                        actions.setConfigurationValue('offset_hour', 0)
                    }
                } else {
                    // Clear timezone and offset when interval is not day or week
                    actions.setConfigurationValue('timezone', null)
                    actions.setConfigurationValue('offset_day', null)
                    actions.setConfigurationValue('offset_hour', null)
                }
            }
        },
        deleteBatchExport: async () => {
            // TODO: support undo'ing a delete
            const batchExportId = values.batchExportConfig?.id
            if (!batchExportId) {
                return
            }
            try {
                await api.batchExports.delete(batchExportId)
                lemonToast.success('Batch export deleted successfully')
                router.actions.replace(urls.destinations('batch'))
            } catch (error: any) {
                // Show error toast with the error message from the API
                const errorMessage = error.detail || error.message || 'Failed to delete'
                lemonToast.error(errorMessage)
            }
        },
    })),
    forms(({ actions, props, values }) => ({
        configuration: {
            errors: (formdata) => {
                const requiredFieldErrors = Object.fromEntries(
                    values.requiredFields.map((field) => [
                        field,
                        formdata[field] ? undefined : 'This field is required',
                    ])
                )

                const destination = formdata.destination as BatchExportService['type'] | undefined
                const definition = destination ? DESTINATIONS[destination] : undefined
                const fieldValidations = definition?.validate?.(formdata) ?? {}

                // Only apply a field validation when it produced a message — a valid (undefined)
                // result must not clobber a "required" error from the same empty field.
                const errors: Record<string, string | undefined> = { ...requiredFieldErrors }
                for (const [field, message] of Object.entries(fieldValidations)) {
                    if (message) {
                        errors[field] = message
                    }
                }
                if (
                    formdata.model === BatchExportModelEnumApi.Hogql &&
                    splitQueries(formdata.hogql_query ?? '').length > 1
                ) {
                    errors.hogql_query =
                        'A batch export runs a single query. Remove the extra statements separated by semicolons.'
                }
                return errors
            },
            submit: async (formdata) => {
                // Check if schedule fields have changed and show confirmation modal
                const scheduleFieldsChanged =
                    formdata.interval !== values.savedConfiguration.interval ||
                    formdata.timezone !== values.savedConfiguration.timezone ||
                    formdata.offset_day !== values.savedConfiguration.offset_day ||
                    formdata.offset_hour !== values.savedConfiguration.offset_hour

                if (!values.isNew && scheduleFieldsChanged) {
                    let userConfirmed = false
                    await new Promise<void>((resolve) => {
                        LemonDialog.open({
                            title: 'Confirm schedule change',
                            description: (
                                <>
                                    <p>
                                        Changing the schedule (interval, timezone, or start time) of a batch export
                                        could result in a gap of data.
                                    </p>
                                    <p>
                                        Make sure to run a backfill if necessary to ensure all data is exported
                                        correctly.
                                    </p>
                                </>
                            ),
                            primaryButton: {
                                children: 'Save changes',
                                onClick: () => {
                                    userConfirmed = true
                                    resolve()
                                },
                            },
                            secondaryButton: {
                                children: 'Cancel',
                                onClick: () => {
                                    userConfirmed = false
                                    resolve()
                                },
                            },
                        })
                    })

                    // Only proceed with submission if user confirmed
                    if (!userConfirmed) {
                        return
                    }
                }

                const data = buildBatchExportPayload(formdata)

                try {
                    if (props.id) {
                        const res = await api.batchExports.update(props.id, data)
                        lemonToast.success('Batch export configuration updated successfully')
                        void addProductIntent({
                            product_type: ProductKey.PIPELINE_BATCH_EXPORTS,
                            intent_context: ProductIntentContext.BATCH_EXPORT_UPDATED,
                        })
                        actions.setBatchExportConfig(res)
                        actions.updateBatchExportConfigSuccess(res)
                        return
                    }
                    const res = await api.batchExports.create(data)
                    actions.resetConfiguration(getConfigurationFromBatchExportConfig(res))

                    void addProductIntent({
                        product_type: ProductKey.PIPELINE_BATCH_EXPORTS,
                        intent_context: ProductIntentContext.BATCH_EXPORT_CREATED,
                    })

                    router.actions.replace(urls.batchExport(res.id))
                    lemonToast.success('Batch export created successfully')
                    actions.updateBatchExportConfigSuccess(res)
                } catch (error: any) {
                    lemonToast.error(error.detail || error.message || 'Could not save the batch export. Try again.')
                    // The toast can be far from the field at fault, such as the query editor, so the error also shows under it
                    if (error.attr && error.detail && error.attr in values.configuration) {
                        actions.setConfigurationManualErrors({ [error.attr]: error.detail })
                    }
                    // kea-forms hides form errors after a successful submit, so a failed save has to fail the submit
                    throw error
                }
            },
        },
    })),
    beforeUnload(({ actions, values }) => ({
        enabled: () => values.configurationChanged,
        message: 'Leave action?\nChanges you made will be discarded.',
        onConfirm: () => {
            values.batchExportConfig
                ? actions.resetConfiguration(getConfigurationFromBatchExportConfig(values.batchExportConfig))
                : values.service
                  ? actions.resetConfiguration(getDefaultConfiguration(values.service))
                  : actions.resetConfiguration()
        },
    })),

    afterMount(({ actions }) => {
        actions.loadBatchExportConfigTest()
    }),
])