:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import { forms } from 'kea-forms'
import type { DeepPartial, DeepPartialMap, FieldName, ValidationErrorType } from 'kea-forms'
import { loaders } from 'kea-loaders'
import { beforeUnload, router } from 'kea-router'
import { LemonDialog, lemonToast } from '@posthog/lemon-ui'
import api from 'lib/api'
import { integrationsLogic } from 'lib/integrations/integrationsLogic'
import { splitQueries } from 'lib/monaco/multiQueryUtils'
import { addProductIntent } from 'lib/utils/product-intents'
import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
import { DatabaseSchemaBatchExportTable, ProductIntentContext, ProductKey } from '~/queries/schema/schema-general'
import {
BatchExportConfiguration,
BatchExportConfigurationTest,
BatchExportConfigurationTestStep,
BatchExportService,
IntegrationType,
} from '~/types'
import { BatchExportModelEnumApi } from 'products/batch_exports/frontend/generated/api.schemas'
import { batchExportDataLogic } from './batchExportDataLogic'
import { DESTINATIONS } from './destinations'
import { genericPersonEventFields, isSelectedCompressionOptionValid } from './destinations/common'
import { humanizeBatchExportName } from './utils'
export interface BatchExportConfigFormLogicProps {
service: BatchExportService['type'] | null
id: string | null
}
// Fields that exist on the form but are not part of destination.config
const TOP_LEVEL_FORM_FIELDS = new Set([
'name',
'destination',
'interval',
'timezone',
'offset_day',
'offset_hour',
'paused',
'created_at',
'start_at',
'end_at',
'model',
'hogql_query',
'hogql_modifiers',
'filters',
'integration_id',
])
const ALLOWED_BASE_CONFIG_KEYS = new Set(['exclude_events', 'include_events'])
function buildDestinationPayload(formValues: Record<string, any>): {
type: string
config: Record<string, any>
integration?: any
} {
const destinationType = formValues.destination as BatchExportService['type']
const definition = DESTINATIONS[destinationType]
// Apply destination-specific transform first (e.g. Redshift's COPY copy_inputs assembly).
// Destinations without a custom serialize get the raw form values passed through.
const intermediate = definition?.serialize ? definition.serialize(formValues) : formValues
// When a destination declares configKeys, drop any config key outside it (plus the base-export
// keys). Mirrors the backend's allowed = destination_fields ∪ base_field_names check, so stale
// fields don't survive a deserialize → serialize round-trip and get rejected on save.
const allowed = definition?.configKeys ? new Set([...definition.configKeys, ...ALLOWED_BASE_CONFIG_KEYS]) : null
// Strip top-level form fields that don't belong in destination.config
const config: Record<string, any> = {}
for (const [key, value] of Object.entries(intermediate)) {
if (TOP_LEVEL_FORM_FIELDS.has(key)) {
continue
}
if (allowed && !allowed.has(key)) {
continue
}
config[key] = value
}
// A persisted compression value can be invalid for the selected file_format (e.g. an
// externally-created JSONLines export still carrying a Parquet-only codec). Drop it on save so
// editing an unrelated field doesn't resubmit a combination the backend rejects.
if ('compression' in config && !isSelectedCompressionOptionValid(config.file_format, config.compression)) {
config.compression = null
}
const result: { type: string; config: Record<string, any>; integration?: any } = {
type: destinationType,
config,
}
if (definition?.usesIntegration) {
result.integration = formValues.integration_id
}
return result
}
function buildBatchExportPayload(formValues: Record<string, any>): Partial<BatchExportConfiguration> {
const interval = formValues.interval
return {
paused: formValues.paused,
name: formValues.name,
interval,
timezone: interval === 'day' || interval === 'week' ? formValues.timezone : null,
offset_day: interval === 'week' ? formValues.offset_day : null,
offset_hour: interval === 'day' || interval === 'week' ? formValues.offset_hour : null,
model: formValues.model,
// Only the 'hogql' model edits the query and its modifiers. The events model would save a query as its
// export schema, and the API rejects modifiers for every other model.
hogql_query: formValues.model === BatchExportModelEnumApi.Hogql ? formValues.hogql_query : undefined,
hogql_modifiers: formValues.model === BatchExportModelEnumApi.Hogql ? formValues.hogql_modifiers : undefined,
// Filters only apply to the events model: the API rejects them for 'hogql' and runs ignore them otherwise
filters: formValues.model === BatchExportModelEnumApi.Events ? formValues.filters : undefined,
destination: buildDestinationPayload(formValues) as any,
}
}
function getConfigurationFromBatchExportConfig(batchExportConfig: BatchExportConfiguration): Record<string, any> {
const destinationType = batchExportConfig.destination.type
const definition = DESTINATIONS[destinationType]
const flatConfig = definition?.deserialize
? definition.deserialize(batchExportConfig.destination.config)
: { ...batchExportConfig.destination.config }
const config: Record<string, any> = {
name: batchExportConfig.name,
destination: destinationType,
paused: batchExportConfig.paused,
interval: batchExportConfig.interval,
timezone: batchExportConfig.timezone,
offset_day: (batchExportConfig as any).offset_day ?? null,
offset_hour: (batchExportConfig as any).offset_hour ?? null,
model: batchExportConfig.model,
hogql_query: batchExportConfig.hogql_query ?? null,
hogql_modifiers: batchExportConfig.hogql_modifiers ?? null,
filters: batchExportConfig.filters,
...flatConfig,
}
if (definition?.usesIntegration) {
config.integration_id = (batchExportConfig.destination as { integration?: number }).integration
}
return config
}
// $feature_flag_called is high volume and is being deprecated from the events table, so new exports
// leave it out by default. It shows up prefilled under "Exclude events" and can be removed there.
export const DEFAULT_EXCLUDE_EVENTS = ['$feature_flag_called']
export function getDefaultConfiguration(service: string): Record<string, any> {
const definition = DESTINATIONS[service as BatchExportService['type']]
return {
name: humanizeBatchExportName(service as BatchExportService['type']),
destination: service,
model: BatchExportModelEnumApi.Events,
paused: true,
exclude_events: [...DEFAULT_EXCLUDE_EVENTS],
...(definition ? definition.defaults() : {}),
}
}
const BASE_EVENT_FIELDS = {
uuid: {
name: 'uuid',
hogql_value: 'toString(uuid)',
type: 'string',
schema_valid: true,
},
timestamp: {
name: 'timestamp',
hogql_value: 'timestamp',
type: 'datetime',
schema_valid: true,
},
event: {
name: 'event',
hogql_value: 'event',
type: 'string',
schema_valid: true,
},
distinct_id: {
name: 'distinct_id',
hogql_value: 'toString(distinct_id)',
type: 'string',
schema_valid: true,
},
properties: {
name: 'properties',
hogql_value: 'properties',
type: 'json',
schema_valid: true,
},
} as const
function getEventTable(service: BatchExportService['type']): DatabaseSchemaBatchExportTable {
const definition = DESTINATIONS[service]
const overrides = definition?.eventTableOverrides ?? {}
const includeGeneric = overrides.includeGenericPersonFields !== false
return {
type: 'batch_export',
id: 'Events',
name: 'events',
fields: {
...BASE_EVENT_FIELDS,
...(includeGeneric
? genericPersonEventFields({
teamIdHogql: overrides.teamIdHogql ?? 'team_id',
setName: overrides.setName ?? 'set',
setOnceName: overrides.setOnceName ?? 'set_once',
})
: {}),
...definition?.eventTableExtraFields,
},
}
}
const personsTable: DatabaseSchemaBatchExportTable = {
type: 'batch_export',
id: 'Persons',
name: 'persons',
fields: {
team_id: {
name: 'team_id',
hogql_value: 'team_id',
type: 'integer',
schema_valid: true,
},
distinct_id: {
name: 'distinct_id',
hogql_value: 'distinct_id',
type: 'string',
schema_valid: true,
},
person_id: {
name: 'person_id',
hogql_value: 'person_id',
type: 'string',
schema_valid: true,
},
properties: {
name: 'properties',
hogql_value: 'properties',
type: 'json',
schema_valid: true,
},
person_version: {
name: 'person_version',
hogql_value: 'person_version',
type: 'integer',
schema_valid: true,
},
person_distinct_id_version: {
name: 'person_distinct_id_version',
hogql_value: 'person_distinct_id_version',
type: 'integer',
schema_valid: true,
},
created_at: {
name: 'created_at',
hogql_value: 'created_at',
type: 'datetime',
schema_valid: true,
},
is_deleted: {
name: 'is_deleted',
hogql_value: 'is_deleted',
type: 'boolean',
schema_valid: true,
},
},
}
const sessionsTable: DatabaseSchemaBatchExportTable = {
type: 'batch_export',
id: 'Sessions',
name: 'sessions',
fields: {
team_id: {
name: 'team_id',
hogql_value: 'team_id',
type: 'integer',
schema_valid: true,
},
session_id: {
name: 'session_id',
type: 'string',
hogql_value: 'session_id',
schema_valid: true,
},
session_id_v7: {
name: 'session_id_v7',
type: 'string',
hogql_value: 'session_id_v7',
schema_valid: true,
},
distinct_id: {
name: 'distinct_id',
type: 'string',
hogql_value: 'distinct_id',
schema_valid: true,
},
start_timestamp: {
name: 'start_timestamp',
type: 'datetime',
hogql_value: 'start_timestamp',
schema_valid: true,
},
end_timestamp: {
name: 'end_timestamp',
type: 'datetime',
hogql_value: 'end_timestamp',
schema_valid: true,
},
urls: {
name: 'urls',
type: 'array',
hogql_value: 'urls',
schema_valid: true,
},
num_uniq_urls: {
name: 'num_uniq_urls',
type: 'integer',
hogql_value: 'num_uniq_urls',
schema_valid: true,
},
entry_current_url: {
name: 'entry_current_url',
type: 'string',
hogql_value: 'entry_current_url',
schema_valid: true,
},
entry_pathname: {
name: 'entry_pathname',
type: 'string',
hogql_value: 'entry_pathname',
schema_valid: true,
},
entry_hostname: {
name: 'entry_hostname',
type: 'string',
hogql_value: 'entry_hostname',
schema_valid: true,
},
end_current_url: {
name: 'end_current_url',
type: 'string',
hogql_value: 'end_current_url',
schema_valid: true,
},
end_pathname: {
name: 'end_pathname',
type: 'string',
hogql_value: 'end_pathname',
schema_valid: true,
},
end_hostname: {
name: 'end_hostname',
type: 'string',
hogql_value: 'end_hostname',
schema_valid: true,
},
entry_utm_source: {
name: 'entry_utm_source',
type: 'string',
hogql_value: 'entry_utm_source',
schema_valid: true,
},
entry_utm_campaign: {
name: 'entry_utm_campaign',
type: 'string',
hogql_value: 'entry_utm_campaign',
schema_valid: true,
},
entry_utm_medium: {
name: 'entry_utm_medium',
type: 'string',
hogql_value: 'entry_utm_medium',
schema_valid: true,
},
entry_utm_term: {
name: 'entry_utm_term',
type: 'string',
hogql_value: 'entry_utm_term',
schema_valid: true,
},
entry_utm_content: {
name: 'entry_utm_content',
type: 'string',
hogql_value: 'entry_utm_content',
schema_valid: true,
},
entry_referring_domain: {
name: 'entry_referring_domain',
type: 'string',
hogql_value: 'entry_referring_domain',
schema_valid: true,
},
entry_gclid: {
name: 'entry_gclid',
type: 'string',
hogql_value: 'entry_gclid',
schema_valid: true,
},
entry_fbclid: {
name: 'entry_fbclid',
type: 'string',
hogql_value: 'entry_fbclid',
schema_valid: true,
},
entry_gad_source: {
name: 'entry_gad_source',
type: 'string',
hogql_value: 'entry_gad_source',
schema_valid: true,
},
pageview_count: {
name: 'pageview_count',
type: 'integer',
hogql_value: 'pageview_count',
schema_valid: true,
},
autocapture_count: {
name: 'autocapture_count',
type: 'integer',
hogql_value: 'autocapture_count',
schema_valid: true,
},
screen_count: {
name: 'screen_count',
type: 'integer',
hogql_value: 'screen_count',
schema_valid: true,
},
channel_type: {
name: 'channel_type',
type: 'string',
hogql_value: 'channel_type',
schema_valid: true,
},
session_duration: {
name: 'session_duration',
type: 'integer',
hogql_value: 'session_duration',
schema_valid: true,
},
duration: {
name: 'duration',
type: 'integer',
hogql_value: 'duration',
schema_valid: true,
},
is_bounce: {
name: 'is_bounce',
type: 'boolean',
hogql_value: 'is_bounce',
schema_valid: true,
},
last_external_click_url: {
name: 'last_external_click_url',
type: 'string',
hogql_value: 'last_external_click_url',
schema_valid: true,
},
page_screen_autocapture_count_up_to: {
name: 'page_screen_autocapture_count_up_to',
type: 'string',
hogql_value: 'page_screen_autocapture_count_up_to',
schema_valid: true,
},
exit_current_url: {
name: 'exit_current_url',
type: 'string',
hogql_value: 'exit_current_url',
schema_valid: true,
},
exit_pathname: {
name: 'exit_pathname',
type: 'string',
hogql_value: 'exit_pathname',
schema_valid: true,
},
vitals_lcp: {
name: 'vitals_lcp',
type: 'float',
hogql_value: 'vitals_lcp',
schema_valid: true,
},
entry_gclsrc: {
name: 'entry_gclsrc',
type: 'string',
hogql_value: 'entry_gclsrc',
schema_valid: true,
},
entry_dclid: {
name: 'entry_dclid',
type: 'string',
hogql_value: 'entry_dclid',
schema_valid: true,
},
entry_gbraid: {
name: 'entry_gbraid',
type: 'string',
hogql_value: 'entry_gbraid',
schema_valid: true,
},
entry_wbraid: {
name: 'entry_wbraid',
type: 'string',
hogql_value: 'entry_wbraid',
schema_valid: true,
},
entry_msclkid: {
name: 'entry_msclkid',
type: 'string',
hogql_value: 'entry_msclkid',
schema_valid: true,
},
entry_twclid: {
name: 'entry_twclid',
type: 'string',
hogql_value: 'entry_twclid',
schema_valid: true,
},
entry_li_fat_id: {
name: 'entry_li_fat_id',
type: 'string',
hogql_value: 'entry_li_fat_id',
schema_valid: true,
},
entry_mc_cid: {
name: 'entry_mc_cid',
type: 'string',
hogql_value: 'entry_mc_cid',
schema_valid: true,
},
entry_igshid: {
name: 'entry_igshid',
type: 'string',
hogql_value: 'entry_igshid',
schema_valid: true,
},
entry_ttclid: {
name: 'entry_ttclid',
type: 'string',
hogql_value: 'entry_ttclid',
schema_valid: true,
},
entry__kx: {
name: 'entry__kx',
type: 'string',
hogql_value: 'entry__kx',
schema_valid: true,
},
entry_irclid: {
name: 'entry_irclid',
type: 'string',
hogql_value: 'entry_irclid',
schema_valid: true,
},
},
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicValues {
batchExportConfig: BatchExportConfiguration | null // batchExportDataLogic
batchExportConfigLoading: boolean // batchExportDataLogic
integrations: IntegrationType[] | null // integrationsLogic
teamTimezone: string // teamLogic
teamWeekStartDay: number // teamLogic
batchExportConfigTest: BatchExportConfigurationTest | null
batchExportConfigTestLoading: boolean
batchExportConfigTestStep: BatchExportConfigurationTestStep | null
batchExportConfigTestStepLoading: boolean
configuration: Record<string, any>
configurationAllErrors: Record<string, any>
configurationChanged: boolean
configurationErrors: DeepPartialMap<Record<string, any>, ValidationErrorType>
configurationHasErrors: boolean
configurationManualErrors: Record<string, any>
configurationTouched: boolean
configurationTouches: Record<string, boolean>
configurationValidationErrors: DeepPartialMap<Record<string, any>, ValidationErrorType>
isConfigurationSubmitting: boolean
isConfigurationValid: boolean
isDatabaseDestination: boolean
isNew: boolean
loading: boolean
logicProps: any
requiredFields: string[]
runningStep: number | null
savedConfiguration: Record<string, any>
selectedIntegration: IntegrationType | null
selectedModel: string
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
showConfigurationErrors: boolean
tables: DatabaseSchemaBatchExportTable[]
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicActions {
loadBatchExportConfig: () => any // batchExportDataLogic
loadBatchExportConfigSuccess: (
batchExportConfig: BatchExportConfiguration | null,
payload?: any
) => {
batchExportConfig: BatchExportConfiguration | null
payload?: any
} // batchExportDataLogic
setBatchExportConfig: (config: BatchExportConfiguration) => BatchExportConfiguration // batchExportDataLogic
deleteBatchExport: () => boolean
loadBatchExportConfigTest: () => any
loadBatchExportConfigTestFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadBatchExportConfigTestSuccess: (
batchExportConfigTest: BatchExportConfigurationTest | null,
payload?: any
) => {
batchExportConfigTest: BatchExportConfigurationTest | null
payload?: any
}
resetConfiguration: (values?: Record<string, any>) => {
values?: Record<string, any>
}
runBatchExportConfigTestStep: (step: any) => any
runBatchExportConfigTestStepFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
runBatchExportConfigTestStepSuccess: (
batchExportConfigTestStep: BatchExportConfigurationTestStep | null,
payload?: any
) => {
batchExportConfigTestStep: BatchExportConfigurationTestStep | null
payload?: any
}
setConfigurationManualErrors: (errors: Record<string, any>) => {
errors: Record<string, any>
}
setConfigurationValue: (
key: FieldName,
value: any
) => {
name: FieldName
value: any
}
setConfigurationValues: (values: DeepPartial<Record<string, any>>) => {
values: DeepPartial<Record<string, any>>
}
setRunningStep: (step: number | null) => {
step: number | null
}
setSavedConfiguration: (configuration: Record<string, any>) => {
configuration: Record<string, any>
}
submitConfiguration: () => {
value: boolean
}
submitConfigurationFailure: (
error: Error,
errors: Record<string, any>
) => {
error: Error
errors: Record<string, any>
}
submitConfigurationRequest: (configuration: Record<string, any>) => {
configuration: Record<string, any>
}
submitConfigurationSuccess: (configuration: Record<string, any>) => {
configuration: Record<string, any>
}
touchConfigurationField: (key: string) => {
key: string
}
updateBatchExportConfigSuccess: (batchExportConfig: BatchExportConfiguration) => {
batchExportConfig: BatchExportConfiguration
}
updateBatchExportConfigTest: (service: any) => any
updateBatchExportConfigTestFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
updateBatchExportConfigTestSuccess: (
batchExportConfigTest: BatchExportConfigurationTest | null,
payload?: any
) => {
batchExportConfigTest: BatchExportConfigurationTest | null
payload?: any
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface batchExportConfigFormLogicMeta {
key: string
__keaTypeGenInternalSelectorTypes: {
logicProps: (arg: any) => any
selectedModel: (configuration: Record<string, any>) => string
service: (
batchExportConfig: BatchExportConfiguration | null,
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
) =>
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
isNew: (id: string | null) => boolean
loading: (batchExportConfigLoading: boolean, batchExportConfigTestLoading: boolean) => boolean
isDatabaseDestination: (
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
) => boolean
selectedIntegration: (
integrations: IntegrationType[] | null,
configuration: Record<string, any>
) => IntegrationType | null
requiredFields: (
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null,
isNew: boolean,
configuration: Record<string, any>,
selectedIntegration: IntegrationType | null
) => string[]
}
}
export type batchExportConfigFormLogicType = MakeLogicType<
batchExportConfigFormLogicValues,
batchExportConfigFormLogicActions,
BatchExportConfigFormLogicProps,
batchExportConfigFormLogicMeta
>
// Form logic for creating and editing batch export configurations.
// Owns form state, validation, dirty-checking, test steps, and save/delete actions.
// Reads the underlying config data from batchExportDataLogic.
// Per-destination behaviour (defaults, required fields, payload assembly, validation, JSX) lives
// in the registry under ./destinations/. This file is destination-agnostic.
export const batchExportConfigFormLogic = kea<batchExportConfigFormLogicType>([
props({ id: null, service: null } as BatchExportConfigFormLogicProps),
key(({ service, id }: BatchExportConfigFormLogicProps) => {
if (id) {
return `ID:${id}`
}
return `NEW:${service}`
}),
path((key) => ['scenes', 'data-pipelines', 'batch-exports', 'batchExportConfigFormLogic', key]),
connect((props: BatchExportConfigFormLogicProps) => ({
values: [
teamLogic,
['timezone as teamTimezone', 'weekStartDay as teamWeekStartDay'],
batchExportDataLogic({ id: props.id }),
['batchExportConfig', 'batchExportConfigLoading'],
integrationsLogic,
['integrations'],
],
actions: [
batchExportDataLogic({ id: props.id }),
['loadBatchExportConfig', 'loadBatchExportConfigSuccess', 'setBatchExportConfig'],
],
})),
actions({
setSavedConfiguration: (configuration: Record<string, any>) => ({ configuration }),
setRunningStep: (step: number | null) => ({ step }),
deleteBatchExport: () => true,
updateBatchExportConfigSuccess: (batchExportConfig: BatchExportConfiguration) => ({ batchExportConfig }),
}),
loaders(({ props, values, actions }) => ({
batchExportConfigTest: [
null as BatchExportConfigurationTest | null,
{
loadBatchExportConfigTest: async () => {
if (props.service) {
try {
return await api.batchExports.test(props.service)
} catch {
return null
}
}
return null
},
updateBatchExportConfigTest: async (service) => {
if (service) {
try {
return await api.batchExports.test(service)
} catch {
return null
}
}
return null
},
},
],
batchExportConfigTestStep: [
null as BatchExportConfigurationTestStep | null,
{
runBatchExportConfigTestStep: async (step) => {
if (!values.batchExportConfigTest) {
return null
}
actions.setRunningStep(step)
if (step === 0) {
// TODO: Allow re-running steps that failed
values.batchExportConfigTest.steps.forEach((step) => {
step.result = null
})
}
const data = buildBatchExportPayload(values.configuration)
if (props.id) {
return await api.batchExports.runTestStep(props.id, step, data)
}
return await api.batchExports.runTestStepNew(step, data)
},
},
],
})),
reducers(({ props }) => ({
tables: [
props.service
? [getEventTable(props.service), personsTable, sessionsTable]
: ([] as DatabaseSchemaBatchExportTable[]),
{
loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return [getEventTable(batchExportConfig.destination.type), personsTable, sessionsTable]
},
updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return [getEventTable(batchExportConfig.destination.type), personsTable, sessionsTable]
},
},
],
configuration: [
props.service ? getDefaultConfiguration(props.service) : ({} as Record<string, any>),
{
loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return getConfigurationFromBatchExportConfig(batchExportConfig)
},
updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return getConfigurationFromBatchExportConfig(batchExportConfig)
},
},
],
savedConfiguration: [
{} as Record<string, any>,
{
loadBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return getConfigurationFromBatchExportConfig(batchExportConfig)
},
updateBatchExportConfigSuccess: (state, { batchExportConfig }) => {
if (!batchExportConfig) {
return state
}
return getConfigurationFromBatchExportConfig(batchExportConfig)
},
},
],
runningStep: [
null as number | null,
{
setRunningStep: (_, { step }) => step,
},
],
})),
selectors(() => ({
logicProps: [() => [(_, props) => props], (props) => props],
selectedModel: [
(s) => [s.configuration],
(configuration: Record<string, any>): string => configuration.model ?? BatchExportModelEnumApi.Events,
],
service: [
(s, p) => [s.batchExportConfig, p.service],
(
config: BatchExportConfiguration | null,
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
) => config?.destination.type || service,
],
isNew: [(_, p) => [p.id], (id: string | null): boolean => !id],
loading: [
(s) => [s.batchExportConfigLoading, s.batchExportConfigTestLoading],
(batchExportConfigLoading: boolean, batchExportConfigTestLoading: boolean) =>
batchExportConfigLoading || batchExportConfigTestLoading,
],
isDatabaseDestination: [
(s) => [s.service],
(
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null
): boolean =>
!!service && ['Postgres', 'Redshift', 'Snowflake', 'Databricks', 'BigQuery'].includes(service),
],
// The Integration picked in the form, for destinations whose required fields depend on what
// the Integration itself stores (a plain Redshift connection carries its own host, an AWS
// one does not).
selectedIntegration: [
(s) => [s.integrations, s.configuration],
(integrations: IntegrationType[] | null, config: Record<string, any>): IntegrationType | null =>
integrations?.find((integration) => integration.id === config.integration_id) ?? null,
],
requiredFields: [
(s) => [s.service, s.isNew, s.configuration, s.selectedIntegration],
(
service:
| 'AwsS3'
| 'AzureBlob'
| 'BigQuery'
| 'Databricks'
| 'HTTP'
| 'Postgres'
| 'Redshift'
| 'S3Compatible'
| 'Snowflake'
| null,
isNew: boolean,
config: Record<string, any>,
selectedIntegration: IntegrationType | null
): string[] => {
const generalRequiredFields = [
'interval',
'name',
'model',
...(config.model === BatchExportModelEnumApi.Hogql ? ['hogql_query'] : []),
]
if (!service) {
return generalRequiredFields
}
const definition = DESTINATIONS[service as BatchExportService['type']]
if (!definition) {
return generalRequiredFields
}
return [
...generalRequiredFields,
...definition.requiredFields({ isNew, formValues: config, selectedIntegration }),
]
},
],
})),
listeners(({ values, actions }) => ({
updateBatchExportConfigSuccess: ({ batchExportConfig }) => {
if (!batchExportConfig) {
return
}
// Reset so that form doesn't think there are unsaved changes.
actions.resetConfiguration(getConfigurationFromBatchExportConfig(batchExportConfig))
},
loadBatchExportConfigSuccess: ({ batchExportConfig }) => {
if (!batchExportConfig) {
return
}
actions.updateBatchExportConfigTest(batchExportConfig.destination.type)
// Set timezone to team's timezone if interval is day/week but timezone is not set
// Check values.configuration since the reducer has already updated it
if (
(values.configuration.interval === 'day' || values.configuration.interval === 'week') &&
!values.configuration.timezone
) {
const teamTz = values.teamTimezone || 'UTC'
actions.setConfigurationValue('timezone', teamTz)
}
},
runBatchExportConfigTestStepSuccess: ({ batchExportConfigTestStep }) => {
if (!values.batchExportConfigTest) {
return
}
const step = batchExportConfigTestStep || values.batchExportConfigTestStep
if (!step) {
return
}
const index = values.batchExportConfigTest.steps.findIndex((item) => item.name === step.name)
if (index > -1) {
values.batchExportConfigTest.steps[index] = step
if (
step.result &&
(step.result.status === 'Passed' || step.result.status === 'Skipped') &&
index < values.batchExportConfigTest.steps.length - 1
) {
actions.runBatchExportConfigTestStep(index + 1)
} else {
actions.setRunningStep(null)
}
}
},
runBatchExportConfigTestStepFailure: () => {
if (!values.batchExportConfigTest || !values.runningStep) {
return
}
values.batchExportConfigTest.steps[values.runningStep].result = {
status: 'Failed',
message: `The batch export configuration could not be correctly serialized. Required fields may be missing or have invalid values`,
}
actions.setRunningStep(null)
},
setConfigurationValue: ({ name, value }) => {
const fieldName = Array.isArray(name) ? name[0] : name
// kea-forms keeps a manual error until the field is touched, and a manual error blocks submitting.
// A save error describes the old value, so it must not outlive an edit.
if (fieldName in values.configurationManualErrors) {
actions.touchConfigurationField(String(fieldName))
}
// Other models do not send the query, so its save error must not block them while the field is hidden
if (fieldName === 'model' && 'hogql_query' in values.configurationManualErrors) {
actions.touchConfigurationField('hogql_query')
}
if (fieldName === 'file_format') {
// Pick a compression that's valid for the newly-selected format, in priority order:
// 1. keep the current codec if it still fits (e.g. gzip works for both formats);
// 2. otherwise, when returning to the format the export was saved with, restore the
// persisted codec — so a Parquet→JSONLines→Parquet round-trip recovers the saved
// compression rather than stranding the export on a default;
// 3. otherwise fall back to the format's default (zstd for Parquet, none for JSONLines).
const current = values.configuration.compression
const saved = values.savedConfiguration
let next: string | null
if (current !== null && isSelectedCompressionOptionValid(value, current)) {
next = current
} else if (value === saved.file_format && isSelectedCompressionOptionValid(value, saved.compression)) {
next = saved.compression
} else {
next = value === 'Parquet' ? 'zstd' : null
}
if (next !== current) {
actions.setConfigurationValue('compression', next)
}
}
if (fieldName === 'interval') {
// if changing to day or week, set the timezone to the team's timezone if not already set
if (value === 'day' || value === 'week') {
// if we didn't have a timezone set before, set it to the team's timezone
if (values.savedConfiguration.interval !== 'day' && values.savedConfiguration.interval !== 'week') {
const teamTz = values.teamTimezone || 'UTC'
actions.setConfigurationValue('timezone', teamTz)
}
// if changing to week, set the day of the week to the team's week start day
if (value === 'week') {
const weekStartDay = values.teamWeekStartDay || 0
actions.setConfigurationValue('offset_day', weekStartDay)
actions.setConfigurationValue('offset_hour', 0)
}
} else {
// Clear timezone and offset when interval is not day or week
actions.setConfigurationValue('timezone', null)
actions.setConfigurationValue('offset_day', null)
actions.setConfigurationValue('offset_hour', null)
}
}
},
deleteBatchExport: async () => {
// TODO: support undo'ing a delete
const batchExportId = values.batchExportConfig?.id
if (!batchExportId) {
return
}
try {
await api.batchExports.delete(batchExportId)
lemonToast.success('Batch export deleted successfully')
router.actions.replace(urls.destinations('batch'))
} catch (error: any) {
// Show error toast with the error message from the API
const errorMessage = error.detail || error.message || 'Failed to delete'
lemonToast.error(errorMessage)
}
},
})),
forms(({ actions, props, values }) => ({
configuration: {
errors: (formdata) => {
const requiredFieldErrors = Object.fromEntries(
values.requiredFields.map((field) => [
field,
formdata[field] ? undefined : 'This field is required',
])
)
const destination = formdata.destination as BatchExportService['type'] | undefined
const definition = destination ? DESTINATIONS[destination] : undefined
const fieldValidations = definition?.validate?.(formdata) ?? {}
// Only apply a field validation when it produced a message — a valid (undefined)
// result must not clobber a "required" error from the same empty field.
const errors: Record<string, string | undefined> = { ...requiredFieldErrors }
for (const [field, message] of Object.entries(fieldValidations)) {
if (message) {
errors[field] = message
}
}
if (
formdata.model === BatchExportModelEnumApi.Hogql &&
splitQueries(formdata.hogql_query ?? '').length > 1
) {
errors.hogql_query =
'A batch export runs a single query. Remove the extra statements separated by semicolons.'
}
return errors
},
submit: async (formdata) => {
// Check if schedule fields have changed and show confirmation modal
const scheduleFieldsChanged =
formdata.interval !== values.savedConfiguration.interval ||
formdata.timezone !== values.savedConfiguration.timezone ||
formdata.offset_day !== values.savedConfiguration.offset_day ||
formdata.offset_hour !== values.savedConfiguration.offset_hour
if (!values.isNew && scheduleFieldsChanged) {
let userConfirmed = false
await new Promise<void>((resolve) => {
LemonDialog.open({
title: 'Confirm schedule change',
description: (
<>
<p>
Changing the schedule (interval, timezone, or start time) of a batch export
could result in a gap of data.
</p>
<p>
Make sure to run a backfill if necessary to ensure all data is exported
correctly.
</p>
</>
),
primaryButton: {
children: 'Save changes',
onClick: () => {
userConfirmed = true
resolve()
},
},
secondaryButton: {
children: 'Cancel',
onClick: () => {
userConfirmed = false
resolve()
},
},
})
})
// Only proceed with submission if user confirmed
if (!userConfirmed) {
return
}
}
const data = buildBatchExportPayload(formdata)
try {
if (props.id) {
const res = await api.batchExports.update(props.id, data)
lemonToast.success('Batch export configuration updated successfully')
void addProductIntent({
product_type: ProductKey.PIPELINE_BATCH_EXPORTS,
intent_context: ProductIntentContext.BATCH_EXPORT_UPDATED,
})
actions.setBatchExportConfig(res)
actions.updateBatchExportConfigSuccess(res)
return
}
const res = await api.batchExports.create(data)
actions.resetConfiguration(getConfigurationFromBatchExportConfig(res))
void addProductIntent({
product_type: ProductKey.PIPELINE_BATCH_EXPORTS,
intent_context: ProductIntentContext.BATCH_EXPORT_CREATED,
})
router.actions.replace(urls.batchExport(res.id))
lemonToast.success('Batch export created successfully')
actions.updateBatchExportConfigSuccess(res)
} catch (error: any) {
lemonToast.error(error.detail || error.message || 'Could not save the batch export. Try again.')
// The toast can be far from the field at fault, such as the query editor, so the error also shows under it
if (error.attr && error.detail && error.attr in values.configuration) {
actions.setConfigurationManualErrors({ [error.attr]: error.detail })
}
// kea-forms hides form errors after a successful submit, so a failed save has to fail the submit
throw error
}
},
},
})),
beforeUnload(({ actions, values }) => ({
enabled: () => values.configurationChanged,
message: 'Leave action?\nChanges you made will be discarded.',
onConfirm: () => {
values.batchExportConfig
? actions.resetConfiguration(getConfigurationFromBatchExportConfig(values.batchExportConfig))
: values.service
? actions.resetConfiguration(getDefaultConfiguration(values.service))
: actions.resetConfiguration()
},
})),
afterMount(({ actions }) => {
actions.loadBatchExportConfigTest()
}),
])