Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): confirm before syncing a migrated self-managed table (#127382)

Tom Owers733d884378d2
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
index b4dd92e6af47..3f7c82688435 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
@@ -41,6 +41,7 @@ import {
SchemaEditorAction,
useSchemaEditorAccess,
} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'
+import { confirmPendingSelfManagedSync } from 'products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync'
import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import {
IncrementalSyncBlockedMessageMap,
@@ -506,6 +507,11 @@ function SyncMethodSection({ sourceId, schema }: { sourceId: string; schema: Ext
return
}
+ if (!schema.should_sync && schema.pending_self_managed_table) {
+ confirmPendingSelfManagedSync(1, () => void applyUpdate())
+ return
+ }
+
await applyUpdate()
}
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.test.ts b/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.test.ts
index c4daf6bc50d8..7ae6e4de5b98 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.test.ts
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.test.ts
@@ -1,6 +1,7 @@
import { expectLogic } from 'kea-test-utils'
import api from 'lib/api'
+import { LemonDialog } from 'lib/lemon-ui/LemonDialog'
import { initKeaTests } from '~/test/init'
import { ExternalDataSchemaStatus, ExternalDataSchemaWithSource } from '~/types'
@@ -77,4 +78,27 @@ describe('schemaSceneLogic', () => {
expect(logic.values.schema?.status).toBe(ExternalDataSchemaStatus.Running)
})
+
+ it.each([true, false])('confirms a pending schema before enabling when pending is %s', async (pending) => {
+ await expectLogic(logic).toFinishAllListeners()
+ logic.actions.loadSchemaSuccess(makeSchema({ should_sync: false, pending_self_managed_table: pending }))
+
+ const updateSpy = jest.spyOn(api.externalDataSchemas, 'update').mockResolvedValue(makeSchema())
+ const dialogSpy = jest.spyOn(LemonDialog, 'open').mockImplementation()
+ const schema = makeSchema({ should_sync: true, pending_self_managed_table: pending })
+
+ logic.actions.updateSchema(schema)
+
+ if (pending) {
+ expect(dialogSpy).toHaveBeenCalledTimes(1)
+ expect(updateSpy).not.toHaveBeenCalled()
+ const confirm = dialogSpy.mock.calls[0][0].primaryButton?.onClick as () => void
+ confirm()
+ } else {
+ expect(dialogSpy).not.toHaveBeenCalled()
+ }
+
+ await expectLogic(logic).toFinishAllListeners()
+ expect(updateSpy).toHaveBeenCalledWith('schema-1', expect.objectContaining({ should_sync: true }))
+ })
})
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.ts b/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.ts
index 7db8587c9a78..6fb4c0197d73 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.ts
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/schemaSceneLogic.ts
@@ -21,6 +21,7 @@ import {
SchemaIncrementalFieldsResponse,
} from '~/types'
+import { confirmPendingSelfManagedSync } from 'products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync'
import { cleanSourceId } from 'products/data_warehouse/frontend/utils'
export const SCHEMA_SCENE_TABS = ['configuration', 'syncs', 'metrics', 'data-quality', 'history'] as const
@@ -332,31 +333,46 @@ export const schemaSceneLogic = kea<schemaSceneLogicType>([
}),
listeners(({ actions, values, props }) => ({
updateSchema: async ({ schema }) => {
- const current = values.schemaData
- if (current) {
- actions.loadSchemaSuccess({ ...current, ...schema })
+ const runUpdate = async (): Promise<void> => {
+ const current = values.schemaData
+ if (current) {
+ actions.loadSchemaSuccess({ ...current, ...schema })
+ }
+ try {
+ await api.externalDataSchemas.update(schema.id, buildSchemaUpdatePayload(schema))
+ } catch (e: any) {
+ lemonToast.error(e?.message || "Can't update schema at this time")
+ } finally {
+ actions.loadSchema()
+ }
}
- try {
- await api.externalDataSchemas.update(schema.id, buildSchemaUpdatePayload(schema))
- } catch (e: any) {
- lemonToast.error(e?.message || "Can't update schema at this time")
- } finally {
- actions.loadSchema()
+ const turnsSyncingOn = schema.should_sync && !values.schemaData?.should_sync
+ if (turnsSyncingOn && schema.pending_self_managed_table) {
+ confirmPendingSelfManagedSync(1, () => void runUpdate())
+ return
}
+ await runUpdate()
},
reloadSchema: async ({ schema }) => {
- const current = values.schemaData
- if (current) {
- actions.loadSchemaSuccess({ ...current, status: ExternalDataSchemaStatus.Running })
+ const runReload = async (): Promise<void> => {
+ const current = values.schemaData
+ if (current) {
+ actions.loadSchemaSuccess({ ...current, status: ExternalDataSchemaStatus.Running })
+ }
+ try {
+ await api.externalDataSchemas.reload(schema.id)
+ posthog.capture('schema reloaded', { sourceType: values.source?.source_type })
+ } catch (e: any) {
+ lemonToast.error(e?.message || 'Cant reload schema at this time')
+ } finally {
+ actions.loadSchema()
+ }
}
- try {
- await api.externalDataSchemas.reload(schema.id)
- posthog.capture('schema reloaded', { sourceType: values.source?.source_type })
- } catch (e: any) {
- lemonToast.error(e?.message || 'Cant reload schema at this time')
- } finally {
- actions.loadSchema()
+ if (schema.pending_self_managed_table) {
+ confirmPendingSelfManagedSync(1, () => void runReload())
+ return
}
+ await runReload()
},
resyncSchema: async ({ schema }) => {
const current = values.schemaData
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index fcc3594a1db4..667ae5b62266 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -52,6 +52,7 @@ import {
SchemaEditorAction,
SourceEditorAction,
} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'
+import { confirmPendingSelfManagedSync } from 'products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync'
import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import { destinationLookupLogic } from 'products/data_warehouse/frontend/shared/logics/destinationLookupLogic'
import { sourceManagementLogic } from 'products/data_warehouse/frontend/shared/logics/sourceManagementLogic'
@@ -870,6 +871,17 @@ function SchemaBulkActions({
const frequencyOptions = allowedSyncFrequencies()
const onEnable = (): void => {
+ const pendingCount = selected.filter(
+ (schema) => !schema.should_sync && schema.pending_self_managed_table
+ ).length
+ if (pendingCount > 0) {
+ confirmPendingSelfManagedSync(pendingCount, enableWithDefaults, 'bulk')
+ return
+ }
+ enableWithDefaults()
+ }
+
+ const enableWithDefaults = (): void => {
const needingDefaults = selected.filter((schema) => !schema.should_sync && !schema.sync_type)
if (needingDefaults.length === 0) {
run(() => bulkEnable(selected))
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/dataWarehouseSourceSettingsLogic.test.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/dataWarehouseSourceSettingsLogic.test.ts
index 84a52e082b21..51647a6ad133 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/dataWarehouseSourceSettingsLogic.test.ts
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/dataWarehouseSourceSettingsLogic.test.ts
@@ -1,6 +1,7 @@
import { expectLogic } from 'kea-test-utils'
import api from 'lib/api'
+import { LemonDialog } from 'lib/lemon-ui/LemonDialog'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
@@ -86,6 +87,35 @@ describe('sourceSettingsLogic', () => {
])
})
+ it('waits for confirmation before enabling a pending schema', async () => {
+ const schema = makeSchema({ sync_type: 'full_refresh', pending_self_managed_table: true })
+ jest.spyOn(api.externalDataSources, 'get').mockResolvedValue(makeSource([schema]))
+ const bulkUpdateSchemasSpy = jest
+ .spyOn(api.externalDataSources, 'bulkUpdateSchemas')
+ .mockImplementation(async (_id, updates) => updates.map((update) => ({ ...schema, ...update })))
+ const dialogSpy = jest.spyOn(LemonDialog, 'open').mockImplementation()
+
+ logic = sourceSettingsLogic({ id: 'source-1' })
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+ jest.useFakeTimers()
+
+ logic.actions.updateSchema({ ...schema, should_sync: true })
+
+ expect(dialogSpy).toHaveBeenCalledTimes(1)
+ expect(logic.values.source?.schemas[0].should_sync).toBe(false)
+ await jest.advanceTimersByTimeAsync(500)
+ expect(bulkUpdateSchemasSpy).not.toHaveBeenCalled()
+
+ const confirm = dialogSpy.mock.calls[0][0].primaryButton?.onClick as () => void
+ confirm()
+ await jest.advanceTimersByTimeAsync(500)
+
+ expect(bulkUpdateSchemasSpy).toHaveBeenCalledWith('source-1', [
+ expect.objectContaining({ id: 'schema-1', should_sync: true }),
+ ])
+ })
+
it('updates a large schema group optimistically with one batch save', async () => {
const schemas = Array.from({ length: 300 }, (_, index) =>
makeSchema({ id: `schema-${index}`, name: `public.table_${index}` })
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
index e0fabc53fb5d..5be02877e35f 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
@@ -36,6 +36,7 @@ import {
} from '~/types'
import { groupTablesBySchema } from 'products/data_warehouse/frontend/shared/components/forms/schemaGroupingUtils'
+import { confirmPendingSelfManagedSync } from 'products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync'
import { FILE_UPLOAD_SOURCE_LABEL, isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import {
clonePayloadPreservingFiles,
@@ -1332,6 +1333,12 @@ export const sourceSettingsLogic = kea<sourceSettingsLogicType>([
return {
updateSchema: (schema) => {
+ const current = values.source?.schemas.find((s) => s.id === schema.id)
+ const turnsSyncingOn = schema.should_sync && !current?.should_sync
+ if (turnsSyncingOn && schema.pending_self_managed_table) {
+ confirmPendingSelfManagedSync(1, () => queueSchemaUpdates([schema]))
+ return
+ }
queueSchemaUpdates([schema])
},
updateSchemas: ({ schemas }) => {
@@ -1498,25 +1505,32 @@ export const sourceSettingsLogic = kea<sourceSettingsLogicType>([
}
},
reloadSchema: async ({ schema }) => {
- // Optimistic UI updates before sending updates to the backend
- const clonedSource = JSON.parse(JSON.stringify(values.source)) as ExternalDataSource
- const schemaIndex = clonedSource.schemas.findIndex((n) => n.id === schema.id)
- clonedSource.status = ExternalDataJobStatus.Running
- clonedSource.schemas[schemaIndex].status = ExternalDataSchemaStatus.Running
+ const runReload = async (): Promise<void> => {
+ // Optimistic UI updates before sending updates to the backend
+ const clonedSource = JSON.parse(JSON.stringify(values.source)) as ExternalDataSource
+ const schemaIndex = clonedSource.schemas.findIndex((n) => n.id === schema.id)
+ clonedSource.status = ExternalDataJobStatus.Running
+ clonedSource.schemas[schemaIndex].status = ExternalDataSchemaStatus.Running
- actions.loadSourceSuccess(clonedSource)
+ actions.loadSourceSuccess(clonedSource)
- try {
- await api.externalDataSchemas.reload(schema.id)
+ try {
+ await api.externalDataSchemas.reload(schema.id)
- posthog.capture('schema reloaded', { sourceType: clonedSource.source_type })
- } catch (e: any) {
- if (e.message) {
- lemonToast.error(e.message)
- } else {
- lemonToast.error('Cant reload schema at this time')
+ posthog.capture('schema reloaded', { sourceType: clonedSource.source_type })
+ } catch (e: any) {
+ if (e.message) {
+ lemonToast.error(e.message)
+ } else {
+ lemonToast.error('Cant reload schema at this time')
+ }
}
}
+ if (schema.pending_self_managed_table) {
+ confirmPendingSelfManagedSync(1, () => void runReload())
+ return
+ }
+ await runReload()
},
resyncSchema: async ({ schema }) => {
// Optimistic UI updates before sending updates to the backend
diff --git a/products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync.ts b/products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync.ts
new file mode 100644
index 000000000000..bf0aa69b9c50
--- /dev/null
+++ b/products/data_warehouse/frontend/shared/confirmPendingSelfManagedSync.ts
@@ -0,0 +1,20 @@
+import { LemonDialog } from 'lib/lemon-ui/LemonDialog'
+
+export function confirmPendingSelfManagedSync(
+ count: number,
+ onConfirm: () => void,
+ scope: 'single' | 'bulk' = 'single'
+): void {
+ LemonDialog.open({
+ title:
+ scope === 'single'
+ ? 'Import this table into PostHog?'
+ : `Import ${count} ${count === 1 ? 'table' : 'tables'} into PostHog?`,
+ description:
+ count === 1
+ ? 'Right now this table reads the files in your bucket directly, at no charge. When you turn on syncing, PostHog imports the data into PostHog and keeps it up to date on the schedule you choose. Synced rows count toward your data warehouse usage and can incur charges.'
+ : 'Right now these tables read the files in your bucket directly, at no charge. When you turn on syncing, PostHog imports the data into PostHog and keeps it up to date on the schedule you choose. Synced rows count toward your data warehouse usage and can incur charges.',
+ primaryButton: { children: 'Turn on syncing', type: 'primary', onClick: onConfirm },
+ secondaryButton: { children: 'Cancel', type: 'tertiary' },
+ })
+}