Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): warn when webhook-sync tables have no webhook (#129039)

Tom Owersf49cdcb190c1
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 3cede28ddf86..a947f277577d 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -11038,6 +11038,10 @@ snapshots:
hash: v1.k794b7964.3bcec4692a9a62b116fc38aee9da58cd33c134602ec1a659a281a564c7d704fb.WOQMHP0MZlV4qdn8etO3YYlIcm5odlOGkPBU5f_kfA4
scenes-app-data-warehouse-settings-schemas--multi-schema--light:
hash: v1.k794b7964.2f4bd6098d8c54c0b25ed9ee0ef7770a59c4d343191d065b0814299a27a42a22.6jWN3F1ozzMy7KZRTYHY1XPxasFL_u2H8Ybw9WcDjfk
+ scenes-app-data-warehouse-settings-webhook-not-registered-banner--many-tables--dark:
+ hash: v1.k794b7964.fec86694675eba5ef60a7c479d4c5e2b8af6db15ac30ea65290c0158a6aa566b.7oBlqrtOAH9HWLo6Aq9vSaWVcVTRmAJ2BbMmcM860dc
+ scenes-app-data-warehouse-settings-webhook-not-registered-banner--many-tables--light:
+ hash: v1.k794b7964.723d1f9d4f0244bf366c07c88b356389f08714e77c7e9db5bf4fa108f51ca332.GG5hDDZIOObo9vzjdwijZl27PDxFjC7TaSjvJSqxKeU
scenes-app-data-warehouse-sql-editor--bi-calculated-measure-editor--dark:
hash: v1.k794b7964.1a82ff22ca1e25304c282c6b0387fa640706bff93ae75bee332169f74568bee2.pOX7yUp-Z9lLTCyUyzIWUN96vePLmipSolnF1e4EP7c
scenes-app-data-warehouse-sql-editor--bi-calculated-measure-editor--light:
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 0e107004f37f..2e074745c8d4 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6473,6 +6473,8 @@ export interface ExternalDataSource {
revenue_analytics_config: ExternalDataSourceRevenueAnalyticsConfig
user_access_level: AccessControlLevel
supports_webhooks?: boolean
+ /** Whether a table of this source uses webhook sync, but the source has no enabled webhook. */
+ webhook_not_registered?: boolean
supports_column_selection?: boolean
api_version?: string | null
api_version_deprecation?: ExternalDataSourceApiVersionDeprecation | null
@@ -6680,6 +6682,8 @@ export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema
* failure is current. A later run that succeeds, or fails for another reason, clears it.
*/
incremental_sync_blocked?: IncrementalSyncBlockedReason | null
+ /** Whether this table uses webhook sync, but its source has no enabled webhook. */
+ webhook_not_registered?: boolean
/**
* User-selected source columns to sync. `null` means "sync all columns".
* Primary-key + active incremental columns are always retained even if not listed.
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/NewSourceScene.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/NewSourceScene.tsx
index ab16b6a75577..a55a84304b18 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/NewSourceScene.tsx
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/NewSourceScene.tsx
@@ -45,6 +45,7 @@ import { BillingLimitNotice } from './components/BillingLimitNotice'
import { DestinationStep } from './components/DestinationStep'
import { FileUploadSourceForm } from './components/FileUploadSourceForm'
import { SelfManagedSourceForm } from './components/SelfManagedSourceForm'
+import { WebhookStepFallback } from './components/WebhookStepFallback'
import { FILE_UPLOAD_SOURCE_NAME } from './fileUploadSource'
import { selfManagedSourceLogic } from './selfManagedSourceLogic'
import { SourceCatalog } from './SourceCatalog'
@@ -696,18 +697,20 @@ function WebhookSetupStep({
.map((s) => ({ name: s.table, label: s.label }))
return (
- <WebhookSetupForm
- sourceName={selectedConnector?.label ?? selectedConnector?.name ?? 'source'}
- sourceConfig={selectedConnector}
- webhookTables={webhookTables}
- webhookResult={webhookResult}
- webhookCreating={webhookCreating}
- webhookFieldsSubmitting={isWebhookFieldInputsSubmitting}
- onCreateWebhook={createWebhook}
- formLogic={sourceWizardLogicProps ? sourceWizardLogic(sourceWizardLogicProps) : sourceWizardLogic}
- formKey="webhookFieldInputs"
- isWizardStep
- />
+ <div className="flex flex-col gap-4">
+ <WebhookSetupForm
+ sourceName={selectedConnector?.label ?? selectedConnector?.name ?? 'source'}
+ sourceConfig={selectedConnector}
+ webhookTables={webhookTables}
+ webhookResult={webhookResult}
+ webhookCreating={webhookCreating}
+ webhookFieldsSubmitting={isWebhookFieldInputsSubmitting}
+ onCreateWebhook={createWebhook}
+ formLogic={sourceWizardLogicProps ? sourceWizardLogic(sourceWizardLogicProps) : sourceWizardLogic}
+ formKey="webhookFieldInputs"
+ />
+ <WebhookStepFallback webhookTableCount={webhookTables.length} />
+ </div>
)
}
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/components/WebhookStepFallback.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/components/WebhookStepFallback.tsx
new file mode 100644
index 000000000000..906f3585e328
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/components/WebhookStepFallback.tsx
@@ -0,0 +1,74 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonButton, LemonCheckbox } from '@posthog/lemon-ui'
+
+import { LemonCard } from 'lib/lemon-ui/LemonCard'
+
+import { sourceWizardLogic } from '../sourceWizardLogic'
+
+/** The ways out of the wizard's webhook step when the webhook is not set up yet. */
+export function WebhookStepFallback({ webhookTableCount }: { webhookTableCount: number }): JSX.Element | null {
+ const {
+ selectedConnector,
+ webhookCreating,
+ webhookStepComplete,
+ webhookNeedsManualConfirmation,
+ webhookManualSetupConfirmed,
+ webhookTablesSwitching,
+ } = useValues(sourceWizardLogic)
+ const { setWebhookManualSetupConfirmed, switchWebhookTablesToPolling, skipWebhookSetup } =
+ useActions(sourceWizardLogic)
+
+ const sourceName = selectedConnector?.label ?? selectedConnector?.name ?? 'the source'
+
+ if (webhookCreating) {
+ return null
+ }
+
+ return (
+ <div className="flex flex-col gap-4">
+ {webhookNeedsManualConfirmation && (
+ <LemonCheckbox
+ checked={webhookManualSetupConfirmed}
+ onChange={setWebhookManualSetupConfirmed}
+ label={`I added the webhook URL in ${sourceName}`}
+ bordered
+ data-attr="source-wizard-webhook-manual-confirm"
+ />
+ )}
+ {!webhookStepComplete && (
+ <LemonCard hoverEffect={false} className="flex flex-col gap-3">
+ <h4 className="font-semibold mb-0">Can't set up the webhook now?</h4>
+ <p className="mb-0">
+ {webhookTableCount.toLocaleString()} {webhookTableCount === 1 ? 'table uses' : 'tables use'}{' '}
+ webhook sync. Without a working webhook, each sync of these tables reads the {sourceName} API
+ instead. This is slower, and a sync can read the whole table each time.
+ </p>
+ <p className="mb-0">
+ Change these tables to incremental sync where {sourceName} supports it, and to full refresh
+ where it doesn't. You can switch them back to webhook sync later.
+ </p>
+ <div className="flex flex-wrap gap-2">
+ <LemonButton
+ type="secondary"
+ onClick={switchWebhookTablesToPolling}
+ loading={webhookTablesSwitching}
+ data-attr="source-wizard-webhook-switch-sync-method"
+ >
+ Use another sync method
+ </LemonButton>
+ <LemonButton
+ type="tertiary"
+ status="danger"
+ onClick={skipWebhookSetup}
+ disabledReason={webhookTablesSwitching ? 'Changing the sync method' : undefined}
+ data-attr="source-wizard-webhook-skip"
+ >
+ Skip for now
+ </LemonButton>
+ </div>
+ </LemonCard>
+ )}
+ </div>
+ )
+}
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
index 2619e50e17cb..84a5a5338751 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
@@ -33,6 +33,11 @@ import {
import type { SourceFieldConfig } from 'products/data_warehouse/frontend/types'
import {
+ externalDataSourcesBulkUpdateSchemasPartialUpdate,
+ externalDataSourcesRetrieve,
+} from 'products/warehouse_sources/frontend/generated/api'
+import {
+ ExternalDataSourceBulkUpdateSchemaApi,
SourceConfigResponseApi,
SourceFieldSwitchGroupConfigApi,
SuggestedTableApi,
@@ -416,6 +421,39 @@ export const bulkToggledShouldSync = (schema: ExternalDataSourceSyncSchema, sele
return selectAll
}
+export interface PollingSyncSettings {
+ sync_type: 'full_refresh' | 'incremental' | 'append'
+ incremental_field?: string
+ incremental_field_type?: string
+}
+
+/** The default sync method for a table that does not sync by CDC or webhook. */
+export function defaultPollingSyncSettings(
+ schema: ExternalDataSourceSyncSchema,
+ connectorName: string
+): PollingSyncSettings {
+ if (schema.prefers_full_refresh) {
+ return { sync_type: 'full_refresh' }
+ }
+ if (schema.incremental_available && !hasIncrementalMergeKey(schema)) {
+ return { sync_type: 'full_refresh' }
+ }
+ if (schema.incremental_available || schema.append_available) {
+ const resolvedField =
+ connectorName === 'Supabase'
+ ? resolveUpdateTrackedIncrementalField(schema.incremental_fields)
+ : resolveIncrementalField(schema.incremental_fields)
+ if (resolvedField) {
+ return {
+ sync_type: schema.incremental_available ? 'incremental' : 'append',
+ incremental_field: resolvedField.field,
+ incremental_field_type: resolvedField.field_type,
+ }
+ }
+ }
+ return { sync_type: 'full_refresh' }
+}
+
function syncExpandedSchemaGroupKeys(
actions: sourceWizardLogicType['actions'],
values: sourceWizardLogicType['values']
@@ -554,8 +592,11 @@ export interface sourceWizardLogicValues {
webhookFieldInputsTouched: boolean
webhookFieldInputsTouches: Record<string, boolean>
webhookFieldInputsValidationErrors: DeepPartialMap<Record<string, any>, ValidationErrorType>
+ webhookManualSetupConfirmed: boolean
+ webhookNeedsManualConfirmation: boolean
webhookResult: WebhookCreateResult | null
webhookStepComplete: boolean
+ webhookTablesSwitching: boolean
wizardAvailableDestinationCount: number
wizardDestinationIds: string[]
}
@@ -796,15 +837,24 @@ export interface sourceWizardLogicActions {
setWebhookFieldInputsValues: (values: DeepPartial<Record<string, any>>) => {
values: DeepPartial<Record<string, any>>
}
+ setWebhookManualSetupConfirmed: (confirmed: boolean) => {
+ confirmed: boolean
+ }
setWebhookResult: (result: WebhookCreateResult | null) => {
result: WebhookCreateResult | null
}
+ setWebhookTablesSwitching: (switching: boolean) => {
+ switching: boolean
+ }
setWizardAvailableDestinationCount: (count: number) => {
count: number
}
setWizardDestinationIds: (destinationIds: string[]) => {
destinationIds: string[]
}
+ skipWebhookSetup: () => {
+ value: true
+ }
submitSourceConnectionDetails: () => {
value: boolean
}
@@ -840,6 +890,9 @@ export interface sourceWizardLogicActions {
submitWebhookFields: () => {
value: true
}
+ switchWebhookTablesToPolling: () => {
+ value: true
+ }
syncExpandedSchemaGroupKeys: (
groupedSchemaKeys: string[],
fingerprint: string
@@ -940,10 +993,15 @@ export interface sourceWizardLogicMeta {
}[]
) => Breadcrumb[]
hasWebhookSchemas: (databaseSchema: ExternalDataSourceSyncSchema[]) => boolean
- webhookStepComplete: (
+ webhookNeedsManualConfirmation: (
webhookResult: WebhookCreateResult | null,
selectedConnector: SourceConfigResponseApi | null
) => boolean
+ webhookStepComplete: (
+ webhookResult: WebhookCreateResult | null,
+ webhookNeedsManualConfirmation: boolean,
+ webhookManualSetupConfirmed: boolean
+ ) => boolean
isManualLinkingSelected: (selectedConnector: SourceConfigResponseApi | null) => boolean
isDirectQueryMode: (
source: {
@@ -1159,6 +1217,10 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
result,
}),
submitWebhookFields: true,
+ setWebhookManualSetupConfirmed: (confirmed: boolean) => ({ confirmed }),
+ switchWebhookTablesToPolling: true,
+ setWebhookTablesSwitching: (switching: boolean) => ({ switching }),
+ skipWebhookSetup: true,
openCdcSelfManagedSetupDialog: true,
closeCdcSelfManagedSetupDialog: true,
clearCdcPrereqsCheckResult: true,
@@ -1482,6 +1544,20 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
setWebhookResult: () => false,
},
],
+ webhookManualSetupConfirmed: [
+ false,
+ {
+ setWebhookManualSetupConfirmed: (_, { confirmed }) => confirmed,
+ setWebhookResult: () => false,
+ onClear: () => false,
+ },
+ ],
+ webhookTablesSwitching: [
+ false,
+ {
+ setWebhookTablesSwitching: (_, { switching }) => switching,
+ },
+ ],
}),
loaders(({ values }) => ({
cdcPrereqsCheckResult: [
@@ -1625,19 +1701,28 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
(databaseSchema: ExternalDataSourceSyncSchema[]): boolean =>
databaseSchema.some((s) => s.supports_webhooks && s.sync_type === 'webhook' && s.should_sync),
],
- webhookStepComplete: [
+ webhookNeedsManualConfirmation: [
(s) => [s.webhookResult, s.selectedConnector],
(webhookResult: WebhookCreateResult | null, selectedConnector: SourceConfigResponseApi | null): boolean => {
- if (webhookResultHasNoPendingInputs(webhookResult)) {
- return true
- }
- if (!webhookResult) {
+ if (!webhookResult?.webhook_url || webhookResultHasNoPendingInputs(webhookResult)) {
return false
}
-
+ // With a required field to save, saving that form is the proof of the manual setup.
const webhookFields = selectedConnector?.webhookFields ?? []
- const requiredFields = webhookFields.filter((f) => 'required' in f && f.required)
- return requiredFields.length === 0
+ return !webhookFields.some((f) => 'required' in f && f.required)
+ },
+ ],
+ webhookStepComplete: [
+ (s) => [s.webhookResult, s.webhookNeedsManualConfirmation, s.webhookManualSetupConfirmed],
+ (
+ webhookResult: WebhookCreateResult | null,
+ webhookNeedsManualConfirmation: boolean,
+ webhookManualSetupConfirmed: boolean
+ ): boolean => {
+ if (webhookResultHasNoPendingInputs(webhookResult)) {
+ return true
+ }
+ return webhookNeedsManualConfirmation && webhookManualSetupConfirmed
},
],
isManualLinkingSelected: [
@@ -1739,7 +1824,7 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
}
if (currentStep === 4 && !webhookStepComplete) {
- return 'Finish setting up the webhook to continue'
+ return 'Set up the webhook, or choose another option below this step'
}
if (currentStep === WIZARD_DESTINATION_STEP) {
@@ -2441,6 +2526,101 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
posthog.captureException(e)
}
},
+ switchWebhookTablesToPolling: async () => {
+ const { sourceId, currentTeamId, selectedConnector } = values
+ if (!sourceId || !currentTeamId || !selectedConnector) {
+ return
+ }
+ actions.setWebhookTablesSwitching(true)
+ try {
+ const source = await externalDataSourcesRetrieve(String(currentTeamId), sourceId)
+ const wizardSchemas = new Map(values.databaseSchema.map((schema) => [schema.table, schema]))
+ const updates: ExternalDataSourceBulkUpdateSchemaApi[] = []
+ const switched = new Map<string, PollingSyncSettings>()
+ let webhookOnlyCount = 0
+ for (const item of source.schemas) {
+ const schemaId = typeof item.id === 'string' ? item.id : null
+ const name = typeof item.name === 'string' ? item.name : null
+ const wizardSchema = name ? wizardSchemas.get(name) : undefined
+ if (!schemaId || !name || !wizardSchema || item.sync_type !== 'webhook' || !item.should_sync) {
+ continue
+ }
+ // A webhook-only table (e.g. Stripe Discount) has no list endpoint to poll, and the
+ // API rejects any other sync type for it, so it stays on webhook sync.
+ if (wizardSchema.webhook_only) {
+ webhookOnlyCount += 1
+ continue
+ }
+ const settings = defaultPollingSyncSettings(wizardSchema, selectedConnector.name)
+ switched.set(name, settings)
+ updates.push({
+ id: schemaId,
+ sync_type: settings.sync_type,
+ incremental_field: settings.incremental_field ?? null,
+ incremental_field_type: settings.incremental_field_type ?? null,
+ })
+ }
+ if (updates.length > 0) {
+ await externalDataSourcesBulkUpdateSchemasPartialUpdate(String(currentTeamId), sourceId, {
+ schemas: updates,
+ })
+ }
+ actions.setDatabaseSchemas(
+ values.databaseSchema.map((schema) => {
+ const settings = switched.get(schema.table)
+ return settings
+ ? {
+ ...schema,
+ sync_type: settings.sync_type,
+ incremental_field: settings.incremental_field ?? null,
+ incremental_field_type: settings.incremental_field_type ?? null,
+ }
+ : schema
+ })
+ )
+ posthog.capture('source wizard webhook tables switched to polling', {
+ sourceType: selectedConnector.name,
+ tableCount: updates.length,
+ })
+ lemonToast.success(
+ `Switched ${updates.length.toLocaleString()} ${updates.length === 1 ? 'table' : 'tables'} to another sync method` +
+ (webhookOnlyCount > 0
+ ? `. ${webhookOnlyCount.toLocaleString()} ${webhookOnlyCount === 1 ? 'table syncs' : 'tables sync'} only by webhook and stayed on webhook sync.`
+ : '')
+ )
+ actions.setStep(5)
+ } catch (e: any) {
+ posthog.captureException(e)
+ lemonToast.error(
+ `Couldn't change the sync method: ${e.data?.detail ?? e.data?.message ?? e.message ?? 'unknown error'}. Try again, or change it on the source's Schemas tab later.`
+ )
+ } finally {
+ actions.setWebhookTablesSwitching(false)
+ }
+ },
+ skipWebhookSetup: () => {
+ const sourceName = values.selectedConnector?.label ?? values.selectedConnector?.name ?? 'the source'
+ LemonDialog.open({
+ title: 'Skip the webhook setup?',
+ description: `Tables that use webhook sync get no webhook updates until you set up the webhook. Until then, each sync reads the ${sourceName} API instead. This is slower, and a sync can read the whole table each time. You can set up the webhook later on the source's Webhook tab.`,
+ primaryButton: {
+ children: 'Skip webhook setup',
+ status: 'danger',
+ onClick: () => {
+ posthog.capture('source wizard webhook setup skipped', {
+ sourceType: values.selectedConnector?.name,
+ webhookCreateFailed: values.webhookResult ? !values.webhookResult.success : null,
+ })
+ actions.setStep(5)
+ },
+ 'data-attr': 'source-wizard-skip-webhook-confirm',
+ },
+ secondaryButton: {
+ children: 'Go back',
+ type: 'tertiary',
+ },
+ })
+ },
submitWebhookFields: async () => {
if (!values.sourceId) {
return
@@ -2545,25 +2725,13 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
schema.sync_type = 'cdc'
} else if (schema.supports_webhooks) {
schema.sync_type = 'webhook'
- } else if (schema.prefers_full_refresh) {
- schema.sync_type = 'full_refresh'
- } else if (schema.incremental_available && !hasIncrementalMergeKey(schema)) {
- schema.sync_type = 'full_refresh'
- } else if (schema.incremental_available || schema.append_available) {
- const method = schema.incremental_available ? 'incremental' : 'append'
- const resolvedField =
- values.selectedConnector.name === 'Supabase'
- ? resolveUpdateTrackedIncrementalField(schema.incremental_fields)
- : resolveIncrementalField(schema.incremental_fields)
- schema.sync_type = method
- if (resolvedField) {
- schema.incremental_field = resolvedField.field
- schema.incremental_field_type = resolvedField.field_type
- } else {
- schema.sync_type = 'full_refresh'
- }
} else {
- schema.sync_type = 'full_refresh'
+ const settings = defaultPollingSyncSettings(schema, values.selectedConnector.name)
+ schema.sync_type = settings.sync_type
+ if (settings.incremental_field) {
+ schema.incremental_field = settings.incremental_field
+ schema.incremental_field_type = settings.incremental_field_type ?? null
+ }
}
}
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
index c2b8212d0180..a45bdfed9917 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
@@ -14,7 +14,14 @@ import { ProductIntentContext, ProductKey, WebStatsBreakdown } from '~/queries/s
import { initKeaTests } from '~/test/init'
import type { AvailableColumn, ExternalDataSource, ExternalDataSourceSyncSchema, IncrementalField } from '~/types'
-import type { SourceConfigResponseApi } from 'products/warehouse_sources/frontend/generated/api.schemas'
+import {
+ externalDataSourcesBulkUpdateSchemasPartialUpdate,
+ externalDataSourcesRetrieve,
+} from 'products/warehouse_sources/frontend/generated/api'
+import type {
+ ExternalDataSourceSerializersApi,
+ SourceConfigResponseApi,
+} from 'products/warehouse_sources/frontend/generated/api.schemas'
import {
buildCreateSourcePayload,
@@ -29,6 +36,12 @@ import {
WIZARD_DESTINATION_STEP,
} from '../sourceWizardLogic'
+jest.mock('products/warehouse_sources/frontend/generated/api', () => ({
+ ...jest.requireActual('products/warehouse_sources/frontend/generated/api'),
+ externalDataSourcesRetrieve: jest.fn(),
+ externalDataSourcesBulkUpdateSchemasPartialUpdate: jest.fn(),
+}))
+
type Deferred = {
resolve: (schemas: ExternalDataSourceSyncSchema[]) => void
reject: (error: unknown) => void
@@ -276,6 +289,135 @@ describe('sourceWizardLogic', () => {
}
})
+ describe('webhook step', () => {
+ const stripeWithSecret = buildSourceConfig({
+ name: 'Stripe',
+ webhookFields: [{ name: 'signing_secret', label: 'Signing secret', type: 'password', required: true }],
+ } as Partial<SourceConfigResponseApi>)
+ const stripeWithoutFields = buildSourceConfig({ name: 'Stripe' })
+
+ it.each([
+ ['auto setup finished', stripeWithSecret, { success: true, webhook_url: 'u' }, false, true],
+ [
+ 'a secret is still pending',
+ stripeWithSecret,
+ { success: true, webhook_url: 'u', pending_inputs: ['signing_secret'] },
+ true,
+ false,
+ ],
+ [
+ 'manual setup with a required field to save',
+ stripeWithSecret,
+ { success: false, webhook_url: 'u', error: 'e' },
+ true,
+ false,
+ ],
+ [
+ 'manual setup not confirmed',
+ stripeWithoutFields,
+ { success: false, webhook_url: 'u', error: 'e' },
+ false,
+ false,
+ ],
+ [
+ 'manual setup confirmed',
+ stripeWithoutFields,
+ { success: false, webhook_url: 'u', error: 'e' },
+ true,
+ true,
+ ],
+ [
+ 'request failed with no URL',
+ stripeWithoutFields,
+ { success: false, webhook_url: '', error: 'e' },
+ true,
+ false,
+ ],
+ ])('blocks Next unless the webhook works: %s', (_name, connector, result, confirmed, expected) => {
+ const logic = sourceWizardLogic({ availableSources: { Stripe: connector } })
+ const unmount = logic.mount()
+
+ try {
+ logic.actions.selectConnector(connector)
+ logic.actions.setStep(4)
+ logic.actions.setWebhookResult(result)
+ logic.actions.setWebhookManualSetupConfirmed(confirmed)
+
+ expect(logic.values.webhookStepComplete).toBe(expected)
+ expect(logic.values.canGoNext).toBe(expected)
+ } finally {
+ unmount()
+ }
+ })
+
+ it('switches only the syncing, pollable webhook tables to a polling sync method', async () => {
+ const schema = (overrides: Partial<ExternalDataSourceSyncSchema>): ExternalDataSourceSyncSchema =>
+ ({
+ table: 'Customer',
+ label: null,
+ should_sync: true,
+ sync_type: 'webhook',
+ incremental_field: null,
+ incremental_field_type: null,
+ incremental_fields: [],
+ incremental_available: false,
+ append_available: false,
+ supports_webhooks: true,
+ available_columns: [],
+ ...overrides,
+ }) as ExternalDataSourceSyncSchema
+ jest.mocked(externalDataSourcesRetrieve).mockResolvedValue({
+ schemas: [
+ { id: 's1', name: 'Customer', sync_type: 'webhook', should_sync: true },
+ { id: 's2', name: 'Invoice', sync_type: 'webhook', should_sync: true },
+ { id: 's3', name: 'Charge', sync_type: 'incremental', should_sync: true },
+ { id: 's4', name: 'Payout', sync_type: 'webhook', should_sync: false },
+ { id: 's5', name: 'Discount', sync_type: 'webhook', should_sync: true },
+ ],
+ } as unknown as ExternalDataSourceSerializersApi)
+ const bulkUpdate = jest.mocked(externalDataSourcesBulkUpdateSchemasPartialUpdate).mockResolvedValue([])
+ const logic = sourceWizardLogic({ availableSources: { Stripe: stripeWithSecret } })
+ const unmount = logic.mount()
+
+ try {
+ logic.actions.selectConnector(stripeWithSecret)
+ logic.actions.setDatabaseSchemas([
+ schema({
+ table: 'Customer',
+ incremental_available: true,
+ incremental_fields: [
+ { label: 'created', type: 'datetime', field: 'created', field_type: 'integer' },
+ ] as IncrementalField[],
+ }),
+ schema({ table: 'Invoice' }),
+ schema({ table: 'Charge', sync_type: 'incremental' }),
+ schema({ table: 'Payout', should_sync: false }),
+ schema({ table: 'Discount', webhook_only: true }),
+ ])
+ logic.actions.setSourceId('source-1')
+ logic.actions.setStep(4)
+
+ await expectLogic(logic, () => logic.actions.switchWebhookTablesToPolling()).toFinishAllListeners()
+
+ expect(bulkUpdate).toHaveBeenCalledWith(expect.any(String), 'source-1', {
+ schemas: [
+ {
+ id: 's1',
+ sync_type: 'incremental',
+ incremental_field: 'created',
+ incremental_field_type: 'integer',
+ },
+ { id: 's2', sync_type: 'full_refresh', incremental_field: null, incremental_field_type: null },
+ ],
+ })
+ expect(logic.values.databaseSchema.find((s) => s.table === 'Discount')?.sync_type).toBe('webhook')
+ expect(logic.values.currentStep).toBe(5)
+ } finally {
+ unmount()
+ }
+ })
+ })
+
it('does not hydrate the same source URL again after the wizard has started', () => {
const postgresSource = buildSourceConfig({ name: 'Postgres' })
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
index 856182caca85..7d71c21bd259 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
@@ -53,6 +53,7 @@ import { SchemasTab } from './tabs/SchemasTab'
import { sourceSettingsLogic } from './tabs/sourceSettingsLogic'
import { SyncsTab } from './tabs/SyncsTab'
import { WebhookTab } from './tabs/WebhookTab'
+import { WebhookNotRegisteredBanner } from './WebhookNotRegisteredBanner'
const SOURCE_SCENE_TABS = [
'schemas',
@@ -384,6 +385,14 @@ function ManagedSourceTabs({
/>
)}
{showDestinationsTab && <FailingDestinationsBanner sourceId={sourceId} />}
+ {source?.webhook_not_registered && showWebhookTab && (
+ <WebhookNotRegisteredBanner
+ sourceName={source.source_type}
+ tableNames={source.schemas.filter((s) => s.webhook_not_registered).map((s) => s.label ?? s.name)}
+ onSetUpWebhook={() => setCurrentTab('webhook')}
+ onChangeSyncMethod={() => setCurrentTab('schemas')}
+ />
+ )}
<LemonTabs
activeKey={currentTab}
tabs={tabs}
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.stories.tsx b/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.stories.tsx
new file mode 100644
index 000000000000..f28811cc0543
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.stories.tsx
@@ -0,0 +1,40 @@
+import type { Meta, StoryObj } from '@storybook/react'
+import type { ComponentProps } from 'react'
+
+import { WebhookNotRegisteredBanner } from './WebhookNotRegisteredBanner'
+
+// LemonBanner is a CSS container, so it collapses without an explicitly sized parent.
+const renderInSizedParent = (props: ComponentProps<typeof WebhookNotRegisteredBanner>): JSX.Element => {
+ return (
+ <div id="target" className="w-200">
+ <WebhookNotRegisteredBanner {...props} />
+ </div>
+ )
+}
+
+type Story = StoryObj<typeof WebhookNotRegisteredBanner>
+const meta: Meta<typeof WebhookNotRegisteredBanner> = {
+ title: 'Scenes-App/Data Warehouse/Settings/Webhook not registered banner',
+ component: WebhookNotRegisteredBanner,
+ parameters: {
+ viewMode: 'story',
+ testOptions: {
+ snapshotBrowsers: ['chromium'],
+ snapshotTargetSelector: '#target',
+ },
+ },
+ args: {
+ sourceName: 'Stripe',
+ onSetUpWebhook: () => {},
+ onChangeSyncMethod: () => {},
+ },
+}
+
+export default meta
+
+export const ManyTables: Story = {
+ render: renderInSizedParent,
+ args: {
+ tableNames: ['Customer', 'Invoice', 'Charge', 'Subscription', 'Price', 'Product', 'Refund'],
+ },
+}
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.tsx b/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.tsx
new file mode 100644
index 000000000000..66fc8ef5333c
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/WebhookNotRegisteredBanner.tsx
@@ -0,0 +1,57 @@
+import { LemonBanner, LemonButton } from '@posthog/lemon-ui'
+
+const MAX_LISTED_TABLES = 5
+
+export interface WebhookNotRegisteredBannerProps {
+ sourceName: string
+ tableNames: string[]
+ onSetUpWebhook: () => void
+ onChangeSyncMethod: () => void
+}
+
+export function WebhookNotRegisteredBanner({
+ sourceName,
+ tableNames,
+ onSetUpWebhook,
+ onChangeSyncMethod,
+}: WebhookNotRegisteredBannerProps): JSX.Element {
+ const listed = tableNames.slice(0, MAX_LISTED_TABLES).join(', ')
+ const hidden = tableNames.length - MAX_LISTED_TABLES
+
+ return (
+ <LemonBanner type="error" alignItems="start">
+ <div className="flex flex-col gap-2 min-w-0 break-words">
+ <div className="font-semibold">Webhook sync is not working for this source</div>
+ <div>
+ Some tables use webhook sync, but this source has no active webhook. Each sync of these tables reads
+ the {sourceName} API instead. This is slower, and a sync can read the whole table each time.
+ </div>
+ {tableNames.length > 0 && (
+ <div className="text-xs">
+ {tableNames.length === 1 ? 'Table' : 'Tables'}: {listed}
+ {hidden > 0 ? ` and ${hidden.toLocaleString()} more` : ''}
+ </div>
+ )}
+ <div>Set up the webhook, or change these tables to another sync method.</div>
+ <div className="flex flex-wrap gap-2">
+ <LemonButton
+ type="primary"
+ size="small"
+ onClick={onSetUpWebhook}
+ data-attr="source-webhook-not-registered-set-up"
+ >
+ Set up webhook
+ </LemonButton>
+ <LemonButton
+ type="secondary"
+ size="small"
+ onClick={onChangeSyncMethod}
+ data-attr="source-webhook-not-registered-change-sync-method"
+ >
+ Change sync method
+ </LemonButton>
+ </div>
+ </div>
+ </LemonBanner>
+ )
+}
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index de5b64e011bd..961d8e55549a 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -637,7 +637,18 @@ function ManagedSchemaTable({
if (isFileUpload) {
return syncMethod
}
- return (
+ const webhookTag = schema.webhook_not_registered ? (
+ <Tooltip title="This table uses webhook sync, but the source has no active webhook. Each sync reads the source API instead, which is slower and can read the whole table. Set up the webhook on the Webhook tab, or change the sync method.">
+ <Link
+ to={urls.dataWarehouseSource(prefixedSourceId, 'webhook')}
+ subtle
+ data-attr="schemas-tab-webhook-not-registered"
+ >
+ <LemonTag type="danger">No webhook</LemonTag>
+ </Link>
+ </Tooltip>
+ ) : null
+ const syncMethodLink = (
<Tooltip
title={
blockedReason
@@ -662,6 +673,15 @@ function ManagedSchemaTable({
</Link>
</Tooltip>
)
+ if (!webhookTag) {
+ return syncMethodLink
+ }
+ return (
+ <div className="flex flex-wrap items-center gap-1">
+ {syncMethodLink}
+ {webhookTag}
+ </div>
+ )
},
},
{
diff --git a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
index ea41d12bd6b6..c8e4f06f4c73 100644
--- a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
+++ b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
@@ -168,13 +168,34 @@ export function ManagedSourcesTable(): JSX.Element {
if (!source.status) {
return null
}
+ const webhookTag = source.webhook_not_registered ? (
+ <Tooltip title="Some tables use webhook sync, but this source has no active webhook. Each sync of these tables reads the source API instead, which is slower. Open the source to set up the webhook or change the sync method.">
+ <LemonTag
+ type="danger"
+ forceClickable
+ onClick={() =>
+ router.actions.push(
+ urls.dataWarehouseSource(`managed-${source.id}`, 'webhook')
+ )
+ }
+ data-attr="managed-sources-webhook-not-registered"
+ >
+ No webhook
+ </LemonTag>
+ </Tooltip>
+ ) : null
const counts = syncingSchemasByStatus(source)
if (counts.length === 0) {
// Source has schemas but none are enabled — source.status can be stale
// ("Running" from before they were disabled), so show a neutral tag instead.
if (source.schemas.length > 0) {
- return <LemonTag type="muted">Not syncing</LemonTag>
+ return (
+ <div className="flex flex-wrap gap-1">
+ <LemonTag type="muted">Not syncing</LemonTag>
+ {webhookTag}
+ </div>
+ )
}
const tagContent = (
<LemonTag type={StatusTagSetting[source.status] || 'default'}>
@@ -214,6 +235,7 @@ export function ManagedSourcesTable(): JSX.Element {
</LemonTag>
</Tooltip>
))}
+ {webhookTag}
</div>
)
},
diff --git a/products/data_warehouse/frontend/shared/components/forms/WebhookSetupForm.tsx b/products/data_warehouse/frontend/shared/components/forms/WebhookSetupForm.tsx
index 82218e4e13bb..2b1f69daa498 100644
--- a/products/data_warehouse/frontend/shared/components/forms/WebhookSetupForm.tsx
+++ b/products/data_warehouse/frontend/shared/components/forms/WebhookSetupForm.tsx
@@ -45,13 +45,6 @@ interface WebhookSetupFormProps {
/** kea-forms logic and formKey for the manual webhook field inputs form */
formLogic?: LogicWrapper | BuiltLogic<any>
formKey?: string
- /**
- * Set from the new-source wizard only. The wizard's own Next button stays disabled on this step
- * until the webhook is set up (tables using webhook sync won't receive data otherwise), so this
- * shows a hint that leaving the wizard is fine and the webhook can be finished later from the
- * source's settings — without claiming the in-wizard Next action itself skips the step.
- */
- isWizardStep?: boolean
}
/**
@@ -69,7 +62,6 @@ export function WebhookSetupForm({
onCreateWebhook,
formLogic,
formKey,
- isWizardStep = false,
}: WebhookSetupFormProps): JSX.Element {
const webhookFields = sourceConfig?.webhookFields ?? []
// A blocked connection is manual-only in practice, so it takes the same path as a source that
@@ -123,13 +115,6 @@ export function WebhookSetupForm({
<LemonButton type="primary" onClick={onCreateWebhook}>
{manualOnly ? 'Generate webhook URL' : 'Create webhook'}
</LemonButton>
- {isWizardStep && (
- <p className="text-sm text-muted mb-0">
- Your source is already connected. If you'd rather not finish this now, you can leave the wizard
- and set up the webhook later from the source's settings — tables using webhook sync just won't
- receive data until then.
- </p>
- )}
</WebhookSetupCard>
)
}
@@ -170,10 +155,11 @@ export function WebhookSetupForm({
return (
<WebhookSetupCard>
- <h3 className="text-lg font-semibold">Almost done — finish setting up your {sourceName} webhook</h3>
- <LemonBanner type="success">
- Webhook created in {sourceName}. We just need a couple more details from your {sourceName} dashboard
- to start receiving events.
+ <h3 className="text-lg font-semibold">Finish setting up your {sourceName} webhook</h3>
+ <LemonBanner type="warning">
+ The webhook is created in {sourceName}, but it can't accept events yet. Copy the details below from
+ your {sourceName} dashboard and save them. Until then, tables that use webhook sync get no webhook
+ updates.
</LemonBanner>
{sourceConfig?.webhookSetupCaption && (
<LemonMarkdown className="text-sm">{sourceConfig.webhookSetupCaption}</LemonMarkdown>
@@ -201,10 +187,19 @@ export function WebhookSetupForm({
<h3 className="text-lg font-semibold">Manual webhook setup for {sourceName}</h3>
{(!manualOnly || !!autoCreationBlockedReason || webhookUrlMissing) && (
<LemonBanner type="warning">
- {(webhookUrlMissing && webhookResult?.error) ||
- autoCreationBlockedReason ||
- webhookResult?.error ||
- 'Could not create the webhook automatically.'}
+ <div className="flex flex-col gap-1">
+ <span>
+ {(webhookUrlMissing && webhookResult?.error) ||
+ autoCreationBlockedReason ||
+ webhookResult?.error ||
+ 'Could not create the webhook automatically.'}
+ </span>
+ <span>
+ {webhookUrlMissing
+ ? 'Tables that use webhook sync get no webhook updates until the webhook is set up.'
+ : `Tables that use webhook sync get no webhook updates until you add the webhook in ${sourceName} by hand, using the steps below.`}
+ </span>
+ </div>
</LemonBanner>
)}
{(!manualOnly || webhookUrlMissing) && (
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 2b0e06f463ae..f67096c3d5b9 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -11149,6 +11149,10 @@ export namespace Schemas {
* * `duplicate_primary_key` - Duplicate primary key
*/
incremental_sync_blocked: IncrementalSyncBlockedReasonEnum | NullEnum;
+ /**
+ * Whether this table uses webhook sync, but its source has no enabled webhook. Such a table gets no webhook updates. Each sync polls the source API instead, which is slower and can read the whole table. Set up the webhook on the source, or change `sync_type`.
+ */
+ webhook_not_registered: boolean;
/**
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
*/
@@ -11379,6 +11383,10 @@ export namespace Schemas {
*/
user_access_level: string | null;
supports_webhooks: boolean;
+ /**
+ * Whether at least one table of this source uses webhook sync, but the source has no enabled webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, which is slower and can read the whole table. Set up the webhook, or change the sync method of those tables.
+ */
+ webhook_not_registered: boolean;
/**
* Whether this source supports per-column sync selection via `enabled_columns`.
*/
@@ -15428,6 +15436,7 @@ export namespace Schemas {
unknown;
user_access_level: string | null;
supports_webhooks: boolean;
+ webhook_not_registered: boolean;
supports_column_selection: boolean;
file_name: string | null;
api_version: string | null;
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index ce68f71dd070..f38f537ecd1c 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -72,6 +72,7 @@ from products.warehouse_sources.backend.models.util import (
from products.warehouse_sources.backend.source_status import effective_source_status
from products.warehouse_sources.backend.types import ExternalDataSourceType
from products.warehouse_sources.backend.view_consumers import saved_queries_feeding_consumers
+from products.warehouse_sources.backend.webhook_registration import schema_needs_webhook, sources_missing_webhook
from . import contracts
@@ -105,6 +106,8 @@ __all__ = [
"list_jobs_for_source",
"list_column_statistics",
"saved_queries_feeding_consumers",
+ "schema_needs_webhook",
+ "sources_missing_webhook",
# framework-free helper transforms
"effective_source_status",
"clickhouse_columns_to_dwh_columns",
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index d533bfd80fa1..c8c9756ae831 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -43,6 +43,7 @@ from products.data_warehouse.backend.facade.api import (
unpause_external_data_schedule,
update_external_job_status,
)
+from products.warehouse_sources.backend.facade.api import schema_needs_webhook, sources_missing_webhook
from products.warehouse_sources.backend.facade.contracts import ConfigValueError, UndecryptedConfigError
from products.warehouse_sources.backend.facade.models import (
CDC_SNAPSHOT_LANE_KEY,
@@ -158,6 +159,7 @@ def source_syncs_on_demand(source_type: str) -> bool:
PINNED_TABLE_IDS_CONTEXT_KEY = "pinned_table_ids"
+WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY = "webhook_missing_by_source"
def pinned_table_ids(schemas: Iterable[ExternalDataSchema], team_id: int) -> dict[uuid.UUID, uuid.UUID]:
@@ -184,6 +186,20 @@ def pinned_table_ids(schemas: Iterable[ExternalDataSchema], team_id: int) -> dic
return {schema_id: table_id for schema_id, table_id in pins.items() if table_id in live}
+def webhook_not_registered(schema: ExternalDataSchema, context: dict[str, Any]) -> bool:
+ """Whether the schema syncs by webhook but its source has no enabled webhook.
+
+ A list fills `WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY` up front with `sources_missing_webhook`. Without it,
+ the first schema of each source looks the source up and keeps the answer for the rest.
+ """
+ if not schema_needs_webhook(schema):
+ return False
+ missing_by_source: dict[uuid.UUID, bool] = context.setdefault(WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY, {})
+ if schema.source_id not in missing_by_source:
+ missing_by_source.update(sources_missing_webhook(schema.team_id, [schema]))
+ return missing_by_source[schema.source_id]
+
+
def linked_table_id(schema: ExternalDataSchema, context: dict[str, Any]) -> uuid.UUID | None:
"""The schema's table id: its linked table, else the live table its metadata pins."""
if schema.table_id is not None:
@@ -917,6 +933,14 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"or fails for another reason, not when an update lands."
),
)
+ webhook_not_registered = serializers.SerializerMethodField(
+ read_only=True,
+ help_text=(
+ "Whether this table uses webhook sync, but its source has no enabled webhook. Such a table "
+ "gets no webhook updates. Each sync polls the source API instead, which is slower and can read "
+ "the whole table. Set up the webhook on the source, or change `sync_type`."
+ ),
+ )
enabled_columns = serializers.ListField(
child=serializers.CharField(),
required=False,
@@ -1029,6 +1053,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"primary_key_columns",
"cdc_table_mode",
"incremental_sync_blocked",
+ "webhook_not_registered",
"enabled_columns",
"masked_columns",
"row_filters",
@@ -1053,6 +1078,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"status",
"pending_self_managed_table",
"incremental_sync_blocked",
+ "webhook_not_registered",
"next_full_refresh_at",
"description",
"row_filter_columns",
@@ -1242,6 +1268,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
def get_incremental(self, schema: ExternalDataSchema) -> bool:
return schema.is_incremental
+ @extend_schema_field(serializers.BooleanField())
+ def get_webhook_not_registered(self, schema: ExternalDataSchema) -> bool:
+ return webhook_not_registered(schema, self.context)
+
@extend_schema_field(serializers.UUIDField(allow_null=True))
def get_table_id(self, schema: ExternalDataSchema) -> uuid.UUID | None:
return linked_table_id(schema, self.context)
@@ -2581,6 +2611,7 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
if kwargs.get("many") and args:
context = kwargs["context"]
context[PINNED_TABLE_IDS_CONTEXT_KEY] = pinned_table_ids(args[0], self.team_id)
+ context[WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY] = sources_missing_webhook(self.team_id, args[0])
serializer_class = self.get_serializer_class()
return serializer_class(*args, **kwargs)
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 8180f1e74788..9b19ee779d72 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -39,7 +39,12 @@ from products.data_warehouse.backend.facade.api import (
is_any_external_data_schema_paused,
is_custom_source_ai_builder_enabled_for_team,
)
-from products.warehouse_sources.backend.facade.api import effective_source_status, validate_source_prefix
+from products.warehouse_sources.backend.facade.api import (
+ effective_source_status,
+ schema_needs_webhook,
+ sources_missing_webhook,
+ validate_source_prefix,
+)
from products.warehouse_sources.backend.facade.contracts import (
CREDENTIALS_REFERENCE_INPUT_KEY,
SharedCredentialsEdit,
@@ -98,6 +103,7 @@ from products.warehouse_sources.backend.presentation.views.destination_links imp
)
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
PINNED_TABLE_IDS_CONTEXT_KEY,
+ WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY,
ExternalDataSchemaListSerializer,
ExternalDataSchemaSerializer,
canonical_masked_columns,
@@ -131,11 +137,27 @@ class ExternalDataSourceListSerializer(serializers.ListSerializer):
child = self.child
assert isinstance(child, ExternalDataSourceSerializers)
child.preloaded_source_inputs = SourceInputsStore.load_many(sources)
+ child.preloaded_webhook_missing_by_source = self._webhook_missing_by_source(child, sources)
return super().to_representation(sources)
+ def _webhook_missing_by_source(
+ self, child: ExternalDataSourceSerializers, sources: list[ExternalDataSource]
+ ) -> dict[UUID, bool] | None:
+ """One webhook lookup for every listed source, or None to look each source up on its own."""
+ schemas: list[ExternalDataSchema] = []
+ for source in sources:
+ prefetched = child._prefetched_schemas(source)
+ if prefetched is None:
+ return None
+ schemas.extend(prefetched)
+ if not sources:
+ return {}
+ return sources_missing_webhook(sources[0].team_id, schemas)
+
class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializers.ModelSerializer):
preloaded_source_inputs: dict[UUID, dict[str, Any]] = {}
+ preloaded_webhook_missing_by_source: dict[UUID, bool] | None = None
account_id = serializers.CharField(write_only=True)
client_secret = serializers.CharField(write_only=True)
@@ -157,6 +179,15 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
)
access_method = serializers.ChoiceField(choices=ExternalDataSource.AccessMethod.choices, read_only=True)
supports_webhooks = serializers.SerializerMethodField(read_only=True)
+ webhook_not_registered = serializers.SerializerMethodField(
+ read_only=True,
+ help_text=(
+ "Whether at least one table of this source uses webhook sync, but the source has no enabled "
+ "webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, "
+ "which is slower and can read the whole table. Set up the webhook, or change the sync method "
+ "of those tables."
+ ),
+ )
supports_column_selection = serializers.SerializerMethodField(
read_only=True,
help_text="Whether this source supports per-column sync selection via `enabled_columns`.",
@@ -264,6 +295,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
"revenue_analytics_config",
"user_access_level",
"supports_webhooks",
+ "webhook_not_registered",
"supports_column_selection",
"file_name",
"api_version",
@@ -284,6 +316,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
"user_access_level",
"access_method",
"supports_webhooks",
+ "webhook_not_registered",
"supports_column_selection",
"file_name",
"api_version",
@@ -354,6 +387,22 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
capture_exception(e)
return False
+ def _webhook_missing_by_source(self, instance: ExternalDataSource) -> dict[UUID, bool]:
+ if self.preloaded_webhook_missing_by_source is not None:
+ return self.preloaded_webhook_missing_by_source
+ cached: dict[UUID, bool] | None = getattr(instance, "_webhook_missing_by_source", None)
+ if cached is None:
+ schemas = self._prefetched_schemas(instance)
+ if schemas is None:
+ schemas = [schema for schema in instance.schemas.exclude(deleted=True) if schema_needs_webhook(schema)]
+ cached = sources_missing_webhook(instance.team_id, schemas)
+ instance._webhook_missing_by_source = cached # type: ignore[attr-defined]
+ return cached
+
+ @extend_schema_field(serializers.BooleanField())
+ def get_webhook_not_registered(self, instance: ExternalDataSource) -> bool:
+ return self._webhook_missing_by_source(instance).get(instance.pk, False)
+
@extend_schema_field(serializers.CharField(allow_null=True))
def get_file_name(self, instance: ExternalDataSource) -> str | None:
try:
@@ -425,6 +474,11 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
**self.context,
"error_redaction_values": self._error_redaction_values(instance),
PINNED_TABLE_IDS_CONTEXT_KEY: pinned_table_ids(schemas, instance.team_id),
+ WEBHOOK_MISSING_BY_SOURCE_CONTEXT_KEY: {
+ source_id: missing
+ for source_id, missing in self._webhook_missing_by_source(instance).items()
+ if source_id == instance.pk
+ },
}
if self.context.get("schemas_list_only"):
return ExternalDataSchemaListSerializer(schemas, many=True, read_only=True, context=context).data
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
index 785a86621b92..687dac7a0104 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
@@ -6,6 +6,7 @@ import functools
import contextlib
import pytest
+from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, call, patch
from django.conf import settings
@@ -17,10 +18,21 @@ import pyarrow as pa
import aioboto3
import pyarrow.parquet as pq
from aiobotocore.config import AioConfig
+from asgiref.sync import async_to_sync
from parameterized import parameterized
+from posthog.sync import database_sync_to_async
+
+from products.cdp.backend.facade.models import HogFunction
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.db import db_read_with_retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import (
+ WebhookFallback,
+ WebhookSourceManager,
+ WebhookState,
+)
_CLOSE_CONNECTIONS_PATH = (
"products.warehouse_sources.backend.temporal.data_imports.sources.common.db.close_old_connections"
@@ -431,6 +443,90 @@ class TestWebhookSourceManager:
assert tables == []
+class TestWebhookStateAndFallbackStamp(BaseTest):
+ def _setup(
+ self, *, sync_type: str, initial_sync_complete: bool, webhook: str | None
+ ) -> tuple[ExternalDataSource, ExternalDataSchema, ExternalDataJob]:
+ source = ExternalDataSource.objects.create(team=self.team, source_type="Stripe")
+ other_source = ExternalDataSource.objects.create(team=self.team, source_type="Stripe")
+ schema = ExternalDataSchema.objects.create(
+ name="Customer",
+ team=self.team,
+ source=source,
+ should_sync=True,
+ sync_type=sync_type,
+ initial_sync_complete=initial_sync_complete,
+ )
+ job = ExternalDataJob.objects.create(
+ team=self.team,
+ pipeline=source,
+ schema=schema,
+ status=ExternalDataJob.Status.RUNNING,
+ pipeline_version=ExternalDataJob.PipelineVersion.V3,
+ schema_snapshot={"name": "Customer", "sync_type": sync_type, "scheduled_full_refresh": True},
+ )
+ if webhook is not None:
+ HogFunction.objects.create(
+ team=self.team,
+ type="warehouse_source_webhook",
+ hog="return event",
+ enabled=webhook != "disabled",
+ deleted=webhook == "deleted",
+ inputs_schema=[{"key": "source_id", "type": "string"}],
+ inputs={"source_id": {"value": str(other_source.pk if webhook == "other_source" else source.pk)}},
+ )
+ return source, schema, job
+
+ @parameterized.expand(
+ [
+ ("no_webhook", "webhook", True, False, None, WebhookState.NO_WEBHOOK),
+ ("disabled_webhook", "webhook", True, False, "disabled", WebhookState.NO_WEBHOOK),
+ ("deleted_webhook", "webhook", True, False, "deleted", WebhookState.NO_WEBHOOK),
+ ("webhook_for_other_source", "webhook", True, False, "other_source", WebhookState.NO_WEBHOOK),
+ ("enabled_webhook", "webhook", True, False, "enabled", WebhookState.ACTIVE),
+ ("initial_sync_pending", "webhook", False, False, None, WebhookState.POLL),
+ ("reset_pipeline", "webhook", True, True, None, WebhookState.POLL),
+ ("not_webhook_sync", "incremental", True, False, None, WebhookState.POLL),
+ ]
+ )
+ def test_fallback_is_stamped_only_when_the_webhook_is_missing(
+ self,
+ _name: str,
+ sync_type: str,
+ initial_sync_complete: bool,
+ reset_pipeline: bool,
+ webhook: str | None,
+ expected_state: WebhookState,
+ ) -> None:
+ source, schema, job = self._setup(
+ sync_type=sync_type, initial_sync_complete=initial_sync_complete, webhook=webhook
+ )
+ inputs = _make_inputs(team_id=self.team.pk, schema_id=str(schema.id), reset_pipeline=reset_pipeline)
+ inputs.source_id = str(source.id)
+ inputs.job_id = str(job.id)
+ manager = WebhookSourceManager(inputs=inputs, logger=AsyncMock())
+
+ # The pool helper runs on another thread, whose connection can't see this test's transaction, and
+ # the read retry closes connections, which would close the test's own.
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3.database_sync_to_async_pool",
+ database_sync_to_async,
+ ),
+ patch(_CLOSE_CONNECTIONS_PATH),
+ ):
+ state = async_to_sync(manager.webhook_state)()
+ async_to_sync(manager.record_webhook_fallback)(WebhookFallback.INCREMENTAL)
+
+ job.refresh_from_db()
+ assert state is expected_state
+ assert manager.webhook_missing is (expected_state is WebhookState.NO_WEBHOOK)
+ expected_snapshot = {"name": "Customer", "sync_type": sync_type, "scheduled_full_refresh": True}
+ if expected_state is WebhookState.NO_WEBHOOK:
+ expected_snapshot["webhook_fallback"] = "incremental"
+ assert job.schema_snapshot == expected_snapshot
+
+
class TestDbReadWithRetry:
def test_rides_out_pool_wait_timeout_then_succeeds(self):
sentinel = object()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
index 41e54b66a516..b080eebdfbae 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
@@ -2,9 +2,11 @@ from __future__ import annotations
import asyncio
from collections.abc import AsyncGenerator, Callable
+from enum import StrEnum
from typing import Optional
from django.conf import settings
+from django.db import transaction
import orjson
import pyarrow as pa
@@ -24,14 +26,52 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bat
from products.warehouse_sources.backend.temporal.data_imports.sources.common.db import db_read_with_retry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
+# Informational key on a job's `schema_snapshot`. The job keeps `sync_type: webhook`, because the next
+# run compares it with the schema's sync type to decide whether it may continue this job's cursor.
+WEBHOOK_FALLBACK_SNAPSHOT_KEY = "webhook_fallback"
+
+
+class WebhookState(StrEnum):
+ # Read the rows that the webhook delivered.
+ ACTIVE = "active"
+ # The run polls the API by design: the table is not on webhook sync, or the run seeds or rebuilds it.
+ POLL = "poll"
+ # The table is on webhook sync, but the source has no enabled webhook, so nothing delivers rows.
+ NO_WEBHOOK = "no_webhook"
+
+
+class WebhookFallback(StrEnum):
+ """How a webhook-sync run with no enabled webhook polls the API instead."""
+
+ FULL_REFRESH = "full_refresh"
+ INCREMENTAL = "incremental"
+
+
+def _stamp_webhook_fallback(job_id: str, team_id: int, fallback: WebhookFallback) -> None:
+ from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+
+ with transaction.atomic():
+ job = (
+ ExternalDataJob.objects.select_for_update()
+ .filter(id=job_id, team_id=team_id)
+ .only("id", "schema_snapshot")
+ .first()
+ )
+ if job is None:
+ return
+ snapshot = {**dict(job.schema_snapshot or {}), WEBHOOK_FALLBACK_SNAPSHOT_KEY: fallback.value}
+ ExternalDataJob.objects.filter(id=job.id, team_id=team_id).update(schema_snapshot=snapshot)
+
class WebhookSourceManager:
_inputs: SourceInputs
_logger: FilteringBoundLogger
+ _state: WebhookState | None
def __init__(self, inputs: SourceInputs, logger: FilteringBoundLogger) -> None:
self._inputs = inputs
self._logger = logger
+ self._state = None
def _get_webhook_s3_prefix(self) -> str:
return f"s3://{settings.DATAWAREHOUSE_BUCKET}/source_webhook_producer/{self._inputs.team_id}/{self._inputs.schema_id}"
@@ -40,8 +80,11 @@ class WebhookSourceManager:
return s3_path.replace("s3://", "")
async def webhook_enabled(self, webhook_only: bool = False) -> bool:
- from products.cdp.backend.models.hog_functions.hog_function import HogFunction
+ return await self.webhook_state(webhook_only=webhook_only) is WebhookState.ACTIVE
+
+ async def webhook_state(self, webhook_only: bool = False) -> WebhookState:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+ from products.warehouse_sources.backend.webhook_registration import source_has_enabled_webhook
schema = await database_sync_to_async_pool(db_read_with_retry)(
lambda: ExternalDataSchema.objects.get(id=self._inputs.schema_id, team_id=self._inputs.team_id)
@@ -60,19 +103,37 @@ class WebhookSourceManager:
f"schema.initial_sync_complete={schema.initial_sync_complete}. "
f"webhook_only={webhook_only}. reset_pipeline={self._inputs.reset_pipeline}"
)
- return False
+ self._state = WebhookState.POLL
+ return self._state
has_webhook_function = await database_sync_to_async_pool(db_read_with_retry)(
- lambda: HogFunction.objects.filter(
- inputs__source_id__value=self._inputs.source_id,
- team_id=self._inputs.team_id,
- type="warehouse_source_webhook",
- enabled=True,
- deleted=False,
- ).exists()
+ lambda: source_has_enabled_webhook(self._inputs.team_id, self._inputs.source_id)
)
- return has_webhook_function
+ self._state = WebhookState.ACTIVE if has_webhook_function else WebhookState.NO_WEBHOOK
+ return self._state
+
+ @property
+ def webhook_missing(self) -> bool:
+ """Whether the last `webhook_state` check found a webhook-sync table with no enabled webhook."""
+ return self._state is WebhookState.NO_WEBHOOK
+
+ async def record_webhook_fallback(self, fallback: WebhookFallback) -> None:
+ """Mark this run's job as one that polls the API because the table has no enabled webhook.
+
+ Call it after `webhook_enabled` returns False and the source starts to poll. It does nothing
+ unless that check found a webhook-sync table with no enabled webhook, so an initial sync or a
+ reset, which poll by design, stay unmarked. Without the mark, the job still reads as a webhook
+ sync, so its rows count as webhook rows.
+ """
+ if not self.webhook_missing:
+ return
+ await self._logger.awarning(
+ "This table uses webhook sync, but the source has no active webhook, so this sync reads the "
+ "source API instead. Set up the webhook on the source's Webhook tab, or change the sync method.",
+ webhook_fallback=fallback.value,
+ )
+ await database_sync_to_async_pool(_stamp_webhook_fallback)(self._inputs.job_id, self._inputs.team_id, fallback)
async def schema_is_webhook(self) -> bool:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index 9d8fbd3d0520..1b8a76fc6f88 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -41,7 +41,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.fan
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import (
+ WebhookFallback,
+ WebhookSourceManager,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.constants import (
ACCOUNT_RESOURCE_NAME,
APPLICATION_FEE_RESOURCE_NAME,
@@ -1029,6 +1032,41 @@ def _stripe_client(
)
+def _lists_every_object(
+ resource: Union[StripeResource, StripeNestedResource],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ db_incremental_field_earliest_value: Optional[Any],
+) -> bool:
+ """Whether a read lists every object, or only the objects created after the stored cursor."""
+ return (
+ not should_use_incremental_field
+ or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)
+ or isinstance(resource, StripeNestedResource)
+ )
+
+
+def _webhook_fallback(
+ api_key: str,
+ endpoint: str,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ db_incremental_field_earliest_value: Optional[Any],
+) -> Optional[WebhookFallback]:
+ """How `get_rows` reads a webhook-sync table whose webhook is missing, or None when it reads nothing."""
+ if endpoint in V2_LIST_PATHS:
+ return WebhookFallback.FULL_REFRESH
+ # Only the kind of resource matters here. Building the resources sends no request.
+ resource = _build_resources(StripeClient(api_key)).get(endpoint)
+ if resource is None:
+ return None
+ if _lists_every_object(
+ resource, should_use_incremental_field, db_incremental_field_last_value, db_incremental_field_earliest_value
+ ):
+ return WebhookFallback.FULL_REFRESH
+ return WebhookFallback.INCREMENTAL
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -1089,10 +1127,10 @@ def get_rows(
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- if (
- not should_use_incremental_field
- or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)
- or isinstance(resource, StripeNestedResource)
+ # The isinstance check repeats one inside `_lists_every_object` so the type checker narrows
+ # `resource` to a flat resource on the incremental path below.
+ if isinstance(resource, StripeNestedResource) or _lists_every_object(
+ resource, should_use_incremental_field, db_incremental_field_last_value, db_incremental_field_earliest_value
):
logger.debug(f"Stripe: iterating all objects from resource")
# Set only on the nested warehouse path; the final flush below is shared with flat
@@ -1683,6 +1721,17 @@ def stripe_source(
warehouse_parent, warehouse_table_id = _reconcile_warehouse_resume(
endpoint, warehouse_parent, resumable_source_manager, logger
)
+ # A webhook-only endpoint has no list to poll, so its run reads nothing and has no fallback.
+ if webhook_source_manager.webhook_missing and not webhook_only:
+ fallback = _webhook_fallback(
+ api_key,
+ endpoint,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ db_incremental_field_earliest_value,
+ )
+ if fallback is not None:
+ async_to_sync(webhook_source_manager.record_webhook_fallback)(fallback)
def items():
if webhook_enabled:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index a315325d722d..e0e4de22dee3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -1574,6 +1574,54 @@ class TestWebhookOnlyResponseWiring:
assert response.webhook_only is False
manager.webhook_enabled.assert_awaited_once_with(webhook_only=False)
+ @parameterized.expand(
+ [
+ ("listing_without_cursor", CUSTOMER_RESOURCE_NAME, False, True, False, None, "full_refresh"),
+ ("incremental_without_value", CUSTOMER_RESOURCE_NAME, False, True, True, None, "full_refresh"),
+ ("cursor_poll", CUSTOMER_RESOURCE_NAME, False, True, True, 1700000000, "incremental"),
+ (
+ "nested_lists_every_parent",
+ CUSTOMER_PAYMENT_METHOD_RESOURCE_NAME,
+ False,
+ True,
+ True,
+ 1700000000,
+ "full_refresh",
+ ),
+ ("webhook_only_endpoint", DISCOUNT_RESOURCE_NAME, False, True, False, None, None),
+ ("webhook_present_or_poll_by_design", CUSTOMER_RESOURCE_NAME, False, False, False, None, None),
+ ("webhook_active", CUSTOMER_RESOURCE_NAME, True, False, False, None, None),
+ ]
+ )
+ def test_records_webhook_fallback(
+ self,
+ _name: str,
+ endpoint: str,
+ enabled: bool,
+ webhook_missing: bool,
+ should_use_incremental_field: bool,
+ last_value: int | None,
+ expected: str | None,
+ ) -> None:
+ manager = self._make_manager(enabled=enabled)
+ manager.webhook_missing = webhook_missing
+ stripe_module.stripe_source(
+ api_key="sk_test_123",
+ account_id=None,
+ endpoint=endpoint,
+ db_incremental_field_last_value=last_value,
+ db_incremental_field_earliest_value=last_value,
+ logger=MagicMock(adebug=mock.AsyncMock()),
+ resumable_source_manager=MagicMock(can_resume=MagicMock(return_value=False)),
+ webhook_source_manager=manager,
+ api_version=STRIPE_API_VERSION_ACACIA,
+ should_use_incremental_field=should_use_incremental_field,
+ )
+ if expected is None:
+ manager.record_webhook_fallback.assert_not_awaited()
+ else:
+ manager.record_webhook_fallback.assert_awaited_once_with(expected)
+
def test_discount_partitions_on_start_not_created(self) -> None:
# Discount objects carry `start`/`end`, not `created`. If the incremental-field entry is
# dropped the partition key falls back to "created" and the Delta partitioner KeyErrors on
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
index 363be89fa294..84818c7f8d3d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
@@ -176,7 +176,9 @@ def _build_schema_snapshot(schema: ExternalDataSchema) -> dict[str, Any]:
`post_import_job` reads `last_synced_at` back, and a CDC history lane's job adds
`cdc_write_mode` for the jobs API. The rest is only ever read by a person: the schema audit log does not diff
- `sync_type_config`, so this is the one record of the cursor and reset flags a run ran with.
+ `sync_type_config`, so this is the one record of the cursor and reset flags a run ran with. A
+ webhook-sync run that polls the API because no webhook is enabled adds `webhook_fallback`
+ (see `WebhookSourceManager.record_webhook_fallback`), and keeps `sync_type` as it is.
"""
sync_type_config = {
key: value
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 0157afbcfb82..043925909ff6 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -32,6 +32,7 @@ from posthog.models.utils import generate_random_token_personal
from posthog.redis import get_client
from posthog.temporal.common.schedule import create_schedule, describe_schedule
+from products.cdp.backend.facade.models import HogFunction
from products.data_modeling.backend.facade.models import Edge, Node
from products.data_warehouse.backend.facade.api import (
DIRECT_POSTGRES_URL_PATTERN,
@@ -5152,6 +5153,47 @@ class TestExternalDataSchemaRetrieveSource(APIBaseTest):
assert {row["id"]: row["table_id"] for row in rows} == expected
+ def _webhook_source(self, *, with_webhook: bool) -> tuple[ExternalDataSource, list[ExternalDataSchema]]:
+ source, _ = self._create()
+ ExternalDataSchema.objects.filter(source=source).delete()
+ schemas = [
+ ExternalDataSchema.objects.create(
+ name=name, team=self.team, source=source, should_sync=True, sync_type="webhook"
+ )
+ for name in ("Customer", "Invoice")
+ ]
+ if with_webhook:
+ HogFunction.objects.create(
+ team=self.team,
+ type="warehouse_source_webhook",
+ hog="return event",
+ enabled=True,
+ inputs_schema=[{"key": "source_id", "type": "string"}],
+ inputs={"source_id": {"value": str(source.pk)}},
+ )
+ return source, schemas
+
+ @parameterized.expand([("no_webhook", False, True), ("enabled_webhook", True, False)])
+ def test_retrieve_webhook_not_registered(self, _name: str, with_webhook: bool, expected: bool) -> None:
+ _, schemas = self._webhook_source(with_webhook=with_webhook)
+
+ detail = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schemas[0].id}/").json()
+
+ assert detail["webhook_not_registered"] is expected
+
+ def test_list_serializer_looks_up_webhooks_once_for_all_sources(self) -> None:
+ _, with_webhook = self._webhook_source(with_webhook=True)
+ _, without_webhook = self._webhook_source(with_webhook=False)
+ view = ExternalDataSchemaViewset()
+ view.team_id = self.team.pk
+
+ with self.assertNumQueries(1):
+ rows = view.get_serializer(
+ [*with_webhook, *without_webhook], many=True, context={"include_columns": False}
+ ).data
+
+ assert [row["webhook_not_registered"] for row in rows] == [False, False, True, True]
+
def test_list_omits_source_summary(self):
self._create()
response = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/")
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 7605873955b2..0f1cd6e1dfdb 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -32,6 +32,7 @@ from posthog.models.user import User
from posthog.models.utils import generate_random_token_personal
from products.access_control.backend.models.access_control import AccessControl
+from products.cdp.backend.facade.models import HogFunction
from products.data_tools.backend.models.join import DataWarehouseJoin
from products.data_warehouse.backend.facade.api import DIRECT_POSTGRES_URL_PATTERN, DIRECT_TRINO_URL_PATTERN
from products.data_warehouse.backend.facade.models import ExternalDataSourceRevenueAnalyticsConfig
@@ -448,6 +449,7 @@ class TestExternalDataSource(APIBaseTest):
latest_error: str | None = None,
should_sync: bool = True,
row_count: int = 0,
+ sync_type: str | None = None,
) -> ExternalDataSchema:
table = DataWarehouseTable.objects.create(
name=name, team=self.team, external_data_source=source, row_count=row_count
@@ -460,8 +462,60 @@ class TestExternalDataSource(APIBaseTest):
status=status,
latest_error=latest_error,
should_sync=should_sync,
+ sync_type=sync_type,
)
+ def _make_webhook_function(
+ self, source: ExternalDataSource, *, enabled: bool = True, deleted: bool = False
+ ) -> None:
+ HogFunction.objects.create(
+ team=self.team,
+ type="warehouse_source_webhook",
+ hog="return event",
+ enabled=enabled,
+ deleted=deleted,
+ inputs_schema=[{"key": "source_id", "type": "string"}],
+ inputs={"source_id": {"value": str(source.pk)}},
+ )
+
+ @parameterized.expand(
+ [
+ ("no_webhook", "webhook", True, None, True),
+ ("disabled_webhook", "webhook", True, "disabled", True),
+ ("deleted_webhook", "webhook", True, "deleted", True),
+ ("webhook_for_other_source", "webhook", True, "other_source", True),
+ ("enabled_webhook", "webhook", True, "enabled", False),
+ ("webhook_table_not_syncing", "webhook", False, None, False),
+ ("no_webhook_table", "incremental", True, None, False),
+ ]
+ )
+ def test_webhook_not_registered(
+ self, _name: str, sync_type: str, should_sync: bool, webhook: str | None, expected: bool
+ ) -> None:
+ source = self._make_source("webhooked")
+ other_source = self._make_source("other")
+ self._make_schema_with_table(source, "Customer", sync_type=sync_type, should_sync=should_sync)
+ self._make_schema_with_table(source, "Invoice", sync_type="full_refresh")
+ if webhook is not None:
+ self._make_webhook_function(
+ other_source if webhook == "other_source" else source,
+ enabled=webhook != "disabled",
+ deleted=webhook == "deleted",
+ )
+
+ listed = self.client.get(f"/api/environments/{self.team.pk}/external_data_sources/").json()["results"]
+ detail = self.client.get(f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/").json()
+
+ assert {row["prefix"]: row["webhook_not_registered"] for row in listed} == {
+ "webhooked": expected,
+ "other": False,
+ }
+ assert detail["webhook_not_registered"] is expected
+ assert {row["name"]: row["webhook_not_registered"] for row in detail["schemas"]} == {
+ "Customer": expected,
+ "Invoice": False,
+ }
+
def test_exists_ignores_deleted_and_other_project_sources_without_loading_schemas(self) -> None:
source = self._make_source("exists")
self._make_schema_with_table(source, "example_customers")
@@ -590,7 +644,7 @@ class TestExternalDataSource(APIBaseTest):
# A regression to per-source credential/source lookups or the duplicate schema prefetch shows up
# here as a rising query count.
first = self._make_source("one")
- self._make_schema_with_table(first, "A")
+ self._make_schema_with_table(first, "A", sync_type="webhook")
# warm any per-request caches so the two measurements compare like with like
self.client.get(f"/api/environments/{self.team.pk}/external_data_sources/")
@@ -600,7 +654,9 @@ class TestExternalDataSource(APIBaseTest):
for index in range(4):
extra = self._make_source(f"many-{index}")
- self._make_schema_with_table(extra, f"S{index}")
+ self._make_schema_with_table(extra, f"S{index}", sync_type="webhook")
+ if index % 2:
+ self._make_webhook_function(extra)
with CaptureQueriesContext(connection) as many_source_queries:
self.client.get(f"/api/environments/{self.team.pk}/external_data_sources/")
@@ -3629,6 +3685,7 @@ class TestExternalDataSource(APIBaseTest):
"revenue_analytics_config",
"user_access_level",
"supports_webhooks",
+ "webhook_not_registered",
"supports_column_selection",
"file_name",
"api_version",
@@ -3665,6 +3722,7 @@ class TestExternalDataSource(APIBaseTest):
"primary_key_columns": None,
"cdc_table_mode": "consolidated",
"incremental_sync_blocked": None,
+ "webhook_not_registered": False,
"enabled_columns": None,
"masked_columns": None,
"row_filters": None,
diff --git a/products/warehouse_sources/backend/webhook_registration.py b/products/warehouse_sources/backend/webhook_registration.py
new file mode 100644
index 000000000000..fce11b62cf12
--- /dev/null
+++ b/products/warehouse_sources/backend/webhook_registration.py
@@ -0,0 +1,63 @@
+"""Whether a source has the webhook that its webhook-sync tables need.
+
+A webhook-sync table gets its rows from a `warehouse_source_webhook` hog function for its source.
+When no such function is enabled, the import worker polls the source API instead, and the API
+warns the user. Both read the rule from this module, so they agree about the same source.
+"""
+
+import uuid
+from collections.abc import Collection, Iterable
+from typing import Protocol
+
+from django.db.models import QuerySet
+
+from products.cdp.backend.facade.models import HogFunction
+from products.warehouse_sources.backend.facade.types import ExternalDataSchemaSyncType
+
+WAREHOUSE_SOURCE_WEBHOOK_TYPE = "warehouse_source_webhook"
+
+
+class WebhookSchemaState(Protocol):
+ @property
+ def source_id(self) -> uuid.UUID: ...
+
+ @property
+ def sync_type(self) -> str | None: ...
+
+ @property
+ def should_sync(self) -> bool: ...
+
+ @property
+ def deleted(self) -> bool | None: ...
+
+
+def schema_needs_webhook(schema: WebhookSchemaState) -> bool:
+ return schema.sync_type == ExternalDataSchemaSyncType.WEBHOOK and schema.should_sync and not schema.deleted
+
+
+def _enabled_webhook_functions(team_id: int) -> QuerySet[HogFunction]:
+ return HogFunction.objects.filter(team_id=team_id, type=WAREHOUSE_SOURCE_WEBHOOK_TYPE, enabled=True, deleted=False)
+
+
+def source_has_enabled_webhook(team_id: int, source_id: uuid.UUID | str) -> bool:
+ return _enabled_webhook_functions(team_id).filter(inputs__source_id__value=str(source_id)).exists()
+
+
+def source_ids_with_enabled_webhook(team_id: int, source_ids: Collection[uuid.UUID | str]) -> set[str]:
+ """The ids, as strings, of the given sources that have an enabled webhook. One query."""
+ if not source_ids:
+ return set()
+ found = _enabled_webhook_functions(team_id).filter(
+ inputs__source_id__value__in=[str(source_id) for source_id in source_ids]
+ )
+ return {str(value) for value in found.values_list("inputs__source_id__value", flat=True)}
+
+
+def sources_missing_webhook(team_id: int, schemas: Iterable[WebhookSchemaState]) -> dict[uuid.UUID, bool]:
+ """For each source with a table that needs a webhook, whether that source has no enabled webhook.
+
+ Sources with no such table are left out. At most one query, whatever the number of sources.
+ """
+ candidates = {schema.source_id for schema in schemas if schema_needs_webhook(schema)}
+ registered = source_ids_with_enabled_webhook(team_id, candidates)
+ return {source_id: str(source_id) not in registered for source_id in candidates}
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index ce010d5a587d..04755de3821a 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -578,6 +578,8 @@ export interface ExternalDataSchemaApi {
* * `missing_primary_key` - Missing primary key
* * `duplicate_primary_key` - Duplicate primary key */
readonly incremental_sync_blocked: IncrementalSyncBlockedReasonEnumApi | null
+ /** Whether this table uses webhook sync, but its source has no enabled webhook. Such a table gets no webhook updates. Each sync polls the source API instead, which is slower and can read the whole table. Set up the webhook on the source, or change `sync_type`. */
+ readonly webhook_not_registered: boolean
/**
* Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.
* @nullable
@@ -787,6 +789,8 @@ export interface PatchedExternalDataSchemaApi {
* * `missing_primary_key` - Missing primary key
* * `duplicate_primary_key` - Duplicate primary key */
readonly incremental_sync_blocked?: IncrementalSyncBlockedReasonEnumApi | null
+ /** Whether this table uses webhook sync, but its source has no enabled webhook. Such a table gets no webhook updates. Each sync polls the source API instead, which is slower and can read the whole table. Set up the webhook on the source, or change `sync_type`. */
+ readonly webhook_not_registered?: boolean
/**
* Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.
* @nullable
@@ -976,6 +980,8 @@ export interface ExternalDataSourceSerializersApi {
*/
readonly user_access_level: string | null
readonly supports_webhooks: boolean
+ /** Whether at least one table of this source uses webhook sync, but the source has no enabled webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, which is slower and can read the whole table. Set up the webhook, or change the sync method of those tables. */
+ readonly webhook_not_registered: boolean
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection: boolean
/**
@@ -1133,6 +1139,8 @@ export interface PatchedExternalDataSourceSerializersApi {
*/
readonly user_access_level?: string | null
readonly supports_webhooks?: boolean
+ /** Whether at least one table of this source uses webhook sync, but the source has no enabled webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, which is slower and can read the whole table. Set up the webhook, or change the sync method of those tables. */
+ readonly webhook_not_registered?: boolean
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection?: boolean
/**
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 1586a1f165b6..e78b343883fc 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -47237,6 +47237,8 @@ export namespace Schemas {
* * `missing_primary_key` - Missing primary key
* * `duplicate_primary_key` - Duplicate primary key */
readonly incremental_sync_blocked: IncrementalSyncBlockedReasonEnum | null;
+ /** Whether this table uses webhook sync, but its source has no enabled webhook. Such a table gets no webhook updates. Each sync polls the source API instead, which is slower and can read the whole table. Set up the webhook on the source, or change `sync_type`. */
+ readonly webhook_not_registered: boolean;
/**
* Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.
* @nullable
@@ -47598,6 +47600,8 @@ export namespace Schemas {
*/
readonly user_access_level: string | null;
readonly supports_webhooks: boolean;
+ /** Whether at least one table of this source uses webhook sync, but the source has no enabled webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, which is slower and can read the whole table. Set up the webhook, or change the sync method of those tables. */
+ readonly webhook_not_registered: boolean;
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection: boolean;
/**
@@ -81712,6 +81716,8 @@ export namespace Schemas {
* * `missing_primary_key` - Missing primary key
* * `duplicate_primary_key` - Duplicate primary key */
readonly incremental_sync_blocked?: IncrementalSyncBlockedReasonEnum | null;
+ /** Whether this table uses webhook sync, but its source has no enabled webhook. Such a table gets no webhook updates. Each sync polls the source API instead, which is slower and can read the whole table. Set up the webhook on the source, or change `sync_type`. */
+ readonly webhook_not_registered?: boolean;
/**
* Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.
* @nullable
@@ -81830,6 +81836,8 @@ export namespace Schemas {
*/
readonly user_access_level?: string | null;
readonly supports_webhooks?: boolean;
+ /** Whether at least one table of this source uses webhook sync, but the source has no enabled webhook. Those tables get no webhook updates. Each of their syncs polls the source API instead, which is slower and can read the whole table. Set up the webhook, or change the sync method of those tables. */
+ readonly webhook_not_registered?: boolean;
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection?: boolean;
/**