Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): remove hog function sync alerts (#121333)

Tom Owers2ca5117b51e2
diff --git a/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.test.ts b/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.test.ts
index cac9980b34fa..dea2f920b922 100644
--- a/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.test.ts
+++ b/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.test.ts
@@ -1,34 +1,18 @@
import { expectLogic } from 'kea-test-utils'
-import posthog from 'posthog-js'
import { lemonToast } from '@posthog/lemon-ui'
-import api from 'lib/api'
import {
HEALTH_ALERT_DESTINATIONS,
HEALTH_ALERT_SUB_TEMPLATE_IDS,
HEALTH_ALERT_TRIGGERS,
} from 'scenes/health-alerts/healthAlertsWizardConfig'
-import { HOG_FUNCTION_SUB_TEMPLATES } from 'scenes/hog-functions/sub-templates/sub-templates'
import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
-import {
- CyclotronJobFilterPropertyFilter,
- CyclotronJobFiltersType,
- PropertyFilterType,
- PropertyOperator,
-} from '~/types'
+import { CyclotronJobFiltersType, PropertyFilterType, PropertyOperator } from '~/types'
-import {
- AlertWizardLogicProps,
- alertWizardLogic,
- applyKindFilter,
- applyPresetFilters,
- buildAlertHogFunctionConfiguration,
- decorateAlertName,
- testInvocationFailureMessage,
-} from './alertWizardLogic'
+import { alertWizardLogic, applyKindFilter, decorateAlertName, testInvocationFailureMessage } from './alertWizardLogic'
describe('alertWizardLogic', () => {
describe('applyKindFilter', () => {
@@ -92,57 +76,6 @@ describe('alertWizardLogic', () => {
})
})
- describe('applyPresetFilters', () => {
- const baseFilters: CyclotronJobFiltersType = {
- events: [{ id: '$data_warehouse_sync_failed', type: 'events' }],
- }
- const sourceFilter: CyclotronJobFilterPropertyFilter = {
- key: 'source_id',
- value: ['source-1'],
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Event,
- }
-
- it.each([
- ['undefined', undefined],
- ['an empty array', [] as CyclotronJobFilterPropertyFilter[]],
- ])('matches applyKindFilter when presetPropertyFilters is %s', (_, preset) => {
- expect(applyPresetFilters(baseFilters, ['job_failed'], preset)).toEqual(
- applyKindFilter(baseFilters, ['job_failed'])
- )
- expect(applyPresetFilters(baseFilters, null, preset)).toBe(baseFilters)
- })
-
- it.each([
- ['no kinds', null, ['source_id']],
- ['kinds', ['job_failed'], ['kind', 'source_id']],
- ])('merges the preset filters beside %s', (_, kinds, expectedKeys) => {
- const result = applyPresetFilters(baseFilters, kinds, [sourceFilter])
- expect(result?.properties?.map((p) => ('key' in p ? p.key : null))).toEqual(expectedKeys)
- expect(result?.events).toEqual(baseFilters.events)
- })
-
- it('returns undefined when base filters are undefined', () => {
- expect(applyPresetFilters(undefined, null, [sourceFilter])).toBeUndefined()
- })
- })
-
- describe('buildAlertHogFunctionConfiguration', () => {
- const base = { templateId: 'template-slack', name: 'n', description: 'd', filters: null, inputs: {} }
-
- it.each([
- ['omitted', undefined, null],
- ['null', null, null],
- [
- 'set',
- { hash: '{event.properties.schema_id}', ttl: 3600, threshold: null },
- { hash: '{event.properties.schema_id}', ttl: 3600, threshold: null },
- ],
- ])('passes masking through when it is %s', (_, masking, expected) => {
- expect(buildAlertHogFunctionConfiguration({ ...base, masking }).masking).toEqual(expected)
- })
- })
-
describe('decorateAlertName', () => {
const baseName = 'Email when a Health check fires'
@@ -164,13 +97,6 @@ describe('alertWizardLogic', () => {
)
})
- it.each([
- ['without kinds', null, 'Email when a Health check fires for Stripe'],
- ['after the kinds', ['sdk_outdated'], 'Email when a Health check fires (SDK outdated) for Stripe'],
- ])('appends the name suffix %s', (_, kinds, expected) => {
- expect(decorateAlertName(baseName, kinds, 'for Stripe')).toBe(expected)
- })
-
it('falls back to the raw kind when no label is registered', () => {
expect(decorateAlertName(baseName, ['some_future_kind'])).toBe(
'Email when a Health check fires (some_future_kind)'
@@ -178,110 +104,6 @@ describe('alertWizardLogic', () => {
})
})
- describe('creating an alert', () => {
- const template = { id: 'template-slack', name: 'Slack', code: '', inputs_schema: [] }
- const sourceFilter: CyclotronJobFilterPropertyFilter = {
- key: 'source_id',
- value: ['source-1'],
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Event,
- }
- const healthSlackName = HOG_FUNCTION_SUB_TEMPLATES['health-check-firing'].find(
- (t) => t.template_id === 'template-slack'
- )?.name
- const dwhTriggers = [
- {
- key: 'data-warehouse-sync-completed',
- name: 'Sync completed',
- description: 'A table finished syncing',
- },
- ] as const
-
- async function createAlert(
- logicProps: Partial<AlertWizardLogicProps>,
- triggerKey: 'health-check-firing' | 'data-warehouse-sync-completed'
- ): Promise<{ payload: Record<string, any>; captureSpy: jest.SpyInstance }> {
- const createSpy = jest.spyOn(api.hogFunctions, 'create').mockResolvedValue({} as any)
- const captureSpy = jest.spyOn(posthog, 'capture').mockReturnValue(undefined)
- jest.spyOn(lemonToast, 'success').mockReturnValue('toast-id')
-
- const logic = alertWizardLogic({
- logicKey: 'create-test',
- subTemplateIds: HEALTH_ALERT_SUB_TEMPLATE_IDS,
- triggers: HEALTH_ALERT_TRIGGERS,
- destinations: HEALTH_ALERT_DESTINATIONS,
- ...logicProps,
- })
- logic.mount()
- logic.actions.setDestinationKey('slack')
- logic.actions.setTriggerKey(triggerKey)
- await expectLogic(logic, () => logic.actions.loadTemplate('template-slack')).toDispatchActions([
- 'loadTemplateSuccess',
- ])
- await expectLogic(logic, () => logic.actions.submitConfiguration()).toDispatchActions([
- 'createAlertSuccess',
- ])
- return { payload: createSpy.mock.calls[0][0] as Record<string, any>, captureSpy }
- }
-
- beforeEach(() => {
- useMocks({
- get: {
- '/api/environments/:team_id/hog_functions/': { count: 0, results: [] },
- '/api/projects/:team_id/hog_functions/': { count: 0, results: [] },
- '/api/environments/:team_id/hog_function_templates/template-slack/': template,
- '/api/projects/:team_id/hog_function_templates/template-slack/': template,
- },
- })
- initKeaTests()
- })
-
- afterEach(() => {
- jest.restoreAllMocks()
- })
-
- it('keeps the payload and event name unchanged when no new props are set', async () => {
- const { payload, captureSpy } = await createAlert({ contextId: 'health-alerts' }, 'health-check-firing')
-
- expect(payload.masking).toBeNull()
- expect(payload.filters.properties).toBeUndefined()
- expect(payload.name).toBe(healthSlackName)
- expect(captureSpy).toHaveBeenCalledWith('error_tracking_alert_created', expect.any(Object))
- })
-
- it('applies the preset filters, name suffix and event name', async () => {
- const { payload, captureSpy } = await createAlert(
- {
- presetTriggerKinds: ['sdk_outdated'],
- presetPropertyFilters: [sourceFilter],
- nameSuffix: 'for Stripe',
- createdEventName: 'custom_alert_created',
- },
- 'health-check-firing'
- )
-
- expect(
- payload.filters.properties.map((p: CyclotronJobFilterPropertyFilter) => 'key' in p && p.key)
- ).toEqual(['kind', 'source_id'])
- expect(payload.name).toBe(`${healthSlackName} (SDK outdated) for Stripe`)
- expect(captureSpy).toHaveBeenCalledWith('custom_alert_created', expect.any(Object))
- expect(captureSpy).not.toHaveBeenCalledWith('error_tracking_alert_created', expect.any(Object))
- })
-
- it('passes the sub-template masking through', async () => {
- const { payload } = await createAlert(
- {
- subTemplateIds: ['data-warehouse-sync-completed'],
- triggers: [...dwhTriggers],
- },
- 'data-warehouse-sync-completed'
- )
-
- expect(payload.masking).toMatchObject({ ttl: 3600 })
- expect(payload.masking.hash).toContain('event.properties.schema_id')
- })
- })
-
describe('test invocation results', () => {
const errorLog = { level: 'error', message: 'invalid_blocks' }
const infoLog = { level: 'info', message: 'done' }
diff --git a/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.ts b/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.ts
index 8d9545f64c77..1ead3bdff79a 100644
--- a/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.ts
+++ b/frontend/src/lib/components/Alerting/AlertWizard/alertWizardLogic.ts
@@ -14,7 +14,6 @@ import {
} from 'scenes/hog-functions/sub-templates/sub-templates'
import {
- CyclotronJobFilterPropertyFilter,
CyclotronJobFiltersType,
CyclotronJobInputType,
CyclotronJobInvocationGlobals,
@@ -70,14 +69,6 @@ export interface AlertWizardLogicProps {
// to one or more health-check kinds. Pass an empty array to mean "all kinds"
// explicitly; omit the prop to leave filters untouched.
presetTriggerKinds?: string[]
- // Property filters merged into `filters.properties` of the created HogFunction,
- // after the `kind` filter from `presetTriggerKinds`. Used to scope an alert to
- // one record, for example `source_id = <id>`.
- presetPropertyFilters?: CyclotronJobFilterPropertyFilter[]
- // Text appended to the generated alert name, for example the name of a source.
- nameSuffix?: string
- // Event name sent when an alert is created.
- createdEventName?: string
// When set, the "Test" button populates the test event via the matching
// SAMPLE_GLOBALS_CONTEXTS loader (real product data) instead of a stub event.
contextId?: HogFunctionConfigurationContextId
@@ -160,21 +151,6 @@ export function applyKindFilter(
}
}
-export function applyPresetFilters(
- baseFilters: CyclotronJobFiltersType | null | undefined,
- selectedKinds: string[] | null,
- presetPropertyFilters: CyclotronJobFilterPropertyFilter[] | null | undefined
-): CyclotronJobFiltersType | null | undefined {
- const withKinds = applyKindFilter(baseFilters, selectedKinds)
- if (!withKinds || !presetPropertyFilters || presetPropertyFilters.length === 0) {
- return withKinds
- }
- return {
- ...withKinds,
- properties: [...(withKinds.properties ?? []), ...presetPropertyFilters],
- }
-}
-
// Renders a selectedKinds list as a short, human-readable parenthetical suffix
// (e.g. "(SDK outdated)" or "(SDK outdated, External data failures)") that can
// be appended to a sub-template's generic name/description, so the created
@@ -187,12 +163,8 @@ function formatKindsSuffix(selectedKinds: string[] | null | undefined): string {
return ` (${labels.join(', ')})`
}
-export function decorateAlertName(
- baseName: string,
- selectedKinds: string[] | null | undefined,
- nameSuffix?: string
-): string {
- return `${baseName}${formatKindsSuffix(selectedKinds)}${nameSuffix ? ` ${nameSuffix}` : ''}`
+export function decorateAlertName(baseName: string, selectedKinds: string[] | null | undefined): string {
+ return `${baseName}${formatKindsSuffix(selectedKinds)}`
}
function lastErrorLogMessage(logs: readonly unknown[]): string | null {
@@ -243,7 +215,7 @@ export interface AlertHogFunctionConfiguration {
description: string
filters: CyclotronJobFiltersType | null | undefined
enabled: true
- masking: HogFunctionSubTemplateType['masking'] | null
+ masking: null
inputs: Record<string, CyclotronJobInputType>
}
@@ -254,14 +226,12 @@ export function buildAlertHogFunctionConfiguration({
description,
filters,
inputs,
- masking,
}: {
templateId: string
name: string
description: string
filters: CyclotronJobFiltersType | null | undefined
inputs: Record<string, CyclotronJobInputType>
- masking?: HogFunctionSubTemplateType['masking'] | null
}): AlertHogFunctionConfiguration {
return {
type: 'internal_destination',
@@ -270,7 +240,7 @@ export function buildAlertHogFunctionConfiguration({
description,
filters,
enabled: true,
- masking: masking ?? null,
+ masking: null,
inputs,
}
}
@@ -796,7 +766,7 @@ export const alertWizardLogic = kea<alertWizardLogicType>([
template_id: destination.templateId,
filters: subTemplate.filters,
enabled: true,
- masking: subTemplate.masking ?? null,
+ masking: null,
inputs: mergedInputs,
inputs_schema: selectedTemplate.inputs_schema,
hog: selectedTemplate.code,
@@ -831,7 +801,7 @@ export const alertWizardLogic = kea<alertWizardLogicType>([
try {
globals = await sampleGlobalsLoader(
globals,
- applyPresetFilters(subTemplate.filters, values.selectedKinds, logicProps.presetPropertyFilters)
+ applyKindFilter(subTemplate.filters, values.selectedKinds)
)
} catch {
// Fall back to the stub test event
@@ -889,12 +859,8 @@ export const alertWizardLogic = kea<alertWizardLogicType>([
values.inputValues
)
- const filters = applyPresetFilters(
- subTemplate.filters,
- values.selectedKinds,
- logicProps.presetPropertyFilters
- )
- const name = decorateAlertName(subTemplate.name ?? '', values.selectedKinds, logicProps.nameSuffix)
+ const filters = applyKindFilter(subTemplate.filters, values.selectedKinds)
+ const name = decorateAlertName(subTemplate.name ?? '', values.selectedKinds)
const description = decorateAlertName(subTemplate.description ?? '', values.selectedKinds)
const configuration = buildAlertHogFunctionConfiguration({
@@ -903,12 +869,10 @@ export const alertWizardLogic = kea<alertWizardLogicType>([
description,
filters,
inputs: mergedInputs,
- masking: subTemplate.masking,
})
await api.hogFunctions.create(configuration)
- // pinned: analytics event name, renaming breaks dashboards
- posthog.capture(logicProps.createdEventName ?? 'error_tracking_alert_created', {
+ posthog.capture('error_tracking_alert_created', {
ui_source: 'wizard',
trigger_event: subTemplate.filters?.events?.[0]?.id ?? null,
subtemplate_id: triggerKey,
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index 4898ad8a436d..d8efe0ac7a8e 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -303,7 +303,6 @@ export const FEATURE_FLAGS = {
DROP_PERSON_LIST_ORDER_BY: 'drop-person-list-order-by', // owner: @arthurdedeus #team-customer-analytics
DWH_POSTGRES_CDC: 'dwh-postgres-cdc', // owner: #team-warehouse-sources
DWH_SOURCE_METRICS: 'dwh-source-metrics', // owner: #team-warehouse-sources
- DWH_SYNC_ALERTS: 'dwh-sync-alerts', // owner: #team-warehouse-sources, gates the data warehouse sync alert sub-templates
EDITOR_DRAFTS: 'editor-drafts', // owner: @EDsCODE #team-data-tools
EMAIL_TEMPLATES_AI_FIRST_NEW: 'email-templates-ai-first-new', // owner: @mayteio #team-workflows
EMOJI_RELATED_SEARCH: 'emoji-related-search', // owner: @pauldambra, gates the model search in `emojiSuggestionsLogic.ts` when the emoji picker finds no match
diff --git a/frontend/src/scenes/hog-functions/configuration/sampleGlobalsContexts.ts b/frontend/src/scenes/hog-functions/configuration/sampleGlobalsContexts.ts
index 380d638748e9..74b118adb9bf 100644
--- a/frontend/src/scenes/hog-functions/configuration/sampleGlobalsContexts.ts
+++ b/frontend/src/scenes/hog-functions/configuration/sampleGlobalsContexts.ts
@@ -88,29 +88,4 @@ export const SAMPLE_GLOBALS_CONTEXTS: Partial<Record<HogFunctionConfigurationCon
},
},
}),
- // The templates read these fields. An empty value renders an empty Slack block, and Slack rejects
- // the whole message.
- 'data-warehouse-alerts': async (exampleGlobals) => ({
- ...exampleGlobals,
- event: {
- ...exampleGlobals.event,
- properties: {
- source_id: '00000000-0000-4000-8000-000000000001',
- source_type: 'Stripe',
- source_prefix: 'stripe_',
- schema_id: '00000000-0000-4000-8000-000000000002',
- schema_name: 'charges',
- job_id: '00000000-0000-4000-8000-000000000003',
- status: 'Failed',
- kind: 'job_failed',
- error: 'This is a test alert from PostHog',
- rows_synced: 1200,
- paused: false,
- failed_runs_in_a_row: 1,
- source_url: 'https://example.com/data-warehouse/sources/00000000-0000-4000-8000-000000000001',
- schema_url: 'https://example.com/data-warehouse/sources/00000000-0000-4000-8000-000000000001/charges',
- finished_at: '2026-01-01T00:00:00.000Z',
- },
- },
- }),
}
diff --git a/frontend/src/scenes/hog-functions/filters/HogFunctionFiltersInternal.tsx b/frontend/src/scenes/hog-functions/filters/HogFunctionFiltersInternal.tsx
index 925f741d60ff..1d9f63aae55e 100644
--- a/frontend/src/scenes/hog-functions/filters/HogFunctionFiltersInternal.tsx
+++ b/frontend/src/scenes/hog-functions/filters/HogFunctionFiltersInternal.tsx
@@ -40,12 +40,10 @@ const setSimpleFilterValue = (
],
}
// Preserve properties bound by Logs alerting (alert_id) and batch export alerts
- // (batch_export_id), and data warehouse alerts (source_id) — the trigger event id changes between the context's events, but the
+ // (batch_export_id) — the trigger event id changes between the context's events, but the
// binding to the parent resource must survive.
if (
- (contextId === 'logs-alerting' ||
- contextId === 'batch-export-alerts' ||
- contextId === 'data-warehouse-alerts') &&
+ (contextId === 'logs-alerting' || contextId === 'batch-export-alerts') &&
previous?.properties &&
previous.properties.length > 0
) {
@@ -81,7 +79,7 @@ export function HogFunctionFiltersInternal(): JSX.Element {
return [TaxonomicFilterGroupType.EventProperties]
} else if (contextId === 'health-alerts') {
return [TaxonomicFilterGroupType.EventProperties]
- } else if (contextId === 'batch-export-alerts' || contextId === 'data-warehouse-alerts') {
+ } else if (contextId === 'batch-export-alerts') {
return [TaxonomicFilterGroupType.EventProperties]
}
return []
diff --git a/frontend/src/scenes/hog-functions/filters/productEventFilterOptions.ts b/frontend/src/scenes/hog-functions/filters/productEventFilterOptions.ts
index d7d36cc6495a..6de69c5efcd4 100644
--- a/frontend/src/scenes/hog-functions/filters/productEventFilterOptions.ts
+++ b/frontend/src/scenes/hog-functions/filters/productEventFilterOptions.ts
@@ -103,13 +103,6 @@ export const getProductEventFilterOptions = (contextId: HogFunctionConfiguration
value: '$batch_export_run_failed',
},
]
- case 'data-warehouse-alerts':
- return [
- { label: 'Sync failed', value: '$data_warehouse_sync_failed' },
- { label: 'Sync recovered', value: '$data_warehouse_sync_recovered' },
- { label: 'Sync completed', value: '$data_warehouse_sync_completed' },
- { label: 'Billing limit reached', value: '$data_warehouse_billing_limit_reached' },
- ]
case 'replay-vision-requests':
return [
{
diff --git a/frontend/src/scenes/hog-functions/list/HogFunctionsList.tsx b/frontend/src/scenes/hog-functions/list/HogFunctionsList.tsx
index bf268e5f163d..40494410ab1d 100644
--- a/frontend/src/scenes/hog-functions/list/HogFunctionsList.tsx
+++ b/frontend/src/scenes/hog-functions/list/HogFunctionsList.tsx
@@ -60,7 +60,6 @@ const INTERNAL_DESTINATION_CONTEXT: Partial<
},
'batch-export-alerts': { label: 'Batch export alerts' },
'billing-alerts': { label: 'Billing alerts' },
- 'data-warehouse-alerts': { label: 'Data warehouse alerts', url: urls.sources() },
'replay-vision-alerts': { label: 'Replay vision alerts' },
'replay-vision-requests': { label: 'Replay vision requests' },
}
diff --git a/frontend/src/scenes/hog-functions/sub-templates/sub-templates.test.ts b/frontend/src/scenes/hog-functions/sub-templates/sub-templates.test.ts
index d65e57887b98..f38abd0bac27 100644
--- a/frontend/src/scenes/hog-functions/sub-templates/sub-templates.test.ts
+++ b/frontend/src/scenes/hog-functions/sub-templates/sub-templates.test.ts
@@ -17,10 +17,6 @@ describe('sub-templates', () => {
['$logs_alert_firing', 'logs-alerting'],
['$health_check_issue_firing', 'health-alerts'],
['$batch_export_run_failed', 'batch-export-alerts'],
- ['$data_warehouse_sync_failed', 'data-warehouse-alerts'],
- ['$data_warehouse_sync_recovered', 'data-warehouse-alerts'],
- ['$data_warehouse_sync_completed', 'data-warehouse-alerts'],
- ['$data_warehouse_billing_limit_reached', 'data-warehouse-alerts'],
['$billing_alert_firing', 'billing-alerts'],
['$replay_vision_alert_match', 'replay-vision-alerts'],
['$replay_vision_request_completed', 'replay-vision-requests'],
@@ -56,17 +52,10 @@ describe('sub-templates', () => {
// A placeholder the sample event leaves empty renders an empty Slack block. Slack then rejects
// the whole test message.
- it.each([
- ['health-alerts', 'health-check-firing'],
- ['health-alerts', 'health-check-resolved'],
- ['data-warehouse-alerts', 'data-warehouse-sync-failed'],
- ['data-warehouse-alerts', 'data-warehouse-sync-recovered'],
- ['data-warehouse-alerts', 'data-warehouse-sync-completed'],
- ['data-warehouse-alerts', 'data-warehouse-billing-limit-reached'],
- ] as const)(
- 'gives the %s sample event a value for every property the %s templates read',
- async (contextId, subTemplateId) => {
- const sample = await SAMPLE_GLOBALS_CONTEXTS[contextId]!({
+ it.each(['health-check-firing', 'health-check-resolved'] as const)(
+ 'gives the %s templates a sample value for every event property they read',
+ async (subTemplateId) => {
+ const sample = await SAMPLE_GLOBALS_CONTEXTS['health-alerts']!({
event: { properties: {} },
} as CyclotronJobInvocationGlobals)
const inputs = JSON.stringify(HOG_FUNCTION_SUB_TEMPLATES[subTemplateId].map((template) => template.inputs))
diff --git a/frontend/src/scenes/hog-functions/sub-templates/sub-templates.ts b/frontend/src/scenes/hog-functions/sub-templates/sub-templates.ts
index 17dc61b43a2e..b3b96da0c52e 100644
--- a/frontend/src/scenes/hog-functions/sub-templates/sub-templates.ts
+++ b/frontend/src/scenes/hog-functions/sub-templates/sub-templates.ts
@@ -159,12 +159,6 @@ const BATCH_EXPORT_ALERT_MASKING_TTL_SECONDS = 60 * 60
const BATCH_EXPORT_ALERT_MASKING_HASH =
"{event.properties.batch_export_id ? event.properties.batch_export_id : 'unknown-batch-export'}"
-// A sync that runs every few minutes would post a message per run, so limit each table to one
-// completed-sync message per hour. HogMaskerService skips masking on falsy hashes, so fall back
-// defensively.
-const DWH_SYNC_COMPLETED_MASKING_TTL_SECONDS = 60 * 60
-const DWH_SYNC_COMPLETED_MASKING_HASH = "{event.properties.schema_id ? event.properties.schema_id : 'unknown-table'}"
-
// The page a rageclick happened on: $pathname when posthog-js set it, else the full URL.
const PA_RAGECLICK_PAGE_EXPR = 'event.properties.$pathname ? event.properties.$pathname : event.properties.$current_url'
@@ -339,42 +333,6 @@ export const HOG_FUNCTION_SUB_TEMPLATE_COMMON_PROPERTIES: Record<
},
flag: FEATURE_FLAGS.BATCH_EXPORT_ALERTS,
},
- 'data-warehouse-sync-failed': {
- sub_template_id: 'data-warehouse-sync-failed',
- type: 'internal_destination',
- context_id: 'data-warehouse-alerts',
- filters: { source: 'internal-events', events: [{ id: '$data_warehouse_sync_failed', type: 'events' }] },
- flag: FEATURE_FLAGS.DWH_SYNC_ALERTS,
- },
- 'data-warehouse-sync-recovered': {
- sub_template_id: 'data-warehouse-sync-recovered',
- type: 'internal_destination',
- context_id: 'data-warehouse-alerts',
- filters: { source: 'internal-events', events: [{ id: '$data_warehouse_sync_recovered', type: 'events' }] },
- flag: FEATURE_FLAGS.DWH_SYNC_ALERTS,
- },
- 'data-warehouse-sync-completed': {
- sub_template_id: 'data-warehouse-sync-completed',
- type: 'internal_destination',
- context_id: 'data-warehouse-alerts',
- filters: { source: 'internal-events', events: [{ id: '$data_warehouse_sync_completed', type: 'events' }] },
- masking: {
- hash: DWH_SYNC_COMPLETED_MASKING_HASH,
- ttl: DWH_SYNC_COMPLETED_MASKING_TTL_SECONDS,
- threshold: null,
- },
- flag: FEATURE_FLAGS.DWH_SYNC_ALERTS,
- },
- 'data-warehouse-billing-limit-reached': {
- sub_template_id: 'data-warehouse-billing-limit-reached',
- type: 'internal_destination',
- context_id: 'data-warehouse-alerts',
- filters: {
- source: 'internal-events',
- events: [{ id: '$data_warehouse_billing_limit_reached', type: 'events' }],
- },
- flag: FEATURE_FLAGS.DWH_SYNC_ALERTS,
- },
'replay-vision-request-completed': {
sub_template_id: 'replay-vision-request-completed',
type: 'internal_destination',
@@ -756,53 +714,6 @@ function notificationVariants({
const BATCH_EXPORT_NAME_SLACK = `{${slackEscapeExpr('event.properties.batch_export_name')}}`
const BATCH_EXPORT_ERROR_SLACK = `{${slackEscapeExpr('event.properties.error', 1000)}}`
-// source_type, schema_name and error come from the source system, so they get the same escaping and
-// bounds as the other producer-controlled notification fields.
-interface DwhMessageFields {
- sourceType: string
- table: string
- error: string
-}
-
-const DWH_SLACK_FIELDS: DwhMessageFields = {
- sourceType: `{${slackEscapeExpr('event.properties.source_type', 100)}}`,
- table: `{${slackEscapeExpr('event.properties.schema_name', 200)}}`,
- error: `{${slackEscapeExpr('event.properties.error', 1000)}}`,
-}
-const DWH_MARKDOWN_FIELDS: DwhMessageFields = {
- sourceType: `{${markdownEscapeExpr('event.properties.source_type', 100)}}`,
- table: `{${markdownEscapeExpr('event.properties.schema_name', 200)}}`,
- error: `{${markdownEscapeExpr('event.properties.error', 1000)}}`,
-}
-
-const DWH_TABLE_URL = '{event.properties.schema_url}'
-const DWH_SOURCE_URL = '{event.properties.source_url}'
-const DWH_MARKDOWN_TABLE_LINK = `[View table](${DWH_TABLE_URL})`
-const DWH_MARKDOWN_SOURCE_LINK = `[View source](${DWH_SOURCE_URL})`
-
-function dwhSyncFailedMessage(fields: DwhMessageFields, bold: string): string {
- return (
- `${bold}Sync failed${bold} for table ${bold}${fields.table}${bold} from ${fields.sourceType}.` +
- `{event.properties.paused ? ' The table is paused.' : ''}\n` +
- `${bold}Error:${bold} ${fields.error}`
- )
-}
-
-function dwhSyncRecoveredMessage(fields: DwhMessageFields, bold: string): string {
- return `Table ${bold}${fields.table}${bold} from ${fields.sourceType} is syncing again.`
-}
-
-function dwhSyncCompletedMessage(fields: DwhMessageFields, bold: string): string {
- return `Table ${bold}${fields.table}${bold} from ${fields.sourceType} finished a sync. Rows synced: {event.properties.rows_synced}.`
-}
-
-function dwhBillingLimitMessage(fields: DwhMessageFields, bold: string): string {
- return (
- `{event.properties.kind == 'too_low' ? 'The billing limit is too low to sync' : 'The billing limit was reached, so this sync did not run for'} ` +
- `table ${bold}${fields.table}${bold} from ${fields.sourceType}.`
- )
-}
-
export const HOG_FUNCTION_SUB_TEMPLATES: Record<HogFunctionSubTemplateIdType, HogFunctionSubTemplateType[]> = {
'mcp-tool-error': notificationVariants({
subTemplateId: 'mcp-tool-error',
@@ -814,46 +725,6 @@ export const HOG_FUNCTION_SUB_TEMPLATES: Record<HogFunctionSubTemplateIdType, Ho
markdownMessage: `${MCP_TOOL_ERROR_MARKDOWN_MESSAGE}\n\n${MCP_TOOL_ERROR_LINK}`,
slackButton: { url: MCP_TOOL_ERROR_LINK, label: MCP_NOTIFICATION_BUTTON_LABELS['mcp-tool-error'] },
}),
- 'data-warehouse-sync-failed': notificationVariants({
- subTemplateId: 'data-warehouse-sync-failed',
- nameSuffix: 'when a table sync fails',
- description: 'Know when a table stops syncing and why',
- webhookDescription: 'Send failed table syncs to your own endpoint',
- slackMessage: dwhSyncFailedMessage(DWH_SLACK_FIELDS, '*'),
- slackFallbackText: 'A table sync failed',
- markdownMessage: `${dwhSyncFailedMessage(DWH_MARKDOWN_FIELDS, '**')}\n\n${DWH_MARKDOWN_TABLE_LINK}`,
- slackButton: { url: DWH_TABLE_URL, label: 'View table' },
- }),
- 'data-warehouse-sync-recovered': notificationVariants({
- subTemplateId: 'data-warehouse-sync-recovered',
- nameSuffix: 'when a table sync recovers',
- description: 'Know when a table that failed is syncing again',
- webhookDescription: 'Send recovered table syncs to your own endpoint',
- slackMessage: dwhSyncRecoveredMessage(DWH_SLACK_FIELDS, '*'),
- slackFallbackText: 'A table sync recovered',
- markdownMessage: `${dwhSyncRecoveredMessage(DWH_MARKDOWN_FIELDS, '**')}\n\n${DWH_MARKDOWN_TABLE_LINK}`,
- slackButton: { url: DWH_TABLE_URL, label: 'View table' },
- }),
- 'data-warehouse-sync-completed': notificationVariants({
- subTemplateId: 'data-warehouse-sync-completed',
- nameSuffix: 'when a table sync completes',
- description: 'Know when a table finishes syncing, at most once an hour for each table',
- webhookDescription: 'Send completed table syncs to your own endpoint',
- slackMessage: dwhSyncCompletedMessage(DWH_SLACK_FIELDS, '*'),
- slackFallbackText: 'A table sync completed',
- markdownMessage: `${dwhSyncCompletedMessage(DWH_MARKDOWN_FIELDS, '**')}\n\n${DWH_MARKDOWN_TABLE_LINK}`,
- slackButton: { url: DWH_TABLE_URL, label: 'View table' },
- }),
- 'data-warehouse-billing-limit-reached': notificationVariants({
- subTemplateId: 'data-warehouse-billing-limit-reached',
- nameSuffix: 'when the billing limit stops a sync',
- description: 'Know when the Sources billing limit stops a table from syncing',
- webhookDescription: 'Send billing limit events to your own endpoint',
- slackMessage: dwhBillingLimitMessage(DWH_SLACK_FIELDS, '*'),
- slackFallbackText: 'The billing limit stopped a table sync',
- markdownMessage: `${dwhBillingLimitMessage(DWH_MARKDOWN_FIELDS, '**')}\n\n${DWH_MARKDOWN_SOURCE_LINK}`,
- slackButton: { url: DWH_SOURCE_URL, label: 'View source' },
- }),
'pa-rageclick': notificationVariants({
subTemplateId: 'pa-rageclick',
nameSuffix: 'when users rage click',
@@ -2023,11 +1894,6 @@ export const eventToHogFunctionContextId = (event: string | undefined): HogFunct
return 'health-alerts'
case '$batch_export_run_failed':
return 'batch-export-alerts'
- case '$data_warehouse_sync_failed':
- case '$data_warehouse_sync_recovered':
- case '$data_warehouse_sync_completed':
- case '$data_warehouse_billing_limit_reached':
- return 'data-warehouse-alerts'
case '$billing_alert_firing':
case '$billing_alert_resolved':
case '$billing_alert_errored':
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index aab5a840f547..1c56c81de115 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -7493,7 +7493,6 @@ export type HogFunctionConfigurationContextId =
| 'batch-export-alerts'
| 'billing-alerts'
| 'replay-vision-alerts'
- | 'data-warehouse-alerts'
| 'replay-vision-requests'
export type HogFunctionSubTemplateIdType =
@@ -7516,10 +7515,6 @@ export type HogFunctionSubTemplateIdType =
| 'health-check-firing'
| 'health-check-resolved'
| 'batch-export-run-failed'
- | 'data-warehouse-sync-failed'
- | 'data-warehouse-sync-recovered'
- | 'data-warehouse-sync-completed'
- | 'data-warehouse-billing-limit-reached'
| 'replay-vision-request-completed'
export type HogFunctionConfigurationType = Omit<
diff --git a/products/data_warehouse/backend/facade/api.py b/products/data_warehouse/backend/facade/api.py
index 8ca5dc745204..15fd89f7a433 100644
--- a/products/data_warehouse/backend/facade/api.py
+++ b/products/data_warehouse/backend/facade/api.py
@@ -47,9 +47,6 @@ _LAZY = {
"unpause_external_data_schedule": "logic.data_load.service",
"create_warehouse_templates_for_source": "logic.data_load.source_templates",
"update_external_job_status": "logic.external_data_source.jobs",
- "SyncAlertEvent": "logic.external_data_source.alerts",
- "SyncAlertKind": "logic.external_data_source.alerts",
- "emit_sync_alert": "logic.external_data_source.alerts",
"create_and_register_webhook": "logic.external_data_source.webhooks",
"delete_webhook_and_hog_function": "logic.external_data_source.webhooks",
"get_or_create_webhook_hog_function": "logic.external_data_source.webhooks",
diff --git a/products/data_warehouse/backend/logic/external_data_source/alerts.py b/products/data_warehouse/backend/logic/external_data_source/alerts.py
deleted file mode 100644
index dc440b6ff0b2..000000000000
--- a/products/data_warehouse/backend/logic/external_data_source/alerts.py
+++ /dev/null
@@ -1,190 +0,0 @@
-from enum import StrEnum
-from typing import Any
-from urllib.parse import quote
-
-from django.conf import settings
-from django.core.cache import cache
-
-import structlog
-from prometheus_client import Counter
-
-from posthog.exceptions_capture import capture_exception
-
-from products.cdp.backend.facade.models import HogFunction
-from products.cdp.backend.internal_events import InternalEventEvent, produce_internal_event
-from products.warehouse_sources.backend.facade.api import get_sync_alert_context
-from products.warehouse_sources.backend.facade.types import ExternalDataJobStatus, ExternalDataSchemaStatus
-
-logger = structlog.get_logger(__name__)
-
-SYNC_ALERT_EVENTS = Counter(
- "dwh_sync_alert_events_total",
- "Sync alert internal events, by event name and outcome",
- labelnames=["event", "outcome"],
-)
-
-MAX_ERROR_LENGTH = 1000
-
-# A destination created less than this long ago can miss a completed sync. The check runs on every
-# completed run of every schema, so it must not query Postgres each time.
-SUBSCRIBER_CACHE_SECONDS = 60
-
-
-class SyncAlertEvent(StrEnum):
- """Internal events a destination can subscribe to. Saved destinations filter on these names."""
-
- FAILED = "$data_warehouse_sync_failed"
- RECOVERED = "$data_warehouse_sync_recovered"
- COMPLETED = "$data_warehouse_sync_completed"
- BILLING_LIMIT_REACHED = "$data_warehouse_billing_limit_reached"
-
-
-class SyncAlertKind(StrEnum):
- JOB_FAILED = "job_failed"
- SCHEMA_PAUSED = "schema_paused"
- CDC_BROKEN = "cdc_broken"
- BILLING_LIMIT_REACHED = "reached"
- BILLING_LIMIT_TOO_LOW = "too_low"
-
-
-BILLING_KIND_BY_STATUS: dict[str, SyncAlertKind] = {
- ExternalDataSchemaStatus.BILLING_LIMIT_REACHED: SyncAlertKind.BILLING_LIMIT_REACHED,
- ExternalDataSchemaStatus.BILLING_LIMIT_TOO_LOW: SyncAlertKind.BILLING_LIMIT_TOO_LOW,
-}
-
-# Events that fire on every run. The others fire on a state change, so their volume is low enough
-# to produce without a subscriber check.
-EVENTS_PRODUCED_ONLY_FOR_SUBSCRIBERS = frozenset({SyncAlertEvent.COMPLETED})
-
-
-def events_for_terminal_run(
- *,
- status: ExternalDataJobStatus,
- previous_schema_status: str | None,
- previous_failed_runs: int,
- counts_as_source_failure: bool,
-) -> list[SyncAlertEvent]:
- """The alert events for the first terminal write of a run.
-
- Failed, recovered and billing events fire on a state change of the schema, so a schema that
- fails each retry sends one alert, not one for each run. `previous_failed_runs` is the failure
- streak before this run moved it. A failure that says nothing about the source does not start
- a streak, so it sends no failed alert and its next completed run sends no recovered alert.
- """
- if status == ExternalDataJobStatus.FAILED:
- return [SyncAlertEvent.FAILED] if counts_as_source_failure and previous_failed_runs == 0 else []
-
- if status == ExternalDataJobStatus.COMPLETED:
- recovered = previous_failed_runs > 0 or previous_schema_status in BILLING_KIND_BY_STATUS
- return [SyncAlertEvent.RECOVERED, SyncAlertEvent.COMPLETED] if recovered else [SyncAlertEvent.COMPLETED]
-
- if status in BILLING_KIND_BY_STATUS and previous_schema_status != status:
- return [SyncAlertEvent.BILLING_LIMIT_REACHED]
-
- return []
-
-
-def emit_terminal_run_alerts(
- *,
- team_id: int,
- schema_id: str,
- job_id: str,
- status: ExternalDataJobStatus,
- previous_schema_status: str | None,
- previous_failed_runs: int,
- counts_as_source_failure: bool,
-) -> None:
- for event in events_for_terminal_run(
- status=status,
- previous_schema_status=previous_schema_status,
- previous_failed_runs=previous_failed_runs,
- counts_as_source_failure=counts_as_source_failure,
- ):
- kind = SyncAlertKind.JOB_FAILED if event == SyncAlertEvent.FAILED else BILLING_KIND_BY_STATUS.get(status)
- emit_sync_alert(team_id=team_id, schema_id=schema_id, event=event, kind=kind, job_id=job_id)
-
-
-def emit_sync_alert(
- *,
- team_id: int,
- schema_id: str,
- event: SyncAlertEvent,
- kind: SyncAlertKind | None = None,
- job_id: str | None = None,
-) -> None:
- """Produce one sync alert event. Never raises, because an alert must not fail the sync that sent it."""
- try:
- if event in EVENTS_PRODUCED_ONLY_FOR_SUBSCRIBERS and not _has_subscriber(team_id, event):
- SYNC_ALERT_EVENTS.labels(event=event.value, outcome="no_subscriber").inc()
- return
-
- properties = _build_properties(team_id=team_id, schema_id=schema_id, event=event, kind=kind, job_id=job_id)
- if properties is None:
- SYNC_ALERT_EVENTS.labels(event=event.value, outcome="schema_missing").inc()
- return
-
- produce_internal_event(
- team_id=team_id,
- event=InternalEventEvent(event=event.value, distinct_id=f"team_{team_id}", properties=properties),
- )
- SYNC_ALERT_EVENTS.labels(event=event.value, outcome="produced").inc()
- except Exception as error:
- SYNC_ALERT_EVENTS.labels(event=event.value, outcome="error").inc()
- logger.exception("dwh_sync_alert_emit_failed", team_id=team_id, schema_id=schema_id, event_name=event.value)
- capture_exception(error)
-
-
-def _has_subscriber(team_id: int, event: SyncAlertEvent) -> bool:
- cache_key = f"dwh_sync_alert_subscriber:{team_id}:{event.value}"
- cached = cache.get(cache_key)
- if cached is not None:
- return bool(cached)
-
- has_subscriber = HogFunction.objects.filter(
- team_id=team_id,
- type="internal_destination",
- enabled=True,
- deleted=False,
- filters__events__contains=[{"id": event.value}],
- ).exists()
- cache.set(cache_key, has_subscriber, SUBSCRIBER_CACHE_SECONDS)
- return has_subscriber
-
-
-def _build_properties(
- *,
- team_id: int,
- schema_id: str,
- event: SyncAlertEvent,
- kind: SyncAlertKind | None,
- job_id: str | None,
-) -> dict[str, Any] | None:
- include_error = event == SyncAlertEvent.FAILED
- context = get_sync_alert_context(team_id, schema_id, job_id, include_error=include_error)
- if context is None:
- return None
-
- source_url = f"{settings.SITE_URL}/project/{team_id}/data-management/sources/managed-{context.source_id}/syncs"
-
- # Only a failed event carries the error. A billing-limited run keeps the error of the last real
- # failure on the schema, which is not the reason for a billing event. `latest_error` is the copy
- # the Syncs tab shows. The internal error can hold driver text and connection details.
- error = (context.latest_error or "Unknown error")[:MAX_ERROR_LENGTH] if include_error else None
-
- return {
- "source_id": str(context.source_id),
- "source_type": context.source_type,
- "source_prefix": (context.source_prefix or "").rstrip("_"),
- "schema_id": str(context.schema_id),
- "schema_name": context.schema_label or context.schema_name,
- "job_id": str(context.job_id) if context.job_id else None,
- "status": context.status,
- "kind": kind.value if kind else None,
- "error": error,
- "rows_synced": context.rows_synced,
- "paused": context.sync_halted,
- "failed_runs_in_a_row": context.failed_runs_in_a_row,
- "source_url": source_url,
- "schema_url": f"{source_url}?schema={quote(context.schema_name)}",
- "finished_at": context.finished_at.isoformat() if context.finished_at else None,
- }
diff --git a/products/data_warehouse/backend/logic/external_data_source/jobs.py b/products/data_warehouse/backend/logic/external_data_source/jobs.py
index e9153eccb0eb..1532099b68f7 100644
--- a/products/data_warehouse/backend/logic/external_data_source/jobs.py
+++ b/products/data_warehouse/backend/logic/external_data_source/jobs.py
@@ -9,7 +9,6 @@ from structlog.types import FilteringBoundLogger
from posthog.exceptions_capture import capture_exception
-from products.data_warehouse.backend.logic.external_data_source.alerts import emit_terminal_run_alerts
from products.data_warehouse.backend.tasks.tasks import schedule_external_data_failure_digest
from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema
from products.warehouse_sources.backend.facade.pipelines import (
@@ -73,9 +72,6 @@ def update_external_job_status(
gap of its next runs (see `retry_limits`).
"""
is_first_terminal_transition = False
- schema_status_written = False
- previous_schema_status: str | None = None
- previous_failed_runs = 0
with transaction.atomic():
model = ExternalDataJob.objects.select_for_update().get(id=job_id, team_id=team_id)
@@ -153,10 +149,6 @@ def update_external_job_status(
# moves: billing is what blocks this schema now.
billing_limited_run = schema_status == ExternalDataSchemaStatus.BILLING_LIMIT_REACHED
- # Read the streak before this run moves it. The sync alerts decide "first failure" and
- # "recovered" from the streak of the earlier runs, not from a streak that includes this run.
- previous_failed_runs = schema.failed_runs_in_a_row
-
# The streak moves for a halted schema too. Its schedule keeps running, and the streak is
# the only thing that spaces out those runs, so skipping it lets a source that never
# answers fail on every slot.
@@ -174,8 +166,6 @@ def update_external_job_status(
if streak_moved:
schema.save(update_fields=["sync_type_config", "updated_at"])
else:
- schema_status_written = True
- previous_schema_status = schema.status
schema.status = schema_status
if not billing_limited_run:
schema.latest_error = error_to_persist
@@ -212,19 +202,6 @@ def update_external_job_status(
except Exception:
logger.exception("Failed to schedule external data failure digest")
- # A CDC-halted schema kept its status, so this run changed nothing a destination should hear
- # about. The CDC code sends its own alert when it halts a schema.
- if schema_status_written:
- emit_terminal_run_alerts(
- team_id=team_id,
- schema_id=str(model.schema_id),
- job_id=job_id,
- status=status,
- previous_schema_status=previous_schema_status,
- previous_failed_runs=previous_failed_runs,
- counts_as_source_failure=counts_as_source_failure,
- )
-
return model
diff --git a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
index 5c28c1d02c37..36880fb8f865 100644
--- a/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
+++ b/products/data_warehouse/backend/logic/external_data_source/test/test_jobs.py
@@ -4,13 +4,10 @@ from datetime import UTC, datetime, timedelta
import pytest
from unittest.mock import MagicMock, patch
-from django.core.cache import cache
-
from temporalio.exceptions import CancelledError
from posthog.models import Organization, Team
-from products.cdp.backend.facade.models import HogFunction
from products.data_warehouse.backend.logic.external_data_source.jobs import update_external_job_status
from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema, ExternalDataSource
from products.warehouse_sources.backend.facade.pipelines import LOCK_TAKEOVER_LATEST_ERROR
@@ -525,254 +522,6 @@ class TestUpdateExternalJobStatus:
assert schema.sync_type_config == {"failure_streak": {"runs": 7, "last_failed_at": last_failed_at}}
-FAILED_EVENT = "$data_warehouse_sync_failed"
-RECOVERED_EVENT = "$data_warehouse_sync_recovered"
-COMPLETED_EVENT = "$data_warehouse_sync_completed"
-BILLING_EVENT = "$data_warehouse_billing_limit_reached"
-STREAK_OF_TWO = {"failure_streak": {"runs": 2, "last_failed_at": "2026-01-01T00:00:00+00:00"}}
-
-
-class TestSyncAlertEvents:
- @pytest.fixture(autouse=True)
- def _clear_subscriber_cache(self):
- cache.clear()
-
- @staticmethod
- def _finalize(team, job, status, **kwargs) -> list[dict]:
- with (
- patch("products.data_warehouse.backend.logic.external_data_source.jobs.emit_data_import_app_metrics"),
- patch(
- "products.data_warehouse.backend.logic.external_data_source.jobs.schedule_external_data_failure_digest"
- ),
- patch(
- "products.data_warehouse.backend.logic.external_data_source.alerts.produce_internal_event"
- ) as mock_produce,
- ):
- update_external_job_status(
- job_id=str(job.id),
- team_id=team.pk,
- status=status,
- logger=MagicMock(),
- latest_error=kwargs.pop("latest_error", "boom") if status == ExternalDataJobStatus.FAILED else None,
- **kwargs,
- )
- assert all(call.kwargs["team_id"] == team.pk for call in mock_produce.call_args_list)
- return [
- {"event": call.kwargs["event"].event, **call.kwargs["event"].properties}
- for call in mock_produce.call_args_list
- ]
-
- @staticmethod
- def _subscribe(team, event_name: str) -> None:
- HogFunction.objects.create(
- team=team,
- type="internal_destination",
- enabled=True,
- hog="return event",
- filters={"source": "internal-events", "events": [{"id": event_name, "type": "events"}]},
- )
-
- @pytest.mark.parametrize(
- "schema_status,sync_type_config,status,counts_as_source_failure,subscribed_to_completed,expected",
- [
- pytest.param(None, {}, ExternalDataJobStatus.FAILED, True, False, [FAILED_EVENT], id="first-failure"),
- pytest.param(
- ExternalDataSchemaStatus.FAILED,
- STREAK_OF_TWO,
- ExternalDataJobStatus.FAILED,
- True,
- False,
- [],
- id="repeat-failure-is-silent",
- ),
- pytest.param(None, {}, ExternalDataJobStatus.FAILED, False, False, [], id="non-source-failure-is-silent"),
- pytest.param(
- None,
- {"cdc_broken": {"reason": "slot_missing"}},
- ExternalDataJobStatus.FAILED,
- True,
- False,
- [],
- id="cdc-halted-schema-is-silent",
- ),
- pytest.param(
- ExternalDataSchemaStatus.FAILED,
- STREAK_OF_TWO,
- ExternalDataJobStatus.COMPLETED,
- True,
- False,
- [RECOVERED_EVENT],
- id="recovered-after-failures",
- ),
- pytest.param(
- ExternalDataSchemaStatus.FAILED,
- STREAK_OF_TWO,
- ExternalDataJobStatus.COMPLETED,
- True,
- True,
- [RECOVERED_EVENT, COMPLETED_EVENT],
- id="recovered-and-completed-with-subscriber",
- ),
- pytest.param(
- ExternalDataSchemaStatus.FAILED,
- {},
- ExternalDataJobStatus.COMPLETED,
- True,
- False,
- [],
- id="completed-after-non-source-failure-is-not-a-recovery",
- ),
- pytest.param(
- ExternalDataSchemaStatus.BILLING_LIMIT_REACHED,
- {},
- ExternalDataJobStatus.COMPLETED,
- True,
- False,
- [RECOVERED_EVENT],
- id="recovered-after-billing-limit",
- ),
- pytest.param(
- ExternalDataSchemaStatus.COMPLETED,
- {},
- ExternalDataJobStatus.COMPLETED,
- True,
- False,
- [],
- id="completed-without-subscriber-is-not-produced",
- ),
- pytest.param(
- ExternalDataSchemaStatus.COMPLETED,
- {},
- ExternalDataJobStatus.COMPLETED,
- True,
- True,
- [COMPLETED_EVENT],
- id="completed-with-subscriber",
- ),
- pytest.param(
- ExternalDataSchemaStatus.COMPLETED,
- {},
- ExternalDataJobStatus.BILLING_LIMIT_REACHED,
- True,
- False,
- [BILLING_EVENT],
- id="billing-limit-reached",
- ),
- pytest.param(
- ExternalDataSchemaStatus.BILLING_LIMIT_REACHED,
- {},
- ExternalDataJobStatus.BILLING_LIMIT_REACHED,
- True,
- False,
- [],
- id="repeat-billing-limit-is-silent",
- ),
- pytest.param(
- ExternalDataSchemaStatus.BILLING_LIMIT_REACHED,
- {},
- ExternalDataJobStatus.BILLING_LIMIT_TOO_LOW,
- True,
- False,
- [BILLING_EVENT],
- id="billing-limit-kind-changed",
- ),
- ],
- )
- def test_terminal_run_produces_alert_events(
- self, schema_status, sync_type_config, status, counts_as_source_failure, subscribed_to_completed, expected
- ):
- team, _source, schema, job = _create_org_team_source_schema_job()
- schema.status = schema_status
- schema.sync_type_config = sync_type_config
- schema.save()
- if subscribed_to_completed:
- self._subscribe(team, COMPLETED_EVENT)
-
- produced = self._finalize(team, job, status, counts_as_source_failure=counts_as_source_failure)
-
- assert [event["event"] for event in produced] == expected
-
- def test_a_repeated_terminal_write_for_the_same_job_produces_no_second_event(self):
- team, _source, _schema, job = _create_org_team_source_schema_job()
-
- first = self._finalize(team, job, ExternalDataJobStatus.FAILED)
- second = self._finalize(team, job, ExternalDataJobStatus.FAILED)
-
- assert [event["event"] for event in first] == [FAILED_EVENT]
- assert second == []
-
- def test_failed_event_properties(self):
- team, source, schema, job = _create_org_team_source_schema_job()
- source.prefix = "billing_"
- source.save()
- schema.label = "Charges"
- schema.save()
-
- with patch("django.conf.settings.SITE_URL", "https://app.example.com"):
- produced = self._finalize(team, job, ExternalDataJobStatus.FAILED)
-
- job.refresh_from_db()
- assert job.finished_at is not None
- source_url = f"https://app.example.com/project/{team.pk}/data-management/sources/managed-{source.id}/syncs"
- assert produced == [
- {
- "event": FAILED_EVENT,
- "source_id": str(source.id),
- "source_type": "Stripe",
- "source_prefix": "billing",
- "schema_id": str(schema.id),
- "schema_name": "Charges",
- "job_id": str(job.id),
- "status": ExternalDataSchemaStatus.FAILED,
- "kind": "job_failed",
- "error": "boom",
- "rows_synced": 100,
- "paused": False,
- "failed_runs_in_a_row": 1,
- "source_url": source_url,
- "schema_url": f"{source_url}?schema=Charge",
- "finished_at": job.finished_at.isoformat(),
- }
- ]
-
- def test_failed_event_error_masks_the_stored_credentials_of_the_source(self):
- team, source, _schema, job = _create_org_team_source_schema_job()
- source.job_inputs = {"stripe_secret_key": "sk_live_stored_credential"}
- source.save()
-
- produced = self._finalize(
- team,
- job,
- ExternalDataJobStatus.FAILED,
- latest_error="401 for key sk_live_stored_credential at https://api.example.com?api_key=other_secret_value",
- )
-
- assert produced[0]["error"] == "401 for key *** at https://api.example.com?api_key=***"
-
- def test_a_produce_error_does_not_fail_the_status_update(self):
- team, _source, _schema, job = _create_org_team_source_schema_job()
-
- with (
- patch("products.data_warehouse.backend.logic.external_data_source.jobs.emit_data_import_app_metrics"),
- patch(
- "products.data_warehouse.backend.logic.external_data_source.jobs.schedule_external_data_failure_digest"
- ),
- patch(
- "products.data_warehouse.backend.logic.external_data_source.alerts.produce_internal_event",
- side_effect=RuntimeError("kafka down"),
- ),
- ):
- updated = update_external_job_status(
- job_id=str(job.id),
- team_id=team.pk,
- status=ExternalDataJobStatus.FAILED,
- logger=MagicMock(),
- latest_error="boom",
- )
-
- assert updated.status == ExternalDataJobStatus.FAILED
-
-
class TestFinalizeQueueSweep:
@pytest.mark.parametrize(
"status",
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
index 7cdd23366974..57d31897e257 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
@@ -42,7 +42,6 @@ import {
SidePanelTab,
} from '~/types'
-import { SourceAlerts } from 'products/data_warehouse/frontend/shared/sourceAlerts/SourceAlerts'
import { cleanSourceId, isSelfManagedSourceId } from 'products/data_warehouse/frontend/utils'
import { FailingDestinationsBanner } from './FailingDestinationsBanner'
@@ -62,7 +61,6 @@ const SOURCE_SCENE_TABS = [
'configuration',
'webhook',
'history',
- 'alerts',
] as const
export type SourceSceneTab = (typeof SOURCE_SCENE_TABS)[number]
@@ -105,13 +103,13 @@ export interface sourceSceneLogicValues {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface sourceSceneLogicActions {
_setCurrentTab: (tab: SourceSceneTab) => {
- tab: 'alerts' | 'configuration' | 'destinations' | 'history' | 'metrics' | 'schemas' | 'syncs' | 'webhook'
+ tab: 'configuration' | 'destinations' | 'history' | 'metrics' | 'schemas' | 'syncs' | 'webhook'
}
setBreadcrumbName: (name: string) => {
name: string
}
setCurrentTab: (tab: SourceSceneTab) => {
- tab: 'alerts' | 'configuration' | 'destinations' | 'history' | 'metrics' | 'schemas' | 'syncs' | 'webhook'
+ tab: 'configuration' | 'destinations' | 'history' | 'metrics' | 'schemas' | 'syncs' | 'webhook'
}
}
@@ -250,7 +248,6 @@ export function SourceScene({ id }: SourceSceneProps): JSX.Element {
{isManagedSourceSceneId(id) ? (
<ManagedSourceTabs
sourceId={sourceId}
- sourceName={breadcrumbName}
currentTab={currentTab}
setCurrentTab={setCurrentTab}
attachTo={logic}
@@ -275,13 +272,11 @@ export function SourceScene({ id }: SourceSceneProps): JSX.Element {
function ManagedSourceTabs({
sourceId,
- sourceName,
currentTab,
setCurrentTab,
attachTo,
}: {
sourceId: string
- sourceName: string
currentTab: SourceSceneTab
setCurrentTab: (tab: SourceSceneTab) => void
attachTo: BuiltLogic | LogicWrapper
@@ -297,7 +292,6 @@ function ManagedSourceTabs({
const showWebhookTab = !!source?.supports_webhooks
const showDestinationsTab = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]
const showMetricsTab = shouldShowManagedSourceMetricsTab(source, !!featureFlags[FEATURE_FLAGS.DWH_SOURCE_METRICS])
- const showAlertsTab = !!featureFlags[FEATURE_FLAGS.DWH_SYNC_ALERTS]
useEffect(() => {
// Wait until the source has loaded before deciding a tab is unavailable.
@@ -316,10 +310,7 @@ function ManagedSourceTabs({
if (!showMetricsTab && currentTab === 'metrics') {
setCurrentTab('schemas')
}
- if (!showAlertsTab && currentTab === 'alerts') {
- setCurrentTab('schemas')
- }
- }, [source, showSyncsTab, showWebhookTab, showMetricsTab, showAlertsTab, currentTab, setCurrentTab])
+ }, [source, showSyncsTab, showWebhookTab, showMetricsTab, currentTab, setCurrentTab])
if (sourceLoading && !source) {
return <LemonSkeleton className="w-full h-12" />
@@ -354,14 +345,6 @@ function ManagedSourceTabs({
})
}
- if (showAlertsTab) {
- tabs.push({
- label: 'Alerts',
- key: 'alerts',
- content: <SourceAlerts sourceId={sourceId} sourceName={sourceName} />,
- })
- }
-
tabs.push({
label: 'History',
key: 'history',
diff --git a/products/data_warehouse/frontend/scenes/SourcesScene/SourcesList.tsx b/products/data_warehouse/frontend/scenes/SourcesScene/SourcesList.tsx
index 3c9199f4641b..e5d5abb6b04c 100644
--- a/products/data_warehouse/frontend/scenes/SourcesScene/SourcesList.tsx
+++ b/products/data_warehouse/frontend/scenes/SourcesScene/SourcesList.tsx
@@ -4,7 +4,6 @@ import { IconPlusSmall } from '@posthog/icons'
import { LemonButton, LemonTag } from '@posthog/lemon-ui'
import { FlaggedFeature } from 'lib/components/FlaggedFeature'
-import { FEATURE_FLAGS } from 'lib/constants'
import { HogFunctionList } from 'scenes/hog-functions/list/HogFunctionsList'
import { urls } from 'scenes/urls'
@@ -18,7 +17,6 @@ import {
import { DirectConnectSourcesTable } from 'products/data_warehouse/frontend/shared/components/DirectConnectSourcesTable'
import { ManagedSourcesTable } from 'products/data_warehouse/frontend/shared/components/ManagedSourcesTable'
import { SelfManagedSourcesTable } from 'products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable'
-import { SourceAlerts } from 'products/data_warehouse/frontend/shared/sourceAlerts/SourceAlerts'
export function SourcesList(): JSX.Element {
return (
@@ -90,18 +88,6 @@ export function SourcesList(): JSX.Element {
>
<SelfManagedSourcesTable />
</SceneSection>
-
- <FlaggedFeature flag={FEATURE_FLAGS.DWH_SYNC_ALERTS}>
- <>
- <SceneDivider />
- <SceneSection
- title="Alerts"
- description="Get a message in Slack, Discord, Teams, or a webhook when a source sync fails, recovers, or finishes"
- >
- <SourceAlerts />
- </SceneSection>
- </>
- </FlaggedFeature>
</div>
)
}
diff --git a/products/data_warehouse/frontend/shared/sourceAlerts/SourceAlerts.tsx b/products/data_warehouse/frontend/shared/sourceAlerts/SourceAlerts.tsx
deleted file mode 100644
index d33312cbe0dc..000000000000
--- a/products/data_warehouse/frontend/shared/sourceAlerts/SourceAlerts.tsx
+++ /dev/null
@@ -1,154 +0,0 @@
-import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'
-import posthog from 'posthog-js'
-
-import { LemonButton } from '@posthog/lemon-ui'
-
-import { AlertWizard } from 'lib/components/Alerting/AlertWizard/AlertWizard'
-import {
- AlertCreationView,
- AlertWizardLogicProps,
- alertWizardLogic,
- decorateAlertName,
-} from 'lib/components/Alerting/AlertWizard/alertWizardLogic'
-import { Link } from 'lib/lemon-ui/Link'
-import { HogFunctionList } from 'scenes/hog-functions/list/HogFunctionsList'
-import { HogFunctionTemplateList } from 'scenes/hog-functions/list/HogFunctionTemplateList'
-import { getFiltersFromSubTemplateId } from 'scenes/hog-functions/list/LinkedHogFunctions'
-import { urls } from 'scenes/urls'
-
-import { HogFunctionSubTemplateType } from '~/types'
-
-import {
- buildSourceAlertFilterGroups,
- buildSourceAlertNameSuffix,
- buildSourceAlertPropertyFilters,
-} from './sourceAlertFilters'
-import { sourceAlertsLogic } from './sourceAlertsLogic'
-import {
- SOURCE_ALERT_DESTINATIONS,
- SOURCE_ALERT_SUB_TEMPLATE_IDS,
- SOURCE_ALERT_TRIGGERS,
-} from './sourceAlertWizardConfig'
-
-export interface SourceAlertsProps {
- sourceId?: string
- sourceName?: string
-}
-
-export function SourceAlerts({ sourceId, sourceName }: SourceAlertsProps): JSX.Element {
- useMountedLogic(sourceAlertsLogic({ sourceId }))
-
- const wizardProps: AlertWizardLogicProps = {
- logicKey: `data-warehouse-alerts-${sourceId ?? 'project'}`,
- subTemplateIds: SOURCE_ALERT_SUB_TEMPLATE_IDS,
- triggers: SOURCE_ALERT_TRIGGERS,
- destinations: SOURCE_ALERT_DESTINATIONS,
- contextId: 'data-warehouse-alerts',
- presetPropertyFilters: buildSourceAlertPropertyFilters(sourceId),
- nameSuffix: buildSourceAlertNameSuffix(sourceId, sourceName),
- createdEventName: 'data_warehouse_alert_created',
- }
-
- return (
- <BindLogic logic={alertWizardLogic} props={wizardProps}>
- <SourceAlertsInner sourceId={sourceId} sourceName={sourceName} />
- </BindLogic>
- )
-}
-
-function SourceAlertsInner({ sourceId, sourceName }: SourceAlertsProps): JSX.Element {
- const { alertCreationView, subTemplateIds } = useValues(alertWizardLogic)
- const { setAlertCreationView, resetWizard } = useActions(alertWizardLogic)
-
- const surface = sourceId ? 'source_tab' : 'sources_page'
-
- if (alertCreationView === AlertCreationView.Wizard) {
- return (
- <AlertWizard
- onCancel={() => {
- setAlertCreationView(AlertCreationView.None)
- resetWizard()
- }}
- onSwitchToTraditional={() => {
- posthog.capture('data_warehouse_alert_creation_switched_to_traditional', { surface })
- setAlertCreationView(AlertCreationView.Traditional)
- resetWizard()
- }}
- />
- )
- }
-
- if (alertCreationView === AlertCreationView.Traditional) {
- const properties = buildSourceAlertPropertyFilters(sourceId)
- const nameSuffix = buildSourceAlertNameSuffix(sourceId, sourceName)
-
- return (
- <HogFunctionTemplateList
- type="destination"
- subTemplateIds={subTemplateIds}
- getConfigurationOverrides={(id, subTemplate) => {
- const filters = id ? getFiltersFromSubTemplateId(id) : undefined
- if (!filters) {
- return undefined
- }
- const overrides: Partial<HogFunctionSubTemplateType> = {
- filters: properties.length > 0 ? { ...filters, properties } : filters,
- }
- if (nameSuffix && subTemplate?.name) {
- overrides.name = decorateAlertName(subTemplate.name, null, nameSuffix)
- }
- return overrides
- }}
- extraControls={
- <LemonButton
- type="secondary"
- size="small"
- onClick={() => setAlertCreationView(AlertCreationView.None)}
- >
- Cancel
- </LemonButton>
- }
- />
- )
- }
-
- return (
- <div className="flex flex-col gap-2">
- {sourceId ? (
- <p className="text-secondary mb-0">
- These alerts apply to this source only. Alerts for all sources are on the{' '}
- <Link to={urls.sources()}>Sources page</Link>.
- </p>
- ) : null}
- <HogFunctionList
- forceFilterGroups={buildSourceAlertFilterGroups(sourceId)}
- type="internal_destination"
- returnTo={sourceId ? urls.dataWarehouseSource(`managed-${sourceId}`, 'alerts') : urls.sources()}
- emptyText={
- sourceId
- ? 'No alerts for this source yet. Create an alert to send a message to Slack, Discord, Teams, or a webhook when a sync fails, recovers, or finishes.'
- : 'No alerts yet. Create an alert to send a message to Slack, Discord, Teams, or a webhook when a sync fails, recovers, or finishes.'
- }
- onDeleteHogFunction={(hogFunction) => {
- posthog.capture('data_warehouse_alert_deleted', { hog_function_id: hogFunction.id, surface })
- }}
- onEditHogFunction={(hogFunction) => {
- posthog.capture('data_warehouse_alert_edit_clicked', { hog_function_id: hogFunction.id, surface })
- }}
- extraControls={
- <LemonButton
- type="primary"
- size="small"
- data-attr="new-data-warehouse-alert"
- onClick={() => {
- posthog.capture('data_warehouse_alert_creation_started', { surface })
- setAlertCreationView(AlertCreationView.Wizard)
- }}
- >
- New alert
- </LemonButton>
- }
- />
- </div>
- )
-}
diff --git a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.test.ts b/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.test.ts
deleted file mode 100644
index deebdbbc64d3..000000000000
--- a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.test.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import {
- buildSourceAlertFilterGroups,
- buildSourceAlertNameSuffix,
- buildSourceAlertPropertyFilters,
-} from './sourceAlertFilters'
-import { SOURCE_ALERT_SUB_TEMPLATE_IDS } from './sourceAlertWizardConfig'
-
-describe('sourceAlertFilters', () => {
- const sourceFilter = { key: 'source_id', type: 'event', value: 'abc-123', operator: 'exact' }
-
- it.each([
- ['no source id', undefined, []],
- ['empty source id', '', []],
- ['a source id', 'abc-123', [sourceFilter]],
- ])('builds property filters for %s', (_name, sourceId, expected) => {
- expect(buildSourceAlertPropertyFilters(sourceId)).toEqual(expected)
- })
-
- it('builds one bare event filter per alert type at project level', () => {
- const groups = buildSourceAlertFilterGroups()
- expect(groups).toHaveLength(SOURCE_ALERT_SUB_TEMPLATE_IDS.length)
- groups.forEach((group) => {
- expect(group.source).toBe('internal-events')
- expect(group.events).toHaveLength(1)
- expect(group.properties).toBeUndefined()
- })
- })
-
- it('adds the source filter to every event filter for one source', () => {
- const groups = buildSourceAlertFilterGroups('abc-123')
- expect(groups).toHaveLength(SOURCE_ALERT_SUB_TEMPLATE_IDS.length)
- groups.forEach((group) => {
- expect(group.properties).toEqual([sourceFilter])
- })
- })
-
- it.each([
- ['both set', 'abc-123', 'Stripe', 'for Stripe'],
- ['no name', 'abc-123', undefined, undefined],
- ['no source id', undefined, 'Stripe', undefined],
- ])('builds the name suffix with %s', (_name, sourceId, sourceName, expected) => {
- expect(buildSourceAlertNameSuffix(sourceId, sourceName)).toBe(expected)
- })
-})
diff --git a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.ts b/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.ts
deleted file mode 100644
index d4795b4c2057..000000000000
--- a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertFilters.ts
+++ /dev/null
@@ -1,39 +0,0 @@
-import { getFiltersFromSubTemplateId } from 'scenes/hog-functions/list/LinkedHogFunctions'
-
-import {
- CyclotronJobFilterPropertyFilter,
- CyclotronJobFiltersType,
- PropertyFilterType,
- PropertyOperator,
-} from '~/types'
-
-import { SOURCE_ALERT_SUB_TEMPLATE_IDS } from './sourceAlertWizardConfig'
-
-export function buildSourceAlertPropertyFilters(sourceId?: string): CyclotronJobFilterPropertyFilter[] {
- if (!sourceId) {
- return []
- }
- return [
- {
- key: 'source_id',
- type: PropertyFilterType.Event,
- value: sourceId,
- operator: PropertyOperator.Exact,
- },
- ]
-}
-
-export function buildSourceAlertFilterGroups(sourceId?: string): CyclotronJobFiltersType[] {
- const properties = buildSourceAlertPropertyFilters(sourceId)
- return SOURCE_ALERT_SUB_TEMPLATE_IDS.flatMap((id) => {
- const filters = getFiltersFromSubTemplateId(id)
- if (!filters) {
- return []
- }
- return [properties.length > 0 ? { ...filters, properties } : filters]
- })
-}
-
-export function buildSourceAlertNameSuffix(sourceId?: string, sourceName?: string): string | undefined {
- return sourceId && sourceName ? `for ${sourceName}` : undefined
-}
diff --git a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertWizardConfig.ts b/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertWizardConfig.ts
deleted file mode 100644
index ae03004e273c..000000000000
--- a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertWizardConfig.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import { WizardDestination, WizardTrigger } from 'lib/components/Alerting/AlertWizard/alertWizardLogic'
-
-import { HogFunctionSubTemplateIdType } from '~/types'
-
-export const SOURCE_ALERT_SUB_TEMPLATE_IDS: HogFunctionSubTemplateIdType[] = [
- 'data-warehouse-sync-failed',
- 'data-warehouse-sync-recovered',
- 'data-warehouse-sync-completed',
- 'data-warehouse-billing-limit-reached',
-]
-
-export const SOURCE_ALERT_TRIGGERS: WizardTrigger[] = [
- {
- key: 'data-warehouse-sync-failed',
- name: 'Sync failed',
- description: 'Get notified when a sync fails',
- },
- {
- key: 'data-warehouse-sync-recovered',
- name: 'Sync recovered',
- description: 'Get notified when a sync works again after it failed',
- },
- {
- key: 'data-warehouse-sync-completed',
- name: 'Sync completed',
- description: 'Get notified when a sync finishes',
- },
- {
- key: 'data-warehouse-billing-limit-reached',
- name: 'Billing limit reached',
- description: 'Get notified when a sync stops because of your billing limit',
- },
-]
-
-export const SOURCE_ALERT_DESTINATIONS: WizardDestination[] = [
- {
- key: 'slack',
- name: 'Slack',
- description: 'Send a message to a channel',
- icon: '/static/services/slack.png',
- templateId: 'template-slack',
- },
- {
- key: 'discord',
- name: 'Discord',
- description: 'Post a notification via webhook',
- icon: '/static/services/discord.png',
- templateId: 'template-discord',
- },
- {
- key: 'microsoft-teams',
- name: 'Teams',
- description: 'Send a message to a channel',
- icon: '/static/services/microsoft-teams.png',
- templateId: 'template-microsoft-teams',
- },
- {
- key: 'webhook',
- name: 'Webhook',
- description: 'Send an HTTP request to any URL',
- icon: '/static/services/webhook.svg',
- templateId: 'template-webhook',
- },
-]
diff --git a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertsLogic.ts b/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertsLogic.ts
deleted file mode 100644
index 774d28e1cdd3..000000000000
--- a/products/data_warehouse/frontend/shared/sourceAlerts/sourceAlertsLogic.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import { LogicWrapper, MakeLogicType, afterMount, kea, key, path, props } from 'kea'
-import posthog from 'posthog-js'
-
-export interface SourceAlertsLogicProps {
- sourceId?: string
-}
-
-// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
-export interface sourceAlertsLogicMeta {
- key: string
-}
-
-export type sourceAlertsLogicType = MakeLogicType<{}, {}, SourceAlertsLogicProps, sourceAlertsLogicMeta>
-
-export const sourceAlertsLogic: LogicWrapper<sourceAlertsLogicType> = kea<sourceAlertsLogicType>([
- props({} as SourceAlertsLogicProps),
- key((props) => props.sourceId ?? 'project'),
- path((key) => ['products', 'dataWarehouse', 'sourceAlerts', 'sourceAlertsLogic', key]),
- afterMount(({ props }) => {
- posthog.capture('data_warehouse_alerts_viewed', {
- surface: props.sourceId ? 'source_tab' : 'sources_page',
- })
- }),
-])
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index 14156c73525b..da2b0dd9460d 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -93,7 +93,6 @@ __all__ = [
"list_tables_for_source",
"list_jobs_for_source",
"list_column_statistics",
- "get_sync_alert_context",
# framework-free helper transforms
"effective_source_status",
"clickhouse_columns_to_dwh_columns",
@@ -635,18 +634,3 @@ def list_column_statistics(team_id: int) -> list[contracts.ColumnStatistics]:
)
for table_id, column_name, null_fraction, min_value, max_value in qs
]
-
-
-def get_sync_alert_context(
- team_id: int, schema_id: str, job_id: str | None = None, *, include_error: bool = False
-) -> contracts.SyncAlertContext | None:
- """The facts a sync alert event reports, or None when the schema no longer exists.
-
- `include_error` adds the schema's error with the source's stored credentials masked. It is
- opt-in because the masking decrypts the source's inputs, which most alerts have no use for.
- """
- from products.warehouse_sources.backend.sync_alert_context import ( # noqa: PLC0415 -- the redaction helpers sit under presentation, which the facade must not import at module level
- build_sync_alert_context,
- )
-
- return build_sync_alert_context(team_id, schema_id, job_id, include_error=include_error)
diff --git a/products/warehouse_sources/backend/facade/contracts.py b/products/warehouse_sources/backend/facade/contracts.py
index 52aacaa70215..bf486f804176 100644
--- a/products/warehouse_sources/backend/facade/contracts.py
+++ b/products/warehouse_sources/backend/facade/contracts.py
@@ -243,28 +243,6 @@ class ExternalDataJob:
source_prefix: str | None
-@dataclass(frozen=True)
-class SyncAlertContext:
- """What a sync alert event reports about one schema and, when given, one run.
-
- `latest_error` is already redacted, so it holds no stored credential of the source.
- """
-
- source_id: UUID
- source_type: str
- source_prefix: str | None
- schema_id: UUID
- schema_name: str
- schema_label: str | None
- status: str | None
- latest_error: str | None
- sync_halted: bool
- failed_runs_in_a_row: int
- job_id: UUID | None
- rows_synced: int | None
- finished_at: datetime | None
-
-
# --- Column annotation ---
diff --git a/products/warehouse_sources/backend/sync_alert_context.py b/products/warehouse_sources/backend/sync_alert_context.py
deleted file mode 100644
index 3ba49d9653a1..000000000000
--- a/products/warehouse_sources/backend/sync_alert_context.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from products.warehouse_sources.backend.facade import contracts
-from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-
-
-def build_sync_alert_context(
- team_id: int, schema_id: str, job_id: str | None, *, include_error: bool
-) -> contracts.SyncAlertContext | None:
- schema = ExternalDataSchema.objects.select_related("source").filter(id=schema_id, team_id=team_id).first()
- if schema is None:
- return None
-
- job = ExternalDataJob.objects.filter(id=job_id, team_id=team_id).first() if job_id else None
- source = schema.source
-
- latest_error = None
- if include_error and schema.latest_error:
- # Deferred: the redaction helpers live under presentation, which imports the facades.
- from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import ( # noqa: PLC0415
- get_error_redaction_values,
- redact_error_message,
- )
-
- latest_error = redact_error_message(schema.latest_error, get_error_redaction_values(source))
-
- return contracts.SyncAlertContext(
- source_id=source.id,
- source_type=source.source_type,
- source_prefix=source.prefix,
- schema_id=schema.id,
- schema_name=schema.name,
- schema_label=schema.label,
- status=schema.status,
- latest_error=latest_error,
- sync_halted=schema.sync_halted,
- failed_runs_in_a_row=schema.failed_runs_in_a_row,
- job_id=job.id if job else None,
- rows_synced=job.rows_synced if job else None,
- finished_at=job.finished_at if job else None,
- )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
index 33ddd5592cf0..e40607cbbf0f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
@@ -1288,13 +1288,10 @@ class CDCExtractActivity:
# Outside an activity nothing retries, so every failure there is terminal.
retries_left = activity.in_activity() and activity.info().attempt < CDC_MAX_EXTRACTION_ATTEMPTS
terminal = not info.retryable or not retries_left
- newly_failed_schema_ids: list[str] = []
for schema in self.cdc_schemas:
# Only a terminal failure paints the schema. A later successful attempt never repaints it,
# because its status belongs to the scheduled sync that consumes the buffer.
if terminal and not marked_broken:
- if schema.status != ExternalDataSchema.Status.FAILED:
- newly_failed_schema_ids.append(str(schema.id))
schema.status = ExternalDataSchema.Status.FAILED
schema.latest_error = friendly
schema.save(update_fields=["status", "latest_error", "updated_at"])
@@ -1328,10 +1325,6 @@ class CDCExtractActivity:
# here, mirroring what update_external_job_status does for non-CDC syncs.
if terminal and not marked_broken:
self._schedule_failure_digest()
- # A paused schedule runs no more, so each schema it stops sends the alert once.
- paused = not info.retryable
- alert_schema_ids = [str(schema.id) for schema in self.cdc_schemas] if paused else newly_failed_schema_ids
- self._emit_sync_alerts(alert_schema_ids, paused=paused)
# An unclassified failure stays retryable and never pauses the schedule, so a deterministic
# one re-fails every scheduled run indefinitely. Only _capture_non_retryable emits analytics,
# so these never reach error triage — capture the terminal case so the taxonomy can be taught
@@ -1341,24 +1334,6 @@ class CDCExtractActivity:
self._emit_run_duration("failed")
return info
- def _emit_sync_alerts(self, schema_ids: list[str], *, paused: bool) -> None:
- # An unclassified failure fails every scheduled run again. Only the schemas this run moved
- # to Failed send an alert, so a failure that persists sends one alert, not one for each run.
- # Deferred: the data_warehouse facade imports this pipeline back.
- from products.data_warehouse.backend.facade.api import ( # noqa: PLC0415
- SyncAlertEvent,
- SyncAlertKind,
- emit_sync_alert,
- )
-
- for schema_id in schema_ids:
- emit_sync_alert(
- team_id=self.inputs.team_id,
- schema_id=schema_id,
- event=SyncAlertEvent.FAILED,
- kind=SyncAlertKind.SCHEMA_PAUSED if paused else SyncAlertKind.JOB_FAILED,
- )
-
def _schedule_failure_digest(self) -> None:
try:
# Deferred: the tasks module pulls Celery wiring onto the import path.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/broken.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/broken.py
index e07a0d01a180..53c3beead768 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/broken.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/broken.py
@@ -119,7 +119,6 @@ def mark_cdc_broken(
if create_visibility_jobs:
_create_failure_visibility_jobs(source, newly_broken, message, log)
_schedule_failure_digest(source, log)
- _emit_sync_alerts(source, newly_broken)
_notify(source, message, log)
_capture(source, reason, paused=pause, log=log)
@@ -254,19 +253,6 @@ def _schedule_failure_digest(source: ExternalDataSource, log: typing.Any) -> Non
log.warning("cdc_broken_digest_schedule_failed", exc_info=True)
-def _emit_sync_alerts(source: ExternalDataSource, schemas: list[ExternalDataSchema]) -> None:
- # Deferred: the data_warehouse facade imports this pipeline back.
- from products.data_warehouse.backend.facade.api import SyncAlertEvent, SyncAlertKind, emit_sync_alert
-
- for schema in schemas:
- emit_sync_alert(
- team_id=source.team_id,
- schema_id=str(schema.id),
- event=SyncAlertEvent.FAILED,
- kind=SyncAlertKind.CDC_BROKEN,
- )
-
-
def _pause_schedules(source: ExternalDataSource, cdc_schemas: list[ExternalDataSchema], log: typing.Any) -> None:
# Deferred: data_load.service participates in the CDC schedule<->workflow import cycle.
from products.data_warehouse.backend.facade.api import pause_cdc_extraction_schedule, pause_external_data_schedule
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
index 4bcaa1f2fb94..38deac34afb9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
@@ -550,7 +550,6 @@ class ConformanceKit:
patch(f"{_FACADE}.api.pause_external_data_schedule"),
patch(f"{_FACADE}.api.unpause_external_data_schedule"),
patch(f"{_FACADE}.api.pause_cdc_extraction_schedule"),
- patch(f"{_FACADE}.api.emit_sync_alert"),
patch(f"{_FACADE}.tasks.schedule_external_data_failure_digest"),
patch(f"{_CDC}.buffer.get_s3_client", return_value=self.store),
patch(f"{_CDC}.buffer.ensure_bucket"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_broken.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_broken.py
index 1589a64a6885..705099df0b4a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_broken.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_broken.py
@@ -98,23 +98,13 @@ def test_re_marking_reports_once_per_reason(team, second_reason, expects_new_evi
source = _source(team)
schema = _cdc_schema(team, source)
- with (
- _mocked_boundaries() as (_pause, mock_digest, _notify, _capture, _table_pause),
- patch(
- "products.data_warehouse.backend.logic.external_data_source.alerts.produce_internal_event"
- ) as mock_produce,
- ):
+ with _mocked_boundaries() as (_pause, mock_digest, _notify, _capture, _table_pause):
mark_cdc_broken(source, "critical_lag_self_managed", "lag too high", pause=False)
mark_cdc_broken(source, second_reason, "still broken", pause=False)
expected_evidence_rounds = 2 if expects_new_evidence else 1
assert ExternalDataJob.objects.filter(schema=schema).count() == expected_evidence_rounds
assert mock_digest.call_count == expected_evidence_rounds
- alerts = [
- (produced.kwargs["event"].event, produced.kwargs["event"].properties["kind"])
- for produced in mock_produce.call_args_list
- ]
- assert alerts == [("$data_warehouse_sync_failed", "cdc_broken")] * expected_evidence_rounds
def test_visibility_jobs_can_be_disabled_for_in_run_callers(team):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
index c4baefca1adc..cacaabd5063d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
@@ -612,17 +612,12 @@ class TestErrorClassification:
inputs = CDCExtractInput(team_id=1, source_id=source.id)
with (
patch("products.data_warehouse.backend.facade.tasks.schedule_external_data_failure_digest") as mock_digest,
- patch(
- "products.data_warehouse.backend.logic.external_data_source.alerts.emit_sync_alert"
- ) as mock_emit_sync_alert,
pytest.raises(NonRetryableException),
):
cdc_extract_activity(inputs)
assert schema.status == "Failed"
assert schema.latest_error == cdc_error_info(CDCErrorCategory.AUTH_FAILED).friendly_message
- mock_emit_sync_alert.assert_called_once()
- assert mock_emit_sync_alert.call_args.kwargs["kind"].value == "schema_paused"
# The schedule pause leaves no DB trace of its own; this marker is what tells the failure
# digest email "paused, action required" instead of "will retry".
assert schema.sync_type_config["cdc_extraction_paused"]["reason"] == "auth_failed"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index d2e1a47ac1ff..ecbbfccbbc4b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -31,11 +31,8 @@ from posthog.usage_ingestion.client import UsageRecord, areport_usage
from posthog.utils import get_machine_id
from products.data_warehouse.backend.facade.api import (
- SyncAlertEvent,
- SyncAlertKind,
a_unpause_external_data_schedule,
create_warehouse_templates_for_source,
- emit_sync_alert,
update_external_job_status,
)
from products.managed_warehouse.backend.facade.temporal import (
@@ -702,18 +699,6 @@ async def _update_job_status(inputs: UpdateExternalDataJobStatusInputs, logger:
counts_as_source_failure=counts_as_source_failure,
)
- if has_non_retryable_error:
- # A non-retryable failure does not count toward the failure streak, so the status write
- # above sends no failed alert for it. The schema stops syncing until a person turns it back
- # on, which is the failure a destination most needs to hear about.
- await database_sync_to_async_pool(emit_sync_alert)(
- team_id=inputs.team_id,
- schema_id=inputs.schema_id,
- event=SyncAlertEvent.FAILED,
- kind=SyncAlertKind.SCHEMA_PAUSED,
- job_id=job_id,
- )
-
if inputs.internal_error and source is not None:
# This activity runs once the workflow has given up, so every error reaching here has
# already exhausted its retries — including the ones no source classified. Those used to
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
index 731caf11a3af..9948e39176a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
@@ -335,17 +335,10 @@ def test_read_only_transaction_disables_the_schema_only_when_the_source_raised_i
mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.external_data_job.update_external_job_status"
) as mock_update_job_status,
- mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.external_data_job.emit_sync_alert"
- ) as mock_emit_sync_alert,
):
asyncio.run(env.run(update_external_data_job_model, inputs))
assert mock_update_should_sync.called is expect_disabled
- paused_alerts = [
- (call.kwargs["event"].value, call.kwargs["kind"].value) for call in mock_emit_sync_alert.call_args_list
- ]
- assert paused_alerts == ([("$data_warehouse_sync_failed", "schema_paused")] if expect_disabled else [])
customer_message = mock_update_job_status.call_args.kwargs["latest_error"] or ""
assert ("tries to write to your database" in customer_message) is expect_disabled