:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { v4 } from 'uuid'
import { INGESTION_WARNINGS_OUTPUT, IngestionWarningsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { PipelineResultType } from '~/ingestion/framework/results'
import { createTestEventHeaders } from '~/tests/helpers/event-headers'
import { createTestMessage } from '~/tests/helpers/kafka-message'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import { createTestPluginEvent } from '~/tests/helpers/plugin-event'
import {
HandleClientIngestionWarningStepInput,
createHandleClientIngestionWarningStep,
} from './handle-client-ingestion-warning-step'
describe('handleClientIngestionWarningStep', () => {
const eventUuid = v4()
const capturedAt = new Date('2023-12-15T14:32:01.987Z')
const baseEvent = createTestPluginEvent({
distinct_id: 'my_id',
site_url: '',
team_id: 1,
now: new Date().toISOString(),
event: '$pageview',
properties: {},
uuid: eventUuid,
})
let mockOutputs: jest.Mocked<IngestionOutputs<IngestionWarningsOutput>>
let baseInput: HandleClientIngestionWarningStepInput
beforeEach(() => {
mockOutputs = createMockIngestionOutputs<IngestionWarningsOutput>()
baseInput = {
event: baseEvent,
team: { id: 1 },
headers: createTestEventHeaders({ now: capturedAt }),
message: createTestMessage(),
}
})
function producedWarning(): { team_id: number; type: string; details: Record<string, any> } {
expect(mockOutputs.queueMessages).toHaveBeenCalledWith(INGESTION_WARNINGS_OUTPUT, [
{ value: expect.any(Buffer) },
])
const value = mockOutputs.queueMessages.mock.calls[0][1][0].value
if (value === null) {
throw new Error('expected warning message to have a value')
}
const parsed = parseJSON(value.toString())
return { ...parsed, details: parseJSON(parsed.details) }
}
describe('$$client_ingestion_warning events', () => {
it('emits the warning and resolves ingested with the event info', async () => {
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: { $$client_ingestion_warning_message: 'Test warning message' },
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.ingested).toHaveLength(1)
expect(result.value.ingested).toEqual(result.sideEffects)
await expect(result.value.ingested[0]).resolves.toEqual({
capturedAt,
topic: 'test-topic',
partition: 5,
})
}
const warning = producedWarning()
expect(warning.team_id).toBe(1)
expect(warning.type).toBe('client_ingestion_warning')
expect(warning.details).toMatchObject({
eventUuid: eventUuid,
event: '$$client_ingestion_warning',
distinctId: 'my_id',
message: 'Test warning message',
})
})
it('handles missing warning message property', async () => {
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
expect(producedWarning().details.message).toBeUndefined()
})
it('resolves ingested with null when the warning could not be produced', async () => {
mockOutputs.queueMessages.mockRejectedValue(new Error('produce failed'))
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
await expect(result.value.ingested[0]).resolves.toBeNull()
}
})
})
describe('structured backend warnings', () => {
it('preserves the structured type, source, and details from the envelope', async () => {
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {
$$client_ingestion_warning_type: 'missing_event_name',
$$client_ingestion_warning_source: 'capture',
$$client_ingestion_warning_details: {
distinctId: 'offending-id',
eventUuid: 'offending-uuid',
count: 3,
pipelineStep: 'capture_validation',
},
},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
const warning = producedWarning()
expect(warning.type).toBe('missing_event_name')
expect((warning as any).source).toBe('capture')
// The producer's own ids win over the synthetic warning event's ids.
expect(warning.details).toMatchObject({
distinctId: 'offending-id',
eventUuid: 'offending-uuid',
count: 3,
pipelineStep: 'capture_validation',
// Category/severity come from the registry, not the envelope.
category: 'event',
severity: 'error',
})
})
it('falls back to the client warning shape for an unrecognized structured type', async () => {
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {
$$client_ingestion_warning_type: 'not_a_real_type',
$$client_ingestion_warning_message: 'legacy',
},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
const warning = producedWarning()
expect(warning.type).toBe('client_ingestion_warning')
expect((warning as any).source).toBe('plugin-server')
expect(warning.details.message).toBe('legacy')
})
it('falls back to the client warning shape for a registered but non-capture type, even with forged details', async () => {
// $$client_ingestion_warning arrives over the public capture path, so a
// client could try to impersonate a plugin-server-only type (registered,
// but never emitted by capture) to reach a renderer that assumes a
// validated shape (e.g. schema_validation_failed's `details.errors`).
// The allowlist must reject this even though the type itself is
// registered in INGESTION_WARNING_TYPES.
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {
$$client_ingestion_warning_type: 'schema_validation_failed',
$$client_ingestion_warning_source: 'capture',
$$client_ingestion_warning_details: {},
},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
const warning = producedWarning()
expect(warning.type).toBe('client_ingestion_warning')
expect((warning as any).source).toBe('plugin-server')
})
// Unique team ids so each row hits a fresh per-(team,type) limiter bucket
// rather than the one the 'preserves' test above already consumed.
it.each([
['a string', 'x'.repeat(10), 90001],
['an array', ['a', 'b', 'c'], 90002],
])(
'drops non-object details (%s) instead of exploding it into indexed keys',
async (_label, badDetails, teamId) => {
// A client can set details to a large string/array; object-rest would
// otherwise expand it into one key per character/element in the hot
// consumer path, before the per-(team,type) limiter can throttle it.
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
team: { id: teamId },
event: {
...baseEvent,
event: '$$client_ingestion_warning',
properties: {
$$client_ingestion_warning_type: 'missing_event_name',
$$client_ingestion_warning_source: 'capture',
$$client_ingestion_warning_details: badDetails,
},
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
const warning = producedWarning()
// Still promoted (type is allowlisted), but details carry only the
// envelope defaults — no indexed keys leaked from the bad shape.
expect(warning.type).toBe('missing_event_name')
const indexedKeys = Object.keys(warning.details).filter((k) => /^\d+$/.test(k))
expect(indexedKeys).toEqual([])
expect(warning.details).toMatchObject({ eventUuid, distinctId: 'my_id' })
}
)
})
describe('non-client ingestion warning events', () => {
it.each([['$pageview'], ['$identify'], ['custom_event']])('DLQs %s events', async (eventName) => {
const step = createHandleClientIngestionWarningStep(mockOutputs)
const input: HandleClientIngestionWarningStepInput = {
...baseInput,
event: {
...baseEvent,
event: eventName,
},
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('unexpected_event_type')
expect(result.error).toBeInstanceOf(Error)
expect((result.error as Error).message).toBe(`Expected $$client_ingestion_warning, got ${eventName}`)
}
expect(mockOutputs.queueMessages).not.toHaveBeenCalled()
})
})
})