Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): count parked lookup failures toward the dead-letter breaker (#122953)

Phil Haack4ffe74c59633
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
index 8ad9283dddf5..b852eca5c76b 100644
--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
@@ -17,7 +17,13 @@ import {
} from '../../../tests/helpers/sql'
import { Hub, Team } from '../../types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
-import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
+import {
+ HOG_EXAMPLES,
+ HOG_FILTERS_EXAMPLES,
+ HOG_FLOW_MASK_EXAMPLES,
+ HOG_INPUTS_EXAMPLES,
+ HOG_MASK_EXAMPLES,
+} from '../_tests/examples'
import {
insertHogFunction as _insertHogFunction,
createHogExecutionGlobals,
@@ -759,26 +765,30 @@ describe('CdpEventsConsumer', () => {
it.each([
{
name: 'a message that is not an event object',
+ step: 'parse',
bad: () => createKafkaMessage('not json at all'),
},
{
name: 'an event whose properties are not JSON',
+ step: 'parse',
bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),
},
{
name: 'an event whose team_id is out of int4 range',
+ step: 'parse',
bad: (teamId: number) =>
createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: 3_000_000_000 }),
},
...lookupSpies.map(({ lookup, spy }) => ({
name: `an event whose ${lookup} lookup throws a non-retriable error`,
+ step: 'lookup',
// The first message in the batch makes the first lookup call, so only it fails.
bad: (teamId: number) => {
spy().mockRejectedValueOnce(new Error('column "bytecode" does not exist'))
return createKafkaMessage(createIncomingEvent(teamId, {}))
},
})),
- ])('parks $name and processes the rest of the batch', async ({ bad }) => {
+ ])('parks $name and processes the rest of the batch', async ({ step, bad }) => {
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
@@ -789,7 +799,7 @@ describe('CdpEventsConsumer', () => {
const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')
expect(parked).toHaveLength(1)
- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_offset: '0' })
+ expect(parked[0].headers).toMatchObject({ dlq_step: step, dlq_offset: '0' })
expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
})
@@ -820,27 +830,79 @@ describe('CdpEventsConsumer', () => {
}
})
- it('fails the batch on a dependency outage while parsing, rather than parking every event', async () => {
- await insertHogFunction({
- ...HOG_EXAMPLES.simple_fetch,
- ...HOG_INPUTS_EXAMPLES.simple_fetch,
- ...HOG_FILTERS_EXAMPLES.no_filters,
- })
- const outage = new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down'))
- const lookup = jest
- .spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam')
- .mockRejectedValue(outage)
+ it.each([
+ {
+ name: 'a dependency outage',
+ error: new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down')),
+ batchSize: 1,
+ rejection: 'Postgres is down',
+ },
+ {
+ name: 'a lookup error a retry cannot fix across the whole batch',
+ error: new Error('column "bytecode" does not exist'),
+ batchSize: 3,
+ rejection: 'Refusing to dead-letter 3 of 3 messages (0 process, 3 lookup)',
+ },
+ ])(
+ 'fails the batch on $name while parsing, rather than parking every event',
+ async ({ error, batchSize, rejection }) => {
+ await insertHogFunction({
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch,
+ ...HOG_FILTERS_EXAMPLES.no_filters,
+ })
+ const lookup = jest
+ .spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam')
+ .mockRejectedValue(error)
+ const messages = Array.from({ length: batchSize }, (_, offset) =>
+ createKafkaMessage(createIncomingEvent(team.id, {}), { offset })
+ )
- await expect(handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])).rejects.toThrow(
- 'Postgres is down'
- )
+ await expect(handleBatch(messages)).rejects.toThrow(rejection)
- // The failed batch never reaches the write, so a failure recorded for it would be
- // parked by the next batch instead.
- lookup.mockRestore()
- await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])
+ // A failure recorded for the failed batch must not be parked by the next batch.
+ lookup.mockRestore()
+ await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])
- expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)
+ expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)
+ }
+ )
+
+ it.each([
+ {
+ name: 'destination',
+ insert: () =>
+ insertHogFunction({
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch,
+ ...HOG_FILTERS_EXAMPLES.no_filters,
+ ...HOG_MASK_EXAMPLES.all,
+ }),
+ },
+ {
+ name: 'workflow',
+ insert: () =>
+ _insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder()
+ .withTeamId(team.id)
+ .withSimpleWorkflow({
+ trigger: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
+ })
+ .withTriggerMasking(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)
+ .build()
+ ),
+ },
+ ])('releases the masking claims of a $name when the batch fails before queueing', async ({ insert }) => {
+ await insert()
+ const globals = await processor._parseKafkaBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])
+
+ await expect(
+ processor.processBatch(globals, () => Promise.reject(new Error('dead-letter write failed')))
+ ).rejects.toThrow('dead-letter write failed')
+
+ const { invocations } = await processor.processBatch(globals)
+ expect(invocations).toHaveLength(1)
})
it('parks nothing when every function builds', async () => {
diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
index 657d0796d2dc..70f2199e4823 100644
--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
@@ -91,22 +91,31 @@ export class CdpEventsConsumer<
// TODO: Add a helper to hog functions to determine if they require groups or not and then only load those
await this.groupsManager.addGroupsToGlobalsList(invocationGlobals)
- const [hogInvocations, hogflowInvocations] = await Promise.all([
- this.hogFunctionPipeline.buildInvocations(invocationGlobals, {
+ const [hogBuild, hogflowBuild] = await Promise.all([
+ this.hogFunctionPipeline.buildInvocationsWithRelease(invocationGlobals, {
hogTypes: this.hogTypes,
filterFn: (fn) => (fn.filters?.source ?? 'events') === 'events',
}),
// Source-compatibility lives in the consumer. The events consumer matches event-triggered
// flows only; other trigger types (data-warehouse-table, batch, schedule, webhook, manual)
// are dispatched from their respective consumers and never reach the executor from here.
- this.hogFlowPipeline.buildInvocations(invocationGlobals, {
+ this.hogFlowPipeline.buildInvocationsWithRelease(invocationGlobals, {
eligibilityFn: (flow) => flow.trigger.type === 'event',
}),
])
+ const hogInvocations = hogBuild.invocations
+ const hogflowInvocations = hogflowBuild.invocations
const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]
- await beforeQueue?.()
+ try {
+ await beforeQueue?.()
+ } catch (error) {
+ // The build already wrote the masking claims. The batch is read again after this throw. A
+ // claim left in place would mask the retried event as a duplicate.
+ await Promise.all([hogBuild.release(), hogflowBuild.release()])
+ throw error
+ }
// Emit a `running` lifecycle row for each freshly-created invocation.
// This fires ONCE per invocation_id at creation — not on every dequeue
@@ -181,8 +190,10 @@ export class CdpEventsConsumer<
@instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')
public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
- const park = (message: Message, error: string): boolean =>
- this.deadLetterService.recordMessageFailure(message, { step: 'parse', error })
+ const park =
+ (step: 'parse' | 'lookup') =>
+ (message: Message, error: string): boolean =>
+ this.deadLetterService.recordMessageFailure(message, { step, error })
return await parseBatch(messages, {
errorLog: 'Error parsing message',
parse: (message) => {
@@ -205,12 +216,13 @@ export class CdpEventsConsumer<
loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),
// Parked rather than dropped: a schema we cannot read today is often one a later
// version can.
- onError: park,
+ onError: park('parse'),
// A lookup error that repeats stops the partition when it fails the batch, so this parks the
// message for a replay. The loaders send the team ids of the whole batch in one query, so one
// failed query parks every message in it. With the dead-letter topic off, nothing keeps the
- // message, so the error fails the batch.
- parkLookupError: park,
+ // message, so the error fails the batch. A failed query that covers enough of the batch trips
+ // the dead-letter circuit breaker, which also fails the batch.
+ parkLookupError: park('lookup'),
})
}
diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts
index 8d6ab9e5c30d..08f1780d6f39 100644
--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts
+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts
@@ -2,7 +2,7 @@ import { Message } from 'node-rdkafka'
import { CdpOutputs } from '../../cdp-services'
import { CDP_EVENTS_DLQ_OUTPUT } from '../../outputs/outputs'
-import { HogFunctionInvocationGlobals, InvocationBuildFailure } from '../../types'
+import { DeadLetterStep, HogFunctionInvocationGlobals, InvocationBuildFailure } from '../../types'
import { CdpDeadLetterService } from './cdp-dead-letter.service'
describe('CdpDeadLetterService', () => {
@@ -34,6 +34,23 @@ describe('CdpDeadLetterService', () => {
...overrides,
})
+ const batchOf = (size: number): Message[] =>
+ Array.from({ length: size }, (_, offset) => message(`event-${offset}`, offset))
+
+ const recordFailure = (service: CdpDeadLetterService, step: DeadLetterStep, source: Message): void => {
+ if (step === 'parse' || step === 'lookup') {
+ service.recordMessageFailure(source, { step, error: 'boom' })
+ return
+ }
+ const globals = globalsFor(2, `event-${source.offset}`)
+ messagesByGlobals.set(globals, source)
+ if (step === 'process') {
+ service.recordProcessFailure(globals, new Error('boom'), 'hog_function')
+ } else {
+ service.recordBuildFailures(globals, [failure({ step })])
+ }
+ }
+
const createService = (enabled = true): CdpDeadLetterService =>
new CdpDeadLetterService(
{ CDP_DLQ_ENABLED: enabled, CDP_DLQ_BATCH_FAIL_RATIO: 0.1 },
@@ -161,28 +178,35 @@ describe('CdpDeadLetterService', () => {
})
})
- it('fails the batch instead of parking it when most of it throws unexpectedly', async () => {
+ it.each<{ name: string; steps: DeadLetterStep[]; counts: string }>([
+ { name: 'throws unexpectedly', steps: ['process', 'process', 'process'], counts: '3 process, 0 lookup' },
+ { name: 'fails its lookups', steps: ['lookup', 'lookup', 'lookup'], counts: '0 process, 3 lookup' },
+ { name: 'throws or fails its lookups', steps: ['process', 'lookup', 'lookup'], counts: '1 process, 2 lookup' },
+ ])('fails the batch instead of parking it when most of it $name', async ({ steps, counts }) => {
const service = createService()
- const messages = [10, 11, 12, 13].map((offset) => message(`event-${offset}`, offset))
- for (const [index, uuid] of ['event-10', 'event-11', 'event-12'].entries()) {
- const globals = globalsFor(2, uuid)
- messagesByGlobals.set(globals, messages[index])
- service.recordProcessFailure(globals, new Error('boom'), 'hog_function')
- }
+ const messages = batchOf(4)
+ steps.forEach((step, index) => recordFailure(service, step, messages[index]))
- await expect(service.produceForBatch(messages)).rejects.toThrow('Refusing to dead-letter 3 of 4')
+ await expect(service.produceForBatch(messages)).rejects.toThrow(
+ `Refusing to dead-letter 3 of 4 messages (${counts})`
+ )
expect(produce).not.toHaveBeenCalled()
})
- it('parks a lone unexpected failure rather than breaking on a small batch', async () => {
+ it.each<{ name: string; step: DeadLetterStep; failing: number; batchSize: number }>([
// A ratio says nothing about one message. Breaking here would stall the partition on the
// very event this path exists to park.
+ { name: 'a lone unexpected failure in a batch of one', step: 'process', failing: 1, batchSize: 1 },
+ // Three meets the floor, but three of forty is under the ratio.
+ { name: 'three lookup failures in a batch of forty', step: 'lookup', failing: 3, batchSize: 40 },
+ ])('parks $name rather than breaking', async ({ step, failing, batchSize }) => {
const service = createService()
- service.recordProcessFailure(defaultGlobals, new Error('boom'), 'hog_function')
+ const messages = batchOf(batchSize)
+ messages.slice(0, failing).forEach((source) => recordFailure(service, step, source))
- await service.produceForBatch([message('event-1')])
+ await service.produceForBatch(messages)
- expect(produce).toHaveBeenCalledTimes(1)
+ expect(produce).toHaveBeenCalledTimes(failing)
})
it('counts events, not failures, when two pipelines both throw on the same event', async () => {
@@ -208,19 +232,17 @@ describe('CdpDeadLetterService', () => {
expect(service.recordProcessFailure(defaultGlobals, new Error('boom'), 'hog_function')).toBe(false)
})
- it('still parks a whole batch of filter and input failures', async () => {
+ it.each<{ name: string; steps: DeadLetterStep[] }>([
// One broken builtin legitimately breaks every function in a batch. Those records are
// exactly what the replay needs, so the ratio must not apply to them.
- //
+ { name: 'filter and input failures', steps: ['filter', 'inputs', 'inputs'] },
+ { name: 'parse failures', steps: ['parse', 'parse', 'parse'] },
+ ])('still parks a whole batch of $name', async ({ steps }) => {
// Sized to clear the breaker on every axis: three events is the floor, and three of four
// messages is past the ratio. Counting these would fail the batch here.
const service = createService()
- const messages = [10, 11, 12, 13].map((offset) => message(`event-${offset}`, offset))
- for (const [index, uuid] of ['event-10', 'event-11', 'event-12'].entries()) {
- const globals = globalsFor(2, uuid)
- messagesByGlobals.set(globals, messages[index])
- service.recordBuildFailures(globals, [failure({ step: index === 0 ? 'filter' : 'inputs' })])
- }
+ const messages = batchOf(4)
+ steps.forEach((step, index) => recordFailure(service, step, messages[index]))
await service.produceForBatch(messages)
diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
index 7b97ab1d8577..9f1484460638 100644
--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
@@ -67,16 +67,26 @@ const MAX_REASON_BYTES = 1024
/** A ratio says nothing on a small batch: one bad event in a batch of one is 100% of it. */
const MIN_UNEXPECTED_TO_BREAK = 3
+/**
+ * The steps that count toward the circuit breaker in `produceForBatch`. `filter` and `inputs` never
+ * count, because one broken builtin legitimately breaks a whole batch and those records are the point.
+ * `parse` never counts either. Most parse errors come from bytes that fail the same way on every read,
+ * so a stall would never clear them. A `lookup` error does not come from the bytes, so it counts with
+ * `process`.
+ */
+const UNEXPECTED_STEPS: ReadonlySet<DeadLetterStep> = new Set<DeadLetterStep>(['lookup', 'process'])
+
export interface CdpDeadLetterServiceConfig {
CDP_DLQ_ENABLED: boolean
CDP_DLQ_BATCH_FAIL_RATIO: number
}
export class DeadLetterCircuitBreakerError extends Error {
- constructor(parked: number, batchSize: number) {
+ constructor(processFailures: number, lookupFailures: number, batchSize: number) {
super(
- `Refusing to dead-letter ${parked} of ${batchSize} messages: an unexpected failure this ` +
- 'wide is a bug in this deployment, not poison in the stream'
+ `Refusing to dead-letter ${processFailures + lookupFailures} of ${batchSize} messages ` +
+ `(${processFailures} process, ${lookupFailures} lookup): an unexpected failure this wide ` +
+ 'is a bug in this deployment or data that fails a batched lookup, not poison in the stream'
)
this.name = 'DeadLetterCircuitBreakerError'
}
@@ -213,21 +223,25 @@ export class CdpDeadLetterService implements InvocationFailureSink {
}
// Stalling the partition beats parking a whole batch: draining the stream into a side topic
- // leaves lag, health and throughput all reading normal, so nobody notices. `filter` and
- // `inputs` never count, because one broken builtin legitimately breaks a whole batch and
- // those records are the point.
- const unexpected = new Set(
+ // leaves lag, health and throughput all reading normal, so nobody notices. An event that fails
+ // the conversion to globals parks as `parse`. This check therefore does not catch a deploy that
+ // breaks that conversion.
+ const unexpectedEvents = new Set(
failures
- .filter((entry) => entry.failures.some((failure) => failure.step === 'process'))
+ .filter((entry) => entry.failures.some((failure) => UNEXPECTED_STEPS.has(failure.step)))
.map((entry) => entry.globals)
- ).size
+ )
+ const unexpectedMessages = new Set(
+ messageFailures.filter(({ failure }) => UNEXPECTED_STEPS.has(failure.step)).map(({ message }) => message)
+ )
+ const unexpected = unexpectedEvents.size + unexpectedMessages.size
if (
unexpected >= MIN_UNEXPECTED_TO_BREAK &&
messages.length &&
unexpected > messages.length * this.config.CDP_DLQ_BATCH_FAIL_RATIO
) {
counterDeadLetterBatchFailures.labels({ output: this.options.output }).inc()
- throw new DeadLetterCircuitBreakerError(unexpected, messages.length)
+ throw new DeadLetterCircuitBreakerError(unexpectedEvents.size, unexpectedMessages.size, messages.length)
}
// Grouped on the globals object itself, so one record covers one event and one step. Both
diff --git a/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts b/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts
index f35e925a4bbc..bbec3b392ddb 100644
--- a/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts
+++ b/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts
@@ -9,6 +9,7 @@ import { CdpValkeyShadowPools } from '../cdp-services'
import { counterHogFunctionStateOnEvent, counterRateLimited } from '../consumers/metrics'
import { shouldBlockInvocationDueToQuota } from '../consumers/quota-limiting-helper'
import {
+ CyclotronJobInvocation,
CyclotronJobInvocationHogFunction,
HogFunctionInvocationGlobals,
HogFunctionType,
@@ -77,11 +78,25 @@ export class HogFunctionInvocationPipeline {
this.hogRateLimiterMirror = new KeyedRateLimiterService(rateLimiterConfig, deps.valkeyShadow.writer)
}
- @instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')
public async buildInvocations(
invocationGlobals: HogFunctionInvocationGlobals[],
opts: BuildHogFunctionInvocationsOptions
): Promise<CyclotronJobInvocationHogFunction[]> {
+ return (await this.buildInvocationsWithRelease(invocationGlobals, opts)).invocations
+ }
+
+ /**
+ * Also returns the masker's `release`. Masking claims are written before the caller queues the invocations,
+ * so a caller that retries after a failed queue must release them, or the retry masks every invocation.
+ */
+ @instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')
+ public async buildInvocationsWithRelease(
+ invocationGlobals: HogFunctionInvocationGlobals[],
+ opts: BuildHogFunctionInvocationsOptions
+ ): Promise<{
+ invocations: CyclotronJobInvocationHogFunction[]
+ release: (persisted?: CyclotronJobInvocation[]) => Promise<void>
+ }> {
const teamsToLoad = [...new Set(invocationGlobals.map((x) => x.project.id))]
const hogFunctionsByTeam = await this.deps.hogFunctionManager.getHogFunctionsForTeams(
teamsToLoad,
@@ -213,7 +228,11 @@ export class HogFunctionInvocationPipeline {
})
)
- const { masked, notMasked: notMaskedInvocations } = await this.deps.hogMasker.filterByMasking(validInvocations)
+ const {
+ masked,
+ notMasked: notMaskedInvocations,
+ release,
+ } = await this.deps.hogMasker.filterByMasking(validInvocations)
this.deps.hogFunctionMonitoringService.queueAppMetrics(
masked.map((item) => ({
@@ -264,6 +283,6 @@ export class HogFunctionInvocationPipeline {
this.deps.hogFunctionMonitoringService.queueAppMetrics(triggeredInvocationsMetrics, 'hog_function')
- return notMaskedInvocations
+ return { invocations: notMaskedInvocations, release }
}
}
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
index a2da97b0d4ec..c4bdd4223116 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
@@ -390,5 +390,45 @@ describe('HogFlowManager', () => {
expect(loaded).not.toBeNull()
expect(loaded).not.toHaveProperty('encrypted_inputs')
})
+
+ it.each<{ name: string; actions: unknown }>([
+ { name: 'actions is not a list', actions: { trigger: { type: 'trigger' } } },
+ { name: 'actions holds a null entry', actions: [null] },
+ {
+ name: 'the trigger action has a null config',
+ actions: [{ id: 'trigger', type: 'trigger', config: null }],
+ },
+ ])(
+ 'leaves out a flow whose secrets cannot be merged when $name, and loads the rest of its batch',
+ async ({ actions }) => {
+ // Served without its secrets, a webhook trigger would skip its auth header check.
+ const flow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder()
+ .withName('Flow that cannot take its secrets')
+ .withTeamId(teamId1)
+ .withStatus('active')
+ .build()
+ )
+ await hub.postgres.query(
+ PostgresUse.COMMON_WRITE,
+ `UPDATE posthog_hogflow SET encrypted_inputs = $2, actions = $3::jsonb WHERE id = $1`,
+ [
+ flow.id,
+ hub.encryptedFields.encrypt(
+ JSON.stringify({ trigger: { auth_header: { value: 'Bearer secret' } } })
+ ),
+ JSON.stringify(actions),
+ ],
+ 'testKey'
+ )
+ manager['onHogFlowsReloaded'](teamId1, [flow.id])
+
+ const loaded = await manager.getHogFlows([flow.id, hogFlows[0].id])
+
+ expect(loaded[flow.id]).toBeNull()
+ expect(loaded[hogFlows[0].id]?.id).toBe(hogFlows[0].id)
+ }
+ )
})
})
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
index e4bb9d9ece3f..d5c3baf2461f 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
@@ -202,7 +202,18 @@ export class HogFlowManagerService {
}
}
}
- this.mergeEncryptedInputs(item)
+ try {
+ this.mergeEncryptedInputs(item)
+ } catch (error) {
+ // A throw here would fail the batched load for every flow in it. Only this flow is left
+ // out, because serving it without its secrets would skip a webhook trigger's auth check.
+ logger.warn('[HogFlowManager]', 'Could not merge encrypted inputs - flow is left out', {
+ hogFlowId: item.id,
+ error: error instanceof Error ? error.message : 'Unknown error',
+ })
+ captureException(error)
+ return acc
+ }
acc[item.id] = item
return acc
}, {})
diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.ts
index 79aa6c4a11f3..c22bc248ae0b 100644
--- a/nodejs/src/cdp/types.ts
+++ b/nodejs/src/cdp/types.ts
@@ -298,11 +298,12 @@ export type AppMetricType = MinimalAppMetric & {
* Where an event stopped, for a record on a dead-letter topic.
*
* `filter` and `inputs` are the two failures the pipeline expects and handles per function.
- * `parse` and `process` are the ones it does not: a message it cannot read, and anything else
- * that throws while an event is being turned into invocations. Those two exist so an unanticipated
- * bug parks the event rather than stalling the partition it arrived on.
+ * `parse`, `lookup` and `process` are the ones it does not: a message it cannot read, a team, hog
+ * function or hog flow lookup that failed with an error that is not retriable, and anything else
+ * that throws while the pipeline turns an event into invocations. Those three park a lone
+ * unanticipated failure rather than stalling the partition it arrived on.
*/
-export type DeadLetterStep = 'parse' | 'filter' | 'inputs' | 'process'
+export type DeadLetterStep = 'parse' | 'lookup' | 'filter' | 'inputs' | 'process'
export type InvocationBuildFailure = {
sourceId: string
diff --git a/nodejs/src/common/utils/team-manager.test.ts b/nodejs/src/common/utils/team-manager.test.ts
index b71599f89934..03e02d75bc6f 100644
--- a/nodejs/src/common/utils/team-manager.test.ts
+++ b/nodejs/src/common/utils/team-manager.test.ts
@@ -274,16 +274,20 @@ describe('TeamManager()', () => {
})
describe('hasAvailableFeature()', () => {
- it('returns false by default', async () => {
- await updateOrganizationAvailableFeatures(postgres, organizationId, [])
- const result = await teamManager.hasAvailableFeature(teamId, 'data_pipelines')
- expect(result).toBe(false)
- })
-
- it('returns false if the available features does not exist', async () => {
- await updateOrganizationAvailableFeatures(postgres, organizationId, [
- { key: 'not_data_pipelines', name: 'Feature 1' },
- ])
+ it.each<{ name: string; features: ({ key: string; name: string } | null)[] }>([
+ { name: 'by default', features: [] },
+ {
+ name: 'if the available features does not exist',
+ features: [{ key: 'not_data_pipelines', name: 'Feature 1' }],
+ },
+ // A malformed row must load, because a throw fails the batched lookup for every team in it.
+ { name: 'if the available features holds a null entry', features: [null] },
+ ])('returns false $name', async ({ features }) => {
+ await updateOrganizationAvailableFeatures(
+ postgres,
+ organizationId,
+ features as { key: string; name: string }[]
+ )
const result = await teamManager.hasAvailableFeature(teamId, 'data_pipelines')
expect(result).toBe(false)
})
diff --git a/nodejs/src/common/utils/team-manager.ts b/nodejs/src/common/utils/team-manager.ts
index f8009e162ffb..a4885ca03fff 100644
--- a/nodejs/src/common/utils/team-manager.ts
+++ b/nodejs/src/common/utils/team-manager.ts
@@ -7,7 +7,18 @@ import { logger } from './logger'
import { captureTeamEvent } from './posthog'
type RawTeam = Omit<Team, 'available_features'> & {
- available_product_features: { key: string; name: string }[]
+ available_product_features: unknown
+}
+
+// One team's row must not throw, because a throw fails the batched lookup for every team in it. The
+// column is a jsonb array, so an entry can be null. An entry without a string key is skipped.
+function availableFeatures(raw: unknown): OrganizationAvailableFeature[] {
+ if (!Array.isArray(raw)) {
+ return []
+ }
+ return raw.flatMap((feature) =>
+ typeof feature?.key === 'string' ? [feature.key as OrganizationAvailableFeature] : []
+ )
}
// Postgres stores team ids in int4 columns. A query that compares one of those columns to a
@@ -210,7 +221,7 @@ export class TeamManager {
teamPartial.drop_events_older_than_seconds !== null
? Number(teamPartial.drop_events_older_than_seconds)
: null,
- available_features: available_product_features?.map((f) => f.key as OrganizationAvailableFeature) || [],
+ available_features: availableFeatures(available_product_features),
}
resultRecord[row.id] = team
resultRecord[row.api_token] = team