:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): count parked lookup failures toward the dead-letter breaker (#122953)
Phil Haack4ffe74c59633
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.tsindex 8ad9283dddf5..b852eca5c76b 100644--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts@@ -17,7 +17,13 @@ import {} from '../../../tests/helpers/sql'import { Hub, Team } from '../../types'import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'-import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'+import {+ HOG_EXAMPLES,+ HOG_FILTERS_EXAMPLES,+ HOG_FLOW_MASK_EXAMPLES,+ HOG_INPUTS_EXAMPLES,+ HOG_MASK_EXAMPLES,+} from '../_tests/examples'import {insertHogFunction as _insertHogFunction,createHogExecutionGlobals,@@ -759,26 +765,30 @@ describe('CdpEventsConsumer', () => {it.each([{name: 'a message that is not an event object',+ step: 'parse',bad: () => createKafkaMessage('not json at all'),},{name: 'an event whose properties are not JSON',+ step: 'parse',bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),},{name: 'an event whose team_id is out of int4 range',+ step: 'parse',bad: (teamId: number) =>createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: 3_000_000_000 }),},...lookupSpies.map(({ lookup, spy }) => ({name: `an event whose ${lookup} lookup throws a non-retriable error`,+ step: 'lookup',// The first message in the batch makes the first lookup call, so only it fails.bad: (teamId: number) => {spy().mockRejectedValueOnce(new Error('column "bytecode" does not exist'))return createKafkaMessage(createIncomingEvent(teamId, {}))},})),- ])('parks $name and processes the rest of the batch', async ({ bad }) => {+ ])('parks $name and processes the rest of the batch', async ({ step, bad }) => {await insertHogFunction({...HOG_EXAMPLES.simple_fetch,...HOG_INPUTS_EXAMPLES.simple_fetch,@@ -789,7 +799,7 @@ describe('CdpEventsConsumer', () => {const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')expect(parked).toHaveLength(1)- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_offset: '0' })+ expect(parked[0].headers).toMatchObject({ dlq_step: step, dlq_offset: '0' })expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])})@@ -820,27 +830,79 @@ describe('CdpEventsConsumer', () => {}})- it('fails the batch on a dependency outage while parsing, rather than parking every event', async () => {- await insertHogFunction({- ...HOG_EXAMPLES.simple_fetch,- ...HOG_INPUTS_EXAMPLES.simple_fetch,- ...HOG_FILTERS_EXAMPLES.no_filters,- })- const outage = new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down'))- const lookup = jest- .spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam')- .mockRejectedValue(outage)+ it.each([+ {+ name: 'a dependency outage',+ error: new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down')),+ batchSize: 1,+ rejection: 'Postgres is down',+ },+ {+ name: 'a lookup error a retry cannot fix across the whole batch',+ error: new Error('column "bytecode" does not exist'),+ batchSize: 3,+ rejection: 'Refusing to dead-letter 3 of 3 messages (0 process, 3 lookup)',+ },+ ])(+ 'fails the batch on $name while parsing, rather than parking every event',+ async ({ error, batchSize, rejection }) => {+ await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.no_filters,+ })+ const lookup = jest+ .spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam')+ .mockRejectedValue(error)+ const messages = Array.from({ length: batchSize }, (_, offset) =>+ createKafkaMessage(createIncomingEvent(team.id, {}), { offset })+ )- await expect(handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])).rejects.toThrow(- 'Postgres is down'- )+ await expect(handleBatch(messages)).rejects.toThrow(rejection)- // The failed batch never reaches the write, so a failure recorded for it would be- // parked by the next batch instead.- lookup.mockRestore()- await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])+ // A failure recorded for the failed batch must not be parked by the next batch.+ lookup.mockRestore()+ await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])- expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)+ expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)+ }+ )++ it.each([+ {+ name: 'destination',+ insert: () =>+ insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.no_filters,+ ...HOG_MASK_EXAMPLES.all,+ }),+ },+ {+ name: 'workflow',+ insert: () =>+ _insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder()+ .withTeamId(team.id)+ .withSimpleWorkflow({+ trigger: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },+ })+ .withTriggerMasking(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)+ .build()+ ),+ },+ ])('releases the masking claims of a $name when the batch fails before queueing', async ({ insert }) => {+ await insert()+ const globals = await processor._parseKafkaBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])++ await expect(+ processor.processBatch(globals, () => Promise.reject(new Error('dead-letter write failed')))+ ).rejects.toThrow('dead-letter write failed')++ const { invocations } = await processor.processBatch(globals)+ expect(invocations).toHaveLength(1)})it('parks nothing when every function builds', async () => {diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex 657d0796d2dc..70f2199e4823 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -91,22 +91,31 @@ export class CdpEventsConsumer<// TODO: Add a helper to hog functions to determine if they require groups or not and then only load thoseawait this.groupsManager.addGroupsToGlobalsList(invocationGlobals)- const [hogInvocations, hogflowInvocations] = await Promise.all([- this.hogFunctionPipeline.buildInvocations(invocationGlobals, {+ const [hogBuild, hogflowBuild] = await Promise.all([+ this.hogFunctionPipeline.buildInvocationsWithRelease(invocationGlobals, {hogTypes: this.hogTypes,filterFn: (fn) => (fn.filters?.source ?? 'events') === 'events',}),// Source-compatibility lives in the consumer. The events consumer matches event-triggered// flows only; other trigger types (data-warehouse-table, batch, schedule, webhook, manual)// are dispatched from their respective consumers and never reach the executor from here.- this.hogFlowPipeline.buildInvocations(invocationGlobals, {+ this.hogFlowPipeline.buildInvocationsWithRelease(invocationGlobals, {eligibilityFn: (flow) => flow.trigger.type === 'event',}),])+ const hogInvocations = hogBuild.invocations+ const hogflowInvocations = hogflowBuild.invocationsconst invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]- await beforeQueue?.()+ try {+ await beforeQueue?.()+ } catch (error) {+ // The build already wrote the masking claims. The batch is read again after this throw. A+ // claim left in place would mask the retried event as a duplicate.+ await Promise.all([hogBuild.release(), hogflowBuild.release()])+ throw error+ }// Emit a `running` lifecycle row for each freshly-created invocation.// This fires ONCE per invocation_id at creation — not on every dequeue@@ -181,8 +190,10 @@ export class CdpEventsConsumer<@instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {- const park = (message: Message, error: string): boolean =>- this.deadLetterService.recordMessageFailure(message, { step: 'parse', error })+ const park =+ (step: 'parse' | 'lookup') =>+ (message: Message, error: string): boolean =>+ this.deadLetterService.recordMessageFailure(message, { step, error })return await parseBatch(messages, {errorLog: 'Error parsing message',parse: (message) => {@@ -205,12 +216,13 @@ export class CdpEventsConsumer<loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),// Parked rather than dropped: a schema we cannot read today is often one a later// version can.- onError: park,+ onError: park('parse'),// A lookup error that repeats stops the partition when it fails the batch, so this parks the// message for a replay. The loaders send the team ids of the whole batch in one query, so one// failed query parks every message in it. With the dead-letter topic off, nothing keeps the- // message, so the error fails the batch.- parkLookupError: park,+ // message, so the error fails the batch. A failed query that covers enough of the batch trips+ // the dead-letter circuit breaker, which also fails the batch.+ parkLookupError: park('lookup'),})}diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.tsindex 8d6ab9e5c30d..08f1780d6f39 100644--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.test.ts@@ -2,7 +2,7 @@ import { Message } from 'node-rdkafka'import { CdpOutputs } from '../../cdp-services'import { CDP_EVENTS_DLQ_OUTPUT } from '../../outputs/outputs'-import { HogFunctionInvocationGlobals, InvocationBuildFailure } from '../../types'+import { DeadLetterStep, HogFunctionInvocationGlobals, InvocationBuildFailure } from '../../types'import { CdpDeadLetterService } from './cdp-dead-letter.service'describe('CdpDeadLetterService', () => {@@ -34,6 +34,23 @@ describe('CdpDeadLetterService', () => {...overrides,})+ const batchOf = (size: number): Message[] =>+ Array.from({ length: size }, (_, offset) => message(`event-${offset}`, offset))++ const recordFailure = (service: CdpDeadLetterService, step: DeadLetterStep, source: Message): void => {+ if (step === 'parse' || step === 'lookup') {+ service.recordMessageFailure(source, { step, error: 'boom' })+ return+ }+ const globals = globalsFor(2, `event-${source.offset}`)+ messagesByGlobals.set(globals, source)+ if (step === 'process') {+ service.recordProcessFailure(globals, new Error('boom'), 'hog_function')+ } else {+ service.recordBuildFailures(globals, [failure({ step })])+ }+ }+const createService = (enabled = true): CdpDeadLetterService =>new CdpDeadLetterService({ CDP_DLQ_ENABLED: enabled, CDP_DLQ_BATCH_FAIL_RATIO: 0.1 },@@ -161,28 +178,35 @@ describe('CdpDeadLetterService', () => {})})- it('fails the batch instead of parking it when most of it throws unexpectedly', async () => {+ it.each<{ name: string; steps: DeadLetterStep[]; counts: string }>([+ { name: 'throws unexpectedly', steps: ['process', 'process', 'process'], counts: '3 process, 0 lookup' },+ { name: 'fails its lookups', steps: ['lookup', 'lookup', 'lookup'], counts: '0 process, 3 lookup' },+ { name: 'throws or fails its lookups', steps: ['process', 'lookup', 'lookup'], counts: '1 process, 2 lookup' },+ ])('fails the batch instead of parking it when most of it $name', async ({ steps, counts }) => {const service = createService()- const messages = [10, 11, 12, 13].map((offset) => message(`event-${offset}`, offset))- for (const [index, uuid] of ['event-10', 'event-11', 'event-12'].entries()) {- const globals = globalsFor(2, uuid)- messagesByGlobals.set(globals, messages[index])- service.recordProcessFailure(globals, new Error('boom'), 'hog_function')- }+ const messages = batchOf(4)+ steps.forEach((step, index) => recordFailure(service, step, messages[index]))- await expect(service.produceForBatch(messages)).rejects.toThrow('Refusing to dead-letter 3 of 4')+ await expect(service.produceForBatch(messages)).rejects.toThrow(+ `Refusing to dead-letter 3 of 4 messages (${counts})`+ )expect(produce).not.toHaveBeenCalled()})- it('parks a lone unexpected failure rather than breaking on a small batch', async () => {+ it.each<{ name: string; step: DeadLetterStep; failing: number; batchSize: number }>([// A ratio says nothing about one message. Breaking here would stall the partition on the// very event this path exists to park.+ { name: 'a lone unexpected failure in a batch of one', step: 'process', failing: 1, batchSize: 1 },+ // Three meets the floor, but three of forty is under the ratio.+ { name: 'three lookup failures in a batch of forty', step: 'lookup', failing: 3, batchSize: 40 },+ ])('parks $name rather than breaking', async ({ step, failing, batchSize }) => {const service = createService()- service.recordProcessFailure(defaultGlobals, new Error('boom'), 'hog_function')+ const messages = batchOf(batchSize)+ messages.slice(0, failing).forEach((source) => recordFailure(service, step, source))- await service.produceForBatch([message('event-1')])+ await service.produceForBatch(messages)- expect(produce).toHaveBeenCalledTimes(1)+ expect(produce).toHaveBeenCalledTimes(failing)})it('counts events, not failures, when two pipelines both throw on the same event', async () => {@@ -208,19 +232,17 @@ describe('CdpDeadLetterService', () => {expect(service.recordProcessFailure(defaultGlobals, new Error('boom'), 'hog_function')).toBe(false)})- it('still parks a whole batch of filter and input failures', async () => {+ it.each<{ name: string; steps: DeadLetterStep[] }>([// One broken builtin legitimately breaks every function in a batch. Those records are// exactly what the replay needs, so the ratio must not apply to them.- //+ { name: 'filter and input failures', steps: ['filter', 'inputs', 'inputs'] },+ { name: 'parse failures', steps: ['parse', 'parse', 'parse'] },+ ])('still parks a whole batch of $name', async ({ steps }) => {// Sized to clear the breaker on every axis: three events is the floor, and three of four// messages is past the ratio. Counting these would fail the batch here.const service = createService()- const messages = [10, 11, 12, 13].map((offset) => message(`event-${offset}`, offset))- for (const [index, uuid] of ['event-10', 'event-11', 'event-12'].entries()) {- const globals = globalsFor(2, uuid)- messagesByGlobals.set(globals, messages[index])- service.recordBuildFailures(globals, [failure({ step: index === 0 ? 'filter' : 'inputs' })])- }+ const messages = batchOf(4)+ steps.forEach((step, index) => recordFailure(service, step, messages[index]))await service.produceForBatch(messages)diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.tsindex 7b97ab1d8577..9f1484460638 100644--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts@@ -67,16 +67,26 @@ const MAX_REASON_BYTES = 1024/** A ratio says nothing on a small batch: one bad event in a batch of one is 100% of it. */const MIN_UNEXPECTED_TO_BREAK = 3+/**+ * The steps that count toward the circuit breaker in `produceForBatch`. `filter` and `inputs` never+ * count, because one broken builtin legitimately breaks a whole batch and those records are the point.+ * `parse` never counts either. Most parse errors come from bytes that fail the same way on every read,+ * so a stall would never clear them. A `lookup` error does not come from the bytes, so it counts with+ * `process`.+ */+const UNEXPECTED_STEPS: ReadonlySet<DeadLetterStep> = new Set<DeadLetterStep>(['lookup', 'process'])+export interface CdpDeadLetterServiceConfig {CDP_DLQ_ENABLED: booleanCDP_DLQ_BATCH_FAIL_RATIO: number}export class DeadLetterCircuitBreakerError extends Error {- constructor(parked: number, batchSize: number) {+ constructor(processFailures: number, lookupFailures: number, batchSize: number) {super(- `Refusing to dead-letter ${parked} of ${batchSize} messages: an unexpected failure this ` +- 'wide is a bug in this deployment, not poison in the stream'+ `Refusing to dead-letter ${processFailures + lookupFailures} of ${batchSize} messages ` ++ `(${processFailures} process, ${lookupFailures} lookup): an unexpected failure this wide ` ++ 'is a bug in this deployment or data that fails a batched lookup, not poison in the stream')this.name = 'DeadLetterCircuitBreakerError'}@@ -213,21 +223,25 @@ export class CdpDeadLetterService implements InvocationFailureSink {}// Stalling the partition beats parking a whole batch: draining the stream into a side topic- // leaves lag, health and throughput all reading normal, so nobody notices. `filter` and- // `inputs` never count, because one broken builtin legitimately breaks a whole batch and- // those records are the point.- const unexpected = new Set(+ // leaves lag, health and throughput all reading normal, so nobody notices. An event that fails+ // the conversion to globals parks as `parse`. This check therefore does not catch a deploy that+ // breaks that conversion.+ const unexpectedEvents = new Set(failures- .filter((entry) => entry.failures.some((failure) => failure.step === 'process'))+ .filter((entry) => entry.failures.some((failure) => UNEXPECTED_STEPS.has(failure.step))).map((entry) => entry.globals)- ).size+ )+ const unexpectedMessages = new Set(+ messageFailures.filter(({ failure }) => UNEXPECTED_STEPS.has(failure.step)).map(({ message }) => message)+ )+ const unexpected = unexpectedEvents.size + unexpectedMessages.sizeif (unexpected >= MIN_UNEXPECTED_TO_BREAK &&messages.length &&unexpected > messages.length * this.config.CDP_DLQ_BATCH_FAIL_RATIO) {counterDeadLetterBatchFailures.labels({ output: this.options.output }).inc()- throw new DeadLetterCircuitBreakerError(unexpected, messages.length)+ throw new DeadLetterCircuitBreakerError(unexpectedEvents.size, unexpectedMessages.size, messages.length)}// Grouped on the globals object itself, so one record covers one event and one step. Bothdiff --git a/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts b/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.tsindex f35e925a4bbc..bbec3b392ddb 100644--- a/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts+++ b/nodejs/src/cdp/services/hog-function-invocation-pipeline.service.ts@@ -9,6 +9,7 @@ import { CdpValkeyShadowPools } from '../cdp-services'import { counterHogFunctionStateOnEvent, counterRateLimited } from '../consumers/metrics'import { shouldBlockInvocationDueToQuota } from '../consumers/quota-limiting-helper'import {+ CyclotronJobInvocation,CyclotronJobInvocationHogFunction,HogFunctionInvocationGlobals,HogFunctionType,@@ -77,11 +78,25 @@ export class HogFunctionInvocationPipeline {this.hogRateLimiterMirror = new KeyedRateLimiterService(rateLimiterConfig, deps.valkeyShadow.writer)}- @instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')public async buildInvocations(invocationGlobals: HogFunctionInvocationGlobals[],opts: BuildHogFunctionInvocationsOptions): Promise<CyclotronJobInvocationHogFunction[]> {+ return (await this.buildInvocationsWithRelease(invocationGlobals, opts)).invocations+ }++ /**+ * Also returns the masker's `release`. Masking claims are written before the caller queues the invocations,+ * so a caller that retries after a failed queue must release them, or the retry masks every invocation.+ */+ @instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')+ public async buildInvocationsWithRelease(+ invocationGlobals: HogFunctionInvocationGlobals[],+ opts: BuildHogFunctionInvocationsOptions+ ): Promise<{+ invocations: CyclotronJobInvocationHogFunction[]+ release: (persisted?: CyclotronJobInvocation[]) => Promise<void>+ }> {const teamsToLoad = [...new Set(invocationGlobals.map((x) => x.project.id))]const hogFunctionsByTeam = await this.deps.hogFunctionManager.getHogFunctionsForTeams(teamsToLoad,@@ -213,7 +228,11 @@ export class HogFunctionInvocationPipeline {}))- const { masked, notMasked: notMaskedInvocations } = await this.deps.hogMasker.filterByMasking(validInvocations)+ const {+ masked,+ notMasked: notMaskedInvocations,+ release,+ } = await this.deps.hogMasker.filterByMasking(validInvocations)this.deps.hogFunctionMonitoringService.queueAppMetrics(masked.map((item) => ({@@ -264,6 +283,6 @@ export class HogFunctionInvocationPipeline {this.deps.hogFunctionMonitoringService.queueAppMetrics(triggeredInvocationsMetrics, 'hog_function')- return notMaskedInvocations+ return { invocations: notMaskedInvocations, release }}}diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.tsindex a2da97b0d4ec..c4bdd4223116 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts@@ -390,5 +390,45 @@ describe('HogFlowManager', () => {expect(loaded).not.toBeNull()expect(loaded).not.toHaveProperty('encrypted_inputs')})++ it.each<{ name: string; actions: unknown }>([+ { name: 'actions is not a list', actions: { trigger: { type: 'trigger' } } },+ { name: 'actions holds a null entry', actions: [null] },+ {+ name: 'the trigger action has a null config',+ actions: [{ id: 'trigger', type: 'trigger', config: null }],+ },+ ])(+ 'leaves out a flow whose secrets cannot be merged when $name, and loads the rest of its batch',+ async ({ actions }) => {+ // Served without its secrets, a webhook trigger would skip its auth header check.+ const flow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder()+ .withName('Flow that cannot take its secrets')+ .withTeamId(teamId1)+ .withStatus('active')+ .build()+ )+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `UPDATE posthog_hogflow SET encrypted_inputs = $2, actions = $3::jsonb WHERE id = $1`,+ [+ flow.id,+ hub.encryptedFields.encrypt(+ JSON.stringify({ trigger: { auth_header: { value: 'Bearer secret' } } })+ ),+ JSON.stringify(actions),+ ],+ 'testKey'+ )+ manager['onHogFlowsReloaded'](teamId1, [flow.id])++ const loaded = await manager.getHogFlows([flow.id, hogFlows[0].id])++ expect(loaded[flow.id]).toBeNull()+ expect(loaded[hogFlows[0].id]?.id).toBe(hogFlows[0].id)+ }+ )})})diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.tsindex e4bb9d9ece3f..d5c3baf2461f 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts@@ -202,7 +202,18 @@ export class HogFlowManagerService {}}}- this.mergeEncryptedInputs(item)+ try {+ this.mergeEncryptedInputs(item)+ } catch (error) {+ // A throw here would fail the batched load for every flow in it. Only this flow is left+ // out, because serving it without its secrets would skip a webhook trigger's auth check.+ logger.warn('[HogFlowManager]', 'Could not merge encrypted inputs - flow is left out', {+ hogFlowId: item.id,+ error: error instanceof Error ? error.message : 'Unknown error',+ })+ captureException(error)+ return acc+ }acc[item.id] = itemreturn acc}, {})diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.tsindex 79aa6c4a11f3..c22bc248ae0b 100644--- a/nodejs/src/cdp/types.ts+++ b/nodejs/src/cdp/types.ts@@ -298,11 +298,12 @@ export type AppMetricType = MinimalAppMetric & {* Where an event stopped, for a record on a dead-letter topic.** `filter` and `inputs` are the two failures the pipeline expects and handles per function.- * `parse` and `process` are the ones it does not: a message it cannot read, and anything else- * that throws while an event is being turned into invocations. Those two exist so an unanticipated- * bug parks the event rather than stalling the partition it arrived on.+ * `parse`, `lookup` and `process` are the ones it does not: a message it cannot read, a team, hog+ * function or hog flow lookup that failed with an error that is not retriable, and anything else+ * that throws while the pipeline turns an event into invocations. Those three park a lone+ * unanticipated failure rather than stalling the partition it arrived on.*/-export type DeadLetterStep = 'parse' | 'filter' | 'inputs' | 'process'+export type DeadLetterStep = 'parse' | 'lookup' | 'filter' | 'inputs' | 'process'export type InvocationBuildFailure = {sourceId: stringdiff --git a/nodejs/src/common/utils/team-manager.test.ts b/nodejs/src/common/utils/team-manager.test.tsindex b71599f89934..03e02d75bc6f 100644--- a/nodejs/src/common/utils/team-manager.test.ts+++ b/nodejs/src/common/utils/team-manager.test.ts@@ -274,16 +274,20 @@ describe('TeamManager()', () => {})describe('hasAvailableFeature()', () => {- it('returns false by default', async () => {- await updateOrganizationAvailableFeatures(postgres, organizationId, [])- const result = await teamManager.hasAvailableFeature(teamId, 'data_pipelines')- expect(result).toBe(false)- })-- it('returns false if the available features does not exist', async () => {- await updateOrganizationAvailableFeatures(postgres, organizationId, [- { key: 'not_data_pipelines', name: 'Feature 1' },- ])+ it.each<{ name: string; features: ({ key: string; name: string } | null)[] }>([+ { name: 'by default', features: [] },+ {+ name: 'if the available features does not exist',+ features: [{ key: 'not_data_pipelines', name: 'Feature 1' }],+ },+ // A malformed row must load, because a throw fails the batched lookup for every team in it.+ { name: 'if the available features holds a null entry', features: [null] },+ ])('returns false $name', async ({ features }) => {+ await updateOrganizationAvailableFeatures(+ postgres,+ organizationId,+ features as { key: string; name: string }[]+ )const result = await teamManager.hasAvailableFeature(teamId, 'data_pipelines')expect(result).toBe(false)})diff --git a/nodejs/src/common/utils/team-manager.ts b/nodejs/src/common/utils/team-manager.tsindex f8009e162ffb..a4885ca03fff 100644--- a/nodejs/src/common/utils/team-manager.ts+++ b/nodejs/src/common/utils/team-manager.ts@@ -7,7 +7,18 @@ import { logger } from './logger'import { captureTeamEvent } from './posthog'type RawTeam = Omit<Team, 'available_features'> & {- available_product_features: { key: string; name: string }[]+ available_product_features: unknown+}++// One team's row must not throw, because a throw fails the batched lookup for every team in it. The+// column is a jsonb array, so an entry can be null. An entry without a string key is skipped.+function availableFeatures(raw: unknown): OrganizationAvailableFeature[] {+ if (!Array.isArray(raw)) {+ return []+ }+ return raw.flatMap((feature) =>+ typeof feature?.key === 'string' ? [feature.key as OrganizationAvailableFeature] : []+ )}// Postgres stores team ids in int4 columns. A query that compares one of those columns to a@@ -210,7 +221,7 @@ export class TeamManager {teamPartial.drop_events_older_than_seconds !== null? Number(teamPartial.drop_events_older_than_seconds): null,- available_features: available_product_features?.map((f) => f.key as OrganizationAvailableFeature) || [],+ available_features: availableFeatures(available_product_features),}resultRecord[row.id] = teamresultRecord[row.api_token] = team