:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* # Chapter 11: Retry Logic
*
* Retry is a per-step option: pass `{ retry }` to `pipe()` or `pipeChunk()` to
* wrap that single step with automatic retry logic. This handles transient
* failures like network hiccups, database connection timeouts, or rate limiting.
*
* ```
* .pipe(step, { retry: { tries: 3, sleepMs: 100 } })
* .pipeChunk(step, { retry: { tries: 3, sleepMs: 100 } })
* ```
*
* ## Key Concepts
*
* - **Retriable errors**: Errors marked with `isRetriable: true` are retried
* - **Non-retriable errors**: Errors marked with `isRetriable: false` go
* directly to DLQ without retrying
* - **Unknown errors**: Errors with no `isRetriable` property are retried, then
* rethrown once retries are exhausted (crashes the process - appropriate for
* unexpected failures)
* - **Exponential backoff**: The delay doubles between retries
* - **Defaults**: 3 tries, 100ms initial sleep
*
* `RetryOptions` is `{ name?, tries?, sleepMs? }`. The `name` labels the
* `ingestion_pipeline_retry_attempts` metric and defaults to the step name.
*
* ## Why Retries Wrap a Single Step
*
* Retry always wraps exactly one step, never a sequence. Retrying a sequence
* would re-run steps that already succeeded, which breaks idempotency: if
* step A writes to a database and step B (a later step) fails, retrying the
* whole block would run step A's write a second time. Keeping retry scoped to
* a single step means only the failing operation is repeated. Compose retries
* by attaching `{ retry }` to each step that needs it, not by wrapping a
* multi-step block.
*
* ## Error Classification
*
* Errors carry an `isRetriable` property to indicate whether they should be
* retried. This distinguishes:
* - Transient failures (network, timeout) - should retry
* - Permanent failures (validation, permission) - should not retry
*/
import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'
import { newChunkPipelineBuilder, newPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { isDlqResult, isOkResult, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
class RetriableError extends Error {
isRetriable = true
}
class NonRetriableError extends Error {
isRetriable = false
}
describe('Single-Step Retries', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
/**
* Retriable errors (those with `isRetriable: true`) are automatically
* retried up to the configured number of times. The step succeeds once
* the transient condition clears.
*/
it('retriable errors are retried until the step succeeds', async () => {
let attempts = 0
interface Input {
value: string
}
function createFlakyStep(): ProcessingStep<Input, { value: string }> {
return function flakyStep(input) {
attempts++
if (attempts < 3) {
throw new RetriableError('Temporary failure')
}
return Promise.resolve(ok({ value: input.value.toUpperCase() }))
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createFlakyStep(), { retry: { tries: 5, sleepMs: 100 } })
.build()
const resultPromise = pipeline.process(createOkContext({ value: 'hello' }, {}))
await jest.advanceTimersByTimeAsync(300) // 100ms + 200ms for two retries (exponential backoff)
const result = await resultPromise
expect(attempts).toBe(3) // Failed twice, succeeded on third
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
expect(result.result.value.value).toBe('HELLO')
}
})
/**
* Non-retriable errors (those with `isRetriable: false`) go directly
* to DLQ without any retry attempts.
*/
it('non-retriable errors go directly to DLQ', async () => {
let attempts = 0
interface Input {
data: string
}
function createValidationStep(): ProcessingStep<Input, { data: string }> {
return function validationStep(_input) {
attempts++
throw new NonRetriableError('Validation failed - invalid format')
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createValidationStep(), { retry: { tries: 5, sleepMs: 100 } })
.build()
// No timer advancement needed - non-retriable errors don't wait
const result = await pipeline.process(createOkContext({ data: 'bad-data' }, {}))
expect(attempts).toBe(1) // Only tried once
expect(isDlqResult(result.result)).toBe(true)
if (isDlqResult(result.result)) {
expect(result.result.reason).toContain('non-retriable')
}
})
/**
* After exhausting the maximum number of retries, a retriable error is
* rethrown. This is a fatal error - the process should report it and crash
* rather than continue in a broken state.
*/
it('after max retries, the error is rethrown', async () => {
let attempts = 0
let caughtError: Error | null = null
interface Input {
value: string
}
function createAlwaysFailsStep(): ProcessingStep<Input, { value: string }> {
return function alwaysFailsStep(_input) {
attempts++
throw new RetriableError(`Attempt ${attempts} failed`)
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createAlwaysFailsStep(), { retry: { tries: 3, sleepMs: 100 } })
.build()
const resultPromise = pipeline.process(createOkContext({ value: 'data' }, {})).catch((e) => {
caughtError = e
})
await jest.advanceTimersByTimeAsync(300) // 100ms + 200ms for retries
await resultPromise
expect(attempts).toBe(3)
expect(caughtError).not.toBeNull()
expect(caughtError!.message).toBe('Attempt 3 failed')
})
/**
* Errors without an `isRetriable` property are retried (in case they are
* transient), then rethrown once retries are exhausted - never sent to DLQ.
* This preserves normal crash behavior for unexpected errors.
*/
it('errors without isRetriable are retried then rethrown', async () => {
let attempts = 0
let caughtError: Error | null = null
interface Input {
value: string
}
function createUnexpectedErrorStep(): ProcessingStep<Input, { value: string }> {
return function unexpectedErrorStep(_input) {
attempts++
throw new Error('Unexpected internal error')
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createUnexpectedErrorStep(), { retry: { tries: 3, sleepMs: 100 } })
.build()
const resultPromise = pipeline.process(createOkContext({ value: 'data' }, {})).catch((e) => {
caughtError = e
})
await jest.advanceTimersByTimeAsync(300) // 100ms + 200ms for retries
await resultPromise
expect(attempts).toBe(3) // Retried, not short-circuited like a non-retriable error
expect(caughtError).not.toBeNull()
expect(caughtError!.message).toBe('Unexpected internal error')
})
})
describe('Retry Configuration', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
/**
* `tries` caps the number of attempts and `sleepMs` sets the initial delay.
* Subsequent retries use exponential backoff (the delay doubles each time).
*/
it('tries and sleepMs control attempts and backoff', async () => {
let attempts = 0
interface Input {
value: string
}
function createSlowlyRecoveringStep(): ProcessingStep<Input, { value: string }> {
return function slowlyRecoveringStep(input) {
attempts++
if (attempts < 4) {
throw new RetriableError('Still recovering')
}
return Promise.resolve(ok({ value: input.value }))
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createSlowlyRecoveringStep(), { retry: { tries: 5, sleepMs: 50 } })
.build()
const resultPromise = pipeline.process(createOkContext({ value: 'data' }, {}))
// First attempt happens immediately
await jest.advanceTimersByTimeAsync(0)
expect(attempts).toBe(1)
// After 50ms (initial sleepMs), second attempt
await jest.advanceTimersByTimeAsync(50)
expect(attempts).toBe(2)
// After 100ms more (50 * 2), third attempt
await jest.advanceTimersByTimeAsync(100)
expect(attempts).toBe(3)
// After 200ms more (100 * 2), fourth attempt succeeds
await jest.advanceTimersByTimeAsync(200)
expect(attempts).toBe(4)
await resultPromise
})
/**
* Omitting `tries`/`sleepMs` uses the defaults: 3 tries, 100ms initial sleep.
*/
it('defaults to 3 tries and 100ms sleep', async () => {
let attempts = 0
interface Input {
value: string
}
function createFlakyStep(): ProcessingStep<Input, { value: string }> {
return function flakyStep(input) {
attempts++
if (attempts < 3) {
throw new RetriableError('retry')
}
return Promise.resolve(ok({ value: input.value }))
}
}
const pipeline = newPipelineBuilder<Input>().pipe(createFlakyStep(), { retry: {} }).build()
const resultPromise = pipeline.process(createOkContext({ value: 'data' }, {}))
await jest.advanceTimersByTimeAsync(300) // 100ms + 200ms for two retries (default sleepMs is 100)
const result = await resultPromise
expect(isOkResult(result.result)).toBe(true)
expect(attempts).toBe(3)
})
})
describe('Chunk Retries', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
/**
* `pipeChunk(step, { retry })` retries the whole chunk step on retriable
* errors, the same way per-item retry works. The chunk is reprocessed as a
* unit until it succeeds or retries are exhausted.
*/
it('pipeChunk retries the chunk step until it succeeds', async () => {
let attempts = 0
interface Event {
id: number
}
function createFlakyChunkStep(): ChunkProcessingStep<Event, Event> {
return function flakyChunkStep(events) {
attempts++
if (attempts < 3) {
throw new RetriableError('Downstream unavailable')
}
return Promise.resolve(events.map((e) => ok(e)))
}
}
const pipeline = newChunkPipelineBuilder<Event>()
.pipeChunk(createFlakyChunkStep(), { retry: { tries: 5, sleepMs: 100 } })
.build()
pipeline.feed([{ id: 1 }, { id: 2 }].map((e) => createOkContext(e, {})))
const resultsPromise = pipeline.next()
await jest.advanceTimersByTimeAsync(300) // 100ms + 200ms for two retries
const results = await resultsPromise
expect(attempts).toBe(3)
expect(results).toHaveLength(2)
expect(results!.every((r) => isOkResult(r.result))).toBe(true)
})
/**
* When a chunk step throws a non-retriable error, every input in the chunk
* is converted to its own DLQ result. The failure is attributed per-input
* so each message is dead-lettered individually.
*/
it('pipeChunk sends one DLQ per input on a non-retriable error', async () => {
interface Event {
id: number
}
function createFailingChunkStep(): ChunkProcessingStep<Event, Event> {
return function failingChunkStep(_events) {
throw new NonRetriableError('Chunk permanently invalid')
}
}
const pipeline = newChunkPipelineBuilder<Event>()
.pipeChunk(createFailingChunkStep(), { retry: { tries: 5, sleepMs: 100 } })
.build()
// Three inputs -> three DLQ results, no retries (non-retriable)
pipeline.feed([{ id: 1 }, { id: 2 }, { id: 3 }].map((e) => createOkContext(e, {})))
const results = await pipeline.next()
expect(results).toHaveLength(3)
expect(results!.every((r) => isDlqResult(r.result))).toBe(true)
})
})