Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.6 KB
Raw Blame
import { Code, ConnectError } from '@connectrpc/connect'

import { logger } from '~/common/utils/logger'

import {
    errorClassLabel,
    personhogHandledVerdictsTotal,
    personhogRetriesTotal,
    personhogTerminalErrorsTotal,
} from './metrics'

const RETRYABLE_CODES = new Set([
    Code.Unavailable,
    Code.DeadlineExceeded,
    Code.ResourceExhausted,
    Code.Aborted,
    Code.Internal,
    Code.Unknown,
    // No caller passes an abort signal, so Canceled is always a stream the peer reset, as a pod leaving does.
    Code.Canceled,
])

// A server sheds with RESOURCE_EXHAUSTED when it is at capacity: identity's
// load shed and the leader's full dirty index both do. A fast retry only adds
// load to a saturated service, so these retries wait longer.
const CAPACITY_INITIAL_DELAY_MS = 250

function isRetryable(error: unknown): error is ConnectError {
    return error instanceof ConnectError && RETRYABLE_CODES.has(error.code)
}

/**
 * A server answer the caller resolves on its own path rather than a failure:
 * the persons store writes a merged-away person's ops to its survivor. The
 * error class declares the verdict so this layer can keep it off the
 * terminal-error series an alert pages on. Only the recovered cases qualify;
 * a fence or a size refusal still leaves the request unfulfilled here.
 */
export interface HandledVerdict {
    readonly handledVerdict: string
}

function handledVerdictOf(error: unknown): string | undefined {
    const verdict = (error as Partial<HandledVerdict> | null)?.handledVerdict
    return typeof verdict === 'string' ? verdict : undefined
}

/**
 * Tag a terminal transient error as retriable so outer retry layers (pipeline
 * chunk retry, LazyLoader loaderRetry) can keep retrying after the client-level
 * budget here is exhausted. Non-transient errors are left untagged so
 * unexpected failures surface loudly instead of being absorbed.
 */
function tagRetriable(error: ConnectError): ConnectError & { isRetriable: true } {
    return Object.assign(error, { isRetriable: true as const })
}

function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

/**
 * Retries beyond the first attempt. Exported so a caller that must wait out
 * another caller's full transport budget can derive its bound from this.
 */
export const TRANSPORT_MAX_RETRIES = 2

export const TRANSPORT_MAX_BACKOFF_MS = Array.from(
    { length: TRANSPORT_MAX_RETRIES },
    (_, attempt) => 2 * CAPACITY_INITIAL_DELAY_MS * Math.pow(2, attempt)
).reduce((total, delay) => total + delay, 0)

function backoffMs(error: ConnectError, attempt: number, initialDelayMs: number): number {
    const base = error.code === Code.ResourceExhausted ? CAPACITY_INITIAL_DELAY_MS : initialDelayMs
    const delay = base * Math.pow(2, attempt)
    return delay + Math.random() * delay
}

/**
 * Retry a function with exponential backoff on transient gRPC errors.
 * Non-transient errors are thrown immediately.
 *
 * Emits `personhog_retries_total` on each retried attempt and
 * `personhog_terminal_errors_total` when retries are exhausted or the
 * error is non-retryable — both tagged with method + client + error_type
 * so they align with `personhog_errors_total` from timedGrpc. A
 * HandledVerdict goes to `personhog_handled_verdicts_total` instead.
 */
export async function withRetry<T>(
    fn: () => Promise<T>,
    client: string,
    method: string,
    maxRetries: number = TRANSPORT_MAX_RETRIES,
    initialDelayMs: number = 50
): Promise<T> {
    let lastError: unknown
    for (let attempt = 0; attempt <= maxRetries; attempt++) {
        try {
            return await fn()
        } catch (error) {
            lastError = error
            const verdict = handledVerdictOf(error)
            if (verdict !== undefined) {
                personhogHandledVerdictsTotal.inc({ method, client, verdict })
                throw error
            }
            if (!isRetryable(error) || attempt === maxRetries) {
                personhogTerminalErrorsTotal.inc({ method, client, error_type: errorClassLabel(error) })
                logger.error(`[${client}/${method}] gRPC call failed`, {
                    error: String(error),
                })
                throw isRetryable(error) ? tagRetriable(error) : error
            }
            personhogRetriesTotal.inc({ method, client, error_type: errorClassLabel(error) })
            logger.warn(`[${client}/${method}] Retryable gRPC error, retrying`, {
                attempt: attempt + 1,
                maxRetries,
                error: String(error),
            })
            await sleep(backoffMs(error, attempt, initialDelayMs))
        }
    }
    throw lastError
}