:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(personhog): back off longer and with jitter on capacity errors (#121836)
Nick Best21f7456b3834
diff --git a/nodejs/src/common/personhog/grpc-retry.test.ts b/nodejs/src/common/personhog/grpc-retry.test.tsindex 0f2439b9659e..3c0b86973c6a 100644--- a/nodejs/src/common/personhog/grpc-retry.test.ts+++ b/nodejs/src/common/personhog/grpc-retry.test.ts@@ -109,6 +109,36 @@ describe('withRetry', () => {expect(callCount).toBe(3)})+ it.each([+ ['Unavailable', Code.Unavailable, 50],+ ['ResourceExhausted', Code.ResourceExhausted, 250],+ ])('waits the %s backoff before the first retry', async (_name, code, firstDelayMs) => {+ jest.useFakeTimers()+ const random = jest.spyOn(Math, 'random').mockReturnValue(0)+ try {+ let callCount = 0+ const result = withRetry(+ () => {+ callCount+++ if (callCount === 1) {+ throw new ConnectError('transient', code)+ }+ return Promise.resolve('ok')+ },+ 'test-client',+ 'test-method'+ )+ await jest.advanceTimersByTimeAsync(firstDelayMs - 1)+ expect(callCount).toBe(1)+ await jest.advanceTimersByTimeAsync(1)+ await expect(result).resolves.toBe('ok')+ expect(callCount).toBe(2)+ } finally {+ random.mockRestore()+ jest.useRealTimers()+ }+ })+it('respects custom maxRetries', async () => {let callCount = 0await expect(diff --git a/nodejs/src/common/personhog/grpc-retry.ts b/nodejs/src/common/personhog/grpc-retry.tsindex d5a8da3fc33c..23671ecd8d21 100644--- a/nodejs/src/common/personhog/grpc-retry.ts+++ b/nodejs/src/common/personhog/grpc-retry.ts@@ -15,6 +15,11 @@ const RETRYABLE_CODES = new Set([Code.Canceled,])+// A server sheds with RESOURCE_EXHAUSTED when it is at capacity: identity's+// load shed and the leader's full dirty index both do. A fast retry only adds+// load to a saturated service, so these retries wait longer.+const CAPACITY_INITIAL_DELAY_MS = 250+function isRetryable(error: unknown): error is ConnectError {return error instanceof ConnectError && RETRYABLE_CODES.has(error.code)}@@ -39,6 +44,17 @@ function sleep(ms: number): Promise<void> {*/export const TRANSPORT_MAX_RETRIES = 2+export const TRANSPORT_MAX_BACKOFF_MS = Array.from(+ { length: TRANSPORT_MAX_RETRIES },+ (_, attempt) => 2 * CAPACITY_INITIAL_DELAY_MS * Math.pow(2, attempt)+).reduce((total, delay) => total + delay, 0)++function backoffMs(error: ConnectError, attempt: number, initialDelayMs: number): number {+ const base = error.code === Code.ResourceExhausted ? CAPACITY_INITIAL_DELAY_MS : initialDelayMs+ const delay = base * Math.pow(2, attempt)+ return delay + Math.random() * delay+}+/*** Retry a function with exponential backoff on transient gRPC errors.* Non-transient errors are thrown immediately.@@ -74,7 +90,7 @@ export async function withRetry<T>(maxRetries,error: String(error),})- await sleep(initialDelayMs * Math.pow(2, attempt))+ await sleep(backoffMs(error, attempt, initialDelayMs))}}throw lastErrordiff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.tsindex 97c35f066402..58b998c0000e 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts@@ -2667,7 +2667,8 @@ describe('PersonhogPersonsStore', () => {() => 'acked' as const,(error: Error) => error.message)- await jest.advanceTimersByTimeAsync(10_000)+ // One round past the budget, which rounds up to a whole round.+ await jest.advanceTimersByTimeAsync((store as any).flushWaitBudgetMs + 1_000)expect(await settled).toMatch(/flush cannot complete/)expect((store as any).entries.get('1:7').segments).toHaveLength(1)} finally {diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.tsindex 52ee0e2db599..607cdd7ec044 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts@@ -3,7 +3,7 @@ import { DateTime } from 'luxon'import pLimit from 'p-limit'import { Counter } from 'prom-client'-import { TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'+import { TRANSPORT_MAX_BACKOFF_MS, TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'import {DistinctIdKey,SEMANTIC_REFUSAL_METADATA_KEY,@@ -290,7 +290,8 @@ export class PersonhogPersonsStore implements PersonsStore {// rather than chosen so the two cannot drift apart; a holder writing// a deeper lane can still exhaust it, and that flush fails to// redelivery instead of acking over unwritten ops.- this.flushWaitBudgetMs = (TRANSPORT_MAX_RETRIES + 1) * this.options.requestTimeoutMs + FLUSH_WAIT_ROUND_MS+ this.flushWaitBudgetMs =+ (TRANSPORT_MAX_RETRIES + 1) * this.options.requestTimeoutMs + TRANSPORT_MAX_BACKOFF_MS + FLUSH_WAIT_ROUND_MS}forBatch(batchId: number): PersonsStoreForBatch {