Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): back off longer and with jitter on capacity errors (#121836)

Nick Best21f7456b3834
diff --git a/nodejs/src/common/personhog/grpc-retry.test.ts b/nodejs/src/common/personhog/grpc-retry.test.ts
index 0f2439b9659e..3c0b86973c6a 100644
--- a/nodejs/src/common/personhog/grpc-retry.test.ts
+++ b/nodejs/src/common/personhog/grpc-retry.test.ts
@@ -109,6 +109,36 @@ describe('withRetry', () => {
expect(callCount).toBe(3)
})
+ it.each([
+ ['Unavailable', Code.Unavailable, 50],
+ ['ResourceExhausted', Code.ResourceExhausted, 250],
+ ])('waits the %s backoff before the first retry', async (_name, code, firstDelayMs) => {
+ jest.useFakeTimers()
+ const random = jest.spyOn(Math, 'random').mockReturnValue(0)
+ try {
+ let callCount = 0
+ const result = withRetry(
+ () => {
+ callCount++
+ if (callCount === 1) {
+ throw new ConnectError('transient', code)
+ }
+ return Promise.resolve('ok')
+ },
+ 'test-client',
+ 'test-method'
+ )
+ await jest.advanceTimersByTimeAsync(firstDelayMs - 1)
+ expect(callCount).toBe(1)
+ await jest.advanceTimersByTimeAsync(1)
+ await expect(result).resolves.toBe('ok')
+ expect(callCount).toBe(2)
+ } finally {
+ random.mockRestore()
+ jest.useRealTimers()
+ }
+ })
+
it('respects custom maxRetries', async () => {
let callCount = 0
await expect(
diff --git a/nodejs/src/common/personhog/grpc-retry.ts b/nodejs/src/common/personhog/grpc-retry.ts
index d5a8da3fc33c..23671ecd8d21 100644
--- a/nodejs/src/common/personhog/grpc-retry.ts
+++ b/nodejs/src/common/personhog/grpc-retry.ts
@@ -15,6 +15,11 @@ const RETRYABLE_CODES = new Set([
Code.Canceled,
])
+// A server sheds with RESOURCE_EXHAUSTED when it is at capacity: identity's
+// load shed and the leader's full dirty index both do. A fast retry only adds
+// load to a saturated service, so these retries wait longer.
+const CAPACITY_INITIAL_DELAY_MS = 250
+
function isRetryable(error: unknown): error is ConnectError {
return error instanceof ConnectError && RETRYABLE_CODES.has(error.code)
}
@@ -39,6 +44,17 @@ function sleep(ms: number): Promise<void> {
*/
export const TRANSPORT_MAX_RETRIES = 2
+export const TRANSPORT_MAX_BACKOFF_MS = Array.from(
+ { length: TRANSPORT_MAX_RETRIES },
+ (_, attempt) => 2 * CAPACITY_INITIAL_DELAY_MS * Math.pow(2, attempt)
+).reduce((total, delay) => total + delay, 0)
+
+function backoffMs(error: ConnectError, attempt: number, initialDelayMs: number): number {
+ const base = error.code === Code.ResourceExhausted ? CAPACITY_INITIAL_DELAY_MS : initialDelayMs
+ const delay = base * Math.pow(2, attempt)
+ return delay + Math.random() * delay
+}
+
/**
* Retry a function with exponential backoff on transient gRPC errors.
* Non-transient errors are thrown immediately.
@@ -74,7 +90,7 @@ export async function withRetry<T>(
maxRetries,
error: String(error),
})
- await sleep(initialDelayMs * Math.pow(2, attempt))
+ await sleep(backoffMs(error, attempt, initialDelayMs))
}
}
throw lastError
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
index 97c35f066402..58b998c0000e 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
@@ -2667,7 +2667,8 @@ describe('PersonhogPersonsStore', () => {
() => 'acked' as const,
(error: Error) => error.message
)
- await jest.advanceTimersByTimeAsync(10_000)
+ // One round past the budget, which rounds up to a whole round.
+ await jest.advanceTimersByTimeAsync((store as any).flushWaitBudgetMs + 1_000)
expect(await settled).toMatch(/flush cannot complete/)
expect((store as any).entries.get('1:7').segments).toHaveLength(1)
} finally {
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
index 52ee0e2db599..607cdd7ec044 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
@@ -3,7 +3,7 @@ import { DateTime } from 'luxon'
import pLimit from 'p-limit'
import { Counter } from 'prom-client'
-import { TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'
+import { TRANSPORT_MAX_BACKOFF_MS, TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'
import {
DistinctIdKey,
SEMANTIC_REFUSAL_METADATA_KEY,
@@ -290,7 +290,8 @@ export class PersonhogPersonsStore implements PersonsStore {
// rather than chosen so the two cannot drift apart; a holder writing
// a deeper lane can still exhaust it, and that flush fails to
// redelivery instead of acking over unwritten ops.
- this.flushWaitBudgetMs = (TRANSPORT_MAX_RETRIES + 1) * this.options.requestTimeoutMs + FLUSH_WAIT_ROUND_MS
+ this.flushWaitBudgetMs =
+ (TRANSPORT_MAX_RETRIES + 1) * this.options.requestTimeoutMs + TRANSPORT_MAX_BACKOFF_MS + FLUSH_WAIT_ROUND_MS
}
forBatch(batchId: number): PersonsStoreForBatch {