Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): stop counting a merged-away person update as a terminal personhog error (#126679)

Nick Bestf4091c31513c
diff --git a/nodejs/src/common/personhog/grpc-retry.test.ts b/nodejs/src/common/personhog/grpc-retry.test.ts
index 3c0b86973c6a..ad12c1b82541 100644
--- a/nodejs/src/common/personhog/grpc-retry.test.ts
+++ b/nodejs/src/common/personhog/grpc-retry.test.ts
@@ -1,10 +1,62 @@
import { Code, ConnectError } from '@connectrpc/connect'
-import { withRetry } from './grpc-retry'
+import { HandledVerdict, withRetry } from './grpc-retry'
+import { personhogHandledVerdictsTotal, personhogTerminalErrorsTotal } from './metrics'
jest.mock('~/common/utils/logger')
+class SurvivorWriteVerdict extends Error implements HandledVerdict {
+ readonly handledVerdict = 'person_not_found'
+}
+
+class UnexpectedStoreError extends Error {}
+
+async function counterValues(counter: typeof personhogTerminalErrorsTotal | typeof personhogHandledVerdictsTotal) {
+ return (await counter.get()).values.map(({ labels, value }) => ({ ...labels, value }))
+}
+
describe('withRetry', () => {
+ beforeEach(() => {
+ personhogTerminalErrorsTotal.reset()
+ personhogHandledVerdictsTotal.reset()
+ })
+
+ it('files a handled verdict under verdicts, not terminal errors, and rethrows it unchanged', async () => {
+ const verdict = new SurvivorWriteVerdict('merged away')
+ let callCount = 0
+ const error: unknown = await withRetry(
+ () => {
+ callCount++
+ throw verdict
+ },
+ 'test-client',
+ 'updatePersonProperties'
+ ).catch((e: unknown) => e)
+
+ expect(error).toBe(verdict)
+ expect(error).not.toHaveProperty('isRetriable')
+ expect(callCount).toBe(1)
+ expect(await counterValues(personhogTerminalErrorsTotal)).toEqual([])
+ expect(await counterValues(personhogHandledVerdictsTotal)).toEqual([
+ { method: 'updatePersonProperties', client: 'test-client', verdict: 'person_not_found', value: 1 },
+ ])
+ })
+
+ it('labels a terminal non-gRPC error with its class so it stays visible on its own series', async () => {
+ await withRetry(
+ () => {
+ throw new UnexpectedStoreError('boom')
+ },
+ 'test-client',
+ 'updatePersonProperties'
+ ).catch(() => undefined)
+
+ expect(await counterValues(personhogHandledVerdictsTotal)).toEqual([])
+ expect(await counterValues(personhogTerminalErrorsTotal)).toEqual([
+ { method: 'updatePersonProperties', client: 'test-client', error_type: 'UnexpectedStoreError', value: 1 },
+ ])
+ })
+
it('returns the result on first success', async () => {
const result = await withRetry(() => Promise.resolve('ok'), 'test-client', 'test-method')
expect(result).toBe('ok')
diff --git a/nodejs/src/common/personhog/grpc-retry.ts b/nodejs/src/common/personhog/grpc-retry.ts
index 23671ecd8d21..dca12c581291 100644
--- a/nodejs/src/common/personhog/grpc-retry.ts
+++ b/nodejs/src/common/personhog/grpc-retry.ts
@@ -2,7 +2,12 @@ import { Code, ConnectError } from '@connectrpc/connect'
import { logger } from '~/common/utils/logger'
-import { grpcErrorType, personhogRetriesTotal, personhogTerminalErrorsTotal } from './metrics'
+import {
+ errorClassLabel,
+ personhogHandledVerdictsTotal,
+ personhogRetriesTotal,
+ personhogTerminalErrorsTotal,
+} from './metrics'
const RETRYABLE_CODES = new Set([
Code.Unavailable,
@@ -24,6 +29,22 @@ function isRetryable(error: unknown): error is ConnectError {
return error instanceof ConnectError && RETRYABLE_CODES.has(error.code)
}
+/**
+ * A server answer the caller resolves on its own path rather than a failure:
+ * the persons store writes a merged-away person's ops to its survivor. The
+ * error class declares the verdict so this layer can keep it off the
+ * terminal-error series an alert pages on. Only the recovered cases qualify;
+ * a fence or a size refusal still leaves the request unfulfilled here.
+ */
+export interface HandledVerdict {
+ readonly handledVerdict: string
+}
+
+function handledVerdictOf(error: unknown): string | undefined {
+ const verdict = (error as Partial<HandledVerdict> | null)?.handledVerdict
+ return typeof verdict === 'string' ? verdict : undefined
+}
+
/**
* Tag a terminal transient error as retriable so outer retry layers (pipeline
* chunk retry, LazyLoader loaderRetry) can keep retrying after the client-level
@@ -62,7 +83,8 @@ function backoffMs(error: ConnectError, attempt: number, initialDelayMs: number)
* Emits `personhog_retries_total` on each retried attempt and
* `personhog_terminal_errors_total` when retries are exhausted or the
* error is non-retryable — both tagged with method + client + error_type
- * so they align with `personhog_errors_total` from timedGrpc.
+ * so they align with `personhog_errors_total` from timedGrpc. A
+ * HandledVerdict goes to `personhog_handled_verdicts_total` instead.
*/
export async function withRetry<T>(
fn: () => Promise<T>,
@@ -77,14 +99,19 @@ export async function withRetry<T>(
return await fn()
} catch (error) {
lastError = error
+ const verdict = handledVerdictOf(error)
+ if (verdict !== undefined) {
+ personhogHandledVerdictsTotal.inc({ method, client, verdict })
+ throw error
+ }
if (!isRetryable(error) || attempt === maxRetries) {
- personhogTerminalErrorsTotal.inc({ method, client, error_type: grpcErrorType(error) })
+ personhogTerminalErrorsTotal.inc({ method, client, error_type: errorClassLabel(error) })
logger.error(`[${client}/${method}] gRPC call failed`, {
error: String(error),
})
throw isRetryable(error) ? tagRetriable(error) : error
}
- personhogRetriesTotal.inc({ method, client, error_type: grpcErrorType(error) })
+ personhogRetriesTotal.inc({ method, client, error_type: errorClassLabel(error) })
logger.warn(`[${client}/${method}] Retryable gRPC error, retrying`, {
attempt: attempt + 1,
maxRetries,
diff --git a/nodejs/src/common/personhog/metrics.ts b/nodejs/src/common/personhog/metrics.ts
index 9e6f50f27828..96741bf5815a 100644
--- a/nodejs/src/common/personhog/metrics.ts
+++ b/nodejs/src/common/personhog/metrics.ts
@@ -63,6 +63,12 @@ export const personhogTerminalErrorsTotal = new Counter({
labelNames: ['method', 'client', 'error_type'] as const,
})
+export const personhogHandledVerdictsTotal = new Counter({
+ name: 'personhog_handled_verdicts_total',
+ help: 'PersonHog answers the caller resolves on its own path (a person merged away is written to its survivor); not failures',
+ labelNames: ['method', 'client', 'verdict'] as const,
+})
+
export function grpcErrorType(error: unknown): string {
if (error instanceof ConnectError) {
return Code[error.code] ?? 'unknown'
@@ -101,7 +107,7 @@ export async function timedGrpc<T>(clientLabel: string, method: string, fn: () =
try {
return await fn()
} catch (error) {
- personhogErrorsTotal.inc({ method, client: clientLabel, error_type: grpcErrorType(error) })
+ personhogErrorsTotal.inc({ method, client: clientLabel, error_type: errorClassLabel(error) })
throw error
} finally {
end()
diff --git a/nodejs/src/common/personhog/persons.test.ts b/nodejs/src/common/personhog/persons.test.ts
index 5a5f0a70c60f..5722560cb2f8 100644
--- a/nodejs/src/common/personhog/persons.test.ts
+++ b/nodejs/src/common/personhog/persons.test.ts
@@ -11,7 +11,12 @@ import type {
import { parseJSON } from '~/common/utils/json-parse'
import { NoRowsUpdatedError } from '~/common/utils/utils'
-import { PersonHogPersonOperations, PersonhogFencedError, PersonhogPropertiesSizeError } from './persons'
+import {
+ PersonHogPersonOperations,
+ PersonhogFencedError,
+ PersonhogPersonNotFoundError,
+ PersonhogPropertiesSizeError,
+} from './persons'
const textEncoder = new TextEncoder()
@@ -279,7 +284,11 @@ describe('PersonHogPersonOperations', () => {
throw new ConnectError('person 42 not found', Code.NotFound)
},
})
- await expect(notFoundOps.updatePersonProperties(foldedUpdate)).rejects.toBeInstanceOf(NoRowsUpdatedError)
+ const notFoundError = await notFoundOps.updatePersonProperties(foldedUpdate).catch((e) => e)
+ expect(notFoundError).toBeInstanceOf(PersonhogPersonNotFoundError)
+ expect(notFoundError).toBeInstanceOf(NoRowsUpdatedError)
+ expect(notFoundError.handledVerdict).toBe('person_not_found')
+ expect(notFoundError.personId).toBe('42')
const { ops: tooBigOps } = createOperations({
updatePersonProperties: () => {
diff --git a/nodejs/src/common/personhog/persons.ts b/nodejs/src/common/personhog/persons.ts
index 5b21f029d186..0c8dd22a0f3d 100644
--- a/nodejs/src/common/personhog/persons.ts
+++ b/nodejs/src/common/personhog/persons.ts
@@ -17,6 +17,7 @@ import { Properties } from '~/plugin-scaffold'
import { InternalPerson } from '~/types'
import { epochMsToDateTime, eventualReadOptions, parseJsonBytes, strongReadOptions } from './client'
+import type { HandledVerdict } from './grpc-retry'
const textEncoder = new TextEncoder()
@@ -43,6 +44,24 @@ export class PersonhogPropertiesSizeError extends Error {
}
}
+/**
+ * The leader knows no such person: deleted, or merged into another. The
+ * persons store redirects the ops to the survivor, so the retry layer files
+ * this under handled verdicts rather than terminal errors. Still a
+ * NoRowsUpdatedError for the callers that branch on that.
+ */
+export class PersonhogPersonNotFoundError extends NoRowsUpdatedError implements HandledVerdict {
+ readonly handledVerdict = 'person_not_found'
+
+ constructor(
+ message: string,
+ public readonly personId: string
+ ) {
+ super(message)
+ this.name = 'PersonhogPersonNotFoundError'
+ }
+}
+
/**
* The leader refused a write because a lifecycle operation holds the
* person; the holder's op id and creator travel for attribution.
@@ -252,7 +271,10 @@ export class PersonHogPersonOperations {
// property-size ceiling. Everything else propagates untouched.
if (error instanceof ConnectError) {
if (error.code === Code.NotFound) {
- throw new NoRowsUpdatedError(`Person ${update.personId} not found by personhog (deleted or merged)`)
+ throw new PersonhogPersonNotFoundError(
+ `Person ${update.personId} not found by personhog (deleted or merged)`,
+ update.personId
+ )
}
if (error.code === Code.InvalidArgument && error.rawMessage.includes('size limit')) {
throw new PersonhogPropertiesSizeError(error.rawMessage, update.teamId, update.personId)