:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): stop counting a merged-away person update as a terminal personhog error (#126679)
Nick Bestf4091c31513c
diff --git a/nodejs/src/common/personhog/grpc-retry.test.ts b/nodejs/src/common/personhog/grpc-retry.test.tsindex 3c0b86973c6a..ad12c1b82541 100644--- a/nodejs/src/common/personhog/grpc-retry.test.ts+++ b/nodejs/src/common/personhog/grpc-retry.test.ts@@ -1,10 +1,62 @@import { Code, ConnectError } from '@connectrpc/connect'-import { withRetry } from './grpc-retry'+import { HandledVerdict, withRetry } from './grpc-retry'+import { personhogHandledVerdictsTotal, personhogTerminalErrorsTotal } from './metrics'jest.mock('~/common/utils/logger')+class SurvivorWriteVerdict extends Error implements HandledVerdict {+ readonly handledVerdict = 'person_not_found'+}++class UnexpectedStoreError extends Error {}++async function counterValues(counter: typeof personhogTerminalErrorsTotal | typeof personhogHandledVerdictsTotal) {+ return (await counter.get()).values.map(({ labels, value }) => ({ ...labels, value }))+}+describe('withRetry', () => {+ beforeEach(() => {+ personhogTerminalErrorsTotal.reset()+ personhogHandledVerdictsTotal.reset()+ })++ it('files a handled verdict under verdicts, not terminal errors, and rethrows it unchanged', async () => {+ const verdict = new SurvivorWriteVerdict('merged away')+ let callCount = 0+ const error: unknown = await withRetry(+ () => {+ callCount+++ throw verdict+ },+ 'test-client',+ 'updatePersonProperties'+ ).catch((e: unknown) => e)++ expect(error).toBe(verdict)+ expect(error).not.toHaveProperty('isRetriable')+ expect(callCount).toBe(1)+ expect(await counterValues(personhogTerminalErrorsTotal)).toEqual([])+ expect(await counterValues(personhogHandledVerdictsTotal)).toEqual([+ { method: 'updatePersonProperties', client: 'test-client', verdict: 'person_not_found', value: 1 },+ ])+ })++ it('labels a terminal non-gRPC error with its class so it stays visible on its own series', async () => {+ await withRetry(+ () => {+ throw new UnexpectedStoreError('boom')+ },+ 'test-client',+ 'updatePersonProperties'+ ).catch(() => undefined)++ expect(await counterValues(personhogHandledVerdictsTotal)).toEqual([])+ expect(await counterValues(personhogTerminalErrorsTotal)).toEqual([+ { method: 'updatePersonProperties', client: 'test-client', error_type: 'UnexpectedStoreError', value: 1 },+ ])+ })+it('returns the result on first success', async () => {const result = await withRetry(() => Promise.resolve('ok'), 'test-client', 'test-method')expect(result).toBe('ok')diff --git a/nodejs/src/common/personhog/grpc-retry.ts b/nodejs/src/common/personhog/grpc-retry.tsindex 23671ecd8d21..dca12c581291 100644--- a/nodejs/src/common/personhog/grpc-retry.ts+++ b/nodejs/src/common/personhog/grpc-retry.ts@@ -2,7 +2,12 @@ import { Code, ConnectError } from '@connectrpc/connect'import { logger } from '~/common/utils/logger'-import { grpcErrorType, personhogRetriesTotal, personhogTerminalErrorsTotal } from './metrics'+import {+ errorClassLabel,+ personhogHandledVerdictsTotal,+ personhogRetriesTotal,+ personhogTerminalErrorsTotal,+} from './metrics'const RETRYABLE_CODES = new Set([Code.Unavailable,@@ -24,6 +29,22 @@ function isRetryable(error: unknown): error is ConnectError {return error instanceof ConnectError && RETRYABLE_CODES.has(error.code)}+/**+ * A server answer the caller resolves on its own path rather than a failure:+ * the persons store writes a merged-away person's ops to its survivor. The+ * error class declares the verdict so this layer can keep it off the+ * terminal-error series an alert pages on. Only the recovered cases qualify;+ * a fence or a size refusal still leaves the request unfulfilled here.+ */+export interface HandledVerdict {+ readonly handledVerdict: string+}++function handledVerdictOf(error: unknown): string | undefined {+ const verdict = (error as Partial<HandledVerdict> | null)?.handledVerdict+ return typeof verdict === 'string' ? verdict : undefined+}+/*** Tag a terminal transient error as retriable so outer retry layers (pipeline* chunk retry, LazyLoader loaderRetry) can keep retrying after the client-level@@ -62,7 +83,8 @@ function backoffMs(error: ConnectError, attempt: number, initialDelayMs: number)* Emits `personhog_retries_total` on each retried attempt and* `personhog_terminal_errors_total` when retries are exhausted or the* error is non-retryable — both tagged with method + client + error_type- * so they align with `personhog_errors_total` from timedGrpc.+ * so they align with `personhog_errors_total` from timedGrpc. A+ * HandledVerdict goes to `personhog_handled_verdicts_total` instead.*/export async function withRetry<T>(fn: () => Promise<T>,@@ -77,14 +99,19 @@ export async function withRetry<T>(return await fn()} catch (error) {lastError = error+ const verdict = handledVerdictOf(error)+ if (verdict !== undefined) {+ personhogHandledVerdictsTotal.inc({ method, client, verdict })+ throw error+ }if (!isRetryable(error) || attempt === maxRetries) {- personhogTerminalErrorsTotal.inc({ method, client, error_type: grpcErrorType(error) })+ personhogTerminalErrorsTotal.inc({ method, client, error_type: errorClassLabel(error) })logger.error(`[${client}/${method}] gRPC call failed`, {error: String(error),})throw isRetryable(error) ? tagRetriable(error) : error}- personhogRetriesTotal.inc({ method, client, error_type: grpcErrorType(error) })+ personhogRetriesTotal.inc({ method, client, error_type: errorClassLabel(error) })logger.warn(`[${client}/${method}] Retryable gRPC error, retrying`, {attempt: attempt + 1,maxRetries,diff --git a/nodejs/src/common/personhog/metrics.ts b/nodejs/src/common/personhog/metrics.tsindex 9e6f50f27828..96741bf5815a 100644--- a/nodejs/src/common/personhog/metrics.ts+++ b/nodejs/src/common/personhog/metrics.ts@@ -63,6 +63,12 @@ export const personhogTerminalErrorsTotal = new Counter({labelNames: ['method', 'client', 'error_type'] as const,})+export const personhogHandledVerdictsTotal = new Counter({+ name: 'personhog_handled_verdicts_total',+ help: 'PersonHog answers the caller resolves on its own path (a person merged away is written to its survivor); not failures',+ labelNames: ['method', 'client', 'verdict'] as const,+})+export function grpcErrorType(error: unknown): string {if (error instanceof ConnectError) {return Code[error.code] ?? 'unknown'@@ -101,7 +107,7 @@ export async function timedGrpc<T>(clientLabel: string, method: string, fn: () =try {return await fn()} catch (error) {- personhogErrorsTotal.inc({ method, client: clientLabel, error_type: grpcErrorType(error) })+ personhogErrorsTotal.inc({ method, client: clientLabel, error_type: errorClassLabel(error) })throw error} finally {end()diff --git a/nodejs/src/common/personhog/persons.test.ts b/nodejs/src/common/personhog/persons.test.tsindex 5a5f0a70c60f..5722560cb2f8 100644--- a/nodejs/src/common/personhog/persons.test.ts+++ b/nodejs/src/common/personhog/persons.test.ts@@ -11,7 +11,12 @@ import type {import { parseJSON } from '~/common/utils/json-parse'import { NoRowsUpdatedError } from '~/common/utils/utils'-import { PersonHogPersonOperations, PersonhogFencedError, PersonhogPropertiesSizeError } from './persons'+import {+ PersonHogPersonOperations,+ PersonhogFencedError,+ PersonhogPersonNotFoundError,+ PersonhogPropertiesSizeError,+} from './persons'const textEncoder = new TextEncoder()@@ -279,7 +284,11 @@ describe('PersonHogPersonOperations', () => {throw new ConnectError('person 42 not found', Code.NotFound)},})- await expect(notFoundOps.updatePersonProperties(foldedUpdate)).rejects.toBeInstanceOf(NoRowsUpdatedError)+ const notFoundError = await notFoundOps.updatePersonProperties(foldedUpdate).catch((e) => e)+ expect(notFoundError).toBeInstanceOf(PersonhogPersonNotFoundError)+ expect(notFoundError).toBeInstanceOf(NoRowsUpdatedError)+ expect(notFoundError.handledVerdict).toBe('person_not_found')+ expect(notFoundError.personId).toBe('42')const { ops: tooBigOps } = createOperations({updatePersonProperties: () => {diff --git a/nodejs/src/common/personhog/persons.ts b/nodejs/src/common/personhog/persons.tsindex 5b21f029d186..0c8dd22a0f3d 100644--- a/nodejs/src/common/personhog/persons.ts+++ b/nodejs/src/common/personhog/persons.ts@@ -17,6 +17,7 @@ import { Properties } from '~/plugin-scaffold'import { InternalPerson } from '~/types'import { epochMsToDateTime, eventualReadOptions, parseJsonBytes, strongReadOptions } from './client'+import type { HandledVerdict } from './grpc-retry'const textEncoder = new TextEncoder()@@ -43,6 +44,24 @@ export class PersonhogPropertiesSizeError extends Error {}}+/**+ * The leader knows no such person: deleted, or merged into another. The+ * persons store redirects the ops to the survivor, so the retry layer files+ * this under handled verdicts rather than terminal errors. Still a+ * NoRowsUpdatedError for the callers that branch on that.+ */+export class PersonhogPersonNotFoundError extends NoRowsUpdatedError implements HandledVerdict {+ readonly handledVerdict = 'person_not_found'++ constructor(+ message: string,+ public readonly personId: string+ ) {+ super(message)+ this.name = 'PersonhogPersonNotFoundError'+ }+}+/*** The leader refused a write because a lifecycle operation holds the* person; the holder's op id and creator travel for attribution.@@ -252,7 +271,10 @@ export class PersonHogPersonOperations {// property-size ceiling. Everything else propagates untouched.if (error instanceof ConnectError) {if (error.code === Code.NotFound) {- throw new NoRowsUpdatedError(`Person ${update.personId} not found by personhog (deleted or merged)`)+ throw new PersonhogPersonNotFoundError(+ `Person ${update.personId} not found by personhog (deleted or merged)`,+ update.personId+ )}if (error.code === Code.InvalidArgument && error.rawMessage.includes('size limit')) {throw new PersonhogPropertiesSizeError(error.rawMessage, update.teamId, update.personId)