Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): retry a load-shed merge refusal instead of settling it as lost (#123738)

Nick Best75b6c96c9d1a
diff --git a/nodejs/src/common/personhog/identity.ts b/nodejs/src/common/personhog/identity.ts
index 474e93f7ecc2..7338fea0579f 100644
--- a/nodejs/src/common/personhog/identity.ts
+++ b/nodejs/src/common/personhog/identity.ts
@@ -84,6 +84,8 @@ export interface MergeSagaRequest {
export const SEMANTIC_REFUSAL_METADATA_KEY = 'x-semantic-refusal'
/** The op id belongs to a different recorded merge; refused before any durable work. */
export const SEMANTIC_REFUSAL_OP_ID_REUSED = 'op_id_reused'
+/** Metadata key on a refusal made before the request reached the service; the value is a reason slug. */
+export const NOT_APPLIED_METADATA_KEY = 'x-not-applied'
/** Only a created person carries a document; a found one is identity only, the leader owns its properties. */
export type GetOrCreatePersonOutcome =
diff --git a/nodejs/src/ingestion/common/persons/person-context.ts b/nodejs/src/ingestion/common/persons/person-context.ts
index a61612ca781b..b9d976801136 100644
--- a/nodejs/src/ingestion/common/persons/person-context.ts
+++ b/nodejs/src/ingestion/common/persons/person-context.ts
@@ -42,7 +42,9 @@ export class PersonContext {
/** Fold plan shared by this event's consecutive $identify run; see person-merge-fold.ts. */
public readonly mergeFoldPlan?: MergeFoldPlan,
/** The personhog identity service publishes the merge verdict warnings it settles, so this consumer must not. */
- public readonly identityEmitsMergeWarnings: boolean = false
+ public readonly identityEmitsMergeWarnings: boolean = false,
+ /** Per-batch wall-clock budget for re-sending merges the backend refused before they ran; 0 escalates at once. */
+ public readonly mergeNotAppliedWaitMs: number = 0
) {
this.eventProperties = event.properties!
}
diff --git a/nodejs/src/ingestion/common/persons/person-merge-service.test.ts b/nodejs/src/ingestion/common/persons/person-merge-service.test.ts
index a749b0605f59..2f1ea68e007d 100644
--- a/nodejs/src/ingestion/common/persons/person-merge-service.test.ts
+++ b/nodejs/src/ingestion/common/persons/person-merge-service.test.ts
@@ -14,7 +14,9 @@ import { MergeFoldPlan } from './person-merge-fold'
import {
PersonMergeService,
mergeClaimDroppedCounter,
+ mergeFinalFailuresCounter,
mergeFoldFallbackCounter,
+ mergeNotAppliedRetriesCounter,
mergeResponseMismatchCounter,
mergeSettledFailureCounter,
mergeUnsettledCounter,
@@ -22,6 +24,7 @@ import {
import {
PersonMergeCallFailedError,
PersonMergeLimitExceededError,
+ PersonMergeNotAppliedError,
PersonMergeResponseMismatchError,
SourcePersonHasDistinctIdsError,
SourcePersonNotFoundError,
@@ -61,7 +64,8 @@ describe('PersonMergeService store-owned merges', () => {
const makeService = (
eventName = '$identify',
mergeFoldPlan?: MergeFoldPlan,
- identityEmitsMergeWarnings = false
+ identityEmitsMergeWarnings = false,
+ mergeNotAppliedWaitMs = 0
) => {
const event = {
event: eventName,
@@ -88,7 +92,8 @@ describe('PersonMergeService store-owned merges', () => {
false,
false,
mergeFoldPlan,
- identityEmitsMergeWarnings
+ identityEmitsMergeWarnings,
+ mergeNotAppliedWaitMs
)
return new PersonMergeService(context)
}
@@ -210,6 +215,112 @@ describe('PersonMergeService store-owned merges', () => {
expect(warned).toContain('merge_settled_failure')
})
+ describe('a merge refused before it ran', () => {
+ const refused = (): Error =>
+ new PersonMergeNotAppliedError('refused before it ran', 'load_shed', new Error('shed'))
+ const warningTypes = (): string[] =>
+ outputs.queueMessages.mock.calls
+ .flat(3)
+ .filter((entry: any) => entry?.value)
+ .map((entry: any) => parseJSON(Buffer.from(entry.value).toString()).type)
+ const outcomes = async (): Promise<Record<string, number>> =>
+ Object.fromEntries(
+ (await mergeNotAppliedRetriesCounter.get()).values.map((entry) => [entry.labels.outcome, entry.value])
+ )
+
+ beforeEach(() => {
+ mergeNotAppliedRetriesCounter.reset()
+ jest.useFakeTimers()
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('is re-sent past the bounded retries and keeps them once capacity returns', async () => {
+ const finalBefore = await counterTotal(mergeFinalFailuresCounter)
+ for (let i = 0; i < 5; i++) {
+ store.mergePersons.mockRejectedValueOnce(refused())
+ }
+ // The first accepted call answers unsettled: without its own bounded
+ // retries the merge would drop with the race warning instead of settling.
+ store.mergePersons.mockResolvedValueOnce({
+ survivor,
+ results: [{ sourceDistinctId: 'anon-1', outcome: 'skipped_conflict' as const, settled: false }],
+ })
+ const service = makeService('$identify', undefined, false, 5_000)
+
+ const merging = service.handleIdentifyOrAlias()
+ await jest.advanceTimersByTimeAsync(5_000)
+ const mergeResult = await merging
+
+ expect(mergeResult).toMatchObject({ success: true, person: survivor })
+ expect(store.mergePersons).toHaveBeenCalledTimes(7)
+ expect(warningTypes()).toEqual([])
+ expect(await counterTotal(mergeFinalFailuresCounter)).toBe(finalBefore)
+ expect(await outcomes()).toEqual({ recovered: 1 })
+ })
+
+ it('escalates to the pipeline retry when the budget runs out, never settling', async () => {
+ const finalBefore = await counterTotal(mergeFinalFailuresCounter)
+ store.mergePersons.mockRejectedValue(refused())
+ const service = makeService('$identify', undefined, false, 1_000)
+
+ const settled = service.handleIdentifyOrAlias().catch((error) => error)
+ await jest.advanceTimersByTimeAsync(10_000)
+ const error = await settled
+
+ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)
+ expect(error.isRetriable).toBe(true)
+ expect(store.mergePersons.mock.calls.length).toBeGreaterThan(4)
+ expect(warningTypes()).not.toContain('merge_settled_failure')
+ expect(await counterTotal(mergeFinalFailuresCounter)).toBe(finalBefore)
+ expect(await outcomes()).toEqual({ escalated: 1 })
+ })
+
+ it('shares one budget across the batch, so a later merge escalates at once when it is spent', async () => {
+ // Merges for one distinct id run serially within a batch; a fresh
+ // budget per event would add up past the consumer stall threshold.
+ store.mergePersons.mockRejectedValue(refused())
+ const first = makeService('$identify', undefined, false, 1_000)
+ .handleIdentifyOrAlias()
+ .catch((e) => e)
+ await jest.advanceTimersByTimeAsync(10_000)
+ await first
+ store.mergePersons.mockClear()
+
+ const second = makeService('$identify', undefined, false, 1_000)
+ .handleIdentifyOrAlias()
+ .catch((e) => e)
+ await jest.advanceTimersByTimeAsync(10)
+ const error = await second
+
+ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)
+ expect(error.isRetriable).toBe(true)
+ expect(store.mergePersons).toHaveBeenCalledTimes(3)
+ // The second merge never waited, so it is not a second escalation.
+ expect(await outcomes()).toEqual({ escalated: 1 })
+ })
+
+ it('an accepted re-send that then fails settles as before, counted apart from recovered', async () => {
+ for (let i = 0; i < 3; i++) {
+ store.mergePersons.mockRejectedValueOnce(refused())
+ }
+ store.mergePersons.mockRejectedValue(
+ new PersonMergeCallFailedError('personhog merge call failed with no verdict', new Error('transport'))
+ )
+ const service = makeService('$identify', undefined, false, 5_000)
+
+ const merging = service.handleIdentifyOrAlias()
+ await jest.advanceTimersByTimeAsync(5_000)
+ const result = await merging
+
+ expect(result.success).toBe(true)
+ expect(warningTypes()).toContain('merge_settled_failure')
+ expect(await outcomes()).toEqual({ accepted_failed: 1 })
+ })
+ })
+
it('a deterministic raw refusal acks once with a warning instead of wedging the batch', async () => {
// The op_id_reused refusal is deterministic, so rethrowing or
// retrying would spin on the same refusal until the recorded op
diff --git a/nodejs/src/ingestion/common/persons/person-merge-service.ts b/nodejs/src/ingestion/common/persons/person-merge-service.ts
index 6eebb9207623..0382c61b364d 100644
--- a/nodejs/src/ingestion/common/persons/person-merge-service.ts
+++ b/nodejs/src/ingestion/common/persons/person-merge-service.ts
@@ -11,12 +11,14 @@ import { timeoutGuard } from '~/common/utils/db/utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { promiseRetry } from '~/common/utils/retries'
+import { sleep } from '~/common/utils/utils'
import { emitIngestionWarning } from '~/ingestion/common/ingestion-warnings'
import { PersonContext } from './person-context'
import {
PersonMergeCallFailedError,
PersonMergeLimitExceededError,
+ PersonMergeNotAppliedError,
PersonMergeRaceConditionError,
PersonMergeResponseMismatchError,
PersonMergeResult,
@@ -34,6 +36,7 @@ import {
MergePersonsResult,
MergePersonsSourceResult,
} from './persons-store'
+import type { PersonsStoreForBatch } from './persons-store-for-batch'
export const mergeFinalFailuresCounter = new Counter({
name: 'person_merge_final_failure_total',
@@ -77,6 +80,18 @@ export const mergeClaimDroppedCounter = new Counter({
labelNames: ['call'],
})
+export const mergeNotAppliedRetriesCounter = new Counter({
+ name: 'person_merge_not_applied_retries_total',
+ help: 'Merges the backend refused before running: recovered once accepted, accepted_failed when the accepted call then failed, escalated when the wait gave up and the pipeline retry took over.',
+ labelNames: ['call', 'outcome'],
+})
+
+const NOT_APPLIED_FIRST_SLEEP_MS = 250
+const NOT_APPLIED_MAX_SLEEP_MS = 2_000
+
+/** One wait budget per batch: merges for one distinct id run serially, so per-event budgets would add up past the stall threshold. */
+const batchNotAppliedDeadlines = new WeakMap<PersonsStoreForBatch, number>()
+
/** Maps a thrown fold failure to the fallback counter's reason label. */
function foldAbandonReason(error: unknown): MergeFoldAbortReason {
if (error instanceof PersonClaimedByLifecycleOpError || error instanceof PersonTombstoneBlockedError) {
@@ -177,6 +192,10 @@ export class PersonMergeService {
// A malformed answer is a bug detector; fail the batch.
throw e
}
+ if (e instanceof PersonMergeNotAppliedError) {
+ // Nothing ran, so there is nothing to settle; the pipeline retry re-sends it.
+ throw e
+ }
// PersonMergeCallFailedError falls through: bounded retries ran,
// so it takes the same give-up the Postgres path has always had.
mergeFinalFailuresCounter
@@ -299,20 +318,14 @@ export class PersonMergeService {
// deliberately unwrapped; retrying re-runs the same comparison.
// Mapping inside the retry lets an unsettled verdict re-present the
// same op id, which the saga's conflict discard was built for.
- try {
- return await promiseRetry(
- async () =>
- this.mapSingleSourceResult(
- await this.context.personStore.mergePersons(request),
- otherPersonDistinctId,
- mergeIntoDistinctId
- ),
- 'merge_distinct_ids',
- undefined,
- undefined,
- undefined,
- [ConnectError, PersonMergeResponseMismatchError]
+ const attempt = async (): Promise<PersonMergeResult> =>
+ this.mapSingleSourceResult(
+ await this.context.personStore.mergePersons(request),
+ otherPersonDistinctId,
+ mergeIntoDistinctId
)
+ try {
+ return await this.mergeWithRetries(attempt)
} catch (error) {
if (!(error instanceof PersonMergeUnsettledError || error instanceof PersonMergeRaceConditionError)) {
throw error
@@ -345,6 +358,75 @@ export class PersonMergeService {
}
}
+ private async mergeWithRetries(attempt: () => Promise<PersonMergeResult>): Promise<PersonMergeResult> {
+ const bounded = (nonRetriable: (new (...args: any[]) => Error)[]): Promise<PersonMergeResult> =>
+ promiseRetry(attempt, 'merge_distinct_ids', undefined, undefined, undefined, nonRetriable)
+ try {
+ return await bounded([ConnectError, PersonMergeResponseMismatchError])
+ } catch (error) {
+ if (!(error instanceof PersonMergeNotAppliedError)) {
+ throw error
+ }
+ // Each re-send is a full bounded run, so a verdict the accepted
+ // call answers keeps the retries it would have had without the wait.
+ return await this.waitOutNotApplied(
+ () => bounded([ConnectError, PersonMergeResponseMismatchError, PersonMergeNotAppliedError]),
+ error
+ )
+ }
+ }
+
+ /** The wait is for capacity, not for a saga, so it cannot stack on a saga deadline. */
+ private async waitOutNotApplied(
+ attempt: () => Promise<PersonMergeResult>,
+ refusal: PersonMergeNotAppliedError
+ ): Promise<PersonMergeResult> {
+ const call = this.context.event.event
+ const deadline = this.batchNotAppliedDeadline()
+ let sleepMs = NOT_APPLIED_FIRST_SLEEP_MS
+ let last = refusal
+ let waited = false
+ for (let remaining = deadline - Date.now(); remaining > 0; remaining = deadline - Date.now()) {
+ waited = true
+ // Jittered so a fleet waiting out one burst does not re-herd the backend.
+ await sleep(Math.min(remaining, sleepMs * (0.5 + Math.random() * 0.5)))
+ sleepMs = Math.min(sleepMs * 2, NOT_APPLIED_MAX_SLEEP_MS)
+ try {
+ const result = await attempt()
+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'recovered' }).inc()
+ return result
+ } catch (error) {
+ if (!(error instanceof PersonMergeNotAppliedError)) {
+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'accepted_failed' }).inc()
+ throw error
+ }
+ last = error
+ }
+ }
+ // A spent budget is re-entered by every pipeline step retry of the same
+ // event, so counting those would report one escalation as five.
+ if (waited) {
+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'escalated' }).inc()
+ logger.warn('🤔', 'merge refused before it ran through every re-send; escalated to the pipeline retry', {
+ team_id: this.context.team.id,
+ distinctId: this.context.distinctId,
+ event_name: call,
+ reason: last.reason,
+ })
+ }
+ throw Object.assign(last, { isRetriable: true as const })
+ }
+
+ private batchNotAppliedDeadline(): number {
+ const store = this.context.personStore
+ let deadline = batchNotAppliedDeadlines.get(store)
+ if (deadline === undefined) {
+ deadline = Date.now() + this.context.mergeNotAppliedWaitMs
+ batchNotAppliedDeadlines.set(store, deadline)
+ }
+ return deadline
+ }
+
/**
* One request for either backend, single-source or folded. The event
* uuid doubles as the op id: retries re-enter with the same id and
diff --git a/nodejs/src/ingestion/common/persons/person-merge-types.ts b/nodejs/src/ingestion/common/persons/person-merge-types.ts
index 05a30ae88133..cf7191a2d5d9 100644
--- a/nodejs/src/ingestion/common/persons/person-merge-types.ts
+++ b/nodejs/src/ingestion/common/persons/person-merge-types.ts
@@ -47,6 +47,19 @@ export class PersonMergeCallFailedError extends PersonMergeError {
}
}
+/** The merge backend refused the call before anything ran (`x-not-applied`); re-sent, never settled as lost. */
+export class PersonMergeNotAppliedError extends PersonMergeError {
+ readonly type = 'NOT_APPLIED' as const
+
+ constructor(
+ message: string,
+ public readonly reason: string,
+ public readonly failure: unknown
+ ) {
+ super(message)
+ }
+}
+
/**
* The merge backend answered an unsettled verdict: a retry under the same
* op id may change the answer. Retried in-process; exhaustion drops the
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
index 58b998c0000e..51a1b39e9978 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
@@ -7,7 +7,11 @@ import { PersonClaimedByLifecycleOpError } from '~/common/persons/repositories/p
import { NoRowsUpdatedError } from '~/common/utils/utils'
import { InternalPerson } from '~/types'
-import { PersonMergeCallFailedError, createDefaultSyncMergeMode } from './person-merge-types'
+import {
+ PersonMergeCallFailedError,
+ PersonMergeNotAppliedError,
+ createDefaultSyncMergeMode,
+} from './person-merge-types'
import { EventOps, extractEventOps } from './person-update'
import { mergeOpIdFromRequest } from './person-uuid'
import {
@@ -530,6 +534,22 @@ describe('PersonhogPersonsStore', () => {
expect(edgeOf('1:d1')).toBeUndefined()
})
+ it('a refusal marked not-applied is typed for the re-send, whatever its status code', async () => {
+ for (const code of [Code.Unavailable, Code.ResourceExhausted]) {
+ repository.mergePersons = jest
+ .fn()
+ .mockRejectedValue(
+ new ConnectError('Server at capacity', code, new Headers({ 'x-not-applied': 'load_shed' }))
+ )
+ const bound = store.forBatch(0)
+ await bound.fetchForUpdate(1, 'd1')
+ const error = await bound.mergePersons(mergeRequest()).catch((e) => e)
+ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)
+ expect(error.reason).toBe('load_shed')
+ expect(edgeOf('1:d1')).toBeUndefined()
+ }
+ })
+
it('derives one op id per delivery regardless of how the event payload drifts', async () => {
// Two deliveries of one event disagree on properties later stages
// refresh, but must present the same op id so the second attaches
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
index 607cdd7ec044..703e1935bcb0 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
@@ -6,6 +6,7 @@ import { Counter } from 'prom-client'
import { TRANSPORT_MAX_BACKOFF_MS, TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'
import {
DistinctIdKey,
+ NOT_APPLIED_METADATA_KEY,
SEMANTIC_REFUSAL_METADATA_KEY,
SEMANTIC_REFUSAL_OP_ID_REUSED,
} from '~/common/personhog/identity'
@@ -23,7 +24,7 @@ import { BatchWritingStoreFlushStats } from '~/ingestion/common/stores/batch-wri
import { Properties } from '~/plugin-scaffold'
import { InternalPerson, PropertiesLastOperation, PropertiesLastUpdatedAt } from '~/types'
-import { MergeMode, PersonMergeCallFailedError } from './person-merge-types'
+import { MergeMode, PersonMergeCallFailedError, PersonMergeNotAppliedError } from './person-merge-types'
import { EventOps, applyEventPropertyUpdates, computeOpsScalarUpdates, foldOps, refineEventOps } from './person-update'
import { mergeOpIdFromRequest } from './person-uuid'
import { FlushResult, MergePersonsRequest, MergePersonsResult, PersonsStore, isFoldRequest } from './persons-store'
@@ -887,6 +888,17 @@ export class PersonhogPersonsStore implements PersonsStore {
personhogStoreMergeCallFailedCounter.inc({ error: 'OpIdReusedSettled' })
throw error
}
+ // Keyed on the header: a hop in between may remap the status code.
+ const notApplied = error instanceof ConnectError ? error.metadata.get(NOT_APPLIED_METADATA_KEY) : null
+ if (notApplied !== null) {
+ const reason = /^[a-z0-9_-]{1,64}$/i.test(notApplied) ? notApplied : 'unknown'
+ personhogStoreMergeCallFailedCounter.inc({ error: `not_applied_${reason}` })
+ throw new PersonMergeNotAppliedError(
+ `personhog merge refused before it ran (${reason}): ${error.message}`,
+ reason,
+ error
+ )
+ }
// No verdict, so an ack could lose the merge. Only the call is
// wrapped, so a post-verdict bug surfaces as itself.
personhogStoreMergeCallFailedCounter.inc({
diff --git a/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts b/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts
index e6216519566b..21da9e801981 100644
--- a/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts
+++ b/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts
@@ -14,7 +14,7 @@ import { logger } from '~/common/utils/logger'
import { defaultRetryConfig } from '~/common/utils/retries'
import { InternalPerson } from '~/types'
-import { PersonMergeCallFailedError } from './person-merge-types'
+import { PersonMergeCallFailedError, PersonMergeNotAppliedError } from './person-merge-types'
import { EventOps } from './person-update'
import { PersonhogPersonsStore } from './personhog-persons-store'
import { MergePersonsResult, PersonsBackend, PersonsStore } from './persons-store'
@@ -944,6 +944,8 @@ describe('RoutingPersonsStore', () => {
})
const noVerdict = (): Error => new PersonMergeCallFailedError('no verdict', new Error('shed'))
+ const notApplied = (): Error =>
+ new PersonMergeNotAppliedError('refused before it ran', 'load_shed', new Error('shed'))
it.each([
['a no-verdict failure that clears is retried under the same request', [noVerdict()], 2, null, false],
@@ -954,6 +956,13 @@ describe('RoutingPersonsStore', () => {
'PersonMergeCallFailedError',
true,
],
+ [
+ 'a refusal before it ran through every attempt is re-driven at a flush, not waited out',
+ [notApplied(), notApplied(), notApplied()],
+ 3,
+ 'PersonMergeNotAppliedError',
+ true,
+ ],
[
'a deterministic refusal is counted without a retry or a re-drive',
[new ConnectError('refused', Code.InvalidArgument)],
diff --git a/nodejs/src/ingestion/common/persons/routing-persons-store.ts b/nodejs/src/ingestion/common/persons/routing-persons-store.ts
index 9b4818cfbe18..31540681fe23 100644
--- a/nodejs/src/ingestion/common/persons/routing-persons-store.ts
+++ b/nodejs/src/ingestion/common/persons/routing-persons-store.ts
@@ -22,7 +22,7 @@ import { BatchWritingStoreFlushStats } from '~/ingestion/common/stores/batch-wri
import { Properties } from '~/plugin-scaffold'
import { InternalPerson, PropertiesLastOperation, PropertiesLastUpdatedAt } from '~/types'
-import { PersonMergeCallFailedError, PersonMergeUnsettledError } from './person-merge-types'
+import { PersonMergeCallFailedError, PersonMergeNotAppliedError, PersonMergeUnsettledError } from './person-merge-types'
import { EventOps } from './person-update'
import { CREATE_EVENT_NAME, PersonhogPersonsStore } from './personhog-persons-store'
import {
@@ -648,8 +648,13 @@ export class RoutingPersonsStore implements PersonsStore {
}
return unsettled
}
- // Abandoned at the ceiling, the merge is re-driven at a flush under the same op id.
- if ((error instanceof PersonMergeCallFailedError || error instanceof ShadowVerbTimeoutError) && defer) {
+ // Abandoned at the ceiling or refused before it ran, the merge is re-driven at a flush under the same op id.
+ if (
+ (error instanceof PersonMergeCallFailedError ||
+ error instanceof PersonMergeNotAppliedError ||
+ error instanceof ShadowVerbTimeoutError) &&
+ defer
+ ) {
this.deferShadowMerge(request, batchId)
}
throw error
diff --git a/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts b/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts
index 9e8eb719c98b..f24be1fdf6ea 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts
@@ -6,6 +6,7 @@ export interface EventPipelineRunnerOptions {
PERSON_MERGE_FOLD_ENABLED: boolean
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: string
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: boolean
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: number
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: number
PERSON_PROPERTIES_UPDATE_ALL: boolean
/** Teams whose $feature_flag_called events default to personless: '*' for all, '' to disable, or comma-separated team IDs */
diff --git a/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts b/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts
index 09e6cee17cc0..c91ea628f0a5 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts
@@ -63,7 +63,8 @@ export function createProcessPersonsStep<TInput extends ProcessPersonsInput>(
options.PERSON_PROPERTIES_UPDATE_ALL,
shouldUpdateLastSeenAt,
input.mergeFold.type === 'planned' ? input.mergeFold.plan : undefined,
- options.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS
+ options.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,
+ options.PERSON_MERGE_NOT_APPLIED_WAIT_MS
)
const processor = new PersonEventProcessor(
diff --git a/nodejs/src/ingestion/config.ts b/nodejs/src/ingestion/config.ts
index 6a532e14c79d..9d88278a0748 100644
--- a/nodejs/src/ingestion/config.ts
+++ b/nodejs/src/ingestion/config.ts
@@ -176,6 +176,10 @@ export type IngestionConsumerConfig = {
// comes back skipped_move_limit for the merge-mode policy.
PERSONHOG_SYNC_MERGE_MOVE_LIMIT: number
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: boolean
+ // Re-send budget per batch for merges refused at admission, measured from the batch's
+ // first refusal and shared by every merge and step retry in it. Keep it well under
+ // CONSUMER_LOOP_STALL_THRESHOLD_MS, which the whole batch has to fit in.
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: number
// Kill switch for emitting person_merge_events to the cohort-stream-processor.
// Enable ordering: (1) create the topic, (2) set INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC
// (startup topic verification is then fatal by design), (3) flip this on. Flipping this on before
@@ -370,6 +374,7 @@ export function getDefaultIngestionConsumerConfig(): IngestionConsumerConfig {
PERSON_MERGE_SYNC_BATCH_SIZE: 0,
PERSONHOG_SYNC_MERGE_MOVE_LIMIT: 10_000,
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: false,
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: 20_000,
PERSON_MERGE_EVENTS_ENABLED: false,
PERSON_MERGE_EVENTS_PARTITION_COUNT: 64,
PERSON_MERGE_EVENTS_TEAM_ALLOWLIST: '2',
diff --git a/nodejs/src/ingestion/ingestion-consumer.ts b/nodejs/src/ingestion/ingestion-consumer.ts
index 6e2f9028369d..633d643d938a 100644
--- a/nodejs/src/ingestion/ingestion-consumer.ts
+++ b/nodejs/src/ingestion/ingestion-consumer.ts
@@ -318,6 +318,7 @@ export class IngestionConsumer {
PERSON_MERGE_FOLD_ENABLED: this.config.PERSON_MERGE_FOLD_ENABLED,
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: this.config.PERSON_MERGE_FOLD_TEAM_ALLOWLIST,
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: this.config.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: this.config.PERSON_MERGE_NOT_APPLIED_WAIT_MS,
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: this.config.PERSON_JSONB_SIZE_ESTIMATE_ENABLE,
PERSON_PROPERTIES_UPDATE_ALL: this.config.PERSON_PROPERTIES_UPDATE_ALL,
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: this.config.FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,
diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.ts
index 98b6429f10e3..2520265466f9 100644
--- a/nodejs/src/servers/ingestion-api-server.ts
+++ b/nodejs/src/servers/ingestion-api-server.ts
@@ -437,6 +437,7 @@ export class IngestionApiServer implements NodeServer {
PERSON_MERGE_FOLD_ENABLED: this.config.PERSON_MERGE_FOLD_ENABLED,
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: this.config.PERSON_MERGE_FOLD_TEAM_ALLOWLIST,
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: this.config.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: this.config.PERSON_MERGE_NOT_APPLIED_WAIT_MS,
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: this.config.PERSON_JSONB_SIZE_ESTIMATE_ENABLE,
PERSON_PROPERTIES_UPDATE_ALL: this.config.PERSON_PROPERTIES_UPDATE_ALL,
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: this.config.FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,
diff --git a/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts b/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts
index f372cd6f2ff5..7ea9c8342c98 100644
--- a/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts
+++ b/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts
@@ -55,6 +55,7 @@ describe('createProcessPersonsStep', () => {
PERSON_MERGE_FOLD_ENABLED: false,
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: '*',
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: false,
+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: 0,
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: 0,
PERSON_PROPERTIES_UPDATE_ALL: false,
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: '*',