:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Serial e2e: drives the Postgres and personhog person backends against real
// infrastructure and compares them through the routing store's own shadow
// machinery. The personhog half is the real stack (identity, leader-mode
// router, leader), not a model, so this catches contract drift a Jest mock
// cannot: proto encoding, uuid derivation, the saga's verdict vocabulary,
// and the identity repointing a merge leaves behind.
//
// Requires the personhog services on top of the usual serial-test infra:
// `docker compose -f docker-compose.dev.yml --profile ingestion up`, or a
// `hogli start` dev stack (the `personhog` capability). Addresses override
// via PERSONHOG_E2E_ROUTER_ADDR / PERSONHOG_E2E_IDENTITY_ADDR.
import { Code, ConnectError } from '@connectrpc/connect'
import { DateTime } from 'luxon'
import { isDeepStrictEqual } from 'node:util'
import { Pool } from 'pg'
import {
KAFKA_INGESTION_WARNINGS,
KAFKA_PERSON,
KAFKA_PERSON_DISTINCT_ID,
KAFKA_PERSON_MERGE_EVENTS,
} from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import {
INGESTION_WARNINGS_OUTPUT,
PERSONS_OUTPUT,
PERSON_DISTINCT_IDS_OUTPUT,
PERSON_MERGE_EVENTS_OUTPUT,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { PersonHogClient } from '~/common/personhog/client'
import { createIdentityClients } from '~/common/personhog/identity-clients'
import { PersonHogPersonWriteRepository } from '~/common/personhog/personhog-person-write-repository'
import {
personhogStoreShadowComparedCounter,
personhogStoreShadowCreateRetriesCounter,
personhogStoreShadowDivergenceCounter,
personhogStoreShadowErrorsCounter,
} from '~/common/persons/metrics'
import { PostgresPersonRepository } from '~/common/persons/repositories/postgres-person-repository'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { UUIDT } from '~/common/utils/utils'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { PersonOutputs } from '~/ingestion/common/persons/person-context'
import { MergeMode, createDefaultSyncMergeMode } from '~/ingestion/common/persons/person-merge-types'
import { extractEventOps } from '~/ingestion/common/persons/person-update'
import { uuidFromDistinctId } from '~/ingestion/common/persons/person-uuid'
import { PersonhogPersonsStore } from '~/ingestion/common/persons/personhog-persons-store'
import { MergePersonsRequest, MergePersonsResult } from '~/ingestion/common/persons/persons-store'
import { RoutingPersonsStore } from '~/ingestion/common/persons/routing-persons-store'
import { Hub } from '~/types'
import { createOrganization, createTeam } from '../helpers/sql'
jest.setTimeout(60000)
const ROUTER_ADDR = process.env.PERSONHOG_E2E_ROUTER_ADDR ?? '127.0.0.1:50054'
const IDENTITY_ADDR = process.env.PERSONHOG_E2E_IDENTITY_ADDR ?? '127.0.0.1:50055'
// Where the personhog services keep their validation tables; the writer
// applies the leader's changelog here. The compose services default to
// the `posthog` database, so a compose-launched stack needs
// POSTHOG_PERSONS_DB_NAME=posthog_persons exported (hogli sets it).
const PERSONS_DATABASE_URL =
process.env.PERSONHOG_E2E_PERSONS_DATABASE_URL ?? 'postgres://posthog:posthog@localhost:5432/posthog_persons'
const counterTotal = async (counter: { get: () => Promise<{ values: { value: number }[] }> }): Promise<number> =>
(await counter.get()).values.reduce((sum, entry) => sum + entry.value, 0)
describe('personhog shadow parity (e2e)', () => {
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let routerClient: PersonHogClient
let closeIdentity: () => void
let writeRepository: PersonHogPersonWriteRepository
let routing: RoutingPersonsStore
let personsDb: Pool = undefined as unknown as Pool
let organizationId: string
let teamId: number
let batchId = 0
// Distinct ids are unique per run: the personhog side writes shared
// dev tables, so a fixed id would meet a person a previous run made.
const runTag = new UUIDT().toString().slice(0, 13)
const id = (name: string) => `e2e-${runTag}-${name}`
const outputs = (): PersonOutputs =>
new IngestionOutputs({
[PERSONS_OUTPUT]: new SingleIngestionOutput(PERSONS_OUTPUT, KAFKA_PERSON, kafkaProducer, 'test'),
[PERSON_DISTINCT_IDS_OUTPUT]: new SingleIngestionOutput(
PERSON_DISTINCT_IDS_OUTPUT,
KAFKA_PERSON_DISTINCT_ID,
kafkaProducer,
'test'
),
[INGESTION_WARNINGS_OUTPUT]: new SingleIngestionOutput(
INGESTION_WARNINGS_OUTPUT,
KAFKA_INGESTION_WARNINGS,
kafkaProducer,
'test'
),
[PERSON_MERGE_EVENTS_OUTPUT]: new SingleIngestionOutput(
PERSON_MERGE_EVENTS_OUTPUT,
KAFKA_PERSON_MERGE_EVENTS,
kafkaProducer,
'test'
),
})
/**
* One ingestion pod: its own caches over the shared databases and the
* shared personhog cluster, so two pods can interleave on one person.
*/
/** One ingestion pod over the shared databases; the personhog repository can be wrapped to fail one call. */
const newPod = (repository: PersonHogPersonWriteRepository = writeRepository): RoutingPersonsStore =>
new RoutingPersonsStore(
new BatchWritingPersonsStore(new PostgresPersonRepository(hub.postgres), outputs(), {
metricEmissionIntervalMs: 0,
}),
new PersonhogPersonsStore(repository, {
maxConcurrentUpdates: 10,
updateAllProperties: false,
syncMergeMoveLimit: 10_000,
}),
'shadow'
)
const ops = (properties: Record<string, unknown>, event = '$set') =>
extractEventOps({
event,
distinct_id: 'e2e',
properties,
team_id: teamId,
uuid: new UUIDT().toString(),
ip: null,
now: new Date().toISOString(),
site_url: '',
} as any)
const createThroughBoth = async (
distinctId: string,
properties: Record<string, unknown>,
batch: number,
extraDistinctIds?: string[],
isIdentified = false
) => {
const result = await routing.createPerson(
DateTime.utc(),
properties,
{},
{},
teamId,
null,
isIdentified,
uuidFromDistinctId(teamId, distinctId),
{ distinctId },
extraDistinctIds?.map((extra) => ({ distinctId: extra })),
undefined,
batch
)
if (!result.success) {
throw new Error('creation failed')
}
return result.person
}
/** One merge through both backends, with the ack awaited. */
const runMerge = async (
targetDistinctId: string,
sourceDistinctIds: string[],
opts: {
allowIdentifiedSources?: boolean
mergeMode?: MergeMode
eventUuid?: string
sourceEventUuids?: string[]
set?: Record<string, unknown>
} = {}
) => {
const mergeOps = ops({ $set: opts.set ?? {} }, '$identify')
// The processor marks the caller identified on every merge-shaped
// event and applies it on the follow-up update; the saga stamps the
// survivor itself, so without this the Postgres row lags behind.
mergeOps.isIdentified = true
const result = await routing.mergePersons(
{
teamId,
targetDistinctId,
sources: sourceDistinctIds.map((distinctId, i) => ({
distinctId,
eventUuid: opts.sourceEventUuids?.[i] ?? new UUIDT().toString(),
})),
eventOps: mergeOps,
eventUuid: opts.eventUuid ?? new UUIDT().toString(),
allowIdentifiedSources: opts.allowIdentifiedSources ?? false,
mergeMode: opts.mergeMode ?? createDefaultSyncMergeMode(),
createdAtMs: Date.now(),
},
batchId
)
await result.kafkaAck
return { result, mergeOps }
}
/**
* The follow-up property update the processor runs after a merge:
* Postgres leaves the event ops to it, the saga already applied them,
* and the idempotent re-fold converges the two.
*/
const applyMergeFollowUp = async (
result: MergePersonsResult,
mergeOps: ReturnType<typeof ops>,
targetDistinctId: string
) => {
if ((result.survivorNeedsUpdate ?? true) && result.survivor) {
await routing.applyEventOps(result.survivor, mergeOps, targetDistinctId, batchId)
}
}
const divergences = () => counterTotal(personhogStoreShadowDivergenceCounter)
const shadowErrors = () => counterTotal(personhogStoreShadowErrorsCounter)
const divergencesByField = async (field: string): Promise<number> =>
(await personhogStoreShadowDivergenceCounter.get()).values
.filter((entry) => entry.labels.field === field)
.reduce((sum, entry) => sum + entry.value, 0)
interface DurableRow {
uuid: string
properties: Record<string, unknown>
is_identified: boolean
created_at: string
last_seen_at: string | null
}
const mainRowByDistinctId = async (distinctId: string): Promise<DurableRow | null> => {
const { rows } = await hub.postgres.query<DurableRow>(
PostgresUse.PERSONS_WRITE,
`SELECT p.uuid, p.properties, p.is_identified,
(extract(epoch from p.created_at) * 1000)::bigint::text AS created_at,
(extract(epoch from p.last_seen_at) * 1000)::bigint::text AS last_seen_at
FROM posthog_persondistinctid d
JOIN posthog_person p ON p.id = d.person_id
WHERE d.team_id = $1 AND d.distinct_id = $2`,
[teamId, distinctId],
'personhog-shadow-parity-main-row'
)
return rows[0] ?? null
}
const tmpRowByDistinctId = async (distinctId: string): Promise<DurableRow | null> => {
const { rows } = await personsDb.query<DurableRow>(
`SELECT p.uuid, p.properties, p.is_identified,
(extract(epoch from p.created_at) * 1000)::bigint::text AS created_at,
(extract(epoch from p.last_seen_at) * 1000)::bigint::text AS last_seen_at
FROM personhog_persondistinctid_tmp d
JOIN personhog_person_tmp p ON p.id = d.person_id AND p.team_id = d.team_id
WHERE d.team_id = $1 AND d.distinct_id = $2
AND d.is_deleted = false AND p.is_deleted = false`,
[teamId, distinctId]
)
return rows[0] ?? null
}
/**
* Asserts the durable rows behind one distinct id agree between the
* main tables (the Postgres backend's writes) and the personhog
* validation tables (identity's mappings plus the writer applying the
* leader's changelog). The writer is asynchronous, so this polls until
* the validation row converges or the deadline passes; the final
* expectations then print whatever it last held.
*/
const expectDurableRowParity = async (distinctId: string): Promise<void> => {
const main = await mainRowByDistinctId(distinctId)
expect(main).not.toBeNull()
const deadline = Date.now() + 20_000
let tmp: DurableRow | null = null
for (;;) {
tmp = await tmpRowByDistinctId(distinctId)
if (
tmp !== null &&
tmp.uuid === main!.uuid &&
tmp.is_identified === main!.is_identified &&
tmp.created_at === main!.created_at &&
tmp.last_seen_at === main!.last_seen_at &&
isDeepStrictEqual(tmp.properties, main!.properties)
) {
break
}
if (Date.now() > deadline) {
break
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
expect(tmp).not.toBeNull()
expect(tmp!.uuid).toBe(main!.uuid)
expect(tmp!.is_identified).toBe(main!.is_identified)
expect(tmp!.created_at).toBe(main!.created_at)
expect(tmp!.last_seen_at).toBe(main!.last_seen_at)
expect(tmp!.properties).toEqual(main!.properties)
}
beforeAll(async () => {
hub = await createHub({})
kafkaProducer = await KafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
routerClient = PersonHogClient.fromConfig({
addr: ROUTER_ADDR,
clientName: 'personhog-shadow-parity-e2e',
timeoutMs: 10_000,
})
const identityClients = createIdentityClients(
{ addr: IDENTITY_ADDR, clientName: 'personhog-shadow-parity-e2e', timeoutMs: 10_000 },
{ mergeTimeoutMs: 35_000 }
)
closeIdentity = identityClients.close
writeRepository = new PersonHogPersonWriteRepository(
routerClient,
identityClients.identity,
'personhog-shadow-parity-e2e'
)
try {
await writeRepository.resolvePersonsByDistinctIds([{ teamId: 1, distinctId: id('ping') }], 'e2e-ping')
} catch (error) {
throw new Error(
`personhog stack not reachable (router ${ROUTER_ADDR}, identity ${IDENTITY_ADDR}); ` +
`start it with \`docker compose -f docker-compose.dev.yml --profile ingestion up\` ` +
`or a hogli dev stack. Cause: ${error instanceof Error ? error.message : String(error)}`
)
}
routing = newPod()
personsDb = new Pool({ connectionString: PERSONS_DATABASE_URL, max: 2 })
organizationId = await createOrganization(hub.postgres)
teamId = await createTeam(hub.postgres, organizationId)
})
afterAll(async () => {
routerClient?.close()
closeIdentity?.()
await personsDb?.end()
await kafkaProducer?.disconnect()
if (hub) {
await closeHub(hub)
}
})
beforeEach(() => {
batchId += 1
personhogStoreShadowDivergenceCounter.reset()
personhogStoreShadowComparedCounter.reset()
personhogStoreShadowErrorsCounter.reset()
personhogStoreShadowCreateRetriesCounter.reset()
})
afterEach(() => {
routing.releaseBatch(batchId)
})
describe('shadow creates', () => {
/**
* A personhog repository whose first create of one distinct id fails. With `after`, identity commits the
* person first and the failure follows, which leaves the person without its properties: what a cancellation
* between the stub commit and the property push leaves.
*/
const failingCreateRepository = (
distinctId: string,
failure: () => Error,
after = false
): PersonHogPersonWriteRepository => {
let failed = false
const real = writeRepository.getOrCreatePersonByDistinctId.bind(writeRepository)
return new Proxy(writeRepository, {
get(target, property, receiver) {
if (property !== 'getOrCreatePersonByDistinctId') {
return Reflect.get(target, property, receiver)
}
return async (entry: Parameters<typeof real>[0], callerTag?: string) => {
if (failed || entry.distinctId !== distinctId) {
return real(entry, callerTag)
}
failed = true
if (after) {
await real({ ...entry, setProperties: {}, setOnceProperties: {} }, callerTag)
}
throw failure()
}
},
})
}
const retriableUnavailable = (): Error =>
Object.assign(new ConnectError('Server at capacity', Code.Unavailable), { isRetriable: true })
it.each([
['refused before identity committed', false, retriableUnavailable],
['cancelled after identity committed', true, () => new ConnectError('canceled', Code.Canceled)],
])('a shadow create %s still lands the person and its creation properties', async (_name, after, failure) => {
const distinctId = id(`create-${after ? 'cancelled' : 'refused'}`)
const pod = newPod(failingCreateRepository(distinctId, failure, after))
await pod.createPerson(
DateTime.utc(),
{ plan: 'free' },
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, distinctId),
{ distinctId },
undefined,
undefined,
batchId
)
// The retry lands inside the create, so nothing is recorded as a shadow failure.
expect(await shadowErrors()).toBe(0)
expect(await counterTotal(personhogStoreShadowCreateRetriesCounter)).toBe(2)
await pod.flush()
pod.releaseBatch(batchId)
// The next event for the id is an update, as Postgres already holds the person; the shadow has to
// catch up on its own.
batchId += 1
const person = await pod.fetchForUpdate(teamId, distinctId, batchId)
await pod.applyEventOps(person!, ops({ $set: { visits: 2 } }, '$pageview'), distinctId, batchId)
await pod.flush()
pod.releaseBatch(batchId)
await expectDurableRowParity(distinctId)
})
it('a set-once right behind a create that failed after identity committed yields to the creation value', async () => {
const distinctId = id('create-order')
const pod = newPod(
failingCreateRepository(distinctId, () => new ConnectError('canceled', Code.Canceled), true)
)
await pod.createPerson(
DateTime.utc(),
{ plan: 'free' },
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, distinctId),
{ distinctId },
undefined,
undefined,
batchId
)
// The event behind the create in its distinct id's sequence: Postgres ignores the set-once.
const person = await pod.fetchForUpdate(teamId, distinctId, batchId)
await pod.applyEventOps(person!, ops({ $set_once: { plan: 'pro' } }), distinctId, batchId)
await pod.flush()
pod.releaseBatch(batchId)
await expectDurableRowParity(distinctId)
})
})
it('creation and updates read back identically through both backends', async () => {
const distinctId = id('reader')
await createThroughBoth(distinctId, { plan: 'free' }, batchId)
const person = await routing.fetchForUpdate(teamId, distinctId, batchId)
expect(person).not.toBeNull()
const [afterSet] = await routing.applyEventOps(
person!,
ops({ $set: { plan: 'pro', level: 3 } }),
distinctId,
batchId
)
// $set_once must not beat the standing value and $unset must land:
// Postgres refines these client-side, the leader server-side, and
// this is where the two refinements could drift. Chained on the
// returned person, as the processor chains events: Postgres refines
// against the caller's snapshot, so a stale one hides the unset.
await routing.applyEventOps(
afterSet,
ops({ $set_once: { plan: 'ignored', fresh: 'kept' }, $unset: ['level'] }),
distinctId,
batchId
)
await routing.flush()
routing.releaseBatch(batchId)
// The durable rows first: waiting for the writer here also makes the
// checking read below deterministic, since the identity resolve
// serves writer-applied state.
await expectDurableRowParity(distinctId)
// A fresh batch, so both sides read their backend rather than a
// cached answer; the shadow comparison is the assertion surface.
// The checking read exercises the other personhog read path, the
// identity resolve without a leader hop.
batchId += 1
const checked = await routing.fetchForChecking(teamId, distinctId, batchId)
expect(checked?.uuid).toBe(uuidFromDistinctId(teamId, distinctId))
const reread = await routing.fetchForUpdate(teamId, distinctId, batchId)
expect(reread?.uuid).toBe(uuidFromDistinctId(teamId, distinctId))
expect(reread?.properties).toEqual({ plan: 'pro', fresh: 'kept' })
expect(await counterTotal(personhogStoreShadowComparedCounter)).toBeGreaterThan(0)
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it('a merge of two unseen ids births the person with both, identically', async () => {
const target = id('birth-target')
const source = id('birth-source')
const { result, mergeOps } = await runMerge(target, [source], { set: { born: 'yes' } })
expect(result.results[0]?.outcome).toBe('attached')
expect(result.survivor?.uuid).toBe(uuidFromDistinctId(teamId, target))
await applyMergeFollowUp(result, mergeOps, target)
await routing.flush()
routing.releaseBatch(batchId)
await expectDurableRowParity(target)
await expectDurableRowParity(source)
batchId += 1
const viaSource = await routing.fetchForUpdate(teamId, source, batchId)
expect(viaSource?.uuid).toBe(uuidFromDistinctId(teamId, target))
expect(viaSource?.properties).toMatchObject({ born: 'yes' })
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it('an unseen target attaches to the one existing person, with the verdict-name divergence pinned', async () => {
const source = id('attach-source')
const target = id('attach-target')
await createThroughBoth(source, { origin: 'source' }, batchId)
const { result, mergeOps } = await runMerge(target, [source])
// The existing person keeps surviving on both backends and only its
// id set grows, but the verdict names differ by design: the saga
// establishes the unresolved target first, so by execution both ids
// share one person and it answers noop_same_person where Postgres's
// one-exists branch says attached.
expect(result.results[0]?.outcome).toBe('attached')
expect(result.survivor?.uuid).toBe(uuidFromDistinctId(teamId, source))
expect(await divergencesByField('outcome')).toBe(1)
expect(await divergencesByField('survivor')).toBe(0)
await applyMergeFollowUp(result, mergeOps, target)
await routing.flush()
routing.releaseBatch(batchId)
await expectDurableRowParity(source)
await expectDurableRowParity(target)
batchId += 1
const viaTarget = await routing.fetchForUpdate(teamId, target, batchId)
expect(viaTarget?.uuid).toBe(uuidFromDistinctId(teamId, source))
expect(await shadowErrors()).toBe(0)
})
it('a merge of two ids already on one person answers noop identically', async () => {
const target = id('same-target')
const source = id('same-source')
await createThroughBoth(target, {}, batchId, [source])
const { result } = await runMerge(target, [source])
expect(result.results[0]?.outcome).toBe('noop_same_person')
expect(result.survivor?.uuid).toBe(uuidFromDistinctId(teamId, target))
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it.each([
[false, 'skipped_already_identified'],
[true, 'merged'],
])(
'an identified source with allowIdentifiedSources=%p answers %s on both backends',
async (allowIdentifiedSources, expected) => {
const target = id(`ident-${allowIdentifiedSources}-target`)
const source = id(`ident-${allowIdentifiedSources}-source`)
await createThroughBoth(target, {}, batchId)
await createThroughBoth(source, {}, batchId, undefined, true)
const { result, mergeOps } = await runMerge(target, [source], { allowIdentifiedSources })
expect(result.results[0]?.outcome).toBe(expected)
if (expected === 'merged') {
await applyMergeFollowUp(result, mergeOps, target)
await routing.flush()
routing.releaseBatch(batchId)
await expectDurableRowParity(target)
await expectDurableRowParity(source)
}
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
}
)
it('a two-source fold settles every verdict identically', async () => {
const target = id('fold-target')
const first = id('fold-first')
const second = id('fold-second')
await createThroughBoth(target, { origin: 'target' }, batchId)
await createThroughBoth(first, { fromFirst: 'yes' }, batchId)
await createThroughBoth(second, { fromSecond: 'yes' }, batchId)
const { result, mergeOps } = await runMerge(target, [first, second], { set: { folded: 'yes' } })
expect(result.foldAborted).toBeUndefined()
expect(result.results.map((source) => source.outcome)).toEqual(['merged', 'merged'])
expect(result.survivor?.uuid).toBe(uuidFromDistinctId(teamId, target))
await applyMergeFollowUp(result, mergeOps, target)
await routing.flush()
routing.releaseBatch(batchId)
await expectDurableRowParity(target)
await expectDurableRowParity(first)
await expectDurableRowParity(second)
batchId += 1
const viaFirst = await routing.fetchForUpdate(teamId, first, batchId)
expect(viaFirst?.properties).toMatchObject({
origin: 'target',
fromFirst: 'yes',
fromSecond: 'yes',
folded: 'yes',
})
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it('a replayed merge pins the one documented verdict divergence', async () => {
const target = id('replay-target')
const source = id('replay-source')
await createThroughBoth(target, {}, batchId)
await createThroughBoth(source, {}, batchId)
const eventUuid = new UUIDT().toString()
const sourceEventUuids = [new UUIDT().toString()]
const { result: firstRun, mergeOps } = await runMerge(target, [source], { eventUuid, sourceEventUuids })
expect(firstRun.results[0]?.outcome).toBe('merged')
await applyMergeFollowUp(firstRun, mergeOps, target)
await routing.flush()
routing.releaseBatch(batchId)
personhogStoreShadowDivergenceCounter.reset()
batchId += 1
// A redelivery of the same event: Postgres re-runs against moved
// rows and lands on noop_same_person; the saga replays the recorded
// 'merged' verdict per op id. Same survivor, different verdict
// name — the one divergence this shape is allowed.
const { result: replay } = await runMerge(target, [source], { eventUuid, sourceEventUuids })
expect(replay.results[0]?.outcome).toBe('noop_same_person')
expect(replay.survivor?.uuid).toBe(uuidFromDistinctId(teamId, target))
expect(await divergencesByField('outcome')).toBe(1)
expect(await divergencesByField('survivor')).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it('an over-limit source skips the merge in LIMIT mode, with the survivor divergence pinned', async () => {
const target = id('limit-target')
const source = id('limit-source')
await createThroughBoth(target, {}, batchId)
await createThroughBoth(source, {}, batchId, [id('limit-source-extra')])
const { result } = await runMerge(target, [source], { mergeMode: { type: 'LIMIT', limit: 1 } })
// Both backends skip on the same verdict; only the survivor field
// differs by design — Postgres answers none on the skip, the saga
// answers the target it resolved. The service maps the verdict to
// an error either way, so nothing reads the survivor.
expect(result.results[0]?.outcome).toBe('skipped_move_limit')
expect(await divergencesByField('outcome')).toBe(0)
expect(await divergencesByField('survivor')).toBe(1)
expect(await shadowErrors()).toBe(0)
})
it('a single-source merge settles the same survivor and verdict on both backends', async () => {
const target = id('merge-target')
const source = id('merge-source')
await createThroughBoth(target, { origin: 'target' }, batchId)
await createThroughBoth(source, { origin: 'source' }, batchId)
const request: MergePersonsRequest = {
teamId,
targetDistinctId: target,
sources: [{ distinctId: source, eventUuid: new UUIDT().toString() }],
eventOps: ops({ $set: { merged: 'yes' } }, '$identify'),
eventUuid: new UUIDT().toString(),
allowIdentifiedSources: false,
mergeMode: createDefaultSyncMergeMode(),
createdAtMs: Date.now(),
}
const result = await routing.mergePersons(request, batchId)
await result.kafkaAck
expect(result.survivor?.uuid).toBe(uuidFromDistinctId(teamId, target))
expect(result.results[0]?.outcome).toBe('merged')
// compareMerge checked the shadow's survivor uuid and per-source
// verdict against these; a saga whose vocabulary or op-id handling
// drifted from the client mapping lands here as a divergence.
expect(await counterTotal(personhogStoreShadowComparedCounter)).toBeGreaterThan(0)
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
it('updates and a merge interleaved in one batch settle identically', async () => {
const target = id('weave-target')
const source = id('weave-source')
await createThroughBoth(target, { shared: 'target', origin: 'target' }, batchId)
await createThroughBoth(source, { shared: 'source', extra: 'source' }, batchId)
// A buffered update on the source before the merge: Postgres folds
// it through its cache, personhog drains the lane to the leader
// before the saga runs; either way it must reach the survivor with
// source precedence, so `shared` still settles to the target's value.
const sourcePerson = await routing.fetchForUpdate(teamId, source, batchId)
await routing.applyEventOps(sourcePerson!, ops({ $set: { updated: 'pre-merge' } }), source, batchId)
const mergeOps = ops({ $set: { mergedBy: 'event' } }, '$identify')
const merged = await routing.mergePersons(
{
teamId,
targetDistinctId: target,
sources: [{ distinctId: source, eventUuid: new UUIDT().toString() }],
eventOps: mergeOps,
eventUuid: new UUIDT().toString(),
allowIdentifiedSources: false,
mergeMode: createDefaultSyncMergeMode(),
createdAtMs: Date.now(),
},
batchId
)
await merged.kafkaAck
expect(merged.results[0]?.outcome).toBe('merged')
// The follow-up property update the processor runs after a merge:
// Postgres leaves the event ops to it, the saga already applied
// them, and the idempotent re-fold converges the two.
if (merged.survivorNeedsUpdate ?? true) {
await routing.applyEventOps(merged.survivor!, mergeOps, target, batchId)
}
// A post-merge update addressed by the merged-away id, still in the
// same batch: both backends must land it on the survivor.
const viaOldId = await routing.fetchForUpdate(teamId, source, batchId)
expect(viaOldId?.uuid).toBe(uuidFromDistinctId(teamId, target))
await routing.applyEventOps(viaOldId!, ops({ $set: { postMerge: 'yes' } }), source, batchId)
await routing.flush()
routing.releaseBatch(batchId)
batchId += 1
const finalState = await routing.fetchForUpdate(teamId, target, batchId)
expect(finalState?.properties).toMatchObject({
shared: 'target',
origin: 'target',
extra: 'source',
updated: 'pre-merge',
mergedBy: 'event',
postMerge: 'yes',
})
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
// The durable rows behind both ids: the main tables against the
// validation tables the identity service and the writer maintain.
await expectDurableRowParity(target)
await expectDurableRowParity(source)
})
it.each([['before'], ['after']])(
"another pod's pending update to a merged-away id lands on the survivor when flushed %s the merge's own update",
async (order) => {
const target = id(`pods-${order}-target`)
const source = id(`pods-${order}-source`)
await createThroughBoth(target, { origin: 'target' }, batchId)
await createThroughBoth(source, { origin: 'source', extra: 'source' }, batchId)
// The other pod reads the source and holds an update to it, with
// a newer last-seen, while this pod merges the source away. Its
// flush finds no row for the person it cached: Postgres re-targets
// from the distinct id, personhog redirects to the survivor.
const other = newPod()
const held = await other.fetchForUpdate(teamId, source, batchId)
expect(held?.uuid).toBe(uuidFromDistinctId(teamId, source))
const otherOps = ops({ $set: { fromOtherPod: 'yes' } })
otherOps.lastSeenAtMs = DateTime.utc().plus({ hours: 1 }).startOf('hour').toMillis()
await other.applyEventOps(held!, otherOps, source, batchId)
const { result, mergeOps } = await runMerge(target, [source], { set: { mergedBy: 'this pod' } })
expect(result.results[0]?.outcome).toBe('merged')
if (order === 'before') {
await other.flush()
}
await applyMergeFollowUp(result, mergeOps, target)
await routing.flush()
if (order === 'after') {
await other.flush()
}
other.releaseBatch(batchId)
routing.releaseBatch(batchId)
batchId += 1
const survivor = await routing.fetchForUpdate(teamId, target, batchId)
expect(survivor?.properties).toMatchObject({
origin: 'target',
extra: 'source',
mergedBy: 'this pod',
fromOtherPod: 'yes',
})
expect(survivor?.last_seen_at?.toMillis()).toBe(otherOps.lastSeenAtMs)
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
await expectDurableRowParity(target)
await expectDurableRowParity(source)
}
)
it('a merged-away id reads the survivor on both backends', async () => {
const target = id('heal-target')
const source = id('heal-source')
await createThroughBoth(target, {}, batchId)
await createThroughBoth(source, {}, batchId)
const merged = await routing.mergePersons(
{
teamId,
targetDistinctId: target,
sources: [{ distinctId: source, eventUuid: new UUIDT().toString() }],
eventOps: ops({}, '$identify'),
eventUuid: new UUIDT().toString(),
allowIdentifiedSources: false,
mergeMode: createDefaultSyncMergeMode(),
createdAtMs: Date.now(),
},
batchId
)
await merged.kafkaAck
routing.releaseBatch(batchId)
// The purge dropped the personhog side's cache for both ids, so this
// read re-resolves through the real identity service, whose mappings
// the saga repointed; Postgres reads its own moved rows.
batchId += 1
const viaSource = await routing.fetchForUpdate(teamId, source, batchId)
expect(viaSource?.uuid).toBe(uuidFromDistinctId(teamId, target))
expect(await divergences()).toBe(0)
expect(await shadowErrors()).toBe(0)
})
})