Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

53.4 KB
Raw Blame
import { DateTime } from 'luxon'
import pLimit from 'p-limit'

import {
    ClickhouseGroupRepository,
    GroupClickhouseMessage,
} from '~/common/groups/repositories/clickhouse-group-repository'
import { GroupRepositoryTransaction } from '~/common/groups/repositories/group-repository-transaction.interface'
import { GroupCreateResult, GroupRepository } from '~/common/groups/repositories/group-repository.interface'
import { logger } from '~/common/utils/logger'
import { promiseRetry } from '~/common/utils/retries'
import { RaceConditionError } from '~/common/utils/utils'
import { BatchWritingStoreFlushStats } from '~/ingestion/common/stores/batch-writing-store'
import { Properties } from '~/plugin-scaffold'
import { Group, GroupTypeIndex, TeamId } from '~/types'

import { logMissingRow, logVersionMismatch } from './group-logging'
import { CacheMetrics, GroupFlushResult, GroupStore } from './group-store.interface'
import { GroupUpdate, calculateUpdate, fromGroup } from './group-update'
import {
    groupBatchCreateExecutedCounter,
    groupBatchCreateFallbackCounter,
    groupBatchCreateSizeHistogram,
    groupCacheOperationsCounter,
    groupCacheSizeHistogram,
    groupDatabaseOperationsPerBatchHistogram,
    groupFetchPromisesCacheOperationsCounter,
    groupOptimisticUpdateConflictsPerBatchCounter,
} from './metrics'

class GroupCache {
    private cache: Map<string, GroupUpdate | null>
    private fetchPromises: Map<string, Promise<GroupUpdate | null>>
    private metrics: CacheMetrics
    private batchGroupKeys: Map<number, Set<string>>
    private groupKeyRefCount: Map<string, number>
    private deferredEvictions: Set<string>
    private pendingPrefetchesByBatchId: Map<number, number>
    private releasedBatchIdsWithPendingPrefetch: Set<number>

    constructor() {
        this.cache = new Map()
        this.fetchPromises = new Map()
        this.batchGroupKeys = new Map()
        this.groupKeyRefCount = new Map()
        this.deferredEvictions = new Set()
        this.pendingPrefetchesByBatchId = new Map()
        this.releasedBatchIdsWithPendingPrefetch = new Set()
        this.metrics = {
            cacheHits: 0,
            cacheMisses: 0,
        }
    }

    obtainForBatchId(batchId: number): BatchBoundGroupCache {
        return new BatchBoundGroupCache(this, batchId)
    }

    get(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): GroupUpdate | null | undefined {
        return this.cache.get(this.getCacheKey(teamId, groupTypeIndex, groupKey))
    }

    hasForBatch(batchId: number, teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): boolean {
        this.trackBatchEntry(batchId, teamId, groupTypeIndex, groupKey)
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        return this.cache.has(key)
    }

    getForBatch(
        batchId: number,
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string
    ): GroupUpdate | null | undefined {
        this.trackBatchEntry(batchId, teamId, groupTypeIndex, groupKey)
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        const result = this.cache.get(key)
        if (result !== undefined) {
            this.metrics.cacheHits++
        } else {
            this.metrics.cacheMisses++
        }
        return result
    }

    setForBatch(
        batchId: number,
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        group: GroupUpdate | null
    ): void {
        this.trackBatchEntry(batchId, teamId, groupTypeIndex, groupKey)
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        this.cache.set(key, group)
    }

    delete(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): void {
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        this.cache.delete(key)
    }

    getFetchPromise(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string
    ): Promise<GroupUpdate | null> | undefined {
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        return this.fetchPromises.get(key)
    }

    setFetchPromise(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        promise: Promise<GroupUpdate | null>
    ): void {
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        this.fetchPromises.set(key, promise)
    }

    deleteFetchPromise(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): void {
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        this.fetchPromises.delete(key)
    }

    getMetrics(): CacheMetrics {
        return this.metrics
    }

    getSize(): number {
        return this.cache.size
    }

    entries(): IterableIterator<[string, GroupUpdate | null]> {
        return this.cache.entries()
    }

    getFlushStats(): BatchWritingStoreFlushStats {
        const dirtyGroupKeys = new Set<string>()
        for (const [groupKey, update] of this.cache.entries()) {
            if (update?.needsWrite) {
                dirtyGroupKeys.add(groupKey)
            }
        }

        const referencedBatchIds = new Set<number>()
        for (const [batchId, groupKeys] of this.batchGroupKeys.entries()) {
            for (const groupKey of groupKeys) {
                if (dirtyGroupKeys.has(groupKey)) {
                    referencedBatchIds.add(batchId)
                    break
                }
            }
        }

        return {
            dirtyEntryCount: dirtyGroupKeys.size,
            referencedBatchCount: referencedBatchIds.size,
            cacheEntryCount: this.cache.size,
        }
    }

    /**
     * Reset per-batch metric accumulators. The cache and in-flight fetch
     * promises persist across batches under the persistent-cache model.
     */
    resetMetrics(): void {
        this.metrics = {
            cacheHits: 0,
            cacheMisses: 0,
        }
    }

    releaseBatchId(batchId: number): void {
        const keys = this.batchGroupKeys.get(batchId)
        if (this.pendingPrefetchesByBatchId.has(batchId)) {
            this.releasedBatchIdsWithPendingPrefetch.add(batchId)
        }
        if (!keys) {
            return
        }

        for (const groupKey of keys) {
            const refCount = (this.groupKeyRefCount.get(groupKey) ?? 1) - 1
            if (refCount <= 0) {
                this.groupKeyRefCount.delete(groupKey)
                this.evictGroupKey(groupKey)
            } else {
                this.groupKeyRefCount.set(groupKey, refCount)
            }
        }

        this.batchGroupKeys.delete(batchId)
    }

    trackPendingPrefetch(batchIds: Set<number>): void {
        for (const batchId of batchIds) {
            this.pendingPrefetchesByBatchId.set(batchId, (this.pendingPrefetchesByBatchId.get(batchId) ?? 0) + 1)
        }
    }

    finishPendingPrefetch(batchIds: Set<number>): void {
        for (const batchId of batchIds) {
            const pendingCount = (this.pendingPrefetchesByBatchId.get(batchId) ?? 1) - 1
            if (pendingCount <= 0) {
                this.pendingPrefetchesByBatchId.delete(batchId)
                this.releasedBatchIdsWithPendingPrefetch.delete(batchId)
            } else {
                this.pendingPrefetchesByBatchId.set(batchId, pendingCount)
            }
        }
    }

    isBatchReleasedWithPendingPrefetch(batchId: number): boolean {
        return this.releasedBatchIdsWithPendingPrefetch.has(batchId)
    }

    processDeferredEvictions(): void {
        for (const groupKey of this.deferredEvictions) {
            const update = this.cache.get(groupKey)
            if (!update || !update.needsWrite) {
                this.cache.delete(groupKey)
                this.deferredEvictions.delete(groupKey)
            }
        }
    }

    // The key must carry group_type_index: two group types can share a group
    // key, and a key without the index would make them read and overwrite
    // each other's cached row.
    private getCacheKey(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): string {
        return `${teamId}:${groupTypeIndex}:${groupKey}`
    }

    private trackBatchEntry(batchId: number, teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): void {
        const key = this.getCacheKey(teamId, groupTypeIndex, groupKey)
        let keys = this.batchGroupKeys.get(batchId)
        if (!keys) {
            keys = new Set()
            this.batchGroupKeys.set(batchId, keys)
        }
        if (!keys.has(key)) {
            keys.add(key)
            this.groupKeyRefCount.set(key, (this.groupKeyRefCount.get(key) ?? 0) + 1)
        }
    }

    private evictGroupKey(groupKey: string): void {
        const update = this.cache.get(groupKey)
        if (update && update.needsWrite) {
            this.deferredEvictions.add(groupKey)
            return
        }

        this.cache.delete(groupKey)
    }
}

class BatchBoundGroupCache {
    constructor(
        private readonly cache: GroupCache,
        private readonly batchId: number
    ) {}

    has(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): boolean {
        return this.cache.hasForBatch(this.batchId, teamId, groupTypeIndex, groupKey)
    }

    get(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): GroupUpdate | null | undefined {
        return this.cache.getForBatch(this.batchId, teamId, groupTypeIndex, groupKey)
    }

    set(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string, group: GroupUpdate | null): void {
        this.cache.setForBatch(this.batchId, teamId, groupTypeIndex, groupKey, group)
    }

    getFetchPromise(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string
    ): Promise<GroupUpdate | null> | undefined {
        return this.cache.getFetchPromise(teamId, groupTypeIndex, groupKey)
    }

    setFetchPromise(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        promise: Promise<GroupUpdate | null>
    ): void {
        this.cache.setFetchPromise(teamId, groupTypeIndex, groupKey, promise)
    }

    deleteFetchPromise(teamId: TeamId, groupTypeIndex: GroupTypeIndex, groupKey: string): void {
        this.cache.deleteFetchPromise(teamId, groupTypeIndex, groupKey)
    }
}

interface PropertiesUpdate {
    updated: boolean
    properties: Properties
}

export interface BatchWritingGroupStoreOptions {
    maxConcurrentUpdates: number
    maxOptimisticUpdateRetries: number
    optimisticUpdateRetryInterval: number
    /**
     * When true, flush writes all dirty groups in a single UNNEST statement
     * with server-side jsonb merge semantics (no version CAS, no conflict
     * retries). When false, each group is written individually with an
     * optimistic version check.
     */
    useBatchUpdates: boolean
    /**
     * When true, groups that don't exist yet are registered as virtual cache
     * entries and inserted at flush time in a single batched statement,
     * instead of an inline single-row insert per new group during event
     * processing.
     */
    useBatchCreates: boolean
    /**
     * Interval at which accumulated group operation metrics are emitted and
     * cleared. Set to 0 to disable the timer (used by tests; production
     * always wants a positive interval).
     */
    metricEmissionIntervalMs: number
}

const DEFAULT_OPTIONS: BatchWritingGroupStoreOptions = {
    maxConcurrentUpdates: 10,
    maxOptimisticUpdateRetries: 5,
    optimisticUpdateRetryInterval: 50,
    useBatchUpdates: true,
    useBatchCreates: false,
    metricEmissionIntervalMs: 30_000,
}

interface PendingGroupWrite {
    update: GroupUpdate
    /** Delta captured (and reset) at the flush linearization point. */
    propertiesToSet: Properties
    cacheKey: string
}

/**
 * Writes groups to the database in batches, accumulating all changes for the
 * same group across events and flushing them on `flush()` calls. The cache
 * persists across batches under concurrentBatches > 1.
 *
 * **Lifecycle:** construction starts a metric-emission timer. Callers MUST
 * invoke `shutdown()` on graceful exit to stop the timer and flush any
 * remaining dirty entries.
 */
export class BatchWritingGroupStore implements GroupStore {
    private groupCache: GroupCache
    /** ClickHouse messages queued by create/fallback paths, drained by flush(). */
    private pendingFlushResults: GroupFlushResult[] = []
    private databaseOperationCounts: Map<string, number>
    private options: BatchWritingGroupStoreOptions
    private groupRepository: GroupRepository
    private clickhouseGroupRepository: ClickhouseGroupRepository
    // Periodic metric emitter — emits accumulated group-operation metrics on
    // a fixed cadence rather than at batch boundaries.
    private metricEmissionTimer: NodeJS.Timeout | undefined

    constructor(
        groupRepository: GroupRepository,
        clickhouseGroupRepository: ClickhouseGroupRepository,
        options?: Partial<BatchWritingGroupStoreOptions>
    ) {
        this.options = { ...DEFAULT_OPTIONS, ...options }
        this.groupCache = new GroupCache()
        this.databaseOperationCounts = new Map()
        this.groupRepository = groupRepository
        this.clickhouseGroupRepository = clickhouseGroupRepository

        if (this.options.metricEmissionIntervalMs > 0) {
            this.metricEmissionTimer = setInterval(
                () => this.emitAccumulatedMetrics(),
                this.options.metricEmissionIntervalMs
            )
            this.metricEmissionTimer.unref?.()
        }
    }

    getGroupCache(): GroupCache {
        return this.groupCache
    }

    async flush(): Promise<GroupFlushResult[]> {
        // SYNCHRONOUS LINEARIZATION POINT for cross-batch correctness.
        // Walk every dirty entry, capture it (and its delta) for writing, and
        // clear `needsWrite` before any await below. Concurrent batches that
        // mutate an entry between this clear and the async DB write will
        // re-set `needsWrite=true` and be picked up by the next flush.
        // DO NOT introduce any `await` inside this block.
        const pendingWrites: PendingGroupWrite[] = []
        const pendingCreates: PendingGroupWrite[] = []
        for (const [key, update] of this.groupCache.entries()) {
            if (!update) {
                continue
            }
            if (!update.needsWrite) {
                continue
            }
            update.needsWrite = false
            const propertiesToSet = update.properties_to_set
            update.properties_to_set = {}
            if (update.pendingCreate) {
                // Clearing here means exactly one flush owns the INSERT: a
                // concurrent flush that captures later dirtying of this key
                // sees a regular update, which converges through the
                // individual-write path even if the insert hasn't landed yet.
                update.pendingCreate = false
                pendingCreates.push({ update, propertiesToSet, cacheKey: key })
            } else {
                pendingWrites.push({ update, propertiesToSet, cacheKey: key })
            }
        }
        // END synchronous linearization point.

        if (pendingWrites.length === 0 && pendingCreates.length === 0) {
            this.groupCache.processDeferredEvictions()
            return this.drainPendingFlushResults()
        }

        try {
            // A key is in exactly one of the two lists, so creates and
            // updates touch disjoint rows and can run concurrently.
            const [createResults, updateResults] = await Promise.all([
                pendingCreates.length > 0 ? this.flushCreates(pendingCreates) : [],
                pendingWrites.length === 0
                    ? []
                    : this.options.useBatchUpdates
                      ? this.flushBatch(pendingWrites)
                      : this.flushIndividual(pendingWrites),
            ])
            this.groupCache.processDeferredEvictions()
            // Drained after the writes so messages queued by fallback paths
            // during this flush ride this flush's side effects too.
            return [...createResults, ...updateResults, ...this.drainPendingFlushResults()]
        } catch (error) {
            logger.error('Failed to flush group updates', {
                error,
                errorMessage: error instanceof Error ? error.message : String(error),
                errorStack: error instanceof Error ? error.stack : undefined,
            })
            throw error
        }
    }

    /**
     * Write every dirty group individually with an optimistic version check.
     */
    private async flushIndividual(pendingWrites: PendingGroupWrite[]): Promise<GroupFlushResult[]> {
        const limit = pLimit(this.options.maxConcurrentUpdates)
        const results = await Promise.all(
            pendingWrites.map(({ update, cacheKey }) => limit(() => this.processGroupUpdate(update, cacheKey)))
        )
        return results.filter((result): result is GroupFlushResult => result !== null)
    }

    /**
     * Write all dirty groups in a single UNNEST statement with server-side
     * jsonb merge semantics. Groups missing from the result (deleted or never
     * created) fall back to the individual path, which creates them via the
     * direct-upsert fallback. A failure of the whole statement falls back to
     * individual optimistic writes of the full property view, so no captured
     * delta is lost.
     */
    private async flushBatch(pendingWrites: PendingGroupWrite[]): Promise<GroupFlushResult[]> {
        this.incrementDatabaseOperation('updateGroupsBatch')

        let updatedGroups: Group[]
        try {
            updatedGroups = await this.groupRepository.updateGroupsBatch(
                pendingWrites.map(({ update, propertiesToSet }) => ({
                    teamId: update.team_id,
                    groupTypeIndex: update.group_type_index,
                    groupKey: update.group_key,
                    propertiesToSet,
                    createdAt: update.created_at,
                }))
            )
        } catch (error) {
            logger.warn('⚠️', 'Batch group update failed, falling back to individual updates', {
                count: pendingWrites.length,
                error,
                errorMessage: error instanceof Error ? error.message : String(error),
            })
            return await this.flushIndividual(pendingWrites)
        }

        const updatedByKey = new Map<string, Group>()
        for (const group of updatedGroups) {
            updatedByKey.set(this.rowIdentity(group), group)
        }

        const results: GroupFlushResult[] = []
        const missingWrites: PendingGroupWrite[] = []

        for (const pendingWrite of pendingWrites) {
            const row = updatedByKey.get(this.rowIdentity(pendingWrite.update))
            if (!row) {
                missingWrites.push(pendingWrite)
                continue
            }

            this.syncCacheEntryFromRow(pendingWrite.update, row)
            results.push(this.buildFlushResultFromRow(row))
        }

        if (missingWrites.length > 0) {
            results.push(...(await this.flushIndividual(missingWrites)))
        }

        return results
    }

    /**
     * Insert all pending group creations in one UNNEST statement. Rows that
     * lost a cross-pod race come back with `inserted: false` — the statement
     * already merged their properties onto the winning row server-side, so no
     * extra convergence round trip is needed. Rows absent from the result
     * (not expected from Postgres, but kept as a data-loss guard) converge
     * through the individual update path. A failure of the whole statement
     * falls back to per-row inserts (the pre-batching behavior), so no
     * captured creation is lost.
     */
    private async flushCreates(pendingCreates: PendingGroupWrite[]): Promise<GroupFlushResult[]> {
        this.incrementDatabaseOperation('insertGroupsBatch')
        groupBatchCreateSizeHistogram.observe(pendingCreates.length)

        let insertedGroups: GroupCreateResult[]
        try {
            insertedGroups = await this.groupRepository.insertGroupsBatch(
                pendingCreates.map(({ update, propertiesToSet }) => ({
                    teamId: update.team_id,
                    groupTypeIndex: update.group_type_index,
                    groupKey: update.group_key,
                    groupProperties: propertiesToSet,
                    createdAt: update.created_at,
                }))
            )
        } catch (error) {
            groupBatchCreateFallbackCounter.inc({ reason: 'statement_failed' })
            logger.warn('⚠️', 'Batch group create failed, falling back to individual inserts', {
                count: pendingCreates.length,
                error,
                errorMessage: error instanceof Error ? error.message : String(error),
            })
            const limit = pLimit(this.options.maxConcurrentUpdates)
            const results = await Promise.all(
                pendingCreates.map((pendingCreate) => limit(() => this.createGroupIndividually(pendingCreate)))
            )
            return results.filter((result): result is GroupFlushResult => result !== null)
        }

        groupBatchCreateExecutedCounter.inc()

        const insertedByKey = new Map<string, GroupCreateResult>()
        for (const group of insertedGroups) {
            insertedByKey.set(this.rowIdentity(group), group)
        }

        const results: GroupFlushResult[] = []
        const missingWrites: PendingGroupWrite[] = []

        for (const pendingCreate of pendingCreates) {
            const row = insertedByKey.get(this.rowIdentity(pendingCreate.update))
            if (!row) {
                missingWrites.push(pendingCreate)
                continue
            }

            if (!row.inserted) {
                groupBatchCreateFallbackCounter.inc({ reason: 'lost_race' })
            }
            this.syncCacheEntryFromRow(pendingCreate.update, row)
            results.push(this.buildFlushResultFromRow(row))
        }

        if (missingWrites.length > 0) {
            groupBatchCreateFallbackCounter.inc({ reason: 'missing_row' }, missingWrites.length)
            results.push(...(await this.flushIndividual(missingWrites)))
        }

        return results
    }

    /**
     * Per-row insert fallback used when the batched create statement fails —
     * equivalent to the inline create that runs when batch creates are
     * disabled. A row that turns out to exist converges through the
     * individual update path instead.
     */
    private async createGroupIndividually(pendingCreate: PendingGroupWrite): Promise<GroupFlushResult | null> {
        const { update, propertiesToSet, cacheKey } = pendingCreate
        try {
            this.incrementDatabaseOperation('insertGroup')
            const insertedVersion = await this.groupRepository.insertGroup(
                update.team_id,
                update.group_type_index,
                update.group_key,
                propertiesToSet,
                update.created_at,
                {},
                {}
            )

            const insertedRow = {
                group_properties: propertiesToSet,
                created_at: update.created_at,
                version: insertedVersion,
            }
            this.syncCacheEntryFromRow(update, insertedRow)
            return {
                messages: [
                    this.clickhouseGroupRepository.buildUpsertMessage(
                        update.team_id,
                        update.group_type_index,
                        update.group_key,
                        insertedRow.group_properties,
                        insertedRow.created_at,
                        insertedRow.version
                    ),
                ],
                teamId: update.team_id,
                groupTypeIndex: update.group_type_index,
                groupKey: update.group_key,
            }
        } catch (error) {
            if (error instanceof RaceConditionError) {
                groupBatchCreateFallbackCounter.inc({ reason: 'lost_race' })
                return await this.processGroupUpdate(update, cacheKey)
            }
            throw error
        }
    }

    // Key by the full row identity — unlike the cache key, which omits
    // group_type_index — so two group types sharing a group key can never
    // sync each other's row.
    private rowIdentity(group: { team_id: TeamId; group_type_index: GroupTypeIndex; group_key: string }): string {
        return `${group.team_id}:${group.group_type_index}:${group.group_key}`
    }

    private buildFlushResultFromRow(row: Group): GroupFlushResult {
        return {
            messages: [
                this.clickhouseGroupRepository.buildUpsertMessage(
                    row.team_id,
                    row.group_type_index,
                    row.group_key,
                    row.group_properties,
                    row.created_at,
                    row.version
                ),
            ],
            teamId: row.team_id,
            groupTypeIndex: row.group_type_index,
            groupKey: row.group_key,
        }
    }

    /**
     * Sync the cached entry with the authoritative row returned by a write,
     * preserving any delta accumulated by concurrent batches since the flush
     * captured this write. A row older than the entry's synced state — a
     * concurrent flush's write landed while this statement was in flight —
     * is ignored: applying it would regress version and properties and
     * manufacture optimistic-update conflicts on the next flush. Cached
     * versions only ever come from authoritative reads and writes, so the
     * comparison is sound.
     */
    private syncCacheEntryFromRow(
        update: GroupUpdate,
        row: Pick<Group, 'group_properties' | 'created_at' | 'version'>
    ): void {
        const cached = this.groupCache.get(update.team_id, update.group_type_index, update.group_key)
        const target = cached ?? update
        if (row.version < target.version) {
            return
        }
        target.group_properties = { ...row.group_properties, ...target.properties_to_set }
        target.created_at = DateTime.min(target.created_at, row.created_at)
        target.version = row.version
    }

    getFlushStats(): BatchWritingStoreFlushStats {
        return this.groupCache.getFlushStats()
    }

    private async processGroupUpdate(update: GroupUpdate, cacheKey: string): Promise<GroupFlushResult | null> {
        try {
            const message = await promiseRetry(
                () => this.executeOptimisticUpdate(update),
                'updateGroupOptimistically',
                this.options.maxOptimisticUpdateRetries,
                // Jitter the starting interval so pods that conflicted on the
                // same group don't retry in lockstep and re-collide.
                this.options.optimisticUpdateRetryInterval * (0.5 + Math.random())
            )
            if (!message) {
                return null
            }
            return {
                messages: [message],
                teamId: update.team_id,
                groupTypeIndex: update.group_type_index,
                groupKey: update.group_key,
            }
        } catch (error) {
            await this.handleOptimisticUpdateFailure(error, update, cacheKey)
            return null
        }
    }

    private async handleOptimisticUpdateFailure(error: unknown, update: GroupUpdate, cacheKey: string): Promise<void> {
        logger.warn('⚠️', 'Optimistic group update failed after max retries', {
            error,
            teamId: update.team_id,
            groupTypeIndex: update.group_type_index,
            groupKey: update.group_key,
            cacheKey,
        })
        await this.fallbackToDirectUpsert(update, cacheKey)
    }

    private async fallbackToDirectUpsert(update: GroupUpdate, cacheKey: string): Promise<void> {
        logger.warn('⚠️', 'Falling back to direct upsert after max retries', {
            teamId: update.team_id,
            groupTypeIndex: update.group_type_index,
            groupKey: update.group_key,
            cacheKey,
        })

        // Remove from cache to prevent retry
        this.groupCache.delete(update.team_id, update.group_type_index, update.group_key)

        try {
            await this.executeGroupUpsert(
                update.team_id,
                update.group_type_index,
                update.group_key,
                update.group_properties,
                update.created_at,
                true, // forUpdate = true, making us not use the cache
                'conflictRetry',
                undefined
            )
        } catch (fallbackError) {
            logger.error('Failed to update group after max retries and direct upsert fallback', {
                error: fallbackError,
                teamId: update.team_id,
                groupTypeIndex: update.group_type_index,
                groupKey: update.group_key,
                errorMessage: fallbackError instanceof Error ? fallbackError.message : String(fallbackError),
                errorStack: fallbackError instanceof Error ? fallbackError.stack : undefined,
            })
            throw fallbackError
        }
    }

    async upsertGroup(
        teamId: TeamId,
        projectId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        batchId?: number
    ): Promise<void> {
        const effectiveBatchId = batchId ?? 0
        try {
            await this.addToBatch(teamId, groupTypeIndex, groupKey, properties, timestamp, effectiveBatchId)
        } catch (error) {
            await this.handleUpsertError(
                error,
                teamId,
                projectId,
                groupTypeIndex,
                groupKey,
                properties,
                timestamp,
                effectiveBatchId
            )
        }
    }

    private async addToBatch(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        batchId: number
    ): Promise<void> {
        const groupCache = this.groupCache.obtainForBatchId(batchId)
        const group = await this.getGroup(teamId, groupTypeIndex, groupKey, false, groupCache)

        if (!group) {
            if (this.options.useBatchCreates) {
                this.queuePendingCreate(teamId, groupTypeIndex, groupKey, properties, timestamp, groupCache)
                return
            }
            await this.createGroup(teamId, groupTypeIndex, groupKey, properties, timestamp, batchId)
            return
        }

        const propertiesUpdate = calculateUpdate(group.group_properties || {}, properties)
        if (propertiesUpdate.updated) {
            groupCache.set(teamId, groupTypeIndex, groupKey, {
                team_id: teamId,
                group_type_index: groupTypeIndex,
                group_key: groupKey,
                group_properties: propertiesUpdate.properties,
                // Accumulate the delta since the last DB sync — a flush in
                // flight has already captured (and reset) the previous delta.
                properties_to_set: { ...group.properties_to_set, ...propertiesUpdate.changedProperties },
                created_at: group.created_at,
                version: group.version,
                needsWrite: true,
                // Carried over so an accumulated-onto entry still routes to
                // the create list; false once a flush has claimed the insert.
                pendingCreate: group.pendingCreate,
            })
        }
    }

    /**
     * Register a group the cache says doesn't exist as a virtual entry; the
     * next flush inserts all pending creations in one batched statement.
     * Later events for the key accumulate onto the entry through the regular
     * update path, so per-lane ordering matches the inline-create flow.
     */
    private queuePendingCreate(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        groupCache: BatchBoundGroupCache
    ): void {
        groupCache.set(teamId, groupTypeIndex, groupKey, {
            team_id: teamId,
            group_type_index: groupTypeIndex,
            group_key: groupKey,
            group_properties: { ...properties },
            properties_to_set: { ...properties },
            created_at: DateTime.min(DateTime.now(), timestamp),
            version: 0,
            needsWrite: true,
            pendingCreate: true,
        })
    }

    /**
     * Create a group that the cache says doesn't exist. Runs without a
     * wrapping transaction — the insert is a single ON CONFLICT DO NOTHING
     * statement, so the transaction added round trips without atomicity value.
     * A racing create surfaces as RaceConditionError, which handleUpsertError
     * turns into a cache refresh and upsert retry.
     */
    private async createGroup(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        batchId: number
    ): Promise<void> {
        const createdAt = DateTime.min(DateTime.now(), timestamp)

        this.incrementDatabaseOperation('insertGroup')
        const insertedVersion = await this.groupRepository.insertGroup(
            teamId,
            groupTypeIndex,
            groupKey,
            properties,
            createdAt,
            {},
            {}
        )

        this.groupCache.obtainForBatchId(batchId).set(teamId, groupTypeIndex, groupKey, {
            team_id: teamId,
            group_type_index: groupTypeIndex,
            group_key: groupKey,
            group_properties: properties,
            properties_to_set: {},
            created_at: createdAt,
            version: insertedVersion,
            needsWrite: false,
        })

        // The ClickHouse message rides the next flush's side effects instead
        // of being awaited here: delivery reports on the downstream producer
        // can take ~seconds under backpressure, and awaiting one inline
        // serializes the per-distinct-id lane (observed in production with
        // create-heavy migration traffic).
        this.queueClickhouseMessage(
            teamId,
            groupTypeIndex,
            groupKey,
            properties,
            createdAt,
            insertedVersion,
            'batch-create'
        )
    }

    private async executeGroupUpsert(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        forUpdate: boolean,
        source: string,
        batchId?: number
    ): Promise<void> {
        const operation = 'upsertGroup' + (source ? `-${source}` : '')
        this.incrementDatabaseOperation(operation)

        const [propertiesUpdate, createdAt, actualVersion] = await this.groupRepository.inTransaction(
            operation,
            async (tx) =>
                this.executeUpsertTransaction(
                    teamId,
                    groupTypeIndex,
                    groupKey,
                    properties,
                    timestamp,
                    forUpdate,
                    tx,
                    batchId
                )
        )

        if (propertiesUpdate.updated) {
            this.queueClickhouseMessage(
                teamId,
                groupTypeIndex,
                groupKey,
                propertiesUpdate.properties,
                createdAt,
                actualVersion,
                source
            )
        }
    }

    /**
     * Queue a group's ClickHouse message for the next flush, which returns it
     * for side-effect production (awaited before the owning batch's offset
     * commit). ReplacingMergeTree ordering by version makes produce order
     * across flushes irrelevant.
     */
    private queueClickhouseMessage(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        createdAt: DateTime,
        actualVersion: number,
        source: string
    ): void {
        this.incrementDatabaseOperation('upsertClickhouse' + (source ? `-${source}` : ''))
        this.pendingFlushResults.push({
            messages: [
                this.clickhouseGroupRepository.buildUpsertMessage(
                    teamId,
                    groupTypeIndex,
                    groupKey,
                    properties,
                    createdAt,
                    actualVersion
                ),
            ],
            teamId,
            groupTypeIndex,
            groupKey,
        })
    }

    private drainPendingFlushResults(): GroupFlushResult[] {
        const pending = this.pendingFlushResults
        this.pendingFlushResults = []
        return pending
    }

    private async executeUpsertTransaction(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        forUpdate: boolean,
        tx: GroupRepositoryTransaction,
        batchId?: number
    ): Promise<[PropertiesUpdate, DateTime, number]> {
        const groupCache = batchId === undefined ? undefined : this.groupCache.obtainForBatchId(batchId)
        const group = await this.getGroup(teamId, groupTypeIndex, groupKey, forUpdate, groupCache, tx)
        const createdAt = DateTime.min(group?.created_at || DateTime.now(), timestamp)
        const expectedVersion = (group?.version || 0) + 1
        const propertiesUpdate = calculateUpdate(group?.group_properties || {}, properties)

        if (!group) {
            propertiesUpdate.updated = true
        }

        let actualVersion = expectedVersion

        if (propertiesUpdate.updated) {
            if (group) {
                actualVersion = await this.updateGroup(
                    teamId,
                    groupTypeIndex,
                    groupKey,
                    propertiesUpdate.properties,
                    createdAt,
                    expectedVersion,
                    'upsertGroup',
                    tx
                )
            } else {
                actualVersion = await this.insertGroup(
                    teamId,
                    groupTypeIndex,
                    groupKey,
                    propertiesUpdate.properties,
                    createdAt,
                    expectedVersion,
                    tx
                )
                groupCache?.set(teamId, groupTypeIndex, groupKey, {
                    team_id: teamId,
                    group_type_index: groupTypeIndex,
                    group_key: groupKey,
                    group_properties: propertiesUpdate.properties,
                    properties_to_set: {},
                    created_at: createdAt,
                    version: actualVersion,
                    needsWrite: false,
                })
            }
        }

        return [propertiesUpdate, createdAt, actualVersion]
    }

    private async updateGroup(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        createdAt: DateTime,
        expectedVersion: number,
        tag: string,
        tx: GroupRepositoryTransaction
    ): Promise<number> {
        this.incrementDatabaseOperation('updateGroup')
        const updatedVersion = await tx.updateGroup(
            teamId,
            groupTypeIndex,
            groupKey,
            properties,
            createdAt,
            {},
            {},
            tag
        )

        if (updatedVersion !== undefined) {
            const versionDisparity = updatedVersion - expectedVersion
            if (versionDisparity > 0) {
                logVersionMismatch(teamId, groupTypeIndex, groupKey, versionDisparity)
            }
            return updatedVersion
        } else {
            logMissingRow(teamId, groupTypeIndex, groupKey)
            return expectedVersion
        }
    }

    private async insertGroup(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        createdAt: DateTime,
        expectedVersion: number,
        tx: GroupRepositoryTransaction
    ): Promise<number> {
        this.incrementDatabaseOperation('insertGroup')
        const insertedVersion = await tx.insertGroup(teamId, groupTypeIndex, groupKey, properties, createdAt, {}, {})
        const versionDisparity = insertedVersion - expectedVersion
        if (versionDisparity > 0) {
            logVersionMismatch(teamId, groupTypeIndex, groupKey, versionDisparity)
        }
        return insertedVersion
    }

    private async executeOptimisticUpdate(update: GroupUpdate): Promise<GroupClickhouseMessage | null> {
        this.incrementDatabaseOperation('updateGroupOptimistically')
        const actualVersion = await this.groupRepository.updateGroupOptimistically(
            update.team_id,
            update.group_type_index,
            update.group_key,
            update.version,
            update.group_properties,
            update.created_at,
            {},
            {}
        )

        if (actualVersion !== undefined) {
            // Keep the cached entry's version in sync with the row we just
            // wrote, so the next flush of this entry doesn't CAS against a
            // stale version and manufacture a conflict.
            update.version = actualVersion
            // The ClickHouse produce is returned to the caller and scheduled as
            // a side effect after flush, so flush latency stays DB-only.
            return this.clickhouseGroupRepository.buildUpsertMessage(
                update.team_id,
                update.group_type_index,
                update.group_key,
                update.group_properties,
                update.created_at,
                actualVersion
            )
        }

        groupOptimisticUpdateConflictsPerBatchCounter.inc()
        this.incrementDatabaseOperation('fetchGroup')
        const latestGroup = await this.groupRepository.fetchGroup(
            update.team_id,
            update.group_type_index,
            update.group_key,
            { callerTag: 'ingestion/group-update-conflict' }
        )
        if (latestGroup) {
            const propertiesUpdate = calculateUpdate(latestGroup.group_properties || {}, update.group_properties)
            if (!propertiesUpdate.updated && update.created_at >= latestGroup.created_at) {
                // The winning writer already persisted everything this update
                // carries — common for hot groups receiving identical
                // $group_set payloads across pods. Sync to the winning row and
                // skip the write (and its ClickHouse produce) entirely.
                update.group_properties = latestGroup.group_properties || {}
                update.created_at = latestGroup.created_at
                update.version = latestGroup.version
                return null
            }
            if (propertiesUpdate.updated) {
                update.group_properties = propertiesUpdate.properties
            }
            update.version = latestGroup.version
        }
        throw new Error('Optimistic update failed, will retry')
    }

    /**
     * Best-effort cache warmer for the given group keys — one batched query
     * instead of a single-row fetch per key. Callers fire this without
     * awaiting it (mirrors prefetchPersons): transient persons-Postgres
     * unavailability is swallowed here so the fire-and-forget copy can't
     * crash the worker, while each per-key promise (awaited by getGroup)
     * still rejects so consumers propagate the error and retry. Any other
     * error is rethrown so it crashes loudly rather than being masked.
     */
    async prefetchGroups(
        entries: { teamId: TeamId; groupTypeIndex: GroupTypeIndex; groupKey: string; batchId: number }[]
    ): Promise<void> {
        if (entries.length === 0) {
            return
        }

        // Filter out entries that are already cached or have pending fetches.
        const uncachedEntries: { teamId: TeamId; groupTypeIndex: GroupTypeIndex; groupKey: string; batchId: number }[] =
            []
        const seenKeys = new Set<string>()
        for (const entry of entries) {
            const key = `${entry.teamId}:${entry.groupTypeIndex}:${entry.groupKey}`
            if (seenKeys.has(key)) {
                continue
            }
            seenKeys.add(key)
            if (this.groupCache.hasForBatch(entry.batchId, entry.teamId, entry.groupTypeIndex, entry.groupKey)) {
                continue
            }
            if (this.groupCache.getFetchPromise(entry.teamId, entry.groupTypeIndex, entry.groupKey)) {
                continue
            }
            uncachedEntries.push(entry)
        }

        if (uncachedEntries.length === 0) {
            return
        }

        const prefetchBatchIds = new Set(uncachedEntries.map(({ batchId }) => batchId))
        this.groupCache.trackPendingPrefetch(prefetchBatchIds)
        this.incrementDatabaseOperation('prefetchGroups')

        const batchFetchPromise = this.groupRepository
            .fetchGroupsByKeys(uncachedEntries, 'ingestion/group-prefetch')
            .then((rows) => {
                const groupsByKey = new Map<string, GroupUpdate>()
                for (const row of rows) {
                    groupsByKey.set(`${row.team_id}:${row.group_type_index}:${row.group_key}`, {
                        team_id: row.team_id,
                        group_type_index: row.group_type_index,
                        group_key: row.group_key,
                        group_properties: row.group_properties,
                        properties_to_set: {},
                        created_at: row.created_at,
                        version: row.version,
                        needsWrite: false,
                    })
                }

                // Cache all results (found groups and nulls for missing ones).
                for (const { teamId, groupTypeIndex, groupKey, batchId } of uncachedEntries) {
                    // Caching under a released batchId would re-register it in
                    // the refcount tracking with nobody left to release it.
                    if (this.groupCache.isBatchReleasedWithPendingPrefetch(batchId)) {
                        continue
                    }
                    // Don't clobber an entry that appeared while the fetch was
                    // in flight (e.g. a group created inline by event processing).
                    if (this.groupCache.get(teamId, groupTypeIndex, groupKey) !== undefined) {
                        continue
                    }
                    this.groupCache.setForBatch(
                        batchId,
                        teamId,
                        groupTypeIndex,
                        groupKey,
                        groupsByKey.get(`${teamId}:${groupTypeIndex}:${groupKey}`) ?? null
                    )
                }

                return groupsByKey
            })
            .finally(() => {
                for (const { teamId, groupTypeIndex, groupKey } of uncachedEntries) {
                    this.groupCache.deleteFetchPromise(teamId, groupTypeIndex, groupKey)
                }
                this.groupCache.finishPendingPrefetch(prefetchBatchIds)
            })

        // Register per-key promises so getGroup waits on the in-flight batch
        // fetch instead of issuing its own single-row query. The throwaway
        // catch only marks the promise handled so an unconsumed key can't
        // become an unhandled rejection — awaiting consumers still observe
        // the rejection.
        for (const { teamId, groupTypeIndex, groupKey } of uncachedEntries) {
            const keyPromise = batchFetchPromise.then(
                (groupsByKey) => groupsByKey.get(`${teamId}:${groupTypeIndex}:${groupKey}`) ?? null
            )
            keyPromise.catch(() => {})
            this.groupCache.setFetchPromise(teamId, groupTypeIndex, groupKey, keyPromise)
        }

        await batchFetchPromise.catch((error) => {
            if (error?.isRetriable === true) {
                logger.warn('⚠️', 'prefetchGroups failed on a retriable persons-Postgres error', {
                    error: String(error),
                })
                return
            }
            throw error
        })
    }

    private async getGroup(
        teamId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        forUpdate: boolean,
        groupCache: BatchBoundGroupCache | undefined,
        tx?: GroupRepositoryTransaction
    ): Promise<GroupUpdate | null> {
        if (groupCache?.has(teamId, groupTypeIndex, groupKey) && !forUpdate) {
            const cachedGroup = groupCache.get(teamId, groupTypeIndex, groupKey)
            if (cachedGroup !== undefined) {
                return cachedGroup
            }
        }

        let fetchPromise = groupCache?.getFetchPromise(teamId, groupTypeIndex, groupKey)
        if (!fetchPromise) {
            groupFetchPromisesCacheOperationsCounter.inc({ operation: 'miss' })
            fetchPromise = (async () => {
                try {
                    this.incrementDatabaseOperation('fetchGroup')
                    const repository = tx || this.groupRepository
                    const existingGroup = await repository.fetchGroup(teamId, groupTypeIndex, groupKey, { forUpdate })
                    if (existingGroup) {
                        const groupUpdate = fromGroup(existingGroup)
                        groupCache?.set(teamId, groupTypeIndex, groupKey, {
                            ...groupUpdate,
                            needsWrite: false,
                        })
                        return groupUpdate
                    } else {
                        groupCache?.set(teamId, groupTypeIndex, groupKey, null)
                        return null
                    }
                } finally {
                    groupCache?.deleteFetchPromise(teamId, groupTypeIndex, groupKey)
                }
            })()
            groupCache?.setFetchPromise(teamId, groupTypeIndex, groupKey, fetchPromise)
        } else {
            groupFetchPromisesCacheOperationsCounter.inc({ operation: 'hit' })
        }
        return fetchPromise
    }

    private async handleUpsertError(
        error: unknown,
        teamId: TeamId,
        projectId: TeamId,
        groupTypeIndex: GroupTypeIndex,
        groupKey: string,
        properties: Properties,
        timestamp: DateTime,
        batchId: number
    ): Promise<void> {
        if (error instanceof RaceConditionError) {
            // Remove from cache to prevent retry, the group was already created by another thread
            this.groupCache.delete(teamId, groupTypeIndex, groupKey)
            return this.upsertGroup(teamId, projectId, groupTypeIndex, groupKey, properties, timestamp, batchId)
        }
        throw error
    }

    releaseBatch(batchId: number): void {
        this.groupCache.releaseBatchId(batchId)
    }

    getCacheMetrics(): CacheMetrics {
        return this.groupCache.getMetrics()
    }

    private incrementDatabaseOperation(operation: string): void {
        this.databaseOperationCounts.set(operation, (this.databaseOperationCounts.get(operation) || 0) + 1)
    }

    /**
     * Emit accumulated group operation metrics to Prometheus and reset the
     * in-memory accumulators. Runs on a fixed-interval timer in production.
     *
     * Under concurrentBatches > 1, per-batch attribution is unreliable
     * (first-flush-wins), so emission is decoupled from batch boundaries.
     * The histogram names retain a "...PerBatch..." suffix for dashboard
     * compatibility but the window is now the emission interval.
     */
    private emitAccumulatedMetrics(): void {
        groupCacheSizeHistogram.observe(this.groupCache.getSize())
        const metrics = this.groupCache.getMetrics()
        groupCacheOperationsCounter.inc({ operation: 'hit' }, metrics.cacheHits)
        groupCacheOperationsCounter.inc({ operation: 'miss' }, metrics.cacheMisses)
        for (const [operation, count] of this.databaseOperationCounts.entries()) {
            groupDatabaseOperationsPerBatchHistogram.observe({ operation }, count)
        }

        this.groupCache.resetMetrics()
        this.databaseOperationCounts.clear()
        groupOptimisticUpdateConflictsPerBatchCounter.reset()
    }

    /**
     * Stop the metric-emission timer and emit accumulated metrics. Idempotent.
     *
     * Callers MUST call `flush()` before `shutdown()`. Reaching shutdown with a
     * dirty cache indicates a drain-ordering bug — writing here without a
     * subsequent offset commit would create duplicate writes when the partition
     * is reprocessed, and silently dropping the data masks the bug. We throw
     * instead and let the caller decide whether to flush, drop, or fail loudly.
     *
     * Does NOT clear the group cache. Cache eviction is intentionally
     * decoupled from this lifecycle hook.
     */
    shutdown(): Promise<void> {
        if (this.metricEmissionTimer) {
            clearInterval(this.metricEmissionTimer)
            this.metricEmissionTimer = undefined
        }

        let dirtyCount = this.pendingFlushResults.length
        for (const [_, entry] of this.groupCache.entries()) {
            if (entry && entry.needsWrite) {
                dirtyCount++
            }
        }
        if (dirtyCount > 0) {
            this.emitAccumulatedMetrics()
            throw new Error(
                `BatchWritingGroupStore.shutdown() called with ${dirtyCount} dirty cache entries — call flush() first`
            )
        }

        this.emitAccumulatedMetrics()
        return Promise.resolve()
    }
}