Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.1 KB
Raw Blame
import {
    FeatureFlagCalledDedupClaim,
    FeatureFlagCalledDedupService,
    featureFlagCalledDedupKey,
} from '~/ingestion/common/feature-flag-called-dedup/feature-flag-called-dedup-service'
import { featureFlagCalledDedupEventsTotal } from '~/ingestion/common/feature-flag-called-dedup/metrics'
import { PipelineResult, drop, ok } from '~/ingestion/framework/results'
import { PluginEvent } from '~/plugin-scaffold'
import { Team } from '~/types'

export interface DedupeFeatureFlagCalledStepInput {
    event: PluginEvent
    team: Team
}

/**
 * Drops redundant $feature_flag_called events using a keep-first Redis claim
 * keyed on (team, distinct_id, flag, response, groups, has_experiment).
 * Server-side SDKs on multi-process fleets re-emit the same exposure from every
 * worker; only the first copy carries signal, so the survivors preserve
 * experiment exposure semantics while the bulk of the volume is dropped.
 * Note that "last called" timestamps become TTL-granular: within a dedup
 * window they reflect the first call, not the most recent one.
 *
 * Claims are tagged with the event uuid, so a batch replayed by at-least-once
 * Kafka delivery recognizes its own claims from a failed prior attempt
 * instead of dropping events that were never written.
 *
 * Must run after cookieless processing (which rewrites event.distinct_id) and
 * before person processing, so dropped events skip the person and ClickHouse
 * writes entirely.
 *
 * In 'shadow' mode the claim is made and counted but nothing is dropped. If no
 * service is provided, this step is a passthrough. Redis failures fail open
 * inside the service: every event passes.
 */
export function createDedupeFeatureFlagCalledStep<T extends DedupeFeatureFlagCalledStepInput>(
    dedupService?: FeatureFlagCalledDedupService
) {
    return async function dedupeFeatureFlagCalledStep(inputs: T[]): Promise<PipelineResult<T>[]> {
        if (!dedupService || inputs.length === 0) {
            return inputs.map((input) => ok(input))
        }

        const claimPerInput: (FeatureFlagCalledDedupClaim | null)[] = inputs.map((input) => {
            if (input.event.event !== '$feature_flag_called' || !dedupService.isEnabledForTeam(input.team.id)) {
                return null
            }
            const properties = input.event.properties ?? {}
            const flagKey = properties['$feature_flag']
            // Without a uuid there is no stable claim identity, so pass the event through.
            if (typeof flagKey !== 'string' || !input.event.uuid) {
                return null
            }
            return {
                key: featureFlagCalledDedupKey(
                    input.team.id,
                    input.event.distinct_id,
                    flagKey,
                    properties['$feature_flag_response'],
                    properties['$groups'],
                    properties['$feature_flag_has_experiment']
                ),
                claimId: input.event.uuid,
            }
        })

        const claims = claimPerInput.filter((claim): claim is FeatureFlagCalledDedupClaim => claim !== null)
        if (claims.length === 0) {
            return inputs.map((input) => ok(input))
        }

        const claimed = await dedupService.claimKeys(claims)

        // `claimed` has exactly one entry per non-null claim, in input order.
        // A missing entry fails open: a short result must never drop events.
        let claimIndex = 0
        return inputs.map((input, index) => {
            if (claimPerInput[index] === null) {
                return ok(input)
            }
            const isFirst = claimed[claimIndex++] ?? true
            if (isFirst) {
                featureFlagCalledDedupEventsTotal.labels('first_seen').inc()
                return ok(input)
            }
            if (dedupService.mode === 'drop') {
                featureFlagCalledDedupEventsTotal.labels('duplicate_dropped').inc()
                return drop('feature_flag_called_duplicate')
            }
            featureFlagCalledDedupEventsTotal.labels('duplicate_shadow').inc()
            return ok(input)
        })
    }
}