Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.4 KB
Raw Blame
// Pins the full flag-evaluations shadow-write path — config → fork step → output
// registration → Kafka topic → ClickHouse Kafka engine → MV → flag_evaluations
// table — which no unit test can: a topic-key typo, a row column ClickHouse can't
// parse (silently eaten by kafka_skip_broken_messages), or a broken MV projection
// all produce "row queued but never lands".
import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'

import { createHogTransformerService } from '~/cdp/hog-transformations/hog-transformer.service'
import { KAFKA_CLICKHOUSE_FLAG_EVALUATIONS, KAFKA_REALTIME_ONLY_EVENTS_JSON } from '~/common/config/kafka-topics'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { parseRawClickHouseEvent } from '~/common/utils/event'
import { parseJSON } from '~/common/utils/json-parse'
import { createFlagEvaluationsService } from '~/ingestion/common/flag-evaluations/flag-evaluations-service'
import { getDefaultIngestionConsumerConfig, getDefaultIngestionOutputsConfig } from '~/ingestion/config'
import { IngestionConsumer } from '~/ingestion/ingestion-consumer'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import {
    EventBuilder,
    createKafkaMessages,
    createTestWithTeamIngester,
    ensureIngestionE2EInfraReady,
    fetchEvents,
    fetchFlagEvaluations,
    waitForKafkaMessages,
} from '~/tests/helpers/ingestion-e2e'
import { createTestIngestionOutputs, createTestMonitoringOutputs } from '~/tests/helpers/ingestion-outputs'
import { fetchPostgresPersons } from '~/tests/helpers/sql'
import { FlagEvaluationsMode } from '~/types'

jest.mock('~/common/utils/logger')

describe('Flag evaluations shadow-routing E2E', () => {
    const testWithTeamIngester = createTestWithTeamIngester({}, (infra, kafkaProducer) => {
        const outputs = createTestIngestionOutputs(kafkaProducer)
        return new IngestionConsumer(infra.config, {
            ...infra,
            hogTransformer: createHogTransformerService(infra.config, {
                ...infra,
                monitoringOutputs: createTestMonitoringOutputs(kafkaProducer),
            }),
            outputs,
            clickhouseGroupRepository: new ClickhouseGroupRepository(outputs),
        })
    })

    let clickhouse: Clickhouse

    beforeAll(async () => {
        clickhouse = Clickhouse.create()
        await ensureIngestionE2EInfraReady()
    })

    afterAll(() => {
        clickhouse.close()
    })

    testWithTeamIngester(
        'dual_write mode: a $feature_flag_called event lands in both flag_evaluations and events',
        {
            pluginServerConfig: {
                INGESTION_FLAG_EVALUATIONS_MODE: 'dual_write',
                INGESTION_FLAG_EVALUATIONS_TEAMS: '*',
                INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: KAFKA_CLICKHOUSE_FLAG_EVALUATIONS,
            },
        },
        async ({ ingester, team, kafkaProducer, token }) => {
            const event = new EventBuilder(team)
                .withEvent('$feature_flag_called')
                .withProperties({
                    $feature_flag: 'my-flag',
                    $feature_flag_response: true,
                    $set: { email: '[email protected]' },
                })
                .build()

            await ingester.handleKafkaBatch(createKafkaMessages([event], token))
            await waitForKafkaMessages(kafkaProducer)

            const flagEvaluations = await waitForExpect(async () => {
                const rows = await fetchFlagEvaluations(clickhouse, team.id)
                expect(rows.length).toBe(1)
                return rows
            }, 30_000)

            // The two rows do not necessarily arrive on the same Kafka flush, and the
            // events row is usually the later of the two, so this read is polled too.
            const events = await waitForExpect(async () => {
                const rows = await fetchEvents(clickhouse, team.id)
                expect(rows.length).toBe(1)
                return rows
            }, 30_000)
            expect(events[0].event).toBe('$feature_flag_called')
            expect(events[0].properties.$feature_flag).toBe('my-flag')

            expect(flagEvaluations[0].event).toBe('$feature_flag_called')
            expect(flagEvaluations[0].uuid).toBe(event.uuid)
            expect(parseJSON(flagEvaluations[0].properties).$feature_flag).toBe('my-flag')
            // The shard materializes these out of properties; reading them
            // through the flag_evaluations proxy proves that wiring.
            expect(flagEvaluations[0].flag_key).toBe('my-flag')
            expect(flagEvaluations[0].response).toBe('true')
            // person_id must match what the events table stamped for the same
            // event — the parity per-flag uniques depend on.
            expect(flagEvaluations[0].person_id).not.toBe('00000000-0000-0000-0000-000000000000')
            expect(flagEvaluations[0].person_id).toBe(events[0].person_id)
            const flagEvaluation = parseRawClickHouseEvent(flagEvaluations[0])
            expect(flagEvaluation.person_properties).toEqual(events[0].person_properties)
            expect(flagEvaluation.person_properties.email).toBe('[email protected]')
            expect(flagEvaluation.person_created_at?.toISO()).toBe(events[0].person_created_at?.toISO())
            expect(flagEvaluations[0].person_mode).toBe(events[0].person_mode)
            // The MV stamps inserted_at with now64() when it processes the row.
            expect(flagEvaluations[0].inserted_at).not.toMatch(/^1970/)
        }
    )

    testWithTeamIngester(
        'FLAG_EVALUATIONS_ONLY team: the call lands only in flag_evaluations and the realtime-only topic, and its person update and $experiment_exposure copy still land',
        {
            teamOverrides: { flag_evaluations_mode: FlagEvaluationsMode.FlagEvaluationsOnly },
            pluginServerConfig: {
                INGESTION_FLAG_EVALUATIONS_MODE: 'dual_write',
                INGESTION_FLAG_EVALUATIONS_TEAMS: '*',
                INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: KAFKA_CLICKHOUSE_FLAG_EVALUATIONS,
                INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: KAFKA_REALTIME_ONLY_EVENTS_JSON,
                EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS: '*',
            },
        },
        async ({ infra, ingester, team, kafkaProducer, token }) => {
            const producerObserver = new KafkaProducerObserver(kafkaProducer)
            // A variant response makes create-event append the $experiment_exposure copy.
            const event = new EventBuilder(team)
                .withEvent('$feature_flag_called')
                .withProperties({
                    $feature_flag: 'my-flag',
                    $feature_flag_response: 'test-variant',
                    $set: { plan: 'pro' },
                })
                .build()

            await ingester.handleKafkaBatch(createKafkaMessages([event], token))
            await waitForKafkaMessages(kafkaProducer)

            const persons = await fetchPostgresPersons(infra.postgres, team.id)
            expect(persons).toEqual([expect.objectContaining({ properties: expect.objectContaining({ plan: 'pro' }) })])

            await waitForExpect(async () => {
                expect(await fetchFlagEvaluations(clickhouse, team.id)).toHaveLength(1)
            }, 30_000)
            const events = await waitForExpect(async () => {
                const rows = await fetchEvents(clickhouse, team.id)
                expect(rows.map((row) => row.event)).toContain('$experiment_exposure')
                return rows
            }, 30_000)
            // create-event puts the call before its copy, and both go to the same
            // single-partition topic. A call written to events would already be visible here.
            expect(events.map((row) => row.event)).toEqual(['$experiment_exposure'])
            expect(events[0].person_id).toBe(persons[0].uuid)

            const realtimeEvents = producerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_REALTIME_ONLY_EVENTS_JSON)
                .map((message) => message.value)
            expect(realtimeEvents).toEqual([
                expect.objectContaining({
                    uuid: event.uuid,
                    event: '$feature_flag_called',
                    person_id: persons[0].uuid,
                }),
            ])
            expect(parseJSON(realtimeEvents[0].person_properties as string)).toMatchObject({ plan: 'pro' })
        }
    )

    // Some lanes run without the fork. An operator can also turn the fork off on any lane.
    // Only the fork step removes a call from events, so a lane without it must still store the call there.
    testWithTeamIngester(
        'FLAG_EVALUATIONS_ONLY team on a lane without the fork: the call lands in events',
        {
            teamOverrides: { flag_evaluations_mode: FlagEvaluationsMode.FlagEvaluationsOnly },
            pluginServerConfig: {
                INGESTION_FLAG_EVALUATIONS_MODE: 'disabled',
                INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: KAFKA_REALTIME_ONLY_EVENTS_JSON,
            },
        },
        async ({ ingester, team, kafkaProducer, token }) => {
            const event = new EventBuilder(team)
                .withEvent('$feature_flag_called')
                .withProperties({ $feature_flag: 'my-flag', $feature_flag_response: true })
                .build()

            await ingester.handleKafkaBatch(createKafkaMessages([event], token))
            await waitForKafkaMessages(kafkaProducer)

            const events = await waitForExpect(async () => {
                const rows = await fetchEvents(clickhouse, team.id)
                expect(rows).toHaveLength(1)
                return rows
            }, 30_000)
            expect(events[0]).toMatchObject({ uuid: event.uuid, event: '$feature_flag_called' })
        }
    )

    it('ships disabled by default', () => {
        // The only thing the round trip above would add for the disabled case:
        // that nothing flipped the fleet default. createFlagEvaluationsService
        // returning undefined is what keeps the step out of the pipeline.
        const config = { ...getDefaultIngestionConsumerConfig(), ...getDefaultIngestionOutputsConfig() }

        expect(config.INGESTION_FLAG_EVALUATIONS_MODE).toBe('disabled')
        expect(createFlagEvaluationsService(config)).toBeUndefined()
    })
})