:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Pins the full flag-evaluations shadow-write path — config → fork step → output
// registration → Kafka topic → ClickHouse Kafka engine → MV → flag_evaluations
// table — which no unit test can: a topic-key typo, a row column ClickHouse can't
// parse (silently eaten by kafka_skip_broken_messages), or a broken MV projection
// all produce "row queued but never lands".
import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'
import { createHogTransformerService } from '~/cdp/hog-transformations/hog-transformer.service'
import { KAFKA_CLICKHOUSE_FLAG_EVALUATIONS, KAFKA_REALTIME_ONLY_EVENTS_JSON } from '~/common/config/kafka-topics'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { parseRawClickHouseEvent } from '~/common/utils/event'
import { parseJSON } from '~/common/utils/json-parse'
import { createFlagEvaluationsService } from '~/ingestion/common/flag-evaluations/flag-evaluations-service'
import { getDefaultIngestionConsumerConfig, getDefaultIngestionOutputsConfig } from '~/ingestion/config'
import { IngestionConsumer } from '~/ingestion/ingestion-consumer'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import {
EventBuilder,
createKafkaMessages,
createTestWithTeamIngester,
ensureIngestionE2EInfraReady,
fetchEvents,
fetchFlagEvaluations,
waitForKafkaMessages,
} from '~/tests/helpers/ingestion-e2e'
import { createTestIngestionOutputs, createTestMonitoringOutputs } from '~/tests/helpers/ingestion-outputs'
import { fetchPostgresPersons } from '~/tests/helpers/sql'
import { FlagEvaluationsMode } from '~/types'
jest.mock('~/common/utils/logger')
describe('Flag evaluations shadow-routing E2E', () => {
const testWithTeamIngester = createTestWithTeamIngester({}, (infra, kafkaProducer) => {
const outputs = createTestIngestionOutputs(kafkaProducer)
return new IngestionConsumer(infra.config, {
...infra,
hogTransformer: createHogTransformerService(infra.config, {
...infra,
monitoringOutputs: createTestMonitoringOutputs(kafkaProducer),
}),
outputs,
clickhouseGroupRepository: new ClickhouseGroupRepository(outputs),
})
})
let clickhouse: Clickhouse
beforeAll(async () => {
clickhouse = Clickhouse.create()
await ensureIngestionE2EInfraReady()
})
afterAll(() => {
clickhouse.close()
})
testWithTeamIngester(
'dual_write mode: a $feature_flag_called event lands in both flag_evaluations and events',
{
pluginServerConfig: {
INGESTION_FLAG_EVALUATIONS_MODE: 'dual_write',
INGESTION_FLAG_EVALUATIONS_TEAMS: '*',
INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: KAFKA_CLICKHOUSE_FLAG_EVALUATIONS,
},
},
async ({ ingester, team, kafkaProducer, token }) => {
const event = new EventBuilder(team)
.withEvent('$feature_flag_called')
.withProperties({
$feature_flag: 'my-flag',
$feature_flag_response: true,
$set: { email: '[email protected]' },
})
.build()
await ingester.handleKafkaBatch(createKafkaMessages([event], token))
await waitForKafkaMessages(kafkaProducer)
const flagEvaluations = await waitForExpect(async () => {
const rows = await fetchFlagEvaluations(clickhouse, team.id)
expect(rows.length).toBe(1)
return rows
}, 30_000)
// The two rows do not necessarily arrive on the same Kafka flush, and the
// events row is usually the later of the two, so this read is polled too.
const events = await waitForExpect(async () => {
const rows = await fetchEvents(clickhouse, team.id)
expect(rows.length).toBe(1)
return rows
}, 30_000)
expect(events[0].event).toBe('$feature_flag_called')
expect(events[0].properties.$feature_flag).toBe('my-flag')
expect(flagEvaluations[0].event).toBe('$feature_flag_called')
expect(flagEvaluations[0].uuid).toBe(event.uuid)
expect(parseJSON(flagEvaluations[0].properties).$feature_flag).toBe('my-flag')
// The shard materializes these out of properties; reading them
// through the flag_evaluations proxy proves that wiring.
expect(flagEvaluations[0].flag_key).toBe('my-flag')
expect(flagEvaluations[0].response).toBe('true')
// person_id must match what the events table stamped for the same
// event — the parity per-flag uniques depend on.
expect(flagEvaluations[0].person_id).not.toBe('00000000-0000-0000-0000-000000000000')
expect(flagEvaluations[0].person_id).toBe(events[0].person_id)
const flagEvaluation = parseRawClickHouseEvent(flagEvaluations[0])
expect(flagEvaluation.person_properties).toEqual(events[0].person_properties)
expect(flagEvaluation.person_properties.email).toBe('[email protected]')
expect(flagEvaluation.person_created_at?.toISO()).toBe(events[0].person_created_at?.toISO())
expect(flagEvaluations[0].person_mode).toBe(events[0].person_mode)
// The MV stamps inserted_at with now64() when it processes the row.
expect(flagEvaluations[0].inserted_at).not.toMatch(/^1970/)
}
)
testWithTeamIngester(
'FLAG_EVALUATIONS_ONLY team: the call lands only in flag_evaluations and the realtime-only topic, and its person update and $experiment_exposure copy still land',
{
teamOverrides: { flag_evaluations_mode: FlagEvaluationsMode.FlagEvaluationsOnly },
pluginServerConfig: {
INGESTION_FLAG_EVALUATIONS_MODE: 'dual_write',
INGESTION_FLAG_EVALUATIONS_TEAMS: '*',
INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: KAFKA_CLICKHOUSE_FLAG_EVALUATIONS,
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: KAFKA_REALTIME_ONLY_EVENTS_JSON,
EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS: '*',
},
},
async ({ infra, ingester, team, kafkaProducer, token }) => {
const producerObserver = new KafkaProducerObserver(kafkaProducer)
// A variant response makes create-event append the $experiment_exposure copy.
const event = new EventBuilder(team)
.withEvent('$feature_flag_called')
.withProperties({
$feature_flag: 'my-flag',
$feature_flag_response: 'test-variant',
$set: { plan: 'pro' },
})
.build()
await ingester.handleKafkaBatch(createKafkaMessages([event], token))
await waitForKafkaMessages(kafkaProducer)
const persons = await fetchPostgresPersons(infra.postgres, team.id)
expect(persons).toEqual([expect.objectContaining({ properties: expect.objectContaining({ plan: 'pro' }) })])
await waitForExpect(async () => {
expect(await fetchFlagEvaluations(clickhouse, team.id)).toHaveLength(1)
}, 30_000)
const events = await waitForExpect(async () => {
const rows = await fetchEvents(clickhouse, team.id)
expect(rows.map((row) => row.event)).toContain('$experiment_exposure')
return rows
}, 30_000)
// create-event puts the call before its copy, and both go to the same
// single-partition topic. A call written to events would already be visible here.
expect(events.map((row) => row.event)).toEqual(['$experiment_exposure'])
expect(events[0].person_id).toBe(persons[0].uuid)
const realtimeEvents = producerObserver
.getProducedKafkaMessagesForTopic(KAFKA_REALTIME_ONLY_EVENTS_JSON)
.map((message) => message.value)
expect(realtimeEvents).toEqual([
expect.objectContaining({
uuid: event.uuid,
event: '$feature_flag_called',
person_id: persons[0].uuid,
}),
])
expect(parseJSON(realtimeEvents[0].person_properties as string)).toMatchObject({ plan: 'pro' })
}
)
// Some lanes run without the fork. An operator can also turn the fork off on any lane.
// Only the fork step removes a call from events, so a lane without it must still store the call there.
testWithTeamIngester(
'FLAG_EVALUATIONS_ONLY team on a lane without the fork: the call lands in events',
{
teamOverrides: { flag_evaluations_mode: FlagEvaluationsMode.FlagEvaluationsOnly },
pluginServerConfig: {
INGESTION_FLAG_EVALUATIONS_MODE: 'disabled',
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: KAFKA_REALTIME_ONLY_EVENTS_JSON,
},
},
async ({ ingester, team, kafkaProducer, token }) => {
const event = new EventBuilder(team)
.withEvent('$feature_flag_called')
.withProperties({ $feature_flag: 'my-flag', $feature_flag_response: true })
.build()
await ingester.handleKafkaBatch(createKafkaMessages([event], token))
await waitForKafkaMessages(kafkaProducer)
const events = await waitForExpect(async () => {
const rows = await fetchEvents(clickhouse, team.id)
expect(rows).toHaveLength(1)
return rows
}, 30_000)
expect(events[0]).toMatchObject({ uuid: event.uuid, event: '$feature_flag_called' })
}
)
it('ships disabled by default', () => {
// The only thing the round trip above would add for the disabled case:
// that nothing flipped the fleet default. createFlagEvaluationsService
// returning undefined is what keeps the step out of the pipeline.
const config = { ...getDefaultIngestionConsumerConfig(), ...getDefaultIngestionOutputsConfig() }
expect(config.INGESTION_FLAG_EVALUATIONS_MODE).toBe('disabled')
expect(createFlagEvaluationsService(config)).toBeUndefined()
})
})