:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Serial: resets shared Postgres and relies on the reset-created default team.
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { DecodedKafkaMessage } from '~/tests/helpers/mocks/producer.spy'
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import { insertHogFunction as _insertHogFunction } from '~/cdp/_tests/fixtures'
import { createHogTransformerService } from '~/cdp/hog-transformations/hog-transformer.service'
import { template as geoipTemplate } from '~/cdp/templates/_transformations/geoip/geoip.template'
import { compileHog } from '~/cdp/templates/compiler'
import { HogFunctionType } from '~/cdp/types'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { COOKIELESS_SENTINEL_VALUE } from '~/common/persons/person-utils'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'
import { COOKIELESS_MODE_FLAG_PROPERTY } from '~/ingestion/common/cookieless/cookieless-manager'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { createPrepareEventStep } from '~/ingestion/common/steps/event-processing/prepare-event-step'
import { createProcessGroupsStep } from '~/ingestion/common/steps/event-processing/process-groups-step'
import { IngestionTestInfra, createIngestionTestInfra } from '~/tests/helpers/ingestion-e2e'
import { createTestIngestionOutputs, createTestMonitoringOutputs } from '~/tests/helpers/ingestion-outputs'
import { forSnapshot } from '~/tests/helpers/snapshots'
import { createTeam, fetchPostgresPersons, getFirstTeam, getTeam, resetTestDatabase } from '~/tests/helpers/sql'
import { CookielessServerHashMode, PipelineEvent, Team } from '~/types'
import { IngestionConsumer, latestOffsetTimestampGauge } from './ingestion-consumer'
const DEFAULT_TEST_TIMEOUT = 5000
jest.setTimeout(DEFAULT_TEST_TIMEOUT)
jest.mock('~/common/utils/posthog', () => {
const original = jest.requireActual('~/common/utils/posthog')
return {
...original,
captureException: jest.fn(),
}
})
// Mock the prepare event step for error testing (a step without a retry option)
jest.mock('~/ingestion/common/steps/event-processing/prepare-event-step', () => ({
createPrepareEventStep: jest.fn(),
}))
// Mock the process groups step for error testing (a step with a retry option)
jest.mock('~/ingestion/common/steps/event-processing/process-groups-step', () => ({
createProcessGroupsStep: jest.fn(),
}))
// Mock the IngestionWarningLimiter to always allow warnings (prevents rate limiting between tests)
jest.mock('~/common/utils/token-bucket', () => {
const mockConsume = jest.fn().mockReturnValue(true)
return {
...jest.requireActual('~/common/utils/token-bucket'),
IngestionWarningLimiter: {
consume: mockConsume,
},
}
})
let offsetIncrementer = 0
const createKafkaMessage = (event: PipelineEvent, token: string): Message => {
// TRICKY: This is the slightly different format that capture sends
const captureEvent = {
uuid: event.uuid,
distinct_id: event.distinct_id,
ip: event.ip,
now: event.now,
token,
data: JSON.stringify(event),
}
return {
key: `${token}:${event.distinct_id}`,
value: Buffer.from(JSON.stringify(captureEvent)),
size: 1,
topic: 'test',
offset: offsetIncrementer++,
timestamp: DateTime.now().toMillis(),
partition: 1,
headers: [
{
distinct_id: Buffer.from(event.distinct_id || ''),
},
{
token: Buffer.from(token),
},
{
event: Buffer.from(event.event || ''),
},
{
uuid: Buffer.from(event.uuid || ''),
},
{
now: Buffer.from(event.now || ''),
},
],
}
}
describe('IngestionConsumer', () => {
let ingester: IngestionConsumer
let infra: IngestionTestInfra
let team: Team
let team2: Team
let fixedTime: DateTime
const createIngestionConsumer = async (
infra: IngestionTestInfra,
overrides?: ConstructorParameters<typeof IngestionConsumer>[2]
) => {
const outputs = createTestIngestionOutputs(mockProducer)
const ingester = new IngestionConsumer(
infra.config,
{
...infra,
outputs,
clickhouseGroupRepository: new ClickhouseGroupRepository(outputs),
hogTransformer: createHogTransformerService(infra.config, {
...infra,
monitoringOutputs: createTestMonitoringOutputs(mockProducer),
}),
},
overrides
)
// NOTE: We don't actually use kafka so we skip instantiation for faster tests
ingester['kafkaConsumer'] = {
connect: jest.fn(),
disconnect: jest.fn(),
isHealthy: jest.fn(),
} as any
await ingester.start()
return ingester
}
const createEvent = (event?: Partial<PipelineEvent>): PipelineEvent => ({
distinct_id: 'user-1',
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'us.posthog.com',
now: fixedTime.toISO()!,
event: '$pageview',
...event,
properties: {
$current_url: 'http://localhost:8000',
...(event?.properties || {}),
},
})
const createCookielessEvent = (event?: Partial<PipelineEvent>): PipelineEvent => ({
distinct_id: COOKIELESS_SENTINEL_VALUE,
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'us.posthog.com',
now: fixedTime.toISO()!,
event: '$pageview',
...event,
properties: {
$current_url: 'http://localhost:8000',
$host: 'localhost:8000',
$raw_user_agent: 'Chrome',
$session_id: null,
$device_id: null,
[COOKIELESS_MODE_FLAG_PROPERTY]: true,
...(event?.properties || {}),
},
})
const createKafkaMessages = (events: PipelineEvent[], token?: string): Message[] => {
return events.map((event) => createKafkaMessage(event, token ?? team.api_token))
}
beforeEach(async () => {
fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
jest.spyOn(Date.prototype, 'toISOString').mockReturnValue(fixedTime.toISO()!)
offsetIncrementer = 0
await resetTestDatabase()
infra = await createIngestionTestInfra()
// mockProducer = mockProducer
team = await getFirstTeam(infra.postgres)
const team2Id = await createTeam(infra.postgres, team.organization_id, 'THIS IS NOT A TOKEN FOR TEAM 3')
team2 = (await getTeam(infra.postgres, team2Id))!
jest.mocked(createPrepareEventStep).mockImplementation((...args) => {
const original = jest.requireActual('~/ingestion/common/steps/event-processing/prepare-event-step')
return original.createPrepareEventStep(...args)
})
jest.mocked(createProcessGroupsStep).mockImplementation((...args) => {
const original = jest.requireActual('~/ingestion/common/steps/event-processing/process-groups-step')
return original.createProcessGroupsStep(...args)
})
ingester = await createIngestionConsumer(infra)
})
afterEach(async () => {
await ingester.stop()
await infra.close()
})
afterAll(() => {
jest.useRealTimers()
})
describe('general', () => {
it('should have the correct config', () => {
expect(ingester['name']).toMatchInlineSnapshot(`"ingestion-consumer-events_plugin_ingestion_test"`)
expect(ingester['groupId']).toMatchInlineSnapshot(`"events-ingestion-consumer"`)
expect(ingester['topic']).toMatchInlineSnapshot(`"events_plugin_ingestion_test"`)
})
it('should process a standard event', async () => {
await ingester.handleKafkaBatch(createKafkaMessages([createEvent()]))
expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
})
it('should stop exporting the processed timestamp of revoked partitions', async () => {
latestOffsetTimestampGauge.reset()
const messages = createKafkaMessages([createEvent(), createEvent()])
messages[1].partition = 2
await ingester.handleKafkaBatch(messages)
const onPartitionsRevoked = jest.mocked(ingester['kafkaConsumer'].connect).mock.calls[0][1]!
await onPartitionsRevoked([{ topic: 'test', partition: 1 }])
const { values } = await latestOffsetTimestampGauge.get()
expect(values.filter((v) => v.labels.groupId === ingester['groupId'])).toEqual([
expect.objectContaining({ labels: expect.objectContaining({ topic: 'test', partition: 2 }) }),
])
})
it('should process a cookieless event', async () => {
await ingester.handleKafkaBatch(createKafkaMessages([createCookielessEvent()]))
expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
})
it('should drop a cookieless event if the team has cookieless disabled', async () => {
await infra.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_team SET cookieless_server_hash_mode = $1 WHERE id = $2`,
[CookielessServerHashMode.Disabled, team.id],
'set cookieless to disabled'
)
await ingester.handleKafkaBatch(createKafkaMessages([createCookielessEvent()]))
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
const warningMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
'clickhouse_ingestion_warnings_test'
)
expect(warningMessages).toHaveLength(1)
expect(warningMessages[0].value).toMatchObject({
team_id: team.id,
type: 'cookieless_team_disabled',
})
expect(mockProducerObserver.getProducedKafkaMessages()).toHaveLength(1)
})
it('should not blend person properties from 2 different cookieless users', async () => {
const events = [
createCookielessEvent({
ip: '1.2.3.4',
properties: {
$set: {
a: 'a',
ab: 'a',
},
},
}),
createCookielessEvent({
ip: '5.6.7.8', // different IP address means different user
properties: {
$set: {
b: 'b',
ab: 'b',
},
},
}),
]
const input = createKafkaMessages(events)
await ingester.handleKafkaBatch(input)
const messages = mockProducerObserver.getProducedKafkaMessages()
const eventsA = messages.filter(
(m) =>
m.topic === 'clickhouse_events_json_test' &&
m.value.person_properties &&
parseJSON(m.value.person_properties as any)?.a
)
const eventsB = messages.filter(
(m) =>
m.topic === 'clickhouse_events_json_test' &&
m.value.person_properties &&
parseJSON(m.value.person_properties as any)?.b
)
expect(eventsA[0].value.person_id).not.toEqual(eventsB[0].value.person_id)
expect(forSnapshot(eventsA)).toMatchSnapshot()
expect(forSnapshot(eventsB)).toMatchSnapshot()
})
describe('overflow', () => {
const now = () => DateTime.now().toMillis()
beforeEach(async () => {
// Just to make it easy to see what is configured
expect(infra.config.EVENT_OVERFLOW_BUCKET_CAPACITY).toEqual(1000)
// Overflow is now gated on explicit mode; the main lane redirects.
infra.config.INGESTION_OVERFLOW_MODE = 'redirect'
await ingester.stop()
ingester = await createIngestionConsumer(infra)
})
it('should emit to overflow if token and distinct_id are overflowed', async () => {
;(ingester['overflowRedirectService'] as any)['strategies'][0]['limiter'].consume(
`${team.api_token}:overflow-distinct-id`,
1000,
now()
)
const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-distinct-id' })])
await ingester.handleKafkaBatch(overflowMessages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(0)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
).toHaveLength(1)
expect(
forSnapshot(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
)
).toMatchSnapshot()
})
it('does not overflow if it is consuming from the overflow topic', async () => {
// Overflow lane consumes the overflow topic; it never redirects back.
infra.config.INGESTION_OVERFLOW_MODE = 'consume'
const overflowIngester = await createIngestionConsumer(infra, {
INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
})
// Overflow consumer doesn't have overflowRedirectService (overflowEnabled() returns false)
// so events are never redirected back to overflow
expect(overflowIngester['overflowRedirectService']).toBeUndefined()
const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-distinct-id' })])
await overflowIngester.handleKafkaBatch(overflowMessages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
).toHaveLength(0)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(1)
await overflowIngester.stop()
})
describe('stateful overflow redirect', () => {
it('refreshes Redis TTL when processing events in overflow lane', async () => {
infra.config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS = 300
infra.config.INGESTION_LANE = 'overflow'
infra.config.INGESTION_OVERFLOW_MODE = 'consume'
// Create overflow lane consumer
const overflowIngester = await createIngestionConsumer(infra, {
INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
})
// Verify TTL refresh service is created
expect(overflowIngester['overflowLaneTTLRefreshService']).toBeDefined()
// Pre-seed Redis with overflow flags (simulating main lane has flagged these keys)
const redis = await infra.redisPool.acquire()
const redisKey = `@posthog/stateful-overflow/events:${team.api_token}:overflow-user`
await redis.set(redisKey, '1', 'EX', 300)
// Get initial TTL
const initialTTL = await redis.ttl(redisKey)
expect(initialTTL).toBeGreaterThan(0)
expect(initialTTL).toBeLessThanOrEqual(300)
// Process an event in the overflow lane
const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-user' })])
await overflowIngester.handleKafkaBatch(overflowMessages)
// Verify TTL was refreshed (should be back to ~300 seconds)
const refreshedTTL = await redis.ttl(redisKey)
expect(refreshedTTL).toBeGreaterThan(0)
expect(refreshedTTL).toBeLessThanOrEqual(300)
// The TTL should be close to the configured value after refresh
expect(refreshedTTL).toBeGreaterThanOrEqual(298) // Allow for some time to pass
// Verify the event was still processed normally
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(1)
await infra.redisPool.release(redis)
await overflowIngester.stop()
})
it('does not create keys when refreshing TTL for non-existent keys', async () => {
infra.config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS = 300
infra.config.INGESTION_LANE = 'overflow'
infra.config.INGESTION_OVERFLOW_MODE = 'consume'
// Create overflow lane consumer
const overflowIngester = await createIngestionConsumer(infra, {
INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
})
const redis = await infra.redisPool.acquire()
const redisKey = `@posthog/stateful-overflow/events:${team.api_token}:new-user`
// Verify key doesn't exist initially
expect(await redis.exists(redisKey)).toBe(0)
// Process an event for a user that wasn't flagged
const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'new-user' })])
await overflowIngester.handleKafkaBatch(overflowMessages)
// Verify key was NOT created (GETEX doesn't create keys, only refreshes existing ones)
expect(await redis.exists(redisKey)).toBe(0)
// Verify the event was still processed
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(1)
await infra.redisPool.release(redis)
await overflowIngester.stop()
})
})
describe('force overflow', () => {
let ingester: IngestionConsumer
afterEach(async () => {
await ingester.stop()
})
it('should force events with matching token:distinct_id to overflow', async () => {
infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `${team.api_token}:team1-user`
ingester = await createIngestionConsumer(infra)
const messages = [
createKafkaMessage(createEvent({ distinct_id: 'team1-user' }), team.api_token), // should overflow
createKafkaMessage(createEvent({ distinct_id: 'team1-other' }), team.api_token), // should not overflow (different distinct_id)
createKafkaMessage(createEvent({ distinct_id: 'team1-user' }), team2.api_token), // should not overflow (different token)
]
await ingester.handleKafkaBatch(messages)
// Only the matching token:distinct_id event should be routed to overflow
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
).toHaveLength(1)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(2)
// Verify the right event went to overflow and the right events were processed normally
const overflowMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_overflow_test'
)
const normalMessages =
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
expect(overflowMessages[0].value.distinct_id).toEqual('team1-user')
expect(overflowMessages[0].value.token).toEqual(team.api_token)
expect(normalMessages.map((m) => m.value.distinct_id).sort()).toEqual(['team1-other', 'team1-user'])
// Add snapshot for the overflow messages
expect(forSnapshot(overflowMessages)).toMatchSnapshot('force overflow messages')
})
it('should handle multiple token:distinct_id pairs in force overflow setting', async () => {
infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `${team.api_token}:user1,${team2.api_token}:user2`
ingester = await createIngestionConsumer(infra)
const messages = [
createKafkaMessage(createEvent({ distinct_id: 'user1' }), team.api_token), // should overflow
createKafkaMessage(createEvent({ distinct_id: 'other' }), team.api_token), // should not overflow
createKafkaMessage(createEvent({ distinct_id: 'user2' }), team2.api_token), // should overflow
createKafkaMessage(createEvent({ distinct_id: 'other' }), team2.api_token), // should not overflow
]
await ingester.handleKafkaBatch(messages)
// Both matching token:distinct_id pairs should be routed to overflow
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
).toHaveLength(2)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(2)
// Verify both matching events went to overflow
const overflowMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_overflow_test'
)
const normalMessages =
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
// Sort messages by distinct_id to make the test deterministic
const sortedOverflowMessages = [...overflowMessages].sort((a, b) =>
String(a.value.distinct_id).localeCompare(String(b.value.distinct_id))
)
expect(sortedOverflowMessages.map((m) => [m.value.token, m.value.distinct_id])).toEqual([
[team.api_token, 'user1'],
[team2.api_token, 'user2'],
])
expect(normalMessages.map((m) => m.value.distinct_id).sort()).toEqual(['other', 'other'])
// Add snapshot for the overflow messages
expect(forSnapshot(sortedOverflowMessages)).toMatchSnapshot(
'force overflow messages multiple pairs'
)
})
describe('via headers (preprocessing)', () => {
it('forces overflow using headers even if payload token/distinct_id differ', async () => {
infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
ingester = await createIngestionConsumer(infra)
const event = createEvent({ distinct_id: 'not-forced' })
const [message] = createKafkaMessages([event])
message.headers = [
{ token: Buffer.from('forced-token') },
{ distinct_id: Buffer.from('forced-id') },
]
await ingester.handleKafkaBatch([message])
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_overflow_test'
)
).toHaveLength(1)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(0)
})
it('preserves partition locality when not skipping person; drops key when skipping person', async () => {
// Not skipping person -> key preserved
infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
infra.config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID = ''
infra.config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY = true
ingester = await createIngestionConsumer(infra)
const eventA = createEvent({ distinct_id: 'not-forced' })
const [messageA] = createKafkaMessages([eventA])
const originalKeyA = messageA.key
messageA.headers = [
{ token: Buffer.from('forced-token') },
{ distinct_id: Buffer.from('forced-id') },
]
await ingester.handleKafkaBatch([messageA])
const overflowA = mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_overflow_test'
)
expect(overflowA).toHaveLength(1)
expect(overflowA[0].key).toEqual(originalKeyA)
// Reset produced messages
mockProducerObserver.resetKafkaProducer()
// Skipping person -> key dropped when preserve locality disabled by config
await ingester.stop()
infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
infra.config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
infra.config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY = false
ingester = await createIngestionConsumer(infra)
const eventB = createEvent({ distinct_id: 'not-forced' })
const [messageB] = createKafkaMessages([eventB])
messageB.headers = [
{ token: Buffer.from('forced-token') },
{ distinct_id: Buffer.from('forced-id') },
]
await ingester.handleKafkaBatch([messageB])
const overflowB = mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_overflow_test'
)
expect(overflowB).toHaveLength(1)
expect(overflowB[0].key).toBeNull()
})
})
})
})
})
describe('dropping events', () => {
it('should DLQ $exception events (wrong consumer)', async () => {
const messages = createKafkaMessages([
createEvent({
event: '$exception',
properties: { $exception_message: 'Test error' },
}),
])
await ingester.handleKafkaBatch(messages)
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_dlq_test')
).toHaveLength(1)
})
it('should DLQ $$client_ingestion_warning events (wrong consumer)', async () => {
const messages = createKafkaMessages([
createEvent({
event: '$$client_ingestion_warning',
properties: { $$client_ingestion_warning_message: 'test' },
}),
])
await ingester.handleKafkaBatch(messages)
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_dlq_test')
).toHaveLength(1)
})
it('should capture ingestion warning for $groupidentify with too long $group_key', async () => {
const longKey = 'x'.repeat(401)
const messages = createKafkaMessages([
createEvent({
event: '$groupidentify',
properties: {
$group_key: longKey,
$group_type: 'company',
$group_set: { name: 'Test Company' },
},
}),
])
const { backgroundTask } = await ingester.handleKafkaBatch(messages)
await backgroundTask
// Event should be dropped
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
// Should produce ingestion warning
const warningMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
'clickhouse_ingestion_warnings_test'
)
expect(warningMessages).toHaveLength(1)
expect(warningMessages[0].value).toMatchObject({
team_id: team.id,
type: 'group_key_too_long',
details: expect.stringContaining(longKey),
})
})
describe.each(['headers', 'payload'] as const)('via %s', (kind) => {
const addMessageHeaders = (message: Message, token?: string, distinctId?: string) => {
if (kind !== 'headers') {
return
}
message.headers = []
if (distinctId) {
message.headers.push({
distinct_id: Buffer.from(distinctId),
})
}
if (token) {
message.headers.push({
token: Buffer.from(token),
})
}
}
beforeEach(() => {
jest.spyOn(logger, 'debug')
})
describe('with DROP_EVENTS_BY_TOKEN_DISTINCT_ID drops events with matching token:distinct_id when only event keys are listed', () => {
beforeEach(async () => {
infra.config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID = `${team.api_token}:distinct-id-to-ignore,phc_other:distinct-id-to-ignore`
ingester = await createIngestionConsumer(infra)
})
it('should drop events with matching token and distinct_id', async () => {
const messages = createKafkaMessages([
createEvent({
distinct_id: 'distinct-id-to-ignore',
}),
])
addMessageHeaders(messages[0], team.api_token, 'distinct-id-to-ignore')
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(0)
})
it('should not drop events for a different team token', async () => {
const messages = createKafkaMessages(
[
createEvent({
distinct_id: 'distinct-id-to-ignore',
}),
],
team2.api_token
)
addMessageHeaders(messages[0], team2.api_token, 'distinct-id-to-ignore')
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).not.toHaveLength(0)
})
it('should not drop events for a different distinct_id', async () => {
const messages = createKafkaMessages([
createEvent({
distinct_id: 'other-id',
}),
])
addMessageHeaders(messages[0], team.api_token, 'not-ignored')
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).not.toHaveLength(0)
})
})
describe('with DROP_EVENTS_BY_TOKEN_DISTINCT_ID drops all team events when only token is listed', () => {
beforeEach(async () => {
const distinct_id_to_drop = 'team1_user_to_drop'
infra.config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID = `${team.api_token}:${distinct_id_to_drop},${team2.api_token}`
ingester = await createIngestionConsumer(infra)
})
it('should still drop events with matching token and distinct_id (event key)', async () => {
const distinct_id_to_drop = 'team1_user_to_drop'
const messages = createKafkaMessages([
createEvent({
distinct_id: distinct_id_to_drop,
}),
])
addMessageHeaders(messages[0], team.api_token, distinct_id_to_drop)
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(0)
})
it('should not drop all events for team with event key listed when distinct_id differs in event', async () => {
const unlisted_distinct_id = 'team1_user_NOT_to_drop'
const messages = createKafkaMessages([
createEvent({
distinct_id: unlisted_distinct_id,
}),
])
addMessageHeaders(messages[0], team.api_token, unlisted_distinct_id)
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).not.toHaveLength(0)
})
it('should drop all events for team with only token listed to be dropped', async () => {
const any_distinct_id = 'any_user'
const other_distinct_id = 'other_user'
const messages = createKafkaMessages(
[
createEvent({
distinct_id: any_distinct_id,
}),
createEvent({
distinct_id: other_distinct_id,
}),
],
team2.api_token
)
addMessageHeaders(messages[0], team2.api_token, any_distinct_id)
addMessageHeaders(messages[1], team2.api_token, other_distinct_id)
await ingester.handleKafkaBatch(messages)
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
).toHaveLength(0)
})
})
})
})
describe('event batching', () => {
beforeEach(async () => {
ingester = await createIngestionConsumer(infra)
})
it('should validate historical_migration flag based on timestamp age', async () => {
const ingestionTime = fixedTime // Now = ingestion time
const oldEventTime = ingestionTime.minus({ hours: 50 }) // Event happened 50 hours ago (>48 hours)
const recentEventTime = ingestionTime.minus({ hours: 24 }) // Event happened 24 hours ago (<48 hours)
const events = [
createEvent({
distinct_id: 'user-old-event',
event: 'old-event',
timestamp: oldEventTime.toISO()!, // Event time in the event payload
now: ingestionTime.toISO()!, // Ingestion time in the event payload
}),
createEvent({
distinct_id: 'user-recent-event',
event: 'recent-event',
timestamp: recentEventTime.toISO()!, // Event time in the event payload
now: ingestionTime.toISO()!, // Ingestion time in the event payload
}),
]
// Create messages with different event timestamps but same ingestion time (now)
// The validate-historical-migration step compares: now - timestamp >= 48 hours
const messages: Message[] = [
{
...createKafkaMessage(events[0], team.api_token),
headers: [
{ token: Buffer.from(team.api_token) },
{ distinct_id: Buffer.from('user-old-event') },
{ timestamp: Buffer.from(oldEventTime.toISO()!) }, // Event time (when it actually happened)
{ now: Buffer.from(ingestionTime.toISO()!) }, // Ingestion time (when we received it)
{ historical_migration: Buffer.from('true') },
],
},
{
...createKafkaMessage(events[1], team.api_token),
headers: [
{ token: Buffer.from(team.api_token) },
{ distinct_id: Buffer.from('user-recent-event') },
{ timestamp: Buffer.from(recentEventTime.toISO()!) }, // Event time (when it actually happened)
{ now: Buffer.from(ingestionTime.toISO()!) }, // Ingestion time (when we received it)
{ historical_migration: Buffer.from('true') },
],
},
]
await ingester.handleKafkaBatch(messages)
const producedMessages = mockProducerObserver.getProducedKafkaMessages()
const eventsTopicMessages = producedMessages.filter((m) => m.topic === 'clickhouse_events_json_test')
expect(eventsTopicMessages).toHaveLength(2)
// Old event should have historical_migration: true (>48 hours)
const oldEventMessage = eventsTopicMessages.find((m) => m.value.event === 'old-event')
expect(oldEventMessage).toBeDefined()
expect(oldEventMessage?.value.historical_migration).toBe(true)
// Recent event should not have historical_migration set (<48 hours, defaults to false)
const recentEventMessage = eventsTopicMessages.find((m) => m.value.event === 'recent-event')
expect(recentEventMessage).toBeDefined()
expect(recentEventMessage?.value.historical_migration).toBeUndefined()
})
})
describe('error handling', () => {
let messages: Message[]
beforeEach(() => {
// Simulate some sort of error happening by mocking out the prepare event step
messages = createKafkaMessages([createEvent()])
jest.spyOn(logger, 'error').mockImplementation(() => {})
})
it('should handle explicitly non retriable errors by sending to DLQ', async () => {
// Non-retriable errors are converted to DLQ results by the retry wrapper,
// so the throwing step must be one that carries a retry option.
const error: any = new Error('test')
error.isRetriable = false
// Mock the process groups step (retry-wrapped) to throw the error
jest.mocked(createProcessGroupsStep).mockImplementation(() => {
return async function processGroupsStepWrapper() {
return Promise.reject(error)
}
})
const ingester = await createIngestionConsumer(infra)
await ingester.handleKafkaBatch(messages)
expect(jest.mocked(logger.error)).toHaveBeenCalledWith(
'🔥',
'Step process_groups failed',
expect.any(Object)
)
expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
})
it.each([undefined, true])('should throw if isRetriable is set to %s', async (isRetriable) => {
const error: any = new Error('test')
error.isRetriable = isRetriable
// Mock the prepare event step to throw the error
jest.mocked(createPrepareEventStep).mockImplementation(() => {
return async function prepareEventStepWrapper() {
return Promise.reject(error)
}
})
const ingester = await createIngestionConsumer(infra)
await expect(ingester.handleKafkaBatch(messages)).rejects.toThrow()
})
it('should throw non retriable errors from steps without a retry option', async () => {
// Steps without a retry option have no error classification: an
// explicitly non-retriable error crashes the batch instead of DLQing.
const error: any = new Error('test')
error.isRetriable = false
// Mock the prepare event step (no retry option) to throw the error
jest.mocked(createPrepareEventStep).mockImplementation(() => {
return async function prepareEventStepWrapper() {
return Promise.reject(error)
}
})
const ingester = await createIngestionConsumer(infra)
await expect(ingester.handleKafkaBatch(messages)).rejects.toThrow()
})
it('should emit failures to dead letter queue for non-retriable errors', async () => {
const error = new Error('Non-retriable processing error')
const errorAny = error as any
errorAny.isRetriable = false
// Mock the process groups step (retry-wrapped) to throw the error
jest.mocked(createProcessGroupsStep).mockImplementation(() => {
return async function processGroupsStepWrapper() {
return Promise.reject(error)
}
})
const ingester = await createIngestionConsumer(infra)
await ingester.handleKafkaBatch(messages)
// Should not produce to clickhouse
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
// Should produce to DLQ
const dlqMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
'events_plugin_ingestion_dlq_test'
)
expect(dlqMessages).toHaveLength(1)
expect(dlqMessages[0].value).toEqual(parseJSON(messages[0].value!.toString()))
expect(dlqMessages[0].headers!['dlq_reason']).toEqual(error.message)
})
})
describe('typical event processing', () => {
/**
* NOTE: The majority of these tests should be done in the event pipeline runner but
* this is a good place to have some high level happy paths
*/
beforeEach(async () => {
ingester = await createIngestionConsumer(infra)
})
const eventTests: [string, () => PipelineEvent[]][] = [
['normal event', () => [createEvent()]],
[
'$identify event',
() => [createEvent({ event: '$identify', properties: { $set: { $email: '[email protected]' } } })],
],
[
'multiple events',
() => [
createEvent({
event: '$pageview',
distinct_id: 'anonymous-id-1',
properties: { $current_url: 'https://example.com/page1' },
}),
createEvent({
event: '$identify',
distinct_id: 'identified-id-1',
properties: { $set: { $email: '[email protected]' }, $anonymous_distinct_id: 'anonymous-id-1' },
}),
createEvent({
event: '$pageview',
distinct_id: 'identified-id-1',
properties: { $current_url: 'https://example.com/page2' },
}),
],
],
// [
// 'heatmap event',
// () => [
// createEvent({
// distinct_id: 'distinct-id-1',
// event: '$$heatmap',
// properties: {
// $heatmap_data: {
// 'http://localhost:3000/': [
// {
// x: 1020,
// y: 363,
// target_fixed: false,
// type: 'mousemove',
// },
// {
// x: 634,
// y: 460,
// target_fixed: false,
// type: 'click',
// },
// ],
// },
// },
// }),
// ],
// ],
[
// NOTE: This currently returns as is - for now we keep this broken but once we release the new ingester we should fix this
'malformed person information',
() => [
createEvent({
distinct_id: 'distinct-id-1',
properties: { $set: 'INVALID', $unset: [[[['definitel invalid']]]] },
}),
],
],
['malformed event', () => [createEvent({ event: '' })]],
['event with common distinct_id that gets dropped', () => [createEvent({ distinct_id: 'distinct_id' })]],
[
'ai event',
() => [
createEvent({
event: '$ai_generation',
properties: {
$ai_model: 'gpt-4',
$ai_provider: 'openai',
$ai_input_tokens: 100,
$ai_output_tokens: 50,
},
}),
],
],
[
'person processing off',
() => [createEvent({ event: '$pageview', properties: { $process_person_profile: false } })],
],
[
'forced person upgrade',
() => [
createEvent({
event: '$pageview',
properties: { $process_person_profile: false, $set: { update1: '1' } },
}),
createEvent({
event: '$identify',
properties: { $process_person_profile: true, $set: { email: '[email protected]' } },
}),
// Add an event at least a minute in the future and it should get force upgraded
createEvent({
event: '$pageview',
properties: { $process_person_profile: false, $set: { update2: '2' } },
timestamp: DateTime.now().plus({ minutes: 2 }).toISO(),
}),
// Add a person-full event and ensure all properties are there that should be
createEvent({
event: '$pageview',
properties: { $process_person_profile: true, $set: { update3: '3' } },
timestamp: DateTime.now().plus({ minutes: 3 }).toISO(),
}),
// Snapshot should contain update2 and update3 but not update1
],
],
[
'groups',
() => [
createEvent({
event: '$pageview',
properties: {
$groups: {
a: 'group-a',
b: 'group-b',
c: 'group-c',
d: 'group-d',
e: 'group-e',
f: 'group-f',
},
},
}),
createEvent({
event: '$groupidentify',
properties: {
$group_type: 'a',
$group_key: 'group-a',
$group_set: {
id: 'group-a',
foo: 'bar',
},
},
}),
// This triggers an event but not a groups clickhouse change as the max groups is already hit
createEvent({
event: '$groupidentify',
properties: {
$group_type: 'f',
$group_key: 'group-f',
$group_set: {
id: 'group-f',
foo: 'bar',
},
},
}),
],
],
[
'person property merging via alias',
() => {
const anonId1 = new UUIDT().toString()
const anonId2 = new UUIDT().toString()
return [
createEvent({
distinct_id: anonId1,
event: 'custom event',
properties: { $set: { k: 'v' } },
}),
createEvent({
distinct_id: anonId2,
event: 'custom event',
properties: { $set: { j: 'w' } },
}),
// final event should have k, j, l
createEvent({
distinct_id: anonId2,
event: '$create_alias',
properties: { alias: anonId1, $set: { l: 'x' } },
}),
]
},
],
]
it.each(eventTests)('%s', async (_, createEvents) => {
const messages = createKafkaMessages(createEvents())
await ingester.handleKafkaBatch(messages)
// Tricky due to some parallel processing race conditions order isn't deterministic
// So we sort by specific properties to make it deterministic
const sortingKey = (message: DecodedKafkaMessage) => {
const value = message.value
return `${value.topic}:${value.team_id}:${value.distinct_id}:${value.properties}`
}
const sortedMessages = mockProducerObserver
.getProducedKafkaMessages()
.sort((a, b) => sortingKey(a).localeCompare(sortingKey(b)))
expect(forSnapshot(sortedMessages)).toMatchSnapshot()
})
})
describe('transformations', () => {
let transformationFunction: HogFunctionType
const TRANSFORMATION_TEST_TIMEOUT = 30000
beforeAll(() => {
jest.setTimeout(TRANSFORMATION_TEST_TIMEOUT)
})
afterAll(() => {
jest.setTimeout(DEFAULT_TEST_TIMEOUT)
})
const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
const { hog, bytecode, name } = hogFunction
const item = await _insertHogFunction(infra.postgres, team.id, {
hog,
bytecode,
name: name || 'Test Function',
type: 'transformation',
})
return item
}
beforeEach(async () => {
// Create a transformation function using the geoip template as an example
const hogByteCode = await compileHog(geoipTemplate.code)
transformationFunction = await insertHogFunction({
name: 'GeoIP Transformation',
hog: geoipTemplate.code,
bytecode: hogByteCode,
})
ingester = await createIngestionConsumer(infra)
})
it(
'should invoke transformation for matching team with error case',
async () => {
// make the geoip lookup fail
const event = createEvent({
ip: '256.256.256.256',
properties: { $ip: '256.256.256.256' },
})
const messages = createKafkaMessages([event])
const result = await ingester.handleKafkaBatch(messages)
await result.backgroundTask
// Verify metrics were published
const metricsMessages =
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
expect(metricsMessages).toEqual([
{
headers: undefined,
key: null,
topic: 'clickhouse_app_metrics2_test',
value: {
app_source: 'hog_function',
app_source_id: transformationFunction.id,
count: 1,
instance_id: '',
metric_kind: 'success',
metric_name: 'succeeded',
team_id: team.id,
timestamp: '2025-01-01 00:00:00.000',
},
},
])
// Verify log entries were published
const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')
expect(logMessages).toEqual([
{
key: null,
topic: 'log_entries_test',
value: {
instance_id: expect.any(String),
level: 'info',
log_source: 'hog_function',
log_source_id: transformationFunction.id,
message: 'geoip lookup failed for ip, 256.256.256.256',
team_id: team.id,
timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
},
},
{
key: null,
topic: 'log_entries_test',
value: {
instance_id: expect.any(String),
level: 'debug',
log_source: 'hog_function',
log_source_id: transformationFunction.id,
message: expect.stringMatching(/^Function completed in/),
team_id: team.id,
timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
},
},
])
},
TRANSFORMATION_TEST_TIMEOUT
)
it(
'should invoke transformation for matching team with success case',
async () => {
const event = createEvent({
ip: '89.160.20.129',
properties: { $ip: '89.160.20.129' },
})
const messages = createKafkaMessages([event])
const result = await ingester.handleKafkaBatch(messages)
await result.backgroundTask
// Verify metrics were published
const metricsMessages =
mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
expect(metricsMessages).toEqual([
{
headers: undefined,
key: null,
topic: 'clickhouse_app_metrics2_test',
value: {
app_source: 'hog_function',
app_source_id: transformationFunction.id,
count: 1,
instance_id: '',
metric_kind: 'success',
metric_name: 'succeeded',
team_id: team.id,
timestamp: '2025-01-01 00:00:00.000',
},
},
])
// Verify log entries were published
const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')
expect(logMessages).toEqual([
{
key: null,
topic: 'log_entries_test',
value: {
instance_id: expect.any(String),
level: 'info',
log_source: 'hog_function',
log_source_id: transformationFunction.id,
message: expect.stringContaining('geoip location data for ip:'),
team_id: team.id,
timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
},
},
{
key: null,
topic: 'log_entries_test',
value: {
instance_id: expect.any(String),
level: 'debug',
log_source: 'hog_function',
log_source_id: transformationFunction.id,
message: expect.stringMatching(/^Function completed in/),
team_id: team.id,
timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
},
},
])
},
TRANSFORMATION_TEST_TIMEOUT
)
})
describe('store flush behavior', () => {
it('should flush stores exactly once per batch regardless of number of events', async () => {
const flushSpy = jest.spyOn(ingester['personsStore'], 'flush')
const groupFlushSpy = jest.spyOn(ingester['groupStore'], 'flush')
const events: PipelineEvent[] = [
{
distinct_id: 'user1',
event: '$pageview',
properties: {},
timestamp: '2025-01-01 00:00:00',
team_id: team.id,
now: '2025-01-01 00:00:00',
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'https://example.com',
},
{
distinct_id: 'user2',
event: '$pageview',
properties: {},
timestamp: '2025-01-01 00:00:01',
team_id: team.id,
now: '2025-01-01 00:00:01',
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'https://example.com',
},
{
distinct_id: 'user3',
event: '$pageview',
properties: {},
timestamp: '2025-01-01 00:00:02',
team_id: team.id,
now: '2025-01-01 00:00:02',
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'https://example.com',
},
]
const messages = createKafkaMessages(events)
await ingester.handleKafkaBatch(messages)
expect(flushSpy).toHaveBeenCalledTimes(1)
expect(groupFlushSpy).toHaveBeenCalledTimes(1)
flushSpy.mockRestore()
groupFlushSpy.mockRestore()
})
it('should flush persons and group stores for each batch', async () => {
const callOrder: string[] = []
const flushSpy = jest.spyOn(ingester['personsStore'], 'flush').mockImplementation(() => {
callOrder.push('personsFlush')
return Promise.resolve([])
})
const groupFlushSpy = jest.spyOn(ingester['groupStore'], 'flush').mockImplementation(() => {
callOrder.push('groupFlush')
return Promise.resolve([])
})
const events: PipelineEvent[] = [
{
distinct_id: 'user1',
event: '$pageview',
properties: {},
timestamp: '2025-01-01 00:00:00',
team_id: team.id,
now: '2025-01-01 00:00:00',
uuid: new UUIDT().toString(),
ip: '127.0.0.1',
site_url: 'https://example.com',
},
]
await ingester.handleKafkaBatch(createKafkaMessages(events))
expect(callOrder).toContain('personsFlush')
expect(callOrder).toContain('groupFlush')
flushSpy.mockRestore()
groupFlushSpy.mockRestore()
})
})
describe('multi-batch cache eviction and ordering', () => {
it('person properties from batch 1 are visible in batch 2', async () => {
await ingester.handleKafkaBatch(
createKafkaMessages([createEvent({ properties: { $set: { name: 'Alice' } } })])
)
const personsAfterBatch1 = await fetchPostgresPersons(infra.postgres, team.id)
expect(personsAfterBatch1).toHaveLength(1)
expect(personsAfterBatch1[0].properties).toMatchObject({ name: 'Alice' })
await ingester.handleKafkaBatch(
createKafkaMessages([createEvent({ properties: { $set: { email: '[email protected]' } } })])
)
const personsAfterBatch2 = await fetchPostgresPersons(infra.postgres, team.id)
expect(personsAfterBatch2).toHaveLength(1)
expect(personsAfterBatch2[0].properties).toMatchObject({
name: 'Alice',
email: '[email protected]',
})
})
it('cache entries are evicted after each batch completes', async () => {
const personsStore = ingester['personsStore'] as BatchWritingPersonsStore
await ingester.handleKafkaBatch(createKafkaMessages([createEvent()]))
const batchDistinctKeys = (personsStore as any)['batchDistinctKeys'] as Map<number, Set<string>>
expect(batchDistinctKeys.size).toBe(0)
expect(personsStore.getCheckCache().size).toBe(0)
})
it('cache does not grow unboundedly across many sequential batches', async () => {
const personsStore = ingester['personsStore'] as BatchWritingPersonsStore
for (let i = 0; i < 5; i++) {
await ingester.handleKafkaBatch(createKafkaMessages([createEvent({ distinct_id: `user-${i}` })]))
}
const batchDistinctKeys = (personsStore as any)['batchDistinctKeys'] as Map<number, Set<string>>
expect(batchDistinctKeys.size).toBe(0)
expect(personsStore.getCheckCache().size).toBe(0)
})
})
describe('stop()', () => {
it('calls shutdown() on persons and group stores', async () => {
const personShutdownSpy = jest.spyOn(ingester['personsStore'], 'shutdown')
const groupShutdownSpy = jest.spyOn(ingester['groupStore'], 'shutdown')
await ingester.stop()
expect(personShutdownSpy).toHaveBeenCalledTimes(1)
expect(groupShutdownSpy).toHaveBeenCalledTimes(1)
})
})
})