Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.7 KB
Raw Blame
/**
 * # Chapter 5: Grouped Processing
 *
 * The `concurrentlyPerGroup()` method partitions items by a key and processes
 * each group as a separate stream. Groups run concurrently; items within a
 * group run sequentially through the provided pipeline.
 *
 * ## Ordering Guarantees
 *
 * **Important:** Items are NOT returned in the original input order. Groups
 * complete independently and are returned as each group finishes. A faster
 * group will return before a slower group, regardless of input order.
 *
 * What grouping DOES guarantee:
 * - Items within the same group maintain their relative input order
 * - Items within a group are processed one at a time, preserving order
 *   within that group
 *
 * ## How It Works
 *
 * ```
 * .concurrentlyPerGroup(keyFn, (group) => group.sequentially((b) => b.pipe(...)))
 * ```
 *
 * - `keyFn` partitions items by key
 * - Groups are processed in parallel (each group returns as it completes)
 * - The callback receives a group builder whose only method is `sequentially`.
 *   Spelling out `sequentially` makes within-group ordering visible at the call
 *   site: items within a group flow through that pipeline one at a time, in
 *   input order.
 *
 * Pass `{ maxConcurrency }` to cap how many groups process at once (the cap
 * is on groups, not items):
 *
 * ```
 * .concurrentlyPerGroup(keyFn, callback, { maxConcurrency: 4 })
 * ```
 *
 * ## Real-World Use Cases
 *
 * - Person operations: All updates to a person must be ordered
 * - Team-scoped processing: Teams can be processed in parallel
 * - User event ordering: Events for the same user maintain order
 */
import { newChunkPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'

import { consumeAll } from './helpers'

interface Event {
    userId: string
    eventId: number
}

describe('Grouped Processing', () => {
    beforeEach(() => {
        jest.useFakeTimers()
    })

    afterEach(() => {
        jest.useRealTimers()
    })
    /**
     * The `concurrentlyPerGroup()` method partitions items by a key function.
     * Items with the same key are grouped together for ordered processing.
     *
     * Processing flow for the example below:
     *
     * ```
     * Input batch:
     *   [alice:1, bob:2, alice:3, bob:4]
     *
     * After grouping by userId:
     *   Group "alice": [1, 3]
     *   Group "bob":   [2, 4]
     *
     * Groups are processed in parallel:
     *   ┌─────────────────────────────────────┐
     *   │  Time ──────────────────────────►   │
     *   │                                     │
     *   │  alice: ─[1]────[3]─►               │
     *   │                       (sequential)  │
     *   │  bob:   ─[2]────[4]─►               │
     *   │                       (sequential)  │
     *   └─────────────────────────────────────┘
     *
     * Within each group, items process sequentially (1 before 3, 2 before 4).
     * Groups alice and bob process concurrently (in parallel).
     * ```
     */
    it('concurrentlyPerGroup() partitions items by a key function', async () => {
        const groupsProcessed = new Map<string, number[]>()

        function createProcessEventStep(): ProcessingStep<Event, Event> {
            return async function processEventStep(event) {
                const eventIds = groupsProcessed.get(event.userId) || []
                eventIds.push(event.eventId)
                groupsProcessed.set(event.userId, eventIds)
                await new Promise((resolve) => setTimeout(resolve, 5))
                return ok(event)
            }
        }

        const pipeline = newChunkPipelineBuilder<Event>()
            .concurrentlyPerGroup(
                (event) => event.userId,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createProcessEventStep()))
            )
            .build()

        const events: Event[] = [
            { userId: 'alice', eventId: 1 },
            { userId: 'bob', eventId: 2 },
            { userId: 'alice', eventId: 3 },
            { userId: 'bob', eventId: 4 },
        ]

        const batch = events.map((e) => createOkContext(e, {}))
        pipeline.feed(batch)

        await consumeAll(pipeline, 20)

        expect(groupsProcessed.get('alice')).toEqual([1, 3])
        expect(groupsProcessed.get('bob')).toEqual([2, 4])
    })

    /**
     * Items in the same group are processed sequentially, maintaining order.
     */
    it('items in the same group are processed sequentially', async () => {
        const processedOrder: number[] = []

        function createProcessEventStep(): ProcessingStep<Event, Event> {
            return async function processEventStep(event) {
                processedOrder.push(event.eventId)
                await new Promise((resolve) => setTimeout(resolve, 10))
                return ok(event)
            }
        }

        const pipeline = newChunkPipelineBuilder<Event>()
            .concurrentlyPerGroup(
                (event) => event.userId,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createProcessEventStep()))
            )
            .build()

        const events: Event[] = [
            { userId: 'alice', eventId: 1 },
            { userId: 'alice', eventId: 2 },
            { userId: 'alice', eventId: 3 },
        ]

        const batch = events.map((e) => createOkContext(e, {}))
        pipeline.feed(batch)

        await consumeAll(pipeline, 30)

        expect(processedOrder).toEqual([1, 2, 3])
    })

    /**
     * Items in different groups can be processed concurrently, improving
     * throughput when groups are independent.
     *
     * This test uses specific delays to prove concurrency:
     * - alice: 30ms delay per event (2 events = 60ms total)
     * - bob: 10ms delay per event (2 events = 20ms total)
     * - charlie: 20ms delay per event (2 events = 40ms total)
     *
     * If sequential (input order): alice would finish first
     * If concurrent (all start together): bob finishes first, then charlie, then alice
     *
     * Each call to next() returns a complete group's results as a chunk.
     */
    it('items in different groups can process concurrently', async () => {
        const delays: Record<string, number> = {
            alice: 30,
            bob: 10,
            charlie: 20,
        }

        function createVariableDelayStep(): ProcessingStep<Event, Event> {
            return async function variableDelayStep(event) {
                await new Promise((resolve) => setTimeout(resolve, delays[event.userId]))
                return ok(event)
            }
        }

        const pipeline = newChunkPipelineBuilder<Event>()
            .concurrentlyPerGroup(
                (event) => event.userId,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createVariableDelayStep()))
            )
            .build()

        const events: Event[] = [
            { userId: 'alice', eventId: 1 },
            { userId: 'bob', eventId: 2 },
            { userId: 'charlie', eventId: 3 },
            { userId: 'alice', eventId: 4 },
            { userId: 'bob', eventId: 5 },
            { userId: 'charlie', eventId: 6 },
        ]

        const batch = events.map((e) => createOkContext(e, {}))
        pipeline.feed(batch)

        // Collect chunks as groups complete
        const chunks: string[][] = []
        const collectResults = (async () => {
            let result = await pipeline.next()
            while (result !== null) {
                chunks.push(
                    result.map((r) => {
                        const event = (r.result as { value: Event }).value
                        return `${event.userId}:${event.eventId}`
                    })
                )
                result = await pipeline.next()
            }
        })()

        // Advance time for all groups to complete (60ms for alice's 2 events)
        await jest.advanceTimersByTimeAsync(60)
        await collectResults

        // Groups complete in delay order, not input order:
        // - bob (2 x 10ms = 20ms) finishes first
        // - charlie (2 x 20ms = 40ms) finishes second
        // - alice (2 x 30ms = 60ms) finishes last
        expect(chunks).toEqual([
            ['bob:2', 'bob:5'],
            ['charlie:3', 'charlie:6'],
            ['alice:1', 'alice:4'],
        ])
    })

    /**
     * Complex group keys: Events are grouped by token + distinct_id.
     * This ensures ordering per-user within a project while allowing parallelism
     * across different users and projects.
     *
     * ```
     * Input events:
     *   [token-A/user-1/login, token-B/user-1/signup, token-A/user-2/pageview, token-A/user-1/purchase]
     *
     * After grouping by token + distinct_id:
     *   Group "token-A:user-1": [login, purchase]  <- same user in same project
     *   Group "token-B:user-1": [signup]           <- different project (different group)
     *   Group "token-A:user-2": [pageview]         <- different user (different group)
     *
     * Same distinct_id with different tokens -> different groups (different projects)
     * Same token with different distinct_ids -> different groups (different users)
     * Same token + distinct_id -> same group (processed sequentially)
     * ```
     */
    it('complex group keys with token + distinct_id', async () => {
        interface IngestionEvent {
            token: string
            distinctId: string
            message: string
        }

        const groupsProcessed = new Map<string, string[]>()

        function createProcessEventStep(): ProcessingStep<IngestionEvent, IngestionEvent> {
            return async function processEventStep(event) {
                const key = `${event.token}:${event.distinctId}`
                const messages = groupsProcessed.get(key) || []
                messages.push(event.message)
                groupsProcessed.set(key, messages)
                await new Promise((resolve) => setTimeout(resolve, 5))
                return ok(event)
            }
        }

        const pipeline = newChunkPipelineBuilder<IngestionEvent>()
            .concurrentlyPerGroup(
                (event) => `${event.token}:${event.distinctId}`,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createProcessEventStep()))
            )
            .build()

        const events: IngestionEvent[] = [
            // Different tokens with same distinct_id are separate groups (different projects)
            { token: 'token-A', distinctId: 'user-1', message: 'login' },
            { token: 'token-B', distinctId: 'user-1', message: 'signup' },
            // Same token with different distinct_id are separate groups (different users)
            { token: 'token-A', distinctId: 'user-2', message: 'pageview' },
            // Same token + distinct_id -> same group, processed in order
            { token: 'token-A', distinctId: 'user-1', message: 'purchase' },
        ]

        const batch = events.map((e) => createOkContext(e, {}))
        pipeline.feed(batch)

        await consumeAll(pipeline, 10)

        // Same token + distinct_id -> same group, processed in order
        expect(groupsProcessed.get('token-A:user-1')).toEqual(['login', 'purchase'])

        // Same distinct_id, different token -> different groups
        expect(groupsProcessed.get('token-B:user-1')).toEqual(['signup'])

        // Same token, different distinct_id -> different groups
        expect(groupsProcessed.get('token-A:user-2')).toEqual(['pageview'])
    })
})