:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* # Chapter 6: Gathering Results
*
* The `gather()` method collects all results from a streaming pipeline into
* a single chunk. This is useful when you need all results together before
* continuing to the next step.
*
* ## When to Use gather()
*
* - After `concurrently()`: collect all concurrent results into one chunk
* - After `concurrentlyPerGroup()`: collect all group results into one chunk
*
* ## How gather() Works
*
* Without gather(), pipelines stream results as they become available:
* - `concurrently()` returns items one at a time (in input order)
* - `concurrentlyPerGroup()` returns groups one at a time as each completes
*
* With gather(), all results are collected and returned in a single chunk.
*/
import { newChunkPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { collectChunks } from './helpers'
interface Event {
userId: string
eventId: number
}
describe('Gathering Results', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
/**
* Without gather(), concurrently() returns items one at a time.
* With gather(), all items are collected into a single chunk.
* In both cases, input order is preserved.
*
* This test uses variable delays to demonstrate ordering:
* - Item 1: 30ms delay (finishes last)
* - Item 2: 10ms delay (finishes first)
* - Item 3: 20ms delay (finishes second)
*/
it('gather() after concurrently() collects items into one chunk', async () => {
const delays: Record<number, number> = { 1: 30, 2: 10, 3: 20 }
function createVariableDelayStep(): ProcessingStep<number, number> {
return async function variableDelayStep(n) {
await new Promise((resolve) => setTimeout(resolve, delays[n]))
return ok(n * 10)
}
}
// Without gather: items stream one at a time in input order
const streamingPipeline = newChunkPipelineBuilder<number>()
.concurrently((builder) => builder.pipe(createVariableDelayStep()))
.build()
streamingPipeline.feed([1, 2, 3].map((n) => createOkContext(n, {})))
const streamingPromise = collectChunks(streamingPipeline)
await jest.advanceTimersByTimeAsync(30)
const streamingChunks = await streamingPromise
// Without gather: 3 separate chunks, one item each, in input order
expect(streamingChunks).toEqual([[10], [20], [30]])
// With gather: all items collected into one chunk
const gatheringPipeline = newChunkPipelineBuilder<number>()
.concurrently((builder) => builder.pipe(createVariableDelayStep()))
.gather()
.build()
gatheringPipeline.feed([1, 2, 3].map((n) => createOkContext(n, {})))
const gatheringPromise = collectChunks(gatheringPipeline)
await jest.advanceTimersByTimeAsync(30)
const gatheringChunks = await gatheringPromise
// With gather: one chunk with all items, input order preserved
expect(gatheringChunks).toEqual([[10, 20, 30]])
})
/**
* Without gather(), concurrentlyPerGroup() returns groups one at a time
* as each group completes. With gather(), all groups are collected into
* a single chunk.
*
* Within each group, events are always processed sequentially (order preserved).
*/
it('gather() after concurrentlyPerGroup() collects groups into one chunk', async () => {
const delays: Record<string, number> = {
alice: 30,
bob: 10,
}
function createVariableDelayStep(): ProcessingStep<Event, Event> {
return async function variableDelayStep(event) {
await new Promise((resolve) => setTimeout(resolve, delays[event.userId]))
return ok(event)
}
}
// Input order: alice:1, bob:2, alice:3, bob:4
const events: Event[] = [
{ userId: 'alice', eventId: 1 },
{ userId: 'bob', eventId: 2 },
{ userId: 'alice', eventId: 3 },
{ userId: 'bob', eventId: 4 },
]
// Without gather: groups stream one at a time in completion order
const streamingPipeline = newChunkPipelineBuilder<Event>()
.concurrentlyPerGroup(
(event) => event.userId,
(group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createVariableDelayStep()))
)
.build()
streamingPipeline.feed(events.map((e) => createOkContext(e, {})))
// bob: 2 events x 10ms = 20ms, alice: 2 events x 30ms = 60ms
const streamingPromise = collectChunks(streamingPipeline)
await jest.advanceTimersByTimeAsync(60)
const streamingChunks = await streamingPromise
// Without gather: bob's group finishes first, then alice's
// Within each group, events are in order (2 before 4, 1 before 3)
expect(streamingChunks).toEqual([
[
{ userId: 'bob', eventId: 2 },
{ userId: 'bob', eventId: 4 },
],
[
{ userId: 'alice', eventId: 1 },
{ userId: 'alice', eventId: 3 },
],
])
// With gather: all groups collected into one chunk
const gatheringPipeline = newChunkPipelineBuilder<Event>()
.concurrentlyPerGroup(
(event) => event.userId,
(group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createVariableDelayStep()))
)
.gather()
.build()
gatheringPipeline.feed(events.map((e) => createOkContext(e, {})))
const gatheringPromise = collectChunks(gatheringPipeline)
await jest.advanceTimersByTimeAsync(60)
const gatheringChunks = await gatheringPromise
// With gather: one chunk with all items
// Groups are in completion order (bob first), within-group order preserved
expect(gatheringChunks).toEqual([
[
{ userId: 'bob', eventId: 2 },
{ userId: 'bob', eventId: 4 },
{ userId: 'alice', eventId: 1 },
{ userId: 'alice', eventId: 3 },
],
])
})
})