Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research): write the v4 index to parquet from kafka (#119750)

Nicholas Waltz39dcf050ea20
diff --git a/nodejs/src/capabilities.ts b/nodejs/src/capabilities.ts
index 829e7e223c50..d2f2a29e2843 100644
--- a/nodejs/src/capabilities.ts
+++ b/nodejs/src/capabilities.ts
@@ -213,6 +213,7 @@ export function getPluginServerCapabilities(
case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:
case PluginServerMode.ai_research_v4_image_fetch:
case PluginServerMode.ai_research_v4_image_fetch_retry:
+ case PluginServerMode.ai_research_v4_parquet_sink:
case PluginServerMode.recording_api:
throw new Error(`Mode ${mode} is handled by IngestionSessionReplayServer, not PluginServer`)
}
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.ts
index d718de4234b6..ad16d4b7a1c7 100644
--- a/nodejs/src/common/config.ts
+++ b/nodejs/src/common/config.ts
@@ -44,6 +44,7 @@ export enum PluginServerMode {
recordings_blob_ingestion_v2_ml_image_fetch_retry = 'recordings-blob-ingestion-v2-ml-image-fetch-retry',
ai_research_v4_image_fetch = 'ai-research-v4-image-fetch',
ai_research_v4_image_fetch_retry = 'ai-research-v4-image-fetch-retry',
+ ai_research_v4_parquet_sink = 'ai-research-v4-parquet-sink',
cdp_processed_events = 'cdp-processed-events',
cdp_person_updates = 'cdp-person-updates',
cdp_data_warehouse_events = 'cdp-data-warehouse-events',
diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.ts
index 61c8aa6299a0..ff32d96febe9 100644
--- a/nodejs/src/common/config/kafka-topics.ts
+++ b/nodejs/src/common/config/kafka-topics.ts
@@ -60,6 +60,8 @@ export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH = `${prefix}ai_research_v4_image_f
export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_v4_image_fetch_retry_1m${suffix}`
export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_v4_image_fetch_retry_10m${suffix}`
export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_v4_image_fetch_retry_1h${suffix}`
+// v4 index rows: v4 recorder -> v4 Parquet sink, one message per Parquet row.
+export const KAFKA_AI_RESEARCH_V4_INDEX = `${prefix}ai_research_v4_index${suffix}`
// images the scrub sidecar cannot process, parked so they stop holding the head of their partition.
// The original bytes are kept: unscrubbed content must never reach the ML bucket, but it must not be
diff --git a/nodejs/src/index.ts b/nodejs/src/index.ts
index 8eb63a6f4773..477175bb9c53 100644
--- a/nodejs/src/index.ts
+++ b/nodejs/src/index.ts
@@ -10,6 +10,7 @@ import {
AiResearchV4ImageFetchRetryServer,
AiResearchV4ImageFetchServer,
} from '~/servers/ai-research-v4-image-fetch-server'
+import { AiResearchV4ParquetSinkServer } from '~/servers/ai-research-v4-parquet-sink-server'
import { NodeServer } from '~/servers/base-server'
import { ErrorTrackingServer } from '~/servers/error-tracking-server'
import { IngestionApiServer } from '~/servers/ingestion-api-server'
@@ -67,6 +68,9 @@ function createServer(): NodeServer {
case PluginServerMode.ai_research_v4_image_fetch_retry:
return new AiResearchV4ImageFetchRetryServer()
+ case PluginServerMode.ai_research_v4_parquet_sink:
+ return new AiResearchV4ParquetSinkServer()
+
case PluginServerMode.recording_api:
return new RecordingApiServer()
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
index c4741bf8c4b9..17ba5c8d7422 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
@@ -130,6 +130,20 @@ export type AiResearchV4Config = {
KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: string
/** A short local delivery timeout, so a slow topic fails a batch fast instead of holding its offsets. */
KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: string
+
+ /** The v4 index sink: the bucket its Parquet files go to, under `index/`. */
+ AI_RESEARCH_V4_RRWEB_BUCKET: string
+ /** The v4 index sink's consumer group. Its own setting, so a v4 sink pod never joins the v3 sink group. */
+ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: string
+ /**
+ * The v4 sink writes its files this long after its last flush, or when the buffer reaches either size below,
+ * whichever comes first. The byte size counts Kafka message bytes, so it also bounds the buffer's memory.
+ */
+ AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS: number
+ AI_RESEARCH_V4_PARQUET_MAX_ROWS: number
+ AI_RESEARCH_V4_PARQUET_MAX_BYTES: number
+ /** Rows per row group in a v4 index file, so a large file has few row groups. */
+ AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS: number
}
export function getDefaultAiResearchV4Config(): AiResearchV4Config {
@@ -181,6 +195,13 @@ export function getDefaultAiResearchV4Config(): AiResearchV4Config {
KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '32768',
KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: `${40 * 1024 * 1024 + 64 * 1024}`,
KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: '20000',
+
+ AI_RESEARCH_V4_RRWEB_BUCKET: '',
+ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: 'ai-research-v4-parquet-sink',
+ AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS: 10 * 60 * 1000,
+ AI_RESEARCH_V4_PARQUET_MAX_ROWS: 1_000_000,
+ AI_RESEARCH_V4_PARQUET_MAX_BYTES: 128 * 1024 * 1024,
+ AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS: 131_072,
}
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts
new file mode 100644
index 000000000000..76b8eaeb4d98
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts
@@ -0,0 +1,286 @@
+import { PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
+import { ParquetReader } from '@dsnp/parquetjs'
+import { Message } from 'node-rdkafka'
+import { register } from 'prom-client'
+
+import { OffsetStore } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+
+import { IndexBatcher, IndexBatcherOptions } from './index-batcher'
+import { BLOCK_INDEX_COLUMNS, V4Index } from './index-columns'
+import { IndexParquetStore } from './index-parquet-store'
+
+const TOPIC = 'ai_research_v4_index'
+
+function blockRow(sessionId: string): Record<string, unknown> {
+ const row: Record<string, unknown> = {}
+ for (const column of BLOCK_INDEX_COLUMNS) {
+ row[column.name] =
+ 'repeated' in column || 'list' in column
+ ? []
+ : column.optional
+ ? null
+ : column.type === 'UTF8'
+ ? 'x'
+ : column.type === 'BOOLEAN'
+ ? false
+ : 1
+ }
+ return {
+ ...row,
+ session_id: sessionId,
+ moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 3 }],
+ url_fingerprints: ['b08517cee3ff5ecc'],
+ window_ids: ['w1'],
+ }
+}
+
+const imageRow = (imageRef: string): Record<string, unknown> => ({
+ team_id: 7,
+ key_month: '2026-10',
+ image_ref: imageRef,
+ pack_s3_key: 'images/packs/dt=2026-10-15/7/1-0',
+ byte_start: 0,
+ byte_end: 99,
+ mime: 'image/png',
+ byte_size: 60,
+})
+
+function message(
+ offset: number,
+ index: string,
+ writeDay: string,
+ value: unknown,
+ partition = 0,
+ raw?: string
+): Message {
+ return {
+ topic: TOPIC,
+ partition,
+ offset,
+ value: Buffer.from(raw ?? JSON.stringify(value)),
+ headers: [{ 'ml-index': Buffer.from(index) }, { 'ml-write-day': Buffer.from(writeDay) }],
+ } as unknown as Message
+}
+
+async function readColumn(body: Buffer, column: string): Promise<unknown[]> {
+ const reader = await ParquetReader.openBuffer(body)
+ const cursor = reader.getCursor()
+ const values: unknown[] = []
+ for (let record = await cursor.next(); record; record = await cursor.next()) {
+ values.push((record as Record<string, unknown>)[column])
+ }
+ await reader.close()
+ return values
+}
+
+async function rejected(reason: string): Promise<number> {
+ const metric = await register.getSingleMetric('ai_research_v4_parquet_sink_rows_rejected_total')!.get()
+ return metric.values.find((value) => value.labels.reason === reason)?.value ?? 0
+}
+
+describe('IndexBatcher', () => {
+ let puts: PutObjectCommandInput[]
+ let send: jest.Mock
+ let offsets: jest.Mocked<OffsetStore>
+
+ const makeBatcher = (options: Partial<IndexBatcherOptions> = {}) =>
+ new IndexBatcher(
+ new IndexParquetStore({ send } as unknown as S3Client, 'rrweb-v4', 1_000),
+ offsets,
+ { flushIntervalMs: 600_000, maxRows: 1_000, maxBytes: 1024 * 1024, ...options },
+ 0
+ )
+
+ const keysOf = (index: V4Index): string[] =>
+ puts.map((put) => put.Key!).filter((key) => key.startsWith(`index/${index}/`))
+
+ beforeEach(() => {
+ puts = []
+ send = jest.fn((command: { input: PutObjectCommandInput }) => {
+ puts.push(command.input)
+ return Promise.resolve({})
+ })
+ offsets = { offsetsStore: jest.fn() }
+ })
+
+ it('writes one file per index and write day, then stores the next offset of each partition', async () => {
+ const batcher = makeBatcher({ maxRows: 4 })
+ await batcher.handleBatch(
+ [
+ message(10, 'blocks', '2026-10-15', blockRow('s1')),
+ message(11, 'images', '2026-10-15', imageRow('image:v4:7:2026-10:a')),
+ message(5, 'blocks', '2026-10-16', blockRow('s3'), 1),
+ ],
+ 1
+ )
+ expect(send).not.toHaveBeenCalled()
+
+ await batcher.handleBatch([message(12, 'blocks', '2026-10-15', blockRow('s2'))], 2)
+
+ expect(puts.map((put) => [put.Bucket, put.IfNoneMatch])).toEqual(Array(3).fill(['rrweb-v4', '*']))
+ const [day15, day16] = keysOf('blocks').sort()
+ expect(day15).toMatch(/^index\/blocks\/dt=2026-10-15\/\d+-[0-9a-f]{16}\.parquet$/)
+ expect(day16).toMatch(/^index\/blocks\/dt=2026-10-16\//)
+ expect(keysOf('images')).toEqual([expect.stringMatching(/^index\/images\/dt=2026-10-15\//)])
+ const body = (key: string): Buffer => puts.find((put) => put.Key === key)!.Body as Buffer
+ expect(await readColumn(body(day15), 'session_id')).toEqual(['s1', 's2'])
+ expect(await readColumn(body(day16), 'session_id')).toEqual(['s3'])
+ expect(await readColumn(body(keysOf('images')[0]), 'image_ref')).toEqual(['image:v4:7:2026-10:a'])
+ expect(offsets.offsetsStore).toHaveBeenCalledTimes(1)
+ expect(offsets.offsetsStore.mock.calls[0][0]).toEqual(
+ expect.arrayContaining([
+ { topic: TOPIC, partition: 0, offset: 13 },
+ { topic: TOPIC, partition: 1, offset: 6 },
+ ])
+ )
+ })
+
+ it.each([
+ ['value that is not JSON', 'parse_failed', message(1, 'blocks', '2026-10-15', null, 0, '{not json')],
+ [
+ 'count above the INT32 range',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), clicks: 2 ** 31 }),
+ ],
+ [
+ 'moment index above the INT32 range',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', {
+ ...blockRow('s1'),
+ moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 }],
+ }),
+ ],
+ [
+ 'flag that is not a boolean',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), moments_truncated: 1 }),
+ ],
+ [
+ 'flush row count above the INT32 range',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), flush_row_count: 2 ** 31 }),
+ ],
+ [
+ 'url fingerprint that is not a string',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), url_fingerprints: [7] }),
+ ],
+ [
+ 'window id list with a null item',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: ['w1', null] }),
+ ],
+ [
+ 'window ids that are not a list',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: 'w1' }),
+ ],
+ [
+ 'missing urls_truncated flag',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), urls_truncated: undefined }),
+ ],
+ [
+ 'moment without its fields',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), moments: [{ kind: 'click' }] }),
+ ],
+ ['block row sent as an image row', 'invalid_row', message(1, 'images', '2026-10-15', blockRow('s1'))],
+ ['unknown index header', 'unknown_index', message(1, 'sessions', '2026-10-15', blockRow('s1'))],
+ ['write day that is not a date', 'invalid_write_day', message(1, 'blocks', '2026-02-30', blockRow('s1'))],
+ ])('rejects a row with a %s as %s, and still moves past it', async (_name, reason, bad) => {
+ const before = await rejected(reason)
+ const batcher = makeBatcher({ flushIntervalMs: 1_000 })
+
+ await batcher.handleBatch([bad], 1_000)
+
+ expect(await rejected(reason)).toBe(before + 1)
+ expect(send).not.toHaveBeenCalled()
+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 2 }])
+ })
+
+ it.each([
+ ['a timestamp outside the JS-safe range', { ...blockRow('s1'), first_ts_ms: 2 ** 53 }, 'first_ts_ms'],
+ [
+ 'a moment index above the INT32 range',
+ { ...blockRow('s1'), moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 }] },
+ 'moments.event_index',
+ ],
+ ])('counts a row rejected for %s by the column that failed', async (_name, row, column) => {
+ const counted = async (): Promise<number> => {
+ const metric = await register.getSingleMetric('ai_research_v4_parquet_sink_invalid_columns_total')!.get()
+ return (
+ metric.values.find((value) => value.labels.index === 'blocks' && value.labels.column === column)
+ ?.value ?? 0
+ )
+ }
+ const before = await counted()
+
+ await makeBatcher().handleBatch([message(1, 'blocks', '2026-10-15', row)], 1)
+
+ expect(await counted()).toBe(before + 1)
+ })
+
+ it('stores no offsets when a write fails, and writes only the unwritten files on the next flush', async () => {
+ const batcher = makeBatcher()
+ await batcher.handleBatch(
+ [
+ message(1, 'blocks', '2026-10-15', blockRow('s1')),
+ message(2, 'images', '2026-10-15', imageRow('image:v4:7:2026-10:a')),
+ ],
+ 1
+ )
+ send.mockImplementationOnce((command: { input: PutObjectCommandInput }) => {
+ puts.push(command.input)
+ return Promise.resolve({})
+ }).mockRejectedValueOnce(new Error('s3 down'))
+
+ await expect(batcher.flush(2)).rejects.toThrow('s3 down')
+ expect(offsets.offsetsStore).not.toHaveBeenCalled()
+ expect(keysOf('blocks')).toHaveLength(1)
+
+ await batcher.flush(3)
+ expect(keysOf('blocks')).toHaveLength(1)
+ expect(keysOf('images')).toHaveLength(1)
+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 3 }])
+ })
+
+ it('reports health while a flush waits on a slow upload, and stops when the flush ends', async () => {
+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask', 'setTimeout', 'clearTimeout'] })
+ const heartbeat = jest.fn()
+ const batcher = makeBatcher({ heartbeat, heartbeatIntervalMs: 10_000 })
+ await batcher.handleBatch([message(1, 'blocks', '2026-10-15', blockRow('s1'))], 1)
+ let finishUpload: ((value: unknown) => void) | undefined
+ send.mockImplementationOnce(() => new Promise((resolve) => (finishUpload = resolve)))
+
+ const flushed = batcher.flush(2)
+ while (!finishUpload) {
+ await new Promise((resolve) => setImmediate(resolve))
+ }
+ jest.advanceTimersByTime(25_000)
+ expect(heartbeat).toHaveBeenCalledTimes(2)
+ finishUpload({})
+ await flushed
+ jest.advanceTimersByTime(20_000)
+
+ expect(heartbeat).toHaveBeenCalledTimes(2)
+ jest.useRealTimers()
+ })
+
+ it('leaves out the rows and offsets of a revoked partition', async () => {
+ const batcher = makeBatcher()
+ await batcher.handleBatch(
+ [
+ message(1, 'blocks', '2026-10-15', blockRow('kept')),
+ message(7, 'blocks', '2026-10-15', blockRow('gone'), 1),
+ ],
+ 1
+ )
+
+ batcher.release([{ topic: TOPIC, partition: 1 }])
+ await batcher.flush(2)
+
+ expect(await readColumn(puts[0].Body as Buffer, 'session_id')).toEqual(['kept'])
+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 2 }])
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.ts
new file mode 100644
index 000000000000..1ff5677a1e09
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.ts
@@ -0,0 +1,215 @@
+/**
+ * Buffers v4 index rows across Kafka batches and writes one Parquet file per index and write day on each flush. Offsets
+ * are stored only after every file that holds their rows is written, so a failed write replays (at-least-once).
+ */
+import { Message, TopicPartition, TopicPartitionOffset } from 'node-rdkafka'
+
+import { findOffsetsToCommit } from '~/common/kafka/consumer/consumer-v1'
+import { logger } from '~/common/utils/logger'
+import { OffsetStore } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+
+import { V4Index, V4IndexRow } from './index-columns'
+import { parseIndexMessage } from './index-message'
+import { V4ParquetSinkMetrics } from './metrics'
+
+export type IndexFileStore = {
+ encode(index: V4Index, rows: readonly V4IndexRow[]): Promise<Buffer>
+ put(index: V4Index, writeDay: string, body: Buffer, nowMs: number): Promise<string>
+}
+
+export interface IndexBatcherOptions {
+ flushIntervalMs: number
+ maxRows: number
+ maxBytes: number
+ /** Called while a flush runs, so the health check does not restart a pod whose uploads are slow but bounded. */
+ heartbeat?: () => void
+ heartbeatIntervalMs?: number
+}
+
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000
+
+interface BufferedRow {
+ row: V4IndexRow
+ partition: string
+ bytes: number
+}
+
+interface IndexFileGroup {
+ index: V4Index
+ writeDay: string
+ rows: BufferedRow[]
+}
+
+const partitionKey = (tp: TopicPartition): string => `${tp.topic}:${tp.partition}`
+
+export class IndexBatcher {
+ private groups = new Map<string, IndexFileGroup>()
+ private rowCount = 0
+ private bufferedBytes = 0
+ private pendingOffsets = new Map<string, TopicPartitionOffset>()
+ /** Partitions revoked since their last message. A flush that started before the revocation must not store their offsets. */
+ private readonly revoked = new Set<string>()
+ private lastFlushMs: number
+ private flushQueue: Promise<void> = Promise.resolve()
+
+ constructor(
+ private readonly store: IndexFileStore,
+ private readonly offsetStore: OffsetStore,
+ private readonly options: IndexBatcherOptions,
+ nowMs: number
+ ) {
+ this.lastFlushMs = nowMs
+ }
+
+ public async handleBatch(messages: Message[], nowMs: number): Promise<void> {
+ for (const message of messages) {
+ const partition = partitionKey(message)
+ // A message proves the partition is assigned to this consumer again.
+ this.revoked.delete(partition)
+ const parsed = parseIndexMessage(message)
+ if (parsed) {
+ this.add(parsed.index, parsed.writeDay, {
+ row: parsed.row,
+ partition,
+ bytes: message.value?.length ?? 0,
+ })
+ }
+ }
+ for (const offset of findOffsetsToCommit(messages)) {
+ this.pendingOffsets.set(partitionKey(offset), offset)
+ }
+ if (this.shouldFlush(nowMs)) {
+ await this.flush(nowMs)
+ }
+ }
+
+ /**
+ * Drops the buffered rows and offsets of revoked partitions. Their new owner reads them again from the committed
+ * offset, so writing them here would only duplicate them, and this consumer can no longer store their offsets.
+ */
+ public release(partitions: readonly TopicPartition[]): void {
+ const keys = new Set(partitions.map(partitionKey))
+ let released = 0
+ for (const key of keys) {
+ this.revoked.add(key)
+ this.pendingOffsets.delete(key)
+ }
+ for (const [groupKey, group] of this.groups) {
+ const kept = group.rows.filter((buffered) => !keys.has(buffered.partition))
+ for (const buffered of group.rows) {
+ if (keys.has(buffered.partition)) {
+ released += 1
+ this.bufferedBytes -= buffered.bytes
+ }
+ }
+ if (kept.length === 0) {
+ this.groups.delete(groupKey)
+ } else {
+ group.rows = kept
+ }
+ }
+ this.rowCount -= released
+ V4ParquetSinkMetrics.incRowsReleased(released)
+ }
+
+ private add(index: V4Index, writeDay: string, buffered: BufferedRow): void {
+ const key = `${index}/${writeDay}`
+ const group = this.groups.get(key) ?? { index, writeDay, rows: [] }
+ group.rows.push(buffered)
+ this.groups.set(key, group)
+ this.rowCount += 1
+ this.bufferedBytes += buffered.bytes
+ }
+
+ private shouldFlush(nowMs: number): boolean {
+ if (this.rowCount >= this.options.maxRows || this.bufferedBytes >= this.options.maxBytes) {
+ return true
+ }
+ // Offsets of messages that gave no rows also flush on the interval, so they do not replay forever.
+ const hasPending = this.rowCount > 0 || this.pendingOffsets.size > 0
+ return hasPending && nowMs - this.lastFlushMs >= this.options.flushIntervalMs
+ }
+
+ public flush(nowMs: number): Promise<void> {
+ // A later flush's offsets also cover rows that an earlier flush is still writing, so flushes run one at a time and in order.
+ const flushed = this.flushQueue.then(() => this.flushNow(nowMs))
+ this.flushQueue = flushed.catch(() => undefined)
+ return flushed
+ }
+
+ private async flushNow(nowMs: number): Promise<void> {
+ const { heartbeat } = this.options
+ // Each upload has its own timeout and encoding blocks the event loop, so this cannot hide a stalled flush.
+ const heartbeats = heartbeat
+ ? setInterval(heartbeat, this.options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS)
+ : undefined
+ heartbeats?.unref()
+ try {
+ await this.writeTaken(nowMs)
+ } finally {
+ if (heartbeats) {
+ clearInterval(heartbeats)
+ }
+ }
+ }
+
+ private async writeTaken(nowMs: number): Promise<void> {
+ this.lastFlushMs = nowMs
+ // The shutdown flush runs while the consumer loop still delivers batches, so a flush writes only what it takes here.
+ const groups = this.groups
+ const offsets = this.pendingOffsets
+ this.groups = new Map()
+ this.pendingOffsets = new Map()
+ this.rowCount = 0
+ this.bufferedBytes = 0
+ const wroteFiles = groups.size > 0
+ const unwritten = new Map(groups)
+ try {
+ for (const [groupKey, group] of groups) {
+ const rows = group.rows.map((buffered) => buffered.row)
+ let body: Buffer
+ let key: string
+ try {
+ body = await this.store.encode(group.index, rows)
+ key = await this.store.put(group.index, group.writeDay, body, Date.now())
+ } catch (error) {
+ V4ParquetSinkMetrics.incWriteError()
+ throw error
+ }
+ unwritten.delete(groupKey)
+ V4ParquetSinkMetrics.observeFile(group.index, rows.length, body.length)
+ logger.info('🪶', 'ai_research_v4_parquet_index_written', {
+ key,
+ rows: rows.length,
+ bytes: body.length,
+ })
+ }
+ const assigned = [...offsets].filter(([partition]) => !this.revoked.has(partition))
+ if (assigned.length > 0) {
+ this.offsetStore.offsetsStore(assigned.map(([, offset]) => offset))
+ }
+ if (offsets.size > 0) {
+ V4ParquetSinkMetrics.incFlush(wroteFiles ? 'written' : 'empty')
+ }
+ } catch (error) {
+ for (const [groupKey, group] of unwritten) {
+ const current = this.groups.get(groupKey)
+ const rows = group.rows.filter((buffered) => !this.revoked.has(buffered.partition))
+ if (current) {
+ current.rows = rows.concat(current.rows)
+ } else if (rows.length > 0) {
+ this.groups.set(groupKey, { ...group, rows })
+ }
+ this.rowCount += rows.length
+ this.bufferedBytes += rows.reduce((bytes, buffered) => bytes + buffered.bytes, 0)
+ }
+ for (const [partition, offset] of offsets) {
+ // A newer offset for the partition stays, because the next flush stores it only after writing every row it covers.
+ if (!this.pendingOffsets.has(partition) && !this.revoked.has(partition)) {
+ this.pendingOffsets.set(partition, offset)
+ }
+ }
+ throw error
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts
new file mode 100644
index 000000000000..f7a69e5b3932
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts
@@ -0,0 +1,187 @@
+/**
+ * The columns of the two v4 index files, in file order, with parquetjs types. This table drives the row validator and
+ * the Parquet writer, as `ml-mirror/block-metadata-columns.ts` does for v3. Kept free of the Parquet dependency so the
+ * message-parsing path does not pull it in.
+ */
+
+export type V4Index = 'blocks' | 'images'
+
+export type ParquetType = 'UTF8' | 'BOOLEAN' | 'INT32' | 'INT64'
+
+export interface V4LeafColumn {
+ name: string
+ type: ParquetType
+ /** Nullable column: it can be null or absent on a row. */
+ optional?: boolean
+ /** Sealed bytes with no useful order. Min and max statistics would copy each long value into the file metadata. */
+ opaque?: boolean
+}
+
+/** A repeated group of required leaves; Parquet readers see it as a non-null list of structs. */
+export interface V4RepeatedColumn {
+ name: string
+ repeated: true
+ fields: readonly V4LeafColumn[]
+}
+
+/** A repeated leaf; Parquet readers see it as a non-null list of non-null values. */
+export interface V4ListColumn {
+ name: string
+ list: ParquetType
+}
+
+export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn
+
+const required = (type: ParquetType, ...names: string[]): V4LeafColumn[] => names.map((name) => ({ name, type }))
+const optional = (type: ParquetType, ...names: string[]): V4LeafColumn[] =>
+ names.map((name) => ({ name, type, optional: true }))
+
+export const MOMENT_COLUMNS: readonly V4LeafColumn[] = [
+ ...required('UTF8', 'kind', 'window_id'),
+ ...required('INT64', 'ts_ms'),
+ ...required('INT32', 'event_index'),
+]
+
+export const IMAGE_INDEX_COLUMNS: readonly V4Column[] = [
+ ...required('INT64', 'team_id'),
+ ...required('UTF8', 'key_month', 'image_ref', 'pack_s3_key'),
+ ...required('INT64', 'byte_start', 'byte_end'),
+ ...required('UTF8', 'mime'),
+ ...required('INT64', 'byte_size'),
+]
+
+export const BLOCK_INDEX_COLUMNS: readonly V4Column[] = [
+ ...required('INT64', 'team_id'),
+ ...required('UTF8', 'session_id', 'key_month', 'block_s3_key', 'topic'),
+ ...required('INT32', 'partition'),
+ ...required('INT64', 'start_offset', 'end_offset', 'flush_end_offset'),
+ ...required('INT32', 'flush_row_count', 'flush_row_index'),
+ ...required('UTF8', 'flush_id'),
+ ...required('INT64', 'flushed_at_ms', 'block_byte_start', 'block_byte_end'),
+ ...optional('INT64', 'first_ts_ms', 'last_ts_ms'),
+ ...required('INT64', 'event_count', 'byte_size'),
+ ...optional('UTF8', 'snapshot_source', 'snapshot_library', 'host'),
+ ...required(
+ 'INT32',
+ 'full_snapshots',
+ 'events_before_first_full_snapshot',
+ 'orphan_mutations',
+ 'decode_failures',
+ 'dom_nodes',
+ 'remote_stylesheets',
+ 'inline_stylesheets',
+ 'iframes',
+ 'canvases',
+ 'videos',
+ 'shadow_roots'
+ ),
+ ...required('INT64', 'max_gap_ms', 'active_ms', 'dom_text_chars'),
+ ...required('BOOLEAN', 'cold_start', 'node_cap_reached', 'moments_truncated', 'touch_input'),
+ ...required(
+ 'INT32',
+ 'clicks',
+ 'mouse_interactions',
+ 'rage_clicks',
+ 'dead_clicks',
+ 'keypresses',
+ 'scrolls',
+ 'mouse_moves',
+ 'touches',
+ 'touch_moves',
+ 'selections',
+ 'media_events',
+ 'viewport_resizes',
+ 'navigations',
+ 'console_errors',
+ 'console_warnings',
+ 'network_requests',
+ 'network_failures',
+ 'images_extracted',
+ 'images_inline',
+ 'image_urls'
+ ),
+ ...required('INT64', 'image_bytes', 'written_jsonl_bytes', 'max_written_event_bytes'),
+ ...optional('INT32', 'viewport_width', 'viewport_height'),
+ ...optional('INT64', 'viewport_ts_ms'),
+ { name: 'moments', repeated: true, fields: MOMENT_COLUMNS },
+ { name: 'url_fingerprints', list: 'UTF8' },
+ { name: 'window_ids', list: 'UTF8' },
+ { name: 'urls', type: 'UTF8', optional: true, opaque: true },
+ ...required('BOOLEAN', 'urls_truncated'),
+]
+
+export const INDEX_COLUMNS: Record<V4Index, readonly V4Column[]> = {
+ blocks: BLOCK_INDEX_COLUMNS,
+ images: IMAGE_INDEX_COLUMNS,
+}
+
+export type V4IndexRow = Record<string, unknown>
+
+const INT32_MIN = -(2 ** 31)
+const INT32_MAX = 2 ** 31 - 1
+
+// JSON numbers above 2^53 lose precision on parse, so an INT64 column takes only safe integers.
+function isEncodableValue(type: ParquetType, value: unknown): boolean {
+ switch (type) {
+ case 'UTF8':
+ return typeof value === 'string'
+ case 'BOOLEAN':
+ return typeof value === 'boolean'
+ case 'INT32':
+ return Number.isInteger(value) && (value as number) >= INT32_MIN && (value as number) <= INT32_MAX
+ case 'INT64':
+ return Number.isSafeInteger(value)
+ }
+}
+
+/** The selected columns of a row, or the first column whose value is missing or does not fit its type. */
+export type IndexRowSelection = { row: V4IndexRow } | { invalidColumn: string }
+
+function selectColumns(value: unknown, columns: readonly V4Column[], prefix = ''): IndexRowSelection {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ return { invalidColumn: prefix ? prefix.slice(0, -1) : 'row' }
+ }
+ const source = value as Record<string, unknown>
+ const row: V4IndexRow = {}
+ for (const column of columns) {
+ const field = source[column.name]
+ const invalid = { invalidColumn: `${prefix}${column.name}` }
+ if ('repeated' in column) {
+ if (!Array.isArray(field)) {
+ return invalid
+ }
+ const items: V4IndexRow[] = []
+ for (const item of field) {
+ const selected = selectColumns(item, column.fields, `${prefix}${column.name}.`)
+ if (!('row' in selected)) {
+ return selected
+ }
+ items.push(selected.row)
+ }
+ row[column.name] = items
+ } else if ('list' in column) {
+ if (!Array.isArray(field) || !field.every((item) => isEncodableValue(column.list, item))) {
+ return invalid
+ }
+ row[column.name] = field
+ } else if (field === null || field === undefined) {
+ if (!column.optional) {
+ return invalid
+ }
+ row[column.name] = null
+ } else if (isEncodableValue(column.type, field)) {
+ row[column.name] = field
+ } else {
+ return invalid
+ }
+ }
+ return { row }
+}
+
+/**
+ * Returns the row's index columns, or the first column that is missing or does not fit its type. Keys that are not
+ * columns are left out, as the v3 sink does. A bad row must not reach the writer, which would fail the whole flush.
+ */
+export function selectIndexRow(index: V4Index, value: unknown): IndexRowSelection {
+ return selectColumns(value, INDEX_COLUMNS[index])
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts
new file mode 100644
index 000000000000..f867c1f0c407
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts
@@ -0,0 +1,69 @@
+/** Reads one v4 index row from its Kafka message, as the recorder sends it. */
+import { Message } from 'node-rdkafka'
+
+import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'
+import { parseJSON } from '~/common/utils/json-parse'
+import { logger } from '~/common/utils/logger'
+
+import { V4Index, V4IndexRow, selectIndexRow } from './index-columns'
+import { V4ParquetSinkMetrics, V4RowRejection } from './metrics'
+
+/** Which index the row belongs to: `blocks` or `images`. */
+export const INDEX_HEADER = 'ml-index'
+/** The UTC day of the objects the row points at, the `dt=` partition of its file. */
+export const WRITE_DAY_HEADER = 'ml-write-day'
+
+export interface V4IndexMessage {
+ index: V4Index
+ writeDay: string
+ row: V4IndexRow
+}
+
+function isIndex(value: string | undefined): value is V4Index {
+ return value === 'blocks' || value === 'images'
+}
+
+function isWriteDay(value: string | undefined): value is string {
+ if (!value || !/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+ return false
+ }
+ const day = new Date(`${value}T00:00:00Z`)
+ return !Number.isNaN(day.getTime()) && day.toISOString().slice(0, 10) === value
+}
+
+function reject(reason: V4RowRejection, message: Message, invalidColumn?: string): null {
+ // Skip a bad row rather than wedge the partition, as the v3 sink does.
+ logger.warn('🪶', 'ai_research_v4_parquet_row_rejected', {
+ reason,
+ partition: message.partition,
+ offset: message.offset,
+ ...(invalidColumn && { column: invalidColumn }),
+ })
+ V4ParquetSinkMetrics.incRowsRejected(reason)
+ return null
+}
+
+export function parseIndexMessage(message: Message): V4IndexMessage | null {
+ const headers = parseKafkaHeaders(message.headers)
+ const index = headers[INDEX_HEADER]
+ if (!isIndex(index)) {
+ return reject('unknown_index', message)
+ }
+ const writeDay = headers[WRITE_DAY_HEADER]
+ if (!isWriteDay(writeDay)) {
+ return reject('invalid_write_day', message)
+ }
+ let value: unknown
+ try {
+ value = parseJSON(message.value?.toString('utf8') ?? '')
+ } catch {
+ return reject('parse_failed', message)
+ }
+ const selected = selectIndexRow(index, value)
+ if (!('row' in selected)) {
+ V4ParquetSinkMetrics.incInvalidColumn(index, selected.invalidColumn)
+ return reject('invalid_row', message, selected.invalidColumn)
+ }
+ V4ParquetSinkMetrics.incRowsParsed(index)
+ return { index, writeDay, row: selected.row }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.ts
new file mode 100644
index 000000000000..d5b28def1b66
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.ts
@@ -0,0 +1,49 @@
+/** Writes one v4 index file to `index/{index}/dt={day}/` in the rrweb bucket, named as the recorder names its files. */
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
+import { randomBytes } from 'crypto'
+
+import { V4Index, V4IndexRow } from './index-columns'
+import { indexRowsToParquet } from './parquet-writer'
+
+// A file can hold up to the sink's byte target of rows, so the upload gets more time than v3's 30 seconds.
+const PUT_TIMEOUT_MS = 120_000
+
+export function indexFileKey(index: V4Index, writeDay: string, nowMs: number): string {
+ return `index/${index}/dt=${writeDay}/${nowMs}-${randomBytes(8).toString('hex')}.parquet`
+}
+
+/** A 412 means the SDK retried a PUT whose first try stored the object, so the object is there. */
+function isPreconditionFailed(error: unknown): boolean {
+ if (typeof error !== 'object' || error === null) {
+ return false
+ }
+ const { $metadata, name } = error as { $metadata?: { httpStatusCode?: unknown }; name?: unknown }
+ return $metadata?.httpStatusCode === 412 || name === 'PreconditionFailed'
+}
+
+export class IndexParquetStore {
+ constructor(
+ private readonly s3Client: S3Client,
+ private readonly bucket: string,
+ private readonly rowGroupSize: number
+ ) {}
+
+ public encode(index: V4Index, rows: readonly V4IndexRow[]): Promise<Buffer> {
+ return indexRowsToParquet(index, rows, this.rowGroupSize)
+ }
+
+ public async put(index: V4Index, writeDay: string, body: Buffer, nowMs: number): Promise<string> {
+ const key = indexFileKey(index, writeDay, nowMs)
+ try {
+ await this.s3Client.send(
+ new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: body, IfNoneMatch: '*' }),
+ { abortSignal: AbortSignal.timeout(PUT_TIMEOUT_MS) }
+ )
+ } catch (error) {
+ if (!isPreconditionFailed(error)) {
+ throw error
+ }
+ }
+ return key
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.ts
new file mode 100644
index 000000000000..4e3d504a2653
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.ts
@@ -0,0 +1,75 @@
+import { Counter } from 'prom-client'
+
+import { V4Index } from './index-columns'
+
+export type V4RowRejection = 'parse_failed' | 'invalid_row' | 'unknown_index' | 'invalid_write_day'
+
+export class V4ParquetSinkMetrics {
+ private static readonly rowsParsed = new Counter({
+ name: 'ai_research_v4_parquet_sink_rows_parsed_total',
+ help: 'v4 index rows read from Kafka and accepted into the Parquet buffer',
+ labelNames: ['index'],
+ })
+ private static readonly rowsRejected = new Counter({
+ name: 'ai_research_v4_parquet_sink_rows_rejected_total',
+ help: 'v4 index messages skipped before buffering, by reason',
+ labelNames: ['reason'],
+ })
+ private static readonly invalidColumns = new Counter({
+ name: 'ai_research_v4_parquet_sink_invalid_columns_total',
+ help: 'v4 index rows rejected as invalid_row, by index and the first column that was missing or did not fit its type',
+ labelNames: ['index', 'column'],
+ })
+ private static readonly rowsReleased = new Counter({
+ name: 'ai_research_v4_parquet_sink_rows_released_total',
+ help: 'Buffered v4 index rows left out of the next file because their partition was revoked; the new owner reads them again',
+ })
+ private static readonly filesWritten = new Counter({
+ name: 'ai_research_v4_parquet_sink_files_written_total',
+ help: 'v4 index Parquet files written',
+ labelNames: ['index'],
+ })
+ private static readonly rowsWritten = new Counter({
+ name: 'ai_research_v4_parquet_sink_rows_written_total',
+ help: 'v4 index rows written as Parquet, including rows written again after a replay',
+ labelNames: ['index'],
+ })
+ private static readonly bytesWritten = new Counter({
+ name: 'ai_research_v4_parquet_sink_bytes_written_total',
+ help: 'v4 index Parquet bytes written',
+ labelNames: ['index'],
+ })
+ private static readonly writeErrors = new Counter({
+ name: 'ai_research_v4_parquet_sink_write_errors_total',
+ help: 'v4 index file writes that threw; the rows stay buffered and their offsets stay uncommitted',
+ })
+ private static readonly flushes = new Counter({
+ name: 'ai_research_v4_parquet_sink_flushes_total',
+ help: 'v4 sink flushes by outcome: wrote files, or committed offsets of messages that gave no rows',
+ labelNames: ['outcome'],
+ })
+
+ public static incRowsParsed(index: V4Index): void {
+ this.rowsParsed.labels(index).inc()
+ }
+ public static incRowsRejected(reason: V4RowRejection): void {
+ this.rowsRejected.labels(reason).inc()
+ }
+ public static incInvalidColumn(index: V4Index, column: string): void {
+ this.invalidColumns.labels(index, column).inc()
+ }
+ public static incRowsReleased(count: number): void {
+ this.rowsReleased.inc(count)
+ }
+ public static observeFile(index: V4Index, rows: number, bytes: number): void {
+ this.filesWritten.labels(index).inc()
+ this.rowsWritten.labels(index).inc(rows)
+ this.bytesWritten.labels(index).inc(bytes)
+ }
+ public static incWriteError(): void {
+ this.writeErrors.inc()
+ }
+ public static incFlush(outcome: 'written' | 'empty'): void {
+ this.flushes.labels(outcome).inc()
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.ts
new file mode 100644
index 000000000000..9c7876f4d07b
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.ts
@@ -0,0 +1,31 @@
+/** Encode already-mapped records into a Snappy/Parquet buffer via an in-memory sink — parquetjs only
+ * writes to a stream, so this is the openStream/collect plumbing. Schema + record shape are the caller's. */
+import { ParquetSchema, ParquetWriter } from '@dsnp/parquetjs'
+import { Writable } from 'stream'
+
+export async function parquetRecordsToBuffer(
+ schema: ParquetSchema,
+ records: Record<string, unknown>[],
+ options: { rowGroupSize?: number } = {}
+): Promise<Buffer> {
+ const chunks: Buffer[] = []
+ const sink = new Writable({
+ write(chunk: Buffer, _encoding, callback) {
+ chunks.push(chunk)
+ callback()
+ },
+ })
+ // parquetjs types want an fs.WriteStream, but it only calls write/end/on at runtime, which Writable provides.
+ const writer = await ParquetWriter.openStream(
+ schema,
+ sink as unknown as Parameters<typeof ParquetWriter.openStream>[1]
+ )
+ if (options.rowGroupSize !== undefined) {
+ writer.setRowGroupSize(options.rowGroupSize)
+ }
+ for (const record of records) {
+ await writer.appendRow(record)
+ }
+ await writer.close()
+ return Buffer.concat(chunks)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts
new file mode 100644
index 000000000000..5226d36f6307
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts
@@ -0,0 +1,190 @@
+import { ParquetReader } from '@dsnp/parquetjs'
+import { CompressionCodec } from '@dsnp/parquetjs/dist/gen-nodejs/parquet_types'
+import { randomBytes } from 'node:crypto'
+
+import { BLOCK_INDEX_COLUMNS, V4Index } from './index-columns'
+import { indexRowsToParquet } from './parquet-writer'
+
+async function describeFile(
+ file: Buffer
+): Promise<{ fields: string[]; rowGroups: number; codecs: unknown[]; rows: unknown[] }> {
+ const reader = await ParquetReader.openBuffer(file)
+ const fields = reader.schema.fieldList.map((field) =>
+ [field.repetitionType, field.primitiveType ?? 'group', field.path.join('.'), field.originalType]
+ .filter(Boolean)
+ .join(' ')
+ )
+ const rowGroups = reader.metadata!.row_groups.length
+ const codecs = reader.metadata!.row_groups.flatMap((group) =>
+ group.columns.map((column) => column.meta_data?.codec)
+ )
+ const rows: unknown[] = []
+ const cursor = reader.getCursor()
+ for (let row = await cursor.next(); row; row = await cursor.next()) {
+ rows.push(row)
+ }
+ await reader.close()
+ return { fields, rowGroups, codecs, rows }
+}
+
+describe('indexRowsToParquet', () => {
+ // The v4 index contract, shared with the recorder's JSON rows: name, order, nullability and type of each column.
+ const SCHEMAS: Record<V4Index, string[]> = {
+ images: [
+ 'REQUIRED INT64 team_id',
+ 'REQUIRED BYTE_ARRAY key_month UTF8',
+ 'REQUIRED BYTE_ARRAY image_ref UTF8',
+ 'REQUIRED BYTE_ARRAY pack_s3_key UTF8',
+ 'REQUIRED INT64 byte_start',
+ 'REQUIRED INT64 byte_end',
+ 'REQUIRED BYTE_ARRAY mime UTF8',
+ 'REQUIRED INT64 byte_size',
+ ],
+ blocks: [
+ 'REQUIRED INT64 team_id',
+ 'REQUIRED BYTE_ARRAY session_id UTF8',
+ 'REQUIRED BYTE_ARRAY key_month UTF8',
+ 'REQUIRED BYTE_ARRAY block_s3_key UTF8',
+ 'REQUIRED BYTE_ARRAY topic UTF8',
+ 'REQUIRED INT32 partition',
+ 'REQUIRED INT64 start_offset',
+ 'REQUIRED INT64 end_offset',
+ 'REQUIRED INT64 flush_end_offset',
+ 'REQUIRED INT32 flush_row_count',
+ 'REQUIRED INT32 flush_row_index',
+ 'REQUIRED BYTE_ARRAY flush_id UTF8',
+ 'REQUIRED INT64 flushed_at_ms',
+ 'REQUIRED INT64 block_byte_start',
+ 'REQUIRED INT64 block_byte_end',
+ 'OPTIONAL INT64 first_ts_ms',
+ 'OPTIONAL INT64 last_ts_ms',
+ 'REQUIRED INT64 event_count',
+ 'REQUIRED INT64 byte_size',
+ 'OPTIONAL BYTE_ARRAY snapshot_source UTF8',
+ 'OPTIONAL BYTE_ARRAY snapshot_library UTF8',
+ 'OPTIONAL BYTE_ARRAY host UTF8',
+ 'REQUIRED INT32 full_snapshots',
+ 'REQUIRED INT32 events_before_first_full_snapshot',
+ 'REQUIRED INT32 orphan_mutations',
+ 'REQUIRED INT32 decode_failures',
+ 'REQUIRED INT32 dom_nodes',
+ 'REQUIRED INT32 remote_stylesheets',
+ 'REQUIRED INT32 inline_stylesheets',
+ 'REQUIRED INT32 iframes',
+ 'REQUIRED INT32 canvases',
+ 'REQUIRED INT32 videos',
+ 'REQUIRED INT32 shadow_roots',
+ 'REQUIRED INT64 max_gap_ms',
+ 'REQUIRED INT64 active_ms',
+ 'REQUIRED INT64 dom_text_chars',
+ 'REQUIRED BOOLEAN cold_start',
+ 'REQUIRED BOOLEAN node_cap_reached',
+ 'REQUIRED BOOLEAN moments_truncated',
+ 'REQUIRED BOOLEAN touch_input',
+ 'REQUIRED INT32 clicks',
+ 'REQUIRED INT32 mouse_interactions',
+ 'REQUIRED INT32 rage_clicks',
+ 'REQUIRED INT32 dead_clicks',
+ 'REQUIRED INT32 keypresses',
+ 'REQUIRED INT32 scrolls',
+ 'REQUIRED INT32 mouse_moves',
+ 'REQUIRED INT32 touches',
+ 'REQUIRED INT32 touch_moves',
+ 'REQUIRED INT32 selections',
+ 'REQUIRED INT32 media_events',
+ 'REQUIRED INT32 viewport_resizes',
+ 'REQUIRED INT32 navigations',
+ 'REQUIRED INT32 console_errors',
+ 'REQUIRED INT32 console_warnings',
+ 'REQUIRED INT32 network_requests',
+ 'REQUIRED INT32 network_failures',
+ 'REQUIRED INT32 images_extracted',
+ 'REQUIRED INT32 images_inline',
+ 'REQUIRED INT32 image_urls',
+ 'REQUIRED INT64 image_bytes',
+ 'REQUIRED INT64 written_jsonl_bytes',
+ 'REQUIRED INT64 max_written_event_bytes',
+ 'OPTIONAL INT32 viewport_width',
+ 'OPTIONAL INT32 viewport_height',
+ 'OPTIONAL INT64 viewport_ts_ms',
+ 'REPEATED group moments',
+ 'REQUIRED BYTE_ARRAY moments.kind UTF8',
+ 'REQUIRED BYTE_ARRAY moments.window_id UTF8',
+ 'REQUIRED INT64 moments.ts_ms',
+ 'REQUIRED INT32 moments.event_index',
+ 'REPEATED BYTE_ARRAY url_fingerprints UTF8',
+ 'REPEATED BYTE_ARRAY window_ids UTF8',
+ 'OPTIONAL BYTE_ARRAY urls UTF8',
+ 'REQUIRED BOOLEAN urls_truncated',
+ ],
+ }
+
+ it.each(['blocks', 'images'] as const)('writes the %s columns of the v4 index contract', async (index) => {
+ const { fields } = await describeFile(await indexRowsToParquet(index, [], 1_000))
+ expect(fields).toEqual(SCHEMAS[index])
+ })
+
+ it('reads back the values it writes, in row groups of the given size, with snappy on every column', async () => {
+ const image = {
+ team_id: 7,
+ key_month: '2026-10',
+ image_ref: 'image:v4:7:2026-10:abc',
+ pack_s3_key: 'images/packs/dt=2026-10-15/7/1-0',
+ byte_start: 0,
+ byte_end: 99,
+ mime: 'image/png',
+ byte_size: 60,
+ }
+ const { rowGroups, codecs, rows } = await describeFile(
+ await indexRowsToParquet('images', Array(5).fill(image), 2)
+ )
+ expect(rowGroups).toBe(3)
+ expect(codecs).toEqual(Array(3 * 8).fill(CompressionCodec.SNAPPY))
+ expect(rows).toEqual(Array(5).fill({ ...image, team_id: 7n, byte_start: 0n, byte_end: 99n, byte_size: 60n }))
+ })
+
+ const block = (moments: unknown[], windowIds: string[]): Record<string, unknown> =>
+ Object.fromEntries(
+ BLOCK_INDEX_COLUMNS.map((column) => [
+ column.name,
+ 'repeated' in column
+ ? moments
+ : 'list' in column
+ ? windowIds
+ : column.optional
+ ? null
+ : column.type === 'UTF8'
+ ? 'x'
+ : column.type === 'BOOLEAN'
+ ? true
+ : 1,
+ ])
+ )
+
+ it('writes a large sealed urls value with no copy of it in the statistics', async () => {
+ const urls = randomBytes(768 * 1024).toString('base64')
+
+ const file = await indexRowsToParquet('blocks', [{ ...block([], []), urls, urls_truncated: false }], 1_000)
+
+ const { rows } = await describeFile(file)
+ expect((rows as Record<string, unknown>[])[0]).toMatchObject({ urls, urls_truncated: false })
+ // Snappy cannot shrink random base64, so the file is about one copy of the value.
+ expect(file.length).toBeLessThan(urls.length * 1.1)
+ })
+
+ it('reads moments back as a list of structs and the string lists as lists', async () => {
+ const moments = [
+ { kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 - 1 },
+ { kind: 'full_snapshot_undecoded', window_id: 'w2', ts_ms: 2_000, event_index: 0 },
+ ]
+ const { rows } = await describeFile(
+ await indexRowsToParquet('blocks', [block(moments, ['w1', 'w2']), block([], [])], 1_000)
+ )
+ const read = rows as Record<string, unknown>[]
+ expect(read.map((row) => [row.moments, row.url_fingerprints, row.window_ids])).toEqual([
+ [moments.map((moment) => ({ ...moment, ts_ms: BigInt(moment.ts_ms) })), ['w1', 'w2'], ['w1', 'w2']],
+ // parquetjs reads an empty repeated field as null; Arrow readers see an empty list.
+ [null, null, null],
+ ])
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts
new file mode 100644
index 000000000000..3d23e79394c8
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts
@@ -0,0 +1,62 @@
+/** Encodes v4 index rows into a Snappy-compressed Parquet buffer, with the types of the COLUMNS table. */
+import { ParquetSchema } from '@dsnp/parquetjs'
+
+import { INDEX_COLUMNS, ParquetType, V4Column, V4Index, V4IndexRow } from './index-columns'
+import { parquetRecordsToBuffer } from './parquet-buffer'
+
+type SchemaDef = ConstructorParameters<typeof ParquetSchema>[0]
+
+// Snappy on every column: compression is a per-field option in parquetjs, not a writer option.
+function schemaDef(columns: readonly V4Column[]): SchemaDef {
+ const def: Record<string, unknown> = {}
+ for (const column of columns) {
+ if ('repeated' in column) {
+ def[column.name] = { repeated: true, fields: schemaDef(column.fields) }
+ } else if ('list' in column) {
+ def[column.name] = { type: column.list, repeated: true, compression: 'SNAPPY' }
+ } else {
+ def[column.name] = {
+ type: column.type,
+ compression: 'SNAPPY',
+ ...(column.optional && { optional: true }),
+ ...(column.opaque && { statistics: false }),
+ }
+ }
+ }
+ return def as SchemaDef
+}
+
+const SCHEMAS: Record<V4Index, ParquetSchema> = {
+ blocks: new ParquetSchema(schemaDef(INDEX_COLUMNS.blocks)),
+ images: new ParquetSchema(schemaDef(INDEX_COLUMNS.images)),
+}
+
+function toValue(type: ParquetType, value: unknown): unknown {
+ return type === 'INT64' ? BigInt(value as number) : value
+}
+
+function toRecord(row: V4IndexRow, columns: readonly V4Column[]): Record<string, unknown> {
+ const record: Record<string, unknown> = {}
+ for (const column of columns) {
+ const value = row[column.name]
+ if ('repeated' in column) {
+ record[column.name] = (value as V4IndexRow[]).map((item) => toRecord(item, column.fields))
+ } else if ('list' in column) {
+ record[column.name] = (value as unknown[]).map((item) => toValue(column.list, item))
+ } else if (value === null || value === undefined) {
+ record[column.name] = null
+ } else {
+ record[column.name] = toValue(column.type, value)
+ }
+ }
+ return record
+}
+
+export function indexRowsToParquet(index: V4Index, rows: readonly V4IndexRow[], rowGroupSize: number): Promise<Buffer> {
+ const columns = INDEX_COLUMNS[index]
+ return parquetRecordsToBuffer(
+ SCHEMAS[index],
+ rows.map((row) => toRecord(row, columns)),
+ { rowGroupSize }
+ )
+}
diff --git a/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.ts b/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.ts
new file mode 100644
index 000000000000..d4561e44a830
--- /dev/null
+++ b/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.ts
@@ -0,0 +1,22 @@
+import { KAFKA_AI_RESEARCH_V4_INDEX } from '~/common/config/kafka-topics'
+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+
+import { buildV4SinkConsumerConfig } from './ai-research-v4-parquet-sink-server'
+
+describe('v4 Parquet sink wiring', () => {
+ it('reads the v4 index topic in its own group, storing offsets only after a flush', () => {
+ const config = buildV4SinkConsumerConfig(buildAiResearchV4ServerConfig({}))
+ expect(config).toMatchObject({
+ topic: KAFKA_AI_RESEARCH_V4_INDEX,
+ groupId: 'ai-research-v4-parquet-sink',
+ autoCommit: true,
+ autoOffsetStore: false,
+ callEachBatchWhenEmpty: true,
+ })
+ })
+
+ it.each(['', 'session-replay-ml-parquet-sink'])('refuses the group %p', (group) => {
+ const config = buildAiResearchV4ServerConfig({ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: group })
+ expect(() => buildV4SinkConsumerConfig(config)).toThrow('AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID')
+ })
+})
diff --git a/nodejs/src/servers/ai-research-v4-parquet-sink-server.ts b/nodejs/src/servers/ai-research-v4-parquet-sink-server.ts
new file mode 100644
index 000000000000..808077e4bd6b
--- /dev/null
+++ b/nodejs/src/servers/ai-research-v4-parquet-sink-server.ts
@@ -0,0 +1,112 @@
+import { Message } from 'node-rdkafka'
+
+import { KAFKA_AI_RESEARCH_V4_INDEX } from '~/common/config/kafka-topics'
+import { KafkaConsumer, KafkaConsumerConfig, parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'
+import { logger } from '~/common/utils/logger'
+import { AiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+import { IndexBatcher } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher'
+import { INDEX_HEADER } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message'
+import { IndexParquetStore } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store'
+import { AiResearchV4ConsumerServer } from '~/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server'
+import {
+ CaptureWatermark,
+ CapturedData,
+ capturedRecords,
+ releasingOffsetStore,
+} from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+import { buildSessionRecordingS3Client } from '~/ingestion/pipelines/sessionreplay/shared/s3-client'
+
+import { CleanupResources } from './base-server'
+
+/** The v3 sink's default group. A v4 sink pod in it would take v3 partitions and read none of their topic. */
+const V3_PARQUET_SINK_GROUP_ID = 'session-replay-ml-parquet-sink'
+
+function assertOwnGroup(config: AiResearchV4ServerConfig): void {
+ const group = config.AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID
+ if (!group || group === V3_PARQUET_SINK_GROUP_ID) {
+ throw new Error('AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID must be set and differ from the v3 sink group')
+ }
+}
+
+/**
+ * Manual offsets and callEachBatchWhenEmpty, as in the v3 sink: the batcher stores offsets only after their files are
+ * written, and idle polls still reach the flush interval.
+ */
+export function buildV4SinkConsumerConfig(config: AiResearchV4ServerConfig): KafkaConsumerConfig {
+ assertOwnGroup(config)
+ return {
+ topic: KAFKA_AI_RESEARCH_V4_INDEX,
+ groupId: config.AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID,
+ autoCommit: true,
+ autoOffsetStore: false,
+ callEachBatchWhenEmpty: true,
+ }
+}
+
+/** Block rows feed the event metadata watermark and image rows the inline image one, as in v3. */
+function indexData(message: Message): CapturedData | undefined {
+ const index = parseKafkaHeaders(message.headers)[INDEX_HEADER]
+ return index === 'blocks' ? 'event_metadata' : index === 'images' ? 'inline_images' : undefined
+}
+
+export class AiResearchV4ParquetSinkServer extends AiResearchV4ConsumerServer {
+ protected async startServices(): Promise<void> {
+ const consumerConfig = buildV4SinkConsumerConfig(this.config)
+ const bucket = this.config.AI_RESEARCH_V4_RRWEB_BUCKET
+ if (!bucket) {
+ throw new Error('AI_RESEARCH_V4_RRWEB_BUCKET must be set for the v4 Parquet sink')
+ }
+ const rowGroupRows = this.config.AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS
+ if (!Number.isInteger(rowGroupRows) || rowGroupRows <= 0) {
+ throw new Error('AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS must be a positive integer')
+ }
+ const s3Client = buildSessionRecordingS3Client(this.config)
+ if (!s3Client) {
+ throw new Error('SESSION_RECORDING_V2_S3_* must be configured for the v4 Parquet sink')
+ }
+
+ const consumer = new KafkaConsumer(consumerConfig)
+ const watermark = new CaptureWatermark('ai_research_v4_parquet_sink')
+ const batcher = new IndexBatcher(
+ new IndexParquetStore(s3Client, bucket, rowGroupRows),
+ releasingOffsetStore(consumer, watermark),
+ {
+ flushIntervalMs: this.config.AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS,
+ maxRows: this.config.AI_RESEARCH_V4_PARQUET_MAX_ROWS,
+ maxBytes: this.config.AI_RESEARCH_V4_PARQUET_MAX_BYTES,
+ heartbeat: () => consumer.reportDeliberateWait(),
+ },
+ Date.now()
+ )
+ await consumer.connect(
+ (messages) => {
+ consumer.heartbeat()
+ watermark.hold(capturedRecords(messages, indexData))
+ return batcher.handleBatch(messages, Date.now())
+ },
+ (partitions) => {
+ batcher.release(partitions)
+ watermark.forget(partitions)
+ return Promise.resolve()
+ }
+ )
+
+ this.lifecycle.services.push({
+ id: 'ai-research-v4-parquet-sink',
+ // Flush the buffer so a graceful restart does not read it again. A failed final flush replays on restart.
+ onShutdown: async () => {
+ try {
+ await batcher.flush(Date.now())
+ } catch (error) {
+ logger.warn('🪶', 'ai_research_v4_parquet_sink_shutdown_flush_failed', { error: String(error) })
+ }
+ await consumer.disconnect()
+ },
+ healthcheck: () => consumer.isHealthy(),
+ })
+ }
+
+ protected getCleanupResources(): CleanupResources {
+ return { kafkaProducers: [], redisPools: [] }
+ }
+}