:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): write the v4 index to parquet from kafka (#119750)
Nicholas Waltz39dcf050ea20
diff --git a/nodejs/src/capabilities.ts b/nodejs/src/capabilities.tsindex 829e7e223c50..d2f2a29e2843 100644--- a/nodejs/src/capabilities.ts+++ b/nodejs/src/capabilities.ts@@ -213,6 +213,7 @@ export function getPluginServerCapabilities(case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:case PluginServerMode.ai_research_v4_image_fetch:case PluginServerMode.ai_research_v4_image_fetch_retry:+ case PluginServerMode.ai_research_v4_parquet_sink:case PluginServerMode.recording_api:throw new Error(`Mode ${mode} is handled by IngestionSessionReplayServer, not PluginServer`)}diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.tsindex d718de4234b6..ad16d4b7a1c7 100644--- a/nodejs/src/common/config.ts+++ b/nodejs/src/common/config.ts@@ -44,6 +44,7 @@ export enum PluginServerMode {recordings_blob_ingestion_v2_ml_image_fetch_retry = 'recordings-blob-ingestion-v2-ml-image-fetch-retry',ai_research_v4_image_fetch = 'ai-research-v4-image-fetch',ai_research_v4_image_fetch_retry = 'ai-research-v4-image-fetch-retry',+ ai_research_v4_parquet_sink = 'ai-research-v4-parquet-sink',cdp_processed_events = 'cdp-processed-events',cdp_person_updates = 'cdp-person-updates',cdp_data_warehouse_events = 'cdp-data-warehouse-events',diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.tsindex 61c8aa6299a0..ff32d96febe9 100644--- a/nodejs/src/common/config/kafka-topics.ts+++ b/nodejs/src/common/config/kafka-topics.ts@@ -60,6 +60,8 @@ export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH = `${prefix}ai_research_v4_image_fexport const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_v4_image_fetch_retry_1m${suffix}`export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_v4_image_fetch_retry_10m${suffix}`export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_v4_image_fetch_retry_1h${suffix}`+// v4 index rows: v4 recorder -> v4 Parquet sink, one message per Parquet row.+export const KAFKA_AI_RESEARCH_V4_INDEX = `${prefix}ai_research_v4_index${suffix}`// images the scrub sidecar cannot process, parked so they stop holding the head of their partition.// The original bytes are kept: unscrubbed content must never reach the ML bucket, but it must not bediff --git a/nodejs/src/index.ts b/nodejs/src/index.tsindex 8eb63a6f4773..477175bb9c53 100644--- a/nodejs/src/index.ts+++ b/nodejs/src/index.ts@@ -10,6 +10,7 @@ import {AiResearchV4ImageFetchRetryServer,AiResearchV4ImageFetchServer,} from '~/servers/ai-research-v4-image-fetch-server'+import { AiResearchV4ParquetSinkServer } from '~/servers/ai-research-v4-parquet-sink-server'import { NodeServer } from '~/servers/base-server'import { ErrorTrackingServer } from '~/servers/error-tracking-server'import { IngestionApiServer } from '~/servers/ingestion-api-server'@@ -67,6 +68,9 @@ function createServer(): NodeServer {case PluginServerMode.ai_research_v4_image_fetch_retry:return new AiResearchV4ImageFetchRetryServer()+ case PluginServerMode.ai_research_v4_parquet_sink:+ return new AiResearchV4ParquetSinkServer()+case PluginServerMode.recording_api:return new RecordingApiServer()diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.tsindex c4741bf8c4b9..17ba5c8d7422 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts@@ -130,6 +130,20 @@ export type AiResearchV4Config = {KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: string/** A short local delivery timeout, so a slow topic fails a batch fast instead of holding its offsets. */KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: string++ /** The v4 index sink: the bucket its Parquet files go to, under `index/`. */+ AI_RESEARCH_V4_RRWEB_BUCKET: string+ /** The v4 index sink's consumer group. Its own setting, so a v4 sink pod never joins the v3 sink group. */+ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: string+ /**+ * The v4 sink writes its files this long after its last flush, or when the buffer reaches either size below,+ * whichever comes first. The byte size counts Kafka message bytes, so it also bounds the buffer's memory.+ */+ AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS: number+ AI_RESEARCH_V4_PARQUET_MAX_ROWS: number+ AI_RESEARCH_V4_PARQUET_MAX_BYTES: number+ /** Rows per row group in a v4 index file, so a large file has few row groups. */+ AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS: number}export function getDefaultAiResearchV4Config(): AiResearchV4Config {@@ -181,6 +195,13 @@ export function getDefaultAiResearchV4Config(): AiResearchV4Config {KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '32768',KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: `${40 * 1024 * 1024 + 64 * 1024}`,KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: '20000',++ AI_RESEARCH_V4_RRWEB_BUCKET: '',+ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: 'ai-research-v4-parquet-sink',+ AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS: 10 * 60 * 1000,+ AI_RESEARCH_V4_PARQUET_MAX_ROWS: 1_000_000,+ AI_RESEARCH_V4_PARQUET_MAX_BYTES: 128 * 1024 * 1024,+ AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS: 131_072,}}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.tsnew file mode 100644index 000000000000..76b8eaeb4d98--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts@@ -0,0 +1,286 @@+import { PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'+import { ParquetReader } from '@dsnp/parquetjs'+import { Message } from 'node-rdkafka'+import { register } from 'prom-client'++import { OffsetStore } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'++import { IndexBatcher, IndexBatcherOptions } from './index-batcher'+import { BLOCK_INDEX_COLUMNS, V4Index } from './index-columns'+import { IndexParquetStore } from './index-parquet-store'++const TOPIC = 'ai_research_v4_index'++function blockRow(sessionId: string): Record<string, unknown> {+ const row: Record<string, unknown> = {}+ for (const column of BLOCK_INDEX_COLUMNS) {+ row[column.name] =+ 'repeated' in column || 'list' in column+ ? []+ : column.optional+ ? null+ : column.type === 'UTF8'+ ? 'x'+ : column.type === 'BOOLEAN'+ ? false+ : 1+ }+ return {+ ...row,+ session_id: sessionId,+ moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 3 }],+ url_fingerprints: ['b08517cee3ff5ecc'],+ window_ids: ['w1'],+ }+}++const imageRow = (imageRef: string): Record<string, unknown> => ({+ team_id: 7,+ key_month: '2026-10',+ image_ref: imageRef,+ pack_s3_key: 'images/packs/dt=2026-10-15/7/1-0',+ byte_start: 0,+ byte_end: 99,+ mime: 'image/png',+ byte_size: 60,+})++function message(+ offset: number,+ index: string,+ writeDay: string,+ value: unknown,+ partition = 0,+ raw?: string+): Message {+ return {+ topic: TOPIC,+ partition,+ offset,+ value: Buffer.from(raw ?? JSON.stringify(value)),+ headers: [{ 'ml-index': Buffer.from(index) }, { 'ml-write-day': Buffer.from(writeDay) }],+ } as unknown as Message+}++async function readColumn(body: Buffer, column: string): Promise<unknown[]> {+ const reader = await ParquetReader.openBuffer(body)+ const cursor = reader.getCursor()+ const values: unknown[] = []+ for (let record = await cursor.next(); record; record = await cursor.next()) {+ values.push((record as Record<string, unknown>)[column])+ }+ await reader.close()+ return values+}++async function rejected(reason: string): Promise<number> {+ const metric = await register.getSingleMetric('ai_research_v4_parquet_sink_rows_rejected_total')!.get()+ return metric.values.find((value) => value.labels.reason === reason)?.value ?? 0+}++describe('IndexBatcher', () => {+ let puts: PutObjectCommandInput[]+ let send: jest.Mock+ let offsets: jest.Mocked<OffsetStore>++ const makeBatcher = (options: Partial<IndexBatcherOptions> = {}) =>+ new IndexBatcher(+ new IndexParquetStore({ send } as unknown as S3Client, 'rrweb-v4', 1_000),+ offsets,+ { flushIntervalMs: 600_000, maxRows: 1_000, maxBytes: 1024 * 1024, ...options },+ 0+ )++ const keysOf = (index: V4Index): string[] =>+ puts.map((put) => put.Key!).filter((key) => key.startsWith(`index/${index}/`))++ beforeEach(() => {+ puts = []+ send = jest.fn((command: { input: PutObjectCommandInput }) => {+ puts.push(command.input)+ return Promise.resolve({})+ })+ offsets = { offsetsStore: jest.fn() }+ })++ it('writes one file per index and write day, then stores the next offset of each partition', async () => {+ const batcher = makeBatcher({ maxRows: 4 })+ await batcher.handleBatch(+ [+ message(10, 'blocks', '2026-10-15', blockRow('s1')),+ message(11, 'images', '2026-10-15', imageRow('image:v4:7:2026-10:a')),+ message(5, 'blocks', '2026-10-16', blockRow('s3'), 1),+ ],+ 1+ )+ expect(send).not.toHaveBeenCalled()++ await batcher.handleBatch([message(12, 'blocks', '2026-10-15', blockRow('s2'))], 2)++ expect(puts.map((put) => [put.Bucket, put.IfNoneMatch])).toEqual(Array(3).fill(['rrweb-v4', '*']))+ const [day15, day16] = keysOf('blocks').sort()+ expect(day15).toMatch(/^index\/blocks\/dt=2026-10-15\/\d+-[0-9a-f]{16}\.parquet$/)+ expect(day16).toMatch(/^index\/blocks\/dt=2026-10-16\//)+ expect(keysOf('images')).toEqual([expect.stringMatching(/^index\/images\/dt=2026-10-15\//)])+ const body = (key: string): Buffer => puts.find((put) => put.Key === key)!.Body as Buffer+ expect(await readColumn(body(day15), 'session_id')).toEqual(['s1', 's2'])+ expect(await readColumn(body(day16), 'session_id')).toEqual(['s3'])+ expect(await readColumn(body(keysOf('images')[0]), 'image_ref')).toEqual(['image:v4:7:2026-10:a'])+ expect(offsets.offsetsStore).toHaveBeenCalledTimes(1)+ expect(offsets.offsetsStore.mock.calls[0][0]).toEqual(+ expect.arrayContaining([+ { topic: TOPIC, partition: 0, offset: 13 },+ { topic: TOPIC, partition: 1, offset: 6 },+ ])+ )+ })++ it.each([+ ['value that is not JSON', 'parse_failed', message(1, 'blocks', '2026-10-15', null, 0, '{not json')],+ [+ 'count above the INT32 range',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), clicks: 2 ** 31 }),+ ],+ [+ 'moment index above the INT32 range',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', {+ ...blockRow('s1'),+ moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 }],+ }),+ ],+ [+ 'flag that is not a boolean',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), moments_truncated: 1 }),+ ],+ [+ 'flush row count above the INT32 range',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), flush_row_count: 2 ** 31 }),+ ],+ [+ 'url fingerprint that is not a string',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), url_fingerprints: [7] }),+ ],+ [+ 'window id list with a null item',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: ['w1', null] }),+ ],+ [+ 'window ids that are not a list',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: 'w1' }),+ ],+ [+ 'missing urls_truncated flag',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), urls_truncated: undefined }),+ ],+ [+ 'moment without its fields',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), moments: [{ kind: 'click' }] }),+ ],+ ['block row sent as an image row', 'invalid_row', message(1, 'images', '2026-10-15', blockRow('s1'))],+ ['unknown index header', 'unknown_index', message(1, 'sessions', '2026-10-15', blockRow('s1'))],+ ['write day that is not a date', 'invalid_write_day', message(1, 'blocks', '2026-02-30', blockRow('s1'))],+ ])('rejects a row with a %s as %s, and still moves past it', async (_name, reason, bad) => {+ const before = await rejected(reason)+ const batcher = makeBatcher({ flushIntervalMs: 1_000 })++ await batcher.handleBatch([bad], 1_000)++ expect(await rejected(reason)).toBe(before + 1)+ expect(send).not.toHaveBeenCalled()+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 2 }])+ })++ it.each([+ ['a timestamp outside the JS-safe range', { ...blockRow('s1'), first_ts_ms: 2 ** 53 }, 'first_ts_ms'],+ [+ 'a moment index above the INT32 range',+ { ...blockRow('s1'), moments: [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 }] },+ 'moments.event_index',+ ],+ ])('counts a row rejected for %s by the column that failed', async (_name, row, column) => {+ const counted = async (): Promise<number> => {+ const metric = await register.getSingleMetric('ai_research_v4_parquet_sink_invalid_columns_total')!.get()+ return (+ metric.values.find((value) => value.labels.index === 'blocks' && value.labels.column === column)+ ?.value ?? 0+ )+ }+ const before = await counted()++ await makeBatcher().handleBatch([message(1, 'blocks', '2026-10-15', row)], 1)++ expect(await counted()).toBe(before + 1)+ })++ it('stores no offsets when a write fails, and writes only the unwritten files on the next flush', async () => {+ const batcher = makeBatcher()+ await batcher.handleBatch(+ [+ message(1, 'blocks', '2026-10-15', blockRow('s1')),+ message(2, 'images', '2026-10-15', imageRow('image:v4:7:2026-10:a')),+ ],+ 1+ )+ send.mockImplementationOnce((command: { input: PutObjectCommandInput }) => {+ puts.push(command.input)+ return Promise.resolve({})+ }).mockRejectedValueOnce(new Error('s3 down'))++ await expect(batcher.flush(2)).rejects.toThrow('s3 down')+ expect(offsets.offsetsStore).not.toHaveBeenCalled()+ expect(keysOf('blocks')).toHaveLength(1)++ await batcher.flush(3)+ expect(keysOf('blocks')).toHaveLength(1)+ expect(keysOf('images')).toHaveLength(1)+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 3 }])+ })++ it('reports health while a flush waits on a slow upload, and stops when the flush ends', async () => {+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask', 'setTimeout', 'clearTimeout'] })+ const heartbeat = jest.fn()+ const batcher = makeBatcher({ heartbeat, heartbeatIntervalMs: 10_000 })+ await batcher.handleBatch([message(1, 'blocks', '2026-10-15', blockRow('s1'))], 1)+ let finishUpload: ((value: unknown) => void) | undefined+ send.mockImplementationOnce(() => new Promise((resolve) => (finishUpload = resolve)))++ const flushed = batcher.flush(2)+ while (!finishUpload) {+ await new Promise((resolve) => setImmediate(resolve))+ }+ jest.advanceTimersByTime(25_000)+ expect(heartbeat).toHaveBeenCalledTimes(2)+ finishUpload({})+ await flushed+ jest.advanceTimersByTime(20_000)++ expect(heartbeat).toHaveBeenCalledTimes(2)+ jest.useRealTimers()+ })++ it('leaves out the rows and offsets of a revoked partition', async () => {+ const batcher = makeBatcher()+ await batcher.handleBatch(+ [+ message(1, 'blocks', '2026-10-15', blockRow('kept')),+ message(7, 'blocks', '2026-10-15', blockRow('gone'), 1),+ ],+ 1+ )++ batcher.release([{ topic: TOPIC, partition: 1 }])+ await batcher.flush(2)++ expect(await readColumn(puts[0].Body as Buffer, 'session_id')).toEqual(['kept'])+ expect(offsets.offsetsStore).toHaveBeenCalledWith([{ topic: TOPIC, partition: 0, offset: 2 }])+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.tsnew file mode 100644index 000000000000..1ff5677a1e09--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.ts@@ -0,0 +1,215 @@+/**+ * Buffers v4 index rows across Kafka batches and writes one Parquet file per index and write day on each flush. Offsets+ * are stored only after every file that holds their rows is written, so a failed write replays (at-least-once).+ */+import { Message, TopicPartition, TopicPartitionOffset } from 'node-rdkafka'++import { findOffsetsToCommit } from '~/common/kafka/consumer/consumer-v1'+import { logger } from '~/common/utils/logger'+import { OffsetStore } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'++import { V4Index, V4IndexRow } from './index-columns'+import { parseIndexMessage } from './index-message'+import { V4ParquetSinkMetrics } from './metrics'++export type IndexFileStore = {+ encode(index: V4Index, rows: readonly V4IndexRow[]): Promise<Buffer>+ put(index: V4Index, writeDay: string, body: Buffer, nowMs: number): Promise<string>+}++export interface IndexBatcherOptions {+ flushIntervalMs: number+ maxRows: number+ maxBytes: number+ /** Called while a flush runs, so the health check does not restart a pod whose uploads are slow but bounded. */+ heartbeat?: () => void+ heartbeatIntervalMs?: number+}++const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000++interface BufferedRow {+ row: V4IndexRow+ partition: string+ bytes: number+}++interface IndexFileGroup {+ index: V4Index+ writeDay: string+ rows: BufferedRow[]+}++const partitionKey = (tp: TopicPartition): string => `${tp.topic}:${tp.partition}`++export class IndexBatcher {+ private groups = new Map<string, IndexFileGroup>()+ private rowCount = 0+ private bufferedBytes = 0+ private pendingOffsets = new Map<string, TopicPartitionOffset>()+ /** Partitions revoked since their last message. A flush that started before the revocation must not store their offsets. */+ private readonly revoked = new Set<string>()+ private lastFlushMs: number+ private flushQueue: Promise<void> = Promise.resolve()++ constructor(+ private readonly store: IndexFileStore,+ private readonly offsetStore: OffsetStore,+ private readonly options: IndexBatcherOptions,+ nowMs: number+ ) {+ this.lastFlushMs = nowMs+ }++ public async handleBatch(messages: Message[], nowMs: number): Promise<void> {+ for (const message of messages) {+ const partition = partitionKey(message)+ // A message proves the partition is assigned to this consumer again.+ this.revoked.delete(partition)+ const parsed = parseIndexMessage(message)+ if (parsed) {+ this.add(parsed.index, parsed.writeDay, {+ row: parsed.row,+ partition,+ bytes: message.value?.length ?? 0,+ })+ }+ }+ for (const offset of findOffsetsToCommit(messages)) {+ this.pendingOffsets.set(partitionKey(offset), offset)+ }+ if (this.shouldFlush(nowMs)) {+ await this.flush(nowMs)+ }+ }++ /**+ * Drops the buffered rows and offsets of revoked partitions. Their new owner reads them again from the committed+ * offset, so writing them here would only duplicate them, and this consumer can no longer store their offsets.+ */+ public release(partitions: readonly TopicPartition[]): void {+ const keys = new Set(partitions.map(partitionKey))+ let released = 0+ for (const key of keys) {+ this.revoked.add(key)+ this.pendingOffsets.delete(key)+ }+ for (const [groupKey, group] of this.groups) {+ const kept = group.rows.filter((buffered) => !keys.has(buffered.partition))+ for (const buffered of group.rows) {+ if (keys.has(buffered.partition)) {+ released += 1+ this.bufferedBytes -= buffered.bytes+ }+ }+ if (kept.length === 0) {+ this.groups.delete(groupKey)+ } else {+ group.rows = kept+ }+ }+ this.rowCount -= released+ V4ParquetSinkMetrics.incRowsReleased(released)+ }++ private add(index: V4Index, writeDay: string, buffered: BufferedRow): void {+ const key = `${index}/${writeDay}`+ const group = this.groups.get(key) ?? { index, writeDay, rows: [] }+ group.rows.push(buffered)+ this.groups.set(key, group)+ this.rowCount += 1+ this.bufferedBytes += buffered.bytes+ }++ private shouldFlush(nowMs: number): boolean {+ if (this.rowCount >= this.options.maxRows || this.bufferedBytes >= this.options.maxBytes) {+ return true+ }+ // Offsets of messages that gave no rows also flush on the interval, so they do not replay forever.+ const hasPending = this.rowCount > 0 || this.pendingOffsets.size > 0+ return hasPending && nowMs - this.lastFlushMs >= this.options.flushIntervalMs+ }++ public flush(nowMs: number): Promise<void> {+ // A later flush's offsets also cover rows that an earlier flush is still writing, so flushes run one at a time and in order.+ const flushed = this.flushQueue.then(() => this.flushNow(nowMs))+ this.flushQueue = flushed.catch(() => undefined)+ return flushed+ }++ private async flushNow(nowMs: number): Promise<void> {+ const { heartbeat } = this.options+ // Each upload has its own timeout and encoding blocks the event loop, so this cannot hide a stalled flush.+ const heartbeats = heartbeat+ ? setInterval(heartbeat, this.options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS)+ : undefined+ heartbeats?.unref()+ try {+ await this.writeTaken(nowMs)+ } finally {+ if (heartbeats) {+ clearInterval(heartbeats)+ }+ }+ }++ private async writeTaken(nowMs: number): Promise<void> {+ this.lastFlushMs = nowMs+ // The shutdown flush runs while the consumer loop still delivers batches, so a flush writes only what it takes here.+ const groups = this.groups+ const offsets = this.pendingOffsets+ this.groups = new Map()+ this.pendingOffsets = new Map()+ this.rowCount = 0+ this.bufferedBytes = 0+ const wroteFiles = groups.size > 0+ const unwritten = new Map(groups)+ try {+ for (const [groupKey, group] of groups) {+ const rows = group.rows.map((buffered) => buffered.row)+ let body: Buffer+ let key: string+ try {+ body = await this.store.encode(group.index, rows)+ key = await this.store.put(group.index, group.writeDay, body, Date.now())+ } catch (error) {+ V4ParquetSinkMetrics.incWriteError()+ throw error+ }+ unwritten.delete(groupKey)+ V4ParquetSinkMetrics.observeFile(group.index, rows.length, body.length)+ logger.info('🪶', 'ai_research_v4_parquet_index_written', {+ key,+ rows: rows.length,+ bytes: body.length,+ })+ }+ const assigned = [...offsets].filter(([partition]) => !this.revoked.has(partition))+ if (assigned.length > 0) {+ this.offsetStore.offsetsStore(assigned.map(([, offset]) => offset))+ }+ if (offsets.size > 0) {+ V4ParquetSinkMetrics.incFlush(wroteFiles ? 'written' : 'empty')+ }+ } catch (error) {+ for (const [groupKey, group] of unwritten) {+ const current = this.groups.get(groupKey)+ const rows = group.rows.filter((buffered) => !this.revoked.has(buffered.partition))+ if (current) {+ current.rows = rows.concat(current.rows)+ } else if (rows.length > 0) {+ this.groups.set(groupKey, { ...group, rows })+ }+ this.rowCount += rows.length+ this.bufferedBytes += rows.reduce((bytes, buffered) => bytes + buffered.bytes, 0)+ }+ for (const [partition, offset] of offsets) {+ // A newer offset for the partition stays, because the next flush stores it only after writing every row it covers.+ if (!this.pendingOffsets.has(partition) && !this.revoked.has(partition)) {+ this.pendingOffsets.set(partition, offset)+ }+ }+ throw error+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.tsnew file mode 100644index 000000000000..f7a69e5b3932--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts@@ -0,0 +1,187 @@+/**+ * The columns of the two v4 index files, in file order, with parquetjs types. This table drives the row validator and+ * the Parquet writer, as `ml-mirror/block-metadata-columns.ts` does for v3. Kept free of the Parquet dependency so the+ * message-parsing path does not pull it in.+ */++export type V4Index = 'blocks' | 'images'++export type ParquetType = 'UTF8' | 'BOOLEAN' | 'INT32' | 'INT64'++export interface V4LeafColumn {+ name: string+ type: ParquetType+ /** Nullable column: it can be null or absent on a row. */+ optional?: boolean+ /** Sealed bytes with no useful order. Min and max statistics would copy each long value into the file metadata. */+ opaque?: boolean+}++/** A repeated group of required leaves; Parquet readers see it as a non-null list of structs. */+export interface V4RepeatedColumn {+ name: string+ repeated: true+ fields: readonly V4LeafColumn[]+}++/** A repeated leaf; Parquet readers see it as a non-null list of non-null values. */+export interface V4ListColumn {+ name: string+ list: ParquetType+}++export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn++const required = (type: ParquetType, ...names: string[]): V4LeafColumn[] => names.map((name) => ({ name, type }))+const optional = (type: ParquetType, ...names: string[]): V4LeafColumn[] =>+ names.map((name) => ({ name, type, optional: true }))++export const MOMENT_COLUMNS: readonly V4LeafColumn[] = [+ ...required('UTF8', 'kind', 'window_id'),+ ...required('INT64', 'ts_ms'),+ ...required('INT32', 'event_index'),+]++export const IMAGE_INDEX_COLUMNS: readonly V4Column[] = [+ ...required('INT64', 'team_id'),+ ...required('UTF8', 'key_month', 'image_ref', 'pack_s3_key'),+ ...required('INT64', 'byte_start', 'byte_end'),+ ...required('UTF8', 'mime'),+ ...required('INT64', 'byte_size'),+]++export const BLOCK_INDEX_COLUMNS: readonly V4Column[] = [+ ...required('INT64', 'team_id'),+ ...required('UTF8', 'session_id', 'key_month', 'block_s3_key', 'topic'),+ ...required('INT32', 'partition'),+ ...required('INT64', 'start_offset', 'end_offset', 'flush_end_offset'),+ ...required('INT32', 'flush_row_count', 'flush_row_index'),+ ...required('UTF8', 'flush_id'),+ ...required('INT64', 'flushed_at_ms', 'block_byte_start', 'block_byte_end'),+ ...optional('INT64', 'first_ts_ms', 'last_ts_ms'),+ ...required('INT64', 'event_count', 'byte_size'),+ ...optional('UTF8', 'snapshot_source', 'snapshot_library', 'host'),+ ...required(+ 'INT32',+ 'full_snapshots',+ 'events_before_first_full_snapshot',+ 'orphan_mutations',+ 'decode_failures',+ 'dom_nodes',+ 'remote_stylesheets',+ 'inline_stylesheets',+ 'iframes',+ 'canvases',+ 'videos',+ 'shadow_roots'+ ),+ ...required('INT64', 'max_gap_ms', 'active_ms', 'dom_text_chars'),+ ...required('BOOLEAN', 'cold_start', 'node_cap_reached', 'moments_truncated', 'touch_input'),+ ...required(+ 'INT32',+ 'clicks',+ 'mouse_interactions',+ 'rage_clicks',+ 'dead_clicks',+ 'keypresses',+ 'scrolls',+ 'mouse_moves',+ 'touches',+ 'touch_moves',+ 'selections',+ 'media_events',+ 'viewport_resizes',+ 'navigations',+ 'console_errors',+ 'console_warnings',+ 'network_requests',+ 'network_failures',+ 'images_extracted',+ 'images_inline',+ 'image_urls'+ ),+ ...required('INT64', 'image_bytes', 'written_jsonl_bytes', 'max_written_event_bytes'),+ ...optional('INT32', 'viewport_width', 'viewport_height'),+ ...optional('INT64', 'viewport_ts_ms'),+ { name: 'moments', repeated: true, fields: MOMENT_COLUMNS },+ { name: 'url_fingerprints', list: 'UTF8' },+ { name: 'window_ids', list: 'UTF8' },+ { name: 'urls', type: 'UTF8', optional: true, opaque: true },+ ...required('BOOLEAN', 'urls_truncated'),+]++export const INDEX_COLUMNS: Record<V4Index, readonly V4Column[]> = {+ blocks: BLOCK_INDEX_COLUMNS,+ images: IMAGE_INDEX_COLUMNS,+}++export type V4IndexRow = Record<string, unknown>++const INT32_MIN = -(2 ** 31)+const INT32_MAX = 2 ** 31 - 1++// JSON numbers above 2^53 lose precision on parse, so an INT64 column takes only safe integers.+function isEncodableValue(type: ParquetType, value: unknown): boolean {+ switch (type) {+ case 'UTF8':+ return typeof value === 'string'+ case 'BOOLEAN':+ return typeof value === 'boolean'+ case 'INT32':+ return Number.isInteger(value) && (value as number) >= INT32_MIN && (value as number) <= INT32_MAX+ case 'INT64':+ return Number.isSafeInteger(value)+ }+}++/** The selected columns of a row, or the first column whose value is missing or does not fit its type. */+export type IndexRowSelection = { row: V4IndexRow } | { invalidColumn: string }++function selectColumns(value: unknown, columns: readonly V4Column[], prefix = ''): IndexRowSelection {+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ return { invalidColumn: prefix ? prefix.slice(0, -1) : 'row' }+ }+ const source = value as Record<string, unknown>+ const row: V4IndexRow = {}+ for (const column of columns) {+ const field = source[column.name]+ const invalid = { invalidColumn: `${prefix}${column.name}` }+ if ('repeated' in column) {+ if (!Array.isArray(field)) {+ return invalid+ }+ const items: V4IndexRow[] = []+ for (const item of field) {+ const selected = selectColumns(item, column.fields, `${prefix}${column.name}.`)+ if (!('row' in selected)) {+ return selected+ }+ items.push(selected.row)+ }+ row[column.name] = items+ } else if ('list' in column) {+ if (!Array.isArray(field) || !field.every((item) => isEncodableValue(column.list, item))) {+ return invalid+ }+ row[column.name] = field+ } else if (field === null || field === undefined) {+ if (!column.optional) {+ return invalid+ }+ row[column.name] = null+ } else if (isEncodableValue(column.type, field)) {+ row[column.name] = field+ } else {+ return invalid+ }+ }+ return { row }+}++/**+ * Returns the row's index columns, or the first column that is missing or does not fit its type. Keys that are not+ * columns are left out, as the v3 sink does. A bad row must not reach the writer, which would fail the whole flush.+ */+export function selectIndexRow(index: V4Index, value: unknown): IndexRowSelection {+ return selectColumns(value, INDEX_COLUMNS[index])+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.tsnew file mode 100644index 000000000000..f867c1f0c407--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts@@ -0,0 +1,69 @@+/** Reads one v4 index row from its Kafka message, as the recorder sends it. */+import { Message } from 'node-rdkafka'++import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'+import { parseJSON } from '~/common/utils/json-parse'+import { logger } from '~/common/utils/logger'++import { V4Index, V4IndexRow, selectIndexRow } from './index-columns'+import { V4ParquetSinkMetrics, V4RowRejection } from './metrics'++/** Which index the row belongs to: `blocks` or `images`. */+export const INDEX_HEADER = 'ml-index'+/** The UTC day of the objects the row points at, the `dt=` partition of its file. */+export const WRITE_DAY_HEADER = 'ml-write-day'++export interface V4IndexMessage {+ index: V4Index+ writeDay: string+ row: V4IndexRow+}++function isIndex(value: string | undefined): value is V4Index {+ return value === 'blocks' || value === 'images'+}++function isWriteDay(value: string | undefined): value is string {+ if (!value || !/^\d{4}-\d{2}-\d{2}$/.test(value)) {+ return false+ }+ const day = new Date(`${value}T00:00:00Z`)+ return !Number.isNaN(day.getTime()) && day.toISOString().slice(0, 10) === value+}++function reject(reason: V4RowRejection, message: Message, invalidColumn?: string): null {+ // Skip a bad row rather than wedge the partition, as the v3 sink does.+ logger.warn('🪶', 'ai_research_v4_parquet_row_rejected', {+ reason,+ partition: message.partition,+ offset: message.offset,+ ...(invalidColumn && { column: invalidColumn }),+ })+ V4ParquetSinkMetrics.incRowsRejected(reason)+ return null+}++export function parseIndexMessage(message: Message): V4IndexMessage | null {+ const headers = parseKafkaHeaders(message.headers)+ const index = headers[INDEX_HEADER]+ if (!isIndex(index)) {+ return reject('unknown_index', message)+ }+ const writeDay = headers[WRITE_DAY_HEADER]+ if (!isWriteDay(writeDay)) {+ return reject('invalid_write_day', message)+ }+ let value: unknown+ try {+ value = parseJSON(message.value?.toString('utf8') ?? '')+ } catch {+ return reject('parse_failed', message)+ }+ const selected = selectIndexRow(index, value)+ if (!('row' in selected)) {+ V4ParquetSinkMetrics.incInvalidColumn(index, selected.invalidColumn)+ return reject('invalid_row', message, selected.invalidColumn)+ }+ V4ParquetSinkMetrics.incRowsParsed(index)+ return { index, writeDay, row: selected.row }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.tsnew file mode 100644index 000000000000..d5b28def1b66--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store.ts@@ -0,0 +1,49 @@+/** Writes one v4 index file to `index/{index}/dt={day}/` in the rrweb bucket, named as the recorder names its files. */+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'+import { randomBytes } from 'crypto'++import { V4Index, V4IndexRow } from './index-columns'+import { indexRowsToParquet } from './parquet-writer'++// A file can hold up to the sink's byte target of rows, so the upload gets more time than v3's 30 seconds.+const PUT_TIMEOUT_MS = 120_000++export function indexFileKey(index: V4Index, writeDay: string, nowMs: number): string {+ return `index/${index}/dt=${writeDay}/${nowMs}-${randomBytes(8).toString('hex')}.parquet`+}++/** A 412 means the SDK retried a PUT whose first try stored the object, so the object is there. */+function isPreconditionFailed(error: unknown): boolean {+ if (typeof error !== 'object' || error === null) {+ return false+ }+ const { $metadata, name } = error as { $metadata?: { httpStatusCode?: unknown }; name?: unknown }+ return $metadata?.httpStatusCode === 412 || name === 'PreconditionFailed'+}++export class IndexParquetStore {+ constructor(+ private readonly s3Client: S3Client,+ private readonly bucket: string,+ private readonly rowGroupSize: number+ ) {}++ public encode(index: V4Index, rows: readonly V4IndexRow[]): Promise<Buffer> {+ return indexRowsToParquet(index, rows, this.rowGroupSize)+ }++ public async put(index: V4Index, writeDay: string, body: Buffer, nowMs: number): Promise<string> {+ const key = indexFileKey(index, writeDay, nowMs)+ try {+ await this.s3Client.send(+ new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: body, IfNoneMatch: '*' }),+ { abortSignal: AbortSignal.timeout(PUT_TIMEOUT_MS) }+ )+ } catch (error) {+ if (!isPreconditionFailed(error)) {+ throw error+ }+ }+ return key+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.tsnew file mode 100644index 000000000000..4e3d504a2653--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/metrics.ts@@ -0,0 +1,75 @@+import { Counter } from 'prom-client'++import { V4Index } from './index-columns'++export type V4RowRejection = 'parse_failed' | 'invalid_row' | 'unknown_index' | 'invalid_write_day'++export class V4ParquetSinkMetrics {+ private static readonly rowsParsed = new Counter({+ name: 'ai_research_v4_parquet_sink_rows_parsed_total',+ help: 'v4 index rows read from Kafka and accepted into the Parquet buffer',+ labelNames: ['index'],+ })+ private static readonly rowsRejected = new Counter({+ name: 'ai_research_v4_parquet_sink_rows_rejected_total',+ help: 'v4 index messages skipped before buffering, by reason',+ labelNames: ['reason'],+ })+ private static readonly invalidColumns = new Counter({+ name: 'ai_research_v4_parquet_sink_invalid_columns_total',+ help: 'v4 index rows rejected as invalid_row, by index and the first column that was missing or did not fit its type',+ labelNames: ['index', 'column'],+ })+ private static readonly rowsReleased = new Counter({+ name: 'ai_research_v4_parquet_sink_rows_released_total',+ help: 'Buffered v4 index rows left out of the next file because their partition was revoked; the new owner reads them again',+ })+ private static readonly filesWritten = new Counter({+ name: 'ai_research_v4_parquet_sink_files_written_total',+ help: 'v4 index Parquet files written',+ labelNames: ['index'],+ })+ private static readonly rowsWritten = new Counter({+ name: 'ai_research_v4_parquet_sink_rows_written_total',+ help: 'v4 index rows written as Parquet, including rows written again after a replay',+ labelNames: ['index'],+ })+ private static readonly bytesWritten = new Counter({+ name: 'ai_research_v4_parquet_sink_bytes_written_total',+ help: 'v4 index Parquet bytes written',+ labelNames: ['index'],+ })+ private static readonly writeErrors = new Counter({+ name: 'ai_research_v4_parquet_sink_write_errors_total',+ help: 'v4 index file writes that threw; the rows stay buffered and their offsets stay uncommitted',+ })+ private static readonly flushes = new Counter({+ name: 'ai_research_v4_parquet_sink_flushes_total',+ help: 'v4 sink flushes by outcome: wrote files, or committed offsets of messages that gave no rows',+ labelNames: ['outcome'],+ })++ public static incRowsParsed(index: V4Index): void {+ this.rowsParsed.labels(index).inc()+ }+ public static incRowsRejected(reason: V4RowRejection): void {+ this.rowsRejected.labels(reason).inc()+ }+ public static incInvalidColumn(index: V4Index, column: string): void {+ this.invalidColumns.labels(index, column).inc()+ }+ public static incRowsReleased(count: number): void {+ this.rowsReleased.inc(count)+ }+ public static observeFile(index: V4Index, rows: number, bytes: number): void {+ this.filesWritten.labels(index).inc()+ this.rowsWritten.labels(index).inc(rows)+ this.bytesWritten.labels(index).inc(bytes)+ }+ public static incWriteError(): void {+ this.writeErrors.inc()+ }+ public static incFlush(outcome: 'written' | 'empty'): void {+ this.flushes.labels(outcome).inc()+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.tsnew file mode 100644index 000000000000..9c7876f4d07b--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-buffer.ts@@ -0,0 +1,31 @@+/** Encode already-mapped records into a Snappy/Parquet buffer via an in-memory sink — parquetjs only+ * writes to a stream, so this is the openStream/collect plumbing. Schema + record shape are the caller's. */+import { ParquetSchema, ParquetWriter } from '@dsnp/parquetjs'+import { Writable } from 'stream'++export async function parquetRecordsToBuffer(+ schema: ParquetSchema,+ records: Record<string, unknown>[],+ options: { rowGroupSize?: number } = {}+): Promise<Buffer> {+ const chunks: Buffer[] = []+ const sink = new Writable({+ write(chunk: Buffer, _encoding, callback) {+ chunks.push(chunk)+ callback()+ },+ })+ // parquetjs types want an fs.WriteStream, but it only calls write/end/on at runtime, which Writable provides.+ const writer = await ParquetWriter.openStream(+ schema,+ sink as unknown as Parameters<typeof ParquetWriter.openStream>[1]+ )+ if (options.rowGroupSize !== undefined) {+ writer.setRowGroupSize(options.rowGroupSize)+ }+ for (const record of records) {+ await writer.appendRow(record)+ }+ await writer.close()+ return Buffer.concat(chunks)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.tsnew file mode 100644index 000000000000..5226d36f6307--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts@@ -0,0 +1,190 @@+import { ParquetReader } from '@dsnp/parquetjs'+import { CompressionCodec } from '@dsnp/parquetjs/dist/gen-nodejs/parquet_types'+import { randomBytes } from 'node:crypto'++import { BLOCK_INDEX_COLUMNS, V4Index } from './index-columns'+import { indexRowsToParquet } from './parquet-writer'++async function describeFile(+ file: Buffer+): Promise<{ fields: string[]; rowGroups: number; codecs: unknown[]; rows: unknown[] }> {+ const reader = await ParquetReader.openBuffer(file)+ const fields = reader.schema.fieldList.map((field) =>+ [field.repetitionType, field.primitiveType ?? 'group', field.path.join('.'), field.originalType]+ .filter(Boolean)+ .join(' ')+ )+ const rowGroups = reader.metadata!.row_groups.length+ const codecs = reader.metadata!.row_groups.flatMap((group) =>+ group.columns.map((column) => column.meta_data?.codec)+ )+ const rows: unknown[] = []+ const cursor = reader.getCursor()+ for (let row = await cursor.next(); row; row = await cursor.next()) {+ rows.push(row)+ }+ await reader.close()+ return { fields, rowGroups, codecs, rows }+}++describe('indexRowsToParquet', () => {+ // The v4 index contract, shared with the recorder's JSON rows: name, order, nullability and type of each column.+ const SCHEMAS: Record<V4Index, string[]> = {+ images: [+ 'REQUIRED INT64 team_id',+ 'REQUIRED BYTE_ARRAY key_month UTF8',+ 'REQUIRED BYTE_ARRAY image_ref UTF8',+ 'REQUIRED BYTE_ARRAY pack_s3_key UTF8',+ 'REQUIRED INT64 byte_start',+ 'REQUIRED INT64 byte_end',+ 'REQUIRED BYTE_ARRAY mime UTF8',+ 'REQUIRED INT64 byte_size',+ ],+ blocks: [+ 'REQUIRED INT64 team_id',+ 'REQUIRED BYTE_ARRAY session_id UTF8',+ 'REQUIRED BYTE_ARRAY key_month UTF8',+ 'REQUIRED BYTE_ARRAY block_s3_key UTF8',+ 'REQUIRED BYTE_ARRAY topic UTF8',+ 'REQUIRED INT32 partition',+ 'REQUIRED INT64 start_offset',+ 'REQUIRED INT64 end_offset',+ 'REQUIRED INT64 flush_end_offset',+ 'REQUIRED INT32 flush_row_count',+ 'REQUIRED INT32 flush_row_index',+ 'REQUIRED BYTE_ARRAY flush_id UTF8',+ 'REQUIRED INT64 flushed_at_ms',+ 'REQUIRED INT64 block_byte_start',+ 'REQUIRED INT64 block_byte_end',+ 'OPTIONAL INT64 first_ts_ms',+ 'OPTIONAL INT64 last_ts_ms',+ 'REQUIRED INT64 event_count',+ 'REQUIRED INT64 byte_size',+ 'OPTIONAL BYTE_ARRAY snapshot_source UTF8',+ 'OPTIONAL BYTE_ARRAY snapshot_library UTF8',+ 'OPTIONAL BYTE_ARRAY host UTF8',+ 'REQUIRED INT32 full_snapshots',+ 'REQUIRED INT32 events_before_first_full_snapshot',+ 'REQUIRED INT32 orphan_mutations',+ 'REQUIRED INT32 decode_failures',+ 'REQUIRED INT32 dom_nodes',+ 'REQUIRED INT32 remote_stylesheets',+ 'REQUIRED INT32 inline_stylesheets',+ 'REQUIRED INT32 iframes',+ 'REQUIRED INT32 canvases',+ 'REQUIRED INT32 videos',+ 'REQUIRED INT32 shadow_roots',+ 'REQUIRED INT64 max_gap_ms',+ 'REQUIRED INT64 active_ms',+ 'REQUIRED INT64 dom_text_chars',+ 'REQUIRED BOOLEAN cold_start',+ 'REQUIRED BOOLEAN node_cap_reached',+ 'REQUIRED BOOLEAN moments_truncated',+ 'REQUIRED BOOLEAN touch_input',+ 'REQUIRED INT32 clicks',+ 'REQUIRED INT32 mouse_interactions',+ 'REQUIRED INT32 rage_clicks',+ 'REQUIRED INT32 dead_clicks',+ 'REQUIRED INT32 keypresses',+ 'REQUIRED INT32 scrolls',+ 'REQUIRED INT32 mouse_moves',+ 'REQUIRED INT32 touches',+ 'REQUIRED INT32 touch_moves',+ 'REQUIRED INT32 selections',+ 'REQUIRED INT32 media_events',+ 'REQUIRED INT32 viewport_resizes',+ 'REQUIRED INT32 navigations',+ 'REQUIRED INT32 console_errors',+ 'REQUIRED INT32 console_warnings',+ 'REQUIRED INT32 network_requests',+ 'REQUIRED INT32 network_failures',+ 'REQUIRED INT32 images_extracted',+ 'REQUIRED INT32 images_inline',+ 'REQUIRED INT32 image_urls',+ 'REQUIRED INT64 image_bytes',+ 'REQUIRED INT64 written_jsonl_bytes',+ 'REQUIRED INT64 max_written_event_bytes',+ 'OPTIONAL INT32 viewport_width',+ 'OPTIONAL INT32 viewport_height',+ 'OPTIONAL INT64 viewport_ts_ms',+ 'REPEATED group moments',+ 'REQUIRED BYTE_ARRAY moments.kind UTF8',+ 'REQUIRED BYTE_ARRAY moments.window_id UTF8',+ 'REQUIRED INT64 moments.ts_ms',+ 'REQUIRED INT32 moments.event_index',+ 'REPEATED BYTE_ARRAY url_fingerprints UTF8',+ 'REPEATED BYTE_ARRAY window_ids UTF8',+ 'OPTIONAL BYTE_ARRAY urls UTF8',+ 'REQUIRED BOOLEAN urls_truncated',+ ],+ }++ it.each(['blocks', 'images'] as const)('writes the %s columns of the v4 index contract', async (index) => {+ const { fields } = await describeFile(await indexRowsToParquet(index, [], 1_000))+ expect(fields).toEqual(SCHEMAS[index])+ })++ it('reads back the values it writes, in row groups of the given size, with snappy on every column', async () => {+ const image = {+ team_id: 7,+ key_month: '2026-10',+ image_ref: 'image:v4:7:2026-10:abc',+ pack_s3_key: 'images/packs/dt=2026-10-15/7/1-0',+ byte_start: 0,+ byte_end: 99,+ mime: 'image/png',+ byte_size: 60,+ }+ const { rowGroups, codecs, rows } = await describeFile(+ await indexRowsToParquet('images', Array(5).fill(image), 2)+ )+ expect(rowGroups).toBe(3)+ expect(codecs).toEqual(Array(3 * 8).fill(CompressionCodec.SNAPPY))+ expect(rows).toEqual(Array(5).fill({ ...image, team_id: 7n, byte_start: 0n, byte_end: 99n, byte_size: 60n }))+ })++ const block = (moments: unknown[], windowIds: string[]): Record<string, unknown> =>+ Object.fromEntries(+ BLOCK_INDEX_COLUMNS.map((column) => [+ column.name,+ 'repeated' in column+ ? moments+ : 'list' in column+ ? windowIds+ : column.optional+ ? null+ : column.type === 'UTF8'+ ? 'x'+ : column.type === 'BOOLEAN'+ ? true+ : 1,+ ])+ )++ it('writes a large sealed urls value with no copy of it in the statistics', async () => {+ const urls = randomBytes(768 * 1024).toString('base64')++ const file = await indexRowsToParquet('blocks', [{ ...block([], []), urls, urls_truncated: false }], 1_000)++ const { rows } = await describeFile(file)+ expect((rows as Record<string, unknown>[])[0]).toMatchObject({ urls, urls_truncated: false })+ // Snappy cannot shrink random base64, so the file is about one copy of the value.+ expect(file.length).toBeLessThan(urls.length * 1.1)+ })++ it('reads moments back as a list of structs and the string lists as lists', async () => {+ const moments = [+ { kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 - 1 },+ { kind: 'full_snapshot_undecoded', window_id: 'w2', ts_ms: 2_000, event_index: 0 },+ ]+ const { rows } = await describeFile(+ await indexRowsToParquet('blocks', [block(moments, ['w1', 'w2']), block([], [])], 1_000)+ )+ const read = rows as Record<string, unknown>[]+ expect(read.map((row) => [row.moments, row.url_fingerprints, row.window_ids])).toEqual([+ [moments.map((moment) => ({ ...moment, ts_ms: BigInt(moment.ts_ms) })), ['w1', 'w2'], ['w1', 'w2']],+ // parquetjs reads an empty repeated field as null; Arrow readers see an empty list.+ [null, null, null],+ ])+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.tsnew file mode 100644index 000000000000..3d23e79394c8--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts@@ -0,0 +1,62 @@+/** Encodes v4 index rows into a Snappy-compressed Parquet buffer, with the types of the COLUMNS table. */+import { ParquetSchema } from '@dsnp/parquetjs'++import { INDEX_COLUMNS, ParquetType, V4Column, V4Index, V4IndexRow } from './index-columns'+import { parquetRecordsToBuffer } from './parquet-buffer'++type SchemaDef = ConstructorParameters<typeof ParquetSchema>[0]++// Snappy on every column: compression is a per-field option in parquetjs, not a writer option.+function schemaDef(columns: readonly V4Column[]): SchemaDef {+ const def: Record<string, unknown> = {}+ for (const column of columns) {+ if ('repeated' in column) {+ def[column.name] = { repeated: true, fields: schemaDef(column.fields) }+ } else if ('list' in column) {+ def[column.name] = { type: column.list, repeated: true, compression: 'SNAPPY' }+ } else {+ def[column.name] = {+ type: column.type,+ compression: 'SNAPPY',+ ...(column.optional && { optional: true }),+ ...(column.opaque && { statistics: false }),+ }+ }+ }+ return def as SchemaDef+}++const SCHEMAS: Record<V4Index, ParquetSchema> = {+ blocks: new ParquetSchema(schemaDef(INDEX_COLUMNS.blocks)),+ images: new ParquetSchema(schemaDef(INDEX_COLUMNS.images)),+}++function toValue(type: ParquetType, value: unknown): unknown {+ return type === 'INT64' ? BigInt(value as number) : value+}++function toRecord(row: V4IndexRow, columns: readonly V4Column[]): Record<string, unknown> {+ const record: Record<string, unknown> = {}+ for (const column of columns) {+ const value = row[column.name]+ if ('repeated' in column) {+ record[column.name] = (value as V4IndexRow[]).map((item) => toRecord(item, column.fields))+ } else if ('list' in column) {+ record[column.name] = (value as unknown[]).map((item) => toValue(column.list, item))+ } else if (value === null || value === undefined) {+ record[column.name] = null+ } else {+ record[column.name] = toValue(column.type, value)+ }+ }+ return record+}++export function indexRowsToParquet(index: V4Index, rows: readonly V4IndexRow[], rowGroupSize: number): Promise<Buffer> {+ const columns = INDEX_COLUMNS[index]+ return parquetRecordsToBuffer(+ SCHEMAS[index],+ rows.map((row) => toRecord(row, columns)),+ { rowGroupSize }+ )+}diff --git a/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.ts b/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.tsnew file mode 100644index 000000000000..d4561e44a830--- /dev/null+++ b/nodejs/src/servers/ai-research-v4-parquet-sink-server.test.ts@@ -0,0 +1,22 @@+import { KAFKA_AI_RESEARCH_V4_INDEX } from '~/common/config/kafka-topics'+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'++import { buildV4SinkConsumerConfig } from './ai-research-v4-parquet-sink-server'++describe('v4 Parquet sink wiring', () => {+ it('reads the v4 index topic in its own group, storing offsets only after a flush', () => {+ const config = buildV4SinkConsumerConfig(buildAiResearchV4ServerConfig({}))+ expect(config).toMatchObject({+ topic: KAFKA_AI_RESEARCH_V4_INDEX,+ groupId: 'ai-research-v4-parquet-sink',+ autoCommit: true,+ autoOffsetStore: false,+ callEachBatchWhenEmpty: true,+ })+ })++ it.each(['', 'session-replay-ml-parquet-sink'])('refuses the group %p', (group) => {+ const config = buildAiResearchV4ServerConfig({ AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID: group })+ expect(() => buildV4SinkConsumerConfig(config)).toThrow('AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID')+ })+})diff --git a/nodejs/src/servers/ai-research-v4-parquet-sink-server.ts b/nodejs/src/servers/ai-research-v4-parquet-sink-server.tsnew file mode 100644index 000000000000..808077e4bd6b--- /dev/null+++ b/nodejs/src/servers/ai-research-v4-parquet-sink-server.ts@@ -0,0 +1,112 @@+import { Message } from 'node-rdkafka'++import { KAFKA_AI_RESEARCH_V4_INDEX } from '~/common/config/kafka-topics'+import { KafkaConsumer, KafkaConsumerConfig, parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'+import { logger } from '~/common/utils/logger'+import { AiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'+import { IndexBatcher } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher'+import { INDEX_HEADER } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message'+import { IndexParquetStore } from '~/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-parquet-store'+import { AiResearchV4ConsumerServer } from '~/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server'+import {+ CaptureWatermark,+ CapturedData,+ capturedRecords,+ releasingOffsetStore,+} from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'+import { buildSessionRecordingS3Client } from '~/ingestion/pipelines/sessionreplay/shared/s3-client'++import { CleanupResources } from './base-server'++/** The v3 sink's default group. A v4 sink pod in it would take v3 partitions and read none of their topic. */+const V3_PARQUET_SINK_GROUP_ID = 'session-replay-ml-parquet-sink'++function assertOwnGroup(config: AiResearchV4ServerConfig): void {+ const group = config.AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID+ if (!group || group === V3_PARQUET_SINK_GROUP_ID) {+ throw new Error('AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID must be set and differ from the v3 sink group')+ }+}++/**+ * Manual offsets and callEachBatchWhenEmpty, as in the v3 sink: the batcher stores offsets only after their files are+ * written, and idle polls still reach the flush interval.+ */+export function buildV4SinkConsumerConfig(config: AiResearchV4ServerConfig): KafkaConsumerConfig {+ assertOwnGroup(config)+ return {+ topic: KAFKA_AI_RESEARCH_V4_INDEX,+ groupId: config.AI_RESEARCH_V4_PARQUET_SINK_GROUP_ID,+ autoCommit: true,+ autoOffsetStore: false,+ callEachBatchWhenEmpty: true,+ }+}++/** Block rows feed the event metadata watermark and image rows the inline image one, as in v3. */+function indexData(message: Message): CapturedData | undefined {+ const index = parseKafkaHeaders(message.headers)[INDEX_HEADER]+ return index === 'blocks' ? 'event_metadata' : index === 'images' ? 'inline_images' : undefined+}++export class AiResearchV4ParquetSinkServer extends AiResearchV4ConsumerServer {+ protected async startServices(): Promise<void> {+ const consumerConfig = buildV4SinkConsumerConfig(this.config)+ const bucket = this.config.AI_RESEARCH_V4_RRWEB_BUCKET+ if (!bucket) {+ throw new Error('AI_RESEARCH_V4_RRWEB_BUCKET must be set for the v4 Parquet sink')+ }+ const rowGroupRows = this.config.AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS+ if (!Number.isInteger(rowGroupRows) || rowGroupRows <= 0) {+ throw new Error('AI_RESEARCH_V4_PARQUET_ROW_GROUP_ROWS must be a positive integer')+ }+ const s3Client = buildSessionRecordingS3Client(this.config)+ if (!s3Client) {+ throw new Error('SESSION_RECORDING_V2_S3_* must be configured for the v4 Parquet sink')+ }++ const consumer = new KafkaConsumer(consumerConfig)+ const watermark = new CaptureWatermark('ai_research_v4_parquet_sink')+ const batcher = new IndexBatcher(+ new IndexParquetStore(s3Client, bucket, rowGroupRows),+ releasingOffsetStore(consumer, watermark),+ {+ flushIntervalMs: this.config.AI_RESEARCH_V4_PARQUET_FLUSH_INTERVAL_MS,+ maxRows: this.config.AI_RESEARCH_V4_PARQUET_MAX_ROWS,+ maxBytes: this.config.AI_RESEARCH_V4_PARQUET_MAX_BYTES,+ heartbeat: () => consumer.reportDeliberateWait(),+ },+ Date.now()+ )+ await consumer.connect(+ (messages) => {+ consumer.heartbeat()+ watermark.hold(capturedRecords(messages, indexData))+ return batcher.handleBatch(messages, Date.now())+ },+ (partitions) => {+ batcher.release(partitions)+ watermark.forget(partitions)+ return Promise.resolve()+ }+ )++ this.lifecycle.services.push({+ id: 'ai-research-v4-parquet-sink',+ // Flush the buffer so a graceful restart does not read it again. A failed final flush replays on restart.+ onShutdown: async () => {+ try {+ await batcher.flush(Date.now())+ } catch (error) {+ logger.warn('🪶', 'ai_research_v4_parquet_sink_shutdown_flush_failed', { error: String(error) })+ }+ await consumer.disconnect()+ },+ healthcheck: () => consumer.isHealthy(),+ })+ }++ protected getCleanupResources(): CleanupResources {+ return { kafkaProducers: [], redisPools: [] }+ }+}