:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ai-research): write v4 block index moments as a parquet list (#125330)
Robbied1bfdc390bd6
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.tsindex f7a69e5b3932..a988a0e37395 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts@@ -17,7 +17,7 @@ export interface V4LeafColumn {opaque?: boolean}-/** A repeated group of required leaves; Parquet readers see it as a non-null list of structs. */+/** A non-null list of structs of required leaves. */export interface V4RepeatedColumn {name: stringrepeated: truediff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.tsindex 8bd528981466..17c2535e1337 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts@@ -1,16 +1,27 @@import { ParquetReader } from '@dsnp/parquetjs'-import { CompressionCodec } from '@dsnp/parquetjs/dist/gen-nodejs/parquet_types'+import { CompressionCodec, ConvertedType } from '@dsnp/parquetjs/dist/gen-nodejs/parquet_types'import { randomBytes } from 'node:crypto'import { BLOCK_INDEX_COLUMNS, V4Index } from './index-columns'import { indexRowsToParquet } from './parquet-writer'+const CONVERTED_TYPE_NAMES: Record<number, string> = Object.fromEntries(+ Object.entries(ConvertedType).map(([name, value]) => [value, name])+)+async function describeFile(file: Buffer): Promise<{ fields: string[]; rowGroups: number; codecs: unknown[]; rows: unknown[] }> {const reader = await ParquetReader.openBuffer(file)- const fields = reader.schema.fieldList.map((field) =>- [field.repetitionType, field.primitiveType ?? 'group', field.path.join('.'), field.originalType]+ // parquetjs's reader drops a group's LIST annotation, so annotations come from the footer, which lists fields in fieldList order.+ const footer = reader.metadata!.schema.slice(1)+ const fields = reader.schema.fieldList.map((field, index) =>+ [+ field.repetitionType,+ field.primitiveType ?? 'group',+ field.path.join('.'),+ CONVERTED_TYPE_NAMES[footer[index].converted_type!],+ ].filter(Boolean).join(' '))@@ -107,11 +118,13 @@ describe('indexRowsToParquet', () => {'OPTIONAL INT32 viewport_width','OPTIONAL INT32 viewport_height','OPTIONAL INT64 viewport_ts_ms',- 'REPEATED group moments',- 'REQUIRED BYTE_ARRAY moments.kind UTF8',- 'REQUIRED BYTE_ARRAY moments.window_id UTF8',- 'REQUIRED INT64 moments.ts_ms',- 'REQUIRED INT32 moments.event_index',+ 'REQUIRED group moments LIST',+ 'REPEATED group moments.list',+ 'REQUIRED group moments.list.element',+ 'REQUIRED BYTE_ARRAY moments.list.element.kind UTF8',+ 'REQUIRED BYTE_ARRAY moments.list.element.window_id UTF8',+ 'REQUIRED INT64 moments.list.element.ts_ms',+ 'REQUIRED INT32 moments.list.element.event_index','REPEATED BYTE_ARRAY url_fingerprints UTF8','REPEATED BYTE_ARRAY window_ids UTF8','OPTIONAL BYTE_ARRAY urls UTF8',@@ -194,7 +207,7 @@ describe('indexRowsToParquet', () => {])})- it('reads moments back as a list of structs and the string lists as lists', async () => {+ it('reads moments back as a 3-level list of structs and the string lists as lists', async () => {const moments = [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 - 1 },{ kind: 'full_snapshot_undecoded', window_id: 'w2', ts_ms: 2_000, event_index: 0 },@@ -204,9 +217,13 @@ describe('indexRowsToParquet', () => {)const read = rows as Record<string, unknown>[]expect(read.map((row) => [row.moments, row.url_fingerprints, row.window_ids])).toEqual([- [moments.map((moment) => ({ ...moment, ts_ms: BigInt(moment.ts_ms) })), ['w1', 'w2'], ['w1', 'w2']],- // parquetjs reads an empty repeated field as null; Arrow readers see an empty list.- [null, null, null],+ [+ { list: moments.map((moment) => ({ element: { ...moment, ts_ms: BigInt(moment.ts_ms) } })) },+ ['w1', 'w2'],+ ['w1', 'w2'],+ ],+ // parquetjs reads an empty repeated field as null and leaves a LIST wrapped; Arrow readers see empty lists.+ [{ list: null }, null, null],])})})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.tsindex c4960c53a18c..b20557ff1542 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts@@ -20,11 +20,15 @@ PARQUET_COMPRESSION_METHODS.ZSTD = {inflate: (value: Buffer) => zstdDecompressAsync(value),}+// A list of structs gets the 3-level LIST layout because Athena reads a bare repeated group of several fields as null.function schemaDef(columns: readonly V4Column[]): SchemaDef {const def: Record<string, unknown> = {}for (const column of columns) {if ('repeated' in column) {- def[column.name] = { repeated: true, fields: schemaDef(column.fields) }+ def[column.name] = {+ type: 'LIST',+ fields: { list: { repeated: true, fields: { element: { fields: schemaDef(column.fields) } } } },+ }} else if ('list' in column) {def[column.name] = { type: column.list, repeated: true, compression: 'ZSTD' }} else {@@ -71,7 +75,9 @@ function toRecord(row: V4IndexRow, columns: readonly V4Column[]): Record<string,for (const column of columns) {const value = row[column.name]if ('repeated' in column) {- record[column.name] = (value as V4IndexRow[]).map((item) => toRecord(item, column.fields))+ record[column.name] = {+ list: (value as V4IndexRow[]).map((item) => ({ element: toRecord(item, column.fields) })),+ }} else if ('list' in column) {record[column.name] = (value as unknown[]).map((item) => toValue(column.list, item))} else if (value === null || value === undefined) {diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.mdindex 8bd015781d0f..7ee00cbf8b27 100644--- a/products/ai_training/docs/replay-data.md+++ b/products/ai_training/docs/replay-data.md@@ -496,7 +496,7 @@ The recorder does not write the two index prefixes itself, as in v3:- After a flush stores its packs and blocks, the recorder sends one JSON record per index row to `ai_research_v4_index`. The record's keys are the Parquet column names.- The recorder commits without waiting for delivery. A row that is not delivered leaves its block or image stored but not in the index.- The v4 Parquet sink reads the topic, groups the rows by index and write day, and writes one file per group and flush as `{ms}-{random}.parquet`. A flush runs every 10 minutes, or at 128 MiB of rows or 1,000,000 rows. The sink commits its offsets only after the files are stored.-- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a repeated group whose item is named `moments`; an Arrow reader sees a list of structs. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.+- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a 3-level `LIST` (`moments`, then the repeated group `list`, then the struct `element`), so Athena, Arrow readers and DuckDB see a non-null list of structs. Files written before the sink used that layout hold `moments` as a bare repeated group whose item is named `moments`. Arrow readers and DuckDB read those as the same list of structs, but Athena reads the column as null, because Trino reads a bare repeated group of more than one field as a missing column. Those files expire with the bucket's lifecycle. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.- The sink can write a row twice after a restart. The copies have the same `flush_id`, so the rule below does not remove them: readers first drop rows that repeat within one `flush_id`.### Block index