:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): write the v4 index as parquet (#117972)
Nicholas Waltz291eecdc64e0
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 14198df61440..57ce90bb9004 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11052,6 +11052,8 @@ name = "replay-recorder-v4"version = "0.1.0"dependencies = ["arc-swap",+ "arrow-array",+ "arrow-schema","async-trait","aws-config","aws-sdk-dynamodb",@@ -11059,6 +11061,7 @@ dependencies = ["aws-sdk-s3","axum 0.7.9","brotlic",+ "bytes","chrono","common-alloc","common-database",@@ -11067,6 +11070,7 @@ dependencies = ["envconfig","lifecycle","metrics",+ "parquet","rand 0.8.5","rdkafka","replay-core-v4",diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex 56d4510be171..a59f22d31e90 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -17,6 +17,9 @@ lifecycle = { path = "../../common/lifecycle" }replay-core-v4 = { path = "../replay-core-v4" }arc-swap = { workspace = true }+# The major version deltalite links, so the workspace builds one arrow.+arrow-array = "58"+arrow-schema = "58"async-trait = { workspace = true }aws-config = { workspace = true }aws-sdk-dynamodb = { workspace = true }@@ -27,6 +30,7 @@ brotlic = "0.8"chrono = { workspace = true }envconfig = { workspace = true }metrics = { workspace = true }+parquet = "58"rand = { workspace = true }rdkafka = { workspace = true }sqlx = { workspace = true }@@ -36,5 +40,6 @@ tracing = { workspace = true }tracing-subscriber = { workspace = true }[dev-dependencies]+bytes = { workspace = true }serde_json = { workspace = true }tokio = { workspace = true, features = ["test-util"] }diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rsnew file mode 100644index 000000000000..b15228543aff--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/index.rs@@ -0,0 +1,147 @@+//! The v4 index files: Parquet in `rrweb-v4/index/`. Column names are the ones in `replay-data.md`.+//!+//! Integer columns are signed, the types parquetjs writes, so a parquetjs writer can produce the same files.++use std::sync::Arc;++use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray};+use arrow_schema::{ArrowError, DataType, Field, Schema};+use parquet::arrow::ArrowWriter;+use parquet::basic::Compression;+use parquet::errors::ParquetError;+use parquet::file::properties::WriterProperties;++#[derive(Debug, thiserror::Error)]+pub enum IndexError {+ #[error(transparent)]+ Arrow(#[from] ArrowError),+ #[error(transparent)]+ Parquet(#[from] ParquetError),+}++/// Writes one Parquet file, snappy-compressed as the v3 catalog is.+pub fn write_parquet(batch: &RecordBatch) -> Result<Vec<u8>, IndexError> {+ let properties = WriterProperties::builder()+ .set_compression(Compression::SNAPPY)+ .build();+ let mut out = Vec::new();+ let mut writer = ArrowWriter::try_new(&mut out, batch.schema(), Some(properties))?;+ writer.write(batch)?;+ writer.close()?;+ Ok(out)+}++/// A count or size as its signed column. Real values fit, and a larger one saturates.+fn int64(value: u64) -> i64 {+ i64::try_from(value).unwrap_or(i64::MAX)+}++/// Where one inline image sits in a pack in `images-v4`.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ImageIndexRow {+ pub team_id: i64,+ pub key_month: String,+ pub image_ref: String,+ pub pack_s3_key: String,+ /// Inclusive byte range of the sealed image in the pack.+ pub byte_start: u64,+ pub byte_end: u64,+ pub mime: String,+ pub byte_size: u64,+}++pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {+ let schema = Schema::new(vec![+ Field::new("team_id", DataType::Int64, false),+ Field::new("key_month", DataType::Utf8, false),+ Field::new("image_ref", DataType::Utf8, false),+ Field::new("pack_s3_key", DataType::Utf8, false),+ Field::new("byte_start", DataType::Int64, false),+ Field::new("byte_end", DataType::Int64, false),+ Field::new("mime", DataType::Utf8, false),+ Field::new("byte_size", DataType::Int64, false),+ ]);+ let columns: Vec<ArrayRef> = vec![+ Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.team_id))),+ Arc::new(StringArray::from_iter_values(+ rows.iter().map(|r| &r.key_month),+ )),+ Arc::new(StringArray::from_iter_values(+ rows.iter().map(|r| &r.image_ref),+ )),+ Arc::new(StringArray::from_iter_values(+ rows.iter().map(|r| &r.pack_s3_key),+ )),+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| int64(r.byte_start)),+ )),+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| int64(r.byte_end)),+ )),+ Arc::new(StringArray::from_iter_values(rows.iter().map(|r| &r.mime))),+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| int64(r.byte_size)),+ )),+ ];+ Ok(RecordBatch::try_new(Arc::new(schema), columns)?)+}++#[cfg(test)]+mod tests {+ use bytes::Bytes;+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;++ use super::*;++ #[test]+ fn image_index_reads_back_with_the_documented_columns() {+ let rows = vec![ImageIndexRow {+ team_id: 7,+ key_month: "2026-10".to_owned(),+ image_ref: "image:v4:7:2026-10:abc".to_owned(),+ pack_s3_key: "images/packs/dt=2026-10-15/7/1-0".to_owned(),+ byte_start: 0,+ byte_end: 99,+ mime: "image/png".to_owned(),+ byte_size: u64::MAX,+ }];+ let file = write_parquet(&image_index(&rows).unwrap()).unwrap();+ let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(file))+ .unwrap()+ .build()+ .unwrap();+ let batch = reader.next().unwrap().unwrap();+ let columns: Vec<_> = batch+ .schema()+ .fields()+ .iter()+ .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))+ .collect();+ let column = |name: &str, data_type| (name.to_owned(), data_type, false);+ assert_eq!(+ columns,+ [+ column("team_id", DataType::Int64),+ column("key_month", DataType::Utf8),+ column("image_ref", DataType::Utf8),+ column("pack_s3_key", DataType::Utf8),+ column("byte_start", DataType::Int64),+ column("byte_end", DataType::Int64),+ column("mime", DataType::Utf8),+ column("byte_size", DataType::Int64),+ ]+ );+ let sizes = batch+ .column(7)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(sizes.value(0), i64::MAX);+ let refs = batch+ .column(2)+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!(refs.value(0), "image:v4:7:2026-10:abc");+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex 72357f281585..e03ec9ea5f19 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -4,6 +4,7 @@pub mod batch;pub mod config;pub mod consumer;+pub mod index;pub mod process;pub mod store;pub mod teams;