:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): consume replay messages behind the v4 gate (#117967)
Nicholas Waltz0bd46172342b
diff --git a/.github/rust-images.yml b/.github/rust-images.ymlindex d8a7fc379c1c..5bc5f67a4f44 100644--- a/.github/rust-images.yml+++ b/.github/rust-images.yml@@ -154,6 +154,12 @@dockerfile: ./rust/Dockerfileproject: lvsrh6c64c+# Records the v4 AI research replay dataset (products/ai_training/docs/replay-data.md). CI builds the image, but it+# has no entry in the deploy matrix of rust-docker-build.yml until the recorder ships.+- image: replay-recorder-v4+ dockerfile: ./rust/Dockerfile+ project: vznmbshh6q+- image: personhog-test-harnessdockerfile: ./rust/Dockerfileproject: 6tl5lmb8l8diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex b4698674352b..439e940dfc85 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11047,6 +11047,25 @@ dependencies = ["zstd 0.13.3",]+[[package]]+name = "replay-recorder-v4"+version = "0.1.0"+dependencies = [+ "axum 0.7.9",+ "common-alloc",+ "common-kafka",+ "common-metrics",+ "envconfig",+ "lifecycle",+ "metrics",+ "rdkafka",+ "replay-core-v4",+ "serde_json",+ "tokio",+ "tracing",+ "tracing-subscriber",+]+[[package]]name = "reqwest"version = "0.11.27"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 01e9f687ea23..aa57147cdede 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -80,6 +80,7 @@ members = ["pgcollector","pgapi","ai-research/replay-core-v4",+ "ai-research/replay-recorder-v4",]# NOTE: cargo allows only one crate with `links = "python"` (i.e. one pyo3 version) perdiff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlnew file mode 100644index 000000000000..699399a22682--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -0,0 +1,27 @@+[package]+name = "replay-recorder-v4"+version = "0.1.0"+edition = "2021"+publish = false+description = "Records the v4 AI research replay dataset: sampled, unscrubbed, sealed recordings with a sampling index"++[lints]+workspace = true++[dependencies]+common-alloc = { path = "../../common/alloc" }+common-kafka = { path = "../../common/kafka" }+common-metrics = { path = "../../common/metrics" }+lifecycle = { path = "../../common/lifecycle" }+replay-core-v4 = { path = "../replay-core-v4" }++axum = { workspace = true }+envconfig = { workspace = true }+metrics = { workspace = true }+rdkafka = { workspace = true }+tokio = { workspace = true }+tracing = { workspace = true }+tracing-subscriber = { workspace = true }++[dev-dependencies]+serde_json = { workspace = true }diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsnew file mode 100644index 000000000000..3869dfc227f9--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -0,0 +1,25 @@+use common_kafka::config::KafkaConfig;+use envconfig::Envconfig;++#[derive(Envconfig, Clone)]+pub struct Config {+ #[envconfig(nested = true)]+ pub kafka: KafkaConfig,++ #[envconfig(default = "session_recording_snapshot_item_events")]+ pub kafka_topic: String,++ #[envconfig(default = "ai-research-v4-recorder")]+ pub kafka_consumer_group: String,++ /// A new group starts at the head of the topic: v4 records sessions from when it starts, not the backlog.+ #[envconfig(default = "latest")]+ pub kafka_consumer_offset_reset: String,++ /// The share of sessions v4 records, from 0 to 100. See `replay_core_v4::gate`.+ #[envconfig(from = "AI_RESEARCH_V4_SAMPLE_PERCENTAGE", default = "0")]+ pub sample_percentage: f64,++ #[envconfig(default = "9104")]+ pub metrics_port: u16,+}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsnew file mode 100644index 000000000000..632d80aa9993--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -0,0 +1,90 @@+//! The consume loop.++use std::time::Duration;++use common_kafka::config::KafkaConfig;+use lifecycle::Handle;+use metrics::counter;+use rdkafka::consumer::{Consumer, StreamConsumer};+use rdkafka::message::{Headers, Message as _};+use rdkafka::ClientConfig;++use crate::process::{read, sampled};++pub fn create_consumer(+ kafka: &KafkaConfig,+ group: &str,+ offset_reset: &str,+ topic: &str,+) -> Result<StreamConsumer, rdkafka::error::KafkaError> {+ let mut config = ClientConfig::new();+ config+ .set("bootstrap.servers", &kafka.kafka_hosts)+ .set("group.id", group)+ .set("auto.offset.reset", offset_reset)+ .set("enable.auto.offset.store", "false")+ .set("partition.assignment.strategy", "cooperative-sticky");+ if kafka.kafka_tls {+ config+ .set("security.protocol", "ssl")+ .set("enable.ssl.certificate.verification", "false");+ }+ if !kafka.kafka_client_rack.is_empty() {+ config.set("client.rack", &kafka.kafka_client_rack);+ }+ let consumer: StreamConsumer = config.create()?;+ consumer.subscribe(&[topic])?;+ Ok(consumer)+}++fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str> {+ message+ .headers()?+ .iter()+ .find(|h| h.key == name)+ .and_then(|h| h.value)+ .and_then(|v| std::str::from_utf8(v).ok())+}++/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.+pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {+ let shutdown = handle.shutdown_signal();+ tokio::pin!(shutdown);+ loop {+ let message = tokio::select! {+ _ = &mut shutdown => break,+ message = consumer.recv() => message,+ };+ handle.report_healthy();+ let message = match message {+ Ok(message) => message,+ Err(error) => {+ tracing::warn!(%error, "kafka receive failed");+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);+ tokio::time::sleep(Duration::from_millis(100)).await;+ continue;+ }+ };+ let outcome = sampled(message.key(), percentage).and_then(|()| {+ read(+ message.payload().unwrap_or_default(),+ header(&message, "content-encoding"),+ )+ });+ match outcome {+ Ok(read) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")+ .increment(1);+ counter!("ai_research_v4_recorder_events_total")+ .increment(read.events.len() as u64);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1)+ }+ }+ if let Err(error) = consumer.store_offset_from_message(&message) {+ tracing::warn!(%error, "storing an offset failed");+ }+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsnew file mode 100644index 000000000000..9a96af680d24--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -0,0 +1,6 @@+//! The v4 recorder: a Kafka consumer beside the replay lanes that records a sample of sessions for AI research,+//! without scrubbing them. The format is in `products/ai_training/docs/replay-data.md`, section "v4 dataset".++pub mod config;+pub mod consumer;+pub mod process;diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsnew file mode 100644index 000000000000..cfb4c5b5f900--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -0,0 +1,78 @@+use std::time::Duration;++use axum::{routing::get, Router};+use common_metrics::setup_metrics_routes;+use envconfig::Envconfig;+use lifecycle::{ComponentOptions, Manager};+use tracing::level_filters::LevelFilter;+use tracing_subscriber::layer::SubscriberExt;+use tracing_subscriber::util::SubscriberInitExt;+use tracing_subscriber::{fmt, EnvFilter};++use replay_recorder_v4::config::Config;+use replay_recorder_v4::consumer::{create_consumer, run};++common_alloc::used!();++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let config = Config::init_from_env().expect("invalid configuration");+ tracing_subscriber::registry()+ .with(fmt::layer().with_target(true).with_level(true))+ .with(+ EnvFilter::builder()+ .with_default_directive(LevelFilter::INFO.into())+ .from_env_lossy(),+ )+ .init();+ tracing::info!(topic = %config.kafka_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");++ let mut manager = Manager::builder("replay-recorder-v4")+ .with_global_shutdown_timeout(Duration::from_secs(30))+ .build();+ let consumer_handle = manager.register(+ "consumer",+ ComponentOptions::new()+ .with_graceful_shutdown(Duration::from_secs(15))+ .with_liveness_deadline(Duration::from_secs(60)),+ );+ let metrics_handle = manager.register(+ "metrics-server",+ ComponentOptions::new().is_observability(true),+ );+ let readiness = manager.readiness_handler();+ let liveness = manager.liveness_handler();+ let monitor = manager.monitor_background();++ let port = config.metrics_port;+ tokio::spawn(async move {+ let _guard = metrics_handle.process_scope();+ let router = Router::new()+ .route(+ "/_readiness",+ get(move || {+ let readiness = readiness.clone();+ async move { readiness.check().await }+ }),+ )+ .route("/_liveness", get(move || async move { liveness.check() }));+ let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))+ .await+ .expect("bind metrics port");+ axum::serve(listener, setup_metrics_routes(router))+ .with_graceful_shutdown(metrics_handle.shutdown_signal())+ .await+ .expect("metrics server");+ });++ let consumer = create_consumer(+ &config.kafka,+ &config.kafka_consumer_group,+ &config.kafka_consumer_offset_reset,+ &config.kafka_topic,+ )?;+ tokio::spawn(run(consumer, config.sample_percentage, consumer_handle));++ monitor.wait().await?;+ Ok(())+}diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rsnew file mode 100644index 000000000000..938330b5f3c4--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/process.rs@@ -0,0 +1,80 @@+//! What the recorder does with one Kafka message, apart from Kafka itself.++use replay_core_v4::gate::in_sample;+use replay_core_v4::payload::decompress_payload;+use replay_core_v4::scan::message::{read_message, Message};++/// Why the recorder leaves a message out. The label is the metric value.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Skip {+ NotSampled,+ NoKey,+ Undecompressable,+ Unreadable,+}++impl Skip {+ pub fn as_str(self) -> &'static str {+ match self {+ Skip::NotSampled => "not_sampled",+ Skip::NoKey => "no_key",+ Skip::Undecompressable => "undecompressable",+ Skip::Unreadable => "unreadable",+ }+ }+}++/// Capture keys every replay message with its session ID, so the gate runs before the payload is read.+pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {+ let key = key.filter(|k| !k.is_empty()).ok_or(Skip::NoKey)?;+ if in_sample(key, percentage) {+ Ok(())+ } else {+ Err(Skip::NotSampled)+ }+}++/// Decompresses and reads a sampled message.+pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {+ let raw = decompress_payload(payload.to_vec(), content_encoding)+ .map_err(|_| Skip::Undecompressable)?;+ read_message(raw).map_err(|_| Skip::Unreadable)+}++#[cfg(test)]+mod tests {+ use replay_core_v4::compression::gzip;+ use serde_json::json;++ use super::*;++ fn capture_message() -> Vec<u8> {+ let inner = json!({"event": "$snapshot_items", "properties": {+ "$session_id": "019a2f3e-8c00-7000-8000-000000000007",+ "$snapshot_items": [{"type": 3, "timestamp": 1, "data": {"source": 1}}]+ }});+ json!({"distinct_id": "d", "data": inner.to_string()})+ .to_string()+ .into_bytes()+ }++ #[test]+ fn gates_on_the_kafka_key_before_reading() {+ let session = b"019a2f3e-8c00-7000-8000-000000000007";+ assert_eq!(sampled(Some(session), 100.0), Ok(()));+ assert_eq!(sampled(Some(session), 0.0), Err(Skip::NotSampled));+ assert_eq!(sampled(None, 100.0), Err(Skip::NoKey));+ assert_eq!(sampled(Some(b""), 100.0), Err(Skip::NoKey));+ }++ #[test]+ fn reads_plain_and_gzipped_capture_messages() {+ let plain = capture_message();+ for payload in [plain.clone(), gzip(&plain).unwrap()] {+ let message = read(&payload, None).unwrap();+ assert_eq!(message.session_id, "019a2f3e-8c00-7000-8000-000000000007");+ assert_eq!(message.events.len(), 1);+ }+ assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);+ }+}