Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): consume replay messages behind the v4 gate (#117967)

Nicholas Waltz0bd46172342b
diff --git a/.github/rust-images.yml b/.github/rust-images.yml
index d8a7fc379c1c..5bc5f67a4f44 100644
--- a/.github/rust-images.yml
+++ b/.github/rust-images.yml
@@ -154,6 +154,12 @@
dockerfile: ./rust/Dockerfile
project: lvsrh6c64c
+# Records the v4 AI research replay dataset (products/ai_training/docs/replay-data.md). CI builds the image, but it
+# has no entry in the deploy matrix of rust-docker-build.yml until the recorder ships.
+- image: replay-recorder-v4
+ dockerfile: ./rust/Dockerfile
+ project: vznmbshh6q
+
- image: personhog-test-harness
dockerfile: ./rust/Dockerfile
project: 6tl5lmb8l8
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index b4698674352b..439e940dfc85 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11047,6 +11047,25 @@ dependencies = [
"zstd 0.13.3",
]
+[[package]]
+name = "replay-recorder-v4"
+version = "0.1.0"
+dependencies = [
+ "axum 0.7.9",
+ "common-alloc",
+ "common-kafka",
+ "common-metrics",
+ "envconfig",
+ "lifecycle",
+ "metrics",
+ "rdkafka",
+ "replay-core-v4",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
[[package]]
name = "reqwest"
version = "0.11.27"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 01e9f687ea23..aa57147cdede 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -80,6 +80,7 @@ members = [
"pgcollector",
"pgapi",
"ai-research/replay-core-v4",
+ "ai-research/replay-recorder-v4",
]
# NOTE: cargo allows only one crate with `links = "python"` (i.e. one pyo3 version) per
diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.toml
new file mode 100644
index 000000000000..699399a22682
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "replay-recorder-v4"
+version = "0.1.0"
+edition = "2021"
+publish = false
+description = "Records the v4 AI research replay dataset: sampled, unscrubbed, sealed recordings with a sampling index"
+
+[lints]
+workspace = true
+
+[dependencies]
+common-alloc = { path = "../../common/alloc" }
+common-kafka = { path = "../../common/kafka" }
+common-metrics = { path = "../../common/metrics" }
+lifecycle = { path = "../../common/lifecycle" }
+replay-core-v4 = { path = "../replay-core-v4" }
+
+axum = { workspace = true }
+envconfig = { workspace = true }
+metrics = { workspace = true }
+rdkafka = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true }
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
new file mode 100644
index 000000000000..3869dfc227f9
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -0,0 +1,25 @@
+use common_kafka::config::KafkaConfig;
+use envconfig::Envconfig;
+
+#[derive(Envconfig, Clone)]
+pub struct Config {
+ #[envconfig(nested = true)]
+ pub kafka: KafkaConfig,
+
+ #[envconfig(default = "session_recording_snapshot_item_events")]
+ pub kafka_topic: String,
+
+ #[envconfig(default = "ai-research-v4-recorder")]
+ pub kafka_consumer_group: String,
+
+ /// A new group starts at the head of the topic: v4 records sessions from when it starts, not the backlog.
+ #[envconfig(default = "latest")]
+ pub kafka_consumer_offset_reset: String,
+
+ /// The share of sessions v4 records, from 0 to 100. See `replay_core_v4::gate`.
+ #[envconfig(from = "AI_RESEARCH_V4_SAMPLE_PERCENTAGE", default = "0")]
+ pub sample_percentage: f64,
+
+ #[envconfig(default = "9104")]
+ pub metrics_port: u16,
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
new file mode 100644
index 000000000000..632d80aa9993
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -0,0 +1,90 @@
+//! The consume loop.
+
+use std::time::Duration;
+
+use common_kafka::config::KafkaConfig;
+use lifecycle::Handle;
+use metrics::counter;
+use rdkafka::consumer::{Consumer, StreamConsumer};
+use rdkafka::message::{Headers, Message as _};
+use rdkafka::ClientConfig;
+
+use crate::process::{read, sampled};
+
+pub fn create_consumer(
+ kafka: &KafkaConfig,
+ group: &str,
+ offset_reset: &str,
+ topic: &str,
+) -> Result<StreamConsumer, rdkafka::error::KafkaError> {
+ let mut config = ClientConfig::new();
+ config
+ .set("bootstrap.servers", &kafka.kafka_hosts)
+ .set("group.id", group)
+ .set("auto.offset.reset", offset_reset)
+ .set("enable.auto.offset.store", "false")
+ .set("partition.assignment.strategy", "cooperative-sticky");
+ if kafka.kafka_tls {
+ config
+ .set("security.protocol", "ssl")
+ .set("enable.ssl.certificate.verification", "false");
+ }
+ if !kafka.kafka_client_rack.is_empty() {
+ config.set("client.rack", &kafka.kafka_client_rack);
+ }
+ let consumer: StreamConsumer = config.create()?;
+ consumer.subscribe(&[topic])?;
+ Ok(consumer)
+}
+
+fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str> {
+ message
+ .headers()?
+ .iter()
+ .find(|h| h.key == name)
+ .and_then(|h| h.value)
+ .and_then(|v| std::str::from_utf8(v).ok())
+}
+
+/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.
+pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {
+ let shutdown = handle.shutdown_signal();
+ tokio::pin!(shutdown);
+ loop {
+ let message = tokio::select! {
+ _ = &mut shutdown => break,
+ message = consumer.recv() => message,
+ };
+ handle.report_healthy();
+ let message = match message {
+ Ok(message) => message,
+ Err(error) => {
+ tracing::warn!(%error, "kafka receive failed");
+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ continue;
+ }
+ };
+ let outcome = sampled(message.key(), percentage).and_then(|()| {
+ read(
+ message.payload().unwrap_or_default(),
+ header(&message, "content-encoding"),
+ )
+ });
+ match outcome {
+ Ok(read) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")
+ .increment(1);
+ counter!("ai_research_v4_recorder_events_total")
+ .increment(read.events.len() as u64);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1)
+ }
+ }
+ if let Err(error) = consumer.store_offset_from_message(&message) {
+ tracing::warn!(%error, "storing an offset failed");
+ }
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
new file mode 100644
index 000000000000..9a96af680d24
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -0,0 +1,6 @@
+//! The v4 recorder: a Kafka consumer beside the replay lanes that records a sample of sessions for AI research,
+//! without scrubbing them. The format is in `products/ai_training/docs/replay-data.md`, section "v4 dataset".
+
+pub mod config;
+pub mod consumer;
+pub mod process;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
new file mode 100644
index 000000000000..cfb4c5b5f900
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -0,0 +1,78 @@
+use std::time::Duration;
+
+use axum::{routing::get, Router};
+use common_metrics::setup_metrics_routes;
+use envconfig::Envconfig;
+use lifecycle::{ComponentOptions, Manager};
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{fmt, EnvFilter};
+
+use replay_recorder_v4::config::Config;
+use replay_recorder_v4::consumer::{create_consumer, run};
+
+common_alloc::used!();
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let config = Config::init_from_env().expect("invalid configuration");
+ tracing_subscriber::registry()
+ .with(fmt::layer().with_target(true).with_level(true))
+ .with(
+ EnvFilter::builder()
+ .with_default_directive(LevelFilter::INFO.into())
+ .from_env_lossy(),
+ )
+ .init();
+ tracing::info!(topic = %config.kafka_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");
+
+ let mut manager = Manager::builder("replay-recorder-v4")
+ .with_global_shutdown_timeout(Duration::from_secs(30))
+ .build();
+ let consumer_handle = manager.register(
+ "consumer",
+ ComponentOptions::new()
+ .with_graceful_shutdown(Duration::from_secs(15))
+ .with_liveness_deadline(Duration::from_secs(60)),
+ );
+ let metrics_handle = manager.register(
+ "metrics-server",
+ ComponentOptions::new().is_observability(true),
+ );
+ let readiness = manager.readiness_handler();
+ let liveness = manager.liveness_handler();
+ let monitor = manager.monitor_background();
+
+ let port = config.metrics_port;
+ tokio::spawn(async move {
+ let _guard = metrics_handle.process_scope();
+ let router = Router::new()
+ .route(
+ "/_readiness",
+ get(move || {
+ let readiness = readiness.clone();
+ async move { readiness.check().await }
+ }),
+ )
+ .route("/_liveness", get(move || async move { liveness.check() }));
+ let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
+ .await
+ .expect("bind metrics port");
+ axum::serve(listener, setup_metrics_routes(router))
+ .with_graceful_shutdown(metrics_handle.shutdown_signal())
+ .await
+ .expect("metrics server");
+ });
+
+ let consumer = create_consumer(
+ &config.kafka,
+ &config.kafka_consumer_group,
+ &config.kafka_consumer_offset_reset,
+ &config.kafka_topic,
+ )?;
+ tokio::spawn(run(consumer, config.sample_percentage, consumer_handle));
+
+ monitor.wait().await?;
+ Ok(())
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rs
new file mode 100644
index 000000000000..938330b5f3c4
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/process.rs
@@ -0,0 +1,80 @@
+//! What the recorder does with one Kafka message, apart from Kafka itself.
+
+use replay_core_v4::gate::in_sample;
+use replay_core_v4::payload::decompress_payload;
+use replay_core_v4::scan::message::{read_message, Message};
+
+/// Why the recorder leaves a message out. The label is the metric value.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Skip {
+ NotSampled,
+ NoKey,
+ Undecompressable,
+ Unreadable,
+}
+
+impl Skip {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Skip::NotSampled => "not_sampled",
+ Skip::NoKey => "no_key",
+ Skip::Undecompressable => "undecompressable",
+ Skip::Unreadable => "unreadable",
+ }
+ }
+}
+
+/// Capture keys every replay message with its session ID, so the gate runs before the payload is read.
+pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {
+ let key = key.filter(|k| !k.is_empty()).ok_or(Skip::NoKey)?;
+ if in_sample(key, percentage) {
+ Ok(())
+ } else {
+ Err(Skip::NotSampled)
+ }
+}
+
+/// Decompresses and reads a sampled message.
+pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {
+ let raw = decompress_payload(payload.to_vec(), content_encoding)
+ .map_err(|_| Skip::Undecompressable)?;
+ read_message(raw).map_err(|_| Skip::Unreadable)
+}
+
+#[cfg(test)]
+mod tests {
+ use replay_core_v4::compression::gzip;
+ use serde_json::json;
+
+ use super::*;
+
+ fn capture_message() -> Vec<u8> {
+ let inner = json!({"event": "$snapshot_items", "properties": {
+ "$session_id": "019a2f3e-8c00-7000-8000-000000000007",
+ "$snapshot_items": [{"type": 3, "timestamp": 1, "data": {"source": 1}}]
+ }});
+ json!({"distinct_id": "d", "data": inner.to_string()})
+ .to_string()
+ .into_bytes()
+ }
+
+ #[test]
+ fn gates_on_the_kafka_key_before_reading() {
+ let session = b"019a2f3e-8c00-7000-8000-000000000007";
+ assert_eq!(sampled(Some(session), 100.0), Ok(()));
+ assert_eq!(sampled(Some(session), 0.0), Err(Skip::NotSampled));
+ assert_eq!(sampled(None, 100.0), Err(Skip::NoKey));
+ assert_eq!(sampled(Some(b""), 100.0), Err(Skip::NoKey));
+ }
+
+ #[test]
+ fn reads_plain_and_gzipped_capture_messages() {
+ let plain = capture_message();
+ for payload in [plain.clone(), gzip(&plain).unwrap()] {
+ let message = read(&payload, None).unwrap();
+ assert_eq!(message.session_id, "019a2f3e-8c00-7000-8000-000000000007");
+ assert_eq!(message.events.len(), 1);
+ }
+ assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);
+ }
+}