:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): batch sessions and inspect them per flush (#117969)
Nicholas Waltzef116139b986
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsnew file mode 100644index 000000000000..367eb09a891c--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -0,0 +1,208 @@+//! Messages buffered between flushes, and the per-session inspectors that outlive them.++use std::collections::{BTreeMap, HashMap};+use std::time::{Duration, Instant};++use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};+use replay_core_v4::inspect::stats::BlockStats;+use replay_core_v4::scan::message::Message;++#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]+pub struct SessionKey {+ pub team_id: i64,+ pub session_id: String,+}++/// One session's messages in the current flush. Capture keys messages by session, so a session has one partition.+pub struct Pending {+ pub partition: i32,+ pub messages: Vec<Message>,+}++/// What a flush hands to the writers.+pub struct Block {+ pub session: SessionKey,+ pub messages: Vec<Message>,+ pub stats: BlockStats,+ pub moments: Vec<Moment>,+}++pub struct Batch {+ pub sessions: BTreeMap<SessionKey, Pending>,+ /// The last offset handled on each partition, recorded or skipped alike.+ pub offsets: HashMap<i32, i64>,+ bytes: usize,+ started: Instant,+}++impl Default for Batch {+ fn default() -> Self {+ Self {+ sessions: BTreeMap::new(),+ offsets: HashMap::new(),+ bytes: 0,+ started: Instant::now(),+ }+ }+}++impl Batch {+ /// Marks an offset handled. A skipped message still moves its partition forward.+ pub fn handled(&mut self, partition: i32, offset: i64) {+ let last = self.offsets.entry(partition).or_insert(offset);+ *last = (*last).max(offset);+ }++ pub fn add(&mut self, partition: i32, offset: i64, session: SessionKey, mut message: Message) {+ self.handled(partition, offset);+ // Decoded once here, for the inspector and the block writer both, and counted toward the flush size.+ message.decode_compressed();+ self.bytes += message.buffered_bytes();+ self.sessions+ .entry(session)+ .or_insert_with(|| Pending {+ partition,+ messages: Vec::new(),+ })+ .messages+ .push(message);+ }++ pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {+ !self.offsets.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)+ }++ /// Forgets partitions this process no longer owns. Their messages are not committed, so the new owner reads them.+ pub fn drop_partitions(&mut self, revoked: &[i32]) {+ self.sessions+ .retain(|_, pending| !revoked.contains(&pending.partition));+ for partition in revoked {+ self.offsets.remove(partition);+ }+ self.bytes = self+ .sessions+ .values()+ .flat_map(|p| &p.messages)+ .map(Message::buffered_bytes)+ .sum();+ }+}++/// Session inspectors by session. They keep DOM and click state across flushes.+#[derive(Default)]+pub struct Inspectors {+ sessions: HashMap<SessionKey, (SessionInspector, i32, Instant)>,+}++impl Inspectors {+ /// Runs every buffered message through its session's inspector and returns one block per session.+ pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<i32, i64>) {+ let now = Instant::now();+ let mut blocks = Vec::with_capacity(batch.sessions.len());+ for (session, pending) in batch.sessions {+ let (inspector, partition, last_seen) = self+ .sessions+ .entry(session.clone())+ .or_insert_with(|| (SessionInspector::default(), pending.partition, now));+ *partition = pending.partition;+ *last_seen = now;+ let mut block = BlockInspection::default();+ for message in &pending.messages {+ inspector.observe_message(&mut block, message);+ }+ let (stats, moments) = block.finish();+ blocks.push(Block {+ session,+ messages: pending.messages,+ stats,+ moments,+ });+ }+ (blocks, batch.offsets)+ }++ pub fn drop_partitions(&mut self, revoked: &[i32]) {+ self.sessions+ .retain(|_, (_, partition, _)| !revoked.contains(partition));+ }++ pub fn evict_idle(&mut self, idle: Duration) {+ self.sessions+ .retain(|_, (_, _, last_seen)| last_seen.elapsed() < idle);+ metrics::gauge!("ai_research_v4_recorder_live_sessions").set(self.sessions.len() as f64);+ }+}++#[cfg(test)]+mod tests {+ use replay_core_v4::scan::message::read_message;+ use serde_json::{json, Value};++ use super::*;++ fn message(session: &str, items: Value) -> Message {+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_items": items}});+ read_message(+ json!({"distinct_id": "d", "data": inner.to_string()})+ .to_string()+ .into_bytes(),+ )+ .unwrap()+ }++ fn key(session: &str) -> SessionKey {+ SessionKey {+ team_id: 7,+ session_id: session.to_owned(),+ }+ }++ fn snapshot() -> Value {+ json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 2, "timestamp": 2, "data": {"node": {"type": 0, "childNodes": [], "id": 1}}}])+ }++ fn mouse() -> Value {+ json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])+ }++ #[test]+ fn a_revoked_partition_leaves_nothing_behind() {+ let mut batch = Batch::default();+ batch.add(0, 10, key("a"), message("a", mouse()));+ batch.add(1, 20, key("b"), message("b", mouse()));+ batch.handled(1, 21);+ let mut inspectors = Inspectors::default();+ batch.drop_partitions(&[1]);+ inspectors.drop_partitions(&[1]);++ let (blocks, offsets) = inspectors.inspect(batch);+ assert_eq!(+ blocks+ .iter()+ .map(|b| b.session.session_id.as_str())+ .collect::<Vec<_>>(),+ vec!["a"]+ );+ assert_eq!(offsets, HashMap::from([(0, 10)]));+ }++ #[test]+ fn session_state_carries_across_flushes() {+ let mut inspectors = Inspectors::default();+ let mut first = Batch::default();+ first.add(0, 1, key("a"), message("a", snapshot()));+ first.add(0, 2, key("b"), message("b", mouse()));+ inspectors.inspect(first);++ let mut second = Batch::default();+ second.add(0, 3, key("a"), message("a", mouse()));+ second.add(0, 4, key("b"), message("b", mouse()));+ let (blocks, _) = inspectors.inspect(second);+ let cold: Vec<_> = blocks+ .iter()+ .map(|b| (b.session.session_id.as_str(), b.stats.cold_start))+ .collect();+ // `a` started with its snapshot in this process. `b` was already running when this process first saw it.+ assert_eq!(cold, vec![("a", false), ("b", true)]);+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 3bdb82fb934e..0951a8ed98c2 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -27,6 +27,18 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]pub team_refresh_secs: u64,+ /// A flush every this many milliseconds. The lane is not latency-sensitive, so long flushes keep objects few.+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_INTERVAL_MS", default = "60000")]+ pub flush_interval_ms: u64,++ /// A flush once the buffered messages reach this many bytes.+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_MAX_BYTES", default = "67108864")]+ pub flush_max_bytes: usize,++ /// A session's inspector is dropped after this long without a message, so memory follows live sessions.+ #[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]+ pub session_idle_secs: u64,+#[envconfig(default = "9104")]pub metrics_port: u16,}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex b2c42823c6df..9438f0183862 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -1,45 +1,109 @@-//! The consume loop.+//! The consume loop: gate, read and admit each message, buffer it, and flush on time or size.-use std::sync::Arc;+use std::sync::{Arc, Mutex};use std::time::{Duration, SystemTime, UNIX_EPOCH};use common_kafka::config::KafkaConfig;use lifecycle::Handle;use metrics::counter;-use rdkafka::consumer::{Consumer, StreamConsumer};-use rdkafka::message::{Headers, Message as _};-use rdkafka::ClientConfig;+use rdkafka::consumer::{+ BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,+};+use rdkafka::message::{BorrowedMessage, Headers, Message as _};+use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};+use crate::batch::{Batch, Block, Inspectors, SessionKey};use crate::process::{admit, read, sampled};use crate::teams::TeamCache;-pub fn create_consumer(- kafka: &KafkaConfig,- group: &str,- offset_reset: &str,- topic: &str,-) -> Result<StreamConsumer, rdkafka::error::KafkaError> {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", &kafka.kafka_hosts)- .set("group.id", group)- .set("auto.offset.reset", offset_reset)- .set("enable.auto.offset.store", "false")- .set("partition.assignment.strategy", "cooperative-sticky");- if kafka.kafka_tls {+/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.+pub struct RebalanceContext {+ revoked: Arc<Mutex<Vec<i32>>>,+}++impl ClientContext for RebalanceContext {}++impl ConsumerContext for RebalanceContext {+ fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {+ if let Rebalance::Revoke(partitions) = rebalance {+ self.revoked+ .lock()+ .unwrap()+ .extend(partitions.elements().iter().map(|e| e.partition()));+ }+ }+}++pub struct RecorderConsumer {+ consumer: StreamConsumer<RebalanceContext>,+ topic: String,+ revoked: Arc<Mutex<Vec<i32>>>,+}++impl RecorderConsumer {+ pub fn new(+ kafka: &KafkaConfig,+ group: &str,+ offset_reset: &str,+ topic: &str,+ ) -> Result<Self, rdkafka::error::KafkaError> {+ let mut config = ClientConfig::new();config- .set("security.protocol", "ssl")- .set("enable.ssl.certificate.verification", "false");+ .set("bootstrap.servers", &kafka.kafka_hosts)+ .set("group.id", group)+ .set("auto.offset.reset", offset_reset)+ // Offsets move only after a flush has stored the blocks for them.+ .set("enable.auto.commit", "false")+ .set("enable.auto.offset.store", "false")+ .set("partition.assignment.strategy", "cooperative-sticky");+ if kafka.kafka_tls {+ config+ .set("security.protocol", "ssl")+ .set("enable.ssl.certificate.verification", "false");+ }+ if !kafka.kafka_client_rack.is_empty() {+ config.set("client.rack", &kafka.kafka_client_rack);+ }+ let revoked = Arc::new(Mutex::new(Vec::new()));+ let consumer: StreamConsumer<RebalanceContext> =+ config.create_with_context(RebalanceContext {+ revoked: revoked.clone(),+ })?;+ consumer.subscribe(&[topic])?;+ Ok(Self {+ consumer,+ topic: topic.to_owned(),+ revoked,+ })+ }++ fn take_revoked(&self) -> Vec<i32> {+ std::mem::take(&mut *self.revoked.lock().unwrap())}- if !kafka.kafka_client_rack.is_empty() {- config.set("client.rack", &kafka.kafka_client_rack);++ /// Commits the next offset of every partition this process still owns.+ fn commit(&self, offsets: &std::collections::HashMap<i32, i64>) {+ let assigned = self.consumer.assignment().unwrap_or_default();+ let mut list = TopicPartitionList::new();+ for (&partition, &offset) in offsets {+ if assigned.find_partition(&self.topic, partition).is_some() {+ if let Err(error) =+ list.add_partition_offset(&self.topic, partition, Offset::Offset(offset + 1))+ {+ tracing::warn!(%error, partition, "building the offset commit failed");+ }+ }+ }+ if list.count() > 0 {+ if let Err(error) = self.consumer.commit(&list, CommitMode::Async) {+ tracing::warn!(%error, "committing offsets failed");+ counter!("ai_research_v4_recorder_commit_failures_total").increment(1);+ }+ }}- let consumer: StreamConsumer = config.create()?;- consumer.subscribe(&[topic])?;- Ok(consumer)}-fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str> {+fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {message.headers()?.iter()@@ -48,53 +112,104 @@ fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str>.and_then(|v| std::str::from_utf8(v).ok())}-/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.-pub async fn run(consumer: StreamConsumer, percentage: f64, teams: Arc<TeamCache>, handle: Handle) {+pub struct Settings {+ pub sample_percentage: f64,+ pub flush_interval: Duration,+ pub flush_max_bytes: usize,+ pub session_idle: Duration,+}++pub async fn run(+ consumer: RecorderConsumer,+ settings: Settings,+ teams: Arc<TeamCache>,+ handle: Handle,+) {let shutdown = handle.shutdown_signal();tokio::pin!(shutdown);+ let mut batch = Batch::default();+ let mut inspectors = Inspectors::default();+ let mut tick = tokio::time::interval(Duration::from_secs(1));loop {- let message = tokio::select! {+ tokio::select! {_ = &mut shutdown => break,- message = consumer.recv() => message,- };+ _ = tick.tick() => {}+ received = consumer.consumer.recv() => match received {+ Ok(message) => handle_message(&message, &settings, &teams, &mut batch),+ Err(error) => {+ tracing::warn!(%error, "kafka receive failed");+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);+ // A fatal error disables the client for good, so only a restart recovers.+ if let Some((code, reason)) = consumer.consumer.client().fatal_error() {+ handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));+ return;+ }+ }+ },+ }handle.report_healthy();- let message = match message {- Ok(message) => message,- Err(error) => {- tracing::warn!(%error, "kafka receive failed");- counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);- tokio::time::sleep(Duration::from_millis(100)).await;- continue;- }- };- let now_ms = SystemTime::now()- .duration_since(UNIX_EPOCH)- .map_or(0, |d| d.as_millis() as i64);- let outcome = sampled(message.key(), percentage)- .and_then(|()| {- read(- message.payload().unwrap_or_default(),- header(&message, "content-encoding"),- )- })- .and_then(|read| {- let team = header(&message, "token").and_then(|token| teams.get(token));- admit(&read, team, now_ms).map(|_| read)- });- match outcome {- Ok(read) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")- .increment(1);- counter!("ai_research_v4_recorder_events_total")- .increment(read.events.len() as u64);- }- Err(skip) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())- .increment(1)- }+ let revoked = consumer.take_revoked();+ if !revoked.is_empty() {+ batch.drop_partitions(&revoked);+ inspectors.drop_partitions(&revoked);+ }+ if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {+ let (blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));+ flush(&blocks);+ consumer.commit(&offsets);+ inspectors.evict_idle(settings.session_idle);+ }+ }+}++fn handle_message(+ message: &BorrowedMessage<'_>,+ settings: &Settings,+ teams: &TeamCache,+ batch: &mut Batch,+) {+ let now_ms = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_or(0, |d| d.as_millis() as i64);+ let admitted = sampled(message.key(), settings.sample_percentage)+ .and_then(|()| {+ read(+ message.payload().unwrap_or_default(),+ header(message, "content-encoding"),+ )+ })+ .and_then(|read| {+ let team = header(message, "token").and_then(|token| teams.get(token));+ admit(&read, team, now_ms).map(|team_id| (team_id, read))+ });+ match admitted {+ Ok((team_id, read)) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")+ .increment(1);+ let session = SessionKey {+ team_id,+ session_id: read.session_id.clone(),+ };+ batch.add(message.partition(), message.offset(), session, read);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1);+ batch.handled(message.partition(), message.offset());+ }+ }+}++/// Reports what a flush inspected. The writers that store blocks come in the next layers.+fn flush(blocks: &[Block]) {+ counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);+ for block in blocks {+ counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);+ if block.stats.cold_start {+ counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);}- if let Err(error) = consumer.store_offset_from_message(&message) {- tracing::warn!(%error, "storing an offset failed");+ if block.stats.orphan_mutations > 0 {+ counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);}}}diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex ae01300ec013..f471229fe802 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -1,6 +1,7 @@//! The v4 recorder: a Kafka consumer beside the replay lanes that records a sample of sessions for AI research,//! without scrubbing them. The format is in `products/ai_training/docs/replay-data.md`, section "v4 dataset".+pub mod batch;pub mod config;pub mod consumer;pub mod process;diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex 239705b13325..0a7a1a40a6eb 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -12,7 +12,7 @@ use tracing_subscriber::util::SubscriberInitExt;use tracing_subscriber::{fmt, EnvFilter};use replay_recorder_v4::config::Config;-use replay_recorder_v4::consumer::{create_consumer, run};+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings};use replay_recorder_v4::teams::TeamCache;common_alloc::used!();@@ -100,18 +100,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {teams_handle,));- let consumer = create_consumer(+ let consumer = RecorderConsumer::new(&config.kafka,&config.kafka_consumer_group,&config.kafka_consumer_offset_reset,&config.kafka_topic,)?;- tokio::spawn(run(- consumer,- config.sample_percentage,- teams,- consumer_handle,- ));+ let settings = Settings {+ sample_percentage: config.sample_percentage,+ flush_interval: Duration::from_millis(config.flush_interval_ms),+ flush_max_bytes: config.flush_max_bytes,+ session_idle: Duration::from_secs(config.session_idle_secs),+ };+ tokio::spawn(run(consumer, settings, teams, consumer_handle));monitor.wait().await?;Ok(())