Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): batch sessions and inspect them per flush (#117969)

Nicholas Waltzef116139b986
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
new file mode 100644
index 000000000000..367eb09a891c
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -0,0 +1,208 @@
+//! Messages buffered between flushes, and the per-session inspectors that outlive them.
+
+use std::collections::{BTreeMap, HashMap};
+use std::time::{Duration, Instant};
+
+use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};
+use replay_core_v4::inspect::stats::BlockStats;
+use replay_core_v4::scan::message::Message;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct SessionKey {
+ pub team_id: i64,
+ pub session_id: String,
+}
+
+/// One session's messages in the current flush. Capture keys messages by session, so a session has one partition.
+pub struct Pending {
+ pub partition: i32,
+ pub messages: Vec<Message>,
+}
+
+/// What a flush hands to the writers.
+pub struct Block {
+ pub session: SessionKey,
+ pub messages: Vec<Message>,
+ pub stats: BlockStats,
+ pub moments: Vec<Moment>,
+}
+
+pub struct Batch {
+ pub sessions: BTreeMap<SessionKey, Pending>,
+ /// The last offset handled on each partition, recorded or skipped alike.
+ pub offsets: HashMap<i32, i64>,
+ bytes: usize,
+ started: Instant,
+}
+
+impl Default for Batch {
+ fn default() -> Self {
+ Self {
+ sessions: BTreeMap::new(),
+ offsets: HashMap::new(),
+ bytes: 0,
+ started: Instant::now(),
+ }
+ }
+}
+
+impl Batch {
+ /// Marks an offset handled. A skipped message still moves its partition forward.
+ pub fn handled(&mut self, partition: i32, offset: i64) {
+ let last = self.offsets.entry(partition).or_insert(offset);
+ *last = (*last).max(offset);
+ }
+
+ pub fn add(&mut self, partition: i32, offset: i64, session: SessionKey, mut message: Message) {
+ self.handled(partition, offset);
+ // Decoded once here, for the inspector and the block writer both, and counted toward the flush size.
+ message.decode_compressed();
+ self.bytes += message.buffered_bytes();
+ self.sessions
+ .entry(session)
+ .or_insert_with(|| Pending {
+ partition,
+ messages: Vec::new(),
+ })
+ .messages
+ .push(message);
+ }
+
+ pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {
+ !self.offsets.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)
+ }
+
+ /// Forgets partitions this process no longer owns. Their messages are not committed, so the new owner reads them.
+ pub fn drop_partitions(&mut self, revoked: &[i32]) {
+ self.sessions
+ .retain(|_, pending| !revoked.contains(&pending.partition));
+ for partition in revoked {
+ self.offsets.remove(partition);
+ }
+ self.bytes = self
+ .sessions
+ .values()
+ .flat_map(|p| &p.messages)
+ .map(Message::buffered_bytes)
+ .sum();
+ }
+}
+
+/// Session inspectors by session. They keep DOM and click state across flushes.
+#[derive(Default)]
+pub struct Inspectors {
+ sessions: HashMap<SessionKey, (SessionInspector, i32, Instant)>,
+}
+
+impl Inspectors {
+ /// Runs every buffered message through its session's inspector and returns one block per session.
+ pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<i32, i64>) {
+ let now = Instant::now();
+ let mut blocks = Vec::with_capacity(batch.sessions.len());
+ for (session, pending) in batch.sessions {
+ let (inspector, partition, last_seen) = self
+ .sessions
+ .entry(session.clone())
+ .or_insert_with(|| (SessionInspector::default(), pending.partition, now));
+ *partition = pending.partition;
+ *last_seen = now;
+ let mut block = BlockInspection::default();
+ for message in &pending.messages {
+ inspector.observe_message(&mut block, message);
+ }
+ let (stats, moments) = block.finish();
+ blocks.push(Block {
+ session,
+ messages: pending.messages,
+ stats,
+ moments,
+ });
+ }
+ (blocks, batch.offsets)
+ }
+
+ pub fn drop_partitions(&mut self, revoked: &[i32]) {
+ self.sessions
+ .retain(|_, (_, partition, _)| !revoked.contains(partition));
+ }
+
+ pub fn evict_idle(&mut self, idle: Duration) {
+ self.sessions
+ .retain(|_, (_, _, last_seen)| last_seen.elapsed() < idle);
+ metrics::gauge!("ai_research_v4_recorder_live_sessions").set(self.sessions.len() as f64);
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use replay_core_v4::scan::message::read_message;
+ use serde_json::{json, Value};
+
+ use super::*;
+
+ fn message(session: &str, items: Value) -> Message {
+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_items": items}});
+ read_message(
+ json!({"distinct_id": "d", "data": inner.to_string()})
+ .to_string()
+ .into_bytes(),
+ )
+ .unwrap()
+ }
+
+ fn key(session: &str) -> SessionKey {
+ SessionKey {
+ team_id: 7,
+ session_id: session.to_owned(),
+ }
+ }
+
+ fn snapshot() -> Value {
+ json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 2, "timestamp": 2, "data": {"node": {"type": 0, "childNodes": [], "id": 1}}}])
+ }
+
+ fn mouse() -> Value {
+ json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])
+ }
+
+ #[test]
+ fn a_revoked_partition_leaves_nothing_behind() {
+ let mut batch = Batch::default();
+ batch.add(0, 10, key("a"), message("a", mouse()));
+ batch.add(1, 20, key("b"), message("b", mouse()));
+ batch.handled(1, 21);
+ let mut inspectors = Inspectors::default();
+ batch.drop_partitions(&[1]);
+ inspectors.drop_partitions(&[1]);
+
+ let (blocks, offsets) = inspectors.inspect(batch);
+ assert_eq!(
+ blocks
+ .iter()
+ .map(|b| b.session.session_id.as_str())
+ .collect::<Vec<_>>(),
+ vec!["a"]
+ );
+ assert_eq!(offsets, HashMap::from([(0, 10)]));
+ }
+
+ #[test]
+ fn session_state_carries_across_flushes() {
+ let mut inspectors = Inspectors::default();
+ let mut first = Batch::default();
+ first.add(0, 1, key("a"), message("a", snapshot()));
+ first.add(0, 2, key("b"), message("b", mouse()));
+ inspectors.inspect(first);
+
+ let mut second = Batch::default();
+ second.add(0, 3, key("a"), message("a", mouse()));
+ second.add(0, 4, key("b"), message("b", mouse()));
+ let (blocks, _) = inspectors.inspect(second);
+ let cold: Vec<_> = blocks
+ .iter()
+ .map(|b| (b.session.session_id.as_str(), b.stats.cold_start))
+ .collect();
+ // `a` started with its snapshot in this process. `b` was already running when this process first saw it.
+ assert_eq!(cold, vec![("a", false), ("b", true)]);
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 3bdb82fb934e..0951a8ed98c2 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -27,6 +27,18 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]
pub team_refresh_secs: u64,
+ /// A flush every this many milliseconds. The lane is not latency-sensitive, so long flushes keep objects few.
+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_INTERVAL_MS", default = "60000")]
+ pub flush_interval_ms: u64,
+
+ /// A flush once the buffered messages reach this many bytes.
+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_MAX_BYTES", default = "67108864")]
+ pub flush_max_bytes: usize,
+
+ /// A session's inspector is dropped after this long without a message, so memory follows live sessions.
+ #[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]
+ pub session_idle_secs: u64,
+
#[envconfig(default = "9104")]
pub metrics_port: u16,
}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index b2c42823c6df..9438f0183862 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -1,45 +1,109 @@
-//! The consume loop.
+//! The consume loop: gate, read and admit each message, buffer it, and flush on time or size.
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use common_kafka::config::KafkaConfig;
use lifecycle::Handle;
use metrics::counter;
-use rdkafka::consumer::{Consumer, StreamConsumer};
-use rdkafka::message::{Headers, Message as _};
-use rdkafka::ClientConfig;
+use rdkafka::consumer::{
+ BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,
+};
+use rdkafka::message::{BorrowedMessage, Headers, Message as _};
+use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
+use crate::batch::{Batch, Block, Inspectors, SessionKey};
use crate::process::{admit, read, sampled};
use crate::teams::TeamCache;
-pub fn create_consumer(
- kafka: &KafkaConfig,
- group: &str,
- offset_reset: &str,
- topic: &str,
-) -> Result<StreamConsumer, rdkafka::error::KafkaError> {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", &kafka.kafka_hosts)
- .set("group.id", group)
- .set("auto.offset.reset", offset_reset)
- .set("enable.auto.offset.store", "false")
- .set("partition.assignment.strategy", "cooperative-sticky");
- if kafka.kafka_tls {
+/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.
+pub struct RebalanceContext {
+ revoked: Arc<Mutex<Vec<i32>>>,
+}
+
+impl ClientContext for RebalanceContext {}
+
+impl ConsumerContext for RebalanceContext {
+ fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {
+ if let Rebalance::Revoke(partitions) = rebalance {
+ self.revoked
+ .lock()
+ .unwrap()
+ .extend(partitions.elements().iter().map(|e| e.partition()));
+ }
+ }
+}
+
+pub struct RecorderConsumer {
+ consumer: StreamConsumer<RebalanceContext>,
+ topic: String,
+ revoked: Arc<Mutex<Vec<i32>>>,
+}
+
+impl RecorderConsumer {
+ pub fn new(
+ kafka: &KafkaConfig,
+ group: &str,
+ offset_reset: &str,
+ topic: &str,
+ ) -> Result<Self, rdkafka::error::KafkaError> {
+ let mut config = ClientConfig::new();
config
- .set("security.protocol", "ssl")
- .set("enable.ssl.certificate.verification", "false");
+ .set("bootstrap.servers", &kafka.kafka_hosts)
+ .set("group.id", group)
+ .set("auto.offset.reset", offset_reset)
+ // Offsets move only after a flush has stored the blocks for them.
+ .set("enable.auto.commit", "false")
+ .set("enable.auto.offset.store", "false")
+ .set("partition.assignment.strategy", "cooperative-sticky");
+ if kafka.kafka_tls {
+ config
+ .set("security.protocol", "ssl")
+ .set("enable.ssl.certificate.verification", "false");
+ }
+ if !kafka.kafka_client_rack.is_empty() {
+ config.set("client.rack", &kafka.kafka_client_rack);
+ }
+ let revoked = Arc::new(Mutex::new(Vec::new()));
+ let consumer: StreamConsumer<RebalanceContext> =
+ config.create_with_context(RebalanceContext {
+ revoked: revoked.clone(),
+ })?;
+ consumer.subscribe(&[topic])?;
+ Ok(Self {
+ consumer,
+ topic: topic.to_owned(),
+ revoked,
+ })
+ }
+
+ fn take_revoked(&self) -> Vec<i32> {
+ std::mem::take(&mut *self.revoked.lock().unwrap())
}
- if !kafka.kafka_client_rack.is_empty() {
- config.set("client.rack", &kafka.kafka_client_rack);
+
+ /// Commits the next offset of every partition this process still owns.
+ fn commit(&self, offsets: &std::collections::HashMap<i32, i64>) {
+ let assigned = self.consumer.assignment().unwrap_or_default();
+ let mut list = TopicPartitionList::new();
+ for (&partition, &offset) in offsets {
+ if assigned.find_partition(&self.topic, partition).is_some() {
+ if let Err(error) =
+ list.add_partition_offset(&self.topic, partition, Offset::Offset(offset + 1))
+ {
+ tracing::warn!(%error, partition, "building the offset commit failed");
+ }
+ }
+ }
+ if list.count() > 0 {
+ if let Err(error) = self.consumer.commit(&list, CommitMode::Async) {
+ tracing::warn!(%error, "committing offsets failed");
+ counter!("ai_research_v4_recorder_commit_failures_total").increment(1);
+ }
+ }
}
- let consumer: StreamConsumer = config.create()?;
- consumer.subscribe(&[topic])?;
- Ok(consumer)
}
-fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str> {
+fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {
message
.headers()?
.iter()
@@ -48,53 +112,104 @@ fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str>
.and_then(|v| std::str::from_utf8(v).ok())
}
-/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.
-pub async fn run(consumer: StreamConsumer, percentage: f64, teams: Arc<TeamCache>, handle: Handle) {
+pub struct Settings {
+ pub sample_percentage: f64,
+ pub flush_interval: Duration,
+ pub flush_max_bytes: usize,
+ pub session_idle: Duration,
+}
+
+pub async fn run(
+ consumer: RecorderConsumer,
+ settings: Settings,
+ teams: Arc<TeamCache>,
+ handle: Handle,
+) {
let shutdown = handle.shutdown_signal();
tokio::pin!(shutdown);
+ let mut batch = Batch::default();
+ let mut inspectors = Inspectors::default();
+ let mut tick = tokio::time::interval(Duration::from_secs(1));
loop {
- let message = tokio::select! {
+ tokio::select! {
_ = &mut shutdown => break,
- message = consumer.recv() => message,
- };
+ _ = tick.tick() => {}
+ received = consumer.consumer.recv() => match received {
+ Ok(message) => handle_message(&message, &settings, &teams, &mut batch),
+ Err(error) => {
+ tracing::warn!(%error, "kafka receive failed");
+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);
+ // A fatal error disables the client for good, so only a restart recovers.
+ if let Some((code, reason)) = consumer.consumer.client().fatal_error() {
+ handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));
+ return;
+ }
+ }
+ },
+ }
handle.report_healthy();
- let message = match message {
- Ok(message) => message,
- Err(error) => {
- tracing::warn!(%error, "kafka receive failed");
- counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);
- tokio::time::sleep(Duration::from_millis(100)).await;
- continue;
- }
- };
- let now_ms = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .map_or(0, |d| d.as_millis() as i64);
- let outcome = sampled(message.key(), percentage)
- .and_then(|()| {
- read(
- message.payload().unwrap_or_default(),
- header(&message, "content-encoding"),
- )
- })
- .and_then(|read| {
- let team = header(&message, "token").and_then(|token| teams.get(token));
- admit(&read, team, now_ms).map(|_| read)
- });
- match outcome {
- Ok(read) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")
- .increment(1);
- counter!("ai_research_v4_recorder_events_total")
- .increment(read.events.len() as u64);
- }
- Err(skip) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
- .increment(1)
- }
+ let revoked = consumer.take_revoked();
+ if !revoked.is_empty() {
+ batch.drop_partitions(&revoked);
+ inspectors.drop_partitions(&revoked);
+ }
+ if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {
+ let (blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));
+ flush(&blocks);
+ consumer.commit(&offsets);
+ inspectors.evict_idle(settings.session_idle);
+ }
+ }
+}
+
+fn handle_message(
+ message: &BorrowedMessage<'_>,
+ settings: &Settings,
+ teams: &TeamCache,
+ batch: &mut Batch,
+) {
+ let now_ms = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_or(0, |d| d.as_millis() as i64);
+ let admitted = sampled(message.key(), settings.sample_percentage)
+ .and_then(|()| {
+ read(
+ message.payload().unwrap_or_default(),
+ header(message, "content-encoding"),
+ )
+ })
+ .and_then(|read| {
+ let team = header(message, "token").and_then(|token| teams.get(token));
+ admit(&read, team, now_ms).map(|team_id| (team_id, read))
+ });
+ match admitted {
+ Ok((team_id, read)) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
+ .increment(1);
+ let session = SessionKey {
+ team_id,
+ session_id: read.session_id.clone(),
+ };
+ batch.add(message.partition(), message.offset(), session, read);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1);
+ batch.handled(message.partition(), message.offset());
+ }
+ }
+}
+
+/// Reports what a flush inspected. The writers that store blocks come in the next layers.
+fn flush(blocks: &[Block]) {
+ counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
+ for block in blocks {
+ counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);
+ if block.stats.cold_start {
+ counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);
}
- if let Err(error) = consumer.store_offset_from_message(&message) {
- tracing::warn!(%error, "storing an offset failed");
+ if block.stats.orphan_mutations > 0 {
+ counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);
}
}
}
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index ae01300ec013..f471229fe802 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -1,6 +1,7 @@
//! The v4 recorder: a Kafka consumer beside the replay lanes that records a sample of sessions for AI research,
//! without scrubbing them. The format is in `products/ai_training/docs/replay-data.md`, section "v4 dataset".
+pub mod batch;
pub mod config;
pub mod consumer;
pub mod process;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 239705b13325..0a7a1a40a6eb 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -12,7 +12,7 @@ use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter};
use replay_recorder_v4::config::Config;
-use replay_recorder_v4::consumer::{create_consumer, run};
+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings};
use replay_recorder_v4::teams::TeamCache;
common_alloc::used!();
@@ -100,18 +100,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
teams_handle,
));
- let consumer = create_consumer(
+ let consumer = RecorderConsumer::new(
&config.kafka,
&config.kafka_consumer_group,
&config.kafka_consumer_offset_reset,
&config.kafka_topic,
)?;
- tokio::spawn(run(
- consumer,
- config.sample_percentage,
- teams,
- consumer_handle,
- ));
+ let settings = Settings {
+ sample_percentage: config.sample_percentage,
+ flush_interval: Duration::from_millis(config.flush_interval_ms),
+ flush_max_bytes: config.flush_max_bytes,
+ session_idle: Duration::from_secs(config.session_idle_secs),
+ };
+ tokio::spawn(run(consumer, settings, teams, consumer_handle));
monitor.wait().await?;
Ok(())