:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): record kafka positions so readers can drop retried blocks (#118421)
Nicholas Waltz5f80da7fae58
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex c081e87a1a0d..ec0b29047134 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -1,6 +1,8 @@//! Messages buffered between flushes, and the per-session inspectors that outlive them.use std::collections::{BTreeMap, HashMap};+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::Arc;use std::time::{Duration, Instant};use futures::stream::{self, StreamExt};@@ -9,9 +11,10 @@ use replay_core_v4::compression::{GZIP_MAGIC, MAX_DECOMPRESSED_BYTES};use replay_core_v4::inspect::dom::UrlLimit;use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};use replay_core_v4::inspect::stats::BlockStats;-use replay_core_v4::scan::message::Message;+use replay_core_v4::scan::cv::{Expansion, MESSAGE_DECODE_BUDGET};+use replay_core_v4::scan::message::{Event, Message};-use crate::process::{check_read, read};+use crate::process::{check_read, read_within};#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct SessionKey {@@ -66,9 +69,23 @@ pub struct Span {pub last: i64,}+/// Where a block's messages came from. A failed flush commits nothing, so every later attempt for the partition+/// starts at the same `start_offset`. Readers use this to remove the rows of earlier attempts.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct Origin {+ pub tp: Tp,+ /// The first offset the flush handled on the partition.+ pub start_offset: i64,+ /// The block's last message.+ pub end_offset: i64,+ /// The last offset the flush handled on the partition, a skipped message included.+ pub flush_end_offset: i64,+}+/// What a flush hands to the writers.pub struct Block {pub session: SessionKey,+ pub origin: Origin,pub messages: Vec<Message>,pub stats: BlockStats,pub moments: Vec<Moment>,@@ -80,6 +97,45 @@ pub struct Block {pub url_limit: Option<UrlLimit>,}+/// The decoded `cv` strings one flush keeps, on top of the decoded payloads that the flush size counts.+const FLUSH_DECODE_BUDGET: usize = 512 * 1024 * 1024;++/// The decoded `cv` strings one flush expands into its blocks, kept or decoded again. Small gzip payloads can expand+/// a thousandfold, so this bounds the work, and the images, of such a flush. Later events stay as they arrived.+const FLUSH_EXPAND_BUDGET: usize = 2 * 1024 * 1024 * 1024;++/// The part of [`FLUSH_EXPAND_BUDGET`] one session may use. A session's block compresses on one thread, so this keeps+/// one session from taking longer than a flush attempt may.+const SESSION_EXPAND_BUDGET: usize = 256 * 1024 * 1024;++/// The parsed events one flush holds. The flush size counts the payloads, and an event as small as `{}` takes far more+/// memory parsed than in its payload, so a message that does not fit is skipped.+const FLUSH_EVENTS_BUDGET: usize = 1024 * 1024 * 1024;++struct DecodeBudget {+ kept: AtomicUsize,+ expanded: AtomicUsize,+ events: AtomicUsize,+}++impl DecodeBudget {+ fn new(kept: usize, expanded: usize) -> Self {+ Self {+ kept: AtomicUsize::new(kept),+ expanded: AtomicUsize::new(expanded),+ events: AtomicUsize::new(FLUSH_EVENTS_BUDGET),+ }+ }+}++fn take(budget: &AtomicUsize, bytes: usize) -> bool {+ budget+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |left| {+ left.checked_sub(bytes)+ })+ .is_ok()+}+pub struct Batch {pub sessions: BTreeMap<(SessionKey, Tp), Pending>,pub spans: HashMap<Tp, Span>,@@ -163,40 +219,106 @@ impl Batch {}}+struct Live {+ inspector: SessionInspector,+ partitions: Vec<Tp>,+ last_seen: Instant,+ bytes: usize,+}++impl Live {+ fn new(+ session: &SessionKey,+ inspector: SessionInspector,+ partitions: Vec<Tp>,+ last_seen: Instant,+ ) -> Self {+ let mut live = Self {+ inspector,+ partitions,+ last_seen,+ bytes: 0,+ };+ live.measure(session);+ live+ }++ fn measure(&mut self, session: &SessionKey) {+ self.bytes = std::mem::size_of::<(SessionKey, Live)>()+ + session.session_id.capacity()+ + self.partitions.capacity() * std::mem::size_of::<Tp>()+ + self.inspector.retained_bytes();+ }++ /// Keeps the session's activity and clicks, so its later blocks still count them.+ fn evict_dom(&mut self, session: &SessionKey) {+ self.inspector.evict_dom();+ self.measure(session);+ }+}+/// Session inspectors by session. They keep DOM and click state across flushes.-#[derive(Default)]pub struct Inspectors {- sessions: HashMap<SessionKey, (SessionInspector, Vec<Tp>, Instant)>,+ sessions: HashMap<SessionKey, Live>,+ max_bytes: usize,+}++impl Default for Inspectors {+ fn default() -> Self {+ Self::new(usize::MAX)+ }}impl Inspectors {+ pub fn new(max_bytes: usize) -> Self {+ Self {+ sessions: HashMap::new(),+ max_bytes,+ }+ }+/// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions- /// at once off the async runtime. Returns one block per session.+ /// at once off the async runtime. Returns one block per session and partition.pub async fn inspect(&mut self, batch: Batch, concurrency: usize) -> Vec<Block> {- let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();+ let mut work: BTreeMap<SessionKey, Vec<(Origin, Vec<Raw>)>> = BTreeMap::new();for ((session, tp), pending) in batch.sessions {- let (tps, raws) = by_session.entry(session).or_default();- tps.push(tp);- raws.extend(pending.messages);+ let Some(span) = batch.spans.get(&tp) else {+ continue;+ };+ let origin = Origin {+ tp,+ start_offset: span.first,+ end_offset: pending+ .messages+ .iter()+ .map(|m| m.offset)+ .max()+ .unwrap_or(span.first),+ flush_end_offset: span.last,+ };+ work.entry(session)+ .or_default()+ .push((origin, pending.messages));}- let tasks: Vec<_> = by_session+ let budget = Arc::new(DecodeBudget::new(FLUSH_DECODE_BUDGET, FLUSH_EXPAND_BUDGET));+ let tasks: Vec<_> = work.into_iter()- .map(|(session, (tps, raws))| {- let (inspector, prior) = match self.sessions.remove(&session) {- Some((inspector, partitions, last_seen)) => {- (inspector, Some((partitions, last_seen)))- }- None => (SessionInspector::default(), None),+ .map(|(session, parts)| {+ let (inspector, partitions, last_seen) = match self.sessions.remove(&session) {+ Some(live) => (live.inspector, live.partitions, Some(live.last_seen)),+ None => Default::default(),};- (session, inspector, prior, tps, raws)+ (session, inspector, partitions, last_seen, parts)}).collect();let done: Vec<_> = stream::iter(tasks)- .map(|(session, inspector, prior, tps, raws)| {- let (key, messages) = (session.clone(), raws.len());+ .map(|(session, inspector, partitions, last_seen, parts)| {+ let budget = budget.clone();+ let key = session.clone();+ let messages: usize = parts.iter().map(|(_, raws)| raws.len()).sum();let task = tokio::task::spawn_blocking(move || {- let (inspector, block) = inspect_session(&session, inspector, raws);- (session, inspector, prior, tps, block)+ let inspected = inspect_session(session, inspector, partitions, parts, &budget);+ (inspected, last_seen)});async move { (key, messages, task.await) }})@@ -207,7 +329,7 @@ impl Inspectors {let mut blocks = Vec::new();for (key, messages, result) in done {// A panic loses that session's block and its inspector state, and the rest of the flush goes on.- let (session, inspector, prior, tps, block) = match result {+ let ((session, inspector, partitions, session_blocks), last_seen) = match result {Ok(inspected) => inspected,Err(error) => {tracing::error!(%error, team_id = key.team_id, "inspecting a session panicked: its block is dropped");@@ -217,79 +339,225 @@ impl Inspectors {}};// A session that decoded nothing left its state as it was, and does not count as seen.- let Some(block) = block else {- if let Some((partitions, last_seen)) = prior {- self.sessions- .insert(session, (inspector, partitions, last_seen));- }- continue;+ let last_seen = if session_blocks.is_empty() {+ last_seen+ } else {+ Some(now)};- let mut partitions = prior.map(|(partitions, _)| partitions).unwrap_or_default();- for tp in tps {- if !partitions.contains(&tp) {- partitions.push(tp);- }+ if let Some(last_seen) = last_seen {+ let live = Live::new(&session, inspector, partitions, last_seen);+ self.sessions.insert(session, live);}- self.sessions.insert(session, (inspector, partitions, now));- blocks.push(block);+ blocks.extend(session_blocks);}- blocks.sort_by(|a, b| a.session.cmp(&b.session));+ self.keep_within_budget();+ blocks.sort_by(|a, b| (&a.session, a.origin.tp).cmp(&(&b.session, b.origin.tp)));blocks}pub fn drop_partitions(&mut self, revoked: &[Tp]) {self.sessions- .retain(|_, (_, partitions, _)| !partitions.iter().any(|tp| revoked.contains(tp)));+ .retain(|_, live| !live.partitions.iter().any(|tp| revoked.contains(tp)));}pub fn evict_idle(&mut self, idle: Duration) {self.sessions- .retain(|_, (_, _, last_seen)| last_seen.elapsed() < idle);+ .retain(|_, live| live.last_seen.elapsed() < idle);metrics::gauge!("ai_research_v4_recorder_live_sessions").set(self.sessions.len() as f64);+ metrics::gauge!("ai_research_v4_recorder_retained_bytes").set(self.retained_bytes() as f64);+ }++ pub fn retained_bytes(&self) -> usize {+ self.sessions.values().map(|live| live.bytes).sum()+ }++ /// Evicts the DOM state of the sessions seen least recently until all fit. An evicted session keeps a small entry+ /// that marks its later blocks unverifiable. Whole sessions go only when those entries alone do not fit.+ fn keep_within_budget(&mut self) {+ let mut total = self.retained_bytes();+ if total > self.max_bytes {+ let mut oldest: Vec<(Instant, SessionKey)> = self+ .sessions+ .iter()+ .map(|(session, live)| (live.last_seen, session.clone()))+ .collect();+ oldest.sort_unstable();+ for (_, session) in &oldest {+ if total <= self.max_bytes {+ break;+ }+ let live = self.sessions.get_mut(session).expect("listed above");+ let bytes = live.bytes;+ live.evict_dom(session);+ total = total - bytes + live.bytes;+ if live.bytes < bytes {+ counter!("ai_research_v4_recorder_retained_evictions_total", "kind" => "dom")+ .increment(1);+ }+ }+ for (_, session) in &oldest {+ if total <= self.max_bytes {+ break;+ }+ if let Some(live) = self.sessions.remove(session) {+ total -= live.bytes;+ counter!("ai_research_v4_recorder_retained_evictions_total", "kind" => "session")+ .increment(1);+ }+ }+ }+ metrics::gauge!("ai_research_v4_recorder_retained_bytes").set(total as f64);}}+/// `partitions` are those the inspector has seen the session on. Its state goes when any of them is revoked.fn inspect_session(- session: &SessionKey,+ session: SessionKey,mut inspector: SessionInspector,- raws: Vec<Raw>,-) -> (SessionInspector, Option<Block>) {- let mut messages = Vec::with_capacity(raws.len());- for raw in raws {- let message = read(raw.payload, raw.content_encoding.as_deref())- .and_then(|m| check_read(&m, &session.session_id).map(|()| m));- match message {- Ok(mut message) => {- // Decoded once here, for the inspector and the block writer both.- message.decode_compressed();- messages.push(message);- }- Err(skip) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())- .increment(1);+ mut partitions: Vec<Tp>,+ parts: Vec<(Origin, Vec<Raw>)>,+ budget: &DecodeBudget,+) -> (SessionKey, SessionInspector, Vec<Tp>, Vec<Block>) {+ let mut decoded = Vec::with_capacity(parts.len());+ let mut session_spent = 0;+ for (origin, raws) in parts {+ let mut messages = Vec::with_capacity(raws.len());+ for raw in raws {+ // A growing list can hold twice its events, so the reader stops at half of what the budget has room for.+ let max_events = budget.events.load(Ordering::Relaxed) / size_of::<Event>() / 2;+ let message = read_within(raw.payload, raw.content_encoding.as_deref(), max_events)+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));+ match message {+ Ok(mut message) => {+ if !take(+ &budget.events,+ message.buffered_bytes() - message.inner.len(),+ ) {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "over_budget")+ .increment(1);+ continue;+ }+ // Decoded once here, for the inspector and the block writer both.+ let flush_room = budget.expanded.load(Ordering::Relaxed);+ let session_room = SESSION_EXPAND_BUDGET.saturating_sub(session_spent);+ let spent = message.decode_compressed_within(flush_room.min(session_room));+ session_spent += spent;+ if session_room < flush_room {+ // The session, not the flush, ran out, so the flush's room stays for the other sessions.+ if message+ .events+ .iter()+ .any(|e| e.expansion == Some(Expansion::Skipped))+ {+ counter!("ai_research_v4_recorder_unexpanded_messages_total")+ .increment(1);+ }+ keep_within(&mut message, spent, usize::MAX, budget);+ } else {+ keep_within(&mut message, spent, flush_room, budget);+ }+ messages.push(message);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1);+ }}}+ if messages.is_empty() {+ continue;+ }+ if !partitions.contains(&origin.tp) {+ partitions.push(origin.tp);+ }+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")+ .increment(messages.len() as u64);+ decoded.push((origin, messages));+ }+ // A session can have messages on the main and the overflow topic in one flush. The inspector takes them in replay+ // order, so that a mutation never comes before the nodes it changes. Each still counts toward its partition's block.+ let mut inspections: Vec<BlockInspection> =+ decoded.iter().map(|_| BlockInspection::default()).collect();+ let times: Vec<Vec<f64>> = decoded+ .iter()+ .map(|(_, messages)| messages.iter().map(replay_time).collect())+ .collect();+ let mut next = vec![0; decoded.len()];+ while let Some(part) = (0..decoded.len())+ .filter(|&part| next[part] < times[part].len())+ .min_by(|&a, &b| times[a][next[a]].total_cmp(×[b][next[b]]))+ {+ inspector.observe_message(&mut inspections[part], &decoded[part].1[next[part]]);+ next[part] += 1;+ }+ let blocks = decoded+ .into_iter()+ .zip(inspections)+ .map(|((origin, messages), block)| {+ let inspected = block.finish();+ Block {+ session: session.clone(),+ origin,+ messages,+ stats: inspected.stats,+ moments: inspected.moments,+ image_urls: inspected.image_urls,+ remote_urls: inspected.remote_urls,+ url_limit: inspected.url_limit,+ }+ })+ .collect();+ (session, inspector, partitions, blocks)+}++/// When a message's events start. One without a timestamp goes as early as its partition's order allows.+fn replay_time(message: &Message) -> f64 {+ message+ .events+ .iter()+ .find_map(|e| e.timestamp_ms)+ .unwrap_or(f64::NEG_INFINITY)+}++/// Drops the message's decoded strings when the flush has no room left to keep them. The inspector and the block+/// writer then each decode them again, which costs CPU but keeps the flush's memory bounded. When the flush has no+/// room left to expand them at all, they stay as they arrived.+///+/// `spent` is what the decode used, failed events included, and `room` what the flush had left before it.+fn keep_within(message: &mut Message, spent: usize, room: usize, budget: &DecodeBudget) {+ // An event that did not fit in the flush's room uses the room up, so that each later message does not decode one+ // event and then stop.+ let overran = room < MESSAGE_DECODE_BUDGET+ && message+ .events+ .iter()+ .any(|e| e.expansion == Some(Expansion::Skipped));+ let charged = budget+ .expanded+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |left| {+ Some(if overran {+ 0+ } else {+ left.saturating_sub(spent)+ })+ })+ .is_ok_and(|left| left >= spent);+ if overran || !charged {+ counter!("ai_research_v4_recorder_unexpanded_messages_total").increment(1);}- if messages.is_empty() {- return (inspector, None);- }- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")- .increment(messages.len() as u64);- let mut block = BlockInspection::default();- for message in &messages {- inspector.observe_message(&mut block, message);- }- let inspected = block.finish();- let block = Block {- session: session.clone(),- messages,- stats: inspected.stats,- moments: inspected.moments,- image_urls: inspected.image_urls,- remote_urls: inspected.remote_urls,- url_limit: inspected.url_limit,+ let decoded = message.decoded_bytes();+ let replacement = if !charged {+ Some(Expansion::Skipped)+ } else if !take(&budget.kept, decoded) {+ None+ } else {+ return;};- (inspector, Some(block))+ for event in &mut message.events {+ if matches!(event.expansion, Some(Expansion::Decoded(_))) {+ event.expansion = replacement.clone();+ }+ }}#[cfg(test)]@@ -345,7 +613,11 @@ mod tests {}fn mouse() -> Value {- json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])+ mouse_at(3)+ }++ fn mouse_at(ts: i64) -> Value {+ json!([{"type": 3, "timestamp": ts, "data": {"source": 1, "positions": []}}])}#[tokio::test]@@ -390,22 +662,160 @@ mod tests {assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));}+ #[test]+ fn a_flush_without_room_keeps_its_expansions_bounded() {+ let cv = |data: &str| -> String {+ gzip(data.as_bytes())+ .unwrap()+ .iter()+ .map(|&b| char::from(b))+ .collect()+ };+ let data = json!({"node": {"type": 0, "childNodes": [], "id": 1}}).to_string();+ let n = data.len();+ // `room` is what the flush had left to expand when the session decoded the message.+ let kept = |data: &str, kept: usize, expanded: usize, room: usize| {+ let items = json!([{"type": 2, "timestamp": 1, "data": cv(data), "cv": "2024-10"}]);+ let mut message = crate::process::read(raw(1, "a", items).payload, None).unwrap();+ let budget = DecodeBudget::new(kept, expanded);+ let spent = message.decode_compressed_within(room);+ keep_within(&mut message, spent, room, &budget);+ let expansion = match &message.events[0].expansion {+ None => "decode again",+ Some(Expansion::Decoded(_)) => "kept",+ Some(Expansion::Skipped) => "as it arrived",+ Some(Expansion::Failed) => "failed",+ Some(other) => panic!("{other:?}"),+ };+ (+ expansion,+ budget.kept.into_inner(),+ budget.expanded.into_inner(),+ )+ };+ assert_eq!(kept(&data, n, n, n), ("kept", 0, 0));+ assert_eq!(kept(&data, n - 1, n, n), ("decode again", n - 1, 0));+ // An event that does not fit uses up the room, so that later messages do not each decode one event.+ assert_eq!(kept(&data, n, n + 1, n - 1), ("as it arrived", n, 0));+ // Another session took the room while this one decoded.+ assert_eq!(kept(&data, n, n - 1, n), ("as it arrived", n, 0));+ // A string that does not decode still used the room it decompressed.+ assert_eq!(kept("not json", n, n + 8, n + 8), ("failed", n, n));+ }+#[tokio::test]- async fn a_session_that_decodes_nothing_keeps_no_state() {+ async fn inspects_a_session_on_both_topics_in_replay_order() {+ let overflow = Tp {+ topic: "overflow",+ partition: 0,+ };let mut batch = Batch::default();- batch.add(- tp(0),- 0,- key("a"),- Raw {- offset: 1,- payload: b"not json".to_vec(),- content_encoding: None,- },+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));+ batch.add(overflow, 0, key("a"), raw(20, "a", snapshot()));+ let blocks = Inspectors::default().inspect(batch, 1).await;+ let cold: Vec<_> = blocks+ .iter()+ .map(|b| (b.origin.tp.topic, b.stats.cold_start))+ .collect();+ assert_eq!(cold, [("main", false), ("overflow", false)]);+ }++ #[tokio::test]+ async fn revoking_any_partition_a_session_was_seen_on_drops_its_state() {+ let mut inspectors = Inspectors::default();+ let mut first = Batch::default();+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));+ inspectors.inspect(first, 1).await;+ let mut second = Batch::default();+ second.add(tp(1), 0, key("a"), raw(1, "a", mouse()));+ inspectors.inspect(second, 1).await;++ inspectors.drop_partitions(&[tp(0)]);+ let mut third = Batch::default();+ third.add(tp(1), 0, key("a"), raw(2, "a", mouse()));+ let blocks = inspectors.inspect(third, 1).await;+ assert!(blocks[0].stats.cold_start);+ }++ #[tokio::test]+ async fn over_budget_the_session_seen_least_recently_loses_its_dom_state() {+ let flush = |items: &[(&str, Value)]| {+ let mut batch = Batch::default();+ for (offset, (session, value)) in items.iter().enumerate() {+ batch.add(+ tp(0),+ 0,+ key(session),+ raw(offset as i64, session, value.clone()),+ );+ }+ batch+ };+ let mut measure = Inspectors::default();+ measure.inspect(flush(&[("a", snapshot())]), 1).await;+ let one = measure.retained_bytes();++ let mut inspectors = Inspectors::new(one + one / 2);+ inspectors+ .inspect(flush(&[("a", snapshot()), ("a", mouse())]), 1)+ .await;+ inspectors.inspect(flush(&[("b", snapshot())]), 1).await;+ assert!(inspectors.retained_bytes() <= one + one / 2);++ let flags = |blocks: Vec<Block>| -> Vec<_> {+ blocks+ .iter()+ .map(|b| {+ (+ b.session.session_id.clone(),+ b.stats.cold_start,+ b.stats.node_cap_reached,+ b.stats.active_ms,+ )+ })+ .collect()+ };+ let blocks = inspectors+ .inspect(flush(&[("a", mouse_at(1003)), ("b", mouse_at(1003))]), 1)+ .await;+ // `a` lost the DOM its snapshot built, so its blocks do not claim to play until it sends a new one. It keeps+ // its activity, so its active time goes on from its mouse move at 3.+ assert_eq!(+ flags(blocks),+ [+ ("a".to_owned(), true, true, 999),+ ("b".to_owned(), false, false, 1)+ ]);+ inspectors.inspect(flush(&[("a", snapshot())]), 1).await;+ let blocks = inspectors.inspect(flush(&[("a", mouse())]), 1).await;+ assert_eq!(flags(blocks), [("a".to_owned(), false, false, 0)]);+ }++ #[tokio::test]+ async fn a_session_that_decodes_nothing_keeps_its_state_as_it_was() {+ let unreadable = |offset: i64| Raw {+ offset,+ payload: b"not json".to_vec(),+ content_encoding: None,+ };let mut inspectors = Inspectors::default();- assert!(inspectors.inspect(batch, 1).await.is_empty());- assert!(inspectors.sessions.is_empty());+ let mut first = Batch::default();+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));+ inspectors.inspect(first, 1).await;+ let retained = inspectors.retained_bytes();++ let mut second = Batch::default();+ second.add(tp(1), 0, key("a"), unreadable(1));+ second.add(tp(1), 0, key("b"), unreadable(2));+ assert!(inspectors.inspect(second, 1).await.is_empty());+ assert_eq!(inspectors.retained_bytes(), retained);++ // `a` was never recorded from partition 1, so its revoke leaves `a`'s DOM state.+ inspectors.drop_partitions(&[tp(1)]);+ let mut third = Batch::default();+ third.add(tp(0), 0, key("a"), raw(2, "a", mouse()));+ assert!(!inspectors.inspect(third, 1).await[0].stats.cold_start);}#[tokio::test]@@ -419,12 +829,22 @@ mod tests {let mut second = Batch::default();second.add(tp(0), 0, key("a"), raw(3, "a", mouse()));second.add(tp(0), 0, key("b"), raw(4, "b", mouse()));+ // A message the flush skips still counts towards the flush's end on the partition.+ second.handled(tp(0), 0, 5);let blocks = inspectors.inspect(second, 2).await;- let cold: Vec<_> = blocks+ let seen: Vec<_> = blocks.iter()- .map(|b| (b.session.session_id.as_str(), b.stats.cold_start))+ .map(|b| {+ (+ b.session.session_id.as_str(),+ b.stats.cold_start,+ b.origin.start_offset,+ b.origin.end_offset,+ b.origin.flush_end_offset,+ )+ }).collect();// `a` started with its snapshot in this process. `b` was already running when this process first saw it.- assert_eq!(cold, vec![("a", false), ("b", true)]);+ assert_eq!(seen, vec![("a", false, 3, 3, 5), ("b", true, 3, 4, 5)]);}}diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 7b8aa2ad33b4..b90da5183cf8 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -96,6 +96,14 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]pub session_idle_secs: u64,+ /// The inspector state all sessions keep between flushes. Past it the sessions seen least recently lose their DOM+ /// state. It only stops runaway growth, such as many huge pages at once, so keep it well above normal use.+ #[envconfig(+ from = "AI_RESEARCH_V4_SESSION_STATE_MAX_BYTES",+ default = "4294967296"+ )]+ pub session_state_max_bytes: usize,+/// Off, the recorder reads, admits and inspects, and stores nothing: no keys, no objects.#[envconfig(from = "AI_RESEARCH_V4_WRITE_ENABLED", default = "false")]pub write_enabled: bool,diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 37c8bc1e1537..43ef92e747cb 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -28,6 +28,10 @@ const KAFKA_TIMEOUT: Duration = Duration::from_secs(10);const COMMIT_ATTEMPTS: u32 = 3;const HEARTBEAT: Duration = Duration::from_secs(5);+/// The consumer's `session.timeout.ms`, the replay lanes' value. Past it the group can give the partitions to another+/// process, so a flush stops storing then.+pub const SESSION_TIMEOUT: Duration = Duration::from_secs(90);+/// The most messages read before the limiter decides on their sessions.const POLL_BATCH: usize = 1000;@@ -241,6 +245,10 @@ impl RecorderConsumer {.set("enable.auto.commit", "false").set("enable.auto.offset.store", "false").set("partition.assignment.strategy", "cooperative-sticky")+ .set(+ "session.timeout.ms",+ SESSION_TIMEOUT.as_millis().to_string(),+ )// `Config::validate` keeps every flush under this, so the group keeps the partitions while one runs..set("max.poll.interval.ms",@@ -393,6 +401,7 @@ pub struct Settings {pub flush_retries: u32,pub shutdown_flush: Duration,pub session_idle: Duration,+ pub session_state_max_bytes: usize,/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.pub redirecting_topic: Option<String>,}@@ -414,7 +423,7 @@ pub async fn run(let shutdown = handle.shutdown_signal();tokio::pin!(shutdown);let mut batch = Batch::default();- let mut inspectors = Inspectors::default();+ let mut inspectors = Inspectors::new(settings.session_state_max_bytes);let mut tick = tokio::time::interval(Duration::from_secs(1));loop {tokio::select! {@@ -673,12 +682,36 @@ async fn flush_steps(.await;report(&blocks);if let Some(writer) = writer {- let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();+ let mut blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();let work = Arc::new(FlushWork::default());+ let started = Instant::now();let mut attempt = 0;loop {+ // Readers keep the newest attempt of a flush, so an attempt after this process lost a partition would+ // win over the new owner's flush. It also loses the partitions when the group hears nothing for the+ // session timeout, so no attempt runs past it.+ let owned = still_owned(+ blocks.clone(),+ &offsets,+ consumer.consumer.assignment_lost(),+ |tp| consumer.current_epoch(tp),+ );+ if owned.is_empty() && !blocks.is_empty() {+ tracing::warn!(+ "the flush lost its partitions: nothing more is written or committed"+ );+ return Ok(());+ }+ blocks = owned;+ let left = SESSION_TIMEOUT.saturating_sub(started.elapsed());+ if left.is_zero() {+ return Err(format!(+ "the flush did not store its blocks within the session timeout of {SESSION_TIMEOUT:?}"+ ));+ }+ let timeout = settings.flush_timeout.min(left);let stored = tokio::time::timeout(- settings.flush_timeout,+ timeout,with_heartbeat(handle,write_flush(&blocks, writer, settings.flush_concurrency, &work),@@ -688,7 +721,7 @@ async fn flush_steps(let error = match stored {Ok(Ok(())) => break,Ok(Err(error)) => error.to_string(),- Err(_) => format!("timed out after {:?}", settings.flush_timeout),+ Err(_) => format!("timed out after {timeout:?}"),};counter!("ai_research_v4_recorder_flush_failures_total").increment(1);attempt += 1;@@ -704,6 +737,28 @@ async fn flush_steps(Ok(())}+/// The blocks this process may still write: none once the group took its assignment away, and otherwise those of the+/// partitions it still owns under the assignment the flush read them in.+fn still_owned(+ blocks: Vec<Arc<Block>>,+ offsets: &HashMap<Tp, (u64, i64)>,+ lost: bool,+ current_epoch: impl Fn(Tp) -> Option<u64>,+) -> Vec<Arc<Block>> {+ if lost {+ return Vec::new();+ }+ blocks+ .into_iter()+ .filter(|block| {+ let tp = block.origin.tp;+ offsets+ .get(&tp)+ .is_some_and(|&(epoch, _)| current_epoch(tp) == Some(epoch))+ })+ .collect()+}+fn report(blocks: &[Block]) {counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);for block in blocks {@@ -719,7 +774,10 @@ fn report(blocks: &[Block]) {#[cfg(test)]mod tests {+ use replay_core_v4::inspect::stats::BlockStats;+use super::*;+ use crate::batch::Origin;fn tp(partition: i32) -> Tp {Tp {@@ -747,4 +805,41 @@ mod tests {state.starts_committed(&starts, true, now);assert!(state.uncommitted.is_empty());}++ #[test]+ fn a_retry_writes_only_the_partitions_this_process_still_owns() {+ let block = |partition: i32| {+ Arc::new(Block {+ session: SessionKey {+ team_id: 7,+ session_id: "s".to_owned(),+ },+ origin: Origin {+ tp: tp(partition),+ start_offset: 0,+ end_offset: 0,+ flush_end_offset: 0,+ },+ messages: Vec::new(),+ stats: BlockStats::default(),+ moments: Vec::new(),+ image_urls: Vec::new(),+ remote_urls: Vec::new(),+ url_limit: None,+ })+ };+ let offsets = HashMap::from([(tp(0), (1, 5)), (tp(1), (1, 5)), (tp(2), (1, 5))]);+ // Partition 1 was assigned again, and partition 2 was revoked.+ let epochs = HashMap::from([(tp(0), 1), (tp(1), 2)]);+ let owned = |lost: bool| -> Vec<i32> {+ still_owned(vec![block(0), block(1), block(2)], &offsets, lost, |tp| {+ epochs.get(&tp).copied()+ })+ .iter()+ .map(|b| b.origin.tp.partition)+ .collect()+ };+ assert_eq!(owned(false), [0]);+ assert!(owned(true).is_empty());+ }}diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsindex 8fc9aa768c48..1261a3267ad0 100644--- a/rust/ai-research/replay-recorder-v4/src/flush.rs+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -1,5 +1,5 @@-//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are-//! unique.+//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are unique,+//! and the block index rows name their attempt.use std::collections::{BTreeMap, BTreeSet, HashMap};use std::sync::{Arc, Mutex, OnceLock, PoisonError};@@ -24,7 +24,7 @@ use crate::store::BlobStore;use crate::urls::{fetch_jobs, records};use crate::write::{compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,- write_block_index, write_blocks, write_images, CompressedBlock, WriteError,+ write_block_index, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,};/// How long a fetch job may wait for room in the producer queue.@@ -127,6 +127,7 @@ pub async fn write_flush(};drop(resolved);let month = key_month(&day);+ let attempt = FlushAttempt::new(now);let object_key = object_key(&day, now);let seals: Vec<_> = blocks@@ -139,12 +140,17 @@ pub async fn write_flush(};let key = keys.sessions.get(&id)?.clone();let team_key = *keys.team_months.get(&block.session.team_id)?;- let (block, month, work) = (block.clone(), month.to_owned(), work.clone());+ let (block, object_key, month, work) = (+ block.clone(),+ object_key.clone(),+ month.to_owned(),+ work.clone(),+ );Some(seal_blocking(&writer.seal_permits, move || {let compressed = work.compressed(Arc::as_ptr(&block) as usize, || {compress_block(&block, &key, &team_key, &month)})?;- seal_compressed(&compressed).map(|sealed| (index, sealed))+ seal_compressed(&compressed, &object_key).map(|sealed| (index, sealed))}))}).collect();@@ -243,6 +249,7 @@ pub async fn write_flush(write_block_index(&entries,&keys,+ &attempt,&writer.rrweb_bucket,writer.store.as_ref(),now,diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rsindex 39045a4c5f69..e0bbfd06d231 100644--- a/rust/ai-research/replay-recorder-v4/src/index.rs+++ b/rust/ai-research/replay-recorder-v4/src/index.rs@@ -1,6 +1,19 @@//! The v4 index files: Parquet in `rrweb-v4/index/`. Column names are the ones in `replay-data.md`.//!//! Integer columns are signed, the types parquetjs writes, so a parquetjs writer can produce the same files.+//!+//! A flush can store its blocks and index rows and then fail before it commits its Kafka offsets. The next attempt+//! for the partition, in this process or after a restart, starts again at the same committed offset and writes the+//! same messages again. Every block index row therefore names its flush attempt (`flush_id`, `flushed_at_ms`) and the+//! Kafka data it came from (`topic`, `partition`, `start_offset` of the flush on that partition, the block's own+//! `end_offset`). A retry can cover a different range than the attempts it replaces, so the rule for readers works on+//! ranges: per (`topic`, `partition`), a flush covers its `start_offset` to its highest `end_offset`. Walk the flushes+//! from the highest (`flushed_at_ms`, `flush_id`) and keep a flush only if its range overlaps no flush already kept.+//!+//! The rows of a flush can also arrive in part. Per (`flush_id`, `topic`, `partition`), every row holds the same+//! `flush_end_offset`, the last offset the flush covers there, and `flush_row_count`, the rows it writes there. The+//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on+//! sessions or teams.use std::sync::Arc;@@ -10,7 +23,8 @@ use arrow_schema::{ArrowError, DataType, Field, Fields, Schema};use replay_core_v4::inspect::session::Moment;use replay_core_v4::inspect::stats::BlockStats;-use crate::write::Written;+use crate::batch::Origin;+use crate::write::{FlushAttempt, Written};use parquet::arrow::ArrowWriter;use parquet::basic::Compression;use parquet::errors::ParquetError;@@ -99,12 +113,18 @@ pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {/// One block's index row: where it is, what the inspector found, and its sealed URLs.pub struct BlockIndexRow<'a> {pub written: &'a Written,+ pub origin: &'a Origin,+ pub attempt: &'a FlushAttempt,pub stats: &'a BlockStats,pub moments: &'a [Moment],pub snapshot_source: Option<&'a str>,pub snapshot_library: Option<&'a str>,/// The block's visited URLs as a v4 JSON envelope under the session key.pub urls: Option<String>,+ /// The block rows this flush writes for the row's partition, and the row's place among them. With `flush_id`,+ /// `topic` and `partition`, the place names the row across retries of the same publish.+ pub flush_row_count: u32,+ pub flush_row_index: u32,}fn moment_fields() -> Fields {@@ -211,6 +231,64 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexErrorutf8!("session_id", |r| r.written.session.session_id.clone());utf8!("key_month", |r| r.written.key_month.clone());utf8!("block_s3_key", |r| r.written.object_key.clone());+ utf8!("topic", |r| r.origin.tp.topic);+ push(+ "partition",+ DataType::Int32,+ false,+ Arc::new(Int32Array::from_iter_values(+ rows.iter().map(|r| r.origin.tp.partition),+ )),+ );+ push(+ "start_offset",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| r.origin.start_offset),+ )),+ );+ push(+ "end_offset",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| r.origin.end_offset),+ )),+ );+ utf8!("flush_id", |r| r.attempt.flush_id.clone());+ push(+ "flushed_at_ms",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| r.attempt.flushed_at_ms),+ )),+ );+ push(+ "flush_end_offset",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| r.origin.flush_end_offset),+ )),+ );+ push(+ "flush_row_count",+ DataType::Int32,+ false,+ Arc::new(Int32Array::from_iter_values(+ rows.iter().map(|r| int32(r.flush_row_count)),+ )),+ );+ push(+ "flush_row_index",+ DataType::Int32,+ false,+ Arc::new(Int32Array::from_iter_values(+ rows.iter().map(|r| int32(r.flush_row_index)),+ )),+ );push("block_byte_start",DataType::Int64,diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex eafc3fdcf7fe..d30f7fb97b72 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -160,6 +160,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {flush_retries: config.flush_retries,shutdown_flush: Duration::from_secs(config.shutdown_flush_secs),session_idle: Duration::from_secs(config.session_idle_secs),+ session_state_max_bytes: config.session_state_max_bytes,redirecting_topic: (config.ingestion_overflow_mode == "redirect").then(|| config.kafka_topic.clone()),};diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rsindex 272ef219a227..e8ac225a9475 100644--- a/rust/ai-research/replay-recorder-v4/src/process.rs+++ b/rust/ai-research/replay-recorder-v4/src/process.rs@@ -7,7 +7,7 @@ use std::borrow::Cow;use replay_core_v4::gate::in_sample;use replay_core_v4::payload::{decompress_payload, is_web_replay};-use replay_core_v4::scan::message::{read_message, Message};+use replay_core_v4::scan::message::{read_message_within, Message, ReadError};use replay_core_v4::session::{drop_reason, DropReason};use crate::teams::Team;@@ -29,6 +29,8 @@ pub enum Skip {Unreadable,SessionMismatch,NotWeb,+ /// The flush has no room left for the message's events.+ OverBudget,}impl Skip {@@ -48,6 +50,7 @@ impl Skip {Skip::Unreadable => "unreadable",Skip::SessionMismatch => "session_mismatch",Skip::NotWeb => "not_web",+ Skip::OverBudget => "over_budget",}}}@@ -145,8 +148,21 @@ pub fn admit(/// Decompresses and reads an admitted message.pub fn read(payload: Vec<u8>, content_encoding: Option<&str>) -> Result<Message, Skip> {+ read_within(payload, content_encoding, usize::MAX)+}++/// As [`read`], with at most `max_events` events, so that a message of many tiny events stops before its event list+/// outgrows the flush's budget.+pub fn read_within(+ payload: Vec<u8>,+ content_encoding: Option<&str>,+ max_events: usize,+) -> Result<Message, Skip> {let raw = decompress_payload(payload, content_encoding).map_err(|_| Skip::Undecompressable)?;- read_message(raw).map_err(|_| Skip::Unreadable)+ read_message_within(raw, max_events).map_err(|error| match error {+ ReadError::TooManyEvents => Skip::OverBudget,+ _ => Skip::Unreadable,+ })}/// The rules that need the payload: the session the headers named, and a web replay.@@ -213,6 +229,12 @@ mod tests {read(b"not json".to_vec(), None).unwrap_err(),Skip::Unreadable);+ // The flush's event budget stops the reader before the event list grows past it.+ assert_eq!(+ read_within(plain.clone(), None, 0).unwrap_err(),+ Skip::OverBudget+ );+ assert!(read_within(plain, None, 1).is_ok());}#[test]diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rsindex 2396007f4458..af25d0355791 100644--- a/rust/ai-research/replay-recorder-v4/src/urls.rs+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs@@ -162,7 +162,7 @@ mod tests {use replay_core_v4::keys::batch::SessionKey as Key;use super::*;- use crate::batch::SessionKey;+ use crate::batch::{Origin, SessionKey, Tp};#[test]fn a_full_sent_cache_still_keeps_new_marks() {@@ -192,6 +192,15 @@ mod tests {team_id: 7,session_id: "s".to_owned(),},+ origin: Origin {+ tp: Tp {+ topic: "main",+ partition: 0,+ },+ start_offset: 0,+ end_offset: 0,+ flush_end_offset: 0,+ },messages: Vec::new(),stats: BlockStats::default(),moments: Vec::new(),diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex 4e811f3ce2b3..056f7a419b12 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -3,7 +3,7 @@//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block//! whose images were never stored would hold references that resolve to nothing.-use std::collections::{BTreeMap, HashSet};+use std::collections::{BTreeMap, HashMap, HashSet};use std::io::{self, Write};use std::sync::Arc;@@ -24,7 +24,7 @@ use replay_core_v4::seal::{image_content_key, url_fingerprint_key};use tokio::sync::Semaphore;-use crate::batch::{Block, SessionKey};+use crate::batch::{Block, SessionKey, Tp};use crate::index::{block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,};@@ -196,14 +196,19 @@ fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {.map_err(seal_error)}-/// Builds, compresses and seals one block.+/// Builds, compresses and seals one block for `object_key`. The key is in the frame's authenticated data, so a+/// frame does not open from another object, even one of the same session.pub fn seal_block(block: &Block,key: &Key,team_key: &[u8; 32],key_month: &str,+ object_key: &str,) -> Result<SealedBlock, WriteError> {- seal_compressed(&compress_block(block, key, team_key, key_month)?)+ seal_compressed(+ &compress_block(block, key, team_key, key_month)?,+ object_key,+ )}/// Builds and compresses one block. This is most of the work of a flush, so a retry does not do it again.@@ -235,12 +240,15 @@ pub fn compress_block(})}-/// Seals a compressed block for one attempt.-pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteError> {+/// Seals a compressed block for one attempt's `object_key`.+pub fn seal_compressed(+ block: &CompressedBlock,+ object_key: &str,+) -> Result<SealedBlock, WriteError> {let frame = seal_frame(&block.key.key,Version::V4,- &block_context(&block.session, &block.key.key_month),+ &block_context(&block.session, &block.key.key_month, object_key),&block.body,).map_err(|e| WriteError::Seal(e.to_string()))?;@@ -255,13 +263,14 @@ pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteErro}/// The authenticated data of a block frame.-pub fn block_context(session: &SessionKey, key_month: &str) -> Context {+pub fn block_context(session: &SessionKey, key_month: &str, object_key: &str) -> Context {Context {team_id: session.team_id,session_id: Some(session.session_id.clone()),key_month: Some(key_month.to_owned()),kind: "rrweb".to_owned(),codec: Some(Codec::Brotli),+ reference: Some(object_key.to_owned()),..Context::default()}}@@ -474,11 +483,41 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>fingerprints}+/// Numbers the rows of each partition in order and gives each row the count of its partition.+fn number_rows(rows: &mut [BlockIndexRow<'_>]) {+ let mut counts: HashMap<Tp, u32> = HashMap::new();+ for row in rows.iter_mut() {+ let count = counts.entry(row.origin.tp).or_default();+ row.flush_row_index = *count;+ *count += 1;+ }+ for row in rows.iter_mut() {+ row.flush_row_count = counts[&row.origin.tp];+ }+}++/// Which flush attempt wrote a block index row. See `crate::index` for how readers use it.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct FlushAttempt {+ pub flush_id: String,+ pub flushed_at_ms: i64,+}++impl FlushAttempt {+ pub fn new(at: DateTime<Utc>) -> Self {+ Self {+ flush_id: format!("{:016x}", rand::thread_rng().gen::<u64>()),+ flushed_at_ms: at.timestamp_millis(),+ }+ }+}+/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored./// Each entry is a block, its stats as this attempt filled them, and where it landed.pub async fn write_block_index(entries: &[(&Block, &BlockStats, &Written)],keys: &FlushKeys,+ attempt: &FlushAttempt,rrweb_bucket: &str,store: &dyn BlobStore,at: DateTime<Utc>,@@ -514,16 +553,21 @@ pub async fn write_block_index(let first = block.messages.first();rows.push(BlockIndexRow {written,+ origin: &block.origin,+ attempt,stats,moments: &block.moments,snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),snapshot_library: first.and_then(|m| m.library.as_deref()),urls,+ flush_row_count: 0,+ flush_row_index: 0,});}if rows.is_empty() {return Ok(0);}+ number_rows(&mut rows);let index_key = format!("index/blocks/dt={}/{}.parquet",write_day(at),@@ -558,6 +602,7 @@ mod tests {use serde_json::{json, Value};use super::*;+ use crate::batch::{Origin, Tp};#[derive(Default)]struct MemoryStore {@@ -611,6 +656,18 @@ mod tests {}}+ fn origin(partition: i32, start_offset: i64, end_offset: i64) -> Origin {+ Origin {+ tp: Tp {+ topic: "main",+ partition,+ },+ start_offset,+ end_offset,+ flush_end_offset: end_offset,+ }+ }+fn block(session: &str, events: Value) -> Block {block_of(session, &events.to_string())}@@ -631,6 +688,7 @@ mod tests {team_id: 7,session_id: session.to_owned(),},+ origin: origin(0, 100, 105),messages: vec![message],stats: BlockStats::default(),moments: Vec::new(),@@ -780,6 +838,35 @@ mod tests {.with_timezone(&Utc)}+ #[test]+ fn block_context_reproduces_the_shared_vector() {+ // The Node writers and readers in other languages check themselves against the same file.+ let vector: Value = serde_json::from_str(include_str!(+ "../../../../nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json"+ ))+ .unwrap();+ let block = &vector["block"];+ let context = &block["context"];+ let session = SessionKey {+ team_id: context["teamId"].as_i64().unwrap(),+ session_id: context["sessionId"].as_str().unwrap().to_owned(),+ };+ let built = block_context(+ &session,+ context["keyMonth"].as_str().unwrap(),+ context["ref"].as_str().unwrap(),+ );+ assert_eq!(+ built.authenticated_data(Version::V4),+ block["aad"].as_str().unwrap()+ );+ let body = STANDARD.decode(block["body"].as_str().unwrap()).unwrap();+ assert_eq!(+ decompress(&body).unwrap(),+ STANDARD.decode(block["plain"].as_str().unwrap()).unwrap()+ );+ }+#[test]fn keeps_one_copy_of_an_image_that_several_messages_repeat() {let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"logo")}}}}]);@@ -841,6 +928,7 @@ mod tests {&keys.sessions[&session_key(&b.session.session_id).0],&[6; 32],"2026-10",+ &object,).unwrap()})@@ -895,7 +983,7 @@ mod tests {let block = &written[0];let frame = &memory.get("rrweb-v4", &block.object_key)[block.byte_start as usize..=block.byte_end as usize];- let context = block_context(&blocks[0].session, "2026-09");+ let context = block_context(&blocks[0].session, "2026-09", &object);let lines = String::from_utf8(decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap(),)@@ -908,6 +996,9 @@ mod tests {),(lines.len() as u64, longest as u64));+ // The frame is bound to its object: it does not open as a block of another one.+ let elsewhere = block_context(&blocks[0].session, "2026-09", "blocks/dt=2026-10-15/other");+ assert!(open_frame(&[5; 32], Version::V4, &elsewhere, frame).is_err());let first: Value = serde_json::from_str(lines.lines().nth(1).unwrap()).unwrap();let reference = first[1]["data"]["node"]["attributes"]["src"].as_str()@@ -951,6 +1042,7 @@ mod tests {write_block_index(&[(&blocks[0], &stats, &written[0])],&keys,+ &FlushAttempt::new(at()),"rrweb-v4",store.as_ref(),at()@@ -1062,4 +1154,169 @@ mod tests {]);}++ #[tokio::test]+ async fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {+ let store = MemoryStore::default();+ let mut keys = FlushKeys::default();+ for session in ["a", "b", "c"] {+ let (id, key) = session_key(session);+ keys.sessions.insert(id, key);+ }+ let mouse = json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}]);+ let stats = BlockStats::default();+ let written = |session: &str| Written {+ session: SessionKey {+ team_id: 7,+ session_id: session.to_owned(),+ },+ key_month: "2026-09".to_owned(),+ object_key: format!("blocks/{session}"),+ byte_start: 0,+ byte_end: 9,+ };+ // Two attempts wrote their rows and failed before the commit: offsets 100-105 and 106-110 of partition 0.+ // The retry starts at the committed offset 100, reads both ranges in one flush, and also writes partition 1.+ let a = block("a", mouse.clone());+ let mut early_b = block("b", mouse.clone());+ early_b.origin = origin(0, 106, 110);+ let mut retried_a = block("a", mouse.clone());+ retried_a.origin.flush_end_offset = 110;+ let mut b = block("b", mouse.clone());+ b.origin = origin(0, 100, 110);+ let mut c = block("c", mouse);+ c.origin = origin(1, 40, 41);+ let (wa, wb, wc) = (written("a"), written("b"), written("c"));+ let first = FlushAttempt {+ flush_id: "0000000000000001".to_owned(),+ flushed_at_ms: 1_000,+ };+ let between = FlushAttempt {+ flush_id: "0000000000000003".to_owned(),+ flushed_at_ms: 1_500,+ };+ let second = FlushAttempt {+ flush_id: "0000000000000002".to_owned(),+ flushed_at_ms: 2_000,+ };+ for (attempt, entries) in [+ (&first, vec![(&a, &stats, &wa)]),+ (&between, vec![(&early_b, &stats, &wb)]),+ (+ &second,+ vec![+ (&retried_a, &stats, &wa),+ (&b, &stats, &wb),+ (&c, &stats, &wc),+ ],+ ),+ ] {+ write_block_index(&entries, &keys, attempt, "rrweb-v4", &store, at())+ .await+ .unwrap();+ }++ let mut rows = Vec::new();+ let mut positions = Vec::new();+ for key in store.keys("index/blocks/") {+ let batch = store.parquet(&key);+ let text = |name: &str| {+ batch+ .column_by_name(name)+ .unwrap()+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap()+ .clone()+ };+ let int = |name: &str| {+ batch+ .column_by_name(name)+ .unwrap()+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .clone()+ };+ let int32 = |name: &str| {+ batch+ .column_by_name(name)+ .unwrap()+ .as_any()+ .downcast_ref::<arrow_array::Int32Array>()+ .unwrap()+ .clone()+ };+ let partition = int32("partition");+ let (topic, session, flush_id) = (text("topic"), text("session_id"), text("flush_id"));+ let (start, end, at_ms) =+ (int("start_offset"), int("end_offset"), int("flushed_at_ms"));+ let flush_end = int("flush_end_offset");+ let (count, index) = (int32("flush_row_count"), int32("flush_row_index"));+ for i in 0..batch.num_rows() {+ positions.push((+ flush_id.value(i).to_owned(),+ partition.value(i),+ flush_end.value(i),+ count.value(i),+ index.value(i),+ ));+ rows.push((+ (topic.value(i).to_owned(), partition.value(i)),+ (start.value(i), end.value(i)),+ (at_ms.value(i), flush_id.value(i).to_owned()),+ session.value(i).to_owned(),+ ));+ }+ }+ assert_eq!(rows.len(), 5);+ // Each partition of a flush numbers its rows, so a reader can tell that all of them arrived.+ positions.sort();+ let retry: Vec<_> = positions+ .iter()+ .filter(|p| p.0 == "0000000000000002")+ .map(|p| (p.1, p.2, p.3, p.4))+ .collect();+ assert_eq!(retry, [(0, 110, 2, 0), (0, 110, 2, 1), (1, 41, 1, 0)]);++ // The documented rule, per (topic, partition): a flush covers its start offset to its highest end offset.+ // Walk the flushes from the latest and keep one only if it overlaps no flush already kept.+ type Flush = ((String, i32), (i64, String));+ let mut ranges: HashMap<Flush, (i64, i64)> = HashMap::new();+ for (tp, (start, end), flush, _) in &rows {+ let range = ranges+ .entry((tp.clone(), flush.clone()))+ .or_insert((*start, *end));+ range.1 = range.1.max(*end);+ }+ let mut flushes: Vec<_> = ranges.into_iter().collect();+ flushes.sort_by(|a, b| b.0 .1.cmp(&a.0 .1));+ let mut kept_flushes: Vec<(Flush, (i64, i64))> = Vec::new();+ for (flush, range) in flushes {+ let overlaps = kept_flushes+ .iter()+ .any(|(k, r)| k.0 == flush.0 && range.0 <= r.1 && r.0 <= range.1);+ if !overlaps {+ kept_flushes.push((flush, range));+ }+ }+ let mut kept: Vec<_> = rows+ .iter()+ .filter(|(tp, _, flush, _)| {+ kept_flushes+ .iter()+ .any(|(k, _)| k.0 == *tp && k.1 == *flush)+ })+ .map(|(_, _, flush, session)| (session.as_str(), flush.1.as_str()))+ .collect();+ kept.sort();+ assert_eq!(+ kept,+ [+ ("a", "0000000000000002"),+ ("b", "0000000000000002"),+ ("c", "0000000000000002")+ ]+ );+ }}