:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay): limit the decoded bytes of one v4 recorder poll (#127327)
Nicholas Waltzcb01b7f53ff0
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex a5f791928dc0..cbae593f95a4 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -38,22 +38,26 @@ pub struct Raw {pub content_encoding: Option<String>,}+/// The bytes a payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer states,+/// or fails, so the flush size counts that instead of the compressed bytes.+pub fn decoded_len(payload: &[u8], content_encoding: Option<&str>) -> usize {+ let stated = if content_encoding == Some("lz4") {+ payload.get(..4)+ } else if payload.starts_with(&GZIP_MAGIC) {+ payload.get(payload.len().saturating_sub(4)..)+ } else {+ None+ };+ let stated = stated+ .and_then(|b| <[u8; 4]>::try_from(b).ok())+ .map(|b| u32::from_le_bytes(b) as usize)+ .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);+ payload.len().max(stated.unwrap_or(0))+}+impl Raw {- /// The bytes the payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer- /// states, or fails, so the flush size counts that instead of the compressed bytes.pub fn decoded_len(&self) -> usize {- let stated = if self.content_encoding.as_deref() == Some("lz4") {- self.payload.get(..4)- } else if self.payload.starts_with(&GZIP_MAGIC) {- self.payload.get(self.payload.len().saturating_sub(4)..)- } else {- None- };- let stated = stated- .and_then(|b| <[u8; 4]>::try_from(b).ok())- .map(|b| u32::from_le_bytes(b) as usize)- .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);- self.payload.len().max(stated.unwrap_or(0))+ decoded_len(&self.payload, self.content_encoding.as_deref())}}@@ -203,6 +207,10 @@ impl Batch {.push(raw);}+ pub fn room(&self, max_bytes: usize) -> usize {+ max_bytes.saturating_sub(self.bytes)+ }+pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {!self.spans.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex c552ceb3c7f2..d8806d5e26ad 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -17,7 +17,7 @@ use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};use rdkafka::types::RDKafkaRespErr;use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};-use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};+use crate::batch::{decoded_len, Batch, Block, Inspectors, Raw, SessionKey, Tp};use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};use crate::flush::{write_flush, FlushWork, Writer, PRODUCER_FLUSH_ON_SHUTDOWN};use crate::limits::{Restrictions, SessionLimiter};@@ -392,6 +392,29 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {.and_then(|v| std::str::from_utf8(v).ok())}+/// A flush holds every decompressed payload until it ends and its size is only checked between polls, so one poll of+/// small compressed messages must not decompress to more than the `room` left in the buffer.+fn take_ready<M, E>(+ first: M,+ room: usize,+ decoded_len: impl Fn(&M) -> usize,+ mut next: impl FnMut() -> Option<Result<M, E>>,+) -> (Vec<M>, Option<E>) {+ let mut room = room.saturating_sub(decoded_len(&first));+ let mut messages = vec![first];+ while messages.len() < POLL_BATCH && room > 0 {+ match next() {+ Some(Ok(message)) => {+ room = room.saturating_sub(decoded_len(&message));+ messages.push(message);+ }+ Some(Err(error)) => return (messages, Some(error)),+ None => break,+ }+ }+ (messages, None)+}+pub struct Settings {pub sample_percentage: f64,pub flush_interval: Duration,@@ -446,18 +469,17 @@ async fn consume(received = consumer.consumer.recv() => match received {Ok(message) => {// The messages Kafka has ready go through the session limiter together, in one Redis round trip.- let mut messages = vec![message];- let mut failed = None;- while messages.len() < POLL_BATCH {- match consumer.consumer.recv().now_or_never() {- Some(Ok(message)) => messages.push(message),- Some(Err(error)) => {- failed = Some(error);- break;- }- None => break,- }- }+ let (messages, failed) = take_ready(+ message,+ batch.room(settings.flush_max_bytes),+ |message| {+ decoded_len(+ message.payload().unwrap_or_default(),+ header(message, "content-encoding"),+ )+ },+ || consumer.consumer.recv().now_or_never(),+ );handle_messages(&messages, consumer, settings, sources, &mut batch).await;drop(messages);if let Some(error) = failed {@@ -801,6 +823,22 @@ mod tests {}}+ #[test]+ fn a_poll_stops_taking_messages_once_they_fill_the_room() {+ let mut ready = vec![10usize; 5].into_iter();+ let (messages, failed) =+ take_ready::<usize, ()>(10, 35, |len| *len, || ready.next().map(Ok));+ assert_eq!(messages.len(), 4);+ assert_eq!(failed, None);+ assert_eq!(ready.len(), 2);++ // A full buffer still takes the first message, which `recv` has already removed from the queue.+ let mut ready = vec![10usize; 5].into_iter();+ let (messages, _) = take_ready::<usize, ()>(10, 0, |len| *len, || ready.next().map(Ok));+ assert_eq!(messages.len(), 1);+ assert_eq!(ready.len(), 5);+ }+#[test]fn a_failed_start_commit_stays_pending_at_the_first_offset() {let mut state = Assignments::default();