Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay): limit the decoded bytes of one v4 recorder poll (#127327)

Nicholas Waltzcb01b7f53ff0
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index a5f791928dc0..cbae593f95a4 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -38,22 +38,26 @@ pub struct Raw {
pub content_encoding: Option<String>,
}
+/// The bytes a payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer states,
+/// or fails, so the flush size counts that instead of the compressed bytes.
+pub fn decoded_len(payload: &[u8], content_encoding: Option<&str>) -> usize {
+ let stated = if content_encoding == Some("lz4") {
+ payload.get(..4)
+ } else if payload.starts_with(&GZIP_MAGIC) {
+ payload.get(payload.len().saturating_sub(4)..)
+ } else {
+ None
+ };
+ let stated = stated
+ .and_then(|b| <[u8; 4]>::try_from(b).ok())
+ .map(|b| u32::from_le_bytes(b) as usize)
+ .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);
+ payload.len().max(stated.unwrap_or(0))
+}
+
impl Raw {
- /// The bytes the payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer
- /// states, or fails, so the flush size counts that instead of the compressed bytes.
pub fn decoded_len(&self) -> usize {
- let stated = if self.content_encoding.as_deref() == Some("lz4") {
- self.payload.get(..4)
- } else if self.payload.starts_with(&GZIP_MAGIC) {
- self.payload.get(self.payload.len().saturating_sub(4)..)
- } else {
- None
- };
- let stated = stated
- .and_then(|b| <[u8; 4]>::try_from(b).ok())
- .map(|b| u32::from_le_bytes(b) as usize)
- .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);
- self.payload.len().max(stated.unwrap_or(0))
+ decoded_len(&self.payload, self.content_encoding.as_deref())
}
}
@@ -203,6 +207,10 @@ impl Batch {
.push(raw);
}
+ pub fn room(&self, max_bytes: usize) -> usize {
+ max_bytes.saturating_sub(self.bytes)
+ }
+
pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {
!self.spans.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)
}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index c552ceb3c7f2..d8806d5e26ad 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -17,7 +17,7 @@ use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};
use rdkafka::types::RDKafkaRespErr;
use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
-use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};
+use crate::batch::{decoded_len, Batch, Block, Inspectors, Raw, SessionKey, Tp};
use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};
use crate::flush::{write_flush, FlushWork, Writer, PRODUCER_FLUSH_ON_SHUTDOWN};
use crate::limits::{Restrictions, SessionLimiter};
@@ -392,6 +392,29 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {
.and_then(|v| std::str::from_utf8(v).ok())
}
+/// A flush holds every decompressed payload until it ends and its size is only checked between polls, so one poll of
+/// small compressed messages must not decompress to more than the `room` left in the buffer.
+fn take_ready<M, E>(
+ first: M,
+ room: usize,
+ decoded_len: impl Fn(&M) -> usize,
+ mut next: impl FnMut() -> Option<Result<M, E>>,
+) -> (Vec<M>, Option<E>) {
+ let mut room = room.saturating_sub(decoded_len(&first));
+ let mut messages = vec![first];
+ while messages.len() < POLL_BATCH && room > 0 {
+ match next() {
+ Some(Ok(message)) => {
+ room = room.saturating_sub(decoded_len(&message));
+ messages.push(message);
+ }
+ Some(Err(error)) => return (messages, Some(error)),
+ None => break,
+ }
+ }
+ (messages, None)
+}
+
pub struct Settings {
pub sample_percentage: f64,
pub flush_interval: Duration,
@@ -446,18 +469,17 @@ async fn consume(
received = consumer.consumer.recv() => match received {
Ok(message) => {
// The messages Kafka has ready go through the session limiter together, in one Redis round trip.
- let mut messages = vec![message];
- let mut failed = None;
- while messages.len() < POLL_BATCH {
- match consumer.consumer.recv().now_or_never() {
- Some(Ok(message)) => messages.push(message),
- Some(Err(error)) => {
- failed = Some(error);
- break;
- }
- None => break,
- }
- }
+ let (messages, failed) = take_ready(
+ message,
+ batch.room(settings.flush_max_bytes),
+ |message| {
+ decoded_len(
+ message.payload().unwrap_or_default(),
+ header(message, "content-encoding"),
+ )
+ },
+ || consumer.consumer.recv().now_or_never(),
+ );
handle_messages(&messages, consumer, settings, sources, &mut batch).await;
drop(messages);
if let Some(error) = failed {
@@ -801,6 +823,22 @@ mod tests {
}
}
+ #[test]
+ fn a_poll_stops_taking_messages_once_they_fill_the_room() {
+ let mut ready = vec![10usize; 5].into_iter();
+ let (messages, failed) =
+ take_ready::<usize, ()>(10, 35, |len| *len, || ready.next().map(Ok));
+ assert_eq!(messages.len(), 4);
+ assert_eq!(failed, None);
+ assert_eq!(ready.len(), 2);
+
+ // A full buffer still takes the first message, which `recv` has already removed from the queue.
+ let mut ready = vec![10usize; 5].into_iter();
+ let (messages, _) = take_ready::<usize, ()>(10, 0, |len| *len, || ready.next().map(Ok));
+ assert_eq!(messages.len(), 1);
+ assert_eq!(ready.len(), 5);
+ }
+
#[test]
fn a_failed_start_commit_stays_pending_at_the_first_offset() {
let mut state = Assignments::default();