Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): consume the replay overflow topic (#118418)

Nicholas Waltz69c46215a41b
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index 4781fdec116f..337a77c92348 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -17,6 +17,13 @@ pub struct SessionKey {
pub session_id: String,
}
+/// A topic partition. The recorder reads a fixed set of topics, so the name is static.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct Tp {
+ pub topic: &'static str,
+ pub partition: i32,
+}
+
/// An admitted message, not decoded yet.
pub struct Raw {
pub offset: i64,
@@ -24,9 +31,9 @@ pub struct Raw {
pub content_encoding: Option<String>,
}
-/// One session's messages in the current flush. Capture keys messages by session, so a session has one partition.
+/// One session's messages from one partition in the current flush. Capture keys messages by session, so a session
+/// has one partition per topic.
pub struct Pending {
- pub partition: i32,
pub messages: Vec<Raw>,
}
@@ -45,9 +52,9 @@ pub struct Block {
}
pub struct Batch {
- pub sessions: BTreeMap<SessionKey, Pending>,
+ pub sessions: BTreeMap<(SessionKey, Tp), Pending>,
/// The last offset handled on each partition, recorded or skipped alike.
- pub offsets: HashMap<i32, i64>,
+ pub offsets: HashMap<Tp, i64>,
bytes: usize,
started: Instant,
}
@@ -65,18 +72,17 @@ impl Default for Batch {
impl Batch {
/// Marks an offset handled. A skipped message still moves its partition forward.
- pub fn handled(&mut self, partition: i32, offset: i64) {
- let last = self.offsets.entry(partition).or_insert(offset);
+ pub fn handled(&mut self, tp: Tp, offset: i64) {
+ let last = self.offsets.entry(tp).or_insert(offset);
*last = (*last).max(offset);
}
- pub fn add(&mut self, partition: i32, session: SessionKey, raw: Raw) {
- self.handled(partition, raw.offset);
+ pub fn add(&mut self, tp: Tp, session: SessionKey, raw: Raw) {
+ self.handled(tp, raw.offset);
self.bytes += raw.payload.len();
self.sessions
- .entry(session)
+ .entry((session, tp))
.or_insert_with(|| Pending {
- partition,
messages: Vec::new(),
})
.messages
@@ -88,11 +94,10 @@ impl Batch {
}
/// Forgets partitions this process no longer owns. Their messages are not committed, so the new owner reads them.
- pub fn drop_partitions(&mut self, revoked: &[i32]) {
- self.sessions
- .retain(|_, pending| !revoked.contains(&pending.partition));
- for partition in revoked {
- self.offsets.remove(partition);
+ pub fn drop_partitions(&mut self, revoked: &[Tp]) {
+ self.sessions.retain(|(_, tp), _| !revoked.contains(tp));
+ for tp in revoked {
+ self.offsets.remove(tp);
}
self.bytes = self
.sessions
@@ -106,17 +111,23 @@ impl Batch {
/// Session inspectors by session. They keep DOM and click state across flushes.
#[derive(Default)]
pub struct Inspectors {
- sessions: HashMap<SessionKey, (SessionInspector, i32, Instant)>,
+ sessions: HashMap<SessionKey, (SessionInspector, Vec<Tp>, Instant)>,
}
impl Inspectors {
/// Decodes every buffered message, runs it through its session's inspector and returns one block per session.
- pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<i32, i64>) {
+ pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<Tp, i64>) {
let now = Instant::now();
- let mut blocks = Vec::with_capacity(batch.sessions.len());
- for (session, pending) in batch.sessions {
- let mut messages = Vec::with_capacity(pending.messages.len());
- for raw in pending.messages {
+ let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();
+ for ((session, tp), pending) in batch.sessions {
+ let (tps, raws) = by_session.entry(session).or_default();
+ tps.push(tp);
+ raws.extend(pending.messages);
+ }
+ let mut blocks = Vec::with_capacity(by_session.len());
+ for (session, (tps, raws)) in by_session {
+ let mut messages = Vec::with_capacity(raws.len());
+ for raw in raws {
let message = read(raw.payload, raw.content_encoding.as_deref())
.and_then(|m| check_read(&m, &session.session_id).map(|()| m));
match message {
@@ -137,11 +148,15 @@ impl Inspectors {
counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
.increment(messages.len() as u64);
// Only a session that decoded something counts as seen.
- let (inspector, partition, last_seen) = self
+ let (inspector, partitions, last_seen) = self
.sessions
.entry(session.clone())
- .or_insert_with(|| (SessionInspector::default(), pending.partition, now));
- *partition = pending.partition;
+ .or_insert_with(|| (SessionInspector::default(), Vec::new(), now));
+ for tp in tps {
+ if !partitions.contains(&tp) {
+ partitions.push(tp);
+ }
+ }
*last_seen = now;
let mut block = BlockInspection::default();
for message in &messages {
@@ -161,9 +176,9 @@ impl Inspectors {
(blocks, batch.offsets)
}
- pub fn drop_partitions(&mut self, revoked: &[i32]) {
+ pub fn drop_partitions(&mut self, revoked: &[Tp]) {
self.sessions
- .retain(|_, (_, partition, _)| !revoked.contains(partition));
+ .retain(|_, (_, partitions, _)| !partitions.iter().any(|tp| revoked.contains(tp)));
}
pub fn evict_idle(&mut self, idle: Duration) {
@@ -190,6 +205,13 @@ mod tests {
}
}
+ fn tp(partition: i32) -> Tp {
+ Tp {
+ topic: "main",
+ partition,
+ }
+ }
+
fn key(session: &str) -> SessionKey {
SessionKey {
team_id: 7,
@@ -208,12 +230,12 @@ mod tests {
#[test]
fn a_revoked_partition_leaves_nothing_behind() {
let mut batch = Batch::default();
- batch.add(0, key("a"), raw(10, "a", mouse()));
- batch.add(1, key("b"), raw(20, "b", mouse()));
- batch.handled(1, 21);
+ batch.add(tp(0), key("a"), raw(10, "a", mouse()));
+ batch.add(tp(1), key("b"), raw(20, "b", mouse()));
+ batch.handled(tp(1), 21);
let mut inspectors = Inspectors::default();
- batch.drop_partitions(&[1]);
- inspectors.drop_partitions(&[1]);
+ batch.drop_partitions(&[tp(1)]);
+ inspectors.drop_partitions(&[tp(1)]);
let (blocks, offsets) = inspectors.inspect(batch);
assert_eq!(
@@ -223,14 +245,14 @@ mod tests {
.collect::<Vec<_>>(),
vec!["a"]
);
- assert_eq!(offsets, HashMap::from([(0, 10)]));
+ assert_eq!(offsets, HashMap::from([(tp(0), 10)]));
}
#[test]
fn a_session_that_decodes_nothing_keeps_no_state() {
let mut batch = Batch::default();
batch.add(
- 0,
+ tp(0),
key("a"),
Raw {
offset: 1,
@@ -247,13 +269,13 @@ mod tests {
fn session_state_carries_across_flushes() {
let mut inspectors = Inspectors::default();
let mut first = Batch::default();
- first.add(0, key("a"), raw(1, "a", snapshot()));
- first.add(0, key("b"), raw(2, "b", mouse()));
+ first.add(tp(0), key("a"), raw(1, "a", snapshot()));
+ first.add(tp(0), key("b"), raw(2, "b", mouse()));
inspectors.inspect(first);
let mut second = Batch::default();
- second.add(0, key("a"), raw(3, "a", mouse()));
- second.add(0, key("b"), raw(4, "b", mouse()));
+ second.add(tp(0), key("a"), raw(3, "a", mouse()));
+ second.add(tp(0), key("b"), raw(4, "b", mouse()));
let (blocks, _) = inspectors.inspect(second);
let cold: Vec<_> = blocks
.iter()
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 711a32566547..6c0892ffc75a 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -9,6 +9,10 @@ pub struct Config {
#[envconfig(default = "session_recording_snapshot_item_events")]
pub kafka_topic: String,
+ /// Capture sends replay-limited and force-overflow sessions here. Empty turns it off.
+ #[envconfig(default = "session_recording_snapshot_item_overflow")]
+ pub kafka_overflow_topic: String,
+
/// The main replay lane's overflow mode. With `redirect`, that lane moves force-overflow messages of the main topic
/// to the overflow topic, so v4 records them there and not twice.
#[envconfig(from = "INGESTION_OVERFLOW_MODE", default = "disabled")]
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index a3f7f1224a03..961c618fb02c 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -1,6 +1,6 @@
//! The consume loop: gate and admit each message from its key and headers, buffer it, and flush on time or size.
-use std::collections::{BTreeSet, HashSet};
+use std::collections::{BTreeMap, HashSet};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
@@ -22,7 +22,7 @@ use replay_core_v4::keys::kms::AwsKms;
use replay_core_v4::keys::rows::{key_month, write_day};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
-use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey};
+use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};
use crate::limits::{Restrictions, SessionLimiter};
use crate::process::{admit, sampled, Admitted, Headers, Skip};
use crate::store::BlobStore;
@@ -38,7 +38,15 @@ const POLL_BATCH: usize = 1000;
/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.
pub struct RebalanceContext {
- revoked: Arc<Mutex<Vec<i32>>>,
+ topics: Vec<&'static str>,
+ revoked: Arc<Mutex<Vec<Tp>>>,
+}
+
+impl RebalanceContext {
+ fn tp(&self, topic: &str, partition: i32) -> Option<Tp> {
+ let topic = *self.topics.iter().find(|t| **t == topic)?;
+ Some(Tp { topic, partition })
+ }
}
impl ClientContext for RebalanceContext {}
@@ -46,27 +54,34 @@ impl ClientContext for RebalanceContext {}
impl ConsumerContext for RebalanceContext {
fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {
if let Rebalance::Revoke(partitions) = rebalance {
- self.revoked
- .lock()
- .unwrap()
- .extend(partitions.elements().iter().map(|e| e.partition()));
+ let revoked = partitions
+ .elements()
+ .iter()
+ .filter_map(|e| self.tp(e.topic(), e.partition()))
+ .collect::<Vec<_>>();
+ self.revoked.lock().unwrap().extend(revoked);
}
}
}
pub struct RecorderConsumer {
consumer: StreamConsumer<RebalanceContext>,
- topic: String,
- revoked: Arc<Mutex<Vec<i32>>>,
+ revoked: Arc<Mutex<Vec<Tp>>>,
}
impl RecorderConsumer {
+ /// `topics` are leaked so a partition is a cheap `Copy` key. There are at most two, created once.
pub fn new(
kafka: &KafkaConfig,
group: &str,
offset_reset: &str,
- topic: &str,
+ topics: &[String],
) -> Result<Self, rdkafka::error::KafkaError> {
+ let topics: Vec<&'static str> = topics
+ .iter()
+ .filter(|t| !t.is_empty())
+ .map(|t| &*Box::leak(t.clone().into_boxed_str()))
+ .collect();
let mut config = ClientConfig::new();
config
.set("bootstrap.servers", &kafka.kafka_hosts)
@@ -87,30 +102,32 @@ impl RecorderConsumer {
let revoked = Arc::new(Mutex::new(Vec::new()));
let consumer: StreamConsumer<RebalanceContext> =
config.create_with_context(RebalanceContext {
+ topics: topics.clone(),
revoked: revoked.clone(),
})?;
- consumer.subscribe(&[topic])?;
- Ok(Self {
- consumer,
- topic: topic.to_owned(),
- revoked,
- })
+ consumer.subscribe(&topics)?;
+ Ok(Self { consumer, revoked })
+ }
+
+ /// The main topic comes first. The overflow lane's limiter only counts, so v4's does the same there.
+ fn is_main(&self, tp: Tp) -> bool {
+ self.consumer.context().topics.first() == Some(&tp.topic)
}
- fn take_revoked(&self) -> Vec<i32> {
+ fn take_revoked(&self) -> Vec<Tp> {
std::mem::take(&mut *self.revoked.lock().unwrap())
}
/// Commits the next offset of every partition this process still owns.
- fn commit(&self, offsets: &std::collections::HashMap<i32, i64>) {
+ fn commit(&self, offsets: &std::collections::HashMap<Tp, i64>) {
let assigned = self.consumer.assignment().unwrap_or_default();
let mut list = TopicPartitionList::new();
- for (&partition, &offset) in offsets {
- if assigned.find_partition(&self.topic, partition).is_some() {
+ for (&tp, &offset) in offsets {
+ if assigned.find_partition(tp.topic, tp.partition).is_some() {
if let Err(error) =
- list.add_partition_offset(&self.topic, partition, Offset::Offset(offset + 1))
+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset + 1))
{
- tracing::warn!(%error, partition, "building the offset commit failed");
+ tracing::warn!(%error, partition = tp.partition, "building the offset commit failed");
}
}
}
@@ -257,25 +274,34 @@ async fn handle_messages(
) {
let mut admitted = Vec::with_capacity(messages.len());
for message in messages {
- let session = admit_message(message, &consumer.topic, settings, sources)
+ let Some(tp) = consumer
+ .consumer
+ .context()
+ .tp(message.topic(), message.partition())
+ else {
+ continue;
+ };
+ let session = admit_message(message, tp, settings, sources)
.await
.map(|admitted| SessionKey {
team_id: admitted.team_id,
session_id: admitted.session_id,
});
- admitted.push((message, session));
+ admitted.push((message, tp, session));
}
// The limiter decides before anything is buffered, so a blocked session's messages never reach a block.
- let sessions: BTreeSet<SessionKey> = admitted
- .iter()
- .filter_map(|(_, session)| session.as_ref().ok().cloned())
- .collect();
+ let mut sessions: BTreeMap<SessionKey, bool> = BTreeMap::new();
+ for (_, tp, session) in &admitted {
+ if let Ok(session) = session {
+ *sessions.entry(session.clone()).or_default() |= consumer.is_main(*tp);
+ }
+ }
let blocked = if sessions.is_empty() {
HashSet::new()
} else {
sources.limiter.blocked(&sessions).await
};
- for (message, session) in admitted {
+ for (message, tp, session) in admitted {
let session = session.and_then(|session| {
if blocked.contains(&session) {
Err(Skip::Blocked)
@@ -292,12 +318,12 @@ async fn handle_messages(
payload: message.payload().unwrap_or_default().to_vec(),
content_encoding: header(message, "content-encoding").map(str::to_owned),
};
- batch.add(message.partition(), session, raw);
+ batch.add(tp, session, raw);
}
Err(skip) => {
counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
.increment(1);
- batch.handled(message.partition(), message.offset());
+ batch.handled(tp, message.offset());
}
}
}
@@ -306,7 +332,7 @@ async fn handle_messages(
/// Everything before the payload: the gate on the Kafka key, then the headers, restrictions and team.
async fn admit_message(
message: &BorrowedMessage<'_>,
- topic: &str,
+ tp: Tp,
settings: &Settings,
sources: &Sources,
) -> Result<Admitted, Skip> {
@@ -323,7 +349,7 @@ async fn admit_message(
};
let restriction = sources.restrictions.get(&headers, now_ms / 1000).await;
let team = headers.token.and_then(|token| sources.teams.get(token));
- let redirect_overflow = settings.redirecting_topic.as_deref() == Some(topic);
+ let redirect_overflow = settings.redirecting_topic.as_deref() == Some(tp.topic);
admit(&headers, restriction, redirect_overflow, team, now_ms)
}
diff --git a/rust/ai-research/replay-recorder-v4/src/limits.rs b/rust/ai-research/replay-recorder-v4/src/limits.rs
index 42cac33a05f1..91bf67c0324b 100644
--- a/rust/ai-research/replay-recorder-v4/src/limits.rs
+++ b/rust/ai-research/replay-recorder-v4/src/limits.rs
@@ -2,7 +2,7 @@
//! own new-session rate limiter with the main lane's rules.
use std::borrow::Cow;
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -165,14 +165,18 @@ impl Buckets {
}
}
-/// The main lane's new-session rate limiter, run by v4 for itself. Each team's new sessions take a token from its
-/// bucket, and a session denied one is blocked as a whole, so v4 never keeps half of it.
+/// The main lane's new-session rate limiter, run by v4 for itself. Each team's new sessions on the main topic take a
+/// token from its bucket, and a session denied one is blocked as a whole, so v4 never keeps half of it.
+///
+/// The replay overflow lane enforces the same blocklist, but its own limiter only counts. v4 does the same: a session
+/// first seen on the overflow topic is never blocked for its own rate. Seen and blocked state is shared by both topics.
///
/// Redis errors fail open, as in the main lane: a session whose state is unknown is not blocked, and a block that
/// is not written holds in this process.
pub struct SessionLimiter {
redis: Arc<dyn Client + Send + Sync>,
- buckets: Mutex<Buckets>,
+ /// The main topic's buckets, then the overflow topic's, which only count.
+ buckets: Mutex<[Buckets; 2]>,
blocking_enabled: bool,
seen: moka::sync::Cache<SessionKey, ()>,
blocked: moka::sync::Cache<SessionKey, bool>,
@@ -209,25 +213,26 @@ impl SessionLimiter {
) -> Self {
Self {
redis,
- buckets: Mutex::new(Buckets {
+ buckets: Mutex::new(std::array::from_fn(|_| Buckets {
capacity,
replenish_per_sec,
teams: HashMap::new(),
- }),
+ })),
blocking_enabled,
seen: local_cache(),
blocked: local_cache(),
}
}
- /// Returns which of a poll batch's sessions are blocked. A session is new when it is neither seen nor blocked,
- /// and only a new session takes a token, so a blocked session does not drain its team's bucket.
- pub async fn blocked(&self, sessions: &BTreeSet<SessionKey>) -> HashSet<SessionKey> {
+ /// Returns which of a poll batch's sessions are blocked. Each session maps to true when a message of it came from
+ /// the main topic. A session is new when it is neither seen nor blocked, and only a new session takes a token, so
+ /// a blocked session does not drain its team's bucket.
+ pub async fn blocked(&self, sessions: &BTreeMap<SessionKey, bool>) -> HashSet<SessionKey> {
let deadline = tokio::time::Instant::now() + LIMITER_DEADLINE;
let mut blocked = HashSet::new();
let mut seen = HashSet::new();
let mut reads: Vec<(String, bool, &SessionKey)> = Vec::new();
- for session in sessions {
+ for session in sessions.keys() {
if self.seen.contains_key(session) {
seen.insert(session);
} else {
@@ -267,14 +272,14 @@ impl SessionLimiter {
let mut writes = Vec::new();
{
let mut buckets = self.buckets.lock().unwrap();
- for session in sessions {
+ for (session, &main) in sessions {
if seen.contains(session) || blocked.contains(session) || unknown.contains(session)
{
continue;
}
counter!("ai_research_v4_recorder_new_sessions_total").increment(1);
- let blocking = self.blocking_enabled;
- let allowed = buckets.take(session.team_id, now);
+ let blocking = main && self.blocking_enabled;
+ let allowed = buckets[usize::from(!main)].take(session.team_id, now);
if !allowed {
let outcome = if blocking { "blocked" } else { "counted" };
counter!("ai_research_v4_recorder_rate_limited_sessions_total", "outcome" => outcome)
@@ -376,8 +381,8 @@ mod tests {
}
}
- fn on(sessions: &[SessionKey]) -> BTreeSet<SessionKey> {
- sessions.iter().cloned().collect()
+ fn on(main: bool, sessions: &[SessionKey]) -> BTreeMap<SessionKey, bool> {
+ sessions.iter().map(|s| (s.clone(), main)).collect()
}
fn sorted(blocked: HashSet<SessionKey>) -> Vec<String> {
@@ -411,11 +416,14 @@ mod tests {
.map(|i| session(7, &format!("s{i}")))
.chain([session(8, "other")])
.collect();
- assert_eq!(sorted(limiter.blocked(&on(&flood)).await), ["s2", "s3"]);
+ assert_eq!(
+ sorted(limiter.blocked(&on(true, &flood)).await),
+ ["s2", "s3"]
+ );
// Later messages: a blocked session stays blocked, and a seen one takes no token from the empty bucket.
let later = [session(7, "s0"), session(7, "s3")];
- assert_eq!(sorted(limiter.blocked(&on(&later)).await), ["s3"]);
+ assert_eq!(sorted(limiter.blocked(&on(true, &later)).await), ["s3"]);
let writes: Vec<String> = redis
.get_calls()
@@ -437,6 +445,17 @@ mod tests {
);
}
+ #[tokio::test(start_paused = true)]
+ async fn the_overflow_topic_only_counts_but_drops_blocked_sessions() {
+ let limiter = SessionLimiter::new(Arc::new(MockRedisClient::new()), 1.0, 0.0, true);
+ let flood: Vec<SessionKey> = (0..3).map(|i| session(7, &format!("o{i}"))).collect();
+ assert!(limiter.blocked(&on(false, &flood)).await.is_empty());
+
+ let main = [session(7, "m0"), session(7, "m1")];
+ assert_eq!(sorted(limiter.blocked(&on(true, &main)).await), ["m1"]);
+ assert_eq!(sorted(limiter.blocked(&on(false, &main)).await), ["m1"]);
+ }
+
#[tokio::test(start_paused = true)]
async fn reads_the_state_other_processes_wrote() {
let mut redis = MockRedisClient::new();
@@ -451,7 +470,7 @@ mod tests {
];
// Neither known session takes the one token, so the first new one gets it.
assert_eq!(
- sorted(limiter.blocked(&on(&sessions)).await),
+ sorted(limiter.blocked(&on(true, &sessions)).await),
["blocked", "new"]
);
}
@@ -463,7 +482,10 @@ mod tests {
"the connection closed in a reply",
));
let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, true);
- assert!(limiter.blocked(&on(&[session(7, "a")])).await.is_empty());
+ assert!(limiter
+ .blocked(&on(true, &[session(7, "a")]))
+ .await
+ .is_empty());
let ops: Vec<_> = redis.get_calls().into_iter().map(|c| c.op).collect();
assert_eq!(ops, ["mget", "heal"]);
@@ -471,7 +493,7 @@ mod tests {
let redis = MockRedisClient::new().pipeline_block(Duration::from_secs(60));
let limiter = SessionLimiter::new(Arc::new(redis), 0.0, 0.0, true);
let started = tokio::time::Instant::now();
- let a = on(&[session(7, "a")]);
+ let a = on(true, &[session(7, "a")]);
assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);
assert_eq!(started.elapsed(), LIMITER_DEADLINE);
assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);
@@ -482,7 +504,7 @@ mod tests {
let redis = MockRedisClient::new();
let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, false);
assert!(limiter
- .blocked(&on(&[session(7, "a"), session(7, "b")]))
+ .blocked(&on(true, &[session(7, "a"), session(7, "b")]))
.await
.is_empty());
assert!(redis
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 18774fc8b630..0f574ba168ef 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -35,7 +35,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.from_env_lossy(),
)
.init();
- tracing::info!(topic = %config.kafka_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");
+ tracing::info!(topic = %config.kafka_topic, overflow_topic = %config.kafka_overflow_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");
let mut manager = Manager::builder("replay-recorder-v4")
.with_global_shutdown_timeout(Duration::from_secs(30))
@@ -143,7 +143,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
&config.kafka,
&config.kafka_consumer_group,
&config.kafka_consumer_offset_reset,
- &config.kafka_topic,
+ &[
+ config.kafka_topic.clone(),
+ config.kafka_overflow_topic.clone(),
+ ],
)?;
let settings = Settings {
sample_percentage: config.sample_percentage,