Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): apply ingestion restrictions and the session filter (#118417)

Nicholas Waltz2b24e2e03f2d
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index fda74543d84a..e871da13c7a6 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11065,12 +11065,15 @@ dependencies = [
"base64 0.22.1",
"brotlic",
"bytes",
+ "capture",
"chrono",
"common-alloc",
"common-database",
"common-kafka",
"common-metrics",
+ "common-redis",
"envconfig",
+ "futures",
"lifecycle",
"metrics",
"moka",
diff --git a/rust/affected-services/tests/integration.rs b/rust/affected-services/tests/integration.rs
index 57351a49d8ed..af4e9a1c4143 100644
--- a/rust/affected-services/tests/integration.rs
+++ b/rust/affected-services/tests/integration.rs
@@ -254,7 +254,12 @@ fn leaf_service_only_affects_itself_and_dependents() {
assert_eq!(result.directly_changed, vec!["capture"]);
assert_eq!(
result.images,
- vec!["capture", "capture-apm-metrics", "capture-logs"]
+ vec![
+ "capture",
+ "capture-apm-metrics",
+ "capture-logs",
+ "replay-recorder-v4"
+ ]
);
}
diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.toml
index 097e412359d0..ed045fbc5545 100644
--- a/rust/ai-research/replay-recorder-v4/Cargo.toml
+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml
@@ -9,10 +9,13 @@ description = "Records the v4 AI research replay dataset: sampled, unscrubbed, s
workspace = true
[dependencies]
+# The event ingestion restrictions capture and the replay lanes apply.
+capture = { path = "../../capture" }
common-alloc = { path = "../../common/alloc" }
common-database = { path = "../../common/database" }
common-kafka = { path = "../../common/kafka" }
common-metrics = { path = "../../common/metrics" }
+common-redis = { path = "../../common/redis" }
lifecycle = { path = "../../common/lifecycle" }
replay-core-v4 = { path = "../replay-core-v4" }
@@ -30,6 +33,7 @@ axum = { workspace = true }
brotlic = "0.8"
chrono = { workspace = true }
envconfig = { workspace = true }
+futures = { workspace = true }
metrics = { workspace = true }
moka = { workspace = true }
parquet = "58"
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index a3f353c8eb32..4781fdec116f 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -3,21 +3,31 @@
use std::collections::{BTreeMap, HashMap};
use std::time::{Duration, Instant};
+use metrics::counter;
use replay_core_v4::inspect::dom::UrlLimit;
use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};
use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::scan::message::Message;
+use crate::process::{check_read, read};
+
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct SessionKey {
pub team_id: i64,
pub session_id: String,
}
+/// An admitted message, not decoded yet.
+pub struct Raw {
+ pub offset: i64,
+ pub payload: Vec<u8>,
+ pub content_encoding: Option<String>,
+}
+
/// One session's messages in the current flush. Capture keys messages by session, so a session has one partition.
pub struct Pending {
pub partition: i32,
- pub messages: Vec<Message>,
+ pub messages: Vec<Raw>,
}
/// What a flush hands to the writers.
@@ -60,11 +70,9 @@ impl Batch {
*last = (*last).max(offset);
}
- pub fn add(&mut self, partition: i32, offset: i64, session: SessionKey, mut message: Message) {
- self.handled(partition, offset);
- // Decoded once here, for the inspector and the block writer both, and counted toward the flush size.
- message.decode_compressed();
- self.bytes += message.buffered_bytes();
+ pub fn add(&mut self, partition: i32, session: SessionKey, raw: Raw) {
+ self.handled(partition, raw.offset);
+ self.bytes += raw.payload.len();
self.sessions
.entry(session)
.or_insert_with(|| Pending {
@@ -72,7 +80,7 @@ impl Batch {
messages: Vec::new(),
})
.messages
- .push(message);
+ .push(raw);
}
pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {
@@ -90,7 +98,7 @@ impl Batch {
.sessions
.values()
.flat_map(|p| &p.messages)
- .map(Message::buffered_bytes)
+ .map(|m| m.payload.len())
.sum();
}
}
@@ -102,11 +110,33 @@ pub struct Inspectors {
}
impl Inspectors {
- /// Runs every buffered message through its session's inspector and returns one block per session.
+ /// Decodes every buffered message, runs it through its session's inspector and returns one block per session.
pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<i32, i64>) {
let now = Instant::now();
let mut blocks = Vec::with_capacity(batch.sessions.len());
for (session, pending) in batch.sessions {
+ let mut messages = Vec::with_capacity(pending.messages.len());
+ for raw in pending.messages {
+ let message = read(raw.payload, raw.content_encoding.as_deref())
+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));
+ match message {
+ Ok(mut message) => {
+ // Decoded once here, for the inspector and the block writer both.
+ message.decode_compressed();
+ messages.push(message);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1);
+ }
+ }
+ }
+ if messages.is_empty() {
+ continue;
+ }
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
+ .increment(messages.len() as u64);
+ // Only a session that decoded something counts as seen.
let (inspector, partition, last_seen) = self
.sessions
.entry(session.clone())
@@ -114,13 +144,13 @@ impl Inspectors {
*partition = pending.partition;
*last_seen = now;
let mut block = BlockInspection::default();
- for message in &pending.messages {
+ for message in &messages {
inspector.observe_message(&mut block, message);
}
let inspected = block.finish();
blocks.push(Block {
session,
- messages: pending.messages,
+ messages,
stats: inspected.stats,
moments: inspected.moments,
image_urls: inspected.image_urls,
@@ -145,19 +175,19 @@ impl Inspectors {
#[cfg(test)]
mod tests {
- use replay_core_v4::scan::message::read_message;
use serde_json::{json, Value};
use super::*;
- fn message(session: &str, items: Value) -> Message {
- let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_items": items}});
- read_message(
- json!({"distinct_id": "d", "data": inner.to_string()})
+ fn raw(offset: i64, session: &str, items: Value) -> Raw {
+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});
+ Raw {
+ offset,
+ payload: json!({"distinct_id": "d", "data": inner.to_string()})
.to_string()
.into_bytes(),
- )
- .unwrap()
+ content_encoding: None,
+ }
}
fn key(session: &str) -> SessionKey {
@@ -178,8 +208,8 @@ mod tests {
#[test]
fn a_revoked_partition_leaves_nothing_behind() {
let mut batch = Batch::default();
- batch.add(0, 10, key("a"), message("a", mouse()));
- batch.add(1, 20, key("b"), message("b", mouse()));
+ batch.add(0, key("a"), raw(10, "a", mouse()));
+ batch.add(1, key("b"), raw(20, "b", mouse()));
batch.handled(1, 21);
let mut inspectors = Inspectors::default();
batch.drop_partitions(&[1]);
@@ -196,17 +226,34 @@ mod tests {
assert_eq!(offsets, HashMap::from([(0, 10)]));
}
+ #[test]
+ fn a_session_that_decodes_nothing_keeps_no_state() {
+ let mut batch = Batch::default();
+ batch.add(
+ 0,
+ key("a"),
+ Raw {
+ offset: 1,
+ payload: b"not json".to_vec(),
+ content_encoding: None,
+ },
+ );
+ let mut inspectors = Inspectors::default();
+ assert!(inspectors.inspect(batch).0.is_empty());
+ assert!(inspectors.sessions.is_empty());
+ }
+
#[test]
fn session_state_carries_across_flushes() {
let mut inspectors = Inspectors::default();
let mut first = Batch::default();
- first.add(0, 1, key("a"), message("a", snapshot()));
- first.add(0, 2, key("b"), message("b", mouse()));
+ first.add(0, key("a"), raw(1, "a", snapshot()));
+ first.add(0, key("b"), raw(2, "b", mouse()));
inspectors.inspect(first);
let mut second = Batch::default();
- second.add(0, 3, key("a"), message("a", mouse()));
- second.add(0, 4, key("b"), message("b", mouse()));
+ second.add(0, key("a"), raw(3, "a", mouse()));
+ second.add(0, key("b"), raw(4, "b", mouse()));
let (blocks, _) = inspectors.inspect(second);
let cold: Vec<_> = blocks
.iter()
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 985ae8c10789..711a32566547 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -9,6 +9,11 @@ pub struct Config {
#[envconfig(default = "session_recording_snapshot_item_events")]
pub kafka_topic: String,
+ /// The main replay lane's overflow mode. With `redirect`, that lane moves force-overflow messages of the main topic
+ /// to the overflow topic, so v4 records them there and not twice.
+ #[envconfig(from = "INGESTION_OVERFLOW_MODE", default = "disabled")]
+ pub ingestion_overflow_mode: String,
+
#[envconfig(default = "ai-research-v4-recorder")]
pub kafka_consumer_group: String,
@@ -23,6 +28,35 @@ pub struct Config {
/// The Postgres database the team list is read from. A read replica is enough.
pub database_url: String,
+ /// The Redis the event ingestion restrictions are read from, as capture and the replay lanes read them.
+ pub event_restrictions_redis_url: String,
+
+ #[envconfig(
+ from = "AI_RESEARCH_V4_EVENT_RESTRICTIONS_REFRESH_SECS",
+ default = "30"
+ )]
+ pub event_restrictions_refresh_secs: u64,
+
+ /// The session recording Redis. v4's new-session rate limiter keeps its seen and blocked sessions there, under its
+ /// own keys.
+ #[envconfig(from = "AI_RESEARCH_V4_REDIS_URL")]
+ pub redis_url: String,
+
+ /// The new-session rate limiter's bucket per team, as in the main lane: this many new sessions at once, and this
+ /// many more each second, scaled by the sample. See [`Config::new_session_bucket`].
+ #[envconfig(from = "AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", default = "3000")]
+ pub new_session_bucket_capacity: f64,
+
+ #[envconfig(
+ from = "AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE",
+ default = "300"
+ )]
+ pub new_session_bucket_replenish_rate: f64,
+
+ /// Off, the limiter only counts the sessions it would block.
+ #[envconfig(from = "AI_RESEARCH_V4_NEW_SESSION_BLOCKING_ENABLED", default = "true")]
+ pub new_session_blocking_enabled: bool,
+
/// How often the team list refreshes. Consent changes take effect within this time, as in v3.
#[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]
pub team_refresh_secs: u64,
@@ -63,6 +97,83 @@ pub struct Config {
)]
pub image_fetch_topic: String,
+ /// A fetch job not delivered in this time fails its flush.
+ #[envconfig(
+ from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",
+ default = "20000"
+ )]
+ pub image_fetch_delivery_timeout_ms: u32,
+
#[envconfig(default = "9104")]
pub metrics_port: u16,
}
+
+/// The slowest refill of a scaled new-session bucket: one session each 100 seconds.
+const MIN_NEW_SESSION_REPLENISH_RATE: f64 = 0.01;
+
+impl Config {
+ /// The bucket v4 runs, as `(capacity, replenish rate per second)`. v4 sees only its sample of the sessions, so both
+ /// scale by the sample, and a flood trips v4 at about the rate that trips the main lane.
+ pub fn new_session_bucket(&self) -> (f64, f64) {
+ let share = self.sample_percentage / 100.0;
+ let scaled = |value: f64, floor: f64| {
+ if value > 0.0 {
+ (value * share).max(floor)
+ } else {
+ value
+ }
+ };
+ (
+ scaled(self.new_session_bucket_capacity, 1.0),
+ scaled(
+ self.new_session_bucket_replenish_rate,
+ MIN_NEW_SESSION_REPLENISH_RATE,
+ ),
+ )
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use super::*;
+
+ fn config(vars: &[(&str, &str)]) -> Config {
+ let mut env: HashMap<String, String> = HashMap::from([
+ (
+ "DATABASE_URL".to_owned(),
+ "postgres://localhost/posthog".to_owned(),
+ ),
+ (
+ "EVENT_RESTRICTIONS_REDIS_URL".to_owned(),
+ "redis://localhost:6379/".to_owned(),
+ ),
+ (
+ "AI_RESEARCH_V4_REDIS_URL".to_owned(),
+ "redis://localhost:6379/".to_owned(),
+ ),
+ ]);
+ env.extend(vars.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())));
+ Config::init_from_hashmap(&env).unwrap()
+ }
+
+ #[test]
+ fn scales_the_new_session_bucket_by_the_sample() {
+ let bucket = |sample: &str, capacity: &str, rate: &str| {
+ config(&[
+ ("AI_RESEARCH_V4_SAMPLE_PERCENTAGE", sample),
+ ("AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", capacity),
+ ("AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE", rate),
+ ])
+ .new_session_bucket()
+ };
+ assert_eq!(bucket("100", "3000", "300"), (3000.0, 300.0));
+ assert_eq!(bucket("10", "3000", "300"), (300.0, 30.0));
+ assert_eq!(
+ bucket("0.01", "3000", "30"),
+ (1.0, MIN_NEW_SESSION_REPLENISH_RATE)
+ );
+ assert_eq!(bucket("10", "0", "0"), (0.0, 0.0));
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 5475ca0beafd..a3f7f1224a03 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -1,15 +1,19 @@
-//! The consume loop: gate, read and admit each message, buffer it, and flush on time or size.
+//! The consume loop: gate and admit each message from its key and headers, buffer it, and flush on time or size.
+use std::collections::{BTreeSet, HashSet};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use common_kafka::config::KafkaConfig;
+use common_kafka::kafka_producer::KafkaContext;
+use futures::FutureExt;
use lifecycle::Handle;
use metrics::counter;
use rdkafka::consumer::{
BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,
};
-use rdkafka::message::{BorrowedMessage, Headers, Message as _};
+use rdkafka::error::KafkaError;
+use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};
use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
use chrono::Utc;
@@ -18,8 +22,9 @@ use replay_core_v4::keys::kms::AwsKms;
use replay_core_v4::keys::rows::{key_month, write_day};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
-use crate::batch::{Batch, Block, Inspectors, SessionKey};
-use crate::process::{admit, read, sampled};
+use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey};
+use crate::limits::{Restrictions, SessionLimiter};
+use crate::process::{admit, sampled, Admitted, Headers, Skip};
use crate::store::BlobStore;
use crate::teams::TeamCache;
use crate::urls::{fetch_jobs, records};
@@ -28,6 +33,9 @@ use rdkafka::message::{Header, OwnedHeaders};
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::util::Timeout;
+/// The most messages read before the limiter decides on their sessions.
+const POLL_BATCH: usize = 1000;
+
/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.
pub struct RebalanceContext {
revoked: Arc<Mutex<Vec<i32>>>,
@@ -130,7 +138,7 @@ pub struct Writer {
pub store: Arc<dyn BlobStore>,
pub rrweb_bucket: String,
pub images_bucket: String,
- pub producer: FutureProducer,
+ pub producer: FutureProducer<KafkaContext>,
pub image_fetch_topic: String,
/// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so
/// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.
@@ -150,12 +158,21 @@ pub struct Settings {
pub flush_interval: Duration,
pub flush_max_bytes: usize,
pub session_idle: Duration,
+ /// The main topic, when the main lane moves its force-overflow messages to the overflow topic.
+ pub redirecting_topic: Option<String>,
+}
+
+/// What the consume loop reads besides Kafka.
+pub struct Sources {
+ pub teams: Arc<TeamCache>,
+ pub restrictions: Restrictions,
+ pub limiter: SessionLimiter,
}
pub async fn run(
consumer: RecorderConsumer,
settings: Settings,
- teams: Arc<TeamCache>,
+ sources: Sources,
writer: Option<Writer>,
handle: Handle,
) {
@@ -169,13 +186,30 @@ pub async fn run(
_ = &mut shutdown => break,
_ = tick.tick() => {}
received = consumer.consumer.recv() => match received {
- Ok(message) => handle_message(&message, &settings, &teams, &mut batch),
+ Ok(message) => {
+ // The messages Kafka has ready go through the session limiter together, in one Redis round trip.
+ let mut messages = vec![message];
+ let mut failed = None;
+ while messages.len() < POLL_BATCH {
+ match consumer.consumer.recv().now_or_never() {
+ Some(Ok(message)) => messages.push(message),
+ Some(Err(error)) => {
+ failed = Some(error);
+ break;
+ }
+ None => break,
+ }
+ }
+ handle_messages(&messages, &consumer, &settings, &sources, &mut batch).await;
+ drop(messages);
+ if let Some(error) = failed {
+ if receive_failed(&consumer, &handle, error) {
+ return;
+ }
+ }
+ }
Err(error) => {
- tracing::warn!(%error, "kafka receive failed");
- counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);
- // A fatal error disables the client for good, so only a restart recovers.
- if let Some((code, reason)) = consumer.consumer.client().fatal_error() {
- handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));
+ if receive_failed(&consumer, &handle, error) {
return;
}
}
@@ -201,44 +235,98 @@ pub async fn run(
}
}
-fn handle_message(
- message: &BorrowedMessage<'_>,
+/// Returns true when the error disabled the client for good, so only a restart recovers.
+fn receive_failed(consumer: &RecorderConsumer, handle: &Handle, error: KafkaError) -> bool {
+ tracing::warn!(%error, "kafka receive failed");
+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);
+ match consumer.consumer.client().fatal_error() {
+ Some((code, reason)) => {
+ handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));
+ true
+ }
+ None => false,
+ }
+}
+
+async fn handle_messages(
+ messages: &[BorrowedMessage<'_>],
+ consumer: &RecorderConsumer,
settings: &Settings,
- teams: &TeamCache,
+ sources: &Sources,
batch: &mut Batch,
) {
- let now_ms = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .map_or(0, |d| d.as_millis() as i64);
- let admitted = sampled(message.key(), settings.sample_percentage)
- .and_then(|()| {
- read(
- message.payload().unwrap_or_default(),
- header(message, "content-encoding"),
- )
- })
- .and_then(|read| {
- let team = header(message, "token").and_then(|token| teams.get(token));
- admit(&read, team, now_ms).map(|team_id| (team_id, read))
+ let mut admitted = Vec::with_capacity(messages.len());
+ for message in messages {
+ let session = admit_message(message, &consumer.topic, settings, sources)
+ .await
+ .map(|admitted| SessionKey {
+ team_id: admitted.team_id,
+ session_id: admitted.session_id,
+ });
+ admitted.push((message, session));
+ }
+ // The limiter decides before anything is buffered, so a blocked session's messages never reach a block.
+ let sessions: BTreeSet<SessionKey> = admitted
+ .iter()
+ .filter_map(|(_, session)| session.as_ref().ok().cloned())
+ .collect();
+ let blocked = if sessions.is_empty() {
+ HashSet::new()
+ } else {
+ sources.limiter.blocked(&sessions).await
+ };
+ for (message, session) in admitted {
+ let session = session.and_then(|session| {
+ if blocked.contains(&session) {
+ Err(Skip::Blocked)
+ } else {
+ Ok(session)
+ }
});
- match admitted {
- Ok((team_id, read)) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
- .increment(1);
- let session = SessionKey {
- team_id,
- session_id: read.session_id.clone(),
- };
- batch.add(message.partition(), message.offset(), session, read);
- }
- Err(skip) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
- .increment(1);
- batch.handled(message.partition(), message.offset());
+ match session {
+ Ok(session) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "admitted")
+ .increment(1);
+ let raw = Raw {
+ offset: message.offset(),
+ payload: message.payload().unwrap_or_default().to_vec(),
+ content_encoding: header(message, "content-encoding").map(str::to_owned),
+ };
+ batch.add(message.partition(), session, raw);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1);
+ batch.handled(message.partition(), message.offset());
+ }
}
}
}
+/// Everything before the payload: the gate on the Kafka key, then the headers, restrictions and team.
+async fn admit_message(
+ message: &BorrowedMessage<'_>,
+ topic: &str,
+ settings: &Settings,
+ sources: &Sources,
+) -> Result<Admitted, Skip> {
+ sampled(message.key(), settings.sample_percentage)?;
+ let now_ms = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_or(0, |d| d.as_millis() as i64);
+ let headers = Headers {
+ token: header(message, "token"),
+ session_id: header(message, "session_id"),
+ distinct_id: header(message, "distinct_id"),
+ event: header(message, "event"),
+ uuid: header(message, "uuid"),
+ };
+ let restriction = sources.restrictions.get(&headers, now_ms / 1000).await;
+ let team = headers.token.and_then(|token| sources.teams.get(token));
+ let redirect_overflow = settings.redirecting_topic.as_deref() == Some(topic);
+ admit(&headers, restriction, redirect_overflow, team, now_ms)
+}
+
/// Stores a flush's blocks, or only reports them when the recorder runs dry.
async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), FlushError> {
counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index b3b3f850faf8..1927a89e129c 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -5,6 +5,7 @@ pub mod batch;
pub mod config;
pub mod consumer;
pub mod index;
+pub mod limits;
pub mod process;
pub mod rewrite;
pub mod store;
diff --git a/rust/ai-research/replay-recorder-v4/src/limits.rs b/rust/ai-research/replay-recorder-v4/src/limits.rs
new file mode 100644
index 000000000000..42cac33a05f1
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/limits.rs
@@ -0,0 +1,493 @@
+//! What drops data before it is buffered: the event ingestion restrictions the replay lanes read from Redis, and v4's
+//! own new-session rate limiter with the main lane's rules.
+
+use std::borrow::Cow;
+use std::collections::{BTreeSet, HashMap, HashSet};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use capture::event_restrictions::{
+ EventContext, EventRestrictionService, EventRestrictionsRepository, Pipeline,
+ RedisRestrictionsRepository,
+};
+use common_redis::{
+ Client, CompressionConfig, CustomRedisError, PipelineCommand, RedisClient, RedisValueFormat,
+};
+use metrics::counter;
+
+use crate::batch::SessionKey;
+use crate::process::{normalize_session_id, Headers, Restriction};
+
+const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// The `session_recordings` restrictions, refreshed in the background as capture refreshes them.
+#[derive(Clone)]
+pub struct Restrictions {
+ service: EventRestrictionService,
+}
+
+impl Default for Restrictions {
+ fn default() -> Self {
+ // The replay lanes keep the last list they read while Redis is away. A stale list never turns into no list.
+ Self {
+ service: EventRestrictionService::new(
+ vec![Pipeline::SessionRecordings],
+ Duration::from_secs(10 * 365 * 86_400),
+ ),
+ }
+ }
+}
+
+impl Restrictions {
+ pub async fn refresh_forever(
+ self,
+ redis_url: String,
+ interval: Duration,
+ handle: lifecycle::Handle,
+ ) {
+ self.service
+ .start_refresh_task(
+ || {
+ let url = redis_url.clone();
+ async move {
+ let repository: Arc<dyn EventRestrictionsRepository> = Arc::new(
+ RedisRestrictionsRepository::new(
+ url,
+ Some(REDIS_TIMEOUT),
+ Some(REDIS_TIMEOUT),
+ )
+ .await?,
+ );
+ Ok(repository)
+ }
+ },
+ interval,
+ handle,
+ )
+ .await;
+ }
+
+ /// Before the first load every message would pass, so the recorder waits for it.
+ pub async fn wait_loaded(&self, timeout: Duration) -> bool {
+ let deadline = tokio::time::Instant::now() + timeout;
+ while !self.service.has_loaded() {
+ if tokio::time::Instant::now() >= deadline {
+ return false;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ true
+ }
+
+ /// Matches on the headers as the main lane's `parseEventHeaders` reads them: NUL replaced in the token, distinct
+ /// ID and session ID, and a UUID session ID in lower case.
+ pub async fn get(&self, headers: &Headers<'_>, now_ts: i64) -> Restriction {
+ let Some(token) = headers.token.map(sanitize) else {
+ return Restriction::default();
+ };
+ let distinct_id = headers.distinct_id.map(sanitize);
+ let session_id = headers.session_id.map(sanitize);
+ let session_id = session_id.as_deref().map(normalize_session_id);
+ let context = EventContext {
+ distinct_id: distinct_id.as_deref(),
+ session_id: session_id.as_deref(),
+ event_name: headers.event,
+ event_uuid: headers.uuid,
+ now_ts,
+ };
+ let applied = self
+ .service
+ .get_restrictions(&token, &context, Pipeline::SessionRecordings)
+ .await;
+ Restriction {
+ drop: applied.should_drop(),
+ dlq: applied.redirect_to_dlq(),
+ force_overflow: applied.force_overflow(),
+ }
+ }
+}
+
+fn sanitize(value: &str) -> Cow<'_, str> {
+ if value.contains('\0') {
+ Cow::Owned(value.replace('\0', "\u{FFFD}"))
+ } else {
+ Cow::Borrowed(value)
+ }
+}
+
+/// The seen and blocked keys of v4's own limiter. Its namespace keeps them apart from the main lane's and the v3
+/// mirror's, as the mirror keeps its own.
+const SEEN_PREFIX: &str = "@posthog/ai-research-v4/session-seen";
+const BLOCKED_PREFIX: &str = "@posthog/ai-research-v4/session-blocked";
+
+/// The main lane's TTL for both keys: twice the longest session.
+const STATE_TTL_SECS: u64 = 2 * 24 * 3600;
+
+/// The main lane's local caches in front of Redis. They are LRU, as the main lane's are: the limiter asks for a seen
+/// session with `contains_key`, which does not count as a read, so moka's default policy would refuse every new
+/// session once the cache is full.
+const LOCAL_CACHE_SIZE: u64 = 100_000;
+const LOCAL_CACHE_TTL: Duration = Duration::from_secs(300);
+
+/// One poll batch's reads and writes together. A Redis that does not answer delays each batch by this much at most.
+const LIMITER_DEADLINE: Duration = Duration::from_secs(1);
+
+fn local_cache<V: Clone + Send + Sync + 'static>() -> moka::sync::Cache<SessionKey, V> {
+ moka::sync::Cache::builder()
+ .max_capacity(LOCAL_CACHE_SIZE)
+ .time_to_live(LOCAL_CACHE_TTL)
+ .eviction_policy(moka::policy::EvictionPolicy::lru())
+ .build()
+}
+
+fn state_key(prefix: &str, session: &SessionKey) -> String {
+ format!("{prefix}:{}:{}", session.team_id, session.session_id)
+}
+
+/// A token bucket per team, as the main lane's `Limiter`.
+struct Buckets {
+ capacity: f64,
+ replenish_per_sec: f64,
+ teams: HashMap<i64, (f64, tokio::time::Instant)>,
+}
+
+impl Buckets {
+ fn take(&mut self, team_id: i64, now: tokio::time::Instant) -> bool {
+ let (tokens, at) = self.teams.entry(team_id).or_insert((self.capacity, now));
+ let refill = now.saturating_duration_since(*at).as_secs_f64() * self.replenish_per_sec;
+ *tokens = (*tokens + refill).min(self.capacity);
+ *at = now;
+ if *tokens < 1.0 {
+ return false;
+ }
+ *tokens -= 1.0;
+ true
+ }
+}
+
+/// The main lane's new-session rate limiter, run by v4 for itself. Each team's new sessions take a token from its
+/// bucket, and a session denied one is blocked as a whole, so v4 never keeps half of it.
+///
+/// Redis errors fail open, as in the main lane: a session whose state is unknown is not blocked, and a block that
+/// is not written holds in this process.
+pub struct SessionLimiter {
+ redis: Arc<dyn Client + Send + Sync>,
+ buckets: Mutex<Buckets>,
+ blocking_enabled: bool,
+ seen: moka::sync::Cache<SessionKey, ()>,
+ blocked: moka::sync::Cache<SessionKey, bool>,
+}
+
+impl SessionLimiter {
+ pub async fn connect(
+ redis_url: String,
+ capacity: f64,
+ replenish_per_sec: f64,
+ blocking_enabled: bool,
+ ) -> Result<Self, CustomRedisError> {
+ let redis = RedisClient::with_config(
+ redis_url,
+ CompressionConfig::disabled(),
+ RedisValueFormat::Utf8,
+ Some(LIMITER_DEADLINE),
+ Some(LIMITER_DEADLINE),
+ )
+ .await?;
+ Ok(Self::new(
+ Arc::new(redis),
+ capacity,
+ replenish_per_sec,
+ blocking_enabled,
+ ))
+ }
+
+ pub fn new(
+ redis: Arc<dyn Client + Send + Sync>,
+ capacity: f64,
+ replenish_per_sec: f64,
+ blocking_enabled: bool,
+ ) -> Self {
+ Self {
+ redis,
+ buckets: Mutex::new(Buckets {
+ capacity,
+ replenish_per_sec,
+ teams: HashMap::new(),
+ }),
+ blocking_enabled,
+ seen: local_cache(),
+ blocked: local_cache(),
+ }
+ }
+
+ /// Returns which of a poll batch's sessions are blocked. A session is new when it is neither seen nor blocked,
+ /// and only a new session takes a token, so a blocked session does not drain its team's bucket.
+ pub async fn blocked(&self, sessions: &BTreeSet<SessionKey>) -> HashSet<SessionKey> {
+ let deadline = tokio::time::Instant::now() + LIMITER_DEADLINE;
+ let mut blocked = HashSet::new();
+ let mut seen = HashSet::new();
+ let mut reads: Vec<(String, bool, &SessionKey)> = Vec::new();
+ for session in sessions {
+ if self.seen.contains_key(session) {
+ seen.insert(session);
+ } else {
+ reads.push((state_key(SEEN_PREFIX, session), false, session));
+ }
+ match self.blocked.get(session) {
+ Some(true) => {
+ blocked.insert(session.clone());
+ }
+ Some(false) => {}
+ None => reads.push((state_key(BLOCKED_PREFIX, session), true, session)),
+ }
+ }
+ let mut unknown = HashSet::new();
+ if !reads.is_empty() {
+ let keys = reads.iter().map(|(key, _, _)| key.clone()).collect();
+ match self.call(deadline, self.redis.mget(keys)).await {
+ Ok(values) => {
+ for ((_, is_block, session), value) in reads.iter().zip(values) {
+ let set = value.is_some();
+ if *is_block {
+ self.blocked.insert((*session).clone(), set);
+ if set {
+ blocked.insert((*session).clone());
+ }
+ } else if set {
+ self.seen.insert((*session).clone(), ());
+ seen.insert(session);
+ }
+ }
+ }
+ Err(()) => unknown.extend(reads.iter().map(|(_, _, session)| *session)),
+ }
+ }
+
+ let now = tokio::time::Instant::now();
+ let mut writes = Vec::new();
+ {
+ let mut buckets = self.buckets.lock().unwrap();
+ for session in sessions {
+ if seen.contains(session) || blocked.contains(session) || unknown.contains(session)
+ {
+ continue;
+ }
+ counter!("ai_research_v4_recorder_new_sessions_total").increment(1);
+ let blocking = self.blocking_enabled;
+ let allowed = buckets.take(session.team_id, now);
+ if !allowed {
+ let outcome = if blocking { "blocked" } else { "counted" };
+ counter!("ai_research_v4_recorder_rate_limited_sessions_total", "outcome" => outcome)
+ .increment(1);
+ }
+ let prefix = if allowed || !blocking {
+ self.seen.insert(session.clone(), ());
+ SEEN_PREFIX
+ } else {
+ self.blocked.insert(session.clone(), true);
+ blocked.insert(session.clone());
+ BLOCKED_PREFIX
+ };
+ writes.push(PipelineCommand::SetEx {
+ key: state_key(prefix, session),
+ value: "1".to_owned(),
+ seconds: STATE_TTL_SECS,
+ format: RedisValueFormat::Utf8,
+ });
+ }
+ }
+ if !writes.is_empty() {
+ let written = self
+ .call(deadline, self.redis.execute_pipeline(writes))
+ .await;
+ if let Ok(results) = written {
+ if let Some(Err(error)) = results.into_iter().find(Result::is_err) {
+ tracing::warn!(%error, "writing the session limiter's state failed");
+ counter!("ai_research_v4_recorder_session_limiter_redis_errors_total")
+ .increment(1);
+ }
+ }
+ }
+ blocked
+ }
+
+ /// Runs one Redis call within the batch's deadline. A failure is counted, and a dead connection reconnects.
+ async fn call<T>(
+ &self,
+ deadline: tokio::time::Instant,
+ call: impl std::future::Future<Output = Result<T, CustomRedisError>>,
+ ) -> Result<T, ()> {
+ let error = match tokio::time::timeout_at(deadline, call).await {
+ Ok(Ok(value)) => return Ok(value),
+ Ok(Err(error)) => error,
+ Err(_) => CustomRedisError::Timeout,
+ };
+ tracing::warn!(%error, "the session limiter's redis call failed");
+ counter!("ai_research_v4_recorder_session_limiter_redis_errors_total").increment(1);
+ // The connection does not come back by itself after a failover.
+ if error.is_unrecoverable_error() {
+ self.redis.heal().await;
+ }
+ Err(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use capture::event_restrictions::{
+ Restriction as Rule, RestrictionFilters, RestrictionManager, RestrictionScope,
+ RestrictionType,
+ };
+ use common_redis::{MockRedisClient, MockRedisValue, RedisErrorKind};
+
+ use super::*;
+
+ #[tokio::test]
+ async fn matches_restrictions_on_the_session_id_as_the_main_lane_reads_it() {
+ let session = "019a2f3e-8c00-7000-8000-00000000000a";
+ let mut manager = RestrictionManager::new();
+ manager.insert_restrictions(
+ Pipeline::SessionRecordings,
+ "token",
+ vec![Rule {
+ restriction_type: RestrictionType::DropEvent,
+ scope: RestrictionScope::Filtered(RestrictionFilters {
+ session_ids: HashSet::from([session.to_owned()]),
+ ..RestrictionFilters::default()
+ }),
+ args: None,
+ }],
+ );
+ let restrictions = Restrictions::default();
+ let now = restrictions.service.update(manager).await;
+ let upper = session.to_ascii_uppercase();
+ let headers = Headers {
+ token: Some("token"),
+ session_id: Some(&upper),
+ ..Headers::default()
+ };
+ assert!(restrictions.get(&headers, now).await.drop);
+ }
+
+ fn session(team_id: i64, id: &str) -> SessionKey {
+ SessionKey {
+ team_id,
+ session_id: id.to_owned(),
+ }
+ }
+
+ fn on(sessions: &[SessionKey]) -> BTreeSet<SessionKey> {
+ sessions.iter().cloned().collect()
+ }
+
+ fn sorted(blocked: HashSet<SessionKey>) -> Vec<String> {
+ let mut ids: Vec<String> = blocked.into_iter().map(|s| s.session_id).collect();
+ ids.sort();
+ ids
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_team_bucket_holds_its_capacity_and_refills_at_its_rate() {
+ let mut buckets = Buckets {
+ capacity: 2.0,
+ replenish_per_sec: 0.5,
+ teams: HashMap::new(),
+ };
+ let now = tokio::time::Instant::now();
+ let taken = |buckets: &mut Buckets, at: tokio::time::Instant| {
+ (0..4).filter(|_| buckets.take(7, at)).count()
+ };
+ assert_eq!(taken(&mut buckets, now), 2);
+ assert_eq!(taken(&mut buckets, now + Duration::from_secs(2)), 1);
+ // A long pause refills to the capacity, not past it.
+ assert_eq!(taken(&mut buckets, now + Duration::from_secs(600)), 2);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_flood_of_new_sessions_blocks_the_excess_of_its_team_only() {
+ let redis = MockRedisClient::new();
+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 2.0, 1.0, true);
+ let flood: Vec<SessionKey> = (0..4)
+ .map(|i| session(7, &format!("s{i}")))
+ .chain([session(8, "other")])
+ .collect();
+ assert_eq!(sorted(limiter.blocked(&on(&flood)).await), ["s2", "s3"]);
+
+ // Later messages: a blocked session stays blocked, and a seen one takes no token from the empty bucket.
+ let later = [session(7, "s0"), session(7, "s3")];
+ assert_eq!(sorted(limiter.blocked(&on(&later)).await), ["s3"]);
+
+ let writes: Vec<String> = redis
+ .get_calls()
+ .into_iter()
+ .filter_map(|c| match c.value {
+ MockRedisValue::StringWithTTLAndFormat(_, STATE_TTL_SECS, _) => Some(c.key),
+ _ => None,
+ })
+ .collect();
+ assert_eq!(
+ writes,
+ [
+ format!("{SEEN_PREFIX}:7:s0"),
+ format!("{SEEN_PREFIX}:7:s1"),
+ format!("{BLOCKED_PREFIX}:7:s2"),
+ format!("{BLOCKED_PREFIX}:7:s3"),
+ format!("{SEEN_PREFIX}:8:other"),
+ ]
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn reads_the_state_other_processes_wrote() {
+ let mut redis = MockRedisClient::new();
+ redis.mget_ret(&format!("{SEEN_PREFIX}:7:seen"), Some(b"1".to_vec()));
+ redis.mget_ret(&format!("{BLOCKED_PREFIX}:7:blocked"), Some(b"1".to_vec()));
+ let limiter = SessionLimiter::new(Arc::new(redis), 1.0, 0.0, true);
+ let sessions = [
+ session(7, "seen"),
+ session(7, "blocked"),
+ session(7, "new"),
+ session(7, "late"),
+ ];
+ // Neither known session takes the one token, so the first new one gets it.
+ assert_eq!(
+ sorted(limiter.blocked(&on(&sessions)).await),
+ ["blocked", "new"]
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn fails_open_when_redis_does_not_answer() {
+ let redis = MockRedisClient::new().mget_error(CustomRedisError::from_redis_kind(
+ RedisErrorKind::ParseError,
+ "the connection closed in a reply",
+ ));
+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, true);
+ assert!(limiter.blocked(&on(&[session(7, "a")])).await.is_empty());
+ let ops: Vec<_> = redis.get_calls().into_iter().map(|c| c.op).collect();
+ assert_eq!(ops, ["mget", "heal"]);
+
+ // A block that misses the deadline holds in this process.
+ let redis = MockRedisClient::new().pipeline_block(Duration::from_secs(60));
+ let limiter = SessionLimiter::new(Arc::new(redis), 0.0, 0.0, true);
+ let started = tokio::time::Instant::now();
+ let a = on(&[session(7, "a")]);
+ assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);
+ assert_eq!(started.elapsed(), LIMITER_DEADLINE);
+ assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn without_blocking_only_counts() {
+ let redis = MockRedisClient::new();
+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, false);
+ assert!(limiter
+ .blocked(&on(&[session(7, "a"), session(7, "b")]))
+ .await
+ .is_empty());
+ assert!(redis
+ .get_calls()
+ .iter()
+ .all(|c| !c.key.starts_with(BLOCKED_PREFIX)));
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 728dcc074838..18774fc8b630 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -3,6 +3,8 @@ use std::time::Duration;
use axum::{routing::get, Router};
use common_database::{get_pool_with_config, PoolConfig};
+use common_kafka::config::KafkaConfig;
+use common_kafka::kafka_producer::create_kafka_producer;
use common_metrics::setup_metrics_routes;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
@@ -15,7 +17,8 @@ use replay_core_v4::keys::batch::KeyStore;
use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use replay_recorder_v4::config::Config;
-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Writer};
+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources, Writer};
+use replay_recorder_v4::limits::{Restrictions, SessionLimiter};
use replay_recorder_v4::store::S3Store;
use replay_recorder_v4::teams::TeamCache;
@@ -43,6 +46,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.with_graceful_shutdown(Duration::from_secs(15))
.with_liveness_deadline(Duration::from_secs(60)),
);
+ let restrictions_handle = manager.register(
+ "event-restrictions",
+ ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
+ );
+ // The producer reports itself live from librdkafka's statistics callback.
+ let producer_handle = config.write_enabled.then(|| {
+ manager.register(
+ "image-fetch-producer",
+ ComponentOptions::new()
+ .with_graceful_shutdown(Duration::from_secs(5))
+ .with_liveness_deadline(Duration::from_secs(60)),
+ )
+ });
let metrics_handle = manager.register(
"metrics-server",
ComponentOptions::new().is_observability(true),
@@ -104,6 +120,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
teams_handle,
));
+ let restrictions = Restrictions::default();
+ tokio::spawn(restrictions.clone().refresh_forever(
+ config.event_restrictions_redis_url.clone(),
+ Duration::from_secs(config.event_restrictions_refresh_secs),
+ restrictions_handle,
+ ));
+ // Before the first load no restriction applies, so nothing is read until it is in.
+ if !restrictions.wait_loaded(Duration::from_secs(60)).await {
+ return Err("the event ingestion restrictions did not load".into());
+ }
+ let (capacity, replenish_rate) = config.new_session_bucket();
+ let limiter = SessionLimiter::connect(
+ config.redis_url.clone(),
+ capacity,
+ replenish_rate,
+ config.new_session_blocking_enabled,
+ )
+ .await?;
+
let consumer = RecorderConsumer::new(
&config.kafka,
&config.kafka_consumer_group,
@@ -115,8 +150,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
flush_interval: Duration::from_millis(config.flush_interval_ms),
flush_max_bytes: config.flush_max_bytes,
session_idle: Duration::from_secs(config.session_idle_secs),
+ redirecting_topic: (config.ingestion_overflow_mode == "redirect")
+ .then(|| config.kafka_topic.clone()),
};
- let writer = if config.write_enabled {
+ let writer = if let Some(producer_handle) = producer_handle {
let aws = aws_config::defaults(aws_config::BehaviorVersion::latest())
.load()
.await;
@@ -144,11 +181,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
store: Arc::new(S3Store::new(aws_sdk_s3::Client::new(&aws))),
rrweb_bucket: config.rrweb_bucket.clone(),
images_bucket: config.images_bucket.clone(),
- producer: rdkafka::ClientConfig::new()
- .set("bootstrap.servers", &config.kafka.kafka_hosts)
- .set("compression.codec", "lz4")
- .set("linger.ms", "20")
- .create()?,
+ producer: create_kafka_producer(
+ &KafkaConfig {
+ kafka_compression_codec: "lz4".to_owned(),
+ kafka_message_timeout_ms: config.image_fetch_delivery_timeout_ms,
+ ..config.kafka.clone()
+ },
+ producer_handle,
+ )
+ .await?,
image_fetch_topic: config.image_fetch_topic.clone(),
sent_urls: replay_recorder_v4::urls::sent_urls(500_000),
})
@@ -156,7 +197,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!("writes are off: the recorder stores nothing");
None
};
- tokio::spawn(run(consumer, settings, teams, writer, consumer_handle));
+ let sources = Sources {
+ teams,
+ restrictions,
+ limiter,
+ };
+ tokio::spawn(run(consumer, settings, sources, writer, consumer_handle));
monitor.wait().await?;
Ok(())
diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rs
index 5a74bc918008..272ef219a227 100644
--- a/rust/ai-research/replay-recorder-v4/src/process.rs
+++ b/rust/ai-research/replay-recorder-v4/src/process.rs
@@ -1,4 +1,9 @@
//! What the recorder does with one Kafka message, apart from Kafka itself.
+//!
+//! Admission reads only the Kafka key and headers, so a message the main lane or the v3 mirror would drop is never
+//! decoded. The payload is read later, at the flush.
+
+use std::borrow::Cow;
use replay_core_v4::gate::in_sample;
use replay_core_v4::payload::{decompress_payload, is_web_replay};
@@ -12,11 +17,17 @@ use crate::teams::Team;
pub enum Skip {
NotSampled,
NoKey,
- Undecompressable,
- Unreadable,
+ Restricted,
+ RestrictedToDlq,
+ Overflowed,
+ MissingHeader,
UnknownTeam,
NotOptedIn,
Session(DropReason),
+ Blocked,
+ Undecompressable,
+ Unreadable,
+ SessionMismatch,
NotWeb,
}
@@ -25,11 +36,17 @@ impl Skip {
match self {
Skip::NotSampled => "not_sampled",
Skip::NoKey => "no_key",
- Skip::Undecompressable => "undecompressable",
- Skip::Unreadable => "unreadable",
+ Skip::Restricted => "blocked_token",
+ Skip::RestrictedToDlq => "restricted_to_dlq",
+ Skip::Overflowed => "redirected_to_overflow",
+ Skip::MissingHeader => "missing_header",
Skip::UnknownTeam => "unknown_team",
Skip::NotOptedIn => "not_opted_in",
Skip::Session(reason) => reason.as_str(),
+ Skip::Blocked => "session_blocked",
+ Skip::Undecompressable => "undecompressable",
+ Skip::Unreadable => "unreadable",
+ Skip::SessionMismatch => "session_mismatch",
Skip::NotWeb => "not_web",
}
}
@@ -45,26 +62,102 @@ pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {
}
}
-/// The v3 mirror's admission rules: a consenting organization, a recent UUIDv7 session, a web replay. Returns the team.
-pub fn admit(message: &Message, team: Option<Team>, now_ms: i64) -> Result<i64, Skip> {
+/// The headers capture sets on every replay message.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct Headers<'a> {
+ pub token: Option<&'a str>,
+ pub session_id: Option<&'a str>,
+ pub distinct_id: Option<&'a str>,
+ pub event: Option<&'a str>,
+ pub uuid: Option<&'a str>,
+}
+
+/// The event ingestion restrictions that apply to a message.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct Restriction {
+ pub drop: bool,
+ pub dlq: bool,
+ pub force_overflow: bool,
+}
+
+/// A message that may be recorded: its team, and its session ID as the replay lanes key it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Admitted {
+ pub team_id: i64,
+ pub session_id: String,
+}
+
+/// The replay lanes lower-case a session ID that is a UUID and keep any other as it is.
+pub fn normalize_session_id(session_id: &str) -> Cow<'_, str> {
+ let bytes = session_id.as_bytes();
+ let uuid = bytes.len() == 36
+ && bytes.iter().enumerate().all(|(i, b)| match i {
+ 8 | 13 | 18 | 23 => *b == b'-',
+ _ => b.is_ascii_hexdigit(),
+ });
+ if uuid && bytes.iter().any(u8::is_ascii_uppercase) {
+ Cow::Owned(session_id.to_ascii_lowercase())
+ } else {
+ Cow::Borrowed(session_id)
+ }
+}
+
+/// The main lane's and the v3 mirror's rules before the payload, in their order: restrictions, the headers capture
+/// guarantees, a team with recordings and a retention period, a recent UUIDv7 session, a consenting organization.
+/// `redirect_overflow` is true on the main topic when the main lane moves force-overflow messages to the overflow
+/// topic.
+pub fn admit(
+ headers: &Headers<'_>,
+ restriction: Restriction,
+ redirect_overflow: bool,
+ team: Option<Team>,
+ now_ms: i64,
+) -> Result<Admitted, Skip> {
+ if restriction.drop {
+ return Err(Skip::Restricted);
+ }
+ if restriction.dlq {
+ return Err(Skip::RestrictedToDlq);
+ }
+ if restriction.force_overflow && redirect_overflow {
+ return Err(Skip::Overflowed);
+ }
+ let (Some(_), Some(session_id), Some(_)) = (
+ headers.token.filter(|t| !t.is_empty()),
+ headers.session_id.filter(|s| !s.is_empty()),
+ headers.distinct_id.filter(|d| !d.is_empty()),
+ ) else {
+ return Err(Skip::MissingHeader);
+ };
let team = team.ok_or(Skip::UnknownTeam)?;
+ let session_id = normalize_session_id(session_id);
+ if let Some(reason) = drop_reason(&session_id, now_ms) {
+ return Err(Skip::Session(reason));
+ }
if !team.ai_training_opted_in {
return Err(Skip::NotOptedIn);
}
- if let Some(reason) = drop_reason(&message.session_id, now_ms) {
- return Err(Skip::Session(reason));
+ Ok(Admitted {
+ team_id: team.id,
+ session_id: session_id.into_owned(),
+ })
+}
+
+/// Decompresses and reads an admitted message.
+pub fn read(payload: Vec<u8>, content_encoding: Option<&str>) -> Result<Message, Skip> {
+ let raw = decompress_payload(payload, content_encoding).map_err(|_| Skip::Undecompressable)?;
+ read_message(raw).map_err(|_| Skip::Unreadable)
+}
+
+/// The rules that need the payload: the session the headers named, and a web replay.
+pub fn check_read(message: &Message, session_id: &str) -> Result<(), Skip> {
+ if normalize_session_id(&message.session_id) != session_id {
+ return Err(Skip::SessionMismatch);
}
if !is_web_replay(message.snapshot_source.as_deref()) {
return Err(Skip::NotWeb);
}
- Ok(team.id)
-}
-
-/// Decompresses and reads a sampled message.
-pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {
- let raw = decompress_payload(payload.to_vec(), content_encoding)
- .map_err(|_| Skip::Undecompressable)?;
- read_message(raw).map_err(|_| Skip::Unreadable)
+ Ok(())
}
#[cfg(test)]
@@ -74,19 +167,34 @@ mod tests {
use super::*;
- fn capture_message() -> Vec<u8> {
- let inner = json!({"event": "$snapshot_items", "properties": {
- "$session_id": "019a2f3e-8c00-7000-8000-000000000007",
+ const SESSION: &str = "019a2f3e-8c00-7000-8000-000000000007";
+
+ fn capture_message(session: &str, source: Option<&str>) -> Vec<u8> {
+ let mut properties = json!({
+ "$session_id": session,
"$snapshot_items": [{"type": 3, "timestamp": 1, "data": {"source": 1}}]
- }});
+ });
+ if let Some(source) = source {
+ properties["$snapshot_source"] = json!(source);
+ }
+ let inner = json!({"event": "$snapshot_items", "properties": properties});
json!({"distinct_id": "d", "data": inner.to_string()})
.to_string()
.into_bytes()
}
+ fn headers(session: &str) -> Headers<'_> {
+ Headers {
+ token: Some("token"),
+ session_id: Some(session),
+ distinct_id: Some("d"),
+ ..Headers::default()
+ }
+ }
+
#[test]
fn gates_on_the_kafka_key_before_reading() {
- let session = b"019a2f3e-8c00-7000-8000-000000000007";
+ let session = SESSION.as_bytes();
assert_eq!(sampled(Some(session), 100.0), Ok(()));
assert_eq!(sampled(Some(session), 0.0), Err(Skip::NotSampled));
assert_eq!(sampled(None, 100.0), Err(Skip::NoKey));
@@ -95,43 +203,121 @@ mod tests {
#[test]
fn reads_plain_and_gzipped_capture_messages() {
- let plain = capture_message();
+ let plain = capture_message(SESSION, Some("web"));
for payload in [plain.clone(), gzip(&plain).unwrap()] {
- let message = read(&payload, None).unwrap();
- assert_eq!(message.session_id, "019a2f3e-8c00-7000-8000-000000000007");
+ let message = read(payload, None).unwrap();
+ assert_eq!(message.session_id, SESSION);
assert_eq!(message.events.len(), 1);
}
- assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);
+ assert_eq!(
+ read(b"not json".to_vec(), None).unwrap_err(),
+ Skip::Unreadable
+ );
}
#[test]
- fn admits_only_what_the_v3_mirror_admits() {
- let session = "019a2f3e-8c00-7000-8000-000000000007";
+ fn checks_the_payload_against_the_headers() {
+ let message = |session: &str, source: Option<&str>| {
+ read(capture_message(session, source), None).unwrap()
+ };
+ let cases = [
+ (message(SESSION, Some("web")), Ok(())),
+ (message(&SESSION.to_ascii_uppercase(), Some("web")), Ok(())),
+ (
+ message("019a2f3e-8c00-7000-8000-000000000008", Some("web")),
+ Err(Skip::SessionMismatch),
+ ),
+ (message(SESSION, Some("mobile")), Err(Skip::NotWeb)),
+ (message(SESSION, None), Err(Skip::NotWeb)),
+ ];
+ for (message, expected) in cases {
+ assert_eq!(check_read(&message, SESSION), expected, "{message:?}");
+ }
+ }
+
+ #[test]
+ fn admits_only_what_the_main_lane_and_the_v3_mirror_admit() {
// 019a2f3e8c00 is the session start in milliseconds.
let now = 0x019a_2f3e_8c00_i64 + 60_000;
- let message = |source: Option<&str>| {
- let mut properties = json!({"$session_id": session, "$snapshot_items": []});
- if let Some(source) = source {
- properties["$snapshot_source"] = json!(source);
- }
- let inner = json!({"event": "$snapshot_items", "properties": properties});
- read(
- json!({"distinct_id": "d", "data": inner.to_string()})
- .to_string()
- .as_bytes(),
- None,
- )
- .unwrap()
- };
+ let upper = SESSION.to_ascii_uppercase();
let opted_in = Some(Team {
id: 7,
ai_training_opted_in: true,
});
+ let open = Restriction::default();
+ let overflow = Restriction {
+ force_overflow: true,
+ ..open
+ };
+ let admitted = Ok(Admitted {
+ team_id: 7,
+ session_id: SESSION.to_owned(),
+ });
let cases = [
- (message(Some("web")), opted_in, now, Ok(7)),
- (message(Some("web")), None, now, Err(Skip::UnknownTeam)),
(
- message(Some("web")),
+ headers(SESSION),
+ open,
+ false,
+ opted_in,
+ now,
+ admitted.clone(),
+ ),
+ (
+ headers(&upper),
+ open,
+ false,
+ opted_in,
+ now,
+ admitted.clone(),
+ ),
+ (headers(SESSION), overflow, false, opted_in, now, admitted),
+ (
+ headers(SESSION),
+ overflow,
+ true,
+ opted_in,
+ now,
+ Err(Skip::Overflowed),
+ ),
+ (
+ headers(SESSION),
+ Restriction { drop: true, ..open },
+ false,
+ opted_in,
+ now,
+ Err(Skip::Restricted),
+ ),
+ (
+ headers(SESSION),
+ Restriction { dlq: true, ..open },
+ false,
+ opted_in,
+ now,
+ Err(Skip::RestrictedToDlq),
+ ),
+ (
+ Headers {
+ distinct_id: None,
+ ..headers(SESSION)
+ },
+ open,
+ false,
+ opted_in,
+ now,
+ Err(Skip::MissingHeader),
+ ),
+ (
+ headers(SESSION),
+ open,
+ false,
+ None,
+ now,
+ Err(Skip::UnknownTeam),
+ ),
+ (
+ headers(SESSION),
+ open,
+ false,
Some(Team {
id: 7,
ai_training_opted_in: false,
@@ -140,20 +326,19 @@ mod tests {
Err(Skip::NotOptedIn),
),
(
- message(Some("web")),
+ headers(SESSION),
+ open,
+ false,
opted_in,
now + 15 * 86_400_000,
Err(Skip::Session(DropReason::TooOld)),
),
- (message(Some("mobile")), opted_in, now, Err(Skip::NotWeb)),
- (message(None), opted_in, now, Err(Skip::NotWeb)),
];
- for (message, team, now, expected) in cases {
+ for (headers, restriction, redirect, team, now, expected) in cases {
assert_eq!(
- admit(&message, team, now),
+ admit(&headers, restriction, redirect, team, now),
expected,
- "{team:?} {:?}",
- message.snapshot_source
+ "{headers:?} {restriction:?} {redirect} {team:?}"
);
}
}