:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): apply ingestion restrictions and the session filter (#118417)
Nicholas Waltz2b24e2e03f2d
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex fda74543d84a..e871da13c7a6 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11065,12 +11065,15 @@ dependencies = ["base64 0.22.1","brotlic","bytes",+ "capture","chrono","common-alloc","common-database","common-kafka","common-metrics",+ "common-redis","envconfig",+ "futures","lifecycle","metrics","moka",diff --git a/rust/affected-services/tests/integration.rs b/rust/affected-services/tests/integration.rsindex 57351a49d8ed..af4e9a1c4143 100644--- a/rust/affected-services/tests/integration.rs+++ b/rust/affected-services/tests/integration.rs@@ -254,7 +254,12 @@ fn leaf_service_only_affects_itself_and_dependents() {assert_eq!(result.directly_changed, vec!["capture"]);assert_eq!(result.images,- vec!["capture", "capture-apm-metrics", "capture-logs"]+ vec![+ "capture",+ "capture-apm-metrics",+ "capture-logs",+ "replay-recorder-v4"+ ]);}diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex 097e412359d0..ed045fbc5545 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -9,10 +9,13 @@ description = "Records the v4 AI research replay dataset: sampled, unscrubbed, sworkspace = true[dependencies]+# The event ingestion restrictions capture and the replay lanes apply.+capture = { path = "../../capture" }common-alloc = { path = "../../common/alloc" }common-database = { path = "../../common/database" }common-kafka = { path = "../../common/kafka" }common-metrics = { path = "../../common/metrics" }+common-redis = { path = "../../common/redis" }lifecycle = { path = "../../common/lifecycle" }replay-core-v4 = { path = "../replay-core-v4" }@@ -30,6 +33,7 @@ axum = { workspace = true }brotlic = "0.8"chrono = { workspace = true }envconfig = { workspace = true }+futures = { workspace = true }metrics = { workspace = true }moka = { workspace = true }parquet = "58"diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex a3f353c8eb32..4781fdec116f 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -3,21 +3,31 @@use std::collections::{BTreeMap, HashMap};use std::time::{Duration, Instant};+use metrics::counter;use replay_core_v4::inspect::dom::UrlLimit;use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::scan::message::Message;+use crate::process::{check_read, read};+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct SessionKey {pub team_id: i64,pub session_id: String,}+/// An admitted message, not decoded yet.+pub struct Raw {+ pub offset: i64,+ pub payload: Vec<u8>,+ pub content_encoding: Option<String>,+}+/// One session's messages in the current flush. Capture keys messages by session, so a session has one partition.pub struct Pending {pub partition: i32,- pub messages: Vec<Message>,+ pub messages: Vec<Raw>,}/// What a flush hands to the writers.@@ -60,11 +70,9 @@ impl Batch {*last = (*last).max(offset);}- pub fn add(&mut self, partition: i32, offset: i64, session: SessionKey, mut message: Message) {- self.handled(partition, offset);- // Decoded once here, for the inspector and the block writer both, and counted toward the flush size.- message.decode_compressed();- self.bytes += message.buffered_bytes();+ pub fn add(&mut self, partition: i32, session: SessionKey, raw: Raw) {+ self.handled(partition, raw.offset);+ self.bytes += raw.payload.len();self.sessions.entry(session).or_insert_with(|| Pending {@@ -72,7 +80,7 @@ impl Batch {messages: Vec::new(),}).messages- .push(message);+ .push(raw);}pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {@@ -90,7 +98,7 @@ impl Batch {.sessions.values().flat_map(|p| &p.messages)- .map(Message::buffered_bytes)+ .map(|m| m.payload.len()).sum();}}@@ -102,11 +110,33 @@ pub struct Inspectors {}impl Inspectors {- /// Runs every buffered message through its session's inspector and returns one block per session.+ /// Decodes every buffered message, runs it through its session's inspector and returns one block per session.pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<i32, i64>) {let now = Instant::now();let mut blocks = Vec::with_capacity(batch.sessions.len());for (session, pending) in batch.sessions {+ let mut messages = Vec::with_capacity(pending.messages.len());+ for raw in pending.messages {+ let message = read(raw.payload, raw.content_encoding.as_deref())+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));+ match message {+ Ok(mut message) => {+ // Decoded once here, for the inspector and the block writer both.+ message.decode_compressed();+ messages.push(message);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1);+ }+ }+ }+ if messages.is_empty() {+ continue;+ }+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")+ .increment(messages.len() as u64);+ // Only a session that decoded something counts as seen.let (inspector, partition, last_seen) = self.sessions.entry(session.clone())@@ -114,13 +144,13 @@ impl Inspectors {*partition = pending.partition;*last_seen = now;let mut block = BlockInspection::default();- for message in &pending.messages {+ for message in &messages {inspector.observe_message(&mut block, message);}let inspected = block.finish();blocks.push(Block {session,- messages: pending.messages,+ messages,stats: inspected.stats,moments: inspected.moments,image_urls: inspected.image_urls,@@ -145,19 +175,19 @@ impl Inspectors {#[cfg(test)]mod tests {- use replay_core_v4::scan::message::read_message;use serde_json::{json, Value};use super::*;- fn message(session: &str, items: Value) -> Message {- let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_items": items}});- read_message(- json!({"distinct_id": "d", "data": inner.to_string()})+ fn raw(offset: i64, session: &str, items: Value) -> Raw {+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});+ Raw {+ offset,+ payload: json!({"distinct_id": "d", "data": inner.to_string()}).to_string().into_bytes(),- )- .unwrap()+ content_encoding: None,+ }}fn key(session: &str) -> SessionKey {@@ -178,8 +208,8 @@ mod tests {#[test]fn a_revoked_partition_leaves_nothing_behind() {let mut batch = Batch::default();- batch.add(0, 10, key("a"), message("a", mouse()));- batch.add(1, 20, key("b"), message("b", mouse()));+ batch.add(0, key("a"), raw(10, "a", mouse()));+ batch.add(1, key("b"), raw(20, "b", mouse()));batch.handled(1, 21);let mut inspectors = Inspectors::default();batch.drop_partitions(&[1]);@@ -196,17 +226,34 @@ mod tests {assert_eq!(offsets, HashMap::from([(0, 10)]));}+ #[test]+ fn a_session_that_decodes_nothing_keeps_no_state() {+ let mut batch = Batch::default();+ batch.add(+ 0,+ key("a"),+ Raw {+ offset: 1,+ payload: b"not json".to_vec(),+ content_encoding: None,+ },+ );+ let mut inspectors = Inspectors::default();+ assert!(inspectors.inspect(batch).0.is_empty());+ assert!(inspectors.sessions.is_empty());+ }+#[test]fn session_state_carries_across_flushes() {let mut inspectors = Inspectors::default();let mut first = Batch::default();- first.add(0, 1, key("a"), message("a", snapshot()));- first.add(0, 2, key("b"), message("b", mouse()));+ first.add(0, key("a"), raw(1, "a", snapshot()));+ first.add(0, key("b"), raw(2, "b", mouse()));inspectors.inspect(first);let mut second = Batch::default();- second.add(0, 3, key("a"), message("a", mouse()));- second.add(0, 4, key("b"), message("b", mouse()));+ second.add(0, key("a"), raw(3, "a", mouse()));+ second.add(0, key("b"), raw(4, "b", mouse()));let (blocks, _) = inspectors.inspect(second);let cold: Vec<_> = blocks.iter()diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 985ae8c10789..711a32566547 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -9,6 +9,11 @@ pub struct Config {#[envconfig(default = "session_recording_snapshot_item_events")]pub kafka_topic: String,+ /// The main replay lane's overflow mode. With `redirect`, that lane moves force-overflow messages of the main topic+ /// to the overflow topic, so v4 records them there and not twice.+ #[envconfig(from = "INGESTION_OVERFLOW_MODE", default = "disabled")]+ pub ingestion_overflow_mode: String,+#[envconfig(default = "ai-research-v4-recorder")]pub kafka_consumer_group: String,@@ -23,6 +28,35 @@ pub struct Config {/// The Postgres database the team list is read from. A read replica is enough.pub database_url: String,+ /// The Redis the event ingestion restrictions are read from, as capture and the replay lanes read them.+ pub event_restrictions_redis_url: String,++ #[envconfig(+ from = "AI_RESEARCH_V4_EVENT_RESTRICTIONS_REFRESH_SECS",+ default = "30"+ )]+ pub event_restrictions_refresh_secs: u64,++ /// The session recording Redis. v4's new-session rate limiter keeps its seen and blocked sessions there, under its+ /// own keys.+ #[envconfig(from = "AI_RESEARCH_V4_REDIS_URL")]+ pub redis_url: String,++ /// The new-session rate limiter's bucket per team, as in the main lane: this many new sessions at once, and this+ /// many more each second, scaled by the sample. See [`Config::new_session_bucket`].+ #[envconfig(from = "AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", default = "3000")]+ pub new_session_bucket_capacity: f64,++ #[envconfig(+ from = "AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE",+ default = "300"+ )]+ pub new_session_bucket_replenish_rate: f64,++ /// Off, the limiter only counts the sessions it would block.+ #[envconfig(from = "AI_RESEARCH_V4_NEW_SESSION_BLOCKING_ENABLED", default = "true")]+ pub new_session_blocking_enabled: bool,+/// How often the team list refreshes. Consent changes take effect within this time, as in v3.#[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]pub team_refresh_secs: u64,@@ -63,6 +97,83 @@ pub struct Config {)]pub image_fetch_topic: String,+ /// A fetch job not delivered in this time fails its flush.+ #[envconfig(+ from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",+ default = "20000"+ )]+ pub image_fetch_delivery_timeout_ms: u32,+#[envconfig(default = "9104")]pub metrics_port: u16,}++/// The slowest refill of a scaled new-session bucket: one session each 100 seconds.+const MIN_NEW_SESSION_REPLENISH_RATE: f64 = 0.01;++impl Config {+ /// The bucket v4 runs, as `(capacity, replenish rate per second)`. v4 sees only its sample of the sessions, so both+ /// scale by the sample, and a flood trips v4 at about the rate that trips the main lane.+ pub fn new_session_bucket(&self) -> (f64, f64) {+ let share = self.sample_percentage / 100.0;+ let scaled = |value: f64, floor: f64| {+ if value > 0.0 {+ (value * share).max(floor)+ } else {+ value+ }+ };+ (+ scaled(self.new_session_bucket_capacity, 1.0),+ scaled(+ self.new_session_bucket_replenish_rate,+ MIN_NEW_SESSION_REPLENISH_RATE,+ ),+ )+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use super::*;++ fn config(vars: &[(&str, &str)]) -> Config {+ let mut env: HashMap<String, String> = HashMap::from([+ (+ "DATABASE_URL".to_owned(),+ "postgres://localhost/posthog".to_owned(),+ ),+ (+ "EVENT_RESTRICTIONS_REDIS_URL".to_owned(),+ "redis://localhost:6379/".to_owned(),+ ),+ (+ "AI_RESEARCH_V4_REDIS_URL".to_owned(),+ "redis://localhost:6379/".to_owned(),+ ),+ ]);+ env.extend(vars.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())));+ Config::init_from_hashmap(&env).unwrap()+ }++ #[test]+ fn scales_the_new_session_bucket_by_the_sample() {+ let bucket = |sample: &str, capacity: &str, rate: &str| {+ config(&[+ ("AI_RESEARCH_V4_SAMPLE_PERCENTAGE", sample),+ ("AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", capacity),+ ("AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE", rate),+ ])+ .new_session_bucket()+ };+ assert_eq!(bucket("100", "3000", "300"), (3000.0, 300.0));+ assert_eq!(bucket("10", "3000", "300"), (300.0, 30.0));+ assert_eq!(+ bucket("0.01", "3000", "30"),+ (1.0, MIN_NEW_SESSION_REPLENISH_RATE)+ );+ assert_eq!(bucket("10", "0", "0"), (0.0, 0.0));+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 5475ca0beafd..a3f7f1224a03 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -1,15 +1,19 @@-//! The consume loop: gate, read and admit each message, buffer it, and flush on time or size.+//! The consume loop: gate and admit each message from its key and headers, buffer it, and flush on time or size.+use std::collections::{BTreeSet, HashSet};use std::sync::{Arc, Mutex};use std::time::{Duration, SystemTime, UNIX_EPOCH};use common_kafka::config::KafkaConfig;+use common_kafka::kafka_producer::KafkaContext;+use futures::FutureExt;use lifecycle::Handle;use metrics::counter;use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,};-use rdkafka::message::{BorrowedMessage, Headers, Message as _};+use rdkafka::error::KafkaError;+use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};use chrono::Utc;@@ -18,8 +22,9 @@ use replay_core_v4::keys::kms::AwsKms;use replay_core_v4::keys::rows::{key_month, write_day};use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};-use crate::batch::{Batch, Block, Inspectors, SessionKey};-use crate::process::{admit, read, sampled};+use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey};+use crate::limits::{Restrictions, SessionLimiter};+use crate::process::{admit, sampled, Admitted, Headers, Skip};use crate::store::BlobStore;use crate::teams::TeamCache;use crate::urls::{fetch_jobs, records};@@ -28,6 +33,9 @@ use rdkafka::message::{Header, OwnedHeaders};use rdkafka::producer::{FutureProducer, FutureRecord};use rdkafka::util::Timeout;+/// The most messages read before the limiter decides on their sessions.+const POLL_BATCH: usize = 1000;+/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.pub struct RebalanceContext {revoked: Arc<Mutex<Vec<i32>>>,@@ -130,7 +138,7 @@ pub struct Writer {pub store: Arc<dyn BlobStore>,pub rrweb_bucket: String,pub images_bucket: String,- pub producer: FutureProducer,+ pub producer: FutureProducer<KafkaContext>,pub image_fetch_topic: String,/// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so/// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.@@ -150,12 +158,21 @@ pub struct Settings {pub flush_interval: Duration,pub flush_max_bytes: usize,pub session_idle: Duration,+ /// The main topic, when the main lane moves its force-overflow messages to the overflow topic.+ pub redirecting_topic: Option<String>,+}++/// What the consume loop reads besides Kafka.+pub struct Sources {+ pub teams: Arc<TeamCache>,+ pub restrictions: Restrictions,+ pub limiter: SessionLimiter,}pub async fn run(consumer: RecorderConsumer,settings: Settings,- teams: Arc<TeamCache>,+ sources: Sources,writer: Option<Writer>,handle: Handle,) {@@ -169,13 +186,30 @@ pub async fn run(_ = &mut shutdown => break,_ = tick.tick() => {}received = consumer.consumer.recv() => match received {- Ok(message) => handle_message(&message, &settings, &teams, &mut batch),+ Ok(message) => {+ // The messages Kafka has ready go through the session limiter together, in one Redis round trip.+ let mut messages = vec![message];+ let mut failed = None;+ while messages.len() < POLL_BATCH {+ match consumer.consumer.recv().now_or_never() {+ Some(Ok(message)) => messages.push(message),+ Some(Err(error)) => {+ failed = Some(error);+ break;+ }+ None => break,+ }+ }+ handle_messages(&messages, &consumer, &settings, &sources, &mut batch).await;+ drop(messages);+ if let Some(error) = failed {+ if receive_failed(&consumer, &handle, error) {+ return;+ }+ }+ }Err(error) => {- tracing::warn!(%error, "kafka receive failed");- counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);- // A fatal error disables the client for good, so only a restart recovers.- if let Some((code, reason)) = consumer.consumer.client().fatal_error() {- handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));+ if receive_failed(&consumer, &handle, error) {return;}}@@ -201,44 +235,98 @@ pub async fn run(}}-fn handle_message(- message: &BorrowedMessage<'_>,+/// Returns true when the error disabled the client for good, so only a restart recovers.+fn receive_failed(consumer: &RecorderConsumer, handle: &Handle, error: KafkaError) -> bool {+ tracing::warn!(%error, "kafka receive failed");+ counter!("ai_research_v4_recorder_kafka_errors_total").increment(1);+ match consumer.consumer.client().fatal_error() {+ Some((code, reason)) => {+ handle.signal_failure(format!("kafka consumer fatal error {code}: {reason}"));+ true+ }+ None => false,+ }+}++async fn handle_messages(+ messages: &[BorrowedMessage<'_>],+ consumer: &RecorderConsumer,settings: &Settings,- teams: &TeamCache,+ sources: &Sources,batch: &mut Batch,) {- let now_ms = SystemTime::now()- .duration_since(UNIX_EPOCH)- .map_or(0, |d| d.as_millis() as i64);- let admitted = sampled(message.key(), settings.sample_percentage)- .and_then(|()| {- read(- message.payload().unwrap_or_default(),- header(message, "content-encoding"),- )- })- .and_then(|read| {- let team = header(message, "token").and_then(|token| teams.get(token));- admit(&read, team, now_ms).map(|team_id| (team_id, read))+ let mut admitted = Vec::with_capacity(messages.len());+ for message in messages {+ let session = admit_message(message, &consumer.topic, settings, sources)+ .await+ .map(|admitted| SessionKey {+ team_id: admitted.team_id,+ session_id: admitted.session_id,+ });+ admitted.push((message, session));+ }+ // The limiter decides before anything is buffered, so a blocked session's messages never reach a block.+ let sessions: BTreeSet<SessionKey> = admitted+ .iter()+ .filter_map(|(_, session)| session.as_ref().ok().cloned())+ .collect();+ let blocked = if sessions.is_empty() {+ HashSet::new()+ } else {+ sources.limiter.blocked(&sessions).await+ };+ for (message, session) in admitted {+ let session = session.and_then(|session| {+ if blocked.contains(&session) {+ Err(Skip::Blocked)+ } else {+ Ok(session)+ }});- match admitted {- Ok((team_id, read)) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")- .increment(1);- let session = SessionKey {- team_id,- session_id: read.session_id.clone(),- };- batch.add(message.partition(), message.offset(), session, read);- }- Err(skip) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())- .increment(1);- batch.handled(message.partition(), message.offset());+ match session {+ Ok(session) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "admitted")+ .increment(1);+ let raw = Raw {+ offset: message.offset(),+ payload: message.payload().unwrap_or_default().to_vec(),+ content_encoding: header(message, "content-encoding").map(str::to_owned),+ };+ batch.add(message.partition(), session, raw);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1);+ batch.handled(message.partition(), message.offset());+ }}}}+/// Everything before the payload: the gate on the Kafka key, then the headers, restrictions and team.+async fn admit_message(+ message: &BorrowedMessage<'_>,+ topic: &str,+ settings: &Settings,+ sources: &Sources,+) -> Result<Admitted, Skip> {+ sampled(message.key(), settings.sample_percentage)?;+ let now_ms = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_or(0, |d| d.as_millis() as i64);+ let headers = Headers {+ token: header(message, "token"),+ session_id: header(message, "session_id"),+ distinct_id: header(message, "distinct_id"),+ event: header(message, "event"),+ uuid: header(message, "uuid"),+ };+ let restriction = sources.restrictions.get(&headers, now_ms / 1000).await;+ let team = headers.token.and_then(|token| sources.teams.get(token));+ let redirect_overflow = settings.redirecting_topic.as_deref() == Some(topic);+ admit(&headers, restriction, redirect_overflow, team, now_ms)+}+/// Stores a flush's blocks, or only reports them when the recorder runs dry.async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), FlushError> {counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex b3b3f850faf8..1927a89e129c 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -5,6 +5,7 @@ pub mod batch;pub mod config;pub mod consumer;pub mod index;+pub mod limits;pub mod process;pub mod rewrite;pub mod store;diff --git a/rust/ai-research/replay-recorder-v4/src/limits.rs b/rust/ai-research/replay-recorder-v4/src/limits.rsnew file mode 100644index 000000000000..42cac33a05f1--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/limits.rs@@ -0,0 +1,493 @@+//! What drops data before it is buffered: the event ingestion restrictions the replay lanes read from Redis, and v4's+//! own new-session rate limiter with the main lane's rules.++use std::borrow::Cow;+use std::collections::{BTreeSet, HashMap, HashSet};+use std::sync::{Arc, Mutex};+use std::time::Duration;++use capture::event_restrictions::{+ EventContext, EventRestrictionService, EventRestrictionsRepository, Pipeline,+ RedisRestrictionsRepository,+};+use common_redis::{+ Client, CompressionConfig, CustomRedisError, PipelineCommand, RedisClient, RedisValueFormat,+};+use metrics::counter;++use crate::batch::SessionKey;+use crate::process::{normalize_session_id, Headers, Restriction};++const REDIS_TIMEOUT: Duration = Duration::from_secs(5);++/// The `session_recordings` restrictions, refreshed in the background as capture refreshes them.+#[derive(Clone)]+pub struct Restrictions {+ service: EventRestrictionService,+}++impl Default for Restrictions {+ fn default() -> Self {+ // The replay lanes keep the last list they read while Redis is away. A stale list never turns into no list.+ Self {+ service: EventRestrictionService::new(+ vec![Pipeline::SessionRecordings],+ Duration::from_secs(10 * 365 * 86_400),+ ),+ }+ }+}++impl Restrictions {+ pub async fn refresh_forever(+ self,+ redis_url: String,+ interval: Duration,+ handle: lifecycle::Handle,+ ) {+ self.service+ .start_refresh_task(+ || {+ let url = redis_url.clone();+ async move {+ let repository: Arc<dyn EventRestrictionsRepository> = Arc::new(+ RedisRestrictionsRepository::new(+ url,+ Some(REDIS_TIMEOUT),+ Some(REDIS_TIMEOUT),+ )+ .await?,+ );+ Ok(repository)+ }+ },+ interval,+ handle,+ )+ .await;+ }++ /// Before the first load every message would pass, so the recorder waits for it.+ pub async fn wait_loaded(&self, timeout: Duration) -> bool {+ let deadline = tokio::time::Instant::now() + timeout;+ while !self.service.has_loaded() {+ if tokio::time::Instant::now() >= deadline {+ return false;+ }+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ true+ }++ /// Matches on the headers as the main lane's `parseEventHeaders` reads them: NUL replaced in the token, distinct+ /// ID and session ID, and a UUID session ID in lower case.+ pub async fn get(&self, headers: &Headers<'_>, now_ts: i64) -> Restriction {+ let Some(token) = headers.token.map(sanitize) else {+ return Restriction::default();+ };+ let distinct_id = headers.distinct_id.map(sanitize);+ let session_id = headers.session_id.map(sanitize);+ let session_id = session_id.as_deref().map(normalize_session_id);+ let context = EventContext {+ distinct_id: distinct_id.as_deref(),+ session_id: session_id.as_deref(),+ event_name: headers.event,+ event_uuid: headers.uuid,+ now_ts,+ };+ let applied = self+ .service+ .get_restrictions(&token, &context, Pipeline::SessionRecordings)+ .await;+ Restriction {+ drop: applied.should_drop(),+ dlq: applied.redirect_to_dlq(),+ force_overflow: applied.force_overflow(),+ }+ }+}++fn sanitize(value: &str) -> Cow<'_, str> {+ if value.contains('\0') {+ Cow::Owned(value.replace('\0', "\u{FFFD}"))+ } else {+ Cow::Borrowed(value)+ }+}++/// The seen and blocked keys of v4's own limiter. Its namespace keeps them apart from the main lane's and the v3+/// mirror's, as the mirror keeps its own.+const SEEN_PREFIX: &str = "@posthog/ai-research-v4/session-seen";+const BLOCKED_PREFIX: &str = "@posthog/ai-research-v4/session-blocked";++/// The main lane's TTL for both keys: twice the longest session.+const STATE_TTL_SECS: u64 = 2 * 24 * 3600;++/// The main lane's local caches in front of Redis. They are LRU, as the main lane's are: the limiter asks for a seen+/// session with `contains_key`, which does not count as a read, so moka's default policy would refuse every new+/// session once the cache is full.+const LOCAL_CACHE_SIZE: u64 = 100_000;+const LOCAL_CACHE_TTL: Duration = Duration::from_secs(300);++/// One poll batch's reads and writes together. A Redis that does not answer delays each batch by this much at most.+const LIMITER_DEADLINE: Duration = Duration::from_secs(1);++fn local_cache<V: Clone + Send + Sync + 'static>() -> moka::sync::Cache<SessionKey, V> {+ moka::sync::Cache::builder()+ .max_capacity(LOCAL_CACHE_SIZE)+ .time_to_live(LOCAL_CACHE_TTL)+ .eviction_policy(moka::policy::EvictionPolicy::lru())+ .build()+}++fn state_key(prefix: &str, session: &SessionKey) -> String {+ format!("{prefix}:{}:{}", session.team_id, session.session_id)+}++/// A token bucket per team, as the main lane's `Limiter`.+struct Buckets {+ capacity: f64,+ replenish_per_sec: f64,+ teams: HashMap<i64, (f64, tokio::time::Instant)>,+}++impl Buckets {+ fn take(&mut self, team_id: i64, now: tokio::time::Instant) -> bool {+ let (tokens, at) = self.teams.entry(team_id).or_insert((self.capacity, now));+ let refill = now.saturating_duration_since(*at).as_secs_f64() * self.replenish_per_sec;+ *tokens = (*tokens + refill).min(self.capacity);+ *at = now;+ if *tokens < 1.0 {+ return false;+ }+ *tokens -= 1.0;+ true+ }+}++/// The main lane's new-session rate limiter, run by v4 for itself. Each team's new sessions take a token from its+/// bucket, and a session denied one is blocked as a whole, so v4 never keeps half of it.+///+/// Redis errors fail open, as in the main lane: a session whose state is unknown is not blocked, and a block that+/// is not written holds in this process.+pub struct SessionLimiter {+ redis: Arc<dyn Client + Send + Sync>,+ buckets: Mutex<Buckets>,+ blocking_enabled: bool,+ seen: moka::sync::Cache<SessionKey, ()>,+ blocked: moka::sync::Cache<SessionKey, bool>,+}++impl SessionLimiter {+ pub async fn connect(+ redis_url: String,+ capacity: f64,+ replenish_per_sec: f64,+ blocking_enabled: bool,+ ) -> Result<Self, CustomRedisError> {+ let redis = RedisClient::with_config(+ redis_url,+ CompressionConfig::disabled(),+ RedisValueFormat::Utf8,+ Some(LIMITER_DEADLINE),+ Some(LIMITER_DEADLINE),+ )+ .await?;+ Ok(Self::new(+ Arc::new(redis),+ capacity,+ replenish_per_sec,+ blocking_enabled,+ ))+ }++ pub fn new(+ redis: Arc<dyn Client + Send + Sync>,+ capacity: f64,+ replenish_per_sec: f64,+ blocking_enabled: bool,+ ) -> Self {+ Self {+ redis,+ buckets: Mutex::new(Buckets {+ capacity,+ replenish_per_sec,+ teams: HashMap::new(),+ }),+ blocking_enabled,+ seen: local_cache(),+ blocked: local_cache(),+ }+ }++ /// Returns which of a poll batch's sessions are blocked. A session is new when it is neither seen nor blocked,+ /// and only a new session takes a token, so a blocked session does not drain its team's bucket.+ pub async fn blocked(&self, sessions: &BTreeSet<SessionKey>) -> HashSet<SessionKey> {+ let deadline = tokio::time::Instant::now() + LIMITER_DEADLINE;+ let mut blocked = HashSet::new();+ let mut seen = HashSet::new();+ let mut reads: Vec<(String, bool, &SessionKey)> = Vec::new();+ for session in sessions {+ if self.seen.contains_key(session) {+ seen.insert(session);+ } else {+ reads.push((state_key(SEEN_PREFIX, session), false, session));+ }+ match self.blocked.get(session) {+ Some(true) => {+ blocked.insert(session.clone());+ }+ Some(false) => {}+ None => reads.push((state_key(BLOCKED_PREFIX, session), true, session)),+ }+ }+ let mut unknown = HashSet::new();+ if !reads.is_empty() {+ let keys = reads.iter().map(|(key, _, _)| key.clone()).collect();+ match self.call(deadline, self.redis.mget(keys)).await {+ Ok(values) => {+ for ((_, is_block, session), value) in reads.iter().zip(values) {+ let set = value.is_some();+ if *is_block {+ self.blocked.insert((*session).clone(), set);+ if set {+ blocked.insert((*session).clone());+ }+ } else if set {+ self.seen.insert((*session).clone(), ());+ seen.insert(session);+ }+ }+ }+ Err(()) => unknown.extend(reads.iter().map(|(_, _, session)| *session)),+ }+ }++ let now = tokio::time::Instant::now();+ let mut writes = Vec::new();+ {+ let mut buckets = self.buckets.lock().unwrap();+ for session in sessions {+ if seen.contains(session) || blocked.contains(session) || unknown.contains(session)+ {+ continue;+ }+ counter!("ai_research_v4_recorder_new_sessions_total").increment(1);+ let blocking = self.blocking_enabled;+ let allowed = buckets.take(session.team_id, now);+ if !allowed {+ let outcome = if blocking { "blocked" } else { "counted" };+ counter!("ai_research_v4_recorder_rate_limited_sessions_total", "outcome" => outcome)+ .increment(1);+ }+ let prefix = if allowed || !blocking {+ self.seen.insert(session.clone(), ());+ SEEN_PREFIX+ } else {+ self.blocked.insert(session.clone(), true);+ blocked.insert(session.clone());+ BLOCKED_PREFIX+ };+ writes.push(PipelineCommand::SetEx {+ key: state_key(prefix, session),+ value: "1".to_owned(),+ seconds: STATE_TTL_SECS,+ format: RedisValueFormat::Utf8,+ });+ }+ }+ if !writes.is_empty() {+ let written = self+ .call(deadline, self.redis.execute_pipeline(writes))+ .await;+ if let Ok(results) = written {+ if let Some(Err(error)) = results.into_iter().find(Result::is_err) {+ tracing::warn!(%error, "writing the session limiter's state failed");+ counter!("ai_research_v4_recorder_session_limiter_redis_errors_total")+ .increment(1);+ }+ }+ }+ blocked+ }++ /// Runs one Redis call within the batch's deadline. A failure is counted, and a dead connection reconnects.+ async fn call<T>(+ &self,+ deadline: tokio::time::Instant,+ call: impl std::future::Future<Output = Result<T, CustomRedisError>>,+ ) -> Result<T, ()> {+ let error = match tokio::time::timeout_at(deadline, call).await {+ Ok(Ok(value)) => return Ok(value),+ Ok(Err(error)) => error,+ Err(_) => CustomRedisError::Timeout,+ };+ tracing::warn!(%error, "the session limiter's redis call failed");+ counter!("ai_research_v4_recorder_session_limiter_redis_errors_total").increment(1);+ // The connection does not come back by itself after a failover.+ if error.is_unrecoverable_error() {+ self.redis.heal().await;+ }+ Err(())+ }+}++#[cfg(test)]+mod tests {+ use capture::event_restrictions::{+ Restriction as Rule, RestrictionFilters, RestrictionManager, RestrictionScope,+ RestrictionType,+ };+ use common_redis::{MockRedisClient, MockRedisValue, RedisErrorKind};++ use super::*;++ #[tokio::test]+ async fn matches_restrictions_on_the_session_id_as_the_main_lane_reads_it() {+ let session = "019a2f3e-8c00-7000-8000-00000000000a";+ let mut manager = RestrictionManager::new();+ manager.insert_restrictions(+ Pipeline::SessionRecordings,+ "token",+ vec![Rule {+ restriction_type: RestrictionType::DropEvent,+ scope: RestrictionScope::Filtered(RestrictionFilters {+ session_ids: HashSet::from([session.to_owned()]),+ ..RestrictionFilters::default()+ }),+ args: None,+ }],+ );+ let restrictions = Restrictions::default();+ let now = restrictions.service.update(manager).await;+ let upper = session.to_ascii_uppercase();+ let headers = Headers {+ token: Some("token"),+ session_id: Some(&upper),+ ..Headers::default()+ };+ assert!(restrictions.get(&headers, now).await.drop);+ }++ fn session(team_id: i64, id: &str) -> SessionKey {+ SessionKey {+ team_id,+ session_id: id.to_owned(),+ }+ }++ fn on(sessions: &[SessionKey]) -> BTreeSet<SessionKey> {+ sessions.iter().cloned().collect()+ }++ fn sorted(blocked: HashSet<SessionKey>) -> Vec<String> {+ let mut ids: Vec<String> = blocked.into_iter().map(|s| s.session_id).collect();+ ids.sort();+ ids+ }++ #[tokio::test(start_paused = true)]+ async fn a_team_bucket_holds_its_capacity_and_refills_at_its_rate() {+ let mut buckets = Buckets {+ capacity: 2.0,+ replenish_per_sec: 0.5,+ teams: HashMap::new(),+ };+ let now = tokio::time::Instant::now();+ let taken = |buckets: &mut Buckets, at: tokio::time::Instant| {+ (0..4).filter(|_| buckets.take(7, at)).count()+ };+ assert_eq!(taken(&mut buckets, now), 2);+ assert_eq!(taken(&mut buckets, now + Duration::from_secs(2)), 1);+ // A long pause refills to the capacity, not past it.+ assert_eq!(taken(&mut buckets, now + Duration::from_secs(600)), 2);+ }++ #[tokio::test(start_paused = true)]+ async fn a_flood_of_new_sessions_blocks_the_excess_of_its_team_only() {+ let redis = MockRedisClient::new();+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 2.0, 1.0, true);+ let flood: Vec<SessionKey> = (0..4)+ .map(|i| session(7, &format!("s{i}")))+ .chain([session(8, "other")])+ .collect();+ assert_eq!(sorted(limiter.blocked(&on(&flood)).await), ["s2", "s3"]);++ // Later messages: a blocked session stays blocked, and a seen one takes no token from the empty bucket.+ let later = [session(7, "s0"), session(7, "s3")];+ assert_eq!(sorted(limiter.blocked(&on(&later)).await), ["s3"]);++ let writes: Vec<String> = redis+ .get_calls()+ .into_iter()+ .filter_map(|c| match c.value {+ MockRedisValue::StringWithTTLAndFormat(_, STATE_TTL_SECS, _) => Some(c.key),+ _ => None,+ })+ .collect();+ assert_eq!(+ writes,+ [+ format!("{SEEN_PREFIX}:7:s0"),+ format!("{SEEN_PREFIX}:7:s1"),+ format!("{BLOCKED_PREFIX}:7:s2"),+ format!("{BLOCKED_PREFIX}:7:s3"),+ format!("{SEEN_PREFIX}:8:other"),+ ]+ );+ }++ #[tokio::test(start_paused = true)]+ async fn reads_the_state_other_processes_wrote() {+ let mut redis = MockRedisClient::new();+ redis.mget_ret(&format!("{SEEN_PREFIX}:7:seen"), Some(b"1".to_vec()));+ redis.mget_ret(&format!("{BLOCKED_PREFIX}:7:blocked"), Some(b"1".to_vec()));+ let limiter = SessionLimiter::new(Arc::new(redis), 1.0, 0.0, true);+ let sessions = [+ session(7, "seen"),+ session(7, "blocked"),+ session(7, "new"),+ session(7, "late"),+ ];+ // Neither known session takes the one token, so the first new one gets it.+ assert_eq!(+ sorted(limiter.blocked(&on(&sessions)).await),+ ["blocked", "new"]+ );+ }++ #[tokio::test(start_paused = true)]+ async fn fails_open_when_redis_does_not_answer() {+ let redis = MockRedisClient::new().mget_error(CustomRedisError::from_redis_kind(+ RedisErrorKind::ParseError,+ "the connection closed in a reply",+ ));+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, true);+ assert!(limiter.blocked(&on(&[session(7, "a")])).await.is_empty());+ let ops: Vec<_> = redis.get_calls().into_iter().map(|c| c.op).collect();+ assert_eq!(ops, ["mget", "heal"]);++ // A block that misses the deadline holds in this process.+ let redis = MockRedisClient::new().pipeline_block(Duration::from_secs(60));+ let limiter = SessionLimiter::new(Arc::new(redis), 0.0, 0.0, true);+ let started = tokio::time::Instant::now();+ let a = on(&[session(7, "a")]);+ assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);+ assert_eq!(started.elapsed(), LIMITER_DEADLINE);+ assert_eq!(sorted(limiter.blocked(&a).await), ["a"]);+ }++ #[tokio::test(start_paused = true)]+ async fn without_blocking_only_counts() {+ let redis = MockRedisClient::new();+ let limiter = SessionLimiter::new(Arc::new(redis.clone()), 0.0, 0.0, false);+ assert!(limiter+ .blocked(&on(&[session(7, "a"), session(7, "b")]))+ .await+ .is_empty());+ assert!(redis+ .get_calls()+ .iter()+ .all(|c| !c.key.starts_with(BLOCKED_PREFIX)));+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex 728dcc074838..18774fc8b630 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -3,6 +3,8 @@ use std::time::Duration;use axum::{routing::get, Router};use common_database::{get_pool_with_config, PoolConfig};+use common_kafka::config::KafkaConfig;+use common_kafka::kafka_producer::create_kafka_producer;use common_metrics::setup_metrics_routes;use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};@@ -15,7 +17,8 @@ use replay_core_v4::keys::batch::KeyStore;use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};use replay_recorder_v4::config::Config;-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Writer};+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources, Writer};+use replay_recorder_v4::limits::{Restrictions, SessionLimiter};use replay_recorder_v4::store::S3Store;use replay_recorder_v4::teams::TeamCache;@@ -43,6 +46,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.with_graceful_shutdown(Duration::from_secs(15)).with_liveness_deadline(Duration::from_secs(60)),);+ let restrictions_handle = manager.register(+ "event-restrictions",+ ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),+ );+ // The producer reports itself live from librdkafka's statistics callback.+ let producer_handle = config.write_enabled.then(|| {+ manager.register(+ "image-fetch-producer",+ ComponentOptions::new()+ .with_graceful_shutdown(Duration::from_secs(5))+ .with_liveness_deadline(Duration::from_secs(60)),+ )+ });let metrics_handle = manager.register("metrics-server",ComponentOptions::new().is_observability(true),@@ -104,6 +120,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {teams_handle,));+ let restrictions = Restrictions::default();+ tokio::spawn(restrictions.clone().refresh_forever(+ config.event_restrictions_redis_url.clone(),+ Duration::from_secs(config.event_restrictions_refresh_secs),+ restrictions_handle,+ ));+ // Before the first load no restriction applies, so nothing is read until it is in.+ if !restrictions.wait_loaded(Duration::from_secs(60)).await {+ return Err("the event ingestion restrictions did not load".into());+ }+ let (capacity, replenish_rate) = config.new_session_bucket();+ let limiter = SessionLimiter::connect(+ config.redis_url.clone(),+ capacity,+ replenish_rate,+ config.new_session_blocking_enabled,+ )+ .await?;+let consumer = RecorderConsumer::new(&config.kafka,&config.kafka_consumer_group,@@ -115,8 +150,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {flush_interval: Duration::from_millis(config.flush_interval_ms),flush_max_bytes: config.flush_max_bytes,session_idle: Duration::from_secs(config.session_idle_secs),+ redirecting_topic: (config.ingestion_overflow_mode == "redirect")+ .then(|| config.kafka_topic.clone()),};- let writer = if config.write_enabled {+ let writer = if let Some(producer_handle) = producer_handle {let aws = aws_config::defaults(aws_config::BehaviorVersion::latest()).load().await;@@ -144,11 +181,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {store: Arc::new(S3Store::new(aws_sdk_s3::Client::new(&aws))),rrweb_bucket: config.rrweb_bucket.clone(),images_bucket: config.images_bucket.clone(),- producer: rdkafka::ClientConfig::new()- .set("bootstrap.servers", &config.kafka.kafka_hosts)- .set("compression.codec", "lz4")- .set("linger.ms", "20")- .create()?,+ producer: create_kafka_producer(+ &KafkaConfig {+ kafka_compression_codec: "lz4".to_owned(),+ kafka_message_timeout_ms: config.image_fetch_delivery_timeout_ms,+ ..config.kafka.clone()+ },+ producer_handle,+ )+ .await?,image_fetch_topic: config.image_fetch_topic.clone(),sent_urls: replay_recorder_v4::urls::sent_urls(500_000),})@@ -156,7 +197,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {tracing::info!("writes are off: the recorder stores nothing");None};- tokio::spawn(run(consumer, settings, teams, writer, consumer_handle));+ let sources = Sources {+ teams,+ restrictions,+ limiter,+ };+ tokio::spawn(run(consumer, settings, sources, writer, consumer_handle));monitor.wait().await?;Ok(())diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rsindex 5a74bc918008..272ef219a227 100644--- a/rust/ai-research/replay-recorder-v4/src/process.rs+++ b/rust/ai-research/replay-recorder-v4/src/process.rs@@ -1,4 +1,9 @@//! What the recorder does with one Kafka message, apart from Kafka itself.+//!+//! Admission reads only the Kafka key and headers, so a message the main lane or the v3 mirror would drop is never+//! decoded. The payload is read later, at the flush.++use std::borrow::Cow;use replay_core_v4::gate::in_sample;use replay_core_v4::payload::{decompress_payload, is_web_replay};@@ -12,11 +17,17 @@ use crate::teams::Team;pub enum Skip {NotSampled,NoKey,- Undecompressable,- Unreadable,+ Restricted,+ RestrictedToDlq,+ Overflowed,+ MissingHeader,UnknownTeam,NotOptedIn,Session(DropReason),+ Blocked,+ Undecompressable,+ Unreadable,+ SessionMismatch,NotWeb,}@@ -25,11 +36,17 @@ impl Skip {match self {Skip::NotSampled => "not_sampled",Skip::NoKey => "no_key",- Skip::Undecompressable => "undecompressable",- Skip::Unreadable => "unreadable",+ Skip::Restricted => "blocked_token",+ Skip::RestrictedToDlq => "restricted_to_dlq",+ Skip::Overflowed => "redirected_to_overflow",+ Skip::MissingHeader => "missing_header",Skip::UnknownTeam => "unknown_team",Skip::NotOptedIn => "not_opted_in",Skip::Session(reason) => reason.as_str(),+ Skip::Blocked => "session_blocked",+ Skip::Undecompressable => "undecompressable",+ Skip::Unreadable => "unreadable",+ Skip::SessionMismatch => "session_mismatch",Skip::NotWeb => "not_web",}}@@ -45,26 +62,102 @@ pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {}}-/// The v3 mirror's admission rules: a consenting organization, a recent UUIDv7 session, a web replay. Returns the team.-pub fn admit(message: &Message, team: Option<Team>, now_ms: i64) -> Result<i64, Skip> {+/// The headers capture sets on every replay message.+#[derive(Debug, Clone, Copy, Default)]+pub struct Headers<'a> {+ pub token: Option<&'a str>,+ pub session_id: Option<&'a str>,+ pub distinct_id: Option<&'a str>,+ pub event: Option<&'a str>,+ pub uuid: Option<&'a str>,+}++/// The event ingestion restrictions that apply to a message.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct Restriction {+ pub drop: bool,+ pub dlq: bool,+ pub force_overflow: bool,+}++/// A message that may be recorded: its team, and its session ID as the replay lanes key it.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Admitted {+ pub team_id: i64,+ pub session_id: String,+}++/// The replay lanes lower-case a session ID that is a UUID and keep any other as it is.+pub fn normalize_session_id(session_id: &str) -> Cow<'_, str> {+ let bytes = session_id.as_bytes();+ let uuid = bytes.len() == 36+ && bytes.iter().enumerate().all(|(i, b)| match i {+ 8 | 13 | 18 | 23 => *b == b'-',+ _ => b.is_ascii_hexdigit(),+ });+ if uuid && bytes.iter().any(u8::is_ascii_uppercase) {+ Cow::Owned(session_id.to_ascii_lowercase())+ } else {+ Cow::Borrowed(session_id)+ }+}++/// The main lane's and the v3 mirror's rules before the payload, in their order: restrictions, the headers capture+/// guarantees, a team with recordings and a retention period, a recent UUIDv7 session, a consenting organization.+/// `redirect_overflow` is true on the main topic when the main lane moves force-overflow messages to the overflow+/// topic.+pub fn admit(+ headers: &Headers<'_>,+ restriction: Restriction,+ redirect_overflow: bool,+ team: Option<Team>,+ now_ms: i64,+) -> Result<Admitted, Skip> {+ if restriction.drop {+ return Err(Skip::Restricted);+ }+ if restriction.dlq {+ return Err(Skip::RestrictedToDlq);+ }+ if restriction.force_overflow && redirect_overflow {+ return Err(Skip::Overflowed);+ }+ let (Some(_), Some(session_id), Some(_)) = (+ headers.token.filter(|t| !t.is_empty()),+ headers.session_id.filter(|s| !s.is_empty()),+ headers.distinct_id.filter(|d| !d.is_empty()),+ ) else {+ return Err(Skip::MissingHeader);+ };let team = team.ok_or(Skip::UnknownTeam)?;+ let session_id = normalize_session_id(session_id);+ if let Some(reason) = drop_reason(&session_id, now_ms) {+ return Err(Skip::Session(reason));+ }if !team.ai_training_opted_in {return Err(Skip::NotOptedIn);}- if let Some(reason) = drop_reason(&message.session_id, now_ms) {- return Err(Skip::Session(reason));+ Ok(Admitted {+ team_id: team.id,+ session_id: session_id.into_owned(),+ })+}++/// Decompresses and reads an admitted message.+pub fn read(payload: Vec<u8>, content_encoding: Option<&str>) -> Result<Message, Skip> {+ let raw = decompress_payload(payload, content_encoding).map_err(|_| Skip::Undecompressable)?;+ read_message(raw).map_err(|_| Skip::Unreadable)+}++/// The rules that need the payload: the session the headers named, and a web replay.+pub fn check_read(message: &Message, session_id: &str) -> Result<(), Skip> {+ if normalize_session_id(&message.session_id) != session_id {+ return Err(Skip::SessionMismatch);}if !is_web_replay(message.snapshot_source.as_deref()) {return Err(Skip::NotWeb);}- Ok(team.id)-}--/// Decompresses and reads a sampled message.-pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {- let raw = decompress_payload(payload.to_vec(), content_encoding)- .map_err(|_| Skip::Undecompressable)?;- read_message(raw).map_err(|_| Skip::Unreadable)+ Ok(())}#[cfg(test)]@@ -74,19 +167,34 @@ mod tests {use super::*;- fn capture_message() -> Vec<u8> {- let inner = json!({"event": "$snapshot_items", "properties": {- "$session_id": "019a2f3e-8c00-7000-8000-000000000007",+ const SESSION: &str = "019a2f3e-8c00-7000-8000-000000000007";++ fn capture_message(session: &str, source: Option<&str>) -> Vec<u8> {+ let mut properties = json!({+ "$session_id": session,"$snapshot_items": [{"type": 3, "timestamp": 1, "data": {"source": 1}}]- }});+ });+ if let Some(source) = source {+ properties["$snapshot_source"] = json!(source);+ }+ let inner = json!({"event": "$snapshot_items", "properties": properties});json!({"distinct_id": "d", "data": inner.to_string()}).to_string().into_bytes()}+ fn headers(session: &str) -> Headers<'_> {+ Headers {+ token: Some("token"),+ session_id: Some(session),+ distinct_id: Some("d"),+ ..Headers::default()+ }+ }+#[test]fn gates_on_the_kafka_key_before_reading() {- let session = b"019a2f3e-8c00-7000-8000-000000000007";+ let session = SESSION.as_bytes();assert_eq!(sampled(Some(session), 100.0), Ok(()));assert_eq!(sampled(Some(session), 0.0), Err(Skip::NotSampled));assert_eq!(sampled(None, 100.0), Err(Skip::NoKey));@@ -95,43 +203,121 @@ mod tests {#[test]fn reads_plain_and_gzipped_capture_messages() {- let plain = capture_message();+ let plain = capture_message(SESSION, Some("web"));for payload in [plain.clone(), gzip(&plain).unwrap()] {- let message = read(&payload, None).unwrap();- assert_eq!(message.session_id, "019a2f3e-8c00-7000-8000-000000000007");+ let message = read(payload, None).unwrap();+ assert_eq!(message.session_id, SESSION);assert_eq!(message.events.len(), 1);}- assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);+ assert_eq!(+ read(b"not json".to_vec(), None).unwrap_err(),+ Skip::Unreadable+ );}#[test]- fn admits_only_what_the_v3_mirror_admits() {- let session = "019a2f3e-8c00-7000-8000-000000000007";+ fn checks_the_payload_against_the_headers() {+ let message = |session: &str, source: Option<&str>| {+ read(capture_message(session, source), None).unwrap()+ };+ let cases = [+ (message(SESSION, Some("web")), Ok(())),+ (message(&SESSION.to_ascii_uppercase(), Some("web")), Ok(())),+ (+ message("019a2f3e-8c00-7000-8000-000000000008", Some("web")),+ Err(Skip::SessionMismatch),+ ),+ (message(SESSION, Some("mobile")), Err(Skip::NotWeb)),+ (message(SESSION, None), Err(Skip::NotWeb)),+ ];+ for (message, expected) in cases {+ assert_eq!(check_read(&message, SESSION), expected, "{message:?}");+ }+ }++ #[test]+ fn admits_only_what_the_main_lane_and_the_v3_mirror_admit() {// 019a2f3e8c00 is the session start in milliseconds.let now = 0x019a_2f3e_8c00_i64 + 60_000;- let message = |source: Option<&str>| {- let mut properties = json!({"$session_id": session, "$snapshot_items": []});- if let Some(source) = source {- properties["$snapshot_source"] = json!(source);- }- let inner = json!({"event": "$snapshot_items", "properties": properties});- read(- json!({"distinct_id": "d", "data": inner.to_string()})- .to_string()- .as_bytes(),- None,- )- .unwrap()- };+ let upper = SESSION.to_ascii_uppercase();let opted_in = Some(Team {id: 7,ai_training_opted_in: true,});+ let open = Restriction::default();+ let overflow = Restriction {+ force_overflow: true,+ ..open+ };+ let admitted = Ok(Admitted {+ team_id: 7,+ session_id: SESSION.to_owned(),+ });let cases = [- (message(Some("web")), opted_in, now, Ok(7)),- (message(Some("web")), None, now, Err(Skip::UnknownTeam)),(- message(Some("web")),+ headers(SESSION),+ open,+ false,+ opted_in,+ now,+ admitted.clone(),+ ),+ (+ headers(&upper),+ open,+ false,+ opted_in,+ now,+ admitted.clone(),+ ),+ (headers(SESSION), overflow, false, opted_in, now, admitted),+ (+ headers(SESSION),+ overflow,+ true,+ opted_in,+ now,+ Err(Skip::Overflowed),+ ),+ (+ headers(SESSION),+ Restriction { drop: true, ..open },+ false,+ opted_in,+ now,+ Err(Skip::Restricted),+ ),+ (+ headers(SESSION),+ Restriction { dlq: true, ..open },+ false,+ opted_in,+ now,+ Err(Skip::RestrictedToDlq),+ ),+ (+ Headers {+ distinct_id: None,+ ..headers(SESSION)+ },+ open,+ false,+ opted_in,+ now,+ Err(Skip::MissingHeader),+ ),+ (+ headers(SESSION),+ open,+ false,+ None,+ now,+ Err(Skip::UnknownTeam),+ ),+ (+ headers(SESSION),+ open,+ false,Some(Team {id: 7,ai_training_opted_in: false,@@ -140,20 +326,19 @@ mod tests {Err(Skip::NotOptedIn),),(- message(Some("web")),+ headers(SESSION),+ open,+ false,opted_in,now + 15 * 86_400_000,Err(Skip::Session(DropReason::TooOld)),),- (message(Some("mobile")), opted_in, now, Err(Skip::NotWeb)),- (message(None), opted_in, now, Err(Skip::NotWeb)),];- for (message, team, now, expected) in cases {+ for (headers, restriction, redirect, team, now, expected) in cases {assert_eq!(- admit(&message, team, now),+ admit(&headers, restriction, redirect, team, now),expected,- "{team:?} {:?}",- message.snapshot_source+ "{headers:?} {restriction:?} {redirect} {team:?}");}}