:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): admit sessions by the v3 mirror's rules (#117968)
Nicholas Waltzcfaa5405d783
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 439e940dfc85..209ef864d212 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11051,8 +11051,10 @@ dependencies = [name = "replay-recorder-v4"version = "0.1.0"dependencies = [+ "arc-swap","axum 0.7.9","common-alloc",+ "common-database","common-kafka","common-metrics","envconfig",@@ -11061,6 +11063,7 @@ dependencies = ["rdkafka","replay-core-v4","serde_json",+ "sqlx","tokio","tracing","tracing-subscriber",diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex 699399a22682..0b8bee87bfa4 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -10,15 +10,18 @@ workspace = true[dependencies]common-alloc = { path = "../../common/alloc" }+common-database = { path = "../../common/database" }common-kafka = { path = "../../common/kafka" }common-metrics = { path = "../../common/metrics" }lifecycle = { path = "../../common/lifecycle" }replay-core-v4 = { path = "../replay-core-v4" }+arc-swap = { workspace = true }axum = { workspace = true }envconfig = { workspace = true }metrics = { workspace = true }rdkafka = { workspace = true }+sqlx = { workspace = true }tokio = { workspace = true }tracing = { workspace = true }tracing-subscriber = { workspace = true }diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 3869dfc227f9..3bdb82fb934e 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -20,6 +20,13 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_SAMPLE_PERCENTAGE", default = "0")]pub sample_percentage: f64,+ /// The Postgres database the team list is read from. A read replica is enough.+ pub database_url: String,++ /// How often the team list refreshes. Consent changes take effect within this time, as in v3.+ #[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]+ pub team_refresh_secs: u64,+#[envconfig(default = "9104")]pub metrics_port: u16,}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 632d80aa9993..b2c42823c6df 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -1,6 +1,7 @@//! The consume loop.-use std::time::Duration;+use std::sync::Arc;+use std::time::{Duration, SystemTime, UNIX_EPOCH};use common_kafka::config::KafkaConfig;use lifecycle::Handle;@@ -9,7 +10,8 @@ use rdkafka::consumer::{Consumer, StreamConsumer};use rdkafka::message::{Headers, Message as _};use rdkafka::ClientConfig;-use crate::process::{read, sampled};+use crate::process::{admit, read, sampled};+use crate::teams::TeamCache;pub fn create_consumer(kafka: &KafkaConfig,@@ -47,7 +49,7 @@ fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str>}/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.-pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {+pub async fn run(consumer: StreamConsumer, percentage: f64, teams: Arc<TeamCache>, handle: Handle) {let shutdown = handle.shutdown_signal();tokio::pin!(shutdown);loop {@@ -65,12 +67,20 @@ pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {continue;}};- let outcome = sampled(message.key(), percentage).and_then(|()| {- read(- message.payload().unwrap_or_default(),- header(&message, "content-encoding"),- )- });+ let now_ms = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_or(0, |d| d.as_millis() as i64);+ let outcome = sampled(message.key(), percentage)+ .and_then(|()| {+ read(+ message.payload().unwrap_or_default(),+ header(&message, "content-encoding"),+ )+ })+ .and_then(|read| {+ let team = header(&message, "token").and_then(|token| teams.get(token));+ admit(&read, team, now_ms).map(|_| read)+ });match outcome {Ok(read) => {counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex 9a96af680d24..ae01300ec013 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -4,3 +4,4 @@pub mod config;pub mod consumer;pub mod process;+pub mod teams;diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex cfb4c5b5f900..239705b13325 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -1,6 +1,8 @@+use std::sync::Arc;use std::time::Duration;use axum::{routing::get, Router};+use common_database::{get_pool_with_config, PoolConfig};use common_metrics::setup_metrics_routes;use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};@@ -11,6 +13,7 @@ use tracing_subscriber::{fmt, EnvFilter};use replay_recorder_v4::config::Config;use replay_recorder_v4::consumer::{create_consumer, run};+use replay_recorder_v4::teams::TeamCache;common_alloc::used!();@@ -40,6 +43,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {"metrics-server",ComponentOptions::new().is_observability(true),);+ let teams_handle = manager.register(+ "team-refresh",+ ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),+ );let readiness = manager.readiness_handler();let liveness = manager.liveness_handler();let monitor = manager.monitor_background();@@ -65,13 +72,46 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.expect("metrics server");});+ if config.team_refresh_secs == 0 {+ return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".into());+ }+ // The first team list must load before any message is read: an empty list would skip every message as unknown.+ let pool = get_pool_with_config(+ &config.database_url,+ PoolConfig {+ max_connections: 2,+ pool_name: Some("replay-recorder-v4".to_owned()),+ ..Default::default()+ },+ )?;+ let teams = Arc::new(TeamCache::new(TeamCache::max_age_for(Duration::from_secs(+ config.team_refresh_secs,+ ))));+ let loaded = tokio::time::timeout(+ replay_recorder_v4::teams::REFRESH_TIMEOUT,+ teams.refresh(&pool),+ )+ .await+ .map_err(|_| "the first team list did not load in time")??;+ tracing::info!(teams = loaded, "loaded the team list");+ tokio::spawn(teams.clone().refresh_every(+ pool,+ Duration::from_secs(config.team_refresh_secs),+ teams_handle,+ ));+let consumer = create_consumer(&config.kafka,&config.kafka_consumer_group,&config.kafka_consumer_offset_reset,&config.kafka_topic,)?;- tokio::spawn(run(consumer, config.sample_percentage, consumer_handle));+ tokio::spawn(run(+ consumer,+ config.sample_percentage,+ teams,+ consumer_handle,+ ));monitor.wait().await?;Ok(())diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rsindex 938330b5f3c4..5a74bc918008 100644--- a/rust/ai-research/replay-recorder-v4/src/process.rs+++ b/rust/ai-research/replay-recorder-v4/src/process.rs@@ -1,8 +1,11 @@//! What the recorder does with one Kafka message, apart from Kafka itself.use replay_core_v4::gate::in_sample;-use replay_core_v4::payload::decompress_payload;+use replay_core_v4::payload::{decompress_payload, is_web_replay};use replay_core_v4::scan::message::{read_message, Message};+use replay_core_v4::session::{drop_reason, DropReason};++use crate::teams::Team;/// Why the recorder leaves a message out. The label is the metric value.#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -11,6 +14,10 @@ pub enum Skip {NoKey,Undecompressable,Unreadable,+ UnknownTeam,+ NotOptedIn,+ Session(DropReason),+ NotWeb,}impl Skip {@@ -20,6 +27,10 @@ impl Skip {Skip::NoKey => "no_key",Skip::Undecompressable => "undecompressable",Skip::Unreadable => "unreadable",+ Skip::UnknownTeam => "unknown_team",+ Skip::NotOptedIn => "not_opted_in",+ Skip::Session(reason) => reason.as_str(),+ Skip::NotWeb => "not_web",}}}@@ -34,6 +45,21 @@ pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {}}+/// The v3 mirror's admission rules: a consenting organization, a recent UUIDv7 session, a web replay. Returns the team.+pub fn admit(message: &Message, team: Option<Team>, now_ms: i64) -> Result<i64, Skip> {+ let team = team.ok_or(Skip::UnknownTeam)?;+ if !team.ai_training_opted_in {+ return Err(Skip::NotOptedIn);+ }+ if let Some(reason) = drop_reason(&message.session_id, now_ms) {+ return Err(Skip::Session(reason));+ }+ if !is_web_replay(message.snapshot_source.as_deref()) {+ return Err(Skip::NotWeb);+ }+ Ok(team.id)+}+/// Decompresses and reads a sampled message.pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {let raw = decompress_payload(payload.to_vec(), content_encoding)@@ -77,4 +103,58 @@ mod tests {}assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);}++ #[test]+ fn admits_only_what_the_v3_mirror_admits() {+ let session = "019a2f3e-8c00-7000-8000-000000000007";+ // 019a2f3e8c00 is the session start in milliseconds.+ let now = 0x019a_2f3e_8c00_i64 + 60_000;+ let message = |source: Option<&str>| {+ let mut properties = json!({"$session_id": session, "$snapshot_items": []});+ if let Some(source) = source {+ properties["$snapshot_source"] = json!(source);+ }+ let inner = json!({"event": "$snapshot_items", "properties": properties});+ read(+ json!({"distinct_id": "d", "data": inner.to_string()})+ .to_string()+ .as_bytes(),+ None,+ )+ .unwrap()+ };+ let opted_in = Some(Team {+ id: 7,+ ai_training_opted_in: true,+ });+ let cases = [+ (message(Some("web")), opted_in, now, Ok(7)),+ (message(Some("web")), None, now, Err(Skip::UnknownTeam)),+ (+ message(Some("web")),+ Some(Team {+ id: 7,+ ai_training_opted_in: false,+ }),+ now,+ Err(Skip::NotOptedIn),+ ),+ (+ message(Some("web")),+ opted_in,+ now + 15 * 86_400_000,+ Err(Skip::Session(DropReason::TooOld)),+ ),+ (message(Some("mobile")), opted_in, now, Err(Skip::NotWeb)),+ (message(None), opted_in, now, Err(Skip::NotWeb)),+ ];+ for (message, team, now, expected) in cases {+ assert_eq!(+ admit(&message, team, now),+ expected,+ "{team:?} {:?}",+ message.snapshot_source+ );+ }+ }}diff --git a/rust/ai-research/replay-recorder-v4/src/teams.rs b/rust/ai-research/replay-recorder-v4/src/teams.rsnew file mode 100644index 000000000000..8ea955f27448--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/teams.rs@@ -0,0 +1,202 @@+//! The teams that may be recorded: the replay teams of organizations that opt in to AI training.++use std::collections::HashMap;+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::Arc;+use std::time::Duration;++use arc_swap::ArcSwap;+use sqlx::PgPool;+use tokio::time::Instant;++/// The query the v3 mirror's team service runs, narrowed to the columns v4 reads. The main lane drops a team without+/// a valid retention period, so the list leaves it out.+const TEAMS: &str = "+ SELECT t.id::bigint, t.api_token, COALESCE(o.is_ai_training_opted_in, false)+ FROM posthog_team t+ LEFT JOIN posthog_organization o ON o.id = t.organization_id+ WHERE t.session_recording_opt_in = true+ AND t.session_recording_retention_period IN ('30d', '90d', '1y', '5y')+";++/// After this many failed refreshes in a row the list is too old to show opt-outs, so the cache admits no team.+pub const MAX_FAILED_REFRESHES: u32 = 20;++/// A refresh that takes longer fails. Without it a hung query would never count as a failure.+pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(60);++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct Team {+ pub id: i64,+ pub ai_training_opted_in: bool,+}++struct Snapshot {+ teams: HashMap<String, Team>,+ loaded_at: Instant,+}++/// Teams by project token. Lookups read a snapshot, so the consume loop never waits on Postgres.+pub struct TeamCache {+ snapshot: ArcSwap<Snapshot>,+ failed_refreshes: AtomicU32,+ max_age: Duration,+}++impl Default for TeamCache {+ fn default() -> Self {+ Self::from_teams(HashMap::new(), Duration::MAX)+ }+}++impl TeamCache {+ pub fn new(max_age: Duration) -> Self {+ Self::from_teams(HashMap::new(), max_age)+ }++ pub fn from_teams(teams: HashMap<String, Team>, max_age: Duration) -> Self {+ Self {+ snapshot: ArcSwap::from_pointee(Snapshot {+ teams,+ loaded_at: Instant::now(),+ }),+ failed_refreshes: AtomicU32::new(0),+ max_age,+ }+ }++ /// The age at which a refreshing cache admits no team: as long as `MAX_FAILED_REFRESHES` intervals that time out.+ pub fn max_age_for(interval: Duration) -> Duration {+ (interval + REFRESH_TIMEOUT) * MAX_FAILED_REFRESHES+ }++ pub fn get(&self, token: &str) -> Option<Team> {+ if self.failed_refreshes.load(Ordering::Relaxed) >= MAX_FAILED_REFRESHES {+ return None;+ }+ let snapshot = self.snapshot.load();+ if snapshot.loaded_at.elapsed() > self.max_age {+ return None;+ }+ snapshot.teams.get(token).copied()+ }++ /// Counts a failed refresh. Returns true when the cache now admits no team.+ pub fn refresh_failed(&self) -> bool {+ let failures = self.failed_refreshes.fetch_add(1, Ordering::Relaxed) + 1;+ let closed = failures >= MAX_FAILED_REFRESHES;+ metrics::gauge!("ai_research_v4_recorder_teams_closed").set(if closed { 1.0 } else { 0.0 });+ closed+ }++ fn store(&self, teams: HashMap<String, Team>) {+ self.snapshot.store(Arc::new(Snapshot {+ teams,+ loaded_at: Instant::now(),+ }));+ self.failed_refreshes.store(0, Ordering::Relaxed);+ metrics::gauge!("ai_research_v4_recorder_teams_closed").set(0.0);+ }++ pub async fn refresh(&self, pool: &PgPool) -> Result<usize, sqlx::Error> {+ let rows: Vec<(i64, String, bool)> = sqlx::query_as(TEAMS).fetch_all(pool).await?;+ let teams: HashMap<String, Team> = rows+ .into_iter()+ .map(|(id, token, opted_in)| {+ (+ token,+ Team {+ id,+ ai_training_opted_in: opted_in,+ },+ )+ })+ .collect();+ let count = teams.len();+ self.store(teams);+ metrics::gauge!("ai_research_v4_recorder_teams").set(count as f64);+ Ok(count)+ }++ /// Refreshes until shutdown. A failed or timed-out refresh keeps the previous list until `MAX_FAILED_REFRESHES` fail+ /// in a row, then the cache admits no team until a refresh succeeds. If this task ends early or panics, the process+ /// stops.+ pub async fn refresh_every(+ self: Arc<Self>,+ pool: PgPool,+ interval: Duration,+ handle: lifecycle::Handle,+ ) {+ let _guard = handle.process_scope();+ let shutdown = handle.shutdown_signal();+ tokio::pin!(shutdown);+ let mut ticker = tokio::time::interval(interval);+ ticker.tick().await;+ loop {+ tokio::select! {+ _ = &mut shutdown => return,+ _ = ticker.tick() => {}+ }+ let refreshed = match tokio::time::timeout(REFRESH_TIMEOUT, self.refresh(&pool)).await {+ Ok(result) => result.map_err(|error| error.to_string()),+ Err(_) => Err(format!("no answer in {REFRESH_TIMEOUT:?}")),+ };+ metrics::gauge!("ai_research_v4_recorder_team_snapshot_age_seconds")+ .set(self.snapshot.load().loaded_at.elapsed().as_secs_f64());+ if let Err(error) = refreshed {+ metrics::counter!("ai_research_v4_recorder_team_refresh_failures_total")+ .increment(1);+ if self.refresh_failed() {+ tracing::error!(%error, "team refreshes keep failing: admitting no team until one succeeds");+ } else {+ tracing::warn!(%error, "team refresh failed");+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn admits_nothing_after_too_many_failed_refreshes_until_one_succeeds() {+ let team = Team {+ id: 7,+ ai_training_opted_in: true,+ };+ let teams = HashMap::from([("token".to_owned(), team)]);+ let cache = TeamCache::from_teams(teams.clone(), Duration::MAX);+ for _ in 1..MAX_FAILED_REFRESHES {+ assert!(!cache.refresh_failed());+ }+ assert_eq!(cache.get("token"), Some(team));+ assert!(cache.refresh_failed());+ assert_eq!(cache.get("token"), None);++ cache.store(teams);+ assert_eq!(cache.get("token"), Some(team));+ }++ #[test]+ fn admits_nothing_once_the_snapshot_is_too_old_until_a_refresh_succeeds() {+ let team = Team {+ id: 7,+ ai_training_opted_in: true,+ };+ let teams = HashMap::from([("token".to_owned(), team)]);+ let max_age = TeamCache::max_age_for(Duration::from_secs(300));+ let cache = TeamCache::from_teams(teams.clone(), max_age);+ assert_eq!(cache.get("token"), Some(team));++ cache.snapshot.store(Arc::new(Snapshot {+ teams: teams.clone(),+ loaded_at: Instant::now() - max_age - Duration::from_secs(1),+ }));+ assert_eq!(cache.get("token"), None);++ cache.store(teams);+ assert_eq!(cache.get("token"), Some(team));+ }+}