Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): admit sessions by the v3 mirror's rules (#117968)

Nicholas Waltzcfaa5405d783
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 439e940dfc85..209ef864d212 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11051,8 +11051,10 @@ dependencies = [
name = "replay-recorder-v4"
version = "0.1.0"
dependencies = [
+ "arc-swap",
"axum 0.7.9",
"common-alloc",
+ "common-database",
"common-kafka",
"common-metrics",
"envconfig",
@@ -11061,6 +11063,7 @@ dependencies = [
"rdkafka",
"replay-core-v4",
"serde_json",
+ "sqlx",
"tokio",
"tracing",
"tracing-subscriber",
diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.toml
index 699399a22682..0b8bee87bfa4 100644
--- a/rust/ai-research/replay-recorder-v4/Cargo.toml
+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml
@@ -10,15 +10,18 @@ workspace = true
[dependencies]
common-alloc = { path = "../../common/alloc" }
+common-database = { path = "../../common/database" }
common-kafka = { path = "../../common/kafka" }
common-metrics = { path = "../../common/metrics" }
lifecycle = { path = "../../common/lifecycle" }
replay-core-v4 = { path = "../replay-core-v4" }
+arc-swap = { workspace = true }
axum = { workspace = true }
envconfig = { workspace = true }
metrics = { workspace = true }
rdkafka = { workspace = true }
+sqlx = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 3869dfc227f9..3bdb82fb934e 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -20,6 +20,13 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_SAMPLE_PERCENTAGE", default = "0")]
pub sample_percentage: f64,
+ /// The Postgres database the team list is read from. A read replica is enough.
+ pub database_url: String,
+
+ /// How often the team list refreshes. Consent changes take effect within this time, as in v3.
+ #[envconfig(from = "AI_RESEARCH_V4_TEAM_REFRESH_SECS", default = "300")]
+ pub team_refresh_secs: u64,
+
#[envconfig(default = "9104")]
pub metrics_port: u16,
}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 632d80aa9993..b2c42823c6df 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -1,6 +1,7 @@
//! The consume loop.
-use std::time::Duration;
+use std::sync::Arc;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
use common_kafka::config::KafkaConfig;
use lifecycle::Handle;
@@ -9,7 +10,8 @@ use rdkafka::consumer::{Consumer, StreamConsumer};
use rdkafka::message::{Headers, Message as _};
use rdkafka::ClientConfig;
-use crate::process::{read, sampled};
+use crate::process::{admit, read, sampled};
+use crate::teams::TeamCache;
pub fn create_consumer(
kafka: &KafkaConfig,
@@ -47,7 +49,7 @@ fn header<'m>(message: &'m impl rdkafka::Message, name: &str) -> Option<&'m str>
}
/// Reads sampled messages and counts what they hold. Offsets are stored once a message is handled.
-pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {
+pub async fn run(consumer: StreamConsumer, percentage: f64, teams: Arc<TeamCache>, handle: Handle) {
let shutdown = handle.shutdown_signal();
tokio::pin!(shutdown);
loop {
@@ -65,12 +67,20 @@ pub async fn run(consumer: StreamConsumer, percentage: f64, handle: Handle) {
continue;
}
};
- let outcome = sampled(message.key(), percentage).and_then(|()| {
- read(
- message.payload().unwrap_or_default(),
- header(&message, "content-encoding"),
- )
- });
+ let now_ms = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_or(0, |d| d.as_millis() as i64);
+ let outcome = sampled(message.key(), percentage)
+ .and_then(|()| {
+ read(
+ message.payload().unwrap_or_default(),
+ header(&message, "content-encoding"),
+ )
+ })
+ .and_then(|read| {
+ let team = header(&message, "token").and_then(|token| teams.get(token));
+ admit(&read, team, now_ms).map(|_| read)
+ });
match outcome {
Ok(read) => {
counter!("ai_research_v4_recorder_messages_total", "outcome" => "read")
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index 9a96af680d24..ae01300ec013 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -4,3 +4,4 @@
pub mod config;
pub mod consumer;
pub mod process;
+pub mod teams;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index cfb4c5b5f900..239705b13325 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -1,6 +1,8 @@
+use std::sync::Arc;
use std::time::Duration;
use axum::{routing::get, Router};
+use common_database::{get_pool_with_config, PoolConfig};
use common_metrics::setup_metrics_routes;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
@@ -11,6 +13,7 @@ use tracing_subscriber::{fmt, EnvFilter};
use replay_recorder_v4::config::Config;
use replay_recorder_v4::consumer::{create_consumer, run};
+use replay_recorder_v4::teams::TeamCache;
common_alloc::used!();
@@ -40,6 +43,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
"metrics-server",
ComponentOptions::new().is_observability(true),
);
+ let teams_handle = manager.register(
+ "team-refresh",
+ ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
+ );
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let monitor = manager.monitor_background();
@@ -65,13 +72,46 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.expect("metrics server");
});
+ if config.team_refresh_secs == 0 {
+ return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".into());
+ }
+ // The first team list must load before any message is read: an empty list would skip every message as unknown.
+ let pool = get_pool_with_config(
+ &config.database_url,
+ PoolConfig {
+ max_connections: 2,
+ pool_name: Some("replay-recorder-v4".to_owned()),
+ ..Default::default()
+ },
+ )?;
+ let teams = Arc::new(TeamCache::new(TeamCache::max_age_for(Duration::from_secs(
+ config.team_refresh_secs,
+ ))));
+ let loaded = tokio::time::timeout(
+ replay_recorder_v4::teams::REFRESH_TIMEOUT,
+ teams.refresh(&pool),
+ )
+ .await
+ .map_err(|_| "the first team list did not load in time")??;
+ tracing::info!(teams = loaded, "loaded the team list");
+ tokio::spawn(teams.clone().refresh_every(
+ pool,
+ Duration::from_secs(config.team_refresh_secs),
+ teams_handle,
+ ));
+
let consumer = create_consumer(
&config.kafka,
&config.kafka_consumer_group,
&config.kafka_consumer_offset_reset,
&config.kafka_topic,
)?;
- tokio::spawn(run(consumer, config.sample_percentage, consumer_handle));
+ tokio::spawn(run(
+ consumer,
+ config.sample_percentage,
+ teams,
+ consumer_handle,
+ ));
monitor.wait().await?;
Ok(())
diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rs
index 938330b5f3c4..5a74bc918008 100644
--- a/rust/ai-research/replay-recorder-v4/src/process.rs
+++ b/rust/ai-research/replay-recorder-v4/src/process.rs
@@ -1,8 +1,11 @@
//! What the recorder does with one Kafka message, apart from Kafka itself.
use replay_core_v4::gate::in_sample;
-use replay_core_v4::payload::decompress_payload;
+use replay_core_v4::payload::{decompress_payload, is_web_replay};
use replay_core_v4::scan::message::{read_message, Message};
+use replay_core_v4::session::{drop_reason, DropReason};
+
+use crate::teams::Team;
/// Why the recorder leaves a message out. The label is the metric value.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -11,6 +14,10 @@ pub enum Skip {
NoKey,
Undecompressable,
Unreadable,
+ UnknownTeam,
+ NotOptedIn,
+ Session(DropReason),
+ NotWeb,
}
impl Skip {
@@ -20,6 +27,10 @@ impl Skip {
Skip::NoKey => "no_key",
Skip::Undecompressable => "undecompressable",
Skip::Unreadable => "unreadable",
+ Skip::UnknownTeam => "unknown_team",
+ Skip::NotOptedIn => "not_opted_in",
+ Skip::Session(reason) => reason.as_str(),
+ Skip::NotWeb => "not_web",
}
}
}
@@ -34,6 +45,21 @@ pub fn sampled(key: Option<&[u8]>, percentage: f64) -> Result<(), Skip> {
}
}
+/// The v3 mirror's admission rules: a consenting organization, a recent UUIDv7 session, a web replay. Returns the team.
+pub fn admit(message: &Message, team: Option<Team>, now_ms: i64) -> Result<i64, Skip> {
+ let team = team.ok_or(Skip::UnknownTeam)?;
+ if !team.ai_training_opted_in {
+ return Err(Skip::NotOptedIn);
+ }
+ if let Some(reason) = drop_reason(&message.session_id, now_ms) {
+ return Err(Skip::Session(reason));
+ }
+ if !is_web_replay(message.snapshot_source.as_deref()) {
+ return Err(Skip::NotWeb);
+ }
+ Ok(team.id)
+}
+
/// Decompresses and reads a sampled message.
pub fn read(payload: &[u8], content_encoding: Option<&str>) -> Result<Message, Skip> {
let raw = decompress_payload(payload.to_vec(), content_encoding)
@@ -77,4 +103,58 @@ mod tests {
}
assert_eq!(read(b"not json", None).unwrap_err(), Skip::Unreadable);
}
+
+ #[test]
+ fn admits_only_what_the_v3_mirror_admits() {
+ let session = "019a2f3e-8c00-7000-8000-000000000007";
+ // 019a2f3e8c00 is the session start in milliseconds.
+ let now = 0x019a_2f3e_8c00_i64 + 60_000;
+ let message = |source: Option<&str>| {
+ let mut properties = json!({"$session_id": session, "$snapshot_items": []});
+ if let Some(source) = source {
+ properties["$snapshot_source"] = json!(source);
+ }
+ let inner = json!({"event": "$snapshot_items", "properties": properties});
+ read(
+ json!({"distinct_id": "d", "data": inner.to_string()})
+ .to_string()
+ .as_bytes(),
+ None,
+ )
+ .unwrap()
+ };
+ let opted_in = Some(Team {
+ id: 7,
+ ai_training_opted_in: true,
+ });
+ let cases = [
+ (message(Some("web")), opted_in, now, Ok(7)),
+ (message(Some("web")), None, now, Err(Skip::UnknownTeam)),
+ (
+ message(Some("web")),
+ Some(Team {
+ id: 7,
+ ai_training_opted_in: false,
+ }),
+ now,
+ Err(Skip::NotOptedIn),
+ ),
+ (
+ message(Some("web")),
+ opted_in,
+ now + 15 * 86_400_000,
+ Err(Skip::Session(DropReason::TooOld)),
+ ),
+ (message(Some("mobile")), opted_in, now, Err(Skip::NotWeb)),
+ (message(None), opted_in, now, Err(Skip::NotWeb)),
+ ];
+ for (message, team, now, expected) in cases {
+ assert_eq!(
+ admit(&message, team, now),
+ expected,
+ "{team:?} {:?}",
+ message.snapshot_source
+ );
+ }
+ }
}
diff --git a/rust/ai-research/replay-recorder-v4/src/teams.rs b/rust/ai-research/replay-recorder-v4/src/teams.rs
new file mode 100644
index 000000000000..8ea955f27448
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/teams.rs
@@ -0,0 +1,202 @@
+//! The teams that may be recorded: the replay teams of organizations that opt in to AI training.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use arc_swap::ArcSwap;
+use sqlx::PgPool;
+use tokio::time::Instant;
+
+/// The query the v3 mirror's team service runs, narrowed to the columns v4 reads. The main lane drops a team without
+/// a valid retention period, so the list leaves it out.
+const TEAMS: &str = "
+ SELECT t.id::bigint, t.api_token, COALESCE(o.is_ai_training_opted_in, false)
+ FROM posthog_team t
+ LEFT JOIN posthog_organization o ON o.id = t.organization_id
+ WHERE t.session_recording_opt_in = true
+ AND t.session_recording_retention_period IN ('30d', '90d', '1y', '5y')
+";
+
+/// After this many failed refreshes in a row the list is too old to show opt-outs, so the cache admits no team.
+pub const MAX_FAILED_REFRESHES: u32 = 20;
+
+/// A refresh that takes longer fails. Without it a hung query would never count as a failure.
+pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(60);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Team {
+ pub id: i64,
+ pub ai_training_opted_in: bool,
+}
+
+struct Snapshot {
+ teams: HashMap<String, Team>,
+ loaded_at: Instant,
+}
+
+/// Teams by project token. Lookups read a snapshot, so the consume loop never waits on Postgres.
+pub struct TeamCache {
+ snapshot: ArcSwap<Snapshot>,
+ failed_refreshes: AtomicU32,
+ max_age: Duration,
+}
+
+impl Default for TeamCache {
+ fn default() -> Self {
+ Self::from_teams(HashMap::new(), Duration::MAX)
+ }
+}
+
+impl TeamCache {
+ pub fn new(max_age: Duration) -> Self {
+ Self::from_teams(HashMap::new(), max_age)
+ }
+
+ pub fn from_teams(teams: HashMap<String, Team>, max_age: Duration) -> Self {
+ Self {
+ snapshot: ArcSwap::from_pointee(Snapshot {
+ teams,
+ loaded_at: Instant::now(),
+ }),
+ failed_refreshes: AtomicU32::new(0),
+ max_age,
+ }
+ }
+
+ /// The age at which a refreshing cache admits no team: as long as `MAX_FAILED_REFRESHES` intervals that time out.
+ pub fn max_age_for(interval: Duration) -> Duration {
+ (interval + REFRESH_TIMEOUT) * MAX_FAILED_REFRESHES
+ }
+
+ pub fn get(&self, token: &str) -> Option<Team> {
+ if self.failed_refreshes.load(Ordering::Relaxed) >= MAX_FAILED_REFRESHES {
+ return None;
+ }
+ let snapshot = self.snapshot.load();
+ if snapshot.loaded_at.elapsed() > self.max_age {
+ return None;
+ }
+ snapshot.teams.get(token).copied()
+ }
+
+ /// Counts a failed refresh. Returns true when the cache now admits no team.
+ pub fn refresh_failed(&self) -> bool {
+ let failures = self.failed_refreshes.fetch_add(1, Ordering::Relaxed) + 1;
+ let closed = failures >= MAX_FAILED_REFRESHES;
+ metrics::gauge!("ai_research_v4_recorder_teams_closed").set(if closed { 1.0 } else { 0.0 });
+ closed
+ }
+
+ fn store(&self, teams: HashMap<String, Team>) {
+ self.snapshot.store(Arc::new(Snapshot {
+ teams,
+ loaded_at: Instant::now(),
+ }));
+ self.failed_refreshes.store(0, Ordering::Relaxed);
+ metrics::gauge!("ai_research_v4_recorder_teams_closed").set(0.0);
+ }
+
+ pub async fn refresh(&self, pool: &PgPool) -> Result<usize, sqlx::Error> {
+ let rows: Vec<(i64, String, bool)> = sqlx::query_as(TEAMS).fetch_all(pool).await?;
+ let teams: HashMap<String, Team> = rows
+ .into_iter()
+ .map(|(id, token, opted_in)| {
+ (
+ token,
+ Team {
+ id,
+ ai_training_opted_in: opted_in,
+ },
+ )
+ })
+ .collect();
+ let count = teams.len();
+ self.store(teams);
+ metrics::gauge!("ai_research_v4_recorder_teams").set(count as f64);
+ Ok(count)
+ }
+
+ /// Refreshes until shutdown. A failed or timed-out refresh keeps the previous list until `MAX_FAILED_REFRESHES` fail
+ /// in a row, then the cache admits no team until a refresh succeeds. If this task ends early or panics, the process
+ /// stops.
+ pub async fn refresh_every(
+ self: Arc<Self>,
+ pool: PgPool,
+ interval: Duration,
+ handle: lifecycle::Handle,
+ ) {
+ let _guard = handle.process_scope();
+ let shutdown = handle.shutdown_signal();
+ tokio::pin!(shutdown);
+ let mut ticker = tokio::time::interval(interval);
+ ticker.tick().await;
+ loop {
+ tokio::select! {
+ _ = &mut shutdown => return,
+ _ = ticker.tick() => {}
+ }
+ let refreshed = match tokio::time::timeout(REFRESH_TIMEOUT, self.refresh(&pool)).await {
+ Ok(result) => result.map_err(|error| error.to_string()),
+ Err(_) => Err(format!("no answer in {REFRESH_TIMEOUT:?}")),
+ };
+ metrics::gauge!("ai_research_v4_recorder_team_snapshot_age_seconds")
+ .set(self.snapshot.load().loaded_at.elapsed().as_secs_f64());
+ if let Err(error) = refreshed {
+ metrics::counter!("ai_research_v4_recorder_team_refresh_failures_total")
+ .increment(1);
+ if self.refresh_failed() {
+ tracing::error!(%error, "team refreshes keep failing: admitting no team until one succeeds");
+ } else {
+ tracing::warn!(%error, "team refresh failed");
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn admits_nothing_after_too_many_failed_refreshes_until_one_succeeds() {
+ let team = Team {
+ id: 7,
+ ai_training_opted_in: true,
+ };
+ let teams = HashMap::from([("token".to_owned(), team)]);
+ let cache = TeamCache::from_teams(teams.clone(), Duration::MAX);
+ for _ in 1..MAX_FAILED_REFRESHES {
+ assert!(!cache.refresh_failed());
+ }
+ assert_eq!(cache.get("token"), Some(team));
+ assert!(cache.refresh_failed());
+ assert_eq!(cache.get("token"), None);
+
+ cache.store(teams);
+ assert_eq!(cache.get("token"), Some(team));
+ }
+
+ #[test]
+ fn admits_nothing_once_the_snapshot_is_too_old_until_a_refresh_succeeds() {
+ let team = Team {
+ id: 7,
+ ai_training_opted_in: true,
+ };
+ let teams = HashMap::from([("token".to_owned(), team)]);
+ let max_age = TeamCache::max_age_for(Duration::from_secs(300));
+ let cache = TeamCache::from_teams(teams.clone(), max_age);
+ assert_eq!(cache.get("token"), Some(team));
+
+ cache.snapshot.store(Arc::new(Snapshot {
+ teams: teams.clone(),
+ loaded_at: Instant::now() - max_age - Duration::from_secs(1),
+ }));
+ assert_eq!(cache.get("token"), None);
+
+ cache.store(teams);
+ assert_eq!(cache.get("token"), Some(team));
+ }
+}