:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-core-v4): add the AISR03 envelope in rust (#117947)
Nicholas Waltz5b0ea8063e7e
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.ymlindex 1b0fa502b52e..85a768b4b3f6 100644--- a/.github/workflows/ci-rust.yml+++ b/.github/workflows/ci-rust.yml@@ -91,6 +91,8 @@ jobs:- 'posthog/models/feature_flag/flags_cache.py'- 'posthog/models/feature_flag/feature_flag.py'- 'rust/feature-flags/tests/fixtures/hypercache_contract.json'+ # replay-core-v4's tests read the encryption vectors the Node writers pin+ - 'nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json'all_crates:# Pins the service images that every crate's tests run against.# A determinator rule would also rebuild and deploy every Rust image.@@ -914,12 +916,14 @@ jobs:# Turning off cone mode ensures that files in the project root are not included during checkout- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:+ # clippy --all-targets compiles replay-core-v4's tests, which read the encryption vectors the Node writers pin.sparse-checkout: |rust/proto/.github/actions/setup-protoc/.github/actions/setup-rust/.github/actions/setup-sccache/+ nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.jsonsparse-checkout-cone-mode: falseclean: falsediff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 990c2ddd060d..f3781c8c728c 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -23,6 +23,16 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"+[[package]]+name = "aead"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"+dependencies = [+ "crypto-common 0.1.7",+ "generic-array",+]+[[package]]name = "aes"version = "0.8.4"@@ -34,6 +44,20 @@ dependencies = ["cpufeatures 0.2.17",]+[[package]]+name = "aes-gcm"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"+dependencies = [+ "aead",+ "aes",+ "cipher",+ "ctr",+ "ghash",+ "subtle",+]+[[package]]name = "affected-services"version = "0.1.0"@@ -3463,6 +3487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"dependencies = ["generic-array",+ "rand_core 0.6.4","typenum",]@@ -3513,6 +3538,15 @@ version = "0.0.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"+[[package]]+name = "ctr"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"+dependencies = [+ "cipher",+]+[[package]]name = "ctutils"version = "0.4.2"@@ -5322,6 +5356,16 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "ghash"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"+dependencies = [+ "opaque-debug",+ "polyval",+]+[[package]]name = "gimli"version = "0.32.3"@@ -8471,6 +8515,12 @@ version = "11.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"+[[package]]+name = "opaque-debug"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"+[[package]]name = "openssl"version = "0.10.75"@@ -9630,6 +9680,18 @@ dependencies = ["arrayvec",]+[[package]]+name = "polyval"+version = "0.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"+dependencies = [+ "cfg-if",+ "cpufeatures 0.2.17",+ "opaque-debug",+ "universal-hash",+]+[[package]]name = "portable-atomic"version = "1.13.1"@@ -10927,6 +10989,16 @@ dependencies = ["sha2 0.10.9",]+[[package]]+name = "replay-core-v4"+version = "0.1.0"+dependencies = [+ "aes-gcm",+ "base64 0.22.1",+ "serde_json",+ "thiserror 2.0.18",+]+[[package]]name = "reqwest"version = "0.11.27"@@ -14268,6 +14340,16 @@ version = "0.2.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"+[[package]]+name = "universal-hash"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"+dependencies = [+ "crypto-common 0.1.7",+ "subtle",+]+[[package]]name = "unsafe-libyaml"version = "0.2.11"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 45496c032fbf..9d91a47e1a20 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -79,6 +79,7 @@ members = ["property-vals-rs","pgcollector","pgapi",+ "ai-research/replay-core-v4",]# NOTE: cargo allows only one crate with `links = "python"` (i.e. one pyo3 version) perdiff --git a/rust/affected-services/determinator-rules.toml b/rust/affected-services/determinator-rules.tomlindex 4aa5b2df4192..42886860cea6 100644--- a/rust/affected-services/determinator-rules.toml+++ b/rust/affected-services/determinator-rules.toml@@ -52,6 +52,12 @@ globs = []mark-changed = ["feature-flags", "hypercache-server", "common-hypercache"]+# replay-core-v4's tests read the encryption vectors the Node writers pin.+# Must come BEFORE the catch-all rule below.+[[path-rule]]+globs = ["../nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json"]+mark-changed = ["replay-core-v4"]+# Django migrations and test environment setup — needed for Rust integration tests.[[path-rule]]globs = [diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.tomlnew file mode 100644index 000000000000..15eb6581be44--- /dev/null+++ b/rust/ai-research/replay-core-v4/Cargo.toml@@ -0,0 +1,17 @@+[package]+name = "replay-core-v4"+version = "0.1.0"+edition = "2021"+publish = false+description = "Core of the v4 AI research replay dataset: the AISR03 envelope, the v4 keys, the session gate and the byte-level scan"++[lints]+workspace = true++[dependencies]+aes-gcm = "0.10"+serde_json = { workspace = true }+thiserror = { workspace = true }++[dev-dependencies]+base64 = { workspace = true }diff --git a/rust/ai-research/replay-core-v4/src/envelope.rs b/rust/ai-research/replay-core-v4/src/envelope.rsnew file mode 100644index 000000000000..4b7e38526324--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/envelope.rs@@ -0,0 +1,212 @@+//! The `AISR03` frame that seals every v3 and v4 object body.+//!+//! The frame is the ASCII magic, a big-endian `u16` length, the additional authenticated data, a 12-byte nonce, and+//! the AES-256-GCM ciphertext followed by its 16-byte tag. The authenticated data is the canonical JSON of+//! `{"v": <version>, "context": <context>}`, so the context binds the owner, the kind and the codec of the body.++use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng, Payload};+use aes_gcm::{Aes256Gcm, Key, Nonce};+use serde_json::{Map, Value};++pub const MAGIC: &[u8; 6] = b"AISR03";+pub const NONCE_BYTES: usize = 12;+pub const TAG_BYTES: usize = 16;+const AAD_LENGTH_BYTES: usize = 2;+const HEADER_BYTES: usize = MAGIC.len() + AAD_LENGTH_BYTES;++#[derive(Debug, thiserror::Error, PartialEq, Eq)]+pub enum EnvelopeError {+ #[error("not an AISR03 frame")]+ NotAFrame,+ #[error("frame is shorter than its header says")]+ Truncated,+ #[error("frame context does not match the requested object")]+ ContextMismatch,+ #[error("authenticated data is too long to frame")]+ ContextTooLong,+ #[error("authentication failed")]+ Authentication,+}++/// The dataset version inside the authenticated data. A reader for one version never opens the other.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Version {+ V3 = 3,+ V4 = 4,+}++/// How the reader must expand the plaintext. It sits inside the authenticated context so it cannot be downgraded.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Codec {+ None,+ Brotli,+ Zstd,+}++impl Codec {+ pub fn as_str(self) -> &'static str {+ match self {+ Codec::None => "none",+ Codec::Brotli => "brotli",+ Codec::Zstd => "zstd",+ }+ }+}++/// The object a frame belongs to. Field names are the camelCase names the Node writers use.+#[derive(Debug, Clone, PartialEq, Eq, Default)]+pub struct Context {+ pub team_id: i64,+ pub session_id: Option<String>,+ pub key_month: Option<String>,+ pub session_month: Option<String>,+ pub organization_id: Option<String>,+ pub kind: String,+ pub codec: Option<Codec>,+ pub reference: Option<String>,+}++impl Context {+ fn to_json(&self) -> Value {+ let mut context = Map::new();+ context.insert("teamId".to_owned(), Value::from(self.team_id));+ let optional = [+ ("sessionId", &self.session_id),+ ("keyMonth", &self.key_month),+ ("sessionMonth", &self.session_month),+ ("organizationId", &self.organization_id),+ ];+ for (name, value) in optional {+ if let Some(value) = value {+ context.insert(name.to_owned(), Value::from(value.as_str()));+ }+ }+ context.insert("kind".to_owned(), Value::from(self.kind.as_str()));+ if let Some(codec) = self.codec {+ context.insert("codec".to_owned(), Value::from(codec.as_str()));+ }+ if let Some(reference) = &self.reference {+ context.insert("ref".to_owned(), Value::from(reference.as_str()));+ }+ Value::Object(context)+ }++ /// The bytes AES-GCM authenticates for this context.+ pub fn authenticated_data(&self, version: Version) -> String {+ let mut wrapper = Map::new();+ wrapper.insert("v".to_owned(), Value::from(version as u8));+ wrapper.insert("context".to_owned(), self.to_json());+ canonical_json(&Value::Object(wrapper))+ }+}++/// JSON with sorted keys and no whitespace, which the Node writers and every reader produce byte for byte.+pub fn canonical_json(value: &Value) -> String {+ let mut out = String::new();+ write_canonical(value, &mut out);+ out+}++fn write_canonical(value: &Value, out: &mut String) {+ match value {+ Value::Array(items) => {+ out.push('[');+ for (index, item) in items.iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ write_canonical(item, out);+ }+ out.push(']');+ }+ Value::Object(entries) => {+ // Sort here rather than rely on the map type: a crate elsewhere in the build can turn on+ // serde_json's `preserve_order` feature, which changes the map to insertion order.+ let mut keys: Vec<&String> = entries.keys().collect();+ keys.sort();+ out.push('{');+ for (index, key) in keys.into_iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ out.push_str(&Value::from(key.as_str()).to_string());+ out.push(':');+ write_canonical(&entries[key], out);+ }+ out.push('}');+ }+ scalar => out.push_str(&scalar.to_string()),+ }+}++/// Seals `body` under `key` with a random nonce.+pub fn seal_frame(+ key: &[u8; 32],+ version: Version,+ context: &Context,+ body: &[u8],+) -> Result<Vec<u8>, EnvelopeError> {+ let nonce = Aes256Gcm::generate_nonce(&mut OsRng);+ seal_frame_with_nonce(key, version, context, body, nonce.into())+}++/// Seals with a caller-chosen nonce. Only the test vectors may fix a nonce: reusing one under a key breaks AES-GCM.+pub fn seal_frame_with_nonce(+ key: &[u8; 32],+ version: Version,+ context: &Context,+ body: &[u8],+ nonce: [u8; NONCE_BYTES],+) -> Result<Vec<u8>, EnvelopeError> {+ let aad = context.authenticated_data(version);+ let aad_length = u16::try_from(aad.len()).map_err(|_| EnvelopeError::ContextTooLong)?;+ let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(key));+ let sealed = cipher+ .encrypt(+ Nonce::from_slice(&nonce),+ Payload {+ msg: body,+ aad: aad.as_bytes(),+ },+ )+ .map_err(|_| EnvelopeError::Authentication)?;+ let mut frame = Vec::with_capacity(HEADER_BYTES + aad.len() + NONCE_BYTES + sealed.len());+ frame.extend_from_slice(MAGIC);+ frame.extend_from_slice(&aad_length.to_be_bytes());+ frame.extend_from_slice(aad.as_bytes());+ frame.extend_from_slice(&nonce);+ frame.extend_from_slice(&sealed);+ Ok(frame)+}++/// Opens a frame and checks it belongs to `context`.+///+/// The check is required: one key seals many objects, and the context is the only thing that tells them apart.+pub fn open_frame(+ key: &[u8; 32],+ version: Version,+ context: &Context,+ frame: &[u8],+) -> Result<Vec<u8>, EnvelopeError> {+ if frame.len() < HEADER_BYTES || &frame[..MAGIC.len()] != MAGIC {+ return Err(EnvelopeError::NotAFrame);+ }+ // The magic and the length sit outside the authenticated data, so bound the frame before slicing it.+ let aad_length = usize::from(u16::from_be_bytes([+ frame[MAGIC.len()],+ frame[MAGIC.len() + 1],+ ]));+ if frame.len() < HEADER_BYTES + aad_length + NONCE_BYTES + TAG_BYTES {+ return Err(EnvelopeError::Truncated);+ }+ let aad = &frame[HEADER_BYTES..HEADER_BYTES + aad_length];+ if aad != context.authenticated_data(version).as_bytes() {+ return Err(EnvelopeError::ContextMismatch);+ }+ let nonce = &frame[HEADER_BYTES + aad_length..HEADER_BYTES + aad_length + NONCE_BYTES];+ let sealed = &frame[HEADER_BYTES + aad_length + NONCE_BYTES..];+ let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(key));+ cipher+ .decrypt(Nonce::from_slice(nonce), Payload { msg: sealed, aad })+ .map_err(|_| EnvelopeError::Authentication)+}diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rsnew file mode 100644index 000000000000..849ab68b9f5e--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/lib.rs@@ -0,0 +1,3 @@+//! The v4 AI research replay dataset: the format in `products/ai_training/docs/replay-data.md`, section "v4 dataset".++pub mod envelope;diff --git a/rust/ai-research/replay-core-v4/tests/v3_vector.rs b/rust/ai-research/replay-core-v4/tests/v3_vector.rsnew file mode 100644index 000000000000..742e61382296--- /dev/null+++ b/rust/ai-research/replay-core-v4/tests/v3_vector.rs@@ -0,0 +1,93 @@+use base64::engine::general_purpose::STANDARD;+use base64::Engine;+use replay_core_v4::envelope::{+ open_frame, seal_frame_with_nonce, Codec, Context, EnvelopeError, Version,+};+use serde_json::Value;++// The Node writer's pinned output. Both languages read this one file, so a drift on either side fails here.+const VECTOR: &str = include_str!("../../../../nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector.json");++fn vector() -> Value {+ serde_json::from_str(VECTOR).unwrap()+}++fn bytes(value: &Value) -> Vec<u8> {+ STANDARD.decode(value.as_str().unwrap()).unwrap()+}++fn frame_context(vector: &Value) -> Context {+ let context = &vector["context"];+ Context {+ team_id: context["teamId"].as_i64().unwrap(),+ session_id: Some(context["sessionId"].as_str().unwrap().to_owned()),+ organization_id: Some(context["organizationId"].as_str().unwrap().to_owned()),+ kind: vector["frame"]["kind"].as_str().unwrap().to_owned(),+ codec: Some(Codec::Brotli),+ ..Context::default()+ }+}++#[test]+fn seals_the_v3_frame_the_node_writer_pinned() {+ let vector = vector();+ let key: [u8; 32] = bytes(&vector["key"]).try_into().unwrap();+ let nonce: [u8; 12] = bytes(&vector["frame"]["nonce"]).try_into().unwrap();+ let context = frame_context(&vector);++ assert_eq!(+ context.authenticated_data(Version::V3),+ vector["frame"]["aad"].as_str().unwrap()+ );+ let frame = seal_frame_with_nonce(+ &key,+ Version::V3,+ &context,+ &bytes(&vector["frame"]["body"]),+ nonce,+ )+ .unwrap();+ assert_eq!(frame, bytes(&vector["frame"]["frame"]));+ assert_eq!(+ open_frame(&key, Version::V3, &context, &frame).unwrap(),+ bytes(&vector["frame"]["body"])+ );+}++#[test]+fn refuses_a_frame_for_another_object() {+ let vector = vector();+ let key: [u8; 32] = bytes(&vector["key"]).try_into().unwrap();+ let frame = bytes(&vector["frame"]["frame"]);+ let context = frame_context(&vector);++ let other_team = Context {+ team_id: 8,+ ..context.clone()+ };+ let other_codec = Context {+ codec: Some(Codec::None),+ ..context.clone()+ };+ for (other, version) in [+ (other_team, Version::V3),+ (other_codec, Version::V3),+ (context.clone(), Version::V4),+ ] {+ assert_eq!(+ open_frame(&key, version, &other, &frame),+ Err(EnvelopeError::ContextMismatch)+ );+ }+ let mut corrupted = frame.clone();+ let last = corrupted.len() - 1;+ corrupted[last] ^= 1;+ assert_eq!(+ open_frame(&key, Version::V3, &context, &corrupted),+ Err(EnvelopeError::Authentication)+ );+ assert_eq!(+ open_frame(&key, Version::V3, &context, &frame[..20]),+ Err(EnvelopeError::Truncated)+ );+}diff --git a/rust/owners.yaml b/rust/owners.yamlindex 511de6aaa92d..d1a7df331c86 100644--- a/rust/owners.yaml+++ b/rust/owners.yaml@@ -33,6 +33,7 @@ rules:- '/property-vals-rs/'owners: team-ingestion- match:+ - '/ai-research/'- '/replay-anonymizer/'- '/replay-anonymizer-node/'owners: ai-research