Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-core-v4): add the AISR03 envelope in rust (#117947)

Nicholas Waltz5b0ea8063e7e
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml
index 1b0fa502b52e..85a768b4b3f6 100644
--- a/.github/workflows/ci-rust.yml
+++ b/.github/workflows/ci-rust.yml
@@ -91,6 +91,8 @@ jobs:
- 'posthog/models/feature_flag/flags_cache.py'
- 'posthog/models/feature_flag/feature_flag.py'
- 'rust/feature-flags/tests/fixtures/hypercache_contract.json'
+ # replay-core-v4's tests read the encryption vectors the Node writers pin
+ - 'nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json'
all_crates:
# Pins the service images that every crate's tests run against.
# A determinator rule would also rebuild and deploy every Rust image.
@@ -914,12 +916,14 @@ jobs:
# Turning off cone mode ensures that files in the project root are not included during checkout
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
+ # clippy --all-targets compiles replay-core-v4's tests, which read the encryption vectors the Node writers pin.
sparse-checkout: |
rust/
proto/
.github/actions/setup-protoc/
.github/actions/setup-rust/
.github/actions/setup-sccache/
+ nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json
sparse-checkout-cone-mode: false
clean: false
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 990c2ddd060d..f3781c8c728c 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -23,6 +23,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
+[[package]]
+name = "aead"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
+dependencies = [
+ "crypto-common 0.1.7",
+ "generic-array",
+]
+
[[package]]
name = "aes"
version = "0.8.4"
@@ -34,6 +44,20 @@ dependencies = [
"cpufeatures 0.2.17",
]
+[[package]]
+name = "aes-gcm"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
[[package]]
name = "affected-services"
version = "0.1.0"
@@ -3463,6 +3487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
+ "rand_core 0.6.4",
"typenum",
]
@@ -3513,6 +3538,15 @@ version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"
+[[package]]
+name = "ctr"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "ctutils"
version = "0.4.2"
@@ -5322,6 +5356,16 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "ghash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"
+dependencies = [
+ "opaque-debug",
+ "polyval",
+]
+
[[package]]
name = "gimli"
version = "0.32.3"
@@ -8471,6 +8515,12 @@ version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+[[package]]
+name = "opaque-debug"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
+
[[package]]
name = "openssl"
version = "0.10.75"
@@ -9630,6 +9680,18 @@ dependencies = [
"arrayvec",
]
+[[package]]
+name = "polyval"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.2.17",
+ "opaque-debug",
+ "universal-hash",
+]
+
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -10927,6 +10989,16 @@ dependencies = [
"sha2 0.10.9",
]
+[[package]]
+name = "replay-core-v4"
+version = "0.1.0"
+dependencies = [
+ "aes-gcm",
+ "base64 0.22.1",
+ "serde_json",
+ "thiserror 2.0.18",
+]
+
[[package]]
name = "reqwest"
version = "0.11.27"
@@ -14268,6 +14340,16 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+[[package]]
+name = "universal-hash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
+dependencies = [
+ "crypto-common 0.1.7",
+ "subtle",
+]
+
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 45496c032fbf..9d91a47e1a20 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -79,6 +79,7 @@ members = [
"property-vals-rs",
"pgcollector",
"pgapi",
+ "ai-research/replay-core-v4",
]
# NOTE: cargo allows only one crate with `links = "python"` (i.e. one pyo3 version) per
diff --git a/rust/affected-services/determinator-rules.toml b/rust/affected-services/determinator-rules.toml
index 4aa5b2df4192..42886860cea6 100644
--- a/rust/affected-services/determinator-rules.toml
+++ b/rust/affected-services/determinator-rules.toml
@@ -52,6 +52,12 @@ globs = [
]
mark-changed = ["feature-flags", "hypercache-server", "common-hypercache"]
+# replay-core-v4's tests read the encryption vectors the Node writers pin.
+# Must come BEFORE the catch-all rule below.
+[[path-rule]]
+globs = ["../nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json"]
+mark-changed = ["replay-core-v4"]
+
# Django migrations and test environment setup — needed for Rust integration tests.
[[path-rule]]
globs = [
diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.toml
new file mode 100644
index 000000000000..15eb6581be44
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "replay-core-v4"
+version = "0.1.0"
+edition = "2021"
+publish = false
+description = "Core of the v4 AI research replay dataset: the AISR03 envelope, the v4 keys, the session gate and the byte-level scan"
+
+[lints]
+workspace = true
+
+[dependencies]
+aes-gcm = "0.10"
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+base64 = { workspace = true }
diff --git a/rust/ai-research/replay-core-v4/src/envelope.rs b/rust/ai-research/replay-core-v4/src/envelope.rs
new file mode 100644
index 000000000000..4b7e38526324
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/envelope.rs
@@ -0,0 +1,212 @@
+//! The `AISR03` frame that seals every v3 and v4 object body.
+//!
+//! The frame is the ASCII magic, a big-endian `u16` length, the additional authenticated data, a 12-byte nonce, and
+//! the AES-256-GCM ciphertext followed by its 16-byte tag. The authenticated data is the canonical JSON of
+//! `{"v": <version>, "context": <context>}`, so the context binds the owner, the kind and the codec of the body.
+
+use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng, Payload};
+use aes_gcm::{Aes256Gcm, Key, Nonce};
+use serde_json::{Map, Value};
+
+pub const MAGIC: &[u8; 6] = b"AISR03";
+pub const NONCE_BYTES: usize = 12;
+pub const TAG_BYTES: usize = 16;
+const AAD_LENGTH_BYTES: usize = 2;
+const HEADER_BYTES: usize = MAGIC.len() + AAD_LENGTH_BYTES;
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum EnvelopeError {
+ #[error("not an AISR03 frame")]
+ NotAFrame,
+ #[error("frame is shorter than its header says")]
+ Truncated,
+ #[error("frame context does not match the requested object")]
+ ContextMismatch,
+ #[error("authenticated data is too long to frame")]
+ ContextTooLong,
+ #[error("authentication failed")]
+ Authentication,
+}
+
+/// The dataset version inside the authenticated data. A reader for one version never opens the other.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Version {
+ V3 = 3,
+ V4 = 4,
+}
+
+/// How the reader must expand the plaintext. It sits inside the authenticated context so it cannot be downgraded.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Codec {
+ None,
+ Brotli,
+ Zstd,
+}
+
+impl Codec {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Codec::None => "none",
+ Codec::Brotli => "brotli",
+ Codec::Zstd => "zstd",
+ }
+ }
+}
+
+/// The object a frame belongs to. Field names are the camelCase names the Node writers use.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct Context {
+ pub team_id: i64,
+ pub session_id: Option<String>,
+ pub key_month: Option<String>,
+ pub session_month: Option<String>,
+ pub organization_id: Option<String>,
+ pub kind: String,
+ pub codec: Option<Codec>,
+ pub reference: Option<String>,
+}
+
+impl Context {
+ fn to_json(&self) -> Value {
+ let mut context = Map::new();
+ context.insert("teamId".to_owned(), Value::from(self.team_id));
+ let optional = [
+ ("sessionId", &self.session_id),
+ ("keyMonth", &self.key_month),
+ ("sessionMonth", &self.session_month),
+ ("organizationId", &self.organization_id),
+ ];
+ for (name, value) in optional {
+ if let Some(value) = value {
+ context.insert(name.to_owned(), Value::from(value.as_str()));
+ }
+ }
+ context.insert("kind".to_owned(), Value::from(self.kind.as_str()));
+ if let Some(codec) = self.codec {
+ context.insert("codec".to_owned(), Value::from(codec.as_str()));
+ }
+ if let Some(reference) = &self.reference {
+ context.insert("ref".to_owned(), Value::from(reference.as_str()));
+ }
+ Value::Object(context)
+ }
+
+ /// The bytes AES-GCM authenticates for this context.
+ pub fn authenticated_data(&self, version: Version) -> String {
+ let mut wrapper = Map::new();
+ wrapper.insert("v".to_owned(), Value::from(version as u8));
+ wrapper.insert("context".to_owned(), self.to_json());
+ canonical_json(&Value::Object(wrapper))
+ }
+}
+
+/// JSON with sorted keys and no whitespace, which the Node writers and every reader produce byte for byte.
+pub fn canonical_json(value: &Value) -> String {
+ let mut out = String::new();
+ write_canonical(value, &mut out);
+ out
+}
+
+fn write_canonical(value: &Value, out: &mut String) {
+ match value {
+ Value::Array(items) => {
+ out.push('[');
+ for (index, item) in items.iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ write_canonical(item, out);
+ }
+ out.push(']');
+ }
+ Value::Object(entries) => {
+ // Sort here rather than rely on the map type: a crate elsewhere in the build can turn on
+ // serde_json's `preserve_order` feature, which changes the map to insertion order.
+ let mut keys: Vec<&String> = entries.keys().collect();
+ keys.sort();
+ out.push('{');
+ for (index, key) in keys.into_iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ out.push_str(&Value::from(key.as_str()).to_string());
+ out.push(':');
+ write_canonical(&entries[key], out);
+ }
+ out.push('}');
+ }
+ scalar => out.push_str(&scalar.to_string()),
+ }
+}
+
+/// Seals `body` under `key` with a random nonce.
+pub fn seal_frame(
+ key: &[u8; 32],
+ version: Version,
+ context: &Context,
+ body: &[u8],
+) -> Result<Vec<u8>, EnvelopeError> {
+ let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
+ seal_frame_with_nonce(key, version, context, body, nonce.into())
+}
+
+/// Seals with a caller-chosen nonce. Only the test vectors may fix a nonce: reusing one under a key breaks AES-GCM.
+pub fn seal_frame_with_nonce(
+ key: &[u8; 32],
+ version: Version,
+ context: &Context,
+ body: &[u8],
+ nonce: [u8; NONCE_BYTES],
+) -> Result<Vec<u8>, EnvelopeError> {
+ let aad = context.authenticated_data(version);
+ let aad_length = u16::try_from(aad.len()).map_err(|_| EnvelopeError::ContextTooLong)?;
+ let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(key));
+ let sealed = cipher
+ .encrypt(
+ Nonce::from_slice(&nonce),
+ Payload {
+ msg: body,
+ aad: aad.as_bytes(),
+ },
+ )
+ .map_err(|_| EnvelopeError::Authentication)?;
+ let mut frame = Vec::with_capacity(HEADER_BYTES + aad.len() + NONCE_BYTES + sealed.len());
+ frame.extend_from_slice(MAGIC);
+ frame.extend_from_slice(&aad_length.to_be_bytes());
+ frame.extend_from_slice(aad.as_bytes());
+ frame.extend_from_slice(&nonce);
+ frame.extend_from_slice(&sealed);
+ Ok(frame)
+}
+
+/// Opens a frame and checks it belongs to `context`.
+///
+/// The check is required: one key seals many objects, and the context is the only thing that tells them apart.
+pub fn open_frame(
+ key: &[u8; 32],
+ version: Version,
+ context: &Context,
+ frame: &[u8],
+) -> Result<Vec<u8>, EnvelopeError> {
+ if frame.len() < HEADER_BYTES || &frame[..MAGIC.len()] != MAGIC {
+ return Err(EnvelopeError::NotAFrame);
+ }
+ // The magic and the length sit outside the authenticated data, so bound the frame before slicing it.
+ let aad_length = usize::from(u16::from_be_bytes([
+ frame[MAGIC.len()],
+ frame[MAGIC.len() + 1],
+ ]));
+ if frame.len() < HEADER_BYTES + aad_length + NONCE_BYTES + TAG_BYTES {
+ return Err(EnvelopeError::Truncated);
+ }
+ let aad = &frame[HEADER_BYTES..HEADER_BYTES + aad_length];
+ if aad != context.authenticated_data(version).as_bytes() {
+ return Err(EnvelopeError::ContextMismatch);
+ }
+ let nonce = &frame[HEADER_BYTES + aad_length..HEADER_BYTES + aad_length + NONCE_BYTES];
+ let sealed = &frame[HEADER_BYTES + aad_length + NONCE_BYTES..];
+ let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(key));
+ cipher
+ .decrypt(Nonce::from_slice(nonce), Payload { msg: sealed, aad })
+ .map_err(|_| EnvelopeError::Authentication)
+}
diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rs
new file mode 100644
index 000000000000..849ab68b9f5e
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/lib.rs
@@ -0,0 +1,3 @@
+//! The v4 AI research replay dataset: the format in `products/ai_training/docs/replay-data.md`, section "v4 dataset".
+
+pub mod envelope;
diff --git a/rust/ai-research/replay-core-v4/tests/v3_vector.rs b/rust/ai-research/replay-core-v4/tests/v3_vector.rs
new file mode 100644
index 000000000000..742e61382296
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/tests/v3_vector.rs
@@ -0,0 +1,93 @@
+use base64::engine::general_purpose::STANDARD;
+use base64::Engine;
+use replay_core_v4::envelope::{
+ open_frame, seal_frame_with_nonce, Codec, Context, EnvelopeError, Version,
+};
+use serde_json::Value;
+
+// The Node writer's pinned output. Both languages read this one file, so a drift on either side fails here.
+const VECTOR: &str = include_str!("../../../../nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector.json");
+
+fn vector() -> Value {
+ serde_json::from_str(VECTOR).unwrap()
+}
+
+fn bytes(value: &Value) -> Vec<u8> {
+ STANDARD.decode(value.as_str().unwrap()).unwrap()
+}
+
+fn frame_context(vector: &Value) -> Context {
+ let context = &vector["context"];
+ Context {
+ team_id: context["teamId"].as_i64().unwrap(),
+ session_id: Some(context["sessionId"].as_str().unwrap().to_owned()),
+ organization_id: Some(context["organizationId"].as_str().unwrap().to_owned()),
+ kind: vector["frame"]["kind"].as_str().unwrap().to_owned(),
+ codec: Some(Codec::Brotli),
+ ..Context::default()
+ }
+}
+
+#[test]
+fn seals_the_v3_frame_the_node_writer_pinned() {
+ let vector = vector();
+ let key: [u8; 32] = bytes(&vector["key"]).try_into().unwrap();
+ let nonce: [u8; 12] = bytes(&vector["frame"]["nonce"]).try_into().unwrap();
+ let context = frame_context(&vector);
+
+ assert_eq!(
+ context.authenticated_data(Version::V3),
+ vector["frame"]["aad"].as_str().unwrap()
+ );
+ let frame = seal_frame_with_nonce(
+ &key,
+ Version::V3,
+ &context,
+ &bytes(&vector["frame"]["body"]),
+ nonce,
+ )
+ .unwrap();
+ assert_eq!(frame, bytes(&vector["frame"]["frame"]));
+ assert_eq!(
+ open_frame(&key, Version::V3, &context, &frame).unwrap(),
+ bytes(&vector["frame"]["body"])
+ );
+}
+
+#[test]
+fn refuses_a_frame_for_another_object() {
+ let vector = vector();
+ let key: [u8; 32] = bytes(&vector["key"]).try_into().unwrap();
+ let frame = bytes(&vector["frame"]["frame"]);
+ let context = frame_context(&vector);
+
+ let other_team = Context {
+ team_id: 8,
+ ..context.clone()
+ };
+ let other_codec = Context {
+ codec: Some(Codec::None),
+ ..context.clone()
+ };
+ for (other, version) in [
+ (other_team, Version::V3),
+ (other_codec, Version::V3),
+ (context.clone(), Version::V4),
+ ] {
+ assert_eq!(
+ open_frame(&key, version, &other, &frame),
+ Err(EnvelopeError::ContextMismatch)
+ );
+ }
+ let mut corrupted = frame.clone();
+ let last = corrupted.len() - 1;
+ corrupted[last] ^= 1;
+ assert_eq!(
+ open_frame(&key, Version::V3, &context, &corrupted),
+ Err(EnvelopeError::Authentication)
+ );
+ assert_eq!(
+ open_frame(&key, Version::V3, &context, &frame[..20]),
+ Err(EnvelopeError::Truncated)
+ );
+}
diff --git a/rust/owners.yaml b/rust/owners.yaml
index 511de6aaa92d..d1a7df331c86 100644
--- a/rust/owners.yaml
+++ b/rust/owners.yaml
@@ -33,6 +33,7 @@ rules:
- '/property-vals-rs/'
owners: team-ingestion
- match:
+ - '/ai-research/'
- '/replay-anonymizer/'
- '/replay-anonymizer-node/'
owners: ai-research