:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-core-v4): define the v4 key table rows (#117953)
Nicholas Waltz05755897b116
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex e0ccc31b5284..f325cdb27ecf 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -10994,7 +10994,9 @@ name = "replay-core-v4"version = "0.1.0"dependencies = ["aes-gcm",+ "aws-sdk-dynamodb","base64 0.22.1",+ "chrono","hkdf","hmac 0.12.1","md5",diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.tomlindex a223528ff423..7fd7a15bbf34 100644--- a/rust/ai-research/replay-core-v4/Cargo.toml+++ b/rust/ai-research/replay-core-v4/Cargo.toml@@ -8,9 +8,16 @@ description = "Core of the v4 AI research replay dataset: the AISR03 envelope, t[lints]workspace = true+[features]+default = ["store"]+# The key table and KMS clients. A reader that only opens frames turns default features off and skips the AWS SDK.+store = ["dep:aws-sdk-dynamodb", "dep:chrono"]+[dependencies]aes-gcm = "0.10"+aws-sdk-dynamodb = { workspace = true, optional = true }base64 = { workspace = true }+chrono = { workspace = true, optional = true }hkdf = "0.12"hmac = { workspace = true }md5 = "0.8"diff --git a/rust/ai-research/replay-core-v4/src/keys/mod.rs b/rust/ai-research/replay-core-v4/src/keys/mod.rsnew file mode 100644index 000000000000..f0519f57d192--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/keys/mod.rs@@ -0,0 +1,3 @@+//! The v4 key table in prod-us: one KMS-wrapped key per team per UTC month, and one session key sealed under it.++pub mod rows;diff --git a/rust/ai-research/replay-core-v4/src/keys/rows.rs b/rust/ai-research/replay-core-v4/src/keys/rows.rsnew file mode 100644index 000000000000..637d511a23ba--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/keys/rows.rs@@ -0,0 +1,402 @@+//! The rows of the v4 key table and their DynamoDB encoding.++use std::collections::HashMap;++use aws_sdk_dynamodb::primitives::Blob;+use aws_sdk_dynamodb::types::AttributeValue;+use chrono::{DateTime, Months, NaiveDate, Utc};+use sha2::{Digest, Sha256};++use crate::envelope::NONCE_BYTES;+use crate::seal::SealedKey;++/// v3 spreads a team's sessions over 32 partitions so one large team cannot overload one partition. v4 keeps this.+pub const SESSION_SHARDS: u32 = 32;+/// A session keeps the key of the month it was first stored and writes at most about 15 days past that month. Raw+/// objects live 90 days and fetched images 98, so five months covers them and leaves room to raise that window a little.+const ROW_LIFETIME_MONTHS: u32 = 5;+/// The attributes that hold key material. A deletion removes all of them.+pub const KEY_MATERIAL: [&str; 3] = ["wrapped_key", "sealed_key", "key_nonce"];++pub type Item = HashMap<String, AttributeValue>;++#[derive(Debug, thiserror::Error, PartialEq, Eq)]+pub enum RowError {+ #[error("key row has no {0}")]+ Missing(&'static str),+ #[error("key row has a malformed {0}")]+ Malformed(&'static str),+ #[error("an unusable key row cannot be written")]+ NotWritable,+}++/// A primary key in the table.+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]+pub struct TableKey {+ pub pk: String,+ pub sk: String,+}++impl TableKey {+ pub fn team_month(team_id: i64, key_month: &str) -> Self {+ Self {+ pk: format!("team:{team_id}"),+ sk: format!("month:{key_month}"),+ }+ }++ pub fn session(team_id: i64, session_id: &str) -> Self {+ Self {+ pk: format!("team:{team_id}:shard:{}", session_shard(session_id)),+ sk: format!("session:{session_id}"),+ }+ }++ pub fn to_item(&self) -> Item {+ HashMap::from([+ ("pk".to_owned(), AttributeValue::S(self.pk.clone())),+ ("sk".to_owned(), AttributeValue::S(self.sk.clone())),+ ])+ }++ pub fn from_item(item: &Item) -> Result<Self, RowError> {+ Ok(Self {+ pk: string(item, "pk")?,+ sk: string(item, "sk")?,+ })+ }+}++/// The v3 shard: the first 8 hex digits of `sha256(session_id)` as an integer, modulo 32.+pub fn session_shard(session_id: &str) -> u32 {+ let digest = Sha256::digest(session_id.as_bytes());+ u32::from_be_bytes([digest[0], digest[1], digest[2], digest[3]]) % SESSION_SHARDS+}++/// The UTC day of an instant, as the object partitions name it. A flush writes on this day.+pub fn write_day(at: DateTime<Utc>) -> String {+ at.format("%Y-%m-%d").to_string()+}++/// The key month of a UTC day `YYYY-MM-DD`, as `key_month` stores it.+pub fn key_month(day: &str) -> &str {+ day.get(..7).unwrap_or(day)+}++/// The start of the month `key_month + 5`, in epoch seconds: when DynamoDB TTL may remove the row.+pub fn expires_at(key_month: &str) -> Result<i64, RowError> {+ let malformed = || RowError::Malformed("key_month");+ let start = NaiveDate::parse_from_str(&format!("{key_month}-01"), "%Y-%m-%d")+ .map_err(|_| malformed())?;+ if start.format("%Y-%m").to_string() != key_month {+ return Err(malformed());+ }+ let end = start+ .checked_add_months(Months::new(ROW_LIFETIME_MONTHS))+ .ok_or_else(malformed)?;+ Ok(end+ .and_hms_opt(0, 0, 0)+ .expect("midnight exists")+ .and_utc()+ .timestamp())+}++/// What the table holds at one key.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Row {+ /// A team month key, wrapped by KMS.+ TeamMonth {+ team_id: i64,+ key_month: String,+ wrapped_key: Vec<u8>,+ },+ /// A session key, sealed under the team month key of `key_month`.+ Session {+ team_id: i64,+ key_month: String,+ sealed: SealedKey,+ },+ /// A deletion replaced the key material. A writer that meets one drops the data it was about to store.+ Deleted,+ /// A row that holds no key a writer can use and no tombstone. A writer drops the data, and never writes over it.+ Unusable(Unusable),+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Unusable {+ Malformed(&'static str),+ /// The row is past `expires_at`, and DynamoDB TTL has not removed it yet.+ Expired,+}++impl Row {+ /// The full item for a conditional put. `key_month` sets the row's expiry.+ pub fn to_item(&self, key: &TableKey) -> Result<Item, RowError> {+ let mut item = key.to_item();+ match self {+ Row::TeamMonth {+ team_id,+ key_month,+ wrapped_key,+ } => {+ insert_common(&mut item, *team_id, key_month)?;+ item.insert(+ "wrapped_key".to_owned(),+ AttributeValue::B(Blob::new(wrapped_key.clone())),+ );+ }+ Row::Session {+ team_id,+ key_month,+ sealed,+ } => {+ insert_common(&mut item, *team_id, key_month)?;+ item.insert(+ "sealed_key".to_owned(),+ AttributeValue::B(Blob::new(sealed.sealed.clone())),+ );+ item.insert(+ "key_nonce".to_owned(),+ AttributeValue::B(Blob::new(sealed.nonce.to_vec())),+ );+ }+ Row::Deleted => {+ item.insert("deleted".to_owned(), AttributeValue::Bool(true));+ }+ Row::Unusable(_) => return Err(RowError::NotWritable),+ }+ Ok(item)+ }++ /// Reads a stored item at `now`, in epoch seconds. A row that cannot give a key reads as `Unusable`, so one bad row+ /// drops only its own data.+ pub fn read(item: &Item, now: i64) -> Self {+ if is_tombstone(item) {+ return Row::Deleted;+ }+ let expires_at = match item.get("expires_at") {+ Some(AttributeValue::N(at)) => at.parse::<i64>().ok(),+ Some(_) => None,+ // Every row this crate writes has an expiry, but a row without one is never removed, so it stays usable.+ None => Some(i64::MAX),+ };+ match (expires_at, Row::from_item(item)) {+ (None, _) => Row::Unusable(Unusable::Malformed("expires_at")),+ // An expired row must not seal new data, because TTL can remove its key at any moment.+ (Some(at), _) if at <= now => Row::Unusable(Unusable::Expired),+ (_, Ok(row)) => row,+ (_, Err(RowError::Missing(name) | RowError::Malformed(name))) => {+ Row::Unusable(Unusable::Malformed(name))+ }+ (_, Err(RowError::NotWritable)) => Row::Unusable(Unusable::Malformed("row")),+ }+ }++ pub fn from_item(item: &Item) -> Result<Self, RowError> {+ if is_tombstone(item) {+ return Ok(Row::Deleted);+ }+ let team_id = string_number(item, "team_id")?;+ let key_month = string(item, "key_month")?;+ if let Some(wrapped) = item.get("wrapped_key") {+ return Ok(Row::TeamMonth {+ team_id,+ key_month,+ wrapped_key: blob(wrapped, "wrapped_key")?,+ });+ }+ let sealed = blob(+ item.get("sealed_key")+ .ok_or(RowError::Missing("sealed_key"))?,+ "sealed_key",+ )?;+ let nonce = blob(+ item.get("key_nonce")+ .ok_or(RowError::Missing("key_nonce"))?,+ "key_nonce",+ )?;+ let nonce: [u8; NONCE_BYTES] = nonce+ .try_into()+ .map_err(|_| RowError::Malformed("key_nonce"))?;+ Ok(Row::Session {+ team_id,+ key_month,+ sealed: SealedKey { sealed, nonce },+ })+ }+}++fn is_tombstone(item: &Item) -> bool {+ matches!(item.get("deleted"), Some(AttributeValue::Bool(true)))+}++/// The DynamoDB update expression and its values for a deletion. It sets the tombstone and removes the key material in+/// one write, and it keeps `expires_at`, so TTL removes the tombstone after every session that could open the key is+/// too old to be admitted.+pub fn tombstone_update() -> (String, Item) {+ (+ format!("SET deleted = :deleted REMOVE {}", KEY_MATERIAL.join(", ")),+ HashMap::from([(":deleted".to_owned(), AttributeValue::Bool(true))]),+ )+}++/// The item `tombstone_update` leaves in place of `item`.+pub fn tombstoned(item: &Item) -> Item {+ let mut item = item.clone();+ for name in KEY_MATERIAL {+ item.remove(name);+ }+ item.insert("deleted".to_owned(), AttributeValue::Bool(true));+ item+}++fn insert_common(item: &mut Item, team_id: i64, key_month: &str) -> Result<(), RowError> {+ item.insert("team_id".to_owned(), AttributeValue::N(team_id.to_string()));+ item.insert(+ "key_month".to_owned(),+ AttributeValue::S(key_month.to_owned()),+ );+ item.insert(+ "expires_at".to_owned(),+ AttributeValue::N(expires_at(key_month)?.to_string()),+ );+ Ok(())+}++fn string(item: &Item, name: &'static str) -> Result<String, RowError> {+ match item.get(name) {+ Some(AttributeValue::S(value)) => Ok(value.clone()),+ Some(_) => Err(RowError::Malformed(name)),+ None => Err(RowError::Missing(name)),+ }+}++fn string_number(item: &Item, name: &'static str) -> Result<i64, RowError> {+ match item.get(name) {+ Some(AttributeValue::N(value)) => value.parse().map_err(|_| RowError::Malformed(name)),+ Some(_) => Err(RowError::Malformed(name)),+ None => Err(RowError::Missing(name)),+ }+}++fn blob(value: &AttributeValue, name: &'static str) -> Result<Vec<u8>, RowError> {+ match value {+ AttributeValue::B(blob) => Ok(blob.as_ref().to_vec()),+ _ => Err(RowError::Malformed(name)),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn keys_sessions_on_the_shards_the_v3_deletion_code_reads() {+ // Shard numbers from the Node `sessionShard`, which the Python deletion store mirrors.+ let cases = [+ ("019a2f3e-8c00-7000-8000-000000000007", 9),+ ("019a2f3e-8c00-7000-8000-000000000008", 11),+ ("00000000-0000-7000-8000-000000000000", 20),+ ("019a2f3e-8c00-7000-8000-00000000ffff", 31),+ ];+ for (session_id, shard) in cases {+ assert_eq!(+ TableKey::session(7, session_id).pk,+ format!("team:7:shard:{shard}")+ );+ }+ }++ fn session_row() -> Row {+ Row::Session {+ team_id: 7,+ key_month: "2026-10".to_owned(),+ sealed: SealedKey {+ sealed: vec![4; 48],+ nonce: [5; NONCE_BYTES],+ },+ }+ }++ fn team_month_row() -> Row {+ Row::TeamMonth {+ team_id: 7,+ key_month: "2026-10".to_owned(),+ wrapped_key: vec![1, 2, 3],+ }+ }++ #[test]+ fn rows_round_trip_and_expire_five_months_after_key_month() {+ let session = TableKey::session(7, "019a2f3e-8c00-7000-8000-000000000007");+ for row in [team_month_row(), session_row(), Row::Deleted] {+ let item = row.to_item(&session).unwrap();+ assert_eq!(Row::from_item(&item).unwrap(), row);+ assert_eq!(TableKey::from_item(&item).unwrap(), session);+ }+ let item = team_month_row()+ .to_item(&TableKey::team_month(7, "2026-10"))+ .unwrap();+ // 2027-03-01T00:00:00Z+ assert_eq!(+ item.get("expires_at"),+ Some(&AttributeValue::N("1803859200".to_owned()))+ );+ // 2027-05-01T00:00:00Z: the lifetime crosses the year.+ assert_eq!(expires_at("2026-12"), Ok(1_809_129_600));+ assert_eq!(+ expires_at("2026-10-15"),+ Err(RowError::Malformed("key_month"))+ );+ assert_eq!(key_month("2026-10-15"), "2026-10");+ assert_eq!(+ Row::from_item(&session.to_item()),+ Err(RowError::Missing("team_id"))+ );+ }++ #[test]+ fn reads_a_row_that_gives_no_key_as_unusable() {+ let key = TableKey::session(7, "019a2f3e-8c00-7000-8000-000000000007");+ let item = session_row().to_item(&key).unwrap();+ let expiry = 1_803_859_200;+ assert_eq!(Row::read(&item, expiry - 1), session_row());+ assert_eq!(Row::read(&item, expiry), Row::Unusable(Unusable::Expired));+ assert_eq!(Row::read(&tombstoned(&item), expiry + 1), Row::Deleted);++ let mut truncated = item.clone();+ truncated.insert(+ "key_nonce".to_owned(),+ AttributeValue::B(Blob::new(vec![5; 4])),+ );+ assert_eq!(+ Row::read(&truncated, 0),+ Row::Unusable(Unusable::Malformed("key_nonce"))+ );+ assert_eq!(+ Row::Unusable(Unusable::Expired).to_item(&key),+ Err(RowError::NotWritable)+ );+ }++ #[test]+ fn a_deletion_update_leaves_a_tombstone_with_no_key_material() {+ let (expression, values) = tombstone_update();+ assert_eq!(+ expression,+ "SET deleted = :deleted REMOVE wrapped_key, sealed_key, key_nonce"+ );+ assert_eq!(values[":deleted"], AttributeValue::Bool(true));+ for (key, row) in [+ (TableKey::team_month(7, "2026-10"), team_month_row()),+ (+ TableKey::session(7, "019a2f3e-8c00-7000-8000-000000000007"),+ session_row(),+ ),+ ] {+ let item = tombstoned(&row.to_item(&key).unwrap());+ assert_eq!(Row::from_item(&item), Ok(Row::Deleted));+ assert!(KEY_MATERIAL.iter().all(|name| !item.contains_key(*name)));+ }+ }+}diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rsindex df98b6dd01c9..b0899ab8c573 100644--- a/rust/ai-research/replay-core-v4/src/lib.rs+++ b/rust/ai-research/replay-core-v4/src/lib.rs@@ -2,6 +2,8 @@pub mod envelope;pub mod gate;+#[cfg(feature = "store")]+pub mod keys;pub mod refs;pub mod seal;pub mod session;