:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-core-v4): wrap team month keys with KMS (#117955)
Nicholas Waltz34ae8f8336cf
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex f325cdb27ecf..a8992f52227d 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -980,6 +980,31 @@ dependencies = ["tracing",]+[[package]]+name = "aws-sdk-kms"+version = "1.111.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c62d6d9b9c60e08be622a6bfc98e483d75683795caaf9704886970aa07ed8529"+dependencies = [+ "arc-swap",+ "aws-credential-types",+ "aws-runtime",+ "aws-smithy-async",+ "aws-smithy-http 0.63.6",+ "aws-smithy-json 0.62.7",+ "aws-smithy-observability",+ "aws-smithy-runtime",+ "aws-smithy-runtime-api",+ "aws-smithy-types",+ "aws-types",+ "bytes",+ "fastrand",+ "http 0.2.12",+ "http 1.4.0",+ "regex-lite",+ "tracing",+]+[[package]]name = "aws-sdk-s3"version = "1.119.0"@@ -10994,15 +11019,20 @@ name = "replay-core-v4"version = "0.1.0"dependencies = ["aes-gcm",+ "async-trait","aws-sdk-dynamodb",+ "aws-sdk-kms","base64 0.22.1","chrono",+ "governor","hkdf","hmac 0.12.1","md5",+ "moka","serde_json","sha2 0.10.9","thiserror 2.0.18",+ "tokio",][[package]]diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 9d91a47e1a20..01e9f687ea23 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -250,6 +250,7 @@ ahash = "0.8.11"aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }aws-sdk-s3 = "1.58.0"aws-sdk-dynamodb = "1.55.0"+aws-sdk-kms = "1"aws-smithy-runtime-api = { version = "1.11.5", features = ["client"] }aws-smithy-http-client = { version = "1.1.11", features = ["rustls-aws-lc"] }mockall = "0.13.0"diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.tomlindex 7fd7a15bbf34..6c23744ae361 100644--- a/rust/ai-research/replay-core-v4/Cargo.toml+++ b/rust/ai-research/replay-core-v4/Cargo.toml@@ -11,16 +11,21 @@ workspace = true[features]default = ["store"]# The key table and KMS clients. A reader that only opens frames turns default features off and skips the AWS SDK.-store = ["dep:aws-sdk-dynamodb", "dep:chrono"]+store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:governor", "dep:moka", "dep:tokio"][dependencies]aes-gcm = "0.10"+async-trait = { workspace = true, optional = true }aws-sdk-dynamodb = { workspace = true, optional = true }+aws-sdk-kms = { workspace = true, optional = true }base64 = { workspace = true }chrono = { workspace = true, optional = true }+governor = { workspace = true, optional = true }hkdf = "0.12"hmac = { workspace = true }md5 = "0.8"+moka = { workspace = true, optional = true }serde_json = { workspace = true }sha2 = { workspace = true }thiserror = { workspace = true }+tokio = { workspace = true, optional = true }diff --git a/rust/ai-research/replay-core-v4/src/keys/kms.rs b/rust/ai-research/replay-core-v4/src/keys/kms.rsnew file mode 100644index 000000000000..11f513fd5156--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/keys/kms.rs@@ -0,0 +1,309 @@+//! KMS wraps every team month key. Unwrapped keys are cached, so a busy team costs one KMS call per cache lifetime per+//! process, not one per flush.++use std::num::NonZeroU32;+use std::sync::Arc;+use std::time::Duration;++use async_trait::async_trait;+use aws_sdk_kms::error::DisplayErrorContext;+use aws_sdk_kms::primitives::Blob;+use aws_sdk_kms::types::DataKeySpec;+use governor::clock::DefaultClock;+use governor::state::{InMemoryState, NotKeyed};+use governor::{Quota, RateLimiter};+use moka::future::Cache;+use tokio::sync::Semaphore;++use crate::seal::WrappingContext;++#[derive(Debug, Clone, thiserror::Error)]+pub enum KmsError {+ #[error("KMS request failed: {0}")]+ Request(String),+ #[error("KMS request timed out")]+ Timeout,+ /// KMS refused this one blob, so retrying cannot help, and other keys still unwrap.+ #[error("KMS refused the wrapped key: {0}")]+ Rejected(String),+ #[error("KMS returned a key that is not 32 bytes")]+ InvalidKey,+}++/// A fresh team month key: the plaintext to seal with, and the KMS blob to store.+#[derive(Clone)]+pub struct DataKey {+ pub plaintext: [u8; 32],+ pub wrapped: Vec<u8>,+}++/// The two KMS calls the key store makes. Tests replace KMS at this boundary.+#[async_trait]+pub trait KeyWrapper: Send + Sync {+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError>;+ async fn unwrap(&self, context: &WrappingContext, wrapped: &[u8])+ -> Result<[u8; 32], KmsError>;+}++/// Limits that match the v3 mirror: KMS throttles a whole account, so one lane must not take all of it.+#[derive(Debug, Clone)]+pub struct KmsLimits {+ pub requests_per_second: NonZeroU32,+ pub max_in_flight: usize,+ pub timeout: Duration,+}++impl Default for KmsLimits {+ fn default() -> Self {+ Self {+ requests_per_second: NonZeroU32::new(150).expect("nonzero"),+ max_in_flight: 8,+ timeout: Duration::from_secs(5),+ }+ }+}++pub struct AwsKms {+ client: aws_sdk_kms::Client,+ key_id: String,+ rate: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,+ in_flight: Semaphore,+ timeout: Duration,+}++impl AwsKms {+ pub fn new(client: aws_sdk_kms::Client, key_id: String, limits: KmsLimits) -> Self {+ Self {+ client,+ key_id,+ rate: RateLimiter::direct(Quota::per_second(limits.requests_per_second)),+ in_flight: Semaphore::new(limits.max_in_flight),+ timeout: limits.timeout,+ }+ }++ async fn call<T, F>(&self, request: F) -> Result<T, KmsError>+ where+ F: std::future::Future<Output = Result<T, KmsError>>,+ {+ let _permit = self+ .in_flight+ .acquire()+ .await+ .map_err(|e| KmsError::Request(e.to_string()))?;+ self.rate.until_ready().await;+ tokio::time::timeout(self.timeout, request)+ .await+ .map_err(|_| KmsError::Timeout)?+ }+}++fn plaintext(bytes: Option<&Blob>) -> Result<[u8; 32], KmsError> {+ bytes+ .map(|blob| blob.as_ref())+ .and_then(|b| b.try_into().ok())+ .ok_or(KmsError::InvalidKey)+}++#[async_trait]+impl KeyWrapper for AwsKms {+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError> {+ let mut request = self+ .client+ .generate_data_key()+ .key_id(&self.key_id)+ .key_spec(DataKeySpec::Aes256);+ for (name, value) in context.entries() {+ request = request.encryption_context(name, value);+ }+ let output = self+ .call(async {+ request+ .send()+ .await+ .map_err(|e| KmsError::Request(DisplayErrorContext(&e).to_string()))+ })+ .await?;+ let wrapped = output+ .ciphertext_blob()+ .map(|blob| blob.as_ref().to_vec())+ .filter(|b| !b.is_empty());+ Ok(DataKey {+ plaintext: plaintext(output.plaintext())?,+ wrapped: wrapped.ok_or(KmsError::InvalidKey)?,+ })+ }++ async fn unwrap(+ &self,+ context: &WrappingContext,+ wrapped: &[u8],+ ) -> Result<[u8; 32], KmsError> {+ let mut request = self+ .client+ .decrypt()+ .key_id(&self.key_id)+ .ciphertext_blob(Blob::new(wrapped));+ for (name, value) in context.entries() {+ request = request.encryption_context(name, value);+ }+ let output = self+ .call(async {+ request.send().await.map_err(|e| {+ // A blob that is corrupt, or that does not match its context, gets one of these. An access+ // denial or a disabled key stops every unwrap, so it stays a failed request.+ let rejected = e.as_service_error().is_some_and(|e| {+ e.is_invalid_ciphertext_exception() || e.is_incorrect_key_exception()+ });+ if rejected {+ KmsError::Rejected(DisplayErrorContext(&e).to_string())+ } else {+ KmsError::Request(DisplayErrorContext(&e).to_string())+ }+ })+ })+ .await?;+ plaintext(output.plaintext())+ }+}++/// The KMS encryption context and the wrapped blob, which together name one unwrapped key.+type UnwrapId = (Vec<(&'static str, String)>, Vec<u8>);++/// Caches unwrapped keys. Concurrent unwraps of one key share a single KMS call.+pub struct CachedKms<W> {+ inner: W,+ cache: Cache<UnwrapId, [u8; 32]>,+}++impl<W: KeyWrapper> CachedKms<W> {+ pub fn new(inner: W, max_keys: u64, lifetime: Duration) -> Self {+ Self {+ inner,+ cache: Cache::builder()+ .max_capacity(max_keys)+ .time_to_live(lifetime)+ .build(),+ }+ }++ /// Records a key this process generated and stored, so reading it back costs no KMS call.+ pub async fn remember(&self, context: &WrappingContext, key: &DataKey) {+ self.cache+ .insert((context.entries(), key.wrapped.clone()), key.plaintext)+ .await;+ }+}++#[async_trait]+impl<W: KeyWrapper> KeyWrapper for CachedKms<W> {+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError> {+ self.inner.generate(context).await+ }++ async fn unwrap(+ &self,+ context: &WrappingContext,+ wrapped: &[u8],+ ) -> Result<[u8; 32], KmsError> {+ self.cache+ .try_get_with(+ (context.entries(), wrapped.to_vec()),+ self.inner.unwrap(context, wrapped),+ )+ .await+ .map_err(|e: Arc<KmsError>| (*e).clone())+ }+}++#[cfg(test)]+mod tests {+ use std::sync::atomic::{AtomicUsize, Ordering};++ use tokio::sync::Notify;++ use super::*;++ /// Holds every unwrap until released, so the test controls when KMS answers.+ struct HeldKms {+ calls: AtomicUsize,+ release: Notify,+ fail_first: bool,+ }++ #[async_trait]+ impl KeyWrapper for HeldKms {+ async fn generate(&self, _: &WrappingContext) -> Result<DataKey, KmsError> {+ unreachable!("the cache never generates")+ }++ async fn unwrap(&self, _: &WrappingContext, _: &[u8]) -> Result<[u8; 32], KmsError> {+ let call = self.calls.fetch_add(1, Ordering::SeqCst);+ self.release.notified().await;+ if self.fail_first && call == 0 {+ return Err(KmsError::Timeout);+ }+ Ok([7; 32])+ }+ }++ fn context() -> WrappingContext {+ WrappingContext {+ team_id: 7,+ key_month: "2026-10".to_owned(),+ session_id: None,+ }+ }++ #[tokio::test]+ async fn concurrent_unwraps_of_one_key_make_one_kms_call() {+ let kms = Arc::new(CachedKms::new(+ HeldKms {+ calls: AtomicUsize::new(0),+ release: Notify::new(),+ fail_first: false,+ },+ 10,+ Duration::from_secs(60),+ ));+ let waiters: Vec<_> = (0..8)+ .map(|_| {+ let kms = kms.clone();+ tokio::spawn(async move { kms.unwrap(&context(), b"wrapped").await })+ })+ .collect();+ while kms.inner.calls.load(Ordering::SeqCst) == 0 {+ tokio::task::yield_now().await;+ }+ kms.inner.release.notify_one();+ for waiter in waiters {+ assert_eq!(waiter.await.unwrap().unwrap(), [7; 32]);+ }+ assert_eq!(kms.inner.calls.load(Ordering::SeqCst), 1);+ }++ #[tokio::test]+ async fn a_failed_unwrap_is_not_cached() {+ let kms = Arc::new(CachedKms::new(+ HeldKms {+ calls: AtomicUsize::new(0),+ release: Notify::new(),+ fail_first: true,+ },+ 10,+ Duration::from_secs(60),+ ));+ for expected_ok in [false, true] {+ let call = {+ let kms = kms.clone();+ tokio::spawn(async move { kms.unwrap(&context(), b"wrapped").await })+ };+ let calls_before = if expected_ok { 1 } else { 0 };+ while kms.inner.calls.load(Ordering::SeqCst) == calls_before {+ tokio::task::yield_now().await;+ }+ kms.inner.release.notify_one();+ assert_eq!(call.await.unwrap().is_ok(), expected_ok);+ }+ }+}diff --git a/rust/ai-research/replay-core-v4/src/keys/mod.rs b/rust/ai-research/replay-core-v4/src/keys/mod.rsindex f0519f57d192..6228b8b2e7b4 100644--- a/rust/ai-research/replay-core-v4/src/keys/mod.rs+++ b/rust/ai-research/replay-core-v4/src/keys/mod.rs@@ -1,3 +1,4 @@//! The v4 key table in prod-us: one KMS-wrapped key per team per UTC month, and one session key sealed under it.+pub mod kms;pub mod rows;