Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-core-v4): wrap team month keys with KMS (#117955)

Nicholas Waltz34ae8f8336cf
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index f325cdb27ecf..a8992f52227d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -980,6 +980,31 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "aws-sdk-kms"
+version = "1.111.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c62d6d9b9c60e08be622a6bfc98e483d75683795caaf9704886970aa07ed8529"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http 0.63.6",
+ "aws-smithy-json 0.62.7",
+ "aws-smithy-observability",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.4.0",
+ "regex-lite",
+ "tracing",
+]
+
[[package]]
name = "aws-sdk-s3"
version = "1.119.0"
@@ -10994,15 +11019,20 @@ name = "replay-core-v4"
version = "0.1.0"
dependencies = [
"aes-gcm",
+ "async-trait",
"aws-sdk-dynamodb",
+ "aws-sdk-kms",
"base64 0.22.1",
"chrono",
+ "governor",
"hkdf",
"hmac 0.12.1",
"md5",
+ "moka",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
+ "tokio",
]
[[package]]
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 9d91a47e1a20..01e9f687ea23 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -250,6 +250,7 @@ ahash = "0.8.11"
aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
aws-sdk-s3 = "1.58.0"
aws-sdk-dynamodb = "1.55.0"
+aws-sdk-kms = "1"
aws-smithy-runtime-api = { version = "1.11.5", features = ["client"] }
aws-smithy-http-client = { version = "1.1.11", features = ["rustls-aws-lc"] }
mockall = "0.13.0"
diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.toml
index 7fd7a15bbf34..6c23744ae361 100644
--- a/rust/ai-research/replay-core-v4/Cargo.toml
+++ b/rust/ai-research/replay-core-v4/Cargo.toml
@@ -11,16 +11,21 @@ workspace = true
[features]
default = ["store"]
# The key table and KMS clients. A reader that only opens frames turns default features off and skips the AWS SDK.
-store = ["dep:aws-sdk-dynamodb", "dep:chrono"]
+store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:governor", "dep:moka", "dep:tokio"]
[dependencies]
aes-gcm = "0.10"
+async-trait = { workspace = true, optional = true }
aws-sdk-dynamodb = { workspace = true, optional = true }
+aws-sdk-kms = { workspace = true, optional = true }
base64 = { workspace = true }
chrono = { workspace = true, optional = true }
+governor = { workspace = true, optional = true }
hkdf = "0.12"
hmac = { workspace = true }
md5 = "0.8"
+moka = { workspace = true, optional = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
+tokio = { workspace = true, optional = true }
diff --git a/rust/ai-research/replay-core-v4/src/keys/kms.rs b/rust/ai-research/replay-core-v4/src/keys/kms.rs
new file mode 100644
index 000000000000..11f513fd5156
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/keys/kms.rs
@@ -0,0 +1,309 @@
+//! KMS wraps every team month key. Unwrapped keys are cached, so a busy team costs one KMS call per cache lifetime per
+//! process, not one per flush.
+
+use std::num::NonZeroU32;
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use aws_sdk_kms::error::DisplayErrorContext;
+use aws_sdk_kms::primitives::Blob;
+use aws_sdk_kms::types::DataKeySpec;
+use governor::clock::DefaultClock;
+use governor::state::{InMemoryState, NotKeyed};
+use governor::{Quota, RateLimiter};
+use moka::future::Cache;
+use tokio::sync::Semaphore;
+
+use crate::seal::WrappingContext;
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub enum KmsError {
+ #[error("KMS request failed: {0}")]
+ Request(String),
+ #[error("KMS request timed out")]
+ Timeout,
+ /// KMS refused this one blob, so retrying cannot help, and other keys still unwrap.
+ #[error("KMS refused the wrapped key: {0}")]
+ Rejected(String),
+ #[error("KMS returned a key that is not 32 bytes")]
+ InvalidKey,
+}
+
+/// A fresh team month key: the plaintext to seal with, and the KMS blob to store.
+#[derive(Clone)]
+pub struct DataKey {
+ pub plaintext: [u8; 32],
+ pub wrapped: Vec<u8>,
+}
+
+/// The two KMS calls the key store makes. Tests replace KMS at this boundary.
+#[async_trait]
+pub trait KeyWrapper: Send + Sync {
+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError>;
+ async fn unwrap(&self, context: &WrappingContext, wrapped: &[u8])
+ -> Result<[u8; 32], KmsError>;
+}
+
+/// Limits that match the v3 mirror: KMS throttles a whole account, so one lane must not take all of it.
+#[derive(Debug, Clone)]
+pub struct KmsLimits {
+ pub requests_per_second: NonZeroU32,
+ pub max_in_flight: usize,
+ pub timeout: Duration,
+}
+
+impl Default for KmsLimits {
+ fn default() -> Self {
+ Self {
+ requests_per_second: NonZeroU32::new(150).expect("nonzero"),
+ max_in_flight: 8,
+ timeout: Duration::from_secs(5),
+ }
+ }
+}
+
+pub struct AwsKms {
+ client: aws_sdk_kms::Client,
+ key_id: String,
+ rate: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,
+ in_flight: Semaphore,
+ timeout: Duration,
+}
+
+impl AwsKms {
+ pub fn new(client: aws_sdk_kms::Client, key_id: String, limits: KmsLimits) -> Self {
+ Self {
+ client,
+ key_id,
+ rate: RateLimiter::direct(Quota::per_second(limits.requests_per_second)),
+ in_flight: Semaphore::new(limits.max_in_flight),
+ timeout: limits.timeout,
+ }
+ }
+
+ async fn call<T, F>(&self, request: F) -> Result<T, KmsError>
+ where
+ F: std::future::Future<Output = Result<T, KmsError>>,
+ {
+ let _permit = self
+ .in_flight
+ .acquire()
+ .await
+ .map_err(|e| KmsError::Request(e.to_string()))?;
+ self.rate.until_ready().await;
+ tokio::time::timeout(self.timeout, request)
+ .await
+ .map_err(|_| KmsError::Timeout)?
+ }
+}
+
+fn plaintext(bytes: Option<&Blob>) -> Result<[u8; 32], KmsError> {
+ bytes
+ .map(|blob| blob.as_ref())
+ .and_then(|b| b.try_into().ok())
+ .ok_or(KmsError::InvalidKey)
+}
+
+#[async_trait]
+impl KeyWrapper for AwsKms {
+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError> {
+ let mut request = self
+ .client
+ .generate_data_key()
+ .key_id(&self.key_id)
+ .key_spec(DataKeySpec::Aes256);
+ for (name, value) in context.entries() {
+ request = request.encryption_context(name, value);
+ }
+ let output = self
+ .call(async {
+ request
+ .send()
+ .await
+ .map_err(|e| KmsError::Request(DisplayErrorContext(&e).to_string()))
+ })
+ .await?;
+ let wrapped = output
+ .ciphertext_blob()
+ .map(|blob| blob.as_ref().to_vec())
+ .filter(|b| !b.is_empty());
+ Ok(DataKey {
+ plaintext: plaintext(output.plaintext())?,
+ wrapped: wrapped.ok_or(KmsError::InvalidKey)?,
+ })
+ }
+
+ async fn unwrap(
+ &self,
+ context: &WrappingContext,
+ wrapped: &[u8],
+ ) -> Result<[u8; 32], KmsError> {
+ let mut request = self
+ .client
+ .decrypt()
+ .key_id(&self.key_id)
+ .ciphertext_blob(Blob::new(wrapped));
+ for (name, value) in context.entries() {
+ request = request.encryption_context(name, value);
+ }
+ let output = self
+ .call(async {
+ request.send().await.map_err(|e| {
+ // A blob that is corrupt, or that does not match its context, gets one of these. An access
+ // denial or a disabled key stops every unwrap, so it stays a failed request.
+ let rejected = e.as_service_error().is_some_and(|e| {
+ e.is_invalid_ciphertext_exception() || e.is_incorrect_key_exception()
+ });
+ if rejected {
+ KmsError::Rejected(DisplayErrorContext(&e).to_string())
+ } else {
+ KmsError::Request(DisplayErrorContext(&e).to_string())
+ }
+ })
+ })
+ .await?;
+ plaintext(output.plaintext())
+ }
+}
+
+/// The KMS encryption context and the wrapped blob, which together name one unwrapped key.
+type UnwrapId = (Vec<(&'static str, String)>, Vec<u8>);
+
+/// Caches unwrapped keys. Concurrent unwraps of one key share a single KMS call.
+pub struct CachedKms<W> {
+ inner: W,
+ cache: Cache<UnwrapId, [u8; 32]>,
+}
+
+impl<W: KeyWrapper> CachedKms<W> {
+ pub fn new(inner: W, max_keys: u64, lifetime: Duration) -> Self {
+ Self {
+ inner,
+ cache: Cache::builder()
+ .max_capacity(max_keys)
+ .time_to_live(lifetime)
+ .build(),
+ }
+ }
+
+ /// Records a key this process generated and stored, so reading it back costs no KMS call.
+ pub async fn remember(&self, context: &WrappingContext, key: &DataKey) {
+ self.cache
+ .insert((context.entries(), key.wrapped.clone()), key.plaintext)
+ .await;
+ }
+}
+
+#[async_trait]
+impl<W: KeyWrapper> KeyWrapper for CachedKms<W> {
+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError> {
+ self.inner.generate(context).await
+ }
+
+ async fn unwrap(
+ &self,
+ context: &WrappingContext,
+ wrapped: &[u8],
+ ) -> Result<[u8; 32], KmsError> {
+ self.cache
+ .try_get_with(
+ (context.entries(), wrapped.to_vec()),
+ self.inner.unwrap(context, wrapped),
+ )
+ .await
+ .map_err(|e: Arc<KmsError>| (*e).clone())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ use tokio::sync::Notify;
+
+ use super::*;
+
+ /// Holds every unwrap until released, so the test controls when KMS answers.
+ struct HeldKms {
+ calls: AtomicUsize,
+ release: Notify,
+ fail_first: bool,
+ }
+
+ #[async_trait]
+ impl KeyWrapper for HeldKms {
+ async fn generate(&self, _: &WrappingContext) -> Result<DataKey, KmsError> {
+ unreachable!("the cache never generates")
+ }
+
+ async fn unwrap(&self, _: &WrappingContext, _: &[u8]) -> Result<[u8; 32], KmsError> {
+ let call = self.calls.fetch_add(1, Ordering::SeqCst);
+ self.release.notified().await;
+ if self.fail_first && call == 0 {
+ return Err(KmsError::Timeout);
+ }
+ Ok([7; 32])
+ }
+ }
+
+ fn context() -> WrappingContext {
+ WrappingContext {
+ team_id: 7,
+ key_month: "2026-10".to_owned(),
+ session_id: None,
+ }
+ }
+
+ #[tokio::test]
+ async fn concurrent_unwraps_of_one_key_make_one_kms_call() {
+ let kms = Arc::new(CachedKms::new(
+ HeldKms {
+ calls: AtomicUsize::new(0),
+ release: Notify::new(),
+ fail_first: false,
+ },
+ 10,
+ Duration::from_secs(60),
+ ));
+ let waiters: Vec<_> = (0..8)
+ .map(|_| {
+ let kms = kms.clone();
+ tokio::spawn(async move { kms.unwrap(&context(), b"wrapped").await })
+ })
+ .collect();
+ while kms.inner.calls.load(Ordering::SeqCst) == 0 {
+ tokio::task::yield_now().await;
+ }
+ kms.inner.release.notify_one();
+ for waiter in waiters {
+ assert_eq!(waiter.await.unwrap().unwrap(), [7; 32]);
+ }
+ assert_eq!(kms.inner.calls.load(Ordering::SeqCst), 1);
+ }
+
+ #[tokio::test]
+ async fn a_failed_unwrap_is_not_cached() {
+ let kms = Arc::new(CachedKms::new(
+ HeldKms {
+ calls: AtomicUsize::new(0),
+ release: Notify::new(),
+ fail_first: true,
+ },
+ 10,
+ Duration::from_secs(60),
+ ));
+ for expected_ok in [false, true] {
+ let call = {
+ let kms = kms.clone();
+ tokio::spawn(async move { kms.unwrap(&context(), b"wrapped").await })
+ };
+ let calls_before = if expected_ok { 1 } else { 0 };
+ while kms.inner.calls.load(Ordering::SeqCst) == calls_before {
+ tokio::task::yield_now().await;
+ }
+ kms.inner.release.notify_one();
+ assert_eq!(call.await.unwrap().is_ok(), expected_ok);
+ }
+ }
+}
diff --git a/rust/ai-research/replay-core-v4/src/keys/mod.rs b/rust/ai-research/replay-core-v4/src/keys/mod.rs
index f0519f57d192..6228b8b2e7b4 100644
--- a/rust/ai-research/replay-core-v4/src/keys/mod.rs
+++ b/rust/ai-research/replay-core-v4/src/keys/mod.rs
@@ -1,3 +1,4 @@
//! The v4 key table in prod-us: one KMS-wrapped key per team per UTC month, and one session key sealed under it.
+pub mod kms;
pub mod rows;