Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-core-v4): read and write the v4 key table (#117956)

Nicholas Waltzf3edbdb1ba95
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a8992f52227d..37a504044347 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11029,6 +11029,7 @@ dependencies = [
"hmac 0.12.1",
"md5",
"moka",
+ "rand 0.8.5",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.toml
index 6c23744ae361..d4023663ba26 100644
--- a/rust/ai-research/replay-core-v4/Cargo.toml
+++ b/rust/ai-research/replay-core-v4/Cargo.toml
@@ -11,7 +11,7 @@ workspace = true
[features]
default = ["store"]
# The key table and KMS clients. A reader that only opens frames turns default features off and skips the AWS SDK.
-store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:governor", "dep:moka", "dep:tokio"]
+store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:governor", "dep:moka", "dep:rand", "dep:tokio"]
[dependencies]
aes-gcm = "0.10"
@@ -25,6 +25,7 @@ hkdf = "0.12"
hmac = { workspace = true }
md5 = "0.8"
moka = { workspace = true, optional = true }
+rand = { workspace = true, optional = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
diff --git a/rust/ai-research/replay-core-v4/src/keys/memory.rs b/rust/ai-research/replay-core-v4/src/keys/memory.rs
new file mode 100644
index 000000000000..f8e2b574f7c1
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/keys/memory.rs
@@ -0,0 +1,46 @@
+//! An in-memory key table for tests. DynamoDB is the boundary, so tests replace it here.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Mutex;
+
+use async_trait::async_trait;
+
+use super::rows::{Row, TableKey};
+use super::table::{KeyTable, PutOutcome, TableError};
+
+#[derive(Default)]
+pub struct MemoryTable {
+ pub rows: Mutex<HashMap<TableKey, Row>>,
+ pub reads: AtomicUsize,
+}
+
+impl MemoryTable {
+ /// Stores a row as another writer would, behind this process's back.
+ pub fn store(&self, key: TableKey, row: Row) {
+ self.rows.lock().unwrap().insert(key, row);
+ }
+}
+
+#[async_trait]
+impl KeyTable for MemoryTable {
+ async fn batch_get(&self, keys: &[TableKey]) -> Result<HashMap<TableKey, Row>, TableError> {
+ self.reads.fetch_add(1, Ordering::SeqCst);
+ let rows = self.rows.lock().unwrap();
+ Ok(keys
+ .iter()
+ .filter_map(|k| rows.get(k).map(|r| (k.clone(), r.clone())))
+ .collect())
+ }
+
+ async fn put_if_absent(&self, key: &TableKey, row: &Row) -> Result<PutOutcome, TableError> {
+ let mut rows = self.rows.lock().unwrap();
+ match rows.get(key) {
+ Some(existing) => Ok(PutOutcome::Existing(existing.clone())),
+ None => {
+ rows.insert(key.clone(), row.clone());
+ Ok(PutOutcome::Stored)
+ }
+ }
+ }
+}
diff --git a/rust/ai-research/replay-core-v4/src/keys/mod.rs b/rust/ai-research/replay-core-v4/src/keys/mod.rs
index 6228b8b2e7b4..e88bf5b304d5 100644
--- a/rust/ai-research/replay-core-v4/src/keys/mod.rs
+++ b/rust/ai-research/replay-core-v4/src/keys/mod.rs
@@ -1,4 +1,7 @@
//! The v4 key table in prod-us: one KMS-wrapped key per team per UTC month, and one session key sealed under it.
pub mod kms;
+#[cfg(test)]
+pub(crate) mod memory;
pub mod rows;
+pub mod table;
diff --git a/rust/ai-research/replay-core-v4/src/keys/rows.rs b/rust/ai-research/replay-core-v4/src/keys/rows.rs
index 637d511a23ba..d7e862d031a3 100644
--- a/rust/ai-research/replay-core-v4/src/keys/rows.rs
+++ b/rust/ai-research/replay-core-v4/src/keys/rows.rs
@@ -20,7 +20,7 @@ pub const KEY_MATERIAL: [&str; 3] = ["wrapped_key", "sealed_key", "key_nonce"];
pub type Item = HashMap<String, AttributeValue>;
-#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
pub enum RowError {
#[error("key row has no {0}")]
Missing(&'static str),
diff --git a/rust/ai-research/replay-core-v4/src/keys/table.rs b/rust/ai-research/replay-core-v4/src/keys/table.rs
new file mode 100644
index 000000000000..bc6f5efc0629
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/keys/table.rs
@@ -0,0 +1,416 @@
+//! Reads and conditional writes on the v4 key table.
+
+use std::collections::HashMap;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use aws_sdk_dynamodb::error::{ProvideErrorMetadata, SdkError};
+use aws_sdk_dynamodb::operation::put_item::PutItemError;
+use aws_sdk_dynamodb::types::{KeysAndAttributes, ReturnValuesOnConditionCheckFailure};
+use moka::future::Cache;
+use rand::Rng;
+
+use super::rows::{Item, Row, RowError, TableKey};
+
+/// DynamoDB's limit on keys in one BatchGetItem request.
+const BATCH_GET_LIMIT: usize = 100;
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub enum TableError {
+ #[error("key table request failed: {0}")]
+ Request(String),
+ #[error("key table read ran out of time with keys still unread")]
+ ReadBudget,
+ #[error("key table write ran out of time")]
+ WriteBudget,
+ #[error(transparent)]
+ Row(#[from] RowError),
+}
+
+/// The result of a conditional put.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PutOutcome {
+ /// Our row is now the row at that key.
+ Stored,
+ /// Another writer stored a row first. The caller adopts it, or drops its data when it is a tombstone.
+ Existing(Row),
+}
+
+#[async_trait]
+pub trait KeyTable: Send + Sync {
+ /// Reads rows with strongly consistent reads. A key with no row is absent from the result, and a row that gives no
+ /// key reads as `Row::Unusable`.
+ async fn batch_get(&self, keys: &[TableKey]) -> Result<HashMap<TableKey, Row>, TableError>;
+ /// Stores `row` at `key` unless a row is already there.
+ async fn put_if_absent(&self, key: &TableKey, row: &Row) -> Result<PutOutcome, TableError>;
+}
+
+pub struct AwsKeyTable {
+ client: aws_sdk_dynamodb::Client,
+ table: String,
+ retry_budget: Duration,
+}
+
+impl AwsKeyTable {
+ pub fn new(client: aws_sdk_dynamodb::Client, table: String) -> Self {
+ Self {
+ client,
+ table,
+ retry_budget: Duration::from_secs(30),
+ }
+ }
+
+ /// Reads rows, retrying until `deadline`. A request still in flight at the deadline is abandoned.
+ async fn batch_get_until(
+ &self,
+ keys: &[TableKey],
+ deadline: Instant,
+ ) -> Result<HashMap<TableKey, Row>, TableError> {
+ let now = now();
+ let mut rows = HashMap::new();
+ for chunk in keys.chunks(BATCH_GET_LIMIT) {
+ let mut pending: Vec<Item> = chunk.iter().map(TableKey::to_item).collect();
+ let mut attempt = 0;
+ while !pending.is_empty() {
+ if attempt > 0 {
+ sleep_until_retry(attempt, deadline)
+ .await
+ .ok_or(TableError::ReadBudget)?;
+ }
+ attempt += 1;
+ let request = KeysAndAttributes::builder()
+ .set_keys(Some(pending.clone()))
+ .consistent_read(true)
+ .build()
+ .map_err(|e| TableError::Request(e.to_string()))?;
+ let sent = self
+ .client
+ .batch_get_item()
+ .request_items(&self.table, request)
+ .send();
+ let output = match tokio::time::timeout(remaining(deadline), sent).await {
+ Err(_) => return Err(TableError::ReadBudget),
+ Ok(Ok(output)) => output,
+ Ok(Err(error)) if transient(&error) => continue,
+ Ok(Err(error)) => return Err(TableError::Request(error.to_string())),
+ };
+ for item in output
+ .responses()
+ .and_then(|r| r.get(&self.table))
+ .into_iter()
+ .flatten()
+ {
+ rows.insert(TableKey::from_item(item)?, Row::read(item, now));
+ }
+ // DynamoDB returns the keys it skipped under throttling, and they must be asked for again.
+ pending = output
+ .unprocessed_keys()
+ .and_then(|u| u.get(&self.table))
+ .map(|u| u.keys().to_vec())
+ .unwrap_or_default();
+ }
+ }
+ Ok(rows)
+ }
+}
+
+/// Throttling, server faults and transport failures pass. Any other error repeats on a retry.
+fn transient<E: ProvideErrorMetadata>(error: &SdkError<E>) -> bool {
+ match error {
+ SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) | SdkError::ResponseError(_) => {
+ true
+ }
+ SdkError::ServiceError(_) => {
+ matches!(
+ error.code(),
+ Some(
+ "ProvisionedThroughputExceededException"
+ | "ThrottlingException"
+ | "RequestLimitExceeded"
+ | "InternalServerError"
+ | "ServiceUnavailable"
+ )
+ ) || error
+ .raw_response()
+ .is_some_and(|r| r.status().is_server_error())
+ }
+ _ => false,
+ }
+}
+
+fn now() -> i64 {
+ chrono::Utc::now().timestamp()
+}
+
+/// The v3 backoff: doubling from 50 ms up to 3 s, with jitter so retrying pods spread out.
+fn backoff(attempt: u32) -> Duration {
+ let ceiling = 3_000u64.min(50u64.saturating_mul(1 << attempt.min(16)));
+ Duration::from_millis(rand::thread_rng().gen_range(ceiling / 2..=ceiling))
+}
+
+fn remaining(deadline: Instant) -> Duration {
+ deadline.saturating_duration_since(Instant::now())
+}
+
+/// Waits out the backoff before a retry. Returns `None` when the retry cannot start before `deadline`.
+async fn sleep_until_retry(attempt: u32, deadline: Instant) -> Option<()> {
+ let delay = backoff(attempt);
+ if delay >= remaining(deadline) {
+ return None;
+ }
+ tokio::time::sleep(delay).await;
+ Some(())
+}
+
+#[async_trait]
+impl KeyTable for AwsKeyTable {
+ async fn batch_get(&self, keys: &[TableKey]) -> Result<HashMap<TableKey, Row>, TableError> {
+ self.batch_get_until(keys, Instant::now() + self.retry_budget)
+ .await
+ }
+
+ async fn put_if_absent(&self, key: &TableKey, row: &Row) -> Result<PutOutcome, TableError> {
+ let item = row.to_item(key)?;
+ // One deadline covers every attempt, the backoff between them and the read of a winner.
+ let deadline = Instant::now() + self.retry_budget;
+ let mut attempt = 0;
+ let result = loop {
+ let sent = self
+ .client
+ .put_item()
+ .table_name(&self.table)
+ .set_item(Some(item.clone()))
+ .condition_expression("attribute_not_exists(pk)")
+ .return_values_on_condition_check_failure(
+ ReturnValuesOnConditionCheckFailure::AllOld,
+ )
+ .send();
+ let result = tokio::time::timeout(remaining(deadline), sent)
+ .await
+ .map_err(|_| TableError::WriteBudget)?;
+ match &result {
+ Err(error) if transient(error) => {}
+ _ => break result,
+ }
+ attempt += 1;
+ if sleep_until_retry(attempt, deadline).await.is_none() {
+ break result;
+ }
+ };
+ match result {
+ Ok(_) => Ok(PutOutcome::Stored),
+ Err(error) => {
+ // A retry after a put that landed meets its own row here, which the caller adopts as it would a winner.
+ let Some(PutItemError::ConditionalCheckFailedException(conflict)) =
+ error.as_service_error()
+ else {
+ return Err(TableError::Request(error.to_string()));
+ };
+ if let Some(item) = conflict.item() {
+ return Ok(PutOutcome::Existing(Row::read(item, now())));
+ }
+ // DynamoDB may omit the winning item, so read it.
+ let winner = self
+ .batch_get_until(std::slice::from_ref(key), deadline)
+ .await?
+ .remove(key);
+ winner.map(PutOutcome::Existing).ok_or_else(|| {
+ TableError::Request("conditional put lost to a row that is gone".to_owned())
+ })
+ }
+ }
+ }
+}
+
+/// Caches the rows the table holds, as v3 does.
+///
+/// The only change a stored key row ever sees is the deletion tombstone, so a cached row is stale only for a deletion,
+/// and the cache lifetime bounds how late a writer sees one. A missing row is never cached: another writer may store
+/// it at any moment, and a cached miss would make this process store a second key for the same session. An unusable
+/// row is not cached either, so a repaired row is seen at once. The cache lifetimes are far shorter than a row's
+/// lifetime of months, so a cached row cannot outlive its expiry by a meaningful time.
+pub struct CachedKeyTable<T> {
+ inner: T,
+ sessions: Cache<TableKey, Row>,
+ team_months: Cache<TableKey, Row>,
+}
+
+impl<T: KeyTable> CachedKeyTable<T> {
+ pub fn new(
+ inner: T,
+ max_rows: u64,
+ session_lifetime: Duration,
+ team_month_lifetime: Duration,
+ ) -> Self {
+ Self {
+ inner,
+ sessions: Cache::builder()
+ .max_capacity(max_rows)
+ .time_to_live(session_lifetime)
+ .build(),
+ team_months: Cache::builder()
+ .max_capacity(max_rows)
+ .time_to_live(team_month_lifetime)
+ .build(),
+ }
+ }
+
+ async fn hold(&self, key: &TableKey, row: &Row) {
+ if !matches!(row, Row::Unusable(_)) {
+ self.cache_for(key).insert(key.clone(), row.clone()).await;
+ }
+ }
+
+ fn cache_for(&self, key: &TableKey) -> &Cache<TableKey, Row> {
+ if key.sk.starts_with("session:") {
+ &self.sessions
+ } else {
+ &self.team_months
+ }
+ }
+}
+
+#[async_trait]
+impl<T: KeyTable> KeyTable for CachedKeyTable<T> {
+ async fn batch_get(&self, keys: &[TableKey]) -> Result<HashMap<TableKey, Row>, TableError> {
+ let mut rows = HashMap::new();
+ let mut missing = Vec::new();
+ for key in keys {
+ match self.cache_for(key).get(key).await {
+ Some(row) => {
+ rows.insert(key.clone(), row);
+ }
+ None => missing.push(key.clone()),
+ }
+ }
+ if !missing.is_empty() {
+ for (key, row) in self.inner.batch_get(&missing).await? {
+ self.hold(&key, &row).await;
+ rows.insert(key, row);
+ }
+ }
+ Ok(rows)
+ }
+
+ async fn put_if_absent(&self, key: &TableKey, row: &Row) -> Result<PutOutcome, TableError> {
+ let outcome = self.inner.put_if_absent(key, row).await?;
+ let stored = match &outcome {
+ PutOutcome::Stored => row,
+ PutOutcome::Existing(winner) => winner,
+ };
+ self.hold(key, stored).await;
+ Ok(outcome)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::atomic::Ordering;
+
+ use super::*;
+ use crate::keys::memory::MemoryTable;
+ use crate::keys::rows::Unusable;
+
+ fn cached() -> CachedKeyTable<MemoryTable> {
+ CachedKeyTable::new(
+ MemoryTable::default(),
+ 100,
+ Duration::from_secs(300),
+ Duration::from_secs(3600),
+ )
+ }
+
+ fn team_month_row(wrapped: u8) -> Row {
+ Row::TeamMonth {
+ team_id: 7,
+ key_month: "2026-10".to_owned(),
+ wrapped_key: vec![wrapped],
+ }
+ }
+
+ #[tokio::test]
+ async fn conditional_put_gives_up_at_its_deadline_when_dynamodb_stalls() {
+ // Accepts connections and never answers them.
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}", listener.local_addr().unwrap());
+ tokio::spawn(async move {
+ let mut held = Vec::new();
+ while let Ok((socket, _)) = listener.accept().await {
+ held.push(socket);
+ }
+ });
+ let config = aws_sdk_dynamodb::Config::builder()
+ .behavior_version(aws_sdk_dynamodb::config::BehaviorVersion::latest())
+ .region(aws_sdk_dynamodb::config::Region::new("us-east-1"))
+ .credentials_provider(aws_sdk_dynamodb::config::Credentials::new(
+ "test", "test", None, None, "test",
+ ))
+ .endpoint_url(endpoint)
+ .build();
+ let mut table = AwsKeyTable::new(
+ aws_sdk_dynamodb::Client::from_conf(config),
+ "keys".to_owned(),
+ );
+ table.retry_budget = Duration::from_millis(200);
+
+ let started = Instant::now();
+ let result = table
+ .put_if_absent(&TableKey::team_month(7, "2026-10"), &team_month_row(1))
+ .await;
+ assert!(matches!(result, Err(TableError::WriteBudget)), "{result:?}");
+ assert!(started.elapsed() < Duration::from_secs(5));
+ }
+
+ #[tokio::test]
+ async fn sees_a_row_another_writer_stores_after_a_miss_or_an_unusable_row() {
+ let unusable = Row::Unusable(Unusable::Malformed("wrapped_key"));
+ for earlier in [None, Some(unusable)] {
+ let table = cached();
+ let key = TableKey::team_month(7, "2026-10");
+ if let Some(row) = &earlier {
+ table.inner.store(key.clone(), row.clone());
+ }
+ let first = table.batch_get(std::slice::from_ref(&key)).await.unwrap();
+ assert_eq!(first.get(&key), earlier.as_ref());
+
+ table.inner.store(key.clone(), team_month_row(1));
+ assert_eq!(
+ table
+ .batch_get(std::slice::from_ref(&key))
+ .await
+ .unwrap()
+ .get(&key),
+ Some(&team_month_row(1))
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn serves_stored_rows_and_tombstones_from_the_cache() {
+ let table = cached();
+ let team_month = TableKey::team_month(7, "2026-10");
+ let session = TableKey::session(7, "019a2f3e-8c00-7000-8000-000000000007");
+ table.inner.store(session.clone(), Row::Deleted);
+ table.inner.store(team_month.clone(), team_month_row(1));
+ let keys = [team_month.clone(), session.clone()];
+
+ table.batch_get(&keys).await.unwrap();
+ let rows = table.batch_get(&keys).await.unwrap();
+ assert_eq!(table.inner.reads.load(Ordering::SeqCst), 1);
+ assert_eq!(rows.get(&session), Some(&Row::Deleted));
+
+ // A lost race returns the winner, and the cache holds the winner rather than the row this writer offered.
+ let outcome = table
+ .put_if_absent(&team_month, &team_month_row(2))
+ .await
+ .unwrap();
+ assert_eq!(outcome, PutOutcome::Existing(team_month_row(1)));
+ assert_eq!(
+ table
+ .batch_get(std::slice::from_ref(&team_month))
+ .await
+ .unwrap()[&team_month],
+ team_month_row(1)
+ );
+ }
+}