:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-core-v4): resolve and store the keys for a flush (#117957)
Nicholas Waltz2e5aa09bc6fe
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 37a504044347..efa466a76984 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11024,6 +11024,7 @@ dependencies = ["aws-sdk-kms","base64 0.22.1","chrono",+ "futures","governor","hkdf","hmac 0.12.1",diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.tomlindex d4023663ba26..331530166ca7 100644--- a/rust/ai-research/replay-core-v4/Cargo.toml+++ b/rust/ai-research/replay-core-v4/Cargo.toml@@ -11,7 +11,7 @@ workspace = true[features]default = ["store"]# The key table and KMS clients. A reader that only opens frames turns default features off and skips the AWS SDK.-store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:governor", "dep:moka", "dep:rand", "dep:tokio"]+store = ["dep:async-trait", "dep:aws-sdk-dynamodb", "dep:aws-sdk-kms", "dep:chrono", "dep:futures", "dep:governor", "dep:moka", "dep:rand", "dep:tokio"][dependencies]aes-gcm = "0.10"@@ -20,6 +20,7 @@ aws-sdk-dynamodb = { workspace = true, optional = true }aws-sdk-kms = { workspace = true, optional = true }base64 = { workspace = true }chrono = { workspace = true, optional = true }+futures = { workspace = true, optional = true }governor = { workspace = true, optional = true }hkdf = "0.12"hmac = { workspace = true }diff --git a/rust/ai-research/replay-core-v4/src/keys/batch.rs b/rust/ai-research/replay-core-v4/src/keys/batch.rsnew file mode 100644index 000000000000..8050b48c7f02--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/keys/batch.rs@@ -0,0 +1,660 @@+//! Resolves the keys for one recorder flush, storing the new ones before anything is sealed with them.++use std::collections::{BTreeSet, HashMap};+use std::fmt;++use futures::stream::{self, StreamExt, TryStreamExt};+use rand::RngCore;++use super::kms::{CachedKms, KeyWrapper, KmsError};+use super::rows::{key_month, Row, TableKey};+use super::table::{KeyTable, PutOutcome, TableError};+use crate::seal::{open_session_key, seal_session_key, SealedKey, WrappingContext};++/// How many conditional puts or team month keys one flush resolves at once, as in v3. `AwsKms` bounds the KMS calls.+const PUT_CONCURRENCY: usize = 32;++/// A failure that may pass on a retry, so the flush fails and runs again. A failure that one row causes drops only+/// that row's sessions instead.+#[derive(Debug, thiserror::Error)]+pub enum KeyError {+ #[error(transparent)]+ Table(#[from] TableError),+ #[error(transparent)]+ Kms(#[from] KmsError),+}++/// A session in a flush.+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]+pub struct SessionId {+ pub team_id: i64,+ pub session_id: String,+}++/// Why a flush leaves a session out.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DropCause {+ /// A deletion tombstoned the session or its team.+ Deleted,+ /// The session's key row names a team month key that is gone, so nothing sealed for it can be opened.+ KeyLost,+ /// The session's key row, or the team month key it is sealed under, is stored but gives no key: the seal does not+ /// open, KMS refuses the wrapped key, or the row is malformed or expired. The row stays, so no new key replaces it.+ KeyUnusable,+}++impl DropCause {+ /// The metric label.+ pub fn as_str(self) -> &'static str {+ match self {+ DropCause::Deleted => "deleted",+ DropCause::KeyLost => "key_lost",+ DropCause::KeyUnusable => "key_unusable",+ }+ }+}++#[derive(Clone, PartialEq, Eq)]+pub struct SessionKey {+ pub key: [u8; 32],+ /// The UTC month the session key was first stored in. Its row expires five months after that month.+ pub key_month: String,+}++impl fmt::Debug for SessionKey {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("SessionKey")+ .field("key", &"<redacted>")+ .field("key_month", &self.key_month)+ .finish()+ }+}++/// The keys a flush seals with.+#[derive(Default)]+pub struct FlushKeys {+ pub sessions: HashMap<SessionId, SessionKey>,+ /// Team month keys of the write day's month, for the flush's images.+ pub team_months: HashMap<i64, [u8; 32]>,+ pub dropped: Vec<(SessionId, DropCause)>,+}++impl fmt::Debug for FlushKeys {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("FlushKeys")+ .field("sessions", &self.sessions)+ .field("team_months", &self.team_months.keys())+ .field("dropped", &self.dropped)+ .finish()+ }+}++enum TeamMonth {+ Key([u8; 32]),+ Deleted,+ Unavailable(DropCause),+}++type TeamMonths = HashMap<(i64, String), TeamMonth>;++pub struct KeyStore<T, W> {+ table: T,+ kms: CachedKms<W>,+}++impl<T: KeyTable, W: KeyWrapper> KeyStore<T, W> {+ pub fn new(table: T, kms: CachedKms<W>) -> Self {+ Self { table, kms }+ }++ /// Resolves a key for every session, storing the new ones. Call it before sealing anything for the flush: a key+ /// that is not stored yet cannot open what it sealed if this process dies. The flush writes on `write_day`, and+ /// new keys belong to its month.+ pub async fn keys_for_flush(+ &self,+ sessions: &[SessionId],+ write_day: &str,+ ) -> Result<FlushKeys, KeyError> {+ let write_month = key_month(write_day);+ let teams: BTreeSet<i64> = sessions.iter().map(|s| s.team_id).collect();+ let mut keys: Vec<TableKey> = teams+ .iter()+ .map(|&team| TableKey::team_month(team, write_month))+ .collect();+ keys.extend(+ sessions+ .iter()+ .map(|s| TableKey::session(s.team_id, &s.session_id)),+ );+ let rows = self.table.batch_get(&keys).await?;++ // The futures are built before they enter the stream, which keeps the flush future `Send`.+ let write_months: Vec<_> = teams+ .iter()+ .map(|&team| {+ let row = rows.get(&TableKey::team_month(team, write_month));+ async move {+ let team_month = self.write_month_key(team, write_month, row).await?;+ Ok::<_, KeyError>(((team, write_month.to_owned()), team_month))+ }+ })+ .collect();+ let mut team_months: TeamMonths = stream::iter(write_months)+ .buffer_unordered(PUT_CONCURRENCY)+ .try_collect()+ .await?;++ // A session stored in an earlier month is sealed under that month's team key, which the first read did not ask+ // for.+ let earlier: BTreeSet<(i64, String)> = rows+ .values()+ .filter_map(|row| match row {+ Row::Session {+ team_id, key_month, ..+ } => Some((*team_id, key_month.clone())),+ _ => None,+ })+ .collect();+ self.resolve_team_months(earlier, &mut team_months).await?;++ let mut flush = FlushKeys::default();+ let mut new_sessions = Vec::new();+ for session in sessions {+ match rows.get(&TableKey::session(session.team_id, &session.session_id)) {+ Some(Row::Session {+ key_month, sealed, ..+ }) => match open_stored(session, key_month, sealed, &team_months) {+ Ok(key) => {+ flush.sessions.insert(session.clone(), key);+ }+ Err(cause) => flush.dropped.push((session.clone(), cause)),+ },+ Some(Row::Deleted) => flush.dropped.push((session.clone(), DropCause::Deleted)),+ Some(_) => flush+ .dropped+ .push((session.clone(), DropCause::KeyUnusable)),+ None => match &team_months[&(session.team_id, write_month.to_owned())] {+ TeamMonth::Key(team_key) => new_sessions.push((session.clone(), *team_key)),+ TeamMonth::Deleted => flush.dropped.push((session.clone(), DropCause::Deleted)),+ TeamMonth::Unavailable(cause) => flush.dropped.push((session.clone(), *cause)),+ },+ }+ }++ let stored: Vec<_> = stream::iter(new_sessions)+ .map(|(session, team_key)| self.store_new_session(session, team_key, write_month))+ .buffer_unordered(PUT_CONCURRENCY)+ .try_collect()+ .await?;+ let winners: BTreeSet<(i64, String)> = stored+ .iter()+ .filter_map(|(session, outcome)| match outcome {+ NewSession::Winner(key_month, _) => Some((session.team_id, key_month.clone())),+ _ => None,+ })+ .collect();+ self.resolve_team_months(winners, &mut team_months).await?;+ for (session, outcome) in stored {+ match outcome {+ NewSession::Key(key) => {+ flush.sessions.insert(session, key);+ }+ NewSession::Winner(key_month, sealed) => {+ match open_stored(&session, &key_month, &sealed, &team_months) {+ Ok(key) => {+ flush.sessions.insert(session, key);+ }+ Err(cause) => flush.dropped.push((session, cause)),+ }+ }+ NewSession::Dropped(cause) => flush.dropped.push((session, cause)),+ }+ }++ for ((team, month), team_month) in team_months {+ if let (TeamMonth::Key(key), true) = (team_month, month == write_month) {+ flush.team_months.insert(team, key);+ }+ }+ Ok(flush)+ }++ /// The team key of the write month: the stored one, or a new one stored now.+ async fn write_month_key(+ &self,+ team: i64,+ write_month: &str,+ row: Option<&Row>,+ ) -> Result<TeamMonth, KeyError> {+ if let Some(row) = row {+ return self.stored_team_month(team, write_month, Some(row)).await;+ }+ let context = WrappingContext {+ team_id: team,+ key_month: write_month.to_owned(),+ session_id: None,+ };+ let generated = self.kms.generate(&context).await?;+ let candidate = Row::TeamMonth {+ team_id: team,+ key_month: write_month.to_owned(),+ wrapped_key: generated.wrapped.clone(),+ };+ match self+ .table+ .put_if_absent(&TableKey::team_month(team, write_month), &candidate)+ .await?+ {+ PutOutcome::Stored => {+ self.kms.remember(&context, &generated).await;+ Ok(TeamMonth::Key(generated.plaintext))+ }+ PutOutcome::Existing(winner) => {+ self.stored_team_month(team, write_month, Some(&winner))+ .await+ }+ }+ }++ async fn resolve_team_months(+ &self,+ slots: BTreeSet<(i64, String)>,+ team_months: &mut TeamMonths,+ ) -> Result<(), KeyError> {+ let slots: Vec<_> = slots+ .into_iter()+ .filter(|slot| !team_months.contains_key(slot))+ .collect();+ if slots.is_empty() {+ return Ok(());+ }+ let keys: Vec<_> = slots+ .iter()+ .map(|(team, month)| TableKey::team_month(*team, month))+ .collect();+ let rows = self.table.batch_get(&keys).await?;+ let unwraps: Vec<_> = slots+ .into_iter()+ .map(|(team, month)| {+ let row = rows.get(&TableKey::team_month(team, &month));+ async move {+ let team_month = self.stored_team_month(team, &month, row).await?;+ Ok::<_, KeyError>(((team, month), team_month))+ }+ })+ .collect();+ let resolved: Vec<_> = stream::iter(unwraps)+ .buffer_unordered(PUT_CONCURRENCY)+ .try_collect()+ .await?;+ team_months.extend(resolved);+ Ok(())+ }++ /// KMS refusing the blob drops that team month's sessions, and any other KMS failure fails the flush.+ async fn stored_team_month(+ &self,+ team: i64,+ key_month: &str,+ row: Option<&Row>,+ ) -> Result<TeamMonth, KeyError> {+ let wrapped = match row {+ Some(Row::TeamMonth { wrapped_key, .. }) => wrapped_key,+ Some(Row::Deleted) => return Ok(TeamMonth::Deleted),+ Some(_) => return Ok(TeamMonth::Unavailable(DropCause::KeyUnusable)),+ None => return Ok(TeamMonth::Unavailable(DropCause::KeyLost)),+ };+ let context = WrappingContext {+ team_id: team,+ key_month: key_month.to_owned(),+ session_id: None,+ };+ match self.kms.unwrap(&context, wrapped).await {+ Ok(key) => Ok(TeamMonth::Key(key)),+ Err(KmsError::Rejected(_) | KmsError::InvalidKey) => {+ Ok(TeamMonth::Unavailable(DropCause::KeyUnusable))+ }+ Err(error) => Err(error.into()),+ }+ }++ async fn store_new_session(+ &self,+ session: SessionId,+ team_key: [u8; 32],+ write_month: &str,+ ) -> Result<(SessionId, NewSession), KeyError> {+ let mut key = [0u8; 32];+ rand::thread_rng().fill_bytes(&mut key);+ let context = WrappingContext {+ team_id: session.team_id,+ key_month: write_month.to_owned(),+ session_id: Some(session.session_id.clone()),+ };+ let row = Row::Session {+ team_id: session.team_id,+ key_month: write_month.to_owned(),+ sealed: seal_session_key(&team_key, &context, &key),+ };+ let outcome = match self+ .table+ .put_if_absent(+ &TableKey::session(session.team_id, &session.session_id),+ &row,+ )+ .await?+ {+ PutOutcome::Stored => NewSession::Key(SessionKey {+ key,+ key_month: write_month.to_owned(),+ }),+ PutOutcome::Existing(Row::Session {+ key_month, sealed, ..+ }) => NewSession::Winner(key_month, sealed),+ PutOutcome::Existing(Row::Deleted) => NewSession::Dropped(DropCause::Deleted),+ PutOutcome::Existing(_) => NewSession::Dropped(DropCause::KeyUnusable),+ };+ Ok((session, outcome))+ }+}++fn open_stored(+ session: &SessionId,+ key_month: &str,+ sealed: &SealedKey,+ team_months: &TeamMonths,+) -> Result<SessionKey, DropCause> {+ let team_key = match team_months.get(&(session.team_id, key_month.to_owned())) {+ Some(TeamMonth::Key(key)) => key,+ Some(TeamMonth::Deleted) => return Err(DropCause::Deleted),+ Some(TeamMonth::Unavailable(cause)) => return Err(*cause),+ None => return Err(DropCause::KeyLost),+ };+ let context = WrappingContext {+ team_id: session.team_id,+ key_month: key_month.to_owned(),+ session_id: Some(session.session_id.clone()),+ };+ // A seal opens under one team month key only, so a failure means the row and that key disagree.+ let key = open_session_key(team_key, &context, sealed).map_err(|_| DropCause::KeyUnusable)?;+ Ok(SessionKey {+ key,+ key_month: key_month.to_owned(),+ })+}++enum NewSession {+ Key(SessionKey),+ /// Another writer stored this session's key first, possibly in another month. The flush adopts it.+ Winner(String, SealedKey),+ /// The put met a tombstone or an unusable row, which it never writes over.+ Dropped(DropCause),+}++#[cfg(test)]+mod tests {+ use std::time::Duration;++ use async_trait::async_trait;++ use super::*;+ use crate::keys::kms::DataKey;+ use crate::keys::memory::MemoryTable;+ use crate::keys::rows::Unusable;++ /// Wraps a key by prefixing its context, so unwrapping under another context fails as KMS would.+ #[derive(Default)]+ struct FakeKms {+ /// Every unwrap times out, as in a KMS outage.+ outage: bool,+ }++ fn label(context: &WrappingContext) -> Vec<u8> {+ format!("{:?}", context.entries()).into_bytes()+ }++ #[async_trait]+ impl KeyWrapper for FakeKms {+ async fn generate(&self, context: &WrappingContext) -> Result<DataKey, KmsError> {+ let plaintext: [u8; 32] = rand::random();+ Ok(DataKey {+ plaintext,+ wrapped: [label(context), plaintext.to_vec()].concat(),+ })+ }++ async fn unwrap(+ &self,+ context: &WrappingContext,+ wrapped: &[u8],+ ) -> Result<[u8; 32], KmsError> {+ if self.outage {+ return Err(KmsError::Timeout);+ }+ let label = label(context);+ if !wrapped.starts_with(&label) {+ return Err(KmsError::Rejected("encryption context mismatch".to_owned()));+ }+ wrapped[label.len()..]+ .try_into()+ .map_err(|_| KmsError::InvalidKey)+ }+ }++ fn store(table: MemoryTable) -> KeyStore<MemoryTable, FakeKms> {+ store_with(table, FakeKms::default())+ }++ fn store_with(table: MemoryTable, kms: FakeKms) -> KeyStore<MemoryTable, FakeKms> {+ KeyStore::new(table, CachedKms::new(kms, 100, Duration::from_secs(60)))+ }++ fn session(team_id: i64, n: u32) -> SessionId {+ SessionId {+ team_id,+ session_id: format!("019a2f3e-8c00-7000-8000-{n:012}"),+ }+ }++ /// Another pod: a fresh process that reads what the first one stored.+ fn rows_of(store: &KeyStore<MemoryTable, FakeKms>) -> MemoryTable {+ MemoryTable {+ rows: std::sync::Mutex::new(store.table.rows.lock().unwrap().clone()),+ ..Default::default()+ }+ }++ #[tokio::test]+ async fn a_new_session_key_is_stored_and_found_again_in_a_later_month() {+ let first = store(MemoryTable::default());+ let a = session(7, 1);+ let october = first+ .keys_for_flush(std::slice::from_ref(&a), "2026-10-15")+ .await+ .unwrap();+ let key = october.sessions[&a].clone();+ assert_eq!(key.key_month, "2026-10");+ let printed = format!("{october:?}");+ for secret in [key.key, october.team_months[&7]] {+ assert!(!printed.contains(&format!("{secret:?}")[1..20]));+ }++ // The team key holds for the whole month, so image references stay the same across its days.+ let second = store(rows_of(&first));+ let late_october = second+ .keys_for_flush(std::slice::from_ref(&a), "2026-10-31")+ .await+ .unwrap();+ assert_eq!(late_october.team_months[&7], october.team_months[&7]);++ // The session keeps its first key in a new month, while the team gets that month's key for its images.+ let november = second+ .keys_for_flush(std::slice::from_ref(&a), "2026-11-02")+ .await+ .unwrap();+ assert_eq!(november.sessions[&a], key);+ assert_ne!(november.team_months[&7], october.team_months[&7]);+ }++ #[tokio::test]+ async fn adopts_the_key_another_writer_stored_first() {+ let a = session(7, 1);+ let winner = store(MemoryTable::default());+ let winning = winner+ .keys_for_flush(std::slice::from_ref(&a), "2026-10-15")+ .await+ .unwrap();++ // The loser read before the winner wrote, so it reaches the conditional put with a candidate of its own.+ let loser = store(rows_of(&winner));+ let (_, outcome) = loser+ .store_new_session(a.clone(), [9; 32], "2026-10")+ .await+ .unwrap();+ let NewSession::Winner(key_month, sealed) = outcome else {+ panic!("the put must lose")+ };+ let mut team_months = HashMap::new();+ loser+ .resolve_team_months(BTreeSet::from([(7, key_month.clone())]), &mut team_months)+ .await+ .unwrap();+ let adopted = open_stored(&a, &key_month, &sealed, &team_months).unwrap();+ assert_eq!(adopted, winning.sessions[&a]);+ }++ #[tokio::test]+ async fn drops_sessions_a_deletion_tombstoned() {+ let table = MemoryTable::default();+ let (deleted_session, deleted_team, kept) = (session(7, 1), session(8, 2), session(7, 3));+ table.store(+ TableKey::session(7, &deleted_session.session_id),+ Row::Deleted,+ );+ table.store(TableKey::team_month(8, "2026-10"), Row::Deleted);+ let store = store(table);++ let flush = store+ .keys_for_flush(+ &[deleted_session.clone(), deleted_team.clone(), kept.clone()],+ "2026-10-15",+ )+ .await+ .unwrap();+ assert_eq!(flush.sessions.keys().collect::<Vec<_>>(), vec![&kept]);+ let mut dropped = flush.dropped.clone();+ dropped.sort_by(|a, b| a.0.cmp(&b.0));+ assert_eq!(+ dropped,+ vec![+ (deleted_session, DropCause::Deleted),+ (deleted_team, DropCause::Deleted)+ ]+ );+ assert!(!flush.team_months.contains_key(&8));+ }++ #[tokio::test]+ async fn a_tombstoned_team_month_drops_the_sessions_sealed_under_it_in_later_months() {+ let first = store(MemoryTable::default());+ let (a, b) = (session(7, 1), session(7, 2));+ first+ .keys_for_flush(std::slice::from_ref(&a), "2026-10-15")+ .await+ .unwrap();+ let rows = rows_of(&first);+ let october = TableKey::team_month(7, "2026-10");+ rows.store(october.clone(), Row::Deleted);++ let second = store(rows);+ let flush = second+ .keys_for_flush(&[a.clone(), b.clone()], "2026-11-02")+ .await+ .unwrap();+ assert_eq!(flush.dropped, vec![(a, DropCause::Deleted)]);+ assert_eq!(flush.sessions.keys().collect::<Vec<_>>(), vec![&b]);+ assert_eq!(second.table.rows.lock().unwrap()[&october], Row::Deleted);+ }++ #[tokio::test]+ async fn a_put_that_meets_a_tombstone_drops_its_data_and_keeps_the_tombstone() {+ // A deletion lands between the flush's read and its conditional put.+ let a = session(7, 1);+ let (team_month, session_row) = (+ TableKey::team_month(8, "2026-10"),+ TableKey::session(7, &a.session_id),+ );+ let table = MemoryTable::default();+ table.store(team_month.clone(), Row::Deleted);+ table.store(session_row.clone(), Row::Deleted);+ let store = store(table);++ let (_, outcome) = store+ .store_new_session(a, [9; 32], "2026-10")+ .await+ .unwrap();+ assert!(matches!(outcome, NewSession::Dropped(DropCause::Deleted)));+ let team = store.write_month_key(8, "2026-10", None).await.unwrap();+ assert!(matches!(team, TeamMonth::Deleted));+ let rows = store.table.rows.lock().unwrap();+ assert_eq!(rows[&team_month], Row::Deleted);+ assert_eq!(rows[&session_row], Row::Deleted);+ }++ #[tokio::test]+ async fn an_unusable_key_row_drops_only_its_sessions_and_an_outage_fails_the_flush() {+ let (a, b, c) = (session(7, 1), session(8, 2), session(9, 3));+ let first = store(MemoryTable::default());+ first+ .keys_for_flush(&[a.clone(), b.clone(), c.clone()], "2026-10-15")+ .await+ .unwrap();+ let rows = rows_of(&first);+ // A seal that does not open, a wrapped key KMS refuses, and an expired row.+ rows.store(+ TableKey::session(7, &a.session_id),+ Row::Session {+ team_id: 7,+ key_month: "2026-10".to_owned(),+ sealed: SealedKey {+ sealed: vec![1; 48],+ nonce: [2; 12],+ },+ },+ );+ rows.store(+ TableKey::team_month(8, "2026-10"),+ Row::TeamMonth {+ team_id: 8,+ key_month: "2026-10".to_owned(),+ wrapped_key: vec![3; 40],+ },+ );+ let expired = Row::Unusable(Unusable::Expired);+ rows.store(TableKey::session(9, &c.session_id), expired.clone());++ let (d, e) = (session(7, 4), session(8, 5));+ let second = store(rows);+ let all = [a.clone(), b.clone(), c.clone(), d.clone(), e.clone()];+ let flush = second.keys_for_flush(&all, "2026-11-02").await.unwrap();+ let mut dropped = flush.dropped.clone();+ dropped.sort_by(|x, y| x.0.cmp(&y.0));+ assert_eq!(+ dropped,+ [a, b, c.clone()].map(|s| (s, DropCause::KeyUnusable))+ );+ let mut kept: Vec<_> = flush.sessions.keys().cloned().collect();+ kept.sort();+ assert_eq!(kept, vec![d, e]);+ assert_eq!(+ second.table.rows.lock().unwrap()[&TableKey::session(9, &c.session_id)],+ expired+ );++ let during_outage = store_with(rows_of(&second), FakeKms { outage: true });+ assert!(matches!(+ during_outage.keys_for_flush(&all, "2026-11-02").await,+ Err(KeyError::Kms(KmsError::Timeout))+ ));+ }+}diff --git a/rust/ai-research/replay-core-v4/src/keys/mod.rs b/rust/ai-research/replay-core-v4/src/keys/mod.rsindex e88bf5b304d5..0b8e4fe37ef6 100644--- a/rust/ai-research/replay-core-v4/src/keys/mod.rs+++ b/rust/ai-research/replay-core-v4/src/keys/mod.rs@@ -1,5 +1,6 @@//! The v4 key table in prod-us: one KMS-wrapped key per team per UTC month, and one session key sealed under it.+pub mod batch;pub mod kms;#[cfg(test)]pub(crate) mod memory;