:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cymbal): add optional Redis shadow writes (#130117)
Hugues Pouillot16ee8aa7e144
diff --git a/rust/common/redis/src/client.rs b/rust/common/redis/src/client.rsindex 2655d8ac25e4..1b6c03113809 100644--- a/rust/common/redis/src/client.rs+++ b/rust/common/redis/src/client.rs@@ -54,9 +54,18 @@ impl RedisClient {/// attempt just waits for the next caller -- the client keeps failing/// open in the meantime, exactly as it would without healing.pub async fn heal_connection(&self) {+ self.reconnect_inner(false).await;+ }++ /// Replace the connection after a shadow timeout, even during the normal heal cooldown.+ pub async fn reconnect(&self) -> bool {+ self.reconnect_inner(true).await+ }++ async fn reconnect_inner(&self, force: bool) -> bool {let mut last_attempt = self.heal_state.lock().await;- if last_attempt.elapsed() < HEAL_COOLDOWN {- return;+ if !force && last_attempt.elapsed() < HEAL_COOLDOWN {+ return false;}*last_attempt = Instant::now();@@ -75,10 +84,12 @@ impl RedisClient {{Ok(connection) => {self.connection.store(Arc::new(connection));- info!("Redis connection healed after unrecoverable error");+ info!("Redis connection replaced");+ true}Err(e) => {- warn!(error = %e, "Redis heal attempt failed; will retry after cooldown");+ warn!(error = %e, "Redis reconnect attempt failed");+ false}}}@@ -336,10 +347,19 @@ pub trait ScriptRunner: Send + Sync {/// Rebuild the underlying connection after a connection-class failure; see/// [`Client::heal`]. The default no-op keeps test fakes trivial.async fn heal(&self) {}++ /// Return whether a fresh connection replaced the old one. Test runners do not reconnect.+ async fn reconnect(&self) -> bool {+ false+ }}#[async_trait]impl ScriptRunner for RedisClient {+ async fn reconnect(&self) -> bool {+ RedisClient::reconnect(self).await+ }+async fn eval_int_vec(&self,script: &str,@@ -360,6 +380,10 @@ impl Client for RedisClient {self.heal_connection().await;}+ async fn reconnect(&self) -> bool {+ RedisClient::reconnect(self).await+ }+async fn zrangebyscore(&self,k: String,diff --git a/rust/common/redis/src/lib.rs b/rust/common/redis/src/lib.rsindex 7c5912061f2b..dd62c6c09701 100644--- a/rust/common/redis/src/lib.rs+++ b/rust/common/redis/src/lib.rs@@ -325,6 +325,11 @@ pub trait Client: Send + Sync {/// default no-op. Must be cheap to call repeatedly -- implementations own/// their own cooldown.async fn heal(&self) {}++ /// Return whether a fresh connection replaced the old one. Test clients do not reconnect.+ async fn reconnect(&self) -> bool {+ false+ }}/// Extension trait providing the `.pipeline()` builder method.diff --git a/rust/cymbal/Cargo.toml b/rust/cymbal/Cargo.tomlindex e18ee1de2f5a..62630f87441f 100644--- a/rust/cymbal/Cargo.toml+++ b/rust/cymbal/Cargo.toml@@ -21,6 +21,7 @@ common-database = { path = "../common/database" }common-types = { path = "../common/types" }common-dns = { path = "../common/dns" }common-redis = { path = "../common/redis" }+redis = { workspace = true }common-posthog = { path = "../common/posthog" }cymbal-proto = { path = "../cymbal-proto" }posthog-symbol-data = { path = "../common/symbol_data" }@@ -62,8 +63,6 @@ common-temporal = { path = "../common/temporal" }[dev-dependencies]common-dns = { path = "../common/dns", features = ["test-util"] }common-compression = { path = "../common/compression" }-# For constructing IO-flavored RedisErrors in redis_heal classifier tests-redis = { workspace = true }httpmock = { workspace = true }mockall = { workspace = true }insta = { version = "1.46.0", features = ["json", "redactions"] }diff --git a/rust/cymbal/README.md b/rust/cymbal/README.mdindex bcf3046b3d75..b6bd51a4b4b1 100644--- a/rust/cymbal/README.md+++ b/rust/cymbal/README.md@@ -50,10 +50,15 @@ are named for what is actually capped. Only issue-created is charged.| Variable | Default | Effect || --- | --- | --- || `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_REDIS_URL` | none | Required. The service does not start without it. |+| `ERROR_TRACKING_NOTIFICATIONS_SHADOW_REDIS_URL` | unset | Optional destination for best-effort shadow charges and refunds. Must be a different Redis instance from the primary. || `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_PER_HOUR` | `1000` | Bucket size, and the tokens a team earns back per hour. Zero or less disables the limit. || `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_KEY_PREFIX` | `@posthog/error-tracking-notifications-rate-limiter` | Key namespace. It must differ from the event limiter's prefix. || `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_BUCKET_TTL_SECONDS` | `3600` | Idle buckets expire and free the memory. A value below 3600 is raised to 3600, because a bucket takes an hour to refill and a shorter TTL would loosen the limit. |+Processing mode can set `ERROR_TRACKING_PROCESSING_SHADOW_REDIS_URL` to copy issue and team spike counters, unique-issue sets, cooldown locks and releases, lifecycle event properties, and event rate-limiter charges to a separate Redis instance. Both shadow URLs are optional and unset by default. The primary Redis still supplies every decision and read. A shadow connection failure at startup disables shadow writes on that pod; runtime errors and the 200 ms shadow deadline are counted in `cymbal_shadow_redis_operations_total` by `operation` and `outcome`, without changing the primary result. Failure warnings are limited to one per 30 seconds per pod. Do not use a DNS alias of any primary Redis URL for the shadow: non-idempotent counters and token buckets would be charged twice. The URL guard detects equivalent host, port and database settings, but cannot prove that different hostnames resolve to different instances.++A shadow write is submitted after a successful primary write, with the same keys, bytes, expiry and Lua arguments. It runs in a bounded background task so it does not delay ingestion; at most 128 shadow writes run per pod, and excess submissions increment the `dropped` outcome rather than queue without a bound. After a timeout, the shadow circuit drops new writes until it can replace the connection; retries occur at most every five seconds while traffic continues. This stops timed-out requests from accumulating in the old connection. The `circuit_open` and `reconnect` outcomes record this state. Background writes can reorder or be lost when a pod exits, and an immediate shadow read is not read-after-write consistent. No historical keys are copied. A 48-hour warm-up covers event properties at their default TTL and the shorter-lived spike buckets only if all pods have a healthy shadow connection for the entire window and no writes fail, time out or drop. It does not guarantee parity for a higher configured TTL, independently advancing token buckets and cooldown locks, or writes that complete in a different order. Cooldown locks are mirrored only when the primary grants them; existing primary locks are not copied. A shadow that fails to connect at startup remains disabled on that pod until restart. Validate key and TTL coverage before switching any reader; this option never switches readers.+The limit covers every team. To size it before it cuts anything, set `PER_HOUR`far above real traffic, watch `cymbal_issue_created_rate_limit_outcomes`, thenlower it. Setting `PER_HOUR` to zero or less switches the limit off.diff --git a/rust/cymbal/src/modes/mod.rs b/rust/cymbal/src/modes/mod.rsindex dda95d984e04..6f4a5f36affd 100644--- a/rust/cymbal/src/modes/mod.rs+++ b/rust/cymbal/src/modes/mod.rs@@ -9,6 +9,7 @@ use std::str::FromStr;pub mod notifications;pub mod processing;pub mod resolution;+pub mod shadow_redis;/// Which server stack the cymbal binary runs. Parsed from `CYMBAL_MODE`.#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]diff --git a/rust/cymbal/src/modes/notifications/config.rs b/rust/cymbal/src/modes/notifications/config.rsindex 70ca2c4bb9ba..fec8ce6469ab 100644--- a/rust/cymbal/src/modes/notifications/config.rs+++ b/rust/cymbal/src/modes/notifications/config.rs@@ -65,6 +65,9 @@ pub struct NotificationsConfig {#[envconfig(from = "ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_REDIS_URL")]pub notifications_rate_limit_redis_url: String,+ #[envconfig(from = "ERROR_TRACKING_NOTIFICATIONS_SHADOW_REDIS_URL")]+ pub shadow_redis_url: Option<String>,+/// Bucket size, and the tokens a team earns back per hour. Zero or less/// disables the limit.#[envconfig(diff --git a/rust/cymbal/src/modes/notifications/rate_limit.rs b/rust/cymbal/src/modes/notifications/rate_limit.rsindex 37bd27ca9a72..27c8509891e8 100644--- a/rust/cymbal/src/modes/notifications/rate_limit.rs+++ b/rust/cymbal/src/modes/notifications/rate_limit.rs@@ -17,7 +17,7 @@ use crate::core::metric_consts::{};use crate::modes::notifications::config::NotificationsConfig;use crate::modes::notifications::token_bucket::TokenBucket;-use crate::modes::processing::redis_heal::is_connection_error;+use crate::modes::{processing::redis_heal::is_connection_error, shadow_redis};pub struct IssueCreatedRateLimiter {/// `None` disables the limit: every notification is admitted.@@ -43,18 +43,33 @@ impl IssueCreatedRateLimiter {).await?;+ let shadow = shadow_redis::connect(+ config.shadow_redis_url.as_deref(),+ &[&config.notifications_rate_limit_redis_url],+ common_redis::RedisValueFormat::Utf8,+ config.redis_response_timeout_ms,+ config.redis_connection_timeout_ms,+ )+ .await;++ let shadow: Option<Arc<dyn common_redis::ScriptRunner>> =+ shadow.map(|client| -> Arc<dyn common_redis::ScriptRunner> { client });+info!(per_hour = config.notifications_rate_limit_per_hour,"Error-tracking issue-created rate limiter enabled for all teams",);Ok(Self {- bucket: Some(TokenBucket::per_hour(- Arc::new(client),- config.notifications_rate_limit_key_prefix.clone(),- config.notifications_rate_limit_per_hour as f64,- config.notifications_rate_limit_bucket_ttl_seconds,- )),+ bucket: Some(+ TokenBucket::per_hour(+ Arc::new(client),+ config.notifications_rate_limit_key_prefix.clone(),+ config.notifications_rate_limit_per_hour as f64,+ config.notifications_rate_limit_bucket_ttl_seconds,+ )+ .with_shadow(shadow),+ ),})}diff --git a/rust/cymbal/src/modes/notifications/token_bucket.rs b/rust/cymbal/src/modes/notifications/token_bucket.rsindex e6de6658c72e..b05581d0f66c 100644--- a/rust/cymbal/src/modes/notifications/token_bucket.rs+++ b/rust/cymbal/src/modes/notifications/token_bucket.rs@@ -11,7 +11,7 @@ use chrono::Utc;use common_redis::{CustomRedisError, ScriptRunner};use tracing::warn;-use crate::modes::processing::redis_heal::HealGate;+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};const SECONDS_PER_HOUR: f64 = 3600.0;@@ -74,6 +74,7 @@ return { 1 }pub struct TokenBucket {redis: Arc<dyn ScriptRunner>,+ shadow: Option<Arc<dyn ScriptRunner>>,key_prefix: String,max: f64,rate: f64,@@ -103,6 +104,7 @@ impl TokenBucket {Self {redis,+ shadow: None,key_prefix,max: per_hour,rate: per_hour / SECONDS_PER_HOUR,@@ -111,6 +113,11 @@ impl TokenBucket {}}+ pub fn with_shadow(mut self, shadow: Option<Arc<dyn ScriptRunner>>) -> Self {+ self.shadow = shadow;+ self+ }+/// Kick a background heal of the bucket's Redis connection, for a caller/// that saw a connection-class error. `MultiplexedConnection` never/// reconnects on its own, so without this one failover leaves the limiter@@ -128,10 +135,21 @@ impl TokenBucket {self.ttl_seconds.to_string(),Utc::now().timestamp().to_string(),];+ let shadow_args = self.shadow.as_ref().map(|_| args.clone());let res = self.redis.eval_int_vec(CHARGE_SCRIPT, vec![self.key(team_id)], args).await?;+ if let (Some(shadow), Some(args)) = (&self.shadow, shadow_args) {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ let key = self.key(team_id);+ shadow_redis::submit(+ "notifications_charge",+ async move { shadow.eval_int_vec(CHARGE_SCRIPT, vec![key], args).await },+ async move { healing.reconnect().await },+ );+ }Ok(res.first().copied().unwrap_or(0) > 0)}@@ -140,9 +158,20 @@ impl TokenBucket {/// one.pub async fn refund(&self, team_id: i32) -> Result<(), CustomRedisError> {let args = vec![self.max.to_string(), self.ttl_seconds.to_string()];+ let shadow_args = self.shadow.as_ref().map(|_| args.clone());self.redis.eval_int_vec(REFUND_SCRIPT, vec![self.key(team_id)], args).await?;+ if let (Some(shadow), Some(args)) = (&self.shadow, shadow_args) {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ let key = self.key(team_id);+ shadow_redis::submit(+ "notifications_refund",+ async move { shadow.eval_int_vec(REFUND_SCRIPT, vec![key], args).await },+ async move { healing.reconnect().await },+ );+ }Ok(())}@@ -163,6 +192,30 @@ mod tests {use crate::test_support::{start_redis, FakeRunner};use common_redis::RedisClient;+ #[tokio::test]+ async fn shadow_failure_does_not_change_charge_or_refund() {+ shadow_redis::with_test_circuit(async {+ let shadow = FakeRunner::failing();+ let primary = FakeRunner::returning(vec![1]);+ let bucket = TokenBucket::per_hour(primary.clone(), "test".into(), 1.0, 3600)+ .with_shadow(Some(shadow.clone()));+ assert!(bucket.charge(42).await.unwrap());+ bucket.refund(42).await.unwrap();+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while shadow.call_count() == 0 {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ let primary_calls = primary.requests();+ let shadow_calls = shadow.requests();+ assert_eq!(primary_calls.len(), 2);+ assert!(shadow_calls.iter().all(|call| primary_calls.contains(call)));+ })+ .await;+ }+fn bucket(client: Arc<RedisClient>, prefix: &str, per_hour: f64) -> TokenBucket {TokenBucket::per_hour(client, prefix.to_string(), per_hour, 3600)}diff --git a/rust/cymbal/src/modes/processing/app_context.rs b/rust/cymbal/src/modes/processing/app_context.rsindex faddb7a720ca..5ae62ae566fd 100644--- a/rust/cymbal/src/modes/processing/app_context.rs+++ b/rust/cymbal/src/modes/processing/app_context.rs@@ -9,7 +9,7 @@ use tokio::{sync::Semaphore, task::JoinHandle};use tracing::info;use uuid::Uuid;-use crate::modes::processing::redis_heal::HealGate;+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};use crate::{core::config::build_pg_pool,error::UnhandledError,@@ -44,6 +44,7 @@ pub struct AppContext {pub team_manager: TeamManager,pub issue_buckets_redis_client: Arc<dyn RedisClientTrait + Send + Sync>,+ pub shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>>,// Debounces heal-task spawns for the issue-buckets client (see redis_heal).pub issue_buckets_heal_gate: HealGate,// Error-tracking rate limiter. `None` when disabled (the default), in which@@ -178,7 +179,20 @@ impl AppContext {let (remote_resolution, remote_resolution_refresh_task) =build_remote_resolution(config).await?;- let rate_limiter = build_rate_limiter(config).await?;+ let shadow = shadow_redis::connect(+ config.shadow_redis_url.as_deref(),+ &[+ &config.issue_buckets_redis_url,+ &config.error_tracking_rate_limiter_redis_url,+ ],+ common_redis::RedisValueFormat::default(),+ config.redis_response_timeout_ms,+ config.redis_connection_timeout_ms,+ )+ .await;+ let rate_limiter = build_rate_limiter(config, shadow.clone()).await?;+ let shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>> =+ shadow.map(|client| -> Arc<dyn RedisClientTrait + Send + Sync> { client });let rate_limiter_enabled_team_ids =parse_team_id_allowlist(&config.error_tracking_rate_limiter_enabled_team_ids);let drop_code_variables_team_ids = Arc::new(parse_strict_team_id_list(@@ -195,6 +209,7 @@ impl AppContext {process_request_limiter,team_manager,issue_buckets_redis_client,+ shadow_redis_client,issue_buckets_heal_gate: HealGate::new(),rate_limiter,rate_limiter_enabled_team_ids,@@ -209,6 +224,7 @@ impl AppContext {async fn build_rate_limiter(config: &ProcessingConfig,+ shadow: Option<Arc<RedisClient>>,) -> Result<Option<Arc<RedisRateLimiter>>, UnhandledError> {if !config.error_tracking_rate_limiter_enabled {return Ok(None);@@ -235,11 +251,16 @@ async fn build_rate_limiter(info!("Error-tracking rate limiter enabled");- Ok(Some(Arc::new(RedisRateLimiter::new(- Arc::new(client),- config.error_tracking_rate_limiter_key_prefix.clone(),- config.error_tracking_rate_limiter_bucket_ttl_seconds,- ))))+ let shadow: Option<Arc<dyn common_redis::ScriptRunner>> =+ shadow.map(|client| -> Arc<dyn common_redis::ScriptRunner> { client });+ Ok(Some(Arc::new(+ RedisRateLimiter::new(+ Arc::new(client),+ config.error_tracking_rate_limiter_key_prefix.clone(),+ config.error_tracking_rate_limiter_bucket_ttl_seconds,+ )+ .with_shadow(shadow),+ )))}/// Parse a comma-separated team-id allowlist. `None` (empty input) means thediff --git a/rust/cymbal/src/modes/processing/config.rs b/rust/cymbal/src/modes/processing/config.rsindex 68b051e43398..04046c9b2b64 100644--- a/rust/cymbal/src/modes/processing/config.rs+++ b/rust/cymbal/src/modes/processing/config.rs@@ -148,6 +148,9 @@ pub struct ProcessingConfig {#[envconfig(from = "ISSUE_BUCKETS_REDIS_URL", default = "redis://localhost:6379/")]pub issue_buckets_redis_url: String,+ #[envconfig(from = "ERROR_TRACKING_PROCESSING_SHADOW_REDIS_URL")]+ pub shadow_redis_url: Option<String>,+#[envconfig(from = "ERROR_TRACKING_EVENT_PROPERTIES_TTL_SECONDS",default = "172800"diff --git a/rust/cymbal/src/modes/processing/issue_resolution.rs b/rust/cymbal/src/modes/processing/issue_resolution.rsindex be0fec793d90..abe8af2599f5 100644--- a/rust/cymbal/src/modes/processing/issue_resolution.rs+++ b/rust/cymbal/src/modes/processing/issue_resolution.rs@@ -500,8 +500,11 @@ pub async fn send_issue_created_notification(let fingerprint = processed_properties.fingerprint().to_string();store_error_tracking_event_properties(&*context.issue_buckets_redis_client,- context.config.event_properties_ttl_seconds,- context.config.event_properties_max_bytes,+ context.shadow_redis_client.as_ref(),+ EventPropertiesLimits {+ ttl_seconds: context.config.event_properties_ttl_seconds,+ max_bytes: context.config.event_properties_max_bytes,+ },issue.team_id,event_uuid,"issue_created",@@ -527,10 +530,15 @@ fn error_tracking_event_properties_key(team_id: i32, event_uuid: Uuid) -> Stringformat!("{ERROR_TRACKING_EVENT_PROPERTIES_KEY_PREFIX}:{team_id}:{event_uuid}")}-async fn store_error_tracking_event_properties(- redis: &(dyn common_redis::Client + Send + Sync),+struct EventPropertiesLimits {ttl_seconds: u64,max_bytes: usize,+}++async fn store_error_tracking_event_properties(+ redis: &(dyn common_redis::Client + Send + Sync),+ shadow: Option<&std::sync::Arc<dyn common_redis::Client + Send + Sync>>,+ limits: EventPropertiesLimits,team_id: i32,event_uuid: Uuid,notification_type: &'static str,@@ -548,14 +556,15 @@ async fn store_error_tracking_event_properties(metrics::histogram!(ISSUE_CREATED_EVENT_PROPERTIES_BYTES, "notification_type" => notification_type).record(payload.len() as f64);- if payload.len() > max_bytes {+ if payload.len() > limits.max_bytes {metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORE_SKIPPED, "reason" => "payload_too_large", "notification_type" => notification_type).increment(1);return;}+ let shadow_payload = shadow.map(|_| payload.clone());if let Err(error) = redis- .set_bytes(key.clone(), payload, Some(ttl_seconds))+ .set_bytes(key.clone(), payload, Some(limits.ttl_seconds)).await{metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORE_FAILED, "reason" => "redis", "notification_type" => notification_type)@@ -564,6 +573,9 @@ async fn store_error_tracking_event_properties(return;}+ if let Some(payload) = shadow_payload {+ crate::modes::shadow_redis::set_bytes(shadow, key, payload, limits.ttl_seconds);+ }metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORED, "notification_type" => notification_type).increment(1);}@@ -578,8 +590,11 @@ pub async fn send_issue_reopened_notification() -> Result<(), UnhandledError> {store_error_tracking_event_properties(&*context.issue_buckets_redis_client,- context.config.event_properties_ttl_seconds,- context.config.event_properties_max_bytes,+ context.shadow_redis_client.as_ref(),+ EventPropertiesLimits {+ ttl_seconds: context.config.event_properties_ttl_seconds,+ max_bytes: context.config.event_properties_max_bytes,+ },issue.team_id,event_uuid,"issue_reopened",@@ -610,8 +625,11 @@ pub async fn send_issue_spiking_notification() -> Result<(), UnhandledError> {store_error_tracking_event_properties(&*context.issue_buckets_redis_client,- context.config.event_properties_ttl_seconds,- context.config.event_properties_max_bytes,+ context.shadow_redis_client.as_ref(),+ EventPropertiesLimits {+ ttl_seconds: context.config.event_properties_ttl_seconds,+ max_bytes: context.config.event_properties_max_bytes,+ },issue.team_id,event_uuid,"issue_spiking",@@ -814,46 +832,71 @@ mod test {#[tokio::test]async fn stores_full_event_properties_as_raw_json_with_a_ttl() {- let redis = MockRedisClient::new();- let event_uuid = Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap();- let properties_json = serde_json::json!({- "$exception_list": [{"type": "Error", "value": "boom 💥"}],- "$exception_fingerprint": "abc",- "$exception_fingerprint_record": [{"type": "manual"}],- "$exception_issue_id": Uuid::nil(),- "$exception_handled": false,- "$exception_types": ["Error"],- "$exception_values": ["boom 💥"],- "$exception_sources": [],- "$exception_functions": [],- "custom": {"nested": [1, null, true]},- });- let properties: ProcessedExceptionProperties =- serde_json::from_value(properties_json.clone()).unwrap();-- super::store_error_tracking_event_properties(- &redis,- 172_800,- 1_048_576,- 42,- event_uuid,- "issue_created",- &properties,- )- .await;-- let calls = redis.get_calls();- assert_eq!(calls.len(), 1);- assert_eq!(calls[0].op, "set_bytes");- match &calls[0].value {- MockRedisValue::Bytes(payload, Some(172_800)) => {- assert_eq!(- serde_json::from_slice::<serde_json::Value>(payload).unwrap(),- properties_json- );+ crate::modes::shadow_redis::with_test_circuit(async {+ let redis = MockRedisClient::new();+ let shadow = std::sync::Arc::new(MockRedisClient::new());+ let shadow_client: std::sync::Arc<dyn common_redis::Client + Send + Sync> =+ shadow.clone();+ let event_uuid = Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap();+ let properties_json = serde_json::json!({+ "$exception_list": [{"type": "Error", "value": "boom 💥"}],+ "$exception_fingerprint": "abc",+ "$exception_fingerprint_record": [{"type": "manual"}],+ "$exception_issue_id": Uuid::nil(),+ "$exception_handled": false,+ "$exception_types": ["Error"],+ "$exception_values": ["boom 💥"],+ "$exception_sources": [],+ "$exception_functions": [],+ "custom": {"nested": [1, null, true]},+ });+ let properties: ProcessedExceptionProperties =+ serde_json::from_value(properties_json.clone()).unwrap();++ super::store_error_tracking_event_properties(+ &redis,+ Some(&shadow_client),+ super::EventPropertiesLimits {+ ttl_seconds: 172_800,+ max_bytes: 1_048_576,+ },+ 42,+ event_uuid,+ "issue_created",+ &properties,+ )+ .await;++ let calls = redis.get_calls();+ assert_eq!(calls.len(), 1);+ assert_eq!(calls[0].op, "set_bytes");+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while shadow.get_calls().is_empty() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ let shadow_calls = shadow.get_calls();+ assert_eq!(shadow_calls.len(), 1);+ assert_eq!(shadow_calls[0].op, calls[0].op);+ assert_eq!(shadow_calls[0].key, calls[0].key);+ assert!(matches!(+ (&shadow_calls[0].value, &calls[0].value),+ (MockRedisValue::Bytes(shadow_bytes, shadow_ttl), MockRedisValue::Bytes(bytes, ttl))+ if shadow_bytes == bytes && shadow_ttl == ttl+ ));+ match &calls[0].value {+ MockRedisValue::Bytes(payload, Some(172_800)) => {+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(payload).unwrap(),+ properties_json+ );+ }+ value => panic!("unexpected Redis value: {value:?}"),}- value => panic!("unexpected Redis value: {value:?}"),- }+ })+ .await;}#[tokio::test]@@ -876,8 +919,11 @@ mod test {super::store_error_tracking_event_properties(&redis,- 172_800,- 128,+ None,+ super::EventPropertiesLimits {+ ttl_seconds: 172_800,+ max_bytes: 128,+ },42,Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap(),"issue_created",diff --git a/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs b/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rsindex 8ea8ac234016..4f7385a14eaa 100644--- a/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs+++ b/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs@@ -92,6 +92,7 @@ fn get_now_rounded_to_minutes(minutes: i64) -> String {async fn try_increment_issue_buckets(heal_gate: &HealGate,redis: &Arc<dyn Client + Send + Sync>,+ shadow: Option<&Arc<dyn Client + Send + Sync>>,issue_counts: &HashMap<Uuid, u32>,) {if issue_counts.is_empty() {@@ -109,12 +110,26 @@ async fn try_increment_issue_buckets(}).collect();+ let shadow_items = shadow.map(|_| items.clone());if let Err(err) = redis.batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS).await{warn!("Failed to increment issue buckets batch: {err}");heal_on_connection_error(heal_gate, redis, &err);+ } else if let Some(shadow) = shadow {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ let items = shadow_items.expect("shadow items exist when shadow is configured");+ crate::modes::shadow_redis::submit(+ "issue_buckets",+ async move {+ shadow+ .batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)+ .await+ },+ async move { healing.reconnect().await },+ );}}@@ -123,6 +138,7 @@ async fn try_increment_team_buckets(redis: &Arc<dyn Client + Send + Sync>,issues_by_id: &HashMap<Uuid, Issue>,issue_counts: &HashMap<Uuid, u32>,+ shadow: Option<&Arc<dyn Client + Send + Sync>>,) {if issue_counts.is_empty() {return;@@ -151,12 +167,26 @@ async fn try_increment_team_buckets(}).collect();+ let shadow_items = shadow.map(|_| items.clone());if let Err(err) = redis.batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS).await{warn!("Failed to increment team buckets batch: {err}");heal_on_connection_error(heal_gate, redis, &err);+ } else if let Some(shadow) = shadow {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ let items = shadow_items.expect("shadow items exist when shadow is configured");+ crate::modes::shadow_redis::submit(+ "team_buckets",+ async move {+ shadow+ .batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)+ .await+ },+ async move { healing.reconnect().await },+ );}// Track unique issues per team bucket using sets@@ -171,12 +201,26 @@ async fn try_increment_team_buckets(}).collect();+ let shadow_items = shadow.map(|_| issue_set_items.clone());if let Err(err) = redis.batch_sadd_expire(issue_set_items, ISSUE_BUCKET_TTL_SECONDS).await{warn!("Failed to add issues to team sets: {err}");heal_on_connection_error(heal_gate, redis, &err);+ } else if let Some(shadow) = shadow {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ let issue_set_items = shadow_items.expect("shadow items exist when shadow is configured");+ crate::modes::shadow_redis::submit(+ "team_issue_sets",+ async move {+ shadow+ .batch_sadd_expire(issue_set_items, ISSUE_BUCKET_TTL_SECONDS)+ .await+ },+ async move { healing.reconnect().await },+ );}}@@ -218,6 +262,7 @@ pub async fn do_spike_detection(try_increment_issue_buckets(&context.issue_buckets_heal_gate,&context.issue_buckets_redis_client,+ context.shadow_redis_client.as_ref(),&issue_counts,).await;@@ -229,6 +274,7 @@ pub async fn do_spike_detection(&context.issue_buckets_redis_client,&issues_by_id,&issue_counts,+ context.shadow_redis_client.as_ref(),).await;team_buckets_timer.fin();@@ -275,6 +321,51 @@ async fn acquire_cooldown_locks(redis.batch_set_nx_ex(batch).await}+async fn acquire_cooldown_locks_mirrored(+ redis: &(dyn Client + Send + Sync),+ shadow: Option<&Arc<dyn Client + Send + Sync>>,+ items: &[(String, usize)],+) -> Result<Vec<bool>, common_redis::CustomRedisError> {+ let result = acquire_cooldown_locks(redis, items).await?;+ if let Some(shadow) = shadow {+ let granted: Vec<_> = items+ .iter()+ .zip(&result)+ .filter(|(_, acquired)| **acquired)+ .map(|(item, _)| item.clone())+ .collect();+ if !granted.is_empty() {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ crate::modes::shadow_redis::submit(+ "cooldown_acquire",+ async move { acquire_cooldown_locks(&*shadow, &granted).await },+ async move { healing.reconnect().await },+ );+ }+ }+ Ok(result)+}++async fn release_cooldown_locks_mirrored(+ redis: &(dyn Client + Send + Sync),+ shadow: Option<&Arc<dyn Client + Send + Sync>>,+ keys: Vec<String>,+) -> Result<(), common_redis::CustomRedisError> {+ let shadow_keys = shadow.map(|_| keys.clone());+ redis.batch_del(keys).await?;+ if let (Some(shadow), Some(keys)) = (shadow, shadow_keys) {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ crate::modes::shadow_redis::submit(+ "cooldown_release",+ async move { shadow.batch_del(keys).await },+ async move { healing.reconnect().await },+ );+ }+ Ok(())+}+async fn emit_spiking_events(context: &AppContext,spiking: Vec<SpikingIssue>,@@ -296,15 +387,20 @@ async fn emit_spiking_events(}).unzip();- let lock_results =- match acquire_cooldown_locks(&*context.issue_buckets_redis_client, &cooldown_items).await {- Ok(results) => results,- Err(e) => {- locks_timer.fin();- warn!("Failed to acquire spike cooldown locks: {e}");- return;- }- };+ let lock_results = match acquire_cooldown_locks_mirrored(+ &*context.issue_buckets_redis_client,+ context.shadow_redis_client.as_ref(),+ &cooldown_items,+ )+ .await+ {+ Ok(results) => results,+ Err(e) => {+ locks_timer.fin();+ warn!("Failed to acquire spike cooldown locks: {e}");+ return;+ }+ };let blocked_count = lock_results.iter().filter(|&&acquired| !acquired).count();metrics::counter!(SPIKE_ISSUES_BLOCKED_BY_COOLDOWN).increment(blocked_count as u64);@@ -340,10 +436,12 @@ async fn emit_spiking_events(}if !failed_keys.is_empty() {- if let Err(e) = context- .issue_buckets_redis_client- .batch_del(failed_keys)- .await+ if let Err(e) = release_cooldown_locks_mirrored(+ &*context.issue_buckets_redis_client,+ context.shadow_redis_client.as_ref(),+ failed_keys,+ )+ .await{warn!("Failed to release cooldown locks after Kafka failure: {e}");}@@ -556,6 +654,113 @@ mod tests {use chrono::TimeZone;use common_redis::MockRedisClient;+ #[tokio::test]+ async fn mirrors_only_cooldown_locks_granted_by_primary() {+ crate::modes::shadow_redis::with_test_circuit(async {+ let denied = "cooldown/denied".to_string();+ let granted = "cooldown/granted".to_string();+ let mut primary_mock = MockRedisClient::new();+ primary_mock.set_nx_ex_ret(&denied, Ok(false));+ primary_mock.set_nx_ex_ret(&granted, Ok(true));+ let shadow = Arc::new(MockRedisClient::new());+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();+ let items = vec![(denied, 600), (granted.clone(), 600)];++ let result = acquire_cooldown_locks_mirrored(+ &primary_mock,+ Some(&shadow_client),+ &items,+ )+ .await+ .unwrap();+ assert_eq!(result, vec![false, true]);+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while shadow.get_calls().is_empty() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ let calls = shadow.get_calls();+ assert_eq!(calls.len(), 1);+ assert!(matches!(&calls[0].value, common_redis::MockRedisValue::VecString(keys) if keys == &vec![granted]));+ })+ .await;+ }++ #[tokio::test]+ async fn does_not_mirror_refused_cooldown_locks() {+ crate::modes::shadow_redis::with_test_circuit(async {+ let primary = MockRedisClient::new();+ let shadow = Arc::new(MockRedisClient::new());+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();+ let items = vec![("cooldown/denied".to_string(), 600)];+ let result = acquire_cooldown_locks_mirrored(&primary, Some(&shadow_client), &items)+ .await+ .unwrap();+ for _ in 0..10 {+ tokio::task::yield_now().await;+ }+ assert_eq!(result, vec![false]);+ assert!(shadow.get_calls().is_empty());+ })+ .await;+ }++ #[tokio::test]+ async fn mirrors_spike_counters_sets_and_cooldowns() {+ let ctx = TestContext::new();+ let key = cooldown_key(&ctx.issue_id);+ let mut primary_mock = MockRedisClient::new();+ primary_mock.set_nx_ex_ret(&key, Ok(true));+ let primary: Arc<dyn Client + Send + Sync> = Arc::new(primary_mock);+ let shadow = Arc::new(MockRedisClient::new());+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();+ let counts = HashMap::from([(ctx.issue_id, 2)]);+ let gate = HealGate::new();+ try_increment_issue_buckets(&gate, &primary, Some(&shadow_client), &counts).await;+ try_increment_team_buckets(+ &gate,+ &primary,+ &ctx.issues_by_id(),+ &counts,+ Some(&shadow_client),+ )+ .await;+ let locks = vec![(key.clone(), 3600)];+ let primary_result =+ acquire_cooldown_locks_mirrored(&*primary, Some(&shadow_client), &locks)+ .await+ .unwrap();+ let keys = vec![cooldown_key(&ctx.issue_id)];+ release_cooldown_locks_mirrored(&*primary, Some(&shadow_client), keys.clone())+ .await+ .unwrap();+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while shadow.get_calls().len() < 5 {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ assert_eq!(primary_result, vec![true]);+ let ops = shadow+ .get_calls()+ .into_iter()+ .map(|call| call.op)+ .collect::<Vec<_>>();+ assert_eq!(+ ops.iter()+ .filter(|op| op.as_str() == "batch_incr_by_expire_nx")+ .count(),+ 2+ );+ assert!(ops.contains(&"batch_sadd_expire".to_string()));+ assert!(ops.contains(&"batch_set_nx_ex".to_string()));+ assert!(ops.contains(&"batch_del".to_string()));+ assert!(shadow.get_calls().iter().any(|call| matches!(&call.value, common_redis::MockRedisValue::VecString(shadow_keys) if call.op == "batch_del" && shadow_keys == &keys)));+ }+fn bytes(v: i64) -> Vec<u8> {v.to_string().into_bytes()}diff --git a/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs b/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rsindex 2a241037d135..c3682d8af211 100644--- a/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs+++ b/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs@@ -4,8 +4,8 @@ use chrono::Utc;use common_redis::{CustomRedisError, ScriptRunner};use uuid::Uuid;-use crate::modes::processing::redis_heal::HealGate;use crate::modes::processing::rules::rate_limit::BucketParams;+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};/// Outcome of one fused rate-limit call for a single issue group of `n` events./// `issue_admitted <= n` passed the per-issue limit; `team_admitted <= issue_admitted`@@ -72,6 +72,7 @@ return { issue_admitted, team_admitted }pub struct RedisRateLimiter {redis: Arc<dyn ScriptRunner>,+ shadow: Option<Arc<dyn ScriptRunner>>,key_prefix: String,bucket_ttl_seconds: u64,heal_gate: HealGate,@@ -81,12 +82,18 @@ impl RedisRateLimiter {pub fn new(redis: Arc<dyn ScriptRunner>, key_prefix: String, bucket_ttl_seconds: u64) -> Self {Self {redis,+ shadow: None,key_prefix,bucket_ttl_seconds,heal_gate: HealGate::new(),}}+ pub fn with_shadow(mut self, shadow: Option<Arc<dyn ScriptRunner>>) -> Self {+ self.shadow = shadow;+ self+ }+fn issue_key(&self, team_id: i32, issue_id: Option<Uuid>) -> String {match issue_id {Some(id) => format!("{}/{{{team_id}}}/issue/{id}", self.key_prefix),@@ -136,7 +143,17 @@ impl RedisRateLimiter {self.bucket_ttl_seconds.to_string(),];+ let shadow_request = self.shadow.as_ref().map(|_| (keys.clone(), args.clone()));let res = self.redis.eval_int_vec(RATE_LIMIT_LUA, keys, args).await?;+ if let (Some(shadow), Some((keys, args))) = (&self.shadow, shadow_request) {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ shadow_redis::submit(+ "processing_rate_limit",+ async move { shadow.eval_int_vec(RATE_LIMIT_LUA, keys, args).await },+ async move { healing.reconnect().await },+ );+ }// Defensive clamps: team_admitted <= issue_admitted <= n.let issue_admitted = (res.first().copied().unwrap_or(0).max(0) as u32).min(n);@@ -164,7 +181,47 @@ impl RedisRateLimiter {#[cfg(test)]mod tests {use super::*;- use crate::test_support::{start_redis, RedisContainer};+ use crate::test_support::{start_redis, FakeRunner, RedisContainer};++ #[tokio::test]+ async fn primary_failure_does_not_charge_the_shadow() {+ let shadow = FakeRunner::returning(vec![1, 1]);+ let limiter = RedisRateLimiter::new(FakeRunner::failing(), "test".into(), 3600)+ .with_shadow(Some(shadow.clone()));+ assert!(limiter.admit(42, None, None, None, 1).await.is_err());+ for _ in 0..10 {+ tokio::task::yield_now().await;+ }+ assert_eq!(shadow.call_count(), 0);+ }++ #[tokio::test]+ async fn shadow_failure_does_not_change_the_primary_admission() {+ shadow_redis::with_test_circuit(async {+ let shadow = FakeRunner::failing();+ let primary = FakeRunner::returning(vec![3, 2]);+ let limiter = RedisRateLimiter::new(primary.clone(), "test".into(), 3600)+ .with_shadow(Some(shadow.clone()));+ let decision = limiter.admit(42, None, None, None, 5).await.unwrap();+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while shadow.call_count() == 0 {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ assert_eq!(primary.requests(), shadow.requests());+ assert_eq!(+ decision,+ RateLimitDecision {+ issue_admitted: 3,+ team_admitted: 2+ }+ );+ })+ .await;+ }+use common_redis::RedisClient;// ===================================================================diff --git a/rust/cymbal/src/modes/shadow_redis.rs b/rust/cymbal/src/modes/shadow_redis.rsnew file mode 100644index 000000000000..6766267dc873--- /dev/null+++ b/rust/cymbal/src/modes/shadow_redis.rs@@ -0,0 +1,507 @@+use std::{+ future::Future,+ sync::{+ atomic::{AtomicBool, AtomicU64, Ordering},+ Arc, Mutex, OnceLock,+ },+ time::{Duration, Instant},+};++use tokio::sync::Semaphore;++use common_redis::{Client, CompressionConfig, CustomRedisError, RedisClient, RedisValueFormat};+use tracing::warn;++use crate::modes::processing::redis_heal::{is_connection_error, HealGate};++const SHADOW_DEADLINE: Duration = Duration::from_millis(200);+const MAX_IN_FLIGHT_SHADOW_WRITES: usize = 128;+const SHADOW_RECOVERY_INTERVAL: Duration = Duration::from_secs(5);+const SHADOW_RECOVERY_DEADLINE: Duration = Duration::from_secs(2);+const SHADOW_WARN_INTERVAL_MS: u64 = 30_000;+static SHADOW_PERMITS: OnceLock<Arc<Semaphore>> = OnceLock::new();+static SHADOW_CIRCUIT: OnceLock<Arc<ShadowCircuit>> = OnceLock::new();+static SHADOW_HEAL_GATE: OnceLock<HealGate> = OnceLock::new();+static WARN_EPOCH: OnceLock<Instant> = OnceLock::new();+static LAST_SHADOW_WARNING_MS: AtomicU64 = AtomicU64::new(0);++#[cfg(test)]+tokio::task_local! {+ static TEST_SHADOW_CIRCUIT: Arc<ShadowCircuit>;+}++#[cfg(test)]+pub(crate) async fn with_test_circuit<T>(future: impl Future<Output = T>) -> T {+ TEST_SHADOW_CIRCUIT+ .scope(Arc::new(ShadowCircuit::new()), future)+ .await+}++fn shadow_circuit() -> Arc<ShadowCircuit> {+ #[cfg(test)]+ if let Ok(circuit) = TEST_SHADOW_CIRCUIT.try_with(Arc::clone) {+ return circuit;+ }+ SHADOW_CIRCUIT+ .get_or_init(|| Arc::new(ShadowCircuit::new()))+ .clone()+}++pub async fn connect(+ url: Option<&str>,+ primary_urls: &[&str],+ format: RedisValueFormat,+ response_timeout_ms: u64,+ connection_timeout_ms: u64,+) -> Option<Arc<RedisClient>> {+ let url = url.filter(|url| !url.trim().is_empty())?;+ if primary_urls+ .iter()+ .any(|primary| same_endpoint(url, primary))+ {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "same_endpoint").increment(1);+ warn!("shadow Redis URL matches a primary Redis URL; shadow writes disabled");+ return None;+ }+ match RedisClient::with_config(+ url.to_string(),+ CompressionConfig::disabled(),+ format,+ Some(Duration::from_millis(shadow_timeout_ms(+ response_timeout_ms,+ 200,+ ))),+ Some(Duration::from_millis(shadow_timeout_ms(+ connection_timeout_ms,+ 1000,+ ))),+ )+ .await+ {+ Ok(client) => {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "ok").increment(1);+ Some(Arc::new(client))+ }+ Err(error) => {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "error").increment(1);+ warn!(%error, "shadow Redis connection unavailable; primary remains active");+ None+ }+ }+}++fn shadow_timeout_ms(configured: u64, cap: u64) -> u64 {+ if configured == 0 {+ cap+ } else {+ configured.min(cap)+ }+}++fn same_endpoint(shadow: &str, primary: &str) -> bool {+ shadow == primary+ || matches!((redis_endpoint(shadow), redis_endpoint(primary)), (Some(a), Some(b)) if a == b)+}++#[derive(PartialEq, Eq)]+enum RedisEndpoint {+ Tcp(String, u16, i64),+ Unix(std::path::PathBuf, i64),+}++fn redis_endpoint(url: &str) -> Option<RedisEndpoint> {+ let client = redis::Client::open(url).ok()?;+ let info = client.get_connection_info();+ let (host, port) = match &info.addr {+ redis::ConnectionAddr::Tcp(host, port) => (host, *port),+ redis::ConnectionAddr::TcpTls { host, port, .. } => (host, *port),+ redis::ConnectionAddr::Unix(path) => {+ return Some(RedisEndpoint::Unix(path.clone(), info.redis.db));+ }+ };+ let host = host.trim_end_matches('.').to_ascii_lowercase();+ let host = match host.as_str() {+ "localhost" | "::1" | "[::1]" => "127.0.0.1".to_string(),+ _ => host,+ };+ Some(RedisEndpoint::Tcp(host, port, info.redis.db))+}++fn should_warn(last: &AtomicU64, now_ms: u64) -> bool {+ let now = now_ms.saturating_add(1);+ let previous = last.load(Ordering::Relaxed);+ (previous == 0 || now.saturating_sub(previous) >= SHADOW_WARN_INTERVAL_MS)+ && last+ .compare_exchange(previous, now, Ordering::Relaxed, Ordering::Relaxed)+ .is_ok()+}++fn warn_for_shadow_failure() -> bool {+ let elapsed_ms = WARN_EPOCH.get_or_init(Instant::now).elapsed().as_millis() as u64;+ should_warn(&LAST_SHADOW_WARNING_MS, elapsed_ms)+}++struct RecoveryState {+ generation: u64,+ next_attempt: Instant,+}++// A timed-out request can stay queued in redis-rs. Block writes until a new connection replaces it.+struct ShadowCircuit {+ blocked: AtomicBool,+ generation: AtomicU64,+ recovery: Mutex<RecoveryState>,+}++impl ShadowCircuit {+ fn new() -> Self {+ Self {+ blocked: AtomicBool::new(false),+ generation: AtomicU64::new(0),+ recovery: Mutex::new(RecoveryState {+ generation: 0,+ next_attempt: Instant::now(),+ }),+ }+ }++ fn trip(self: &Arc<Self>, generation: u64, heal: impl Future<Output = bool> + Send + 'static) {+ let mut recovery = self.recovery.lock().unwrap();+ if recovery.generation != generation {+ return;+ }+ self.blocked.store(true, Ordering::Release);+ self.schedule_recovery(&mut recovery, heal);+ }++ fn retry(self: &Arc<Self>, heal: impl Future<Output = bool> + Send + 'static) {+ let mut recovery = self.recovery.lock().unwrap();+ self.schedule_recovery(&mut recovery, heal);+ }++ fn schedule_recovery(+ self: &Arc<Self>,+ recovery: &mut RecoveryState,+ heal: impl Future<Output = bool> + Send + 'static,+ ) {+ if Instant::now() < recovery.next_attempt {+ return;+ }+ recovery.next_attempt = Instant::now() + SHADOW_RECOVERY_INTERVAL;+ let circuit = self.clone();+ tokio::spawn(async move {+ if let Ok(true) = tokio::time::timeout(SHADOW_RECOVERY_DEADLINE, heal).await {+ let mut recovery = circuit.recovery.lock().unwrap();+ recovery.generation += 1;+ circuit+ .generation+ .store(recovery.generation, Ordering::Release);+ circuit.blocked.store(false, Ordering::Release);+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "reconnect", "outcome" => "ok").increment(1);+ } else {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "reconnect", "outcome" => "error").increment(1);+ }+ });+ }+}++pub fn submit<T: Send + 'static>(+ operation: &'static str,+ write: impl Future<Output = Result<T, CustomRedisError>> + Send + 'static,+ heal: impl Future<Output = bool> + Send + 'static,+) {+ let permits =+ SHADOW_PERMITS.get_or_init(|| Arc::new(Semaphore::new(MAX_IN_FLIGHT_SHADOW_WRITES)));+ let circuit = shadow_circuit();+ submit_with_permits(permits, &circuit, operation, write, heal);+}++fn submit_with_permits<T: Send + 'static>(+ permits: &Arc<Semaphore>,+ circuit: &Arc<ShadowCircuit>,+ operation: &'static str,+ write: impl Future<Output = Result<T, CustomRedisError>> + Send + 'static,+ heal: impl Future<Output = bool> + Send + 'static,+) {+ if circuit.blocked.load(Ordering::Acquire) {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "circuit_open").increment(1);+ circuit.retry(heal);+ return;+ }+ let generation = circuit.generation.load(Ordering::Acquire);+ let Ok(permit) = permits.clone().try_acquire_owned() else {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "dropped").increment(1);+ return;+ };+ let circuit = circuit.clone();+ tokio::spawn(async move {+ let _permit = permit;+ match tokio::time::timeout(SHADOW_DEADLINE, write).await {+ Ok(Ok(_)) => metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "ok").increment(1),+ Ok(Err(CustomRedisError::Timeout)) | Err(_) => {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "timeout").increment(1);+ if warn_for_shadow_failure() {+ warn!(operation, "shadow Redis write timed out");+ }+ circuit.trip(generation, heal);+ }+ Ok(Err(error)) => {+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "error").increment(1);+ if warn_for_shadow_failure() {+ warn!(%error, operation, "shadow Redis write failed");+ }+ if is_connection_error(&error) {+ SHADOW_HEAL_GATE.get_or_init(HealGate::new).spawn_heal(async move {+ let _ = heal.await;+ });+ }+ }+ }+ });+}++pub fn set_bytes(+ shadow: Option<&Arc<dyn Client + Send + Sync>>,+ key: String,+ payload: Vec<u8>,+ ttl_seconds: u64,+) {+ if let Some(shadow) = shadow {+ let shadow = shadow.clone();+ let healing = shadow.clone();+ submit(+ "event_properties",+ async move { shadow.set_bytes(key, payload, Some(ttl_seconds)).await },+ async move { healing.reconnect().await },+ );+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use metrics_util::debugging::DebuggingRecorder;++ #[tokio::test]+ async fn exact_primary_url_cannot_be_shadowed() {+ let shadow = connect(+ Some("redis://localhost:6379/"),+ &["redis://localhost:6379/"],+ RedisValueFormat::Utf8,+ 100,+ 100,+ )+ .await;+ assert!(shadow.is_none());+ }++ #[test]+ fn equivalent_primary_urls_cannot_be_shadowed() {+ assert!(same_endpoint(+ "redis://primary-host:6379/0",+ "redis://primary-host:6379"+ ));+ assert!(same_endpoint(+ "redis://localhost",+ "redis://127.0.0.1:6379/"+ ));+ assert!(same_endpoint(+ "redis+unix:///var/run/redis/redis.sock",+ "unix:///var/run/redis/redis.sock"+ ));+ assert!(same_endpoint(+ "rediss://PRIMARY-HOST:6379/",+ "redis://primary-host:6379/0"+ ));+ assert!(!same_endpoint(+ "redis://primary-host:6379/1",+ "redis://primary-host:6379/0"+ ));+ assert!(!same_endpoint(+ "redis://new-host:6379",+ "redis://primary-host:6379"+ ));+ }++ #[test]+ fn zero_primary_timeout_uses_bounded_shadow_timeout() {+ assert_eq!(shadow_timeout_ms(0, 200), 200);+ assert_eq!(shadow_timeout_ms(0, 1000), 1000);+ assert_eq!(shadow_timeout_ms(100, 200), 100);+ assert_eq!(shadow_timeout_ms(5000, 1000), 1000);+ }++ #[test]+ fn repeated_failures_do_not_repeat_warnings() {+ let last = AtomicU64::new(0);+ assert!(should_warn(&last, 0));+ assert!(!should_warn(&last, 1));+ assert!(!should_warn(&last, SHADOW_WARN_INTERVAL_MS - 1));+ assert!(should_warn(&last, SHADOW_WARN_INTERVAL_MS));+ }++ #[tokio::test]+ async fn timeout_blocks_new_writes_until_reconnect_succeeds() {+ let permits = Arc::new(Semaphore::new(2));+ let circuit = Arc::new(ShadowCircuit::new());+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();+ let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();+ submit_with_permits(+ &permits,+ &circuit,+ "test_timeout",+ std::future::pending::<Result<(), CustomRedisError>>(),+ async move {+ let _ = started_tx.send(());+ let _ = resume_rx.await;+ true+ },+ );+ tokio::time::timeout(Duration::from_secs(1), started_rx)+ .await+ .unwrap()+ .unwrap();+ assert!(circuit.blocked.load(Ordering::Acquire));++ let called = Arc::new(AtomicBool::new(false));+ let rejected = called.clone();+ submit_with_permits(+ &permits,+ &circuit,+ "test_blocked",+ async move {+ rejected.store(true, Ordering::Release);+ Ok::<(), CustomRedisError>(())+ },+ async { true },+ );+ assert!(!called.load(Ordering::Acquire));++ resume_tx.send(()).unwrap();+ tokio::time::timeout(Duration::from_secs(1), async {+ while circuit.blocked.load(Ordering::Acquire) {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ assert_eq!(circuit.generation.load(Ordering::Acquire), 1);+ }++ #[tokio::test]+ async fn failed_reconnect_keeps_shadow_circuit_open() {+ let permits = Arc::new(Semaphore::new(1));+ let circuit = Arc::new(ShadowCircuit::new());+ let (tried_tx, tried_rx) = tokio::sync::oneshot::channel();+ submit_with_permits(+ &permits,+ &circuit,+ "test_failure",+ async { Err::<(), _>(CustomRedisError::Timeout) },+ async move {+ let _ = tried_tx.send(());+ false+ },+ );+ tokio::time::timeout(Duration::from_secs(1), tried_rx)+ .await+ .unwrap()+ .unwrap();+ assert!(circuit.blocked.load(Ordering::Acquire));+ circuit.recovery.lock().unwrap().next_attempt = Instant::now();+ let (recovered_tx, recovered_rx) = tokio::sync::oneshot::channel();+ submit_with_permits(+ &permits,+ &circuit,+ "test_retry",+ async { Ok::<(), CustomRedisError>(()) },+ async move {+ let _ = recovered_tx.send(());+ true+ },+ );+ tokio::time::timeout(Duration::from_secs(1), recovered_rx)+ .await+ .unwrap()+ .unwrap();+ tokio::time::timeout(Duration::from_secs(1), async {+ while circuit.blocked.load(Ordering::Acquire) {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ }++ #[test]+ fn client_timeout_records_timeout_instead_of_error() {+ let recorder = DebuggingRecorder::new();+ let circuit = Arc::new(ShadowCircuit::new());+ let permits = Arc::new(Semaphore::new(1));+ metrics::with_local_recorder(&recorder, || {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ submit_with_permits(+ &permits,+ &circuit,+ "test_client_timeout",+ async { Err::<(), _>(CustomRedisError::Timeout) },+ async { false },+ );+ tokio::time::timeout(Duration::from_secs(1), async {+ while !circuit.blocked.load(Ordering::Acquire) {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ });+ });+ let counts = recorder.snapshotter().snapshot().into_vec();+ assert!(counts.iter().any(|(key, _, _, _)| {+ key.key().name() == "cymbal_shadow_redis_operations_total"+ && key.key().labels().any(|label| {+ label.key() == "operation" && label.value() == "test_client_timeout"+ })+ && key+ .key()+ .labels()+ .any(|label| label.key() == "outcome" && label.value() == "timeout")+ }));+ assert!(!counts.iter().any(|(key, _, _, _)| {+ key.key().name() == "cymbal_shadow_redis_operations_total"+ && key.key().labels().any(|label| {+ label.key() == "operation" && label.value() == "test_client_timeout"+ })+ && key+ .key()+ .labels()+ .any(|label| label.key() == "outcome" && label.value() == "error")+ }));+ }++ #[test]+ fn saturated_shadow_capacity_records_dropped_write() {+ let recorder = DebuggingRecorder::new();+ metrics::with_local_recorder(&recorder, || {+ submit_with_permits(+ &Arc::new(Semaphore::new(0)),+ &Arc::new(ShadowCircuit::new()),+ "issue_buckets",+ async { Ok::<(), CustomRedisError>(()) },+ async { true },+ );+ });+ let snapshot = recorder.snapshotter().snapshot().into_vec();+ assert!(snapshot.iter().any(|(key, _, _, _)| {+ key.key().name() == "cymbal_shadow_redis_operations_total"+ && key+ .key()+ .labels()+ .any(|label| label.key() == "outcome" && label.value() == "dropped")+ }));+ }+}diff --git a/rust/cymbal/src/test_support.rs b/rust/cymbal/src/test_support.rsindex 7db7882b722f..a7b57869d241 100644--- a/rust/cymbal/src/test_support.rs+++ b/rust/cymbal/src/test_support.rs@@ -51,18 +51,35 @@ pub async fn start_redis() -> (Arc<RedisClient>, RedisContainer) {panic!("redis container never became ready");}+pub type ScriptRequest = (String, Vec<String>, Vec<String>);+/// A [`ScriptRunner`] that never touches Redis: a canned reply, or an error.pub struct FakeRunner {reply: Option<Vec<i64>>,+ calls: std::sync::Mutex<Vec<ScriptRequest>>,}impl FakeRunner {pub fn returning(reply: Vec<i64>) -> Arc<Self> {- Arc::new(Self { reply: Some(reply) })+ Arc::new(Self {+ reply: Some(reply),+ calls: std::sync::Mutex::new(Vec::new()),+ })}pub fn failing() -> Arc<Self> {- Arc::new(Self { reply: None })+ Arc::new(Self {+ reply: None,+ calls: std::sync::Mutex::new(Vec::new()),+ })+ }++ pub fn call_count(&self) -> usize {+ self.requests().len()+ }++ pub fn requests(&self) -> Vec<ScriptRequest> {+ self.calls.lock().unwrap().clone()}}@@ -70,10 +87,14 @@ impl FakeRunner {impl ScriptRunner for FakeRunner {async fn eval_int_vec(&self,- _script: &str,- _keys: Vec<String>,- _args: Vec<String>,+ script: &str,+ keys: Vec<String>,+ args: Vec<String>,) -> Result<Vec<i64>, CustomRedisError> {+ self.calls+ .lock()+ .unwrap()+ .push((script.to_string(), keys, args));self.reply.clone().ok_or(CustomRedisError::Timeout)}}