Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cymbal): add optional Redis shadow writes (#130117)

Hugues Pouillot16ee8aa7e144
diff --git a/rust/common/redis/src/client.rs b/rust/common/redis/src/client.rs
index 2655d8ac25e4..1b6c03113809 100644
--- a/rust/common/redis/src/client.rs
+++ b/rust/common/redis/src/client.rs
@@ -54,9 +54,18 @@ impl RedisClient {
/// attempt just waits for the next caller -- the client keeps failing
/// open in the meantime, exactly as it would without healing.
pub async fn heal_connection(&self) {
+ self.reconnect_inner(false).await;
+ }
+
+ /// Replace the connection after a shadow timeout, even during the normal heal cooldown.
+ pub async fn reconnect(&self) -> bool {
+ self.reconnect_inner(true).await
+ }
+
+ async fn reconnect_inner(&self, force: bool) -> bool {
let mut last_attempt = self.heal_state.lock().await;
- if last_attempt.elapsed() < HEAL_COOLDOWN {
- return;
+ if !force && last_attempt.elapsed() < HEAL_COOLDOWN {
+ return false;
}
*last_attempt = Instant::now();
@@ -75,10 +84,12 @@ impl RedisClient {
{
Ok(connection) => {
self.connection.store(Arc::new(connection));
- info!("Redis connection healed after unrecoverable error");
+ info!("Redis connection replaced");
+ true
}
Err(e) => {
- warn!(error = %e, "Redis heal attempt failed; will retry after cooldown");
+ warn!(error = %e, "Redis reconnect attempt failed");
+ false
}
}
}
@@ -336,10 +347,19 @@ pub trait ScriptRunner: Send + Sync {
/// Rebuild the underlying connection after a connection-class failure; see
/// [`Client::heal`]. The default no-op keeps test fakes trivial.
async fn heal(&self) {}
+
+ /// Return whether a fresh connection replaced the old one. Test runners do not reconnect.
+ async fn reconnect(&self) -> bool {
+ false
+ }
}
#[async_trait]
impl ScriptRunner for RedisClient {
+ async fn reconnect(&self) -> bool {
+ RedisClient::reconnect(self).await
+ }
+
async fn eval_int_vec(
&self,
script: &str,
@@ -360,6 +380,10 @@ impl Client for RedisClient {
self.heal_connection().await;
}
+ async fn reconnect(&self) -> bool {
+ RedisClient::reconnect(self).await
+ }
+
async fn zrangebyscore(
&self,
k: String,
diff --git a/rust/common/redis/src/lib.rs b/rust/common/redis/src/lib.rs
index 7c5912061f2b..dd62c6c09701 100644
--- a/rust/common/redis/src/lib.rs
+++ b/rust/common/redis/src/lib.rs
@@ -325,6 +325,11 @@ pub trait Client: Send + Sync {
/// default no-op. Must be cheap to call repeatedly -- implementations own
/// their own cooldown.
async fn heal(&self) {}
+
+ /// Return whether a fresh connection replaced the old one. Test clients do not reconnect.
+ async fn reconnect(&self) -> bool {
+ false
+ }
}
/// Extension trait providing the `.pipeline()` builder method.
diff --git a/rust/cymbal/Cargo.toml b/rust/cymbal/Cargo.toml
index e18ee1de2f5a..62630f87441f 100644
--- a/rust/cymbal/Cargo.toml
+++ b/rust/cymbal/Cargo.toml
@@ -21,6 +21,7 @@ common-database = { path = "../common/database" }
common-types = { path = "../common/types" }
common-dns = { path = "../common/dns" }
common-redis = { path = "../common/redis" }
+redis = { workspace = true }
common-posthog = { path = "../common/posthog" }
cymbal-proto = { path = "../cymbal-proto" }
posthog-symbol-data = { path = "../common/symbol_data" }
@@ -62,8 +63,6 @@ common-temporal = { path = "../common/temporal" }
[dev-dependencies]
common-dns = { path = "../common/dns", features = ["test-util"] }
common-compression = { path = "../common/compression" }
-# For constructing IO-flavored RedisErrors in redis_heal classifier tests
-redis = { workspace = true }
httpmock = { workspace = true }
mockall = { workspace = true }
insta = { version = "1.46.0", features = ["json", "redactions"] }
diff --git a/rust/cymbal/README.md b/rust/cymbal/README.md
index bcf3046b3d75..b6bd51a4b4b1 100644
--- a/rust/cymbal/README.md
+++ b/rust/cymbal/README.md
@@ -50,10 +50,15 @@ are named for what is actually capped. Only issue-created is charged.
| Variable | Default | Effect |
| --- | --- | --- |
| `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_REDIS_URL` | none | Required. The service does not start without it. |
+| `ERROR_TRACKING_NOTIFICATIONS_SHADOW_REDIS_URL` | unset | Optional destination for best-effort shadow charges and refunds. Must be a different Redis instance from the primary. |
| `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_PER_HOUR` | `1000` | Bucket size, and the tokens a team earns back per hour. Zero or less disables the limit. |
| `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_KEY_PREFIX` | `@posthog/error-tracking-notifications-rate-limiter` | Key namespace. It must differ from the event limiter's prefix. |
| `ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_BUCKET_TTL_SECONDS` | `3600` | Idle buckets expire and free the memory. A value below 3600 is raised to 3600, because a bucket takes an hour to refill and a shorter TTL would loosen the limit. |
+Processing mode can set `ERROR_TRACKING_PROCESSING_SHADOW_REDIS_URL` to copy issue and team spike counters, unique-issue sets, cooldown locks and releases, lifecycle event properties, and event rate-limiter charges to a separate Redis instance. Both shadow URLs are optional and unset by default. The primary Redis still supplies every decision and read. A shadow connection failure at startup disables shadow writes on that pod; runtime errors and the 200 ms shadow deadline are counted in `cymbal_shadow_redis_operations_total` by `operation` and `outcome`, without changing the primary result. Failure warnings are limited to one per 30 seconds per pod. Do not use a DNS alias of any primary Redis URL for the shadow: non-idempotent counters and token buckets would be charged twice. The URL guard detects equivalent host, port and database settings, but cannot prove that different hostnames resolve to different instances.
+
+A shadow write is submitted after a successful primary write, with the same keys, bytes, expiry and Lua arguments. It runs in a bounded background task so it does not delay ingestion; at most 128 shadow writes run per pod, and excess submissions increment the `dropped` outcome rather than queue without a bound. After a timeout, the shadow circuit drops new writes until it can replace the connection; retries occur at most every five seconds while traffic continues. This stops timed-out requests from accumulating in the old connection. The `circuit_open` and `reconnect` outcomes record this state. Background writes can reorder or be lost when a pod exits, and an immediate shadow read is not read-after-write consistent. No historical keys are copied. A 48-hour warm-up covers event properties at their default TTL and the shorter-lived spike buckets only if all pods have a healthy shadow connection for the entire window and no writes fail, time out or drop. It does not guarantee parity for a higher configured TTL, independently advancing token buckets and cooldown locks, or writes that complete in a different order. Cooldown locks are mirrored only when the primary grants them; existing primary locks are not copied. A shadow that fails to connect at startup remains disabled on that pod until restart. Validate key and TTL coverage before switching any reader; this option never switches readers.
+
The limit covers every team. To size it before it cuts anything, set `PER_HOUR`
far above real traffic, watch `cymbal_issue_created_rate_limit_outcomes`, then
lower it. Setting `PER_HOUR` to zero or less switches the limit off.
diff --git a/rust/cymbal/src/modes/mod.rs b/rust/cymbal/src/modes/mod.rs
index dda95d984e04..6f4a5f36affd 100644
--- a/rust/cymbal/src/modes/mod.rs
+++ b/rust/cymbal/src/modes/mod.rs
@@ -9,6 +9,7 @@ use std::str::FromStr;
pub mod notifications;
pub mod processing;
pub mod resolution;
+pub mod shadow_redis;
/// Which server stack the cymbal binary runs. Parsed from `CYMBAL_MODE`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
diff --git a/rust/cymbal/src/modes/notifications/config.rs b/rust/cymbal/src/modes/notifications/config.rs
index 70ca2c4bb9ba..fec8ce6469ab 100644
--- a/rust/cymbal/src/modes/notifications/config.rs
+++ b/rust/cymbal/src/modes/notifications/config.rs
@@ -65,6 +65,9 @@ pub struct NotificationsConfig {
#[envconfig(from = "ERROR_TRACKING_NOTIFICATIONS_RATE_LIMIT_REDIS_URL")]
pub notifications_rate_limit_redis_url: String,
+ #[envconfig(from = "ERROR_TRACKING_NOTIFICATIONS_SHADOW_REDIS_URL")]
+ pub shadow_redis_url: Option<String>,
+
/// Bucket size, and the tokens a team earns back per hour. Zero or less
/// disables the limit.
#[envconfig(
diff --git a/rust/cymbal/src/modes/notifications/rate_limit.rs b/rust/cymbal/src/modes/notifications/rate_limit.rs
index 37bd27ca9a72..27c8509891e8 100644
--- a/rust/cymbal/src/modes/notifications/rate_limit.rs
+++ b/rust/cymbal/src/modes/notifications/rate_limit.rs
@@ -17,7 +17,7 @@ use crate::core::metric_consts::{
};
use crate::modes::notifications::config::NotificationsConfig;
use crate::modes::notifications::token_bucket::TokenBucket;
-use crate::modes::processing::redis_heal::is_connection_error;
+use crate::modes::{processing::redis_heal::is_connection_error, shadow_redis};
pub struct IssueCreatedRateLimiter {
/// `None` disables the limit: every notification is admitted.
@@ -43,18 +43,33 @@ impl IssueCreatedRateLimiter {
)
.await?;
+ let shadow = shadow_redis::connect(
+ config.shadow_redis_url.as_deref(),
+ &[&config.notifications_rate_limit_redis_url],
+ common_redis::RedisValueFormat::Utf8,
+ config.redis_response_timeout_ms,
+ config.redis_connection_timeout_ms,
+ )
+ .await;
+
+ let shadow: Option<Arc<dyn common_redis::ScriptRunner>> =
+ shadow.map(|client| -> Arc<dyn common_redis::ScriptRunner> { client });
+
info!(
per_hour = config.notifications_rate_limit_per_hour,
"Error-tracking issue-created rate limiter enabled for all teams",
);
Ok(Self {
- bucket: Some(TokenBucket::per_hour(
- Arc::new(client),
- config.notifications_rate_limit_key_prefix.clone(),
- config.notifications_rate_limit_per_hour as f64,
- config.notifications_rate_limit_bucket_ttl_seconds,
- )),
+ bucket: Some(
+ TokenBucket::per_hour(
+ Arc::new(client),
+ config.notifications_rate_limit_key_prefix.clone(),
+ config.notifications_rate_limit_per_hour as f64,
+ config.notifications_rate_limit_bucket_ttl_seconds,
+ )
+ .with_shadow(shadow),
+ ),
})
}
diff --git a/rust/cymbal/src/modes/notifications/token_bucket.rs b/rust/cymbal/src/modes/notifications/token_bucket.rs
index e6de6658c72e..b05581d0f66c 100644
--- a/rust/cymbal/src/modes/notifications/token_bucket.rs
+++ b/rust/cymbal/src/modes/notifications/token_bucket.rs
@@ -11,7 +11,7 @@ use chrono::Utc;
use common_redis::{CustomRedisError, ScriptRunner};
use tracing::warn;
-use crate::modes::processing::redis_heal::HealGate;
+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};
const SECONDS_PER_HOUR: f64 = 3600.0;
@@ -74,6 +74,7 @@ return { 1 }
pub struct TokenBucket {
redis: Arc<dyn ScriptRunner>,
+ shadow: Option<Arc<dyn ScriptRunner>>,
key_prefix: String,
max: f64,
rate: f64,
@@ -103,6 +104,7 @@ impl TokenBucket {
Self {
redis,
+ shadow: None,
key_prefix,
max: per_hour,
rate: per_hour / SECONDS_PER_HOUR,
@@ -111,6 +113,11 @@ impl TokenBucket {
}
}
+ pub fn with_shadow(mut self, shadow: Option<Arc<dyn ScriptRunner>>) -> Self {
+ self.shadow = shadow;
+ self
+ }
+
/// Kick a background heal of the bucket's Redis connection, for a caller
/// that saw a connection-class error. `MultiplexedConnection` never
/// reconnects on its own, so without this one failover leaves the limiter
@@ -128,10 +135,21 @@ impl TokenBucket {
self.ttl_seconds.to_string(),
Utc::now().timestamp().to_string(),
];
+ let shadow_args = self.shadow.as_ref().map(|_| args.clone());
let res = self
.redis
.eval_int_vec(CHARGE_SCRIPT, vec![self.key(team_id)], args)
.await?;
+ if let (Some(shadow), Some(args)) = (&self.shadow, shadow_args) {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ let key = self.key(team_id);
+ shadow_redis::submit(
+ "notifications_charge",
+ async move { shadow.eval_int_vec(CHARGE_SCRIPT, vec![key], args).await },
+ async move { healing.reconnect().await },
+ );
+ }
Ok(res.first().copied().unwrap_or(0) > 0)
}
@@ -140,9 +158,20 @@ impl TokenBucket {
/// one.
pub async fn refund(&self, team_id: i32) -> Result<(), CustomRedisError> {
let args = vec![self.max.to_string(), self.ttl_seconds.to_string()];
+ let shadow_args = self.shadow.as_ref().map(|_| args.clone());
self.redis
.eval_int_vec(REFUND_SCRIPT, vec![self.key(team_id)], args)
.await?;
+ if let (Some(shadow), Some(args)) = (&self.shadow, shadow_args) {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ let key = self.key(team_id);
+ shadow_redis::submit(
+ "notifications_refund",
+ async move { shadow.eval_int_vec(REFUND_SCRIPT, vec![key], args).await },
+ async move { healing.reconnect().await },
+ );
+ }
Ok(())
}
@@ -163,6 +192,30 @@ mod tests {
use crate::test_support::{start_redis, FakeRunner};
use common_redis::RedisClient;
+ #[tokio::test]
+ async fn shadow_failure_does_not_change_charge_or_refund() {
+ shadow_redis::with_test_circuit(async {
+ let shadow = FakeRunner::failing();
+ let primary = FakeRunner::returning(vec![1]);
+ let bucket = TokenBucket::per_hour(primary.clone(), "test".into(), 1.0, 3600)
+ .with_shadow(Some(shadow.clone()));
+ assert!(bucket.charge(42).await.unwrap());
+ bucket.refund(42).await.unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while shadow.call_count() == 0 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ let primary_calls = primary.requests();
+ let shadow_calls = shadow.requests();
+ assert_eq!(primary_calls.len(), 2);
+ assert!(shadow_calls.iter().all(|call| primary_calls.contains(call)));
+ })
+ .await;
+ }
+
fn bucket(client: Arc<RedisClient>, prefix: &str, per_hour: f64) -> TokenBucket {
TokenBucket::per_hour(client, prefix.to_string(), per_hour, 3600)
}
diff --git a/rust/cymbal/src/modes/processing/app_context.rs b/rust/cymbal/src/modes/processing/app_context.rs
index faddb7a720ca..5ae62ae566fd 100644
--- a/rust/cymbal/src/modes/processing/app_context.rs
+++ b/rust/cymbal/src/modes/processing/app_context.rs
@@ -9,7 +9,7 @@ use tokio::{sync::Semaphore, task::JoinHandle};
use tracing::info;
use uuid::Uuid;
-use crate::modes::processing::redis_heal::HealGate;
+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};
use crate::{
core::config::build_pg_pool,
error::UnhandledError,
@@ -44,6 +44,7 @@ pub struct AppContext {
pub team_manager: TeamManager,
pub issue_buckets_redis_client: Arc<dyn RedisClientTrait + Send + Sync>,
+ pub shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>>,
// Debounces heal-task spawns for the issue-buckets client (see redis_heal).
pub issue_buckets_heal_gate: HealGate,
// Error-tracking rate limiter. `None` when disabled (the default), in which
@@ -178,7 +179,20 @@ impl AppContext {
let (remote_resolution, remote_resolution_refresh_task) =
build_remote_resolution(config).await?;
- let rate_limiter = build_rate_limiter(config).await?;
+ let shadow = shadow_redis::connect(
+ config.shadow_redis_url.as_deref(),
+ &[
+ &config.issue_buckets_redis_url,
+ &config.error_tracking_rate_limiter_redis_url,
+ ],
+ common_redis::RedisValueFormat::default(),
+ config.redis_response_timeout_ms,
+ config.redis_connection_timeout_ms,
+ )
+ .await;
+ let rate_limiter = build_rate_limiter(config, shadow.clone()).await?;
+ let shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>> =
+ shadow.map(|client| -> Arc<dyn RedisClientTrait + Send + Sync> { client });
let rate_limiter_enabled_team_ids =
parse_team_id_allowlist(&config.error_tracking_rate_limiter_enabled_team_ids);
let drop_code_variables_team_ids = Arc::new(parse_strict_team_id_list(
@@ -195,6 +209,7 @@ impl AppContext {
process_request_limiter,
team_manager,
issue_buckets_redis_client,
+ shadow_redis_client,
issue_buckets_heal_gate: HealGate::new(),
rate_limiter,
rate_limiter_enabled_team_ids,
@@ -209,6 +224,7 @@ impl AppContext {
async fn build_rate_limiter(
config: &ProcessingConfig,
+ shadow: Option<Arc<RedisClient>>,
) -> Result<Option<Arc<RedisRateLimiter>>, UnhandledError> {
if !config.error_tracking_rate_limiter_enabled {
return Ok(None);
@@ -235,11 +251,16 @@ async fn build_rate_limiter(
info!("Error-tracking rate limiter enabled");
- Ok(Some(Arc::new(RedisRateLimiter::new(
- Arc::new(client),
- config.error_tracking_rate_limiter_key_prefix.clone(),
- config.error_tracking_rate_limiter_bucket_ttl_seconds,
- ))))
+ let shadow: Option<Arc<dyn common_redis::ScriptRunner>> =
+ shadow.map(|client| -> Arc<dyn common_redis::ScriptRunner> { client });
+ Ok(Some(Arc::new(
+ RedisRateLimiter::new(
+ Arc::new(client),
+ config.error_tracking_rate_limiter_key_prefix.clone(),
+ config.error_tracking_rate_limiter_bucket_ttl_seconds,
+ )
+ .with_shadow(shadow),
+ )))
}
/// Parse a comma-separated team-id allowlist. `None` (empty input) means the
diff --git a/rust/cymbal/src/modes/processing/config.rs b/rust/cymbal/src/modes/processing/config.rs
index 68b051e43398..04046c9b2b64 100644
--- a/rust/cymbal/src/modes/processing/config.rs
+++ b/rust/cymbal/src/modes/processing/config.rs
@@ -148,6 +148,9 @@ pub struct ProcessingConfig {
#[envconfig(from = "ISSUE_BUCKETS_REDIS_URL", default = "redis://localhost:6379/")]
pub issue_buckets_redis_url: String,
+ #[envconfig(from = "ERROR_TRACKING_PROCESSING_SHADOW_REDIS_URL")]
+ pub shadow_redis_url: Option<String>,
+
#[envconfig(
from = "ERROR_TRACKING_EVENT_PROPERTIES_TTL_SECONDS",
default = "172800"
diff --git a/rust/cymbal/src/modes/processing/issue_resolution.rs b/rust/cymbal/src/modes/processing/issue_resolution.rs
index be0fec793d90..abe8af2599f5 100644
--- a/rust/cymbal/src/modes/processing/issue_resolution.rs
+++ b/rust/cymbal/src/modes/processing/issue_resolution.rs
@@ -500,8 +500,11 @@ pub async fn send_issue_created_notification(
let fingerprint = processed_properties.fingerprint().to_string();
store_error_tracking_event_properties(
&*context.issue_buckets_redis_client,
- context.config.event_properties_ttl_seconds,
- context.config.event_properties_max_bytes,
+ context.shadow_redis_client.as_ref(),
+ EventPropertiesLimits {
+ ttl_seconds: context.config.event_properties_ttl_seconds,
+ max_bytes: context.config.event_properties_max_bytes,
+ },
issue.team_id,
event_uuid,
"issue_created",
@@ -527,10 +530,15 @@ fn error_tracking_event_properties_key(team_id: i32, event_uuid: Uuid) -> String
format!("{ERROR_TRACKING_EVENT_PROPERTIES_KEY_PREFIX}:{team_id}:{event_uuid}")
}
-async fn store_error_tracking_event_properties(
- redis: &(dyn common_redis::Client + Send + Sync),
+struct EventPropertiesLimits {
ttl_seconds: u64,
max_bytes: usize,
+}
+
+async fn store_error_tracking_event_properties(
+ redis: &(dyn common_redis::Client + Send + Sync),
+ shadow: Option<&std::sync::Arc<dyn common_redis::Client + Send + Sync>>,
+ limits: EventPropertiesLimits,
team_id: i32,
event_uuid: Uuid,
notification_type: &'static str,
@@ -548,14 +556,15 @@ async fn store_error_tracking_event_properties(
metrics::histogram!(ISSUE_CREATED_EVENT_PROPERTIES_BYTES, "notification_type" => notification_type)
.record(payload.len() as f64);
- if payload.len() > max_bytes {
+ if payload.len() > limits.max_bytes {
metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORE_SKIPPED, "reason" => "payload_too_large", "notification_type" => notification_type)
.increment(1);
return;
}
+ let shadow_payload = shadow.map(|_| payload.clone());
if let Err(error) = redis
- .set_bytes(key.clone(), payload, Some(ttl_seconds))
+ .set_bytes(key.clone(), payload, Some(limits.ttl_seconds))
.await
{
metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORE_FAILED, "reason" => "redis", "notification_type" => notification_type)
@@ -564,6 +573,9 @@ async fn store_error_tracking_event_properties(
return;
}
+ if let Some(payload) = shadow_payload {
+ crate::modes::shadow_redis::set_bytes(shadow, key, payload, limits.ttl_seconds);
+ }
metrics::counter!(ISSUE_CREATED_EVENT_PROPERTIES_STORED, "notification_type" => notification_type)
.increment(1);
}
@@ -578,8 +590,11 @@ pub async fn send_issue_reopened_notification(
) -> Result<(), UnhandledError> {
store_error_tracking_event_properties(
&*context.issue_buckets_redis_client,
- context.config.event_properties_ttl_seconds,
- context.config.event_properties_max_bytes,
+ context.shadow_redis_client.as_ref(),
+ EventPropertiesLimits {
+ ttl_seconds: context.config.event_properties_ttl_seconds,
+ max_bytes: context.config.event_properties_max_bytes,
+ },
issue.team_id,
event_uuid,
"issue_reopened",
@@ -610,8 +625,11 @@ pub async fn send_issue_spiking_notification(
) -> Result<(), UnhandledError> {
store_error_tracking_event_properties(
&*context.issue_buckets_redis_client,
- context.config.event_properties_ttl_seconds,
- context.config.event_properties_max_bytes,
+ context.shadow_redis_client.as_ref(),
+ EventPropertiesLimits {
+ ttl_seconds: context.config.event_properties_ttl_seconds,
+ max_bytes: context.config.event_properties_max_bytes,
+ },
issue.team_id,
event_uuid,
"issue_spiking",
@@ -814,46 +832,71 @@ mod test {
#[tokio::test]
async fn stores_full_event_properties_as_raw_json_with_a_ttl() {
- let redis = MockRedisClient::new();
- let event_uuid = Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap();
- let properties_json = serde_json::json!({
- "$exception_list": [{"type": "Error", "value": "boom 💥"}],
- "$exception_fingerprint": "abc",
- "$exception_fingerprint_record": [{"type": "manual"}],
- "$exception_issue_id": Uuid::nil(),
- "$exception_handled": false,
- "$exception_types": ["Error"],
- "$exception_values": ["boom 💥"],
- "$exception_sources": [],
- "$exception_functions": [],
- "custom": {"nested": [1, null, true]},
- });
- let properties: ProcessedExceptionProperties =
- serde_json::from_value(properties_json.clone()).unwrap();
-
- super::store_error_tracking_event_properties(
- &redis,
- 172_800,
- 1_048_576,
- 42,
- event_uuid,
- "issue_created",
- &properties,
- )
- .await;
-
- let calls = redis.get_calls();
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].op, "set_bytes");
- match &calls[0].value {
- MockRedisValue::Bytes(payload, Some(172_800)) => {
- assert_eq!(
- serde_json::from_slice::<serde_json::Value>(payload).unwrap(),
- properties_json
- );
+ crate::modes::shadow_redis::with_test_circuit(async {
+ let redis = MockRedisClient::new();
+ let shadow = std::sync::Arc::new(MockRedisClient::new());
+ let shadow_client: std::sync::Arc<dyn common_redis::Client + Send + Sync> =
+ shadow.clone();
+ let event_uuid = Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap();
+ let properties_json = serde_json::json!({
+ "$exception_list": [{"type": "Error", "value": "boom 💥"}],
+ "$exception_fingerprint": "abc",
+ "$exception_fingerprint_record": [{"type": "manual"}],
+ "$exception_issue_id": Uuid::nil(),
+ "$exception_handled": false,
+ "$exception_types": ["Error"],
+ "$exception_values": ["boom 💥"],
+ "$exception_sources": [],
+ "$exception_functions": [],
+ "custom": {"nested": [1, null, true]},
+ });
+ let properties: ProcessedExceptionProperties =
+ serde_json::from_value(properties_json.clone()).unwrap();
+
+ super::store_error_tracking_event_properties(
+ &redis,
+ Some(&shadow_client),
+ super::EventPropertiesLimits {
+ ttl_seconds: 172_800,
+ max_bytes: 1_048_576,
+ },
+ 42,
+ event_uuid,
+ "issue_created",
+ &properties,
+ )
+ .await;
+
+ let calls = redis.get_calls();
+ assert_eq!(calls.len(), 1);
+ assert_eq!(calls[0].op, "set_bytes");
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while shadow.get_calls().is_empty() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ let shadow_calls = shadow.get_calls();
+ assert_eq!(shadow_calls.len(), 1);
+ assert_eq!(shadow_calls[0].op, calls[0].op);
+ assert_eq!(shadow_calls[0].key, calls[0].key);
+ assert!(matches!(
+ (&shadow_calls[0].value, &calls[0].value),
+ (MockRedisValue::Bytes(shadow_bytes, shadow_ttl), MockRedisValue::Bytes(bytes, ttl))
+ if shadow_bytes == bytes && shadow_ttl == ttl
+ ));
+ match &calls[0].value {
+ MockRedisValue::Bytes(payload, Some(172_800)) => {
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(payload).unwrap(),
+ properties_json
+ );
+ }
+ value => panic!("unexpected Redis value: {value:?}"),
}
- value => panic!("unexpected Redis value: {value:?}"),
- }
+ })
+ .await;
}
#[tokio::test]
@@ -876,8 +919,11 @@ mod test {
super::store_error_tracking_event_properties(
&redis,
- 172_800,
- 128,
+ None,
+ super::EventPropertiesLimits {
+ ttl_seconds: 172_800,
+ max_bytes: 128,
+ },
42,
Uuid::parse_str("01982721-5e00-7000-8000-000000000001").unwrap(),
"issue_created",
diff --git a/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs b/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs
index 8ea8ac234016..4f7385a14eaa 100644
--- a/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs
+++ b/rust/cymbal/src/modes/processing/stages/alerting/spike_detection.rs
@@ -92,6 +92,7 @@ fn get_now_rounded_to_minutes(minutes: i64) -> String {
async fn try_increment_issue_buckets(
heal_gate: &HealGate,
redis: &Arc<dyn Client + Send + Sync>,
+ shadow: Option<&Arc<dyn Client + Send + Sync>>,
issue_counts: &HashMap<Uuid, u32>,
) {
if issue_counts.is_empty() {
@@ -109,12 +110,26 @@ async fn try_increment_issue_buckets(
})
.collect();
+ let shadow_items = shadow.map(|_| items.clone());
if let Err(err) = redis
.batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)
.await
{
warn!("Failed to increment issue buckets batch: {err}");
heal_on_connection_error(heal_gate, redis, &err);
+ } else if let Some(shadow) = shadow {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ let items = shadow_items.expect("shadow items exist when shadow is configured");
+ crate::modes::shadow_redis::submit(
+ "issue_buckets",
+ async move {
+ shadow
+ .batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)
+ .await
+ },
+ async move { healing.reconnect().await },
+ );
}
}
@@ -123,6 +138,7 @@ async fn try_increment_team_buckets(
redis: &Arc<dyn Client + Send + Sync>,
issues_by_id: &HashMap<Uuid, Issue>,
issue_counts: &HashMap<Uuid, u32>,
+ shadow: Option<&Arc<dyn Client + Send + Sync>>,
) {
if issue_counts.is_empty() {
return;
@@ -151,12 +167,26 @@ async fn try_increment_team_buckets(
})
.collect();
+ let shadow_items = shadow.map(|_| items.clone());
if let Err(err) = redis
.batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)
.await
{
warn!("Failed to increment team buckets batch: {err}");
heal_on_connection_error(heal_gate, redis, &err);
+ } else if let Some(shadow) = shadow {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ let items = shadow_items.expect("shadow items exist when shadow is configured");
+ crate::modes::shadow_redis::submit(
+ "team_buckets",
+ async move {
+ shadow
+ .batch_incr_by_expire_nx(items, ISSUE_BUCKET_TTL_SECONDS)
+ .await
+ },
+ async move { healing.reconnect().await },
+ );
}
// Track unique issues per team bucket using sets
@@ -171,12 +201,26 @@ async fn try_increment_team_buckets(
})
.collect();
+ let shadow_items = shadow.map(|_| issue_set_items.clone());
if let Err(err) = redis
.batch_sadd_expire(issue_set_items, ISSUE_BUCKET_TTL_SECONDS)
.await
{
warn!("Failed to add issues to team sets: {err}");
heal_on_connection_error(heal_gate, redis, &err);
+ } else if let Some(shadow) = shadow {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ let issue_set_items = shadow_items.expect("shadow items exist when shadow is configured");
+ crate::modes::shadow_redis::submit(
+ "team_issue_sets",
+ async move {
+ shadow
+ .batch_sadd_expire(issue_set_items, ISSUE_BUCKET_TTL_SECONDS)
+ .await
+ },
+ async move { healing.reconnect().await },
+ );
}
}
@@ -218,6 +262,7 @@ pub async fn do_spike_detection(
try_increment_issue_buckets(
&context.issue_buckets_heal_gate,
&context.issue_buckets_redis_client,
+ context.shadow_redis_client.as_ref(),
&issue_counts,
)
.await;
@@ -229,6 +274,7 @@ pub async fn do_spike_detection(
&context.issue_buckets_redis_client,
&issues_by_id,
&issue_counts,
+ context.shadow_redis_client.as_ref(),
)
.await;
team_buckets_timer.fin();
@@ -275,6 +321,51 @@ async fn acquire_cooldown_locks(
redis.batch_set_nx_ex(batch).await
}
+async fn acquire_cooldown_locks_mirrored(
+ redis: &(dyn Client + Send + Sync),
+ shadow: Option<&Arc<dyn Client + Send + Sync>>,
+ items: &[(String, usize)],
+) -> Result<Vec<bool>, common_redis::CustomRedisError> {
+ let result = acquire_cooldown_locks(redis, items).await?;
+ if let Some(shadow) = shadow {
+ let granted: Vec<_> = items
+ .iter()
+ .zip(&result)
+ .filter(|(_, acquired)| **acquired)
+ .map(|(item, _)| item.clone())
+ .collect();
+ if !granted.is_empty() {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ crate::modes::shadow_redis::submit(
+ "cooldown_acquire",
+ async move { acquire_cooldown_locks(&*shadow, &granted).await },
+ async move { healing.reconnect().await },
+ );
+ }
+ }
+ Ok(result)
+}
+
+async fn release_cooldown_locks_mirrored(
+ redis: &(dyn Client + Send + Sync),
+ shadow: Option<&Arc<dyn Client + Send + Sync>>,
+ keys: Vec<String>,
+) -> Result<(), common_redis::CustomRedisError> {
+ let shadow_keys = shadow.map(|_| keys.clone());
+ redis.batch_del(keys).await?;
+ if let (Some(shadow), Some(keys)) = (shadow, shadow_keys) {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ crate::modes::shadow_redis::submit(
+ "cooldown_release",
+ async move { shadow.batch_del(keys).await },
+ async move { healing.reconnect().await },
+ );
+ }
+ Ok(())
+}
+
async fn emit_spiking_events(
context: &AppContext,
spiking: Vec<SpikingIssue>,
@@ -296,15 +387,20 @@ async fn emit_spiking_events(
})
.unzip();
- let lock_results =
- match acquire_cooldown_locks(&*context.issue_buckets_redis_client, &cooldown_items).await {
- Ok(results) => results,
- Err(e) => {
- locks_timer.fin();
- warn!("Failed to acquire spike cooldown locks: {e}");
- return;
- }
- };
+ let lock_results = match acquire_cooldown_locks_mirrored(
+ &*context.issue_buckets_redis_client,
+ context.shadow_redis_client.as_ref(),
+ &cooldown_items,
+ )
+ .await
+ {
+ Ok(results) => results,
+ Err(e) => {
+ locks_timer.fin();
+ warn!("Failed to acquire spike cooldown locks: {e}");
+ return;
+ }
+ };
let blocked_count = lock_results.iter().filter(|&&acquired| !acquired).count();
metrics::counter!(SPIKE_ISSUES_BLOCKED_BY_COOLDOWN).increment(blocked_count as u64);
@@ -340,10 +436,12 @@ async fn emit_spiking_events(
}
if !failed_keys.is_empty() {
- if let Err(e) = context
- .issue_buckets_redis_client
- .batch_del(failed_keys)
- .await
+ if let Err(e) = release_cooldown_locks_mirrored(
+ &*context.issue_buckets_redis_client,
+ context.shadow_redis_client.as_ref(),
+ failed_keys,
+ )
+ .await
{
warn!("Failed to release cooldown locks after Kafka failure: {e}");
}
@@ -556,6 +654,113 @@ mod tests {
use chrono::TimeZone;
use common_redis::MockRedisClient;
+ #[tokio::test]
+ async fn mirrors_only_cooldown_locks_granted_by_primary() {
+ crate::modes::shadow_redis::with_test_circuit(async {
+ let denied = "cooldown/denied".to_string();
+ let granted = "cooldown/granted".to_string();
+ let mut primary_mock = MockRedisClient::new();
+ primary_mock.set_nx_ex_ret(&denied, Ok(false));
+ primary_mock.set_nx_ex_ret(&granted, Ok(true));
+ let shadow = Arc::new(MockRedisClient::new());
+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();
+ let items = vec![(denied, 600), (granted.clone(), 600)];
+
+ let result = acquire_cooldown_locks_mirrored(
+ &primary_mock,
+ Some(&shadow_client),
+ &items,
+ )
+ .await
+ .unwrap();
+ assert_eq!(result, vec![false, true]);
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while shadow.get_calls().is_empty() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ let calls = shadow.get_calls();
+ assert_eq!(calls.len(), 1);
+ assert!(matches!(&calls[0].value, common_redis::MockRedisValue::VecString(keys) if keys == &vec![granted]));
+ })
+ .await;
+ }
+
+ #[tokio::test]
+ async fn does_not_mirror_refused_cooldown_locks() {
+ crate::modes::shadow_redis::with_test_circuit(async {
+ let primary = MockRedisClient::new();
+ let shadow = Arc::new(MockRedisClient::new());
+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();
+ let items = vec![("cooldown/denied".to_string(), 600)];
+ let result = acquire_cooldown_locks_mirrored(&primary, Some(&shadow_client), &items)
+ .await
+ .unwrap();
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(result, vec![false]);
+ assert!(shadow.get_calls().is_empty());
+ })
+ .await;
+ }
+
+ #[tokio::test]
+ async fn mirrors_spike_counters_sets_and_cooldowns() {
+ let ctx = TestContext::new();
+ let key = cooldown_key(&ctx.issue_id);
+ let mut primary_mock = MockRedisClient::new();
+ primary_mock.set_nx_ex_ret(&key, Ok(true));
+ let primary: Arc<dyn Client + Send + Sync> = Arc::new(primary_mock);
+ let shadow = Arc::new(MockRedisClient::new());
+ let shadow_client: Arc<dyn Client + Send + Sync> = shadow.clone();
+ let counts = HashMap::from([(ctx.issue_id, 2)]);
+ let gate = HealGate::new();
+ try_increment_issue_buckets(&gate, &primary, Some(&shadow_client), &counts).await;
+ try_increment_team_buckets(
+ &gate,
+ &primary,
+ &ctx.issues_by_id(),
+ &counts,
+ Some(&shadow_client),
+ )
+ .await;
+ let locks = vec![(key.clone(), 3600)];
+ let primary_result =
+ acquire_cooldown_locks_mirrored(&*primary, Some(&shadow_client), &locks)
+ .await
+ .unwrap();
+ let keys = vec![cooldown_key(&ctx.issue_id)];
+ release_cooldown_locks_mirrored(&*primary, Some(&shadow_client), keys.clone())
+ .await
+ .unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while shadow.get_calls().len() < 5 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(primary_result, vec![true]);
+ let ops = shadow
+ .get_calls()
+ .into_iter()
+ .map(|call| call.op)
+ .collect::<Vec<_>>();
+ assert_eq!(
+ ops.iter()
+ .filter(|op| op.as_str() == "batch_incr_by_expire_nx")
+ .count(),
+ 2
+ );
+ assert!(ops.contains(&"batch_sadd_expire".to_string()));
+ assert!(ops.contains(&"batch_set_nx_ex".to_string()));
+ assert!(ops.contains(&"batch_del".to_string()));
+ assert!(shadow.get_calls().iter().any(|call| matches!(&call.value, common_redis::MockRedisValue::VecString(shadow_keys) if call.op == "batch_del" && shadow_keys == &keys)));
+ }
+
fn bytes(v: i64) -> Vec<u8> {
v.to_string().into_bytes()
}
diff --git a/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs b/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs
index 2a241037d135..c3682d8af211 100644
--- a/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs
+++ b/rust/cymbal/src/modes/processing/stages/rate_limiting/limiter.rs
@@ -4,8 +4,8 @@ use chrono::Utc;
use common_redis::{CustomRedisError, ScriptRunner};
use uuid::Uuid;
-use crate::modes::processing::redis_heal::HealGate;
use crate::modes::processing::rules::rate_limit::BucketParams;
+use crate::modes::{processing::redis_heal::HealGate, shadow_redis};
/// Outcome of one fused rate-limit call for a single issue group of `n` events.
/// `issue_admitted <= n` passed the per-issue limit; `team_admitted <= issue_admitted`
@@ -72,6 +72,7 @@ return { issue_admitted, team_admitted }
pub struct RedisRateLimiter {
redis: Arc<dyn ScriptRunner>,
+ shadow: Option<Arc<dyn ScriptRunner>>,
key_prefix: String,
bucket_ttl_seconds: u64,
heal_gate: HealGate,
@@ -81,12 +82,18 @@ impl RedisRateLimiter {
pub fn new(redis: Arc<dyn ScriptRunner>, key_prefix: String, bucket_ttl_seconds: u64) -> Self {
Self {
redis,
+ shadow: None,
key_prefix,
bucket_ttl_seconds,
heal_gate: HealGate::new(),
}
}
+ pub fn with_shadow(mut self, shadow: Option<Arc<dyn ScriptRunner>>) -> Self {
+ self.shadow = shadow;
+ self
+ }
+
fn issue_key(&self, team_id: i32, issue_id: Option<Uuid>) -> String {
match issue_id {
Some(id) => format!("{}/{{{team_id}}}/issue/{id}", self.key_prefix),
@@ -136,7 +143,17 @@ impl RedisRateLimiter {
self.bucket_ttl_seconds.to_string(),
];
+ let shadow_request = self.shadow.as_ref().map(|_| (keys.clone(), args.clone()));
let res = self.redis.eval_int_vec(RATE_LIMIT_LUA, keys, args).await?;
+ if let (Some(shadow), Some((keys, args))) = (&self.shadow, shadow_request) {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ shadow_redis::submit(
+ "processing_rate_limit",
+ async move { shadow.eval_int_vec(RATE_LIMIT_LUA, keys, args).await },
+ async move { healing.reconnect().await },
+ );
+ }
// Defensive clamps: team_admitted <= issue_admitted <= n.
let issue_admitted = (res.first().copied().unwrap_or(0).max(0) as u32).min(n);
@@ -164,7 +181,47 @@ impl RedisRateLimiter {
#[cfg(test)]
mod tests {
use super::*;
- use crate::test_support::{start_redis, RedisContainer};
+ use crate::test_support::{start_redis, FakeRunner, RedisContainer};
+
+ #[tokio::test]
+ async fn primary_failure_does_not_charge_the_shadow() {
+ let shadow = FakeRunner::returning(vec![1, 1]);
+ let limiter = RedisRateLimiter::new(FakeRunner::failing(), "test".into(), 3600)
+ .with_shadow(Some(shadow.clone()));
+ assert!(limiter.admit(42, None, None, None, 1).await.is_err());
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(shadow.call_count(), 0);
+ }
+
+ #[tokio::test]
+ async fn shadow_failure_does_not_change_the_primary_admission() {
+ shadow_redis::with_test_circuit(async {
+ let shadow = FakeRunner::failing();
+ let primary = FakeRunner::returning(vec![3, 2]);
+ let limiter = RedisRateLimiter::new(primary.clone(), "test".into(), 3600)
+ .with_shadow(Some(shadow.clone()));
+ let decision = limiter.admit(42, None, None, None, 5).await.unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while shadow.call_count() == 0 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(primary.requests(), shadow.requests());
+ assert_eq!(
+ decision,
+ RateLimitDecision {
+ issue_admitted: 3,
+ team_admitted: 2
+ }
+ );
+ })
+ .await;
+ }
+
use common_redis::RedisClient;
// ===================================================================
diff --git a/rust/cymbal/src/modes/shadow_redis.rs b/rust/cymbal/src/modes/shadow_redis.rs
new file mode 100644
index 000000000000..6766267dc873
--- /dev/null
+++ b/rust/cymbal/src/modes/shadow_redis.rs
@@ -0,0 +1,507 @@
+use std::{
+ future::Future,
+ sync::{
+ atomic::{AtomicBool, AtomicU64, Ordering},
+ Arc, Mutex, OnceLock,
+ },
+ time::{Duration, Instant},
+};
+
+use tokio::sync::Semaphore;
+
+use common_redis::{Client, CompressionConfig, CustomRedisError, RedisClient, RedisValueFormat};
+use tracing::warn;
+
+use crate::modes::processing::redis_heal::{is_connection_error, HealGate};
+
+const SHADOW_DEADLINE: Duration = Duration::from_millis(200);
+const MAX_IN_FLIGHT_SHADOW_WRITES: usize = 128;
+const SHADOW_RECOVERY_INTERVAL: Duration = Duration::from_secs(5);
+const SHADOW_RECOVERY_DEADLINE: Duration = Duration::from_secs(2);
+const SHADOW_WARN_INTERVAL_MS: u64 = 30_000;
+static SHADOW_PERMITS: OnceLock<Arc<Semaphore>> = OnceLock::new();
+static SHADOW_CIRCUIT: OnceLock<Arc<ShadowCircuit>> = OnceLock::new();
+static SHADOW_HEAL_GATE: OnceLock<HealGate> = OnceLock::new();
+static WARN_EPOCH: OnceLock<Instant> = OnceLock::new();
+static LAST_SHADOW_WARNING_MS: AtomicU64 = AtomicU64::new(0);
+
+#[cfg(test)]
+tokio::task_local! {
+ static TEST_SHADOW_CIRCUIT: Arc<ShadowCircuit>;
+}
+
+#[cfg(test)]
+pub(crate) async fn with_test_circuit<T>(future: impl Future<Output = T>) -> T {
+ TEST_SHADOW_CIRCUIT
+ .scope(Arc::new(ShadowCircuit::new()), future)
+ .await
+}
+
+fn shadow_circuit() -> Arc<ShadowCircuit> {
+ #[cfg(test)]
+ if let Ok(circuit) = TEST_SHADOW_CIRCUIT.try_with(Arc::clone) {
+ return circuit;
+ }
+ SHADOW_CIRCUIT
+ .get_or_init(|| Arc::new(ShadowCircuit::new()))
+ .clone()
+}
+
+pub async fn connect(
+ url: Option<&str>,
+ primary_urls: &[&str],
+ format: RedisValueFormat,
+ response_timeout_ms: u64,
+ connection_timeout_ms: u64,
+) -> Option<Arc<RedisClient>> {
+ let url = url.filter(|url| !url.trim().is_empty())?;
+ if primary_urls
+ .iter()
+ .any(|primary| same_endpoint(url, primary))
+ {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "same_endpoint").increment(1);
+ warn!("shadow Redis URL matches a primary Redis URL; shadow writes disabled");
+ return None;
+ }
+ match RedisClient::with_config(
+ url.to_string(),
+ CompressionConfig::disabled(),
+ format,
+ Some(Duration::from_millis(shadow_timeout_ms(
+ response_timeout_ms,
+ 200,
+ ))),
+ Some(Duration::from_millis(shadow_timeout_ms(
+ connection_timeout_ms,
+ 1000,
+ ))),
+ )
+ .await
+ {
+ Ok(client) => {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "ok").increment(1);
+ Some(Arc::new(client))
+ }
+ Err(error) => {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "connect", "outcome" => "error").increment(1);
+ warn!(%error, "shadow Redis connection unavailable; primary remains active");
+ None
+ }
+ }
+}
+
+fn shadow_timeout_ms(configured: u64, cap: u64) -> u64 {
+ if configured == 0 {
+ cap
+ } else {
+ configured.min(cap)
+ }
+}
+
+fn same_endpoint(shadow: &str, primary: &str) -> bool {
+ shadow == primary
+ || matches!((redis_endpoint(shadow), redis_endpoint(primary)), (Some(a), Some(b)) if a == b)
+}
+
+#[derive(PartialEq, Eq)]
+enum RedisEndpoint {
+ Tcp(String, u16, i64),
+ Unix(std::path::PathBuf, i64),
+}
+
+fn redis_endpoint(url: &str) -> Option<RedisEndpoint> {
+ let client = redis::Client::open(url).ok()?;
+ let info = client.get_connection_info();
+ let (host, port) = match &info.addr {
+ redis::ConnectionAddr::Tcp(host, port) => (host, *port),
+ redis::ConnectionAddr::TcpTls { host, port, .. } => (host, *port),
+ redis::ConnectionAddr::Unix(path) => {
+ return Some(RedisEndpoint::Unix(path.clone(), info.redis.db));
+ }
+ };
+ let host = host.trim_end_matches('.').to_ascii_lowercase();
+ let host = match host.as_str() {
+ "localhost" | "::1" | "[::1]" => "127.0.0.1".to_string(),
+ _ => host,
+ };
+ Some(RedisEndpoint::Tcp(host, port, info.redis.db))
+}
+
+fn should_warn(last: &AtomicU64, now_ms: u64) -> bool {
+ let now = now_ms.saturating_add(1);
+ let previous = last.load(Ordering::Relaxed);
+ (previous == 0 || now.saturating_sub(previous) >= SHADOW_WARN_INTERVAL_MS)
+ && last
+ .compare_exchange(previous, now, Ordering::Relaxed, Ordering::Relaxed)
+ .is_ok()
+}
+
+fn warn_for_shadow_failure() -> bool {
+ let elapsed_ms = WARN_EPOCH.get_or_init(Instant::now).elapsed().as_millis() as u64;
+ should_warn(&LAST_SHADOW_WARNING_MS, elapsed_ms)
+}
+
+struct RecoveryState {
+ generation: u64,
+ next_attempt: Instant,
+}
+
+// A timed-out request can stay queued in redis-rs. Block writes until a new connection replaces it.
+struct ShadowCircuit {
+ blocked: AtomicBool,
+ generation: AtomicU64,
+ recovery: Mutex<RecoveryState>,
+}
+
+impl ShadowCircuit {
+ fn new() -> Self {
+ Self {
+ blocked: AtomicBool::new(false),
+ generation: AtomicU64::new(0),
+ recovery: Mutex::new(RecoveryState {
+ generation: 0,
+ next_attempt: Instant::now(),
+ }),
+ }
+ }
+
+ fn trip(self: &Arc<Self>, generation: u64, heal: impl Future<Output = bool> + Send + 'static) {
+ let mut recovery = self.recovery.lock().unwrap();
+ if recovery.generation != generation {
+ return;
+ }
+ self.blocked.store(true, Ordering::Release);
+ self.schedule_recovery(&mut recovery, heal);
+ }
+
+ fn retry(self: &Arc<Self>, heal: impl Future<Output = bool> + Send + 'static) {
+ let mut recovery = self.recovery.lock().unwrap();
+ self.schedule_recovery(&mut recovery, heal);
+ }
+
+ fn schedule_recovery(
+ self: &Arc<Self>,
+ recovery: &mut RecoveryState,
+ heal: impl Future<Output = bool> + Send + 'static,
+ ) {
+ if Instant::now() < recovery.next_attempt {
+ return;
+ }
+ recovery.next_attempt = Instant::now() + SHADOW_RECOVERY_INTERVAL;
+ let circuit = self.clone();
+ tokio::spawn(async move {
+ if let Ok(true) = tokio::time::timeout(SHADOW_RECOVERY_DEADLINE, heal).await {
+ let mut recovery = circuit.recovery.lock().unwrap();
+ recovery.generation += 1;
+ circuit
+ .generation
+ .store(recovery.generation, Ordering::Release);
+ circuit.blocked.store(false, Ordering::Release);
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "reconnect", "outcome" => "ok").increment(1);
+ } else {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => "reconnect", "outcome" => "error").increment(1);
+ }
+ });
+ }
+}
+
+pub fn submit<T: Send + 'static>(
+ operation: &'static str,
+ write: impl Future<Output = Result<T, CustomRedisError>> + Send + 'static,
+ heal: impl Future<Output = bool> + Send + 'static,
+) {
+ let permits =
+ SHADOW_PERMITS.get_or_init(|| Arc::new(Semaphore::new(MAX_IN_FLIGHT_SHADOW_WRITES)));
+ let circuit = shadow_circuit();
+ submit_with_permits(permits, &circuit, operation, write, heal);
+}
+
+fn submit_with_permits<T: Send + 'static>(
+ permits: &Arc<Semaphore>,
+ circuit: &Arc<ShadowCircuit>,
+ operation: &'static str,
+ write: impl Future<Output = Result<T, CustomRedisError>> + Send + 'static,
+ heal: impl Future<Output = bool> + Send + 'static,
+) {
+ if circuit.blocked.load(Ordering::Acquire) {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "circuit_open").increment(1);
+ circuit.retry(heal);
+ return;
+ }
+ let generation = circuit.generation.load(Ordering::Acquire);
+ let Ok(permit) = permits.clone().try_acquire_owned() else {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "dropped").increment(1);
+ return;
+ };
+ let circuit = circuit.clone();
+ tokio::spawn(async move {
+ let _permit = permit;
+ match tokio::time::timeout(SHADOW_DEADLINE, write).await {
+ Ok(Ok(_)) => metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "ok").increment(1),
+ Ok(Err(CustomRedisError::Timeout)) | Err(_) => {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "timeout").increment(1);
+ if warn_for_shadow_failure() {
+ warn!(operation, "shadow Redis write timed out");
+ }
+ circuit.trip(generation, heal);
+ }
+ Ok(Err(error)) => {
+ metrics::counter!("cymbal_shadow_redis_operations_total", "operation" => operation, "outcome" => "error").increment(1);
+ if warn_for_shadow_failure() {
+ warn!(%error, operation, "shadow Redis write failed");
+ }
+ if is_connection_error(&error) {
+ SHADOW_HEAL_GATE.get_or_init(HealGate::new).spawn_heal(async move {
+ let _ = heal.await;
+ });
+ }
+ }
+ }
+ });
+}
+
+pub fn set_bytes(
+ shadow: Option<&Arc<dyn Client + Send + Sync>>,
+ key: String,
+ payload: Vec<u8>,
+ ttl_seconds: u64,
+) {
+ if let Some(shadow) = shadow {
+ let shadow = shadow.clone();
+ let healing = shadow.clone();
+ submit(
+ "event_properties",
+ async move { shadow.set_bytes(key, payload, Some(ttl_seconds)).await },
+ async move { healing.reconnect().await },
+ );
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use metrics_util::debugging::DebuggingRecorder;
+
+ #[tokio::test]
+ async fn exact_primary_url_cannot_be_shadowed() {
+ let shadow = connect(
+ Some("redis://localhost:6379/"),
+ &["redis://localhost:6379/"],
+ RedisValueFormat::Utf8,
+ 100,
+ 100,
+ )
+ .await;
+ assert!(shadow.is_none());
+ }
+
+ #[test]
+ fn equivalent_primary_urls_cannot_be_shadowed() {
+ assert!(same_endpoint(
+ "redis://primary-host:6379/0",
+ "redis://primary-host:6379"
+ ));
+ assert!(same_endpoint(
+ "redis://localhost",
+ "redis://127.0.0.1:6379/"
+ ));
+ assert!(same_endpoint(
+ "redis+unix:///var/run/redis/redis.sock",
+ "unix:///var/run/redis/redis.sock"
+ ));
+ assert!(same_endpoint(
+ "rediss://PRIMARY-HOST:6379/",
+ "redis://primary-host:6379/0"
+ ));
+ assert!(!same_endpoint(
+ "redis://primary-host:6379/1",
+ "redis://primary-host:6379/0"
+ ));
+ assert!(!same_endpoint(
+ "redis://new-host:6379",
+ "redis://primary-host:6379"
+ ));
+ }
+
+ #[test]
+ fn zero_primary_timeout_uses_bounded_shadow_timeout() {
+ assert_eq!(shadow_timeout_ms(0, 200), 200);
+ assert_eq!(shadow_timeout_ms(0, 1000), 1000);
+ assert_eq!(shadow_timeout_ms(100, 200), 100);
+ assert_eq!(shadow_timeout_ms(5000, 1000), 1000);
+ }
+
+ #[test]
+ fn repeated_failures_do_not_repeat_warnings() {
+ let last = AtomicU64::new(0);
+ assert!(should_warn(&last, 0));
+ assert!(!should_warn(&last, 1));
+ assert!(!should_warn(&last, SHADOW_WARN_INTERVAL_MS - 1));
+ assert!(should_warn(&last, SHADOW_WARN_INTERVAL_MS));
+ }
+
+ #[tokio::test]
+ async fn timeout_blocks_new_writes_until_reconnect_succeeds() {
+ let permits = Arc::new(Semaphore::new(2));
+ let circuit = Arc::new(ShadowCircuit::new());
+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+ let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
+ submit_with_permits(
+ &permits,
+ &circuit,
+ "test_timeout",
+ std::future::pending::<Result<(), CustomRedisError>>(),
+ async move {
+ let _ = started_tx.send(());
+ let _ = resume_rx.await;
+ true
+ },
+ );
+ tokio::time::timeout(Duration::from_secs(1), started_rx)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(circuit.blocked.load(Ordering::Acquire));
+
+ let called = Arc::new(AtomicBool::new(false));
+ let rejected = called.clone();
+ submit_with_permits(
+ &permits,
+ &circuit,
+ "test_blocked",
+ async move {
+ rejected.store(true, Ordering::Release);
+ Ok::<(), CustomRedisError>(())
+ },
+ async { true },
+ );
+ assert!(!called.load(Ordering::Acquire));
+
+ resume_tx.send(()).unwrap();
+ tokio::time::timeout(Duration::from_secs(1), async {
+ while circuit.blocked.load(Ordering::Acquire) {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(circuit.generation.load(Ordering::Acquire), 1);
+ }
+
+ #[tokio::test]
+ async fn failed_reconnect_keeps_shadow_circuit_open() {
+ let permits = Arc::new(Semaphore::new(1));
+ let circuit = Arc::new(ShadowCircuit::new());
+ let (tried_tx, tried_rx) = tokio::sync::oneshot::channel();
+ submit_with_permits(
+ &permits,
+ &circuit,
+ "test_failure",
+ async { Err::<(), _>(CustomRedisError::Timeout) },
+ async move {
+ let _ = tried_tx.send(());
+ false
+ },
+ );
+ tokio::time::timeout(Duration::from_secs(1), tried_rx)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(circuit.blocked.load(Ordering::Acquire));
+ circuit.recovery.lock().unwrap().next_attempt = Instant::now();
+ let (recovered_tx, recovered_rx) = tokio::sync::oneshot::channel();
+ submit_with_permits(
+ &permits,
+ &circuit,
+ "test_retry",
+ async { Ok::<(), CustomRedisError>(()) },
+ async move {
+ let _ = recovered_tx.send(());
+ true
+ },
+ );
+ tokio::time::timeout(Duration::from_secs(1), recovered_rx)
+ .await
+ .unwrap()
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(1), async {
+ while circuit.blocked.load(Ordering::Acquire) {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ }
+
+ #[test]
+ fn client_timeout_records_timeout_instead_of_error() {
+ let recorder = DebuggingRecorder::new();
+ let circuit = Arc::new(ShadowCircuit::new());
+ let permits = Arc::new(Semaphore::new(1));
+ metrics::with_local_recorder(&recorder, || {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ submit_with_permits(
+ &permits,
+ &circuit,
+ "test_client_timeout",
+ async { Err::<(), _>(CustomRedisError::Timeout) },
+ async { false },
+ );
+ tokio::time::timeout(Duration::from_secs(1), async {
+ while !circuit.blocked.load(Ordering::Acquire) {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ });
+ });
+ let counts = recorder.snapshotter().snapshot().into_vec();
+ assert!(counts.iter().any(|(key, _, _, _)| {
+ key.key().name() == "cymbal_shadow_redis_operations_total"
+ && key.key().labels().any(|label| {
+ label.key() == "operation" && label.value() == "test_client_timeout"
+ })
+ && key
+ .key()
+ .labels()
+ .any(|label| label.key() == "outcome" && label.value() == "timeout")
+ }));
+ assert!(!counts.iter().any(|(key, _, _, _)| {
+ key.key().name() == "cymbal_shadow_redis_operations_total"
+ && key.key().labels().any(|label| {
+ label.key() == "operation" && label.value() == "test_client_timeout"
+ })
+ && key
+ .key()
+ .labels()
+ .any(|label| label.key() == "outcome" && label.value() == "error")
+ }));
+ }
+
+ #[test]
+ fn saturated_shadow_capacity_records_dropped_write() {
+ let recorder = DebuggingRecorder::new();
+ metrics::with_local_recorder(&recorder, || {
+ submit_with_permits(
+ &Arc::new(Semaphore::new(0)),
+ &Arc::new(ShadowCircuit::new()),
+ "issue_buckets",
+ async { Ok::<(), CustomRedisError>(()) },
+ async { true },
+ );
+ });
+ let snapshot = recorder.snapshotter().snapshot().into_vec();
+ assert!(snapshot.iter().any(|(key, _, _, _)| {
+ key.key().name() == "cymbal_shadow_redis_operations_total"
+ && key
+ .key()
+ .labels()
+ .any(|label| label.key() == "outcome" && label.value() == "dropped")
+ }));
+ }
+}
diff --git a/rust/cymbal/src/test_support.rs b/rust/cymbal/src/test_support.rs
index 7db7882b722f..a7b57869d241 100644
--- a/rust/cymbal/src/test_support.rs
+++ b/rust/cymbal/src/test_support.rs
@@ -51,18 +51,35 @@ pub async fn start_redis() -> (Arc<RedisClient>, RedisContainer) {
panic!("redis container never became ready");
}
+pub type ScriptRequest = (String, Vec<String>, Vec<String>);
+
/// A [`ScriptRunner`] that never touches Redis: a canned reply, or an error.
pub struct FakeRunner {
reply: Option<Vec<i64>>,
+ calls: std::sync::Mutex<Vec<ScriptRequest>>,
}
impl FakeRunner {
pub fn returning(reply: Vec<i64>) -> Arc<Self> {
- Arc::new(Self { reply: Some(reply) })
+ Arc::new(Self {
+ reply: Some(reply),
+ calls: std::sync::Mutex::new(Vec::new()),
+ })
}
pub fn failing() -> Arc<Self> {
- Arc::new(Self { reply: None })
+ Arc::new(Self {
+ reply: None,
+ calls: std::sync::Mutex::new(Vec::new()),
+ })
+ }
+
+ pub fn call_count(&self) -> usize {
+ self.requests().len()
+ }
+
+ pub fn requests(&self) -> Vec<ScriptRequest> {
+ self.calls.lock().unwrap().clone()
}
}
@@ -70,10 +87,14 @@ impl FakeRunner {
impl ScriptRunner for FakeRunner {
async fn eval_int_vec(
&self,
- _script: &str,
- _keys: Vec<String>,
- _args: Vec<String>,
+ script: &str,
+ keys: Vec<String>,
+ args: Vec<String>,
) -> Result<Vec<i64>, CustomRedisError> {
+ self.calls
+ .lock()
+ .unwrap()
+ .push((script.to_string(), keys, args));
self.reply.clone().ok_or(CustomRedisError::Timeout)
}
}