Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(pathlock): observe Redis acquisition latency and slow calls (#6722)

zhanglistare3768cccfd08
diff --git a/crates/ragfs/src/lock/cache_provder.rs b/crates/ragfs/src/lock/cache_provder.rs
index 47fbd49de..fb8697d09 100644
--- a/crates/ragfs/src/lock/cache_provder.rs
+++ b/crates/ragfs/src/lock/cache_provder.rs
@@ -2,10 +2,14 @@
//! system scope; global roots cannot atomically cover the separate scope HASHes.
use std::collections::BTreeMap;
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
+use std::time::{Duration, Instant};
+use crate::metrics::duration::DurationMetrics;
use async_trait::async_trait;
use bytes::Bytes;
+use opentelemetry::KeyValue;
use crate::cache_runtime::{
CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue,
@@ -31,6 +35,8 @@ const IS_LOCKED_ID: &str = "pathlock.is_locked.v1";
const SCAN_DESCENDANTS_ID: &str = "pathlock.scan_descendants.v1";
const READ_MANY_ID: &str = "pathlock.read_many.v1";
const CREATE_ID: &str = "pathlock.create.v1";
+// Observe requests approaching the default 20ms Redis command timeout.
+const SLOW_ACQUIRE_THRESHOLD: Duration = Duration::from_millis(10);
macro_rules! token_script {
($body:literal) => {
@@ -502,6 +508,8 @@ pub struct RedisPathLockProvider {
runtime: Arc<CacheRuntime>,
service_name: String,
hash_ttl_ms: i64,
+ acquire_duration: DurationMetrics,
+ slow_acquisitions: AtomicU64,
}
impl RedisPathLockProvider {
@@ -544,6 +552,8 @@ impl RedisPathLockProvider {
runtime,
service_name: service_name.to_string(),
hash_ttl_ms: hash_ttl_ms as i64,
+ acquire_duration: DurationMetrics::default(),
+ slow_acquisitions: AtomicU64::new(0),
})
}
@@ -801,6 +811,11 @@ impl RedisPathLockProvider {
#[async_trait]
impl PathLockProvider for RedisPathLockProvider {
+ fn update_metrics(&self, metrics: &mut super::metrics::LockMetrics) {
+ metrics.redis_acquire = Some(self.acquire_duration.clone());
+ metrics.redis_slow_acquisitions = self.slow_acquisitions.load(Ordering::Relaxed);
+ }
+
/// Return the provider name used by configuration and diagnostics.
fn name(&self) -> &'static str {
"cache"
@@ -852,7 +867,51 @@ impl PathLockProvider for RedisPathLockProvider {
args.push(Self::bytes(&request.path));
args.push(Self::bytes(Self::kind_code(request.kind)));
}
- let values = self.execute_array(ACQUIRE_BATCH_ID, key, args).await?;
+ let started = Instant::now();
+ let response = self
+ .execute_array(ACQUIRE_BATCH_ID, key.clone(), args)
+ .await;
+ let elapsed = started.elapsed();
+ let result = match &response {
+ Ok(values) => match Self::result_status(values, "acquire_batch") {
+ Ok("ok") => "ok",
+ Ok("conflict") => "conflict",
+ Ok("invalid_token") => "invalid_token",
+ Ok("invalid_request") => "invalid_request",
+ _ => "invalid_response",
+ },
+ Err(_) => "error",
+ };
+ let includes_tree = requests
+ .iter()
+ .any(|request| request.kind == PathLockKind::Tree);
+ self.acquire_duration.record(
+ elapsed,
+ &[
+ KeyValue::new("kind", if includes_tree { "tree" } else { "exact" }),
+ KeyValue::new("result", result),
+ ],
+ );
+ if elapsed > SLOW_ACQUIRE_THRESHOLD {
+ self.slow_acquisitions.fetch_add(1, Ordering::Relaxed);
+ let scope = key
+ .split_once(":ov:pathlock:")
+ .and_then(|(_, suffix)| suffix.strip_suffix(":tokens"))
+ .unwrap_or(&key);
+ tracing::warn!(
+ script_id = ACQUIRE_BATCH_ID,
+ service_name = %self.service_name,
+ scope,
+ request_count = requests.len(),
+ includes_tree,
+ elapsed_ms = elapsed.as_secs_f64() * 1_000.0,
+ threshold_ms = SLOW_ACQUIRE_THRESHOLD.as_millis() as u64,
+ result,
+ error = ?response.as_ref().err(),
+ "slow Redis pathlock acquire"
+ );
+ }
+ let values = response?;
let status = Self::result_status(&values, "acquire_batch")?;
if status != "ok" {
return Err(Self::decode_error_result("acquire_batch", &values)?);
@@ -1231,6 +1290,102 @@ return redis.call("HGET", KEYS[1], ARGV[2])
Conflict,
}
+ #[tokio::test]
+ async fn redis_acquire_metrics_export_outcomes_and_slow_calls() {
+ let calls = AtomicU64::new(0);
+ let mock = crate::cache_runtime::MemoryMockProvider::new().with_script_handler(move |_| {
+ // Inject real response latency so the slow counter follows the timed call.
+ std::thread::sleep(Duration::from_millis(15));
+ let values = match calls.fetch_add(1, Ordering::Relaxed) {
+ 0 => vec!["ok", "0", "/local/account-a/file", "reentrant", "", ""],
+ 1 => vec!["conflict", "/local/account-a/file", "other:100:E"],
+ 2 => return Err(CacheError::Unavailable("injected failure".into())),
+ _ => panic!("unexpected Redis call"),
+ };
+ crate::cache_runtime::ScriptResult::encode(&ScriptValue::Array(
+ values
+ .into_iter()
+ .map(|value| ScriptValue::Bytes(value.as_bytes().to_vec()))
+ .collect(),
+ ))
+ });
+ let runtime = CacheRuntime::memory_with_provider(Arc::new(mock));
+ let provider =
+ Arc::new(RedisPathLockProvider::new(runtime.clone(), "metrics-test", 30.0).unwrap());
+ assert!(provider
+ .try_acquire_batch_atomic(&[], "owner", 100, 0)
+ .await
+ .is_err());
+ for (index, kind) in [PathLockKind::Exact, PathLockKind::Tree, PathLockKind::Exact]
+ .into_iter()
+ .enumerate()
+ {
+ let result = provider
+ .try_acquire_batch_atomic(
+ &[PathLockRequest {
+ path: "/local/account-a/file".into(),
+ kind,
+ }],
+ "owner",
+ 100,
+ 0,
+ )
+ .await;
+ match index {
+ 0 => assert!(result.is_ok()),
+ 1 => assert!(matches!(result, Err(PathLockError::Conflict { .. }))),
+ _ => assert!(matches!(result, Err(PathLockError::Io(_)))),
+ }
+ }
+ let manager = PathLockManager::new(
+ Arc::new(MemFileSystem::new()),
+ provider,
+ PathLockConfig::default(),
+ );
+ // Repeated exports must retain cumulative observations.
+ for _ in 0..2 {
+ let metrics = crate::metrics::lock_metrics(manager.metrics_snapshot().await);
+ let slow = metrics
+ .iter()
+ .find(|metric| metric.name == "lock_redis_slow_acquisitions_total")
+ .unwrap();
+ assert!(matches!(
+ slow.value,
+ crate::metrics::RagfsMetricValue::Counter { value: 3, .. }
+ ));
+ let durations = metrics
+ .iter()
+ .filter(|metric| metric.name == "lock_redis_acquire_duration_seconds")
+ .collect::<Vec<_>>();
+ assert_eq!(durations.len(), 3);
+ for (kind, result) in [("exact", "ok"), ("tree", "conflict"), ("exact", "error")] {
+ let metric = durations
+ .iter()
+ .find(|metric| {
+ metric.labels["kind"] == kind && metric.labels["result"] == result
+ })
+ .unwrap();
+ assert_eq!(metric.labels.len(), 2);
+ match &metric.value {
+ crate::metrics::RagfsMetricValue::Histogram {
+ count,
+ sum,
+ bucket_counts,
+ scale,
+ ..
+ } => {
+ assert_eq!(*count, 1);
+ assert_eq!(bucket_counts.iter().sum::<u64>(), 1);
+ assert!(*sum > 10_000_000);
+ assert_eq!(*scale, 1e-9);
+ }
+ _ => panic!("expected duration histogram"),
+ }
+ }
+ }
+ runtime.close().await.unwrap();
+ }
+
/// Logical test provider for atomic Manager failure paths.
struct FaultProvider {
inner: super::super::provider::MemoryPathLockProvider,
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index a45faea89..5e5b729b1 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -2380,6 +2380,7 @@ impl PathLockManager {
metrics.active_lock_count = self.lease_registry.active_count();
metrics.waiting_lock_count = self.waiting_lock_count.load(AtomicOrdering::Relaxed);
metrics.wait_duration_ns = self.wait_duration_ns.load(AtomicOrdering::Relaxed);
+ self.provider.update_metrics(&mut metrics);
metrics
}
}
diff --git a/crates/ragfs/src/lock/metrics.rs b/crates/ragfs/src/lock/metrics.rs
index b768e6436..c3aaec3f9 100644
--- a/crates/ragfs/src/lock/metrics.rs
+++ b/crates/ragfs/src/lock/metrics.rs
@@ -21,6 +21,10 @@ pub struct LockMetrics {
pub stale_cleanup_failures: usize,
/// SDK-owned descendant scan duration distribution.
pub(crate) descendant_scan: DurationMetrics,
+ /// SDK-owned Redis acquisition round-trip durations, when using Redis.
+ pub(crate) redis_acquire: Option<DurationMetrics>,
+ /// Redis acquisitions exceeding the slow-log threshold.
+ pub redis_slow_acquisitions: u64,
/// Total nanoseconds spent waiting for successfully acquired locks.
pub wait_duration_ns: u64,
/// Count of conflicts by type (exact/exact, tree/exact, etc.).
diff --git a/crates/ragfs/src/lock/provider.rs b/crates/ragfs/src/lock/provider.rs
index 52121d971..4d9e1a7d1 100644
--- a/crates/ragfs/src/lock/provider.rs
+++ b/crates/ragfs/src/lock/provider.rs
@@ -65,6 +65,9 @@ pub struct AtomicAcquisitionResult {
/// Storage abstraction for lock tokens. Only `PathLockManager` should call these methods.
#[async_trait]
pub trait PathLockProvider: Send + Sync {
+ /// Add provider-owned observations to the manager's metrics snapshot.
+ fn update_metrics(&self, _metrics: &mut super::metrics::LockMetrics) {}
+
/// Human-readable provider name for config/logging/metrics.
fn name(&self) -> &'static str;
diff --git a/crates/ragfs/src/metrics/mod.rs b/crates/ragfs/src/metrics/mod.rs
index 97602e028..d7fdec7c5 100644
--- a/crates/ragfs/src/metrics/mod.rs
+++ b/crates/ragfs/src/metrics/mod.rs
@@ -129,10 +129,34 @@ pub(crate) fn cache_metrics(stats: crate::cache::CacheMetricsSnapshot) -> Vec<Ra
metrics
}
-/// Convert the single lock manager's snapshot into unlabeled metric records.
+/// Convert the single lock manager's snapshot into bounded metric records.
pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {
let mut metrics = Vec::new();
let scan = stats.descendant_scan.snapshot().into_iter().next();
+ if let Some(acquire) = &stats.redis_acquire {
+ metrics.push(RagfsMetric::counter(
+ "lock_redis_slow_acquisitions_total",
+ &[],
+ stats.redis_slow_acquisitions,
+ 1.0,
+ ));
+ for point in acquire.snapshot() {
+ metrics.push(RagfsMetric {
+ name: "lock_redis_acquire_duration_seconds".into(),
+ labels: point
+ .attributes()
+ .map(|attribute| (attribute.key.to_string(), attribute.value.to_string()))
+ .collect(),
+ value: RagfsMetricValue::Histogram {
+ bucket_bounds: point.bounds().map(|bound| bound as u64).collect(),
+ bucket_counts: point.bucket_counts().collect(),
+ count: point.count(),
+ sum: point.sum(),
+ scale: 1e-9,
+ },
+ });
+ }
+ }
for (name, value) in [
("lock_active", stats.active_lock_count),
("lock_waiting", stats.waiting_lock_count),
diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md
index e52b98590..644db585e 100644
--- a/docs/en/concepts/12-metrics.md
+++ b/docs/en/concepts/12-metrics.md
@@ -286,6 +286,12 @@ Notes:
| `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration |
| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | completed descendant scan duration distribution, supporting mean, P95, and P99 |
+| `openviking_lock_redis_acquire_duration_seconds` | Histogram | `kind, result, bucket_schema` | Redis acquisition round trip including response decoding; `kind` is `tree` if the batch includes Tree, otherwise `exact`; `result` is `ok`, `conflict`, `invalid_token`, `invalid_request`, `invalid_response`, or `error` |
+| `openviking_lock_redis_slow_acquisitions_total` | Counter | none | Redis acquisition round trips exceeding 10ms, including failed requests |
+
+Redis acquisition metrics cover returned script calls, excluding local validation,
+manager conflict waiting and calls cancelled before a response. Scope, owner and
+service name are retained in slow logs, not metric labels.
The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.
diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md
index d07ee5cbd..73363b27c 100644
--- a/docs/zh/concepts/12-metrics.md
+++ b/docs/zh/concepts/12-metrics.md
@@ -282,6 +282,11 @@ Telemetry bridge 还识别旧摘要键,对应 `request`、`process`、`parse`
| `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 |
| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | 已完成后代锁扫描的耗时分布,可计算平均、P95 和 P99 |
+| `openviking_lock_redis_acquire_duration_seconds` | Histogram | `kind, result, bucket_schema` | Redis 获取往返耗时,包含响应解码;batch 包含 Tree 时 `kind=tree`,否则为 `exact`;`result` 为 `ok`、`conflict`、`invalid_token`、`invalid_request`、`invalid_response` 或 `error` |
+| `openviking_lock_redis_slow_acquisitions_total` | Counter | 无 | Redis 获取往返超过 10ms 的累计次数,包含失败请求 |
+
+Redis 获取指标统计已返回的脚本调用,不包含本地校验、manager 冲突等待,
+以及响应前被取消的调用。scope、owner 和服务名保留在慢日志中,不作为指标标签。
队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。