Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(pathlock): observe Redis acquisition latency and slow calls (#6722)
zhanglistare3768cccfd08
diff --git a/crates/ragfs/src/lock/cache_provder.rs b/crates/ragfs/src/lock/cache_provder.rsindex 47fbd49de..fb8697d09 100644--- a/crates/ragfs/src/lock/cache_provder.rs+++ b/crates/ragfs/src/lock/cache_provder.rs@@ -2,10 +2,14 @@//! system scope; global roots cannot atomically cover the separate scope HASHes.use std::collections::BTreeMap;+use std::sync::atomic::{AtomicU64, Ordering};use std::sync::Arc;+use std::time::{Duration, Instant};+use crate::metrics::duration::DurationMetrics;use async_trait::async_trait;use bytes::Bytes;+use opentelemetry::KeyValue;use crate::cache_runtime::{CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue,@@ -31,6 +35,8 @@ const IS_LOCKED_ID: &str = "pathlock.is_locked.v1";const SCAN_DESCENDANTS_ID: &str = "pathlock.scan_descendants.v1";const READ_MANY_ID: &str = "pathlock.read_many.v1";const CREATE_ID: &str = "pathlock.create.v1";+// Observe requests approaching the default 20ms Redis command timeout.+const SLOW_ACQUIRE_THRESHOLD: Duration = Duration::from_millis(10);macro_rules! token_script {($body:literal) => {@@ -502,6 +508,8 @@ pub struct RedisPathLockProvider {runtime: Arc<CacheRuntime>,service_name: String,hash_ttl_ms: i64,+ acquire_duration: DurationMetrics,+ slow_acquisitions: AtomicU64,}impl RedisPathLockProvider {@@ -544,6 +552,8 @@ impl RedisPathLockProvider {runtime,service_name: service_name.to_string(),hash_ttl_ms: hash_ttl_ms as i64,+ acquire_duration: DurationMetrics::default(),+ slow_acquisitions: AtomicU64::new(0),})}@@ -801,6 +811,11 @@ impl RedisPathLockProvider {#[async_trait]impl PathLockProvider for RedisPathLockProvider {+ fn update_metrics(&self, metrics: &mut super::metrics::LockMetrics) {+ metrics.redis_acquire = Some(self.acquire_duration.clone());+ metrics.redis_slow_acquisitions = self.slow_acquisitions.load(Ordering::Relaxed);+ }+/// Return the provider name used by configuration and diagnostics.fn name(&self) -> &'static str {"cache"@@ -852,7 +867,51 @@ impl PathLockProvider for RedisPathLockProvider {args.push(Self::bytes(&request.path));args.push(Self::bytes(Self::kind_code(request.kind)));}- let values = self.execute_array(ACQUIRE_BATCH_ID, key, args).await?;+ let started = Instant::now();+ let response = self+ .execute_array(ACQUIRE_BATCH_ID, key.clone(), args)+ .await;+ let elapsed = started.elapsed();+ let result = match &response {+ Ok(values) => match Self::result_status(values, "acquire_batch") {+ Ok("ok") => "ok",+ Ok("conflict") => "conflict",+ Ok("invalid_token") => "invalid_token",+ Ok("invalid_request") => "invalid_request",+ _ => "invalid_response",+ },+ Err(_) => "error",+ };+ let includes_tree = requests+ .iter()+ .any(|request| request.kind == PathLockKind::Tree);+ self.acquire_duration.record(+ elapsed,+ &[+ KeyValue::new("kind", if includes_tree { "tree" } else { "exact" }),+ KeyValue::new("result", result),+ ],+ );+ if elapsed > SLOW_ACQUIRE_THRESHOLD {+ self.slow_acquisitions.fetch_add(1, Ordering::Relaxed);+ let scope = key+ .split_once(":ov:pathlock:")+ .and_then(|(_, suffix)| suffix.strip_suffix(":tokens"))+ .unwrap_or(&key);+ tracing::warn!(+ script_id = ACQUIRE_BATCH_ID,+ service_name = %self.service_name,+ scope,+ request_count = requests.len(),+ includes_tree,+ elapsed_ms = elapsed.as_secs_f64() * 1_000.0,+ threshold_ms = SLOW_ACQUIRE_THRESHOLD.as_millis() as u64,+ result,+ error = ?response.as_ref().err(),+ "slow Redis pathlock acquire"+ );+ }+ let values = response?;let status = Self::result_status(&values, "acquire_batch")?;if status != "ok" {return Err(Self::decode_error_result("acquire_batch", &values)?);@@ -1231,6 +1290,102 @@ return redis.call("HGET", KEYS[1], ARGV[2])Conflict,}+ #[tokio::test]+ async fn redis_acquire_metrics_export_outcomes_and_slow_calls() {+ let calls = AtomicU64::new(0);+ let mock = crate::cache_runtime::MemoryMockProvider::new().with_script_handler(move |_| {+ // Inject real response latency so the slow counter follows the timed call.+ std::thread::sleep(Duration::from_millis(15));+ let values = match calls.fetch_add(1, Ordering::Relaxed) {+ 0 => vec!["ok", "0", "/local/account-a/file", "reentrant", "", ""],+ 1 => vec!["conflict", "/local/account-a/file", "other:100:E"],+ 2 => return Err(CacheError::Unavailable("injected failure".into())),+ _ => panic!("unexpected Redis call"),+ };+ crate::cache_runtime::ScriptResult::encode(&ScriptValue::Array(+ values+ .into_iter()+ .map(|value| ScriptValue::Bytes(value.as_bytes().to_vec()))+ .collect(),+ ))+ });+ let runtime = CacheRuntime::memory_with_provider(Arc::new(mock));+ let provider =+ Arc::new(RedisPathLockProvider::new(runtime.clone(), "metrics-test", 30.0).unwrap());+ assert!(provider+ .try_acquire_batch_atomic(&[], "owner", 100, 0)+ .await+ .is_err());+ for (index, kind) in [PathLockKind::Exact, PathLockKind::Tree, PathLockKind::Exact]+ .into_iter()+ .enumerate()+ {+ let result = provider+ .try_acquire_batch_atomic(+ &[PathLockRequest {+ path: "/local/account-a/file".into(),+ kind,+ }],+ "owner",+ 100,+ 0,+ )+ .await;+ match index {+ 0 => assert!(result.is_ok()),+ 1 => assert!(matches!(result, Err(PathLockError::Conflict { .. }))),+ _ => assert!(matches!(result, Err(PathLockError::Io(_)))),+ }+ }+ let manager = PathLockManager::new(+ Arc::new(MemFileSystem::new()),+ provider,+ PathLockConfig::default(),+ );+ // Repeated exports must retain cumulative observations.+ for _ in 0..2 {+ let metrics = crate::metrics::lock_metrics(manager.metrics_snapshot().await);+ let slow = metrics+ .iter()+ .find(|metric| metric.name == "lock_redis_slow_acquisitions_total")+ .unwrap();+ assert!(matches!(+ slow.value,+ crate::metrics::RagfsMetricValue::Counter { value: 3, .. }+ ));+ let durations = metrics+ .iter()+ .filter(|metric| metric.name == "lock_redis_acquire_duration_seconds")+ .collect::<Vec<_>>();+ assert_eq!(durations.len(), 3);+ for (kind, result) in [("exact", "ok"), ("tree", "conflict"), ("exact", "error")] {+ let metric = durations+ .iter()+ .find(|metric| {+ metric.labels["kind"] == kind && metric.labels["result"] == result+ })+ .unwrap();+ assert_eq!(metric.labels.len(), 2);+ match &metric.value {+ crate::metrics::RagfsMetricValue::Histogram {+ count,+ sum,+ bucket_counts,+ scale,+ ..+ } => {+ assert_eq!(*count, 1);+ assert_eq!(bucket_counts.iter().sum::<u64>(), 1);+ assert!(*sum > 10_000_000);+ assert_eq!(*scale, 1e-9);+ }+ _ => panic!("expected duration histogram"),+ }+ }+ }+ runtime.close().await.unwrap();+ }+/// Logical test provider for atomic Manager failure paths.struct FaultProvider {inner: super::super::provider::MemoryPathLockProvider,diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rsindex a45faea89..5e5b729b1 100644--- a/crates/ragfs/src/lock/manager.rs+++ b/crates/ragfs/src/lock/manager.rs@@ -2380,6 +2380,7 @@ impl PathLockManager {metrics.active_lock_count = self.lease_registry.active_count();metrics.waiting_lock_count = self.waiting_lock_count.load(AtomicOrdering::Relaxed);metrics.wait_duration_ns = self.wait_duration_ns.load(AtomicOrdering::Relaxed);+ self.provider.update_metrics(&mut metrics);metrics}}diff --git a/crates/ragfs/src/lock/metrics.rs b/crates/ragfs/src/lock/metrics.rsindex b768e6436..c3aaec3f9 100644--- a/crates/ragfs/src/lock/metrics.rs+++ b/crates/ragfs/src/lock/metrics.rs@@ -21,6 +21,10 @@ pub struct LockMetrics {pub stale_cleanup_failures: usize,/// SDK-owned descendant scan duration distribution.pub(crate) descendant_scan: DurationMetrics,+ /// SDK-owned Redis acquisition round-trip durations, when using Redis.+ pub(crate) redis_acquire: Option<DurationMetrics>,+ /// Redis acquisitions exceeding the slow-log threshold.+ pub redis_slow_acquisitions: u64,/// Total nanoseconds spent waiting for successfully acquired locks.pub wait_duration_ns: u64,/// Count of conflicts by type (exact/exact, tree/exact, etc.).diff --git a/crates/ragfs/src/lock/provider.rs b/crates/ragfs/src/lock/provider.rsindex 52121d971..4d9e1a7d1 100644--- a/crates/ragfs/src/lock/provider.rs+++ b/crates/ragfs/src/lock/provider.rs@@ -65,6 +65,9 @@ pub struct AtomicAcquisitionResult {/// Storage abstraction for lock tokens. Only `PathLockManager` should call these methods.#[async_trait]pub trait PathLockProvider: Send + Sync {+ /// Add provider-owned observations to the manager's metrics snapshot.+ fn update_metrics(&self, _metrics: &mut super::metrics::LockMetrics) {}+/// Human-readable provider name for config/logging/metrics.fn name(&self) -> &'static str;diff --git a/crates/ragfs/src/metrics/mod.rs b/crates/ragfs/src/metrics/mod.rsindex 97602e028..d7fdec7c5 100644--- a/crates/ragfs/src/metrics/mod.rs+++ b/crates/ragfs/src/metrics/mod.rs@@ -129,10 +129,34 @@ pub(crate) fn cache_metrics(stats: crate::cache::CacheMetricsSnapshot) -> Vec<Rametrics}-/// Convert the single lock manager's snapshot into unlabeled metric records.+/// Convert the single lock manager's snapshot into bounded metric records.pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {let mut metrics = Vec::new();let scan = stats.descendant_scan.snapshot().into_iter().next();+ if let Some(acquire) = &stats.redis_acquire {+ metrics.push(RagfsMetric::counter(+ "lock_redis_slow_acquisitions_total",+ &[],+ stats.redis_slow_acquisitions,+ 1.0,+ ));+ for point in acquire.snapshot() {+ metrics.push(RagfsMetric {+ name: "lock_redis_acquire_duration_seconds".into(),+ labels: point+ .attributes()+ .map(|attribute| (attribute.key.to_string(), attribute.value.to_string()))+ .collect(),+ value: RagfsMetricValue::Histogram {+ bucket_bounds: point.bounds().map(|bound| bound as u64).collect(),+ bucket_counts: point.bucket_counts().collect(),+ count: point.count(),+ sum: point.sum(),+ scale: 1e-9,+ },+ });+ }+ }for (name, value) in [("lock_active", stats.active_lock_count),("lock_waiting", stats.waiting_lock_count),diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.mdindex e52b98590..644db585e 100644--- a/docs/en/concepts/12-metrics.md+++ b/docs/en/concepts/12-metrics.md@@ -286,6 +286,12 @@ Notes:| `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration || `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | completed descendant scan duration distribution, supporting mean, P95, and P99 |+| `openviking_lock_redis_acquire_duration_seconds` | Histogram | `kind, result, bucket_schema` | Redis acquisition round trip including response decoding; `kind` is `tree` if the batch includes Tree, otherwise `exact`; `result` is `ok`, `conflict`, `invalid_token`, `invalid_request`, `invalid_response`, or `error` |+| `openviking_lock_redis_slow_acquisitions_total` | Counter | none | Redis acquisition round trips exceeding 10ms, including failed requests |++Redis acquisition metrics cover returned script calls, excluding local validation,+manager conflict waiting and calls cancelled before a response. Scope, owner and+service name are retained in slow logs, not metric labels.The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.mdindex d07ee5cbd..73363b27c 100644--- a/docs/zh/concepts/12-metrics.md+++ b/docs/zh/concepts/12-metrics.md@@ -282,6 +282,11 @@ Telemetry bridge 还识别旧摘要键,对应 `request`、`process`、`parse`| `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 || `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | 已完成后代锁扫描的耗时分布,可计算平均、P95 和 P99 |+| `openviking_lock_redis_acquire_duration_seconds` | Histogram | `kind, result, bucket_schema` | Redis 获取往返耗时,包含响应解码;batch 包含 Tree 时 `kind=tree`,否则为 `exact`;`result` 为 `ok`、`conflict`、`invalid_token`、`invalid_request`、`invalid_response` 或 `error` |+| `openviking_lock_redis_slow_acquisitions_total` | Counter | 无 | Redis 获取往返超过 10ms 的累计次数,包含失败请求 |++Redis 获取指标统计已返回的脚本调用,不包含本地校验、manager 冲突等待,+以及响应前被取消的调用。scope、owner 和服务名保留在慢日志中,不作为指标标签。队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。