Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(metrics): track queue processing latency (#5365)
zhanggaoyuana78c61297697
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rsindex 67dac8898..73407d708 100644--- a/crates/ragfs/src/plugins/queuefs/backend.rs+++ b/crates/ragfs/src/plugins/queuefs/backend.rs@@ -4,7 +4,7 @@//! storage implementations (memory, SQLite, etc.) while maintaining a consistent interface.use crate::core::errors::{Error, Result};-use chrono::{DateTime, Utc};+use chrono::{DateTime, SecondsFormat, Utc};use rusqlite::{params, types::ValueRef, Connection, Row};use serde::{Deserialize, Deserializer, Serialize};use std::collections::{HashMap, VecDeque};@@ -22,6 +22,8 @@ pub struct Message {pub data: Vec<u8>,/// Timestamp when the message was enqueuedpub timestamp: SystemTime,+ /// Whether `timestamp` came from a real persisted enqueue time.+ pub timestamp_valid: bool,}impl Message {@@ -31,6 +33,7 @@ impl Message {id: Uuid::new_v4().to_string(),data,timestamp: SystemTime::now(),+ timestamp_valid: true,}}}@@ -58,16 +61,21 @@ impl StoredMessage {Self {id: msg.id.clone(),data: msg.data.clone(),- // Prefer unix seconds for compatibility with older queue.db producers.- timestamp: Some(serde_json::Value::Number(unix_secs(msg.timestamp).into())),+ // Older readers already accept RFC3339 strings; nanoseconds preserve latency accuracy.+ timestamp: Some(serde_json::Value::String(+ DateTime::<Utc>::from(msg.timestamp)+ .to_rfc3339_opts(SecondsFormat::Nanos, true),+ )),}}pub(super) fn into_message(self) -> Message {+ let (timestamp, timestamp_valid) = parse_stored_timestamp(self.timestamp);Message {id: self.id,data: self.data,- timestamp: parse_stored_timestamp(self.timestamp),+ timestamp,+ timestamp_valid,}}}@@ -101,26 +109,28 @@ where}}-fn parse_stored_timestamp(raw: Option<serde_json::Value>) -> SystemTime {+fn parse_stored_timestamp(raw: Option<serde_json::Value>) -> (SystemTime, bool) {match raw {Some(serde_json::Value::String(ts)) => DateTime::parse_from_rfc3339(&ts)- .map(|dt| {- let secs = dt.timestamp();- let secs_u64 = if secs <= 0 { 0 } else { secs as u64 };- UNIX_EPOCH + Duration::from_secs(secs_u64)- })- .unwrap_or_else(|_| SystemTime::now()),+ .ok()+ .and_then(|dt| timestamp_from_parts(dt.timestamp(), dt.timestamp_subsec_nanos()))+ .map(|timestamp| (timestamp, true))+ .unwrap_or_else(|| (SystemTime::now(), false)),Some(serde_json::Value::Number(num)) => num.as_i64()- .map(|secs| {- let secs_u64 = if secs <= 0 { 0 } else { secs as u64 };- UNIX_EPOCH + Duration::from_secs(secs_u64)- })- .unwrap_or_else(SystemTime::now),- _ => SystemTime::now(),+ .and_then(|secs| timestamp_from_parts(secs, 0))+ .map(|timestamp| (timestamp, true))+ .unwrap_or_else(|| (SystemTime::now(), false)),+ _ => (SystemTime::now(), false),}}+fn timestamp_from_parts(secs: i64, nanos: u32) -> Option<SystemTime> {+ (secs > 0)+ .then(|| Duration::new(secs as u64, nanos))+ .and_then(|duration| UNIX_EPOCH.checked_add(duration))+}+fn unix_secs(time: SystemTime) -> i64 {time.duration_since(UNIX_EPOCH).unwrap_or_default()@@ -1081,6 +1091,67 @@ mod tests {assert_eq!(dequeued.data, payload);}+ #[test]+ fn test_stored_message_preserves_timestamp_validity() {+ let missing = StoredMessage {+ id: "missing".to_string(),+ data: b"payload".to_vec(),+ timestamp: None,+ }+ .into_message();+ assert!(!missing.timestamp_valid);++ let invalid = StoredMessage {+ id: "invalid".to_string(),+ data: b"payload".to_vec(),+ timestamp: Some(serde_json::Value::String("not-a-time".to_string())),+ }+ .into_message();+ assert!(!invalid.timestamp_valid);++ let valid = StoredMessage {+ id: "valid".to_string(),+ data: b"payload".to_vec(),+ timestamp: Some(serde_json::Value::Number(1234.into())),+ }+ .into_message();+ assert!(valid.timestamp_valid);+ assert_eq!(unix_secs(valid.timestamp), 1234);++ let precise_timestamp = UNIX_EPOCH + Duration::new(1_234, 567_890_123);+ let precise = StoredMessage::from_message(&Message {+ id: "precise".to_string(),+ data: b"payload".to_vec(),+ timestamp: precise_timestamp,+ timestamp_valid: true,+ })+ .into_message();+ assert!(precise.timestamp_valid);+ assert_eq!(precise.timestamp, precise_timestamp);+ }++ #[test]+ fn test_sqlite_backend_preserves_subsecond_enqueue_timestamp() {+ let (_dir, _db_path, mut backend) = sqlite_backend();+ backend.create_queue("test").unwrap();++ let timestamp = UNIX_EPOCH + Duration::new(1_700_000_000, 987_654_321);+ backend+ .enqueue(+ "test",+ Message {+ id: "precise".to_string(),+ data: b"payload".to_vec(),+ timestamp,+ timestamp_valid: true,+ },+ )+ .unwrap();++ let dequeued = backend.dequeue("test").unwrap().unwrap();+ assert_eq!(dequeued.timestamp, timestamp);+ }+#[test]fn test_sqlite_backend_recover_stale() {let (_dir, db_path_str, _backend) = sqlite_backend();diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rsindex bce789b6a..f3da2c4b3 100644--- a/crates/ragfs/src/plugins/queuefs/mod.rs+++ b/crates/ragfs/src/plugins/queuefs/mod.rs@@ -79,6 +79,17 @@ const CONTROL_FILES: &[ControlFileSpec] = &[struct QueueMessage {id: String,data: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ timestamp: Option<f64>,+}++fn queue_message_timestamp(msg: &Message) -> Option<f64> {+ msg.timestamp_valid.then(|| {+ msg.timestamp+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .as_secs_f64()+ })}#[derive(Debug, Clone, Copy)]@@ -374,9 +385,11 @@ impl FileSystem for QueueFileSystem {};// Return in Go libagfsbinding format: {"id": "...", "data": "..."}let data_str = String::from_utf8_lossy(&msg.data).to_string();+ let timestamp = queue_message_timestamp(&msg);let response = QueueMessage {id: msg.id,data: data_str,+ timestamp,};Ok(serde_json::to_vec(&response)?)}@@ -389,6 +402,7 @@ impl FileSystem for QueueFileSystem {let response = QueueMessage {id: msg.id.clone(),data: data_str,+ timestamp: queue_message_timestamp(&msg),};Ok(serde_json::to_vec(&response)?)}@@ -406,9 +420,13 @@ impl FileSystem for QueueFileSystem {.list_unacked(&queue_name).await?.into_iter()- .map(|msg| QueueMessage {+ .map(|msg| {+ let timestamp = queue_message_timestamp(&msg);+ QueueMessage {id: msg.id,data: String::from_utf8_lossy(&msg.data).to_string(),+ timestamp,+ }}).collect::<Vec<_>>();Ok(serde_json::to_vec(&messages)?)@@ -898,6 +916,18 @@ mod tests {struct TestQueueMessage {id: String,data: String,+ timestamp: Option<f64>,+ }++ #[test]+ fn test_queue_message_omits_missing_timestamp() {+ let response = QueueMessage {+ id: "legacy".to_string(),+ data: "payload".to_string(),+ timestamp: None,+ };+ let value = serde_json::to_value(response).unwrap();+ assert!(value.get("timestamp").is_none());}/// Create a queue filesystem with one initialized queue.@@ -940,10 +970,12 @@ mod tests {let msg1 = dequeue_msg(&fs, "test").await;assert!(!msg1.id.is_empty());assert_eq!(msg1.data.as_bytes(), data1);+ assert!(msg1.timestamp.is_some_and(|value| value > 0.0));let msg2 = dequeue_msg(&fs, "test").await;assert!(!msg2.id.is_empty());assert_eq!(msg2.data.as_bytes(), data2);+ assert!(msg2.timestamp.is_some_and(|value| value > 0.0));// Queue should be emptylet result = fs.read("/test/dequeue", 0, 0).await.unwrap();diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.mdindex 07191d558..1b3f088b4 100644--- a/docs/en/concepts/12-metrics.md+++ b/docs/en/concepts/12-metrics.md@@ -228,6 +228,8 @@ Notes:| `openviking_queue_errors_total` | Counter | `queue` | total error count per queue || `openviking_queue_pending` | Gauge | `queue` | pending queue items || `openviking_queue_in_progress` | Gauge | `queue` | in-progress queue items |+| `openviking_queue_process_duration_seconds` | Histogram | `queue, outcome` | handler execution time after a worker dequeues a message |+| `openviking_queue_end_to_end_duration_seconds` | Histogram | `queue, outcome` | enqueue-to-handler-completion latency, including queue wait time || `openviking_executor_max_workers` | Gauge | `pool, process_role, worker` | maximum workers in the asyncio default executor || `openviking_executor_threads` | Gauge | `pool, process_role, worker` | threads created by the asyncio default executor || `openviking_executor_active_tasks` | Gauge | `pool, process_role, worker` | default executor tasks currently running |@@ -243,9 +245,12 @@ Notes:| `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration |+The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.+These help answer:- Is there queue backlog?+- Which queue is slow, and is the latency caused by waiting or processing?- Is there lock contention or stale locking?- Is the default executor near its worker limit or building a queue?diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.mdindex 50c123003..ab6c40525 100644--- a/docs/zh/concepts/12-metrics.md+++ b/docs/zh/concepts/12-metrics.md@@ -231,6 +231,8 @@ scrape_configs:| `openviking_queue_errors_total` | Counter | `queue` | 队列累计错误量 || `openviking_queue_pending` | Gauge | `queue` | 队列待处理数 || `openviking_queue_in_progress` | Gauge | `queue` | 队列执行中数量 |+| `openviking_queue_process_duration_seconds` | Histogram | `queue, outcome` | 消息被 worker 取出后到 handler 结束的处理耗时 |+| `openviking_queue_end_to_end_duration_seconds` | Histogram | `queue, outcome` | 消息从入队到 handler 结束的端到端耗时,包含排队时间 || `openviking_executor_max_workers` | Gauge | `pool, process_role, worker` | asyncio 默认 executor 最大 worker 数 || `openviking_executor_threads` | Gauge | `pool, process_role, worker` | asyncio 默认 executor 已创建线程数 || `openviking_executor_active_tasks` | Gauge | `pool, process_role, worker` | 默认 executor 当前执行中的任务数 |@@ -246,9 +248,12 @@ scrape_configs:| `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 |+队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。+这些指标适合回答:- 是否有队列堆积?+- 哪个队列处理慢,以及耗时是否主要来自排队?- 是否有锁竞争或 stale lock?- 默认 executor 是否接近线程上限或出现排队?diff --git a/openviking/metrics/collectors/__init__.py b/openviking/metrics/collectors/__init__.pyindex 716fde7a9..b55a07e4b 100644--- a/openviking/metrics/collectors/__init__.py+++ b/openviking/metrics/collectors/__init__.py@@ -22,7 +22,7 @@ from .model_provider_probe import ModelProviderProbeCollectorfrom .model_usage import ModelUsageCollectorfrom .observer_health import ObserverHealthCollectorfrom .observer_state import ObserverStateCollector-from .queue import QueueCollector+from .queue import QueueCollector, QueueDurationCollectorfrom .ragfs import RagfsMetricCollectorfrom .rerank import RerankCollectorfrom .retrieval import RetrievalCollector@@ -47,6 +47,7 @@ __all__ = ["EncryptionCollector","FeedbackCollector","QueueCollector",+ "QueueDurationCollector","RerankCollector","RagfsMetricCollector","VikingDBCollector",diff --git a/openviking/metrics/collectors/queue.py b/openviking/metrics/collectors/queue.pyindex b819dbf07..e8b1ac417 100644--- a/openviking/metrics/collectors/queue.py+++ b/openviking/metrics/collectors/queue.py@@ -3,13 +3,91 @@from __future__ import annotations+import mathfrom dataclasses import dataclass, fieldfrom typing import ClassVarfrom openviking.metrics.core.base import MetricCollectorfrom openviking.metrics.datasources.queue import QueuePipelineStateDataSource-from .base import CollectorConfig, StateMetricCollector+from .base import CollectorConfig, EventMetricCollector, StateMetricCollector++QUEUE_DURATION_BUCKETS: tuple[float, ...] = (+ 0.01,+ 0.05,+ 0.1,+ 0.25,+ 0.5,+ 1.0,+ 2.5,+ 5.0,+ 10.0,+ 30.0,+ 60.0,+ 120.0,+ 300.0,+ 600.0,+ 1800.0,+ 3600.0,+ 21600.0,+ 86400.0,+)+_QUEUE_OUTCOMES = frozenset({"success", "failed", "requeued", "cancelled", "exception"})+++@dataclass+class QueueDurationCollector(EventMetricCollector):+ """Export processing and end-to-end latency for completed queue deliveries."""++ DOMAIN: ClassVar[str] = "queue"+ PROCESS_DURATION_SECONDS: ClassVar[str] = MetricCollector.metric_name(+ DOMAIN, "process_duration", unit="seconds"+ )+ END_TO_END_DURATION_SECONDS: ClassVar[str] = MetricCollector.metric_name(+ DOMAIN, "end_to_end_duration", unit="seconds"+ )+ SUPPORTED_EVENTS: ClassVar[frozenset[str]] = frozenset({"queue.processed"})++ def collect(self, registry=None) -> None:+ return None++ def receive_hook(self, event_name: str, payload: dict, registry) -> None:+ queue = str(payload.get("queue") or "").strip()+ outcome = str(payload.get("outcome") or "").strip()+ if not queue or outcome not in _QUEUE_OUTCOMES:+ return+ try:+ process_duration = float(payload["process_duration_seconds"])+ except (KeyError, TypeError, ValueError):+ return+ if not math.isfinite(process_duration) or process_duration < 0:+ return++ labels = {"queue": queue, "outcome": outcome}+ registry.observe_histogram(+ self.PROCESS_DURATION_SECONDS,+ process_duration,+ labels=labels,+ label_names=("queue", "outcome"),+ buckets=QUEUE_DURATION_BUCKETS,+ )++ raw_end_to_end = payload.get("end_to_end_duration_seconds")+ if raw_end_to_end is None:+ return+ try:+ end_to_end_duration = float(raw_end_to_end)+ except (TypeError, ValueError):+ return+ if not math.isfinite(end_to_end_duration) or end_to_end_duration < 0:+ return+ registry.observe_histogram(+ self.END_TO_END_DURATION_SECONDS,+ end_to_end_duration,+ labels=labels,+ label_names=("queue", "outcome"),+ buckets=QUEUE_DURATION_BUCKETS,+ )@dataclassdiff --git a/openviking/metrics/global_api.py b/openviking/metrics/global_api.pyindex 5b9a65120..09eb51157 100644--- a/openviking/metrics/global_api.py+++ b/openviking/metrics/global_api.py@@ -44,6 +44,7 @@ from .collectors.cache import CacheCollectorfrom .collectors.embedding import EmbeddingCollectorfrom .collectors.encryption import EncryptionCollectorfrom .collectors.http import HTTPCollector+from .collectors.queue import QueueDurationCollectorfrom .collectors.rerank import RerankCollectorfrom .collectors.resource import ResourceIngestionCollectorfrom .collectors.retrieval import RetrievalCollector@@ -172,7 +173,9 @@ def init_metrics_from_server_config(max_active_accounts=int(account_dimension.max_active_accounts),)_registry = registry or _create_metric_registry()- collector_manager = create_default_collector_manager(app=app, service=service, config=config)+ collector_manager = create_default_collector_manager(+ app=app, service=service, config=config+ )_event_router = _build_event_router(_registry)register_event_subscriber(_METRICS_EVENT_SUBSCRIBER,@@ -304,6 +307,7 @@ def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:retrieval_collector = RetrievalCollector()encryption_collector = EncryptionCollector()telemetry_bridge_collector = TelemetryBridgeCollector()+ queue_duration_collector = QueueDurationCollector()def _receiver(collector, event_name: str):"""@@ -345,6 +349,7 @@ def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:("encryption.key_cache_miss", encryption_collector),("encryption.key_version_usage", encryption_collector),("telemetry.summary", telemetry_bridge_collector),+ ("queue.processed", queue_duration_collector),):router.register(event_name, _receiver(collector, event_name))return routerdiff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex ee08d3449..9b17120d1 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -3,7 +3,9 @@import abcimport asyncioimport json+import mathimport threading+import timefrom dataclasses import dataclass, fieldfrom datetime import datetimefrom functools import partial@@ -263,6 +265,7 @@ class NamedQueue:handler = self._dequeue_handlerif handler is None:return ProcessResult.success(data)+ process_started = time.perf_counter()async def process(ctx: ProcessContext) -> ProcessResult:return self._validate_result(await handler.on_dequeue(ctx.message))@@ -273,12 +276,44 @@ class NamedQueue:ctx = ProcessContext(self.name, data, cancel=cancel)try:result = self._validate_result(await self._run_middleware("process", ctx, process))+ except asyncio.CancelledError:+ self._publish_duration(data, process_started, "exception")+ raiseexcept Exception as exc:self._record_error(str(exc), data)+ self._publish_duration(data, process_started, "exception")raiseself._record_result(result, data)+ self._publish_duration(data, process_started, result.outcome.value)return result+ def _publish_duration(self, data: Dict[str, Any], process_started: float, outcome: str) -> None:+ """Publish best-effort processing and end-to-end timings for one delivery."""+ from openviking.observability.events import try_publish_event++ payload: Dict[str, Any] = {+ "queue": self.name,+ "outcome": str(outcome),+ "process_duration_seconds": max(0.0, time.perf_counter() - process_started),+ }+ enqueued_at = self._extract_enqueued_at(data)+ if enqueued_at is not None:+ end_to_end = time.time() - enqueued_at+ if end_to_end >= 0:+ payload["end_to_end_duration_seconds"] = end_to_end+ try_publish_event("queue.processed", payload)++ @staticmethod+ def _extract_enqueued_at(data: Dict[str, Any]) -> Optional[float]:+ """Read the QueueFS enqueue timestamp from a dequeued message envelope."""+ if not isinstance(data, dict) or "id" not in data or "data" not in data:+ return None+ try:+ value = float(data.get("timestamp"))+ except (TypeError, ValueError):+ return None+ return value if math.isfinite(value) and value >= 0 else None+@staticmethoddef _validate_result(result: ProcessResult) -> ProcessResult:if not isinstance(result, ProcessResult):diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.pyindex a4afea66f..f87e8f463 100644--- a/tests/metrics/collectors/test_event_collectors.py+++ b/tests/metrics/collectors/test_event_collectors.py@@ -9,6 +9,7 @@ import openviking.metrics.global_api as global_apifrom openviking.metrics.collectors.base import EventMetricCollectorfrom openviking.metrics.collectors.cache import CacheCollectorfrom openviking.metrics.collectors.embedding import EmbeddingCollector+from openviking.metrics.collectors.queue import QueueDurationCollectorfrom openviking.metrics.collectors.rerank import RerankCollectorfrom openviking.metrics.collectors.retrieval import RetrievalCollectorfrom openviking.metrics.collectors.session import SessionCollector@@ -57,6 +58,76 @@ def test_cache_collector_uses_supported_events_and_receive_hook_routing(assert 'openviking_cache_misses_total{level="L1"} 1' in text+def test_queue_duration_collector_records_processing_and_end_to_end_histograms(+ registry, render_prometheus+):+ collector = QueueDurationCollector()++ collector.receive(+ "queue.processed",+ {+ "queue": "Semantic",+ "outcome": "success",+ "process_duration_seconds": 2.5,+ "end_to_end_duration_seconds": 8.0,+ },+ registry,+ )++ text = render_prometheus(registry)+ labels = 'outcome="success",queue="Semantic"'+ assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text+ assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 2.5" in text+ assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text+ assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 8.0" in text+ assert (+ "openviking_queue_end_to_end_duration_seconds_bucket{"+ 'le="86400.0",outcome="success",queue="Semantic"} 1' in text+ )+++def test_queue_duration_collector_allows_processing_only_and_ignores_bad_payloads(+ registry, render_prometheus+):+ collector = QueueDurationCollector()+ collector.receive(+ "queue.processed",+ {+ "queue": "Embedding",+ "outcome": "requeued",+ "process_duration_seconds": 0.5,+ },+ registry,+ )+ collector.receive(+ "queue.processed",+ {+ "queue": "Embedding",+ "outcome": "unknown",+ "process_duration_seconds": 1.0,+ },+ registry,+ )+ collector.receive("queue.processed", {"queue": "Embedding"}, registry)+ collector.receive(+ "queue.processed",+ {+ "queue": "Embedding",+ "outcome": "success",+ "process_duration_seconds": float("nan"),+ },+ registry,+ )++ text = render_prometheus(registry)+ assert (+ "openviking_queue_process_duration_seconds_count{"+ 'outcome="requeued",queue="Embedding"} 1' in text+ )+ assert "openviking_queue_end_to_end_duration_seconds" not in text+ assert 'outcome="unknown"' not in text++def test_session_collector_records_lifecycle_and_archive(registry, render_prometheus):c = SessionCollector()c.receive("session.lifecycle", {"action": "create", "status": "ok"}, registry)@@ -229,7 +300,9 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome)-def test_embedding_collector_records_failed_call_duration_with_error_code(registry, render_prometheus):+def test_embedding_collector_records_failed_call_duration_with_error_code(+ registry, render_prometheus+):EmbeddingCollector().receive("embedding.call",{@@ -335,7 +408,9 @@ def _emit_retrieval(registry, *, result_count, context_type="search", rerank_use)-def test_retrieval_collector_zero_result_records_downstream_without_raising(registry, render_prometheus):+def test_retrieval_collector_zero_result_records_downstream_without_raising(+ registry, render_prometheus+):"""Regression for #2922: a zero-result retrieval is a normal outcome, not an error.Previously ``record_completed`` incremented RESULTS_TOTAL by ``max(0, result_count)``,@@ -347,10 +422,17 @@ def test_retrieval_collector_zero_result_records_downstream_without_raising(regi_emit_retrieval(registry, result_count=0)text = render_prometheus(registry)- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)- assert re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)+ assert re.search(+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text+ )+ assert re.search(+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text+ )# Latency histogram still observed one sample (proving the method did not abort early).- assert re.search(r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)+ assert re.search(+ r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?',+ text,+ )# No positive results -> RESULTS_TOTAL has no series for this context.assert not re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"', text)@@ -361,18 +443,33 @@ def test_retrieval_collector_positive_result_increments_results_total(registry,_emit_retrieval(registry, result_count=3, rerank_used=True)text = render_prometheus(registry)- assert re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"[^}]*\} 3(?:\.0)?', text)- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)- assert not re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"', text)+ assert re.search(+ r'openviking_retrieval_results_total\{[^}]*context_type="search"[^}]*\} 3(?:\.0)?', text+ )+ assert re.search(+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text+ )+ assert not re.search(+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"', text+ )-def test_retrieval_collector_negative_result_count_is_bucketed_as_zero_result(registry, render_prometheus):+def test_retrieval_collector_negative_result_count_is_bucketed_as_zero_result(+ registry, render_prometheus+):"""A malformed negative result_count must not raise or drop metrics; it is classifiedas an empty retrieval (no positive results) so telemetry stays internally consistent."""_emit_retrieval(registry, result_count=-1)text = render_prometheus(registry)- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)- assert re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)- assert re.search(r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)+ assert re.search(+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text+ )+ assert re.search(+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text+ )+ assert re.search(+ r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?',+ text,+ )assert not re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"', text)diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.pyindex edb0c8b5f..29118905f 100644--- a/tests/metrics/integration/test_telemetry_bridge.py+++ b/tests/metrics/integration/test_telemetry_bridge.py@@ -3,9 +3,34 @@from openviking.metrics.datasources.telemetry_bridge import TelemetryBridgeEventDataSourcefrom openviking.metrics.global_api import init_metrics_from_server_config, shutdown_metrics+from openviking.observability.events import try_publish_eventfrom openviking.server.config import MetricsConfig, ObservabilityConfig, ServerConfig+def test_queue_duration_event_is_wired_to_metrics_registry(registry, render_prometheus):+ init_metrics_from_server_config(+ ServerConfig(observability=ObservabilityConfig(metrics=MetricsConfig(enabled=True))),+ app=None,+ registry=registry,+ )+ try:+ try_publish_event(+ "queue.processed",+ {+ "queue": "SessionCommit",+ "outcome": "success",+ "process_duration_seconds": 3.0,+ "end_to_end_duration_seconds": 9.0,+ },+ )+ text = render_prometheus(registry)+ labels = 'outcome="success",queue="SessionCommit"'+ assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text+ assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text+ finally:+ shutdown_metrics()++def test_telemetry_bridge_records_operation_and_resource_metrics(registry, render_prometheus):init_metrics_from_server_config(ServerConfig(observability=ObservabilityConfig(metrics=MetricsConfig(enabled=True))),diff --git a/tests/storage/test_named_queue_middleware.py b/tests/storage/test_named_queue_middleware.pyindex 1ac31f2d9..f015a51c2 100644--- a/tests/storage/test_named_queue_middleware.py+++ b/tests/storage/test_named_queue_middleware.py@@ -109,6 +109,104 @@ async def test_enqueue_middleware_rewrites_payload(transport):assert json.loads(transport.write.await_args.args[1]) == {"input": "input", "middleware": True}+async def test_process_dequeued_publishes_processing_and_end_to_end_durations(+ transport, monkeypatch+):+ events = []+ wall_times = iter((105.0,))+ perf_times = iter((10.0, 12.5))+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.time.time", lambda: next(wall_times)+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.time.perf_counter",+ lambda: next(perf_times),+ )+ monkeypatch.setattr(+ "openviking.observability.events.try_publish_event",+ lambda event_name, payload: events.append((event_name, payload)),+ )+ queue = NamedQueue(object(), "/queue", "Semantic", dequeue_handler=Handler())+ message = {+ "id": "m",+ "data": "{}",+ "timestamp": 100.0,+ }++ result = await queue.process_dequeued(message)++ assert result.outcome is ProcessOutcome.SUCCESS+ assert events == [+ (+ "queue.processed",+ {+ "queue": "Semantic",+ "outcome": "success",+ "process_duration_seconds": 2.5,+ "end_to_end_duration_seconds": 5.0,+ },+ )+ ]+++async def test_process_dequeued_old_message_only_publishes_processing_duration(+ transport, monkeypatch+):+ events = []+ perf_times = iter((20.0, 20.25))+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.time.perf_counter",+ lambda: next(perf_times),+ )+ monkeypatch.setattr(+ "openviking.observability.events.try_publish_event",+ lambda event_name, payload: events.append((event_name, payload)),+ )+ queue = NamedQueue(object(), "/queue", "Embedding", dequeue_handler=Handler())++ await queue.process_dequeued({"id": "legacy", "data": "{}"})++ assert events == [+ (+ "queue.processed",+ {+ "queue": "Embedding",+ "outcome": "success",+ "process_duration_seconds": 0.25,+ },+ )+ ]+++async def test_process_dequeued_exception_publishes_exception_outcome(transport, monkeypatch):+ events = []+ perf_times = iter((30.0, 31.0))++ class FailingHandler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ raise RuntimeError("failed")++ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.time.perf_counter",+ lambda: next(perf_times),+ )+ monkeypatch.setattr(+ "openviking.observability.events.try_publish_event",+ lambda event_name, payload: events.append((event_name, payload)),+ )+ queue = NamedQueue(object(), "/queue", "AddResource", dequeue_handler=FailingHandler())++ with pytest.raises(RuntimeError, match="failed"):+ await queue.process_dequeued({"id": "m", "data": "{}"})++ assert events[0][0] == "queue.processed"+ assert events[0][1] == {+ "queue": "AddResource",+ "outcome": "exception",+ "process_duration_seconds": 1.0,+ }++@pytest.mark.parametrize("failure", [RuntimeError("handler failed"), asyncio.CancelledError()])async def test_handler_exception_still_skips_ack(transport, failure):class FailingHandler(DequeueHandlerBase):@@ -178,7 +276,13 @@ async def test_queue_without_middleware_remains_generic(transport):ProcessResult.cancelled(),],)-async def test_process_short_circuit_settles_once_without_acking(transport, result):+async def test_process_short_circuit_settles_once_without_acking(transport, result, monkeypatch):+ events = []+ monkeypatch.setattr(+ "openviking.observability.events.try_publish_event",+ lambda event_name, payload: events.append((event_name, payload)),+ )+class ShortCircuit(QueueMiddleware):async def process(self, ctx, call_next):return result@@ -195,6 +299,8 @@ async def test_process_short_circuit_settles_once_without_acking(transport, resuassert queue._error_count == (result.outcome is ProcessOutcome.FAILED)assert queue._processed == (result.outcome is not ProcessOutcome.FAILED)assert queue._requeue_count == (result.outcome is ProcessOutcome.REQUEUED)+ assert events[0][0] == "queue.processed"+ assert events[0][1]["outcome"] == result.outcome.valuetransport.write.assert_not_awaited()diff --git a/tests/storage/test_queue_duration_binding.py b/tests/storage/test_queue_duration_binding.pynew file mode 100644index 000000000..93055e6b3--- /dev/null+++ b/tests/storage/test_queue_duration_binding.py@@ -0,0 +1,29 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++"""Queue timing integration through the real RAGFS Python binding."""++import time++import pytest++from openviking.storage.queuefs.named_queue import NamedQueue+from openviking.utils.agfs_utils import RagfsBindingConfig, create_agfs_client+from openviking_cli.utils.config.agfs_config import AGFSConfig+++async def test_queuefs_dequeue_exposes_original_enqueue_timestamp(tmp_path):+ client = create_agfs_client(+ RagfsBindingConfig(agfs=AGFSConfig(path=str(tmp_path), backend="memory"))+ )+ queue = NamedQueue(client, "/queue", "TimingTest")+ before = time.time()++ await queue.enqueue({"value": 1})+ message = await queue.dequeue_raw()++ assert message is not None+ assert message["id"]+ assert message["data"] == '{"value": 1}'+ assert before <= float(message["timestamp"]) <= time.time()