Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(metrics): track queue processing latency (#5365)

zhanggaoyuana78c61297697
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rs
index 67dac8898..73407d708 100644
--- a/crates/ragfs/src/plugins/queuefs/backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/backend.rs
@@ -4,7 +4,7 @@
//! storage implementations (memory, SQLite, etc.) while maintaining a consistent interface.
use crate::core::errors::{Error, Result};
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, SecondsFormat, Utc};
use rusqlite::{params, types::ValueRef, Connection, Row};
use serde::{Deserialize, Deserializer, Serialize};
use std::collections::{HashMap, VecDeque};
@@ -22,6 +22,8 @@ pub struct Message {
pub data: Vec<u8>,
/// Timestamp when the message was enqueued
pub timestamp: SystemTime,
+ /// Whether `timestamp` came from a real persisted enqueue time.
+ pub timestamp_valid: bool,
}
impl Message {
@@ -31,6 +33,7 @@ impl Message {
id: Uuid::new_v4().to_string(),
data,
timestamp: SystemTime::now(),
+ timestamp_valid: true,
}
}
}
@@ -58,16 +61,21 @@ impl StoredMessage {
Self {
id: msg.id.clone(),
data: msg.data.clone(),
- // Prefer unix seconds for compatibility with older queue.db producers.
- timestamp: Some(serde_json::Value::Number(unix_secs(msg.timestamp).into())),
+ // Older readers already accept RFC3339 strings; nanoseconds preserve latency accuracy.
+ timestamp: Some(serde_json::Value::String(
+ DateTime::<Utc>::from(msg.timestamp)
+ .to_rfc3339_opts(SecondsFormat::Nanos, true),
+ )),
}
}
pub(super) fn into_message(self) -> Message {
+ let (timestamp, timestamp_valid) = parse_stored_timestamp(self.timestamp);
Message {
id: self.id,
data: self.data,
- timestamp: parse_stored_timestamp(self.timestamp),
+ timestamp,
+ timestamp_valid,
}
}
}
@@ -101,26 +109,28 @@ where
}
}
-fn parse_stored_timestamp(raw: Option<serde_json::Value>) -> SystemTime {
+fn parse_stored_timestamp(raw: Option<serde_json::Value>) -> (SystemTime, bool) {
match raw {
Some(serde_json::Value::String(ts)) => DateTime::parse_from_rfc3339(&ts)
- .map(|dt| {
- let secs = dt.timestamp();
- let secs_u64 = if secs <= 0 { 0 } else { secs as u64 };
- UNIX_EPOCH + Duration::from_secs(secs_u64)
- })
- .unwrap_or_else(|_| SystemTime::now()),
+ .ok()
+ .and_then(|dt| timestamp_from_parts(dt.timestamp(), dt.timestamp_subsec_nanos()))
+ .map(|timestamp| (timestamp, true))
+ .unwrap_or_else(|| (SystemTime::now(), false)),
Some(serde_json::Value::Number(num)) => num
.as_i64()
- .map(|secs| {
- let secs_u64 = if secs <= 0 { 0 } else { secs as u64 };
- UNIX_EPOCH + Duration::from_secs(secs_u64)
- })
- .unwrap_or_else(SystemTime::now),
- _ => SystemTime::now(),
+ .and_then(|secs| timestamp_from_parts(secs, 0))
+ .map(|timestamp| (timestamp, true))
+ .unwrap_or_else(|| (SystemTime::now(), false)),
+ _ => (SystemTime::now(), false),
}
}
+fn timestamp_from_parts(secs: i64, nanos: u32) -> Option<SystemTime> {
+ (secs > 0)
+ .then(|| Duration::new(secs as u64, nanos))
+ .and_then(|duration| UNIX_EPOCH.checked_add(duration))
+}
+
fn unix_secs(time: SystemTime) -> i64 {
time.duration_since(UNIX_EPOCH)
.unwrap_or_default()
@@ -1081,6 +1091,67 @@ mod tests {
assert_eq!(dequeued.data, payload);
}
+ #[test]
+ fn test_stored_message_preserves_timestamp_validity() {
+ let missing = StoredMessage {
+ id: "missing".to_string(),
+ data: b"payload".to_vec(),
+ timestamp: None,
+ }
+ .into_message();
+ assert!(!missing.timestamp_valid);
+
+ let invalid = StoredMessage {
+ id: "invalid".to_string(),
+ data: b"payload".to_vec(),
+ timestamp: Some(serde_json::Value::String("not-a-time".to_string())),
+ }
+ .into_message();
+ assert!(!invalid.timestamp_valid);
+
+ let valid = StoredMessage {
+ id: "valid".to_string(),
+ data: b"payload".to_vec(),
+ timestamp: Some(serde_json::Value::Number(1234.into())),
+ }
+ .into_message();
+ assert!(valid.timestamp_valid);
+ assert_eq!(unix_secs(valid.timestamp), 1234);
+
+ let precise_timestamp = UNIX_EPOCH + Duration::new(1_234, 567_890_123);
+ let precise = StoredMessage::from_message(&Message {
+ id: "precise".to_string(),
+ data: b"payload".to_vec(),
+ timestamp: precise_timestamp,
+ timestamp_valid: true,
+ })
+ .into_message();
+ assert!(precise.timestamp_valid);
+ assert_eq!(precise.timestamp, precise_timestamp);
+ }
+
+ #[test]
+ fn test_sqlite_backend_preserves_subsecond_enqueue_timestamp() {
+ let (_dir, _db_path, mut backend) = sqlite_backend();
+ backend.create_queue("test").unwrap();
+
+ let timestamp = UNIX_EPOCH + Duration::new(1_700_000_000, 987_654_321);
+ backend
+ .enqueue(
+ "test",
+ Message {
+ id: "precise".to_string(),
+ data: b"payload".to_vec(),
+ timestamp,
+ timestamp_valid: true,
+ },
+ )
+ .unwrap();
+
+ let dequeued = backend.dequeue("test").unwrap().unwrap();
+ assert_eq!(dequeued.timestamp, timestamp);
+ }
+
#[test]
fn test_sqlite_backend_recover_stale() {
let (_dir, db_path_str, _backend) = sqlite_backend();
diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs
index bce789b6a..f3da2c4b3 100644
--- a/crates/ragfs/src/plugins/queuefs/mod.rs
+++ b/crates/ragfs/src/plugins/queuefs/mod.rs
@@ -79,6 +79,17 @@ const CONTROL_FILES: &[ControlFileSpec] = &[
struct QueueMessage {
id: String,
data: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ timestamp: Option<f64>,
+}
+
+fn queue_message_timestamp(msg: &Message) -> Option<f64> {
+ msg.timestamp_valid.then(|| {
+ msg.timestamp
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs_f64()
+ })
}
#[derive(Debug, Clone, Copy)]
@@ -374,9 +385,11 @@ impl FileSystem for QueueFileSystem {
};
// Return in Go libagfsbinding format: {"id": "...", "data": "..."}
let data_str = String::from_utf8_lossy(&msg.data).to_string();
+ let timestamp = queue_message_timestamp(&msg);
let response = QueueMessage {
id: msg.id,
data: data_str,
+ timestamp,
};
Ok(serde_json::to_vec(&response)?)
}
@@ -389,6 +402,7 @@ impl FileSystem for QueueFileSystem {
let response = QueueMessage {
id: msg.id.clone(),
data: data_str,
+ timestamp: queue_message_timestamp(&msg),
};
Ok(serde_json::to_vec(&response)?)
}
@@ -406,9 +420,13 @@ impl FileSystem for QueueFileSystem {
.list_unacked(&queue_name)
.await?
.into_iter()
- .map(|msg| QueueMessage {
+ .map(|msg| {
+ let timestamp = queue_message_timestamp(&msg);
+ QueueMessage {
id: msg.id,
data: String::from_utf8_lossy(&msg.data).to_string(),
+ timestamp,
+ }
})
.collect::<Vec<_>>();
Ok(serde_json::to_vec(&messages)?)
@@ -898,6 +916,18 @@ mod tests {
struct TestQueueMessage {
id: String,
data: String,
+ timestamp: Option<f64>,
+ }
+
+ #[test]
+ fn test_queue_message_omits_missing_timestamp() {
+ let response = QueueMessage {
+ id: "legacy".to_string(),
+ data: "payload".to_string(),
+ timestamp: None,
+ };
+ let value = serde_json::to_value(response).unwrap();
+ assert!(value.get("timestamp").is_none());
}
/// Create a queue filesystem with one initialized queue.
@@ -940,10 +970,12 @@ mod tests {
let msg1 = dequeue_msg(&fs, "test").await;
assert!(!msg1.id.is_empty());
assert_eq!(msg1.data.as_bytes(), data1);
+ assert!(msg1.timestamp.is_some_and(|value| value > 0.0));
let msg2 = dequeue_msg(&fs, "test").await;
assert!(!msg2.id.is_empty());
assert_eq!(msg2.data.as_bytes(), data2);
+ assert!(msg2.timestamp.is_some_and(|value| value > 0.0));
// Queue should be empty
let result = fs.read("/test/dequeue", 0, 0).await.unwrap();
diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md
index 07191d558..1b3f088b4 100644
--- a/docs/en/concepts/12-metrics.md
+++ b/docs/en/concepts/12-metrics.md
@@ -228,6 +228,8 @@ Notes:
| `openviking_queue_errors_total` | Counter | `queue` | total error count per queue |
| `openviking_queue_pending` | Gauge | `queue` | pending queue items |
| `openviking_queue_in_progress` | Gauge | `queue` | in-progress queue items |
+| `openviking_queue_process_duration_seconds` | Histogram | `queue, outcome` | handler execution time after a worker dequeues a message |
+| `openviking_queue_end_to_end_duration_seconds` | Histogram | `queue, outcome` | enqueue-to-handler-completion latency, including queue wait time |
| `openviking_executor_max_workers` | Gauge | `pool, process_role, worker` | maximum workers in the asyncio default executor |
| `openviking_executor_threads` | Gauge | `pool, process_role, worker` | threads created by the asyncio default executor |
| `openviking_executor_active_tasks` | Gauge | `pool, process_role, worker` | default executor tasks currently running |
@@ -243,9 +245,12 @@ Notes:
| `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration |
+The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.
+
These help answer:
- Is there queue backlog?
+- Which queue is slow, and is the latency caused by waiting or processing?
- Is there lock contention or stale locking?
- Is the default executor near its worker limit or building a queue?
diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md
index 50c123003..ab6c40525 100644
--- a/docs/zh/concepts/12-metrics.md
+++ b/docs/zh/concepts/12-metrics.md
@@ -231,6 +231,8 @@ scrape_configs:
| `openviking_queue_errors_total` | Counter | `queue` | 队列累计错误量 |
| `openviking_queue_pending` | Gauge | `queue` | 队列待处理数 |
| `openviking_queue_in_progress` | Gauge | `queue` | 队列执行中数量 |
+| `openviking_queue_process_duration_seconds` | Histogram | `queue, outcome` | 消息被 worker 取出后到 handler 结束的处理耗时 |
+| `openviking_queue_end_to_end_duration_seconds` | Histogram | `queue, outcome` | 消息从入队到 handler 结束的端到端耗时,包含排队时间 |
| `openviking_executor_max_workers` | Gauge | `pool, process_role, worker` | asyncio 默认 executor 最大 worker 数 |
| `openviking_executor_threads` | Gauge | `pool, process_role, worker` | asyncio 默认 executor 已创建线程数 |
| `openviking_executor_active_tasks` | Gauge | `pool, process_role, worker` | 默认 executor 当前执行中的任务数 |
@@ -246,9 +248,12 @@ scrape_configs:
| `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 |
+队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。
+
这些指标适合回答:
- 是否有队列堆积?
+- 哪个队列处理慢,以及耗时是否主要来自排队?
- 是否有锁竞争或 stale lock?
- 默认 executor 是否接近线程上限或出现排队?
diff --git a/openviking/metrics/collectors/__init__.py b/openviking/metrics/collectors/__init__.py
index 716fde7a9..b55a07e4b 100644
--- a/openviking/metrics/collectors/__init__.py
+++ b/openviking/metrics/collectors/__init__.py
@@ -22,7 +22,7 @@ from .model_provider_probe import ModelProviderProbeCollector
from .model_usage import ModelUsageCollector
from .observer_health import ObserverHealthCollector
from .observer_state import ObserverStateCollector
-from .queue import QueueCollector
+from .queue import QueueCollector, QueueDurationCollector
from .ragfs import RagfsMetricCollector
from .rerank import RerankCollector
from .retrieval import RetrievalCollector
@@ -47,6 +47,7 @@ __all__ = [
"EncryptionCollector",
"FeedbackCollector",
"QueueCollector",
+ "QueueDurationCollector",
"RerankCollector",
"RagfsMetricCollector",
"VikingDBCollector",
diff --git a/openviking/metrics/collectors/queue.py b/openviking/metrics/collectors/queue.py
index b819dbf07..e8b1ac417 100644
--- a/openviking/metrics/collectors/queue.py
+++ b/openviking/metrics/collectors/queue.py
@@ -3,13 +3,91 @@
from __future__ import annotations
+import math
from dataclasses import dataclass, field
from typing import ClassVar
from openviking.metrics.core.base import MetricCollector
from openviking.metrics.datasources.queue import QueuePipelineStateDataSource
-from .base import CollectorConfig, StateMetricCollector
+from .base import CollectorConfig, EventMetricCollector, StateMetricCollector
+
+QUEUE_DURATION_BUCKETS: tuple[float, ...] = (
+ 0.01,
+ 0.05,
+ 0.1,
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.5,
+ 5.0,
+ 10.0,
+ 30.0,
+ 60.0,
+ 120.0,
+ 300.0,
+ 600.0,
+ 1800.0,
+ 3600.0,
+ 21600.0,
+ 86400.0,
+)
+_QUEUE_OUTCOMES = frozenset({"success", "failed", "requeued", "cancelled", "exception"})
+
+
+@dataclass
+class QueueDurationCollector(EventMetricCollector):
+ """Export processing and end-to-end latency for completed queue deliveries."""
+
+ DOMAIN: ClassVar[str] = "queue"
+ PROCESS_DURATION_SECONDS: ClassVar[str] = MetricCollector.metric_name(
+ DOMAIN, "process_duration", unit="seconds"
+ )
+ END_TO_END_DURATION_SECONDS: ClassVar[str] = MetricCollector.metric_name(
+ DOMAIN, "end_to_end_duration", unit="seconds"
+ )
+ SUPPORTED_EVENTS: ClassVar[frozenset[str]] = frozenset({"queue.processed"})
+
+ def collect(self, registry=None) -> None:
+ return None
+
+ def receive_hook(self, event_name: str, payload: dict, registry) -> None:
+ queue = str(payload.get("queue") or "").strip()
+ outcome = str(payload.get("outcome") or "").strip()
+ if not queue or outcome not in _QUEUE_OUTCOMES:
+ return
+ try:
+ process_duration = float(payload["process_duration_seconds"])
+ except (KeyError, TypeError, ValueError):
+ return
+ if not math.isfinite(process_duration) or process_duration < 0:
+ return
+
+ labels = {"queue": queue, "outcome": outcome}
+ registry.observe_histogram(
+ self.PROCESS_DURATION_SECONDS,
+ process_duration,
+ labels=labels,
+ label_names=("queue", "outcome"),
+ buckets=QUEUE_DURATION_BUCKETS,
+ )
+
+ raw_end_to_end = payload.get("end_to_end_duration_seconds")
+ if raw_end_to_end is None:
+ return
+ try:
+ end_to_end_duration = float(raw_end_to_end)
+ except (TypeError, ValueError):
+ return
+ if not math.isfinite(end_to_end_duration) or end_to_end_duration < 0:
+ return
+ registry.observe_histogram(
+ self.END_TO_END_DURATION_SECONDS,
+ end_to_end_duration,
+ labels=labels,
+ label_names=("queue", "outcome"),
+ buckets=QUEUE_DURATION_BUCKETS,
+ )
@dataclass
diff --git a/openviking/metrics/global_api.py b/openviking/metrics/global_api.py
index 5b9a65120..09eb51157 100644
--- a/openviking/metrics/global_api.py
+++ b/openviking/metrics/global_api.py
@@ -44,6 +44,7 @@ from .collectors.cache import CacheCollector
from .collectors.embedding import EmbeddingCollector
from .collectors.encryption import EncryptionCollector
from .collectors.http import HTTPCollector
+from .collectors.queue import QueueDurationCollector
from .collectors.rerank import RerankCollector
from .collectors.resource import ResourceIngestionCollector
from .collectors.retrieval import RetrievalCollector
@@ -172,7 +173,9 @@ def init_metrics_from_server_config(
max_active_accounts=int(account_dimension.max_active_accounts),
)
_registry = registry or _create_metric_registry()
- collector_manager = create_default_collector_manager(app=app, service=service, config=config)
+ collector_manager = create_default_collector_manager(
+ app=app, service=service, config=config
+ )
_event_router = _build_event_router(_registry)
register_event_subscriber(
_METRICS_EVENT_SUBSCRIBER,
@@ -304,6 +307,7 @@ def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:
retrieval_collector = RetrievalCollector()
encryption_collector = EncryptionCollector()
telemetry_bridge_collector = TelemetryBridgeCollector()
+ queue_duration_collector = QueueDurationCollector()
def _receiver(collector, event_name: str):
"""
@@ -345,6 +349,7 @@ def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:
("encryption.key_cache_miss", encryption_collector),
("encryption.key_version_usage", encryption_collector),
("telemetry.summary", telemetry_bridge_collector),
+ ("queue.processed", queue_duration_collector),
):
router.register(event_name, _receiver(collector, event_name))
return router
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index ee08d3449..9b17120d1 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -3,7 +3,9 @@
import abc
import asyncio
import json
+import math
import threading
+import time
from dataclasses import dataclass, field
from datetime import datetime
from functools import partial
@@ -263,6 +265,7 @@ class NamedQueue:
handler = self._dequeue_handler
if handler is None:
return ProcessResult.success(data)
+ process_started = time.perf_counter()
async def process(ctx: ProcessContext) -> ProcessResult:
return self._validate_result(await handler.on_dequeue(ctx.message))
@@ -273,12 +276,44 @@ class NamedQueue:
ctx = ProcessContext(self.name, data, cancel=cancel)
try:
result = self._validate_result(await self._run_middleware("process", ctx, process))
+ except asyncio.CancelledError:
+ self._publish_duration(data, process_started, "exception")
+ raise
except Exception as exc:
self._record_error(str(exc), data)
+ self._publish_duration(data, process_started, "exception")
raise
self._record_result(result, data)
+ self._publish_duration(data, process_started, result.outcome.value)
return result
+ def _publish_duration(self, data: Dict[str, Any], process_started: float, outcome: str) -> None:
+ """Publish best-effort processing and end-to-end timings for one delivery."""
+ from openviking.observability.events import try_publish_event
+
+ payload: Dict[str, Any] = {
+ "queue": self.name,
+ "outcome": str(outcome),
+ "process_duration_seconds": max(0.0, time.perf_counter() - process_started),
+ }
+ enqueued_at = self._extract_enqueued_at(data)
+ if enqueued_at is not None:
+ end_to_end = time.time() - enqueued_at
+ if end_to_end >= 0:
+ payload["end_to_end_duration_seconds"] = end_to_end
+ try_publish_event("queue.processed", payload)
+
+ @staticmethod
+ def _extract_enqueued_at(data: Dict[str, Any]) -> Optional[float]:
+ """Read the QueueFS enqueue timestamp from a dequeued message envelope."""
+ if not isinstance(data, dict) or "id" not in data or "data" not in data:
+ return None
+ try:
+ value = float(data.get("timestamp"))
+ except (TypeError, ValueError):
+ return None
+ return value if math.isfinite(value) and value >= 0 else None
+
@staticmethod
def _validate_result(result: ProcessResult) -> ProcessResult:
if not isinstance(result, ProcessResult):
diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.py
index a4afea66f..f87e8f463 100644
--- a/tests/metrics/collectors/test_event_collectors.py
+++ b/tests/metrics/collectors/test_event_collectors.py
@@ -9,6 +9,7 @@ import openviking.metrics.global_api as global_api
from openviking.metrics.collectors.base import EventMetricCollector
from openviking.metrics.collectors.cache import CacheCollector
from openviking.metrics.collectors.embedding import EmbeddingCollector
+from openviking.metrics.collectors.queue import QueueDurationCollector
from openviking.metrics.collectors.rerank import RerankCollector
from openviking.metrics.collectors.retrieval import RetrievalCollector
from openviking.metrics.collectors.session import SessionCollector
@@ -57,6 +58,76 @@ def test_cache_collector_uses_supported_events_and_receive_hook_routing(
assert 'openviking_cache_misses_total{level="L1"} 1' in text
+def test_queue_duration_collector_records_processing_and_end_to_end_histograms(
+ registry, render_prometheus
+):
+ collector = QueueDurationCollector()
+
+ collector.receive(
+ "queue.processed",
+ {
+ "queue": "Semantic",
+ "outcome": "success",
+ "process_duration_seconds": 2.5,
+ "end_to_end_duration_seconds": 8.0,
+ },
+ registry,
+ )
+
+ text = render_prometheus(registry)
+ labels = 'outcome="success",queue="Semantic"'
+ assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text
+ assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 2.5" in text
+ assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text
+ assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 8.0" in text
+ assert (
+ "openviking_queue_end_to_end_duration_seconds_bucket{"
+ 'le="86400.0",outcome="success",queue="Semantic"} 1' in text
+ )
+
+
+def test_queue_duration_collector_allows_processing_only_and_ignores_bad_payloads(
+ registry, render_prometheus
+):
+ collector = QueueDurationCollector()
+ collector.receive(
+ "queue.processed",
+ {
+ "queue": "Embedding",
+ "outcome": "requeued",
+ "process_duration_seconds": 0.5,
+ },
+ registry,
+ )
+ collector.receive(
+ "queue.processed",
+ {
+ "queue": "Embedding",
+ "outcome": "unknown",
+ "process_duration_seconds": 1.0,
+ },
+ registry,
+ )
+ collector.receive("queue.processed", {"queue": "Embedding"}, registry)
+ collector.receive(
+ "queue.processed",
+ {
+ "queue": "Embedding",
+ "outcome": "success",
+ "process_duration_seconds": float("nan"),
+ },
+ registry,
+ )
+
+ text = render_prometheus(registry)
+ assert (
+ "openviking_queue_process_duration_seconds_count{"
+ 'outcome="requeued",queue="Embedding"} 1' in text
+ )
+ assert "openviking_queue_end_to_end_duration_seconds" not in text
+ assert 'outcome="unknown"' not in text
+
+
def test_session_collector_records_lifecycle_and_archive(registry, render_prometheus):
c = SessionCollector()
c.receive("session.lifecycle", {"action": "create", "status": "ok"}, registry)
@@ -229,7 +300,9 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome
)
-def test_embedding_collector_records_failed_call_duration_with_error_code(registry, render_prometheus):
+def test_embedding_collector_records_failed_call_duration_with_error_code(
+ registry, render_prometheus
+):
EmbeddingCollector().receive(
"embedding.call",
{
@@ -335,7 +408,9 @@ def _emit_retrieval(registry, *, result_count, context_type="search", rerank_use
)
-def test_retrieval_collector_zero_result_records_downstream_without_raising(registry, render_prometheus):
+def test_retrieval_collector_zero_result_records_downstream_without_raising(
+ registry, render_prometheus
+):
"""Regression for #2922: a zero-result retrieval is a normal outcome, not an error.
Previously ``record_completed`` incremented RESULTS_TOTAL by ``max(0, result_count)``,
@@ -347,10 +422,17 @@ def test_retrieval_collector_zero_result_records_downstream_without_raising(regi
_emit_retrieval(registry, result_count=0)
text = render_prometheus(registry)
- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
- assert re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
+ assert re.search(
+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text
+ )
+ assert re.search(
+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text
+ )
# Latency histogram still observed one sample (proving the method did not abort early).
- assert re.search(r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
+ assert re.search(
+ r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?',
+ text,
+ )
# No positive results -> RESULTS_TOTAL has no series for this context.
assert not re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"', text)
@@ -361,18 +443,33 @@ def test_retrieval_collector_positive_result_increments_results_total(registry,
_emit_retrieval(registry, result_count=3, rerank_used=True)
text = render_prometheus(registry)
- assert re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"[^}]*\} 3(?:\.0)?', text)
- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
- assert not re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"', text)
+ assert re.search(
+ r'openviking_retrieval_results_total\{[^}]*context_type="search"[^}]*\} 3(?:\.0)?', text
+ )
+ assert re.search(
+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text
+ )
+ assert not re.search(
+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"', text
+ )
-def test_retrieval_collector_negative_result_count_is_bucketed_as_zero_result(registry, render_prometheus):
+def test_retrieval_collector_negative_result_count_is_bucketed_as_zero_result(
+ registry, render_prometheus
+):
"""A malformed negative result_count must not raise or drop metrics; it is classified
as an empty retrieval (no positive results) so telemetry stays internally consistent."""
_emit_retrieval(registry, result_count=-1)
text = render_prometheus(registry)
- assert re.search(r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
- assert re.search(r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
- assert re.search(r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text)
+ assert re.search(
+ r'openviking_retrieval_requests_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text
+ )
+ assert re.search(
+ r'openviking_retrieval_zero_result_total\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?', text
+ )
+ assert re.search(
+ r'openviking_retrieval_latency_seconds_count\{[^}]*context_type="search"[^}]*\} 1(?:\.0)?',
+ text,
+ )
assert not re.search(r'openviking_retrieval_results_total\{[^}]*context_type="search"', text)
diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.py
index edb0c8b5f..29118905f 100644
--- a/tests/metrics/integration/test_telemetry_bridge.py
+++ b/tests/metrics/integration/test_telemetry_bridge.py
@@ -3,9 +3,34 @@
from openviking.metrics.datasources.telemetry_bridge import TelemetryBridgeEventDataSource
from openviking.metrics.global_api import init_metrics_from_server_config, shutdown_metrics
+from openviking.observability.events import try_publish_event
from openviking.server.config import MetricsConfig, ObservabilityConfig, ServerConfig
+def test_queue_duration_event_is_wired_to_metrics_registry(registry, render_prometheus):
+ init_metrics_from_server_config(
+ ServerConfig(observability=ObservabilityConfig(metrics=MetricsConfig(enabled=True))),
+ app=None,
+ registry=registry,
+ )
+ try:
+ try_publish_event(
+ "queue.processed",
+ {
+ "queue": "SessionCommit",
+ "outcome": "success",
+ "process_duration_seconds": 3.0,
+ "end_to_end_duration_seconds": 9.0,
+ },
+ )
+ text = render_prometheus(registry)
+ labels = 'outcome="success",queue="SessionCommit"'
+ assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text
+ assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text
+ finally:
+ shutdown_metrics()
+
+
def test_telemetry_bridge_records_operation_and_resource_metrics(registry, render_prometheus):
init_metrics_from_server_config(
ServerConfig(observability=ObservabilityConfig(metrics=MetricsConfig(enabled=True))),
diff --git a/tests/storage/test_named_queue_middleware.py b/tests/storage/test_named_queue_middleware.py
index 1ac31f2d9..f015a51c2 100644
--- a/tests/storage/test_named_queue_middleware.py
+++ b/tests/storage/test_named_queue_middleware.py
@@ -109,6 +109,104 @@ async def test_enqueue_middleware_rewrites_payload(transport):
assert json.loads(transport.write.await_args.args[1]) == {"input": "input", "middleware": True}
+async def test_process_dequeued_publishes_processing_and_end_to_end_durations(
+ transport, monkeypatch
+):
+ events = []
+ wall_times = iter((105.0,))
+ perf_times = iter((10.0, 12.5))
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.time.time", lambda: next(wall_times)
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.time.perf_counter",
+ lambda: next(perf_times),
+ )
+ monkeypatch.setattr(
+ "openviking.observability.events.try_publish_event",
+ lambda event_name, payload: events.append((event_name, payload)),
+ )
+ queue = NamedQueue(object(), "/queue", "Semantic", dequeue_handler=Handler())
+ message = {
+ "id": "m",
+ "data": "{}",
+ "timestamp": 100.0,
+ }
+
+ result = await queue.process_dequeued(message)
+
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert events == [
+ (
+ "queue.processed",
+ {
+ "queue": "Semantic",
+ "outcome": "success",
+ "process_duration_seconds": 2.5,
+ "end_to_end_duration_seconds": 5.0,
+ },
+ )
+ ]
+
+
+async def test_process_dequeued_old_message_only_publishes_processing_duration(
+ transport, monkeypatch
+):
+ events = []
+ perf_times = iter((20.0, 20.25))
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.time.perf_counter",
+ lambda: next(perf_times),
+ )
+ monkeypatch.setattr(
+ "openviking.observability.events.try_publish_event",
+ lambda event_name, payload: events.append((event_name, payload)),
+ )
+ queue = NamedQueue(object(), "/queue", "Embedding", dequeue_handler=Handler())
+
+ await queue.process_dequeued({"id": "legacy", "data": "{}"})
+
+ assert events == [
+ (
+ "queue.processed",
+ {
+ "queue": "Embedding",
+ "outcome": "success",
+ "process_duration_seconds": 0.25,
+ },
+ )
+ ]
+
+
+async def test_process_dequeued_exception_publishes_exception_outcome(transport, monkeypatch):
+ events = []
+ perf_times = iter((30.0, 31.0))
+
+ class FailingHandler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ raise RuntimeError("failed")
+
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.time.perf_counter",
+ lambda: next(perf_times),
+ )
+ monkeypatch.setattr(
+ "openviking.observability.events.try_publish_event",
+ lambda event_name, payload: events.append((event_name, payload)),
+ )
+ queue = NamedQueue(object(), "/queue", "AddResource", dequeue_handler=FailingHandler())
+
+ with pytest.raises(RuntimeError, match="failed"):
+ await queue.process_dequeued({"id": "m", "data": "{}"})
+
+ assert events[0][0] == "queue.processed"
+ assert events[0][1] == {
+ "queue": "AddResource",
+ "outcome": "exception",
+ "process_duration_seconds": 1.0,
+ }
+
+
@pytest.mark.parametrize("failure", [RuntimeError("handler failed"), asyncio.CancelledError()])
async def test_handler_exception_still_skips_ack(transport, failure):
class FailingHandler(DequeueHandlerBase):
@@ -178,7 +276,13 @@ async def test_queue_without_middleware_remains_generic(transport):
ProcessResult.cancelled(),
],
)
-async def test_process_short_circuit_settles_once_without_acking(transport, result):
+async def test_process_short_circuit_settles_once_without_acking(transport, result, monkeypatch):
+ events = []
+ monkeypatch.setattr(
+ "openviking.observability.events.try_publish_event",
+ lambda event_name, payload: events.append((event_name, payload)),
+ )
+
class ShortCircuit(QueueMiddleware):
async def process(self, ctx, call_next):
return result
@@ -195,6 +299,8 @@ async def test_process_short_circuit_settles_once_without_acking(transport, resu
assert queue._error_count == (result.outcome is ProcessOutcome.FAILED)
assert queue._processed == (result.outcome is not ProcessOutcome.FAILED)
assert queue._requeue_count == (result.outcome is ProcessOutcome.REQUEUED)
+ assert events[0][0] == "queue.processed"
+ assert events[0][1]["outcome"] == result.outcome.value
transport.write.assert_not_awaited()
diff --git a/tests/storage/test_queue_duration_binding.py b/tests/storage/test_queue_duration_binding.py
new file mode 100644
index 000000000..93055e6b3
--- /dev/null
+++ b/tests/storage/test_queue_duration_binding.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+"""Queue timing integration through the real RAGFS Python binding."""
+
+import time
+
+import pytest
+
+from openviking.storage.queuefs.named_queue import NamedQueue
+from openviking.utils.agfs_utils import RagfsBindingConfig, create_agfs_client
+from openviking_cli.utils.config.agfs_config import AGFSConfig
+
+
+async def test_queuefs_dequeue_exposes_original_enqueue_timestamp(tmp_path):
+ client = create_agfs_client(
+ RagfsBindingConfig(agfs=AGFSConfig(path=str(tmp_path), backend="memory"))
+ )
+ queue = NamedQueue(client, "/queue", "TimingTest")
+ before = time.time()
+
+ await queue.enqueue({"value": 1})
+ message = await queue.dequeue_raw()
+
+ assert message is not None
+ assert message["id"]
+ assert message["data"] == '{"value": 1}'
+ assert before <= float(message["timestamp"]) <= time.time()