Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(metrics): 复用标准 SDK 并修复长耗时统计 (#6474)

Qin Haojie07b5181d61c1
diff --git a/Cargo.lock b/Cargo.lock
index 585b8b6e4..bde98e932 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3564,6 +3564,29 @@ dependencies = [
"vcpkg",
]
+[[package]]
+name = "opentelemetry"
+version = "0.33.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee"
+dependencies = [
+ "js-sys",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.33.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061"
+dependencies = [
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "opentelemetry",
+ "portable-atomic",
+ "thiserror 2.0.18",
+]
+
[[package]]
name = "option-ext"
version = "0.2.0"
@@ -4094,6 +4117,8 @@ dependencies = [
"lru",
"mime_guess",
"native-tls",
+ "opentelemetry",
+ "opentelemetry_sdk",
"path-clean",
"radix_trie",
"rand 0.8.5",
diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml
index c39fb87d7..32b8f7129 100644
--- a/crates/ragfs/Cargo.toml
+++ b/crates/ragfs/Cargo.toml
@@ -45,6 +45,10 @@ serde_json = "1.0"
# Logging
tracing = "0.1"
+# Metrics aggregation; the binding reads cumulative SDK snapshots on demand.
+opentelemetry = { version = "0.33", default-features = false, features = ["metrics"] }
+opentelemetry_sdk = { version = "0.33", default-features = false, features = ["metrics", "experimental_metrics_custom_reader"] }
+
# Git (gitoxide)
gix-hash = "0.14"
gix-object = "0.42"
diff --git a/crates/ragfs/src/core/stats.rs b/crates/ragfs/src/core/stats.rs
index 6907babee..b0eedfe80 100644
--- a/crates/ragfs/src/core/stats.rs
+++ b/crates/ragfs/src/core/stats.rs
@@ -3,12 +3,15 @@
//! This module provides functionality to track filesystem operations,
//! including operation counts and latency statistics.
-use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
+use opentelemetry::KeyValue;
+use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
+use crate::metrics::duration::DurationMetrics;
+
/// Type of filesystem operation
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum FsOperation {
@@ -92,12 +95,6 @@ impl FsOperation {
}
}
-/// Inclusive upper bounds for operation duration buckets, in nanoseconds.
-pub const DURATION_BUCKET_BOUNDS_NS: [u64; 11] = [
- 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000,
- 100_000_000, 500_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000,
-];
-
/// Statistics for a single operation type
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OperationStats {
@@ -113,8 +110,10 @@ pub struct OperationStats {
pub min_time_ns: u64,
/// Maximum time spent (nanoseconds).
pub max_time_ns: u64,
+ /// Inclusive duration boundaries returned by the SDK, in nanoseconds.
+ pub duration_bucket_bounds_ns: Vec<u64>,
/// Noncumulative duration counts; the last bucket holds overflow samples.
- pub duration_bucket_counts: [u64; 12],
+ pub duration_bucket_counts: Vec<u64>,
}
impl Default for OperationStats {
@@ -127,26 +126,13 @@ impl Default for OperationStats {
total_time_ns: 0,
min_time_ns: u64::MAX,
max_time_ns: 0,
- duration_bucket_counts: [0; 12],
+ duration_bucket_bounds_ns: Vec::new(),
+ duration_bucket_counts: Vec::new(),
}
}
}
impl OperationStats {
- /// Record the supplied duration and success flag, updating totals in place.
- pub fn record(&mut self, duration: Duration, success: bool) {
- let ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);
- self.count = self.count.saturating_add(1);
- let outcome = if success { &mut self.success_count } else { &mut self.error_count };
- *outcome = outcome.saturating_add(1);
- self.total_time_ns = self.total_time_ns.saturating_add(ns);
- self.min_time_ns = self.min_time_ns.min(ns);
- self.max_time_ns = self.max_time_ns.max(ns);
- let bucket = DURATION_BUCKET_BOUNDS_NS.partition_point(|bound| *bound < ns);
- self.duration_bucket_counts[bucket] =
- self.duration_bucket_counts[bucket].saturating_add(1);
- }
-
/// Return this operation's average nanoseconds, or zero when no samples exist.
pub fn avg_time_ns(&self) -> f64 {
if self.count == 0 {
@@ -185,15 +171,11 @@ impl FilesystemStats {
self.operations.values().map(|s| s.count).sum()
}
- /// Reset all statistics
- pub fn reset(&mut self) {
- *self = Self::default();
- }
}
/// Thread-safe statistics collector
pub struct StatsCollector {
- stats: Arc<RwLock<FilesystemStats>>,
+ durations: RwLock<DurationMetrics>,
}
impl Default for StatsCollector {
@@ -206,25 +188,65 @@ impl StatsCollector {
/// Create a new statistics collector
pub fn new() -> Self {
Self {
- stats: Arc::new(RwLock::new(FilesystemStats::default())),
+ durations: RwLock::new(DurationMetrics::default()),
}
}
/// Record the supplied operation, duration, and success flag; return unit.
pub async fn record(&self, op: FsOperation, duration: Duration, success: bool) {
- let mut stats = self.stats.write().await;
- stats.operations.entry(op).or_default().record(duration, success);
+ self.durations.read().await.record(
+ duration,
+ &[
+ KeyValue::new("operation", op.as_str()),
+ KeyValue::new("status", if success { "success" } else { "error" }),
+ ],
+ );
}
/// Get a snapshot of current statistics
pub async fn snapshot(&self) -> FilesystemStats {
- self.stats.read().await.clone()
+ let mut snapshot = FilesystemStats::default();
+ for point in self.durations.read().await.snapshot() {
+ let operation = point
+ .attributes()
+ .find(|attribute| attribute.key.as_str() == "operation")
+ .expect("filesystem duration has an operation attribute");
+ let operation = FsOperation::all()
+ .iter()
+ .find(|op| op.as_str() == operation.value.as_str())
+ .expect("filesystem duration uses a known operation");
+ let success = point.attributes().any(|attribute| {
+ attribute.key.as_str() == "status" && attribute.value.as_str() == "success"
+ });
+ let stats = snapshot.operations.get_mut(operation).unwrap();
+ stats.count = stats.count.saturating_add(point.count());
+ if success {
+ stats.success_count = point.count();
+ } else {
+ stats.error_count = point.count();
+ }
+ stats.total_time_ns = stats.total_time_ns.saturating_add(point.sum());
+ stats.min_time_ns = stats.min_time_ns.min(point.min().unwrap_or(u64::MAX));
+ stats.max_time_ns = stats.max_time_ns.max(point.max().unwrap_or(0));
+ if stats.duration_bucket_counts.is_empty() {
+ stats.duration_bucket_bounds_ns = point.bounds().map(|bound| bound as u64).collect();
+ stats.duration_bucket_counts = point.bucket_counts().collect();
+ } else {
+ for (count, incoming) in stats
+ .duration_bucket_counts
+ .iter_mut()
+ .zip(point.bucket_counts())
+ {
+ *count = count.saturating_add(incoming);
+ }
+ }
+ }
+ snapshot
}
/// Reset all statistics
pub async fn reset(&self) {
- let mut stats = self.stats.write().await;
- stats.reset();
+ *self.durations.write().await = DurationMetrics::default();
}
}
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index d7fcbdb19..ece5885c4 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -1293,11 +1293,8 @@ impl PathLockManager {
let descendants = self.provider.scan_descendant_locks(path).await?;
let now_ns = Self::now_ns();
- let mut metrics = self.metrics.write().await;
- metrics.descendant_scan_count += 1;
- metrics.descendant_scan_duration_ns = metrics
- .descendant_scan_duration_ns
- .saturating_add(u64::try_from(scan_start.elapsed().as_nanos()).unwrap_or(u64::MAX));
+ let metrics = self.metrics.read().await;
+ metrics.descendant_scan.record(scan_start.elapsed(), &[]);
drop(metrics);
debug!(path = %path, descendant_count = descendants.len(), scan_ms = scan_start.elapsed().as_millis() as u64, "scanned descendant pathlock tokens");
diff --git a/crates/ragfs/src/lock/metrics.rs b/crates/ragfs/src/lock/metrics.rs
index ca01d4ec3..b768e6436 100644
--- a/crates/ragfs/src/lock/metrics.rs
+++ b/crates/ragfs/src/lock/metrics.rs
@@ -1,6 +1,7 @@
//! Lock observability metrics.
use crate::lock::types::PathLockConflict;
+use crate::metrics::duration::DurationMetrics;
/// Max number of recent conflicts retained for observability.
const MAX_RECENT_CONFLICTS: usize = 32;
@@ -18,10 +19,8 @@ pub struct LockMetrics {
pub stale_leases_released: usize,
/// Cumulative count of stale cleanup failures.
pub stale_cleanup_failures: usize,
- /// Cumulative count of descendant scan operations.
- pub descendant_scan_count: usize,
- /// Total nanoseconds spent in descendant scans.
- pub descendant_scan_duration_ns: u64,
+ /// SDK-owned descendant scan duration distribution.
+ pub(crate) descendant_scan: DurationMetrics,
/// Total nanoseconds spent waiting for successfully acquired locks.
pub wait_duration_ns: u64,
/// Count of conflicts by type (exact/exact, tree/exact, etc.).
diff --git a/crates/ragfs/src/metrics/duration.rs b/crates/ragfs/src/metrics/duration.rs
new file mode 100644
index 000000000..5e9b7bf20
--- /dev/null
+++ b/crates/ragfs/src/metrics/duration.rs
@@ -0,0 +1,102 @@
+//! SDK-owned duration aggregation shared by filesystem and path-lock instrumentation.
+
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+
+use opentelemetry::metrics::{Histogram, MeterProvider};
+use opentelemetry::KeyValue;
+use opentelemetry_sdk::error::OTelSdkResult;
+use opentelemetry_sdk::metrics::data::{
+ AggregatedMetrics, HistogramDataPoint, MetricData, ResourceMetrics,
+};
+use opentelemetry_sdk::metrics::reader::MetricReader;
+use opentelemetry_sdk::metrics::{
+ InstrumentKind, ManualReader, Pipeline, SdkMeterProvider, Temporality,
+};
+
+/// Retain the same reader in the provider and the native snapshot owner.
+#[derive(Clone, Debug)]
+struct SharedReader(Arc<ManualReader>);
+
+impl MetricReader for SharedReader {
+ fn register_pipeline(&self, pipeline: Weak<Pipeline>) {
+ self.0.register_pipeline(pipeline);
+ }
+
+ fn collect(&self, metrics: &mut ResourceMetrics) -> OTelSdkResult {
+ self.0.collect(metrics)
+ }
+
+ fn force_flush(&self) -> OTelSdkResult {
+ self.0.force_flush()
+ }
+
+ fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
+ self.0.shutdown_with_timeout(timeout)
+ }
+
+ fn temporality(&self, kind: InstrumentKind) -> Temporality {
+ self.0.temporality(kind)
+ }
+}
+
+/// Own one SDK provider for the lifetime of the measured subsystem.
+#[derive(Clone, Debug)]
+pub(crate) struct DurationMetrics {
+ // Dropping the final provider handle shuts down its reader.
+ _provider: SdkMeterProvider,
+ reader: SharedReader,
+ histogram: Histogram<u64>,
+}
+
+impl Default for DurationMetrics {
+ fn default() -> Self {
+ let reader = SharedReader(Arc::new(ManualReader::default()));
+ let provider = SdkMeterProvider::builder()
+ .with_reader(reader.clone())
+ .build();
+ // A bounded 1/2/5 scale covers short I/O through multi-day background work.
+ // This is SDK collection policy, not a timeout or a per-operation setting.
+ let boundaries = (5..=15)
+ .flat_map(|power| [1, 2, 5].map(|factor| (factor * 10_u64.pow(power)) as f64))
+ .collect();
+ let histogram = provider
+ .meter("ragfs")
+ .u64_histogram("duration")
+ .with_unit("ns")
+ .with_boundaries(boundaries)
+ .build();
+ Self {
+ _provider: provider,
+ reader,
+ histogram,
+ }
+ }
+}
+
+impl DurationMetrics {
+ pub(crate) fn record(&self, duration: Duration, attributes: &[KeyValue]) {
+ self.histogram.record(
+ u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX),
+ attributes,
+ );
+ }
+
+ pub(crate) fn snapshot(&self) -> Vec<HistogramDataPoint<u64>> {
+ let mut metrics = ResourceMetrics::default();
+ // The provider and reader have the same owner; neither is shut down independently.
+ self.reader
+ .collect(&mut metrics)
+ .expect("live duration metrics provider");
+ metrics
+ .scope_metrics()
+ .flat_map(|scope| scope.metrics())
+ .flat_map(|metric| match metric.data() {
+ AggregatedMetrics::U64(MetricData::Histogram(histogram)) => {
+ histogram.data_points().cloned().collect::<Vec<_>>()
+ }
+ _ => unreachable!("duration provider contains only a u64 histogram"),
+ })
+ .collect()
+ }
+}
diff --git a/crates/ragfs/src/metrics/mod.rs b/crates/ragfs/src/metrics/mod.rs
index 7f6f1ec47..97602e028 100644
--- a/crates/ragfs/src/metrics/mod.rs
+++ b/crates/ragfs/src/metrics/mod.rs
@@ -2,10 +2,11 @@
use std::collections::BTreeMap;
-use crate::core::stats::DURATION_BUCKET_BOUNDS_NS;
use crate::core::{Error, FilesystemStats, FsOperation, Result};
use crate::lock::metrics::LockMetrics;
+pub(crate) mod duration;
+
/// One metric series, identified by its name and bounded labels.
#[derive(Debug, Clone, PartialEq)]
pub struct RagfsMetric {
@@ -68,6 +69,9 @@ pub(crate) fn operation_metrics(plugin: &str, stats: &FilesystemStats) -> Vec<Ra
1.0,
));
}
+ if stats.count == 0 {
+ continue;
+ }
metrics.push(RagfsMetric {
name: "ragfs_operation_duration_seconds".into(),
labels: [
@@ -75,7 +79,7 @@ pub(crate) fn operation_metrics(plugin: &str, stats: &FilesystemStats) -> Vec<Ra
("operation".into(), operation.as_str().into()),
].into(),
value: RagfsMetricValue::Histogram {
- bucket_bounds: DURATION_BUCKET_BOUNDS_NS.to_vec(),
+ bucket_bounds: stats.duration_bucket_bounds_ns.clone(),
bucket_counts: stats.duration_bucket_counts.to_vec(),
count: stats.count,
sum: stats.total_time_ns,
@@ -128,6 +132,7 @@ pub(crate) fn cache_metrics(stats: crate::cache::CacheMetricsSnapshot) -> Vec<Ra
/// Convert the single lock manager's snapshot into unlabeled metric records.
pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {
let mut metrics = Vec::new();
+ let scan = stats.descendant_scan.snapshot().into_iter().next();
for (name, value) in [
("lock_active", stats.active_lock_count),
("lock_waiting", stats.waiting_lock_count),
@@ -142,13 +147,28 @@ pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {
for (name, value) in [
("lock_conflicts_total", stats.conflict_count),
("lock_stale_leases_released_total", stats.stale_leases_released),
- ("lock_descendant_scans_total", stats.descendant_scan_count),
] {
metrics.push(RagfsMetric::counter(name, &[], value as u64, 1.0));
}
metrics.push(RagfsMetric::counter(
- "lock_descendant_scan_duration_seconds_total", &[], stats.descendant_scan_duration_ns, 1e-9,
+ "lock_descendant_scans_total", &[], scan.as_ref().map_or(0, |point| point.count()), 1.0,
));
+ metrics.push(RagfsMetric::counter(
+ "lock_descendant_scan_duration_seconds_total", &[], scan.as_ref().map_or(0, |point| point.sum()), 1e-9,
+ ));
+ if let Some(scan) = scan {
+ metrics.push(RagfsMetric {
+ name: "lock_descendant_scan_duration_seconds".into(),
+ labels: BTreeMap::new(),
+ value: RagfsMetricValue::Histogram {
+ bucket_bounds: scan.bounds().map(|bound| bound as u64).collect(),
+ bucket_counts: scan.bucket_counts().collect(),
+ count: scan.count(),
+ sum: scan.sum(),
+ scale: 1e-9,
+ },
+ });
+ }
metrics
}
diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md
index 8f0451a40..e52b98590 100644
--- a/docs/en/concepts/12-metrics.md
+++ b/docs/en/concepts/12-metrics.md
@@ -84,11 +84,11 @@ Collectors turn inputs into metric semantics:
### MetricRegistry
-The MetricRegistry is the in-process metric store that keeps the current metric values and serves them to the exporter.
+MetricRegistry keeps OpenViking metric names, label contracts, and account-dimension rules. Event counts and durations are accumulated by the OpenTelemetry SDK. Cumulative values from RAGFS and state collectors enter as snapshots, without being replayed as new observations.
### Exporter
-The first exporter implementation is the Prometheus exporter, which renders registry contents into Prometheus exposition text.
+`/metrics` uses the official Prometheus Python client for text exposition. OTLP HTTP/gRPC uses the official OpenTelemetry exporters for encoding, delivery, and retries. Existing endpoint, headers, TLS, and export-interval settings remain unchanged.
## Usage
@@ -139,6 +139,36 @@ Notes:
The tables list representative metrics defined by the collectors. A series appears only when its feature is enabled, the relevant event occurs, and collection succeeds. An absent series does not necessarily mean zero.
+### Duration distributions and percentiles
+
+Instrumentation records actual durations and labels; Python and RAGFS delegate duration
+aggregation to their respective OpenTelemetry SDKs. The integration layer supplies the default
+collection policy, so business collectors do not maintain bucket boundaries. Python starts at
+0.01 seconds and RAGFS at 0.0001 seconds, using a 1/2/5 scale per decade through 5,000,000 seconds
+(about 58 days). Larger observations remain in `+Inf`. These boundaries are not timeouts;
+dashboards must not filter slow operations with `< 10` or a fixed axis maximum.
+
+Prometheus `_bucket`, `_count`, `_sum` series and OTLP histograms carry `bucket_schema`,
+derived from their actual boundaries; collectors cannot set it themselves.
+**Keep `bucket_schema` when aggregating percentiles**
+so rolling upgrades cannot mix incompatible bucket layouts. Older series without this label
+form a separate group; their lost overflow distribution cannot be reconstructed.
+
+```promql
+histogram_quantile(0.99,
+ sum by (le, bucket_schema, operation) (
+ rate(openviking_operation_duration_seconds_bucket[5m])
+ )
+)
+```
+
+Use `0.95` for P95. Compute the mean from `rate(_sum) / rate(_count)` with the same
+`operation, bucket_schema` grouping, showing it only when the count is positive.
+No completed observations must remain a gap, not zero latency. Percentiles are still
+bucket-interpolated estimates, particularly with low traffic or coarse buckets.
+Each default Python duration histogram label set exports 30 Prometheus series; RAGFS exports 36.
+Account for the extra buckets in scrape and storage capacity planning.
+
### Requests and Operations
| Metric Family | Type | Common Labels | Meaning |
@@ -255,6 +285,7 @@ Notes:
| `openviking_lock_stale_leases_released_total` | Counter | none | stale leases released |
| `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration |
+| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | completed descendant scan duration distribution, supporting mean, P95, and P99 |
The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.
@@ -302,7 +333,10 @@ seconds, for example `123 ns = 0.000000123 seconds`.
These metrics have no `mount` or `account_id` label. Cache and multi-backend
families are absent when those capabilities are not enabled. `status` is
`success` or `error`; `exists=false` is successful and `replace` uses `rename`.
-Histogram buckets cover all outcomes, with finite bounds from 0.0001 to 10 seconds.
+Histograms cover all outcomes and are accumulated by the Rust OpenTelemetry SDK;
+exported boundaries and counts come directly from SDK snapshots.
+Histograms are emitted only after an operation has observations, avoiding empty long-tail
+buckets for every plugin/operation pair; existing operation counters remain available.
Python `get_stats()` retains its microsecond fields.
The protocol family always contains both closed-label series. One unreadable
diff --git a/docs/en/guides/11-grafana-prometheus.md b/docs/en/guides/11-grafana-prometheus.md
index 67bd83e8e..ee1f0b74d 100644
--- a/docs/en/guides/11-grafana-prometheus.md
+++ b/docs/en/guides/11-grafana-prometheus.md
@@ -333,7 +333,7 @@ sum by (route, status) (rate(openviking_http_requests_total[5m]))
P95 latency:
```promql
-histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[5m])))
+histogram_quantile(0.95, sum by (le, route, bucket_schema) (rate(openviking_http_request_duration_seconds_bucket[5m])))
```
Queue backlog:
diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md
index 283fba384..d07ee5cbd 100644
--- a/docs/zh/concepts/12-metrics.md
+++ b/docs/zh/concepts/12-metrics.md
@@ -84,11 +84,11 @@ Collector 负责把输入转成指标语义:
### MetricRegistry
-MetricRegistry 是进程内的指标注册中心,用于保存当前指标值,并在导出时统一读取。
+MetricRegistry 保留 OpenViking 的指标名、标签约束和账户维度规则。单次事件的计数与耗时直接交给 OpenTelemetry SDK 累计;RAGFS 和状态采集器返回的累计值作为快照接入,不会重新作为事件计数。
### Exporter
-当前首个落地导出器是 Prometheus Exporter,用于把 registry 中的指标渲染成 Prometheus exposition 文本。
+`/metrics` 使用官方 Prometheus Python 客户端输出文本;OTLP HTTP/gRPC 使用 OpenTelemetry 官方导出器处理编码、发送和重试。现有 endpoint、headers、TLS 和上报周期配置保持不变。
## 使用方式
@@ -139,6 +139,31 @@ scrape_configs:
下面列出 Collector 定义的代表性指标。某个指标是否出现,取决于功能是否启用、是否发生对应事件,以及 Collector 是否采集成功;未出现不能直接解释为数值为零。
+### 耗时分布与分位数
+
+业务打点只记录实际耗时和标签;Python 与 RAGFS 的耗时累计由各自的 OpenTelemetry SDK 完成。
+接入层统一配置默认统计范围,业务采集器无需维护桶边界。Python 从 0.01 秒、RAGFS 从 0.0001 秒起,
+按每个十倍数量级的 1、2、5 档覆盖到 5,000,000 秒(约 58 天),超出范围仍计入 `+Inf`。
+这里的范围不是任务超时;看板不应通过 `< 10` 或固定纵轴上限过滤慢请求。
+
+Prometheus 的 `_bucket`、`_count`、`_sum` 和 OTLP Histogram 都带有 `bucket_schema`,
+该标签由实际桶边界计算,采集器不能自行设置。**聚合分位数时必须保留 `bucket_schema`**,避免滚动升级时把
+新旧分桶混算。没有该标签的旧数据会形成独立的一组;其已经合并到最后一桶的细节无法恢复。
+
+```promql
+histogram_quantile(0.99,
+ sum by (le, bucket_schema, operation) (
+ rate(openviking_operation_duration_seconds_bucket[5m])
+ )
+)
+```
+
+P95 将 `0.99` 改为 `0.95`。平均耗时使用同一组 `operation, bucket_schema` 的
+`rate(_sum) / rate(_count)`,仅在次数大于零时显示;没有完成记录时不能用零耗时代替。
+分位数仍是桶内插值估算,低流量或粗粒度桶内的结果不等于精确的单次请求耗时。
+默认每组 Python 耗时直方图输出 30 条 Prometheus 时序,RAGFS 输出 36 条;
+应将新增桶的采集和存储成本计入容量规划。
+
### 请求与操作
| 指标族 | 类型 | 常见标签 | 含义 |
@@ -256,6 +281,7 @@ Telemetry bridge 还识别旧摘要键,对应 `request`、`process`、`parse`
| `openviking_lock_stale_leases_released_total` | Counter | 无 | 已释放的过期租约数 |
| `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 |
| `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 |
+| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | 已完成后代锁扫描的耗时分布,可计算平均、P95 和 P99 |
队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。
@@ -301,7 +327,8 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba
这些指标不带 `mount` 和 `account_id` 标签。未启用 Cache 或 multi-backend
时,不生成对应指标族。`status` 为 `success` 或 `error`。
`exists=false` 计为成功;`replace` 使用 `rename` 操作标签。
-Histogram 包含全部操作结果,有限桶边界范围为 0.0001 至 10 秒。
+Histogram 包含全部操作结果,使用 Rust OpenTelemetry SDK 统计;输出边界和计数直接来自 SDK。
+尚未发生的操作不导出空直方图,避免为每个插件和操作持续发布无样本的长尾桶;原有操作计数器保留。
Python `get_stats()` 保留原有微秒字段。
protocol 指标始终包含闭集中的两条 series。任何 protocol、account 或 segment
diff --git a/docs/zh/guides/11-grafana-prometheus.md b/docs/zh/guides/11-grafana-prometheus.md
index 5d84f7fd8..fdf636f45 100644
--- a/docs/zh/guides/11-grafana-prometheus.md
+++ b/docs/zh/guides/11-grafana-prometheus.md
@@ -333,7 +333,7 @@ sum by (route, status) (rate(openviking_http_requests_total[5m]))
P95 延迟:
```promql
-histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[5m])))
+histogram_quantile(0.95, sum by (le, route, bucket_schema) (rate(openviking_http_request_duration_seconds_bucket[5m])))
```
队列积压:
diff --git a/examples/grafana/openviking_demo_dashboard.json b/examples/grafana/openviking_demo_dashboard.json
index da863a8c7..71b1cca12 100644
--- a/examples/grafana/openviking_demo_dashboard.json
+++ b/examples/grafana/openviking_demo_dashboard.json
@@ -324,8 +324,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "HTTP P95",
+ "expr": "histogram_quantile(0.95, sum by (le, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "HTTP P95 / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -1586,8 +1586,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{route}}",
+ "expr": "histogram_quantile(0.95, sum by (le, route, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{route}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -1687,8 +1687,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -2219,8 +2219,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, context_type) (rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{context_type}}",
+ "expr": "histogram_quantile(0.95, sum by (le, context_type, bucket_schema) (label_replace(rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{context_type}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -2320,8 +2320,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -2623,8 +2623,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(openviking_operation_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{operation}}",
+ "expr": "histogram_quantile(0.95, sum by (le, operation, bucket_schema) (label_replace(rate(openviking_operation_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{operation}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -2724,8 +2724,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_operation_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_operation_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -3041,8 +3041,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{stage}}",
+ "expr": "histogram_quantile(0.95, sum by (le, stage, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{stage}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -3142,8 +3142,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -3858,8 +3858,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, provider, model_name) (rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{provider}} / {{model_name}}",
+ "expr": "histogram_quantile(0.95, sum by (le, provider, model_name, bucket_schema) (label_replace(rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{provider}} / {{model_name}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -3959,8 +3959,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -4262,8 +4262,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, status) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{status}}",
+ "expr": "histogram_quantile(0.95, sum by (le, status, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{status}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -4363,8 +4363,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval])))",
- "legendFormat": "{{account_id}}",
+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
@@ -4895,7 +4895,24 @@
},
"unit": "short"
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "D"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ },
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 8,
@@ -4953,8 +4970,8 @@
},
{
"editorMode": "code",
- "expr": "topk(10, histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]))))",
- "legendFormat": "embedding p95 {{account_id}}",
+ "expr": "(histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))) and on (account_id) (topk(10, sum by (account_id) (rate(openviking_vlm_calls_total[$__rate_interval]))))",
+ "legendFormat": "embedding p95 {{account_id}} / {{bucket_schema}}",
"range": true,
"refId": "D"
}
@@ -5054,7 +5071,7 @@
"x": 0,
"y": 196
},
- "id": 59,
+ "id": 64,
"panels": [],
"title": "Memory Extraction",
"type": "row"
@@ -5089,7 +5106,7 @@
"x": 0,
"y": 197
},
- "id": 60,
+ "id": 65,
"options": {
"legend": {
"calcs": [],
@@ -5245,8 +5262,8 @@
"targets": [
{
"editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(openviking_resource_stage_duration_seconds_bucket{stage=~\"semantic_queue_wait|semantic_execute|embedding_queue_wait|embedding_execute\"}[$__rate_interval])))",
- "legendFormat": "{{stage}}",
+ "expr": "histogram_quantile(0.95, sum by (le, stage, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket{stage=~\"semantic_queue_wait|semantic_execute|embedding_queue_wait|embedding_execute\"}[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",
+ "legendFormat": "{{stage}} / {{bucket_schema}}",
"range": true,
"refId": "A"
}
diff --git a/openviking/metrics/collectors/base.py b/openviking/metrics/collectors/base.py
index 936a1ed98..443115275 100644
--- a/openviking/metrics/collectors/base.py
+++ b/openviking/metrics/collectors/base.py
@@ -170,7 +170,7 @@ class CollectorMetricWriter:
kwargs["buckets"] = buckets
self._registry.observe_histogram(name, value, **kwargs)
- def gauge_delete_matching(
+ def delete_matching(
self,
name: str,
*,
@@ -179,7 +179,7 @@ class CollectorMetricWriter:
) -> None:
"""Delete matching gauge series while preserving the current account partition."""
if not metric_supports_account_dimension(name):
- self._registry.gauge_delete_matching(name, match_labels=dict(match_labels))
+ self._registry.delete_matching(name, match_labels=dict(match_labels))
return
final_account_id = resolve_metric_account_label(
metric_name=name,
@@ -188,7 +188,7 @@ class CollectorMetricWriter:
)
labels = dict(match_labels)
labels["account_id"] = final_account_id
- self._registry.gauge_delete_matching(name, match_labels=labels)
+ self._registry.delete_matching(name, match_labels=labels)
def _with_account_labels(
self,
@@ -341,7 +341,7 @@ class StateMetricCollector(MetricCollector, Refreshable, ABC):
label_names: tuple[str, ...] = (),
) -> None:
"""Replace the full gauge series selected by `match_labels` with one fresh value."""
- registry.gauge_delete_matching(str(metric_name), match_labels=match_labels)
+ registry.delete_matching(str(metric_name), match_labels=match_labels)
if labels is None:
registry.set_gauge(str(metric_name), float(value))
return
@@ -421,7 +421,7 @@ class DomainStatsMetricCollector(MetricCollector, Refreshable, ABC):
) -> None:
"""Replace all existing global gauge series for the provided metric names in one pass."""
for metric_name, value in zip(metric_names, values, strict=True):
- registry.gauge_delete_matching(str(metric_name), match_labels={})
+ registry.delete_matching(str(metric_name), match_labels={})
registry.set_gauge(
str(metric_name),
float(value),
@@ -534,7 +534,7 @@ class ProbeMetricCollector(MetricCollector, Refreshable, ABC):
label_names: tuple[str, ...] = (),
) -> None:
"""Replace the selected gauge series with one fresh readiness-style value."""
- registry.gauge_delete_matching(str(metric_name), match_labels=match_labels)
+ registry.delete_matching(str(metric_name), match_labels=match_labels)
if labels is None:
registry.set_gauge(str(metric_name), float(value))
return
@@ -601,4 +601,4 @@ class ProbeMetricCollector(MetricCollector, Refreshable, ABC):
) -> None:
"""Delete all gauge series whose label values are listed in `values`."""
for value in values:
- registry.gauge_delete_matching(str(metric_name), match_labels={label_key: str(value)})
+ registry.delete_matching(str(metric_name), match_labels={label_key: str(value)})
diff --git a/openviking/metrics/collectors/feedback.py b/openviking/metrics/collectors/feedback.py
index 285c0cc22..5cf6cc859 100644
--- a/openviking/metrics/collectors/feedback.py
+++ b/openviking/metrics/collectors/feedback.py
@@ -169,7 +169,7 @@ class FeedbackCollector(DomainStatsMetricCollector):
channels: dict[str, dict[str, Any]],
) -> None:
for metric_name, _ in self._SUMMARY_METRICS:
- registry.gauge_delete_matching(metric_name, match_labels={})
+ registry.delete_matching(metric_name, match_labels={})
for metric_name, key in self._SUMMARY_METRICS:
registry.set_gauge(
metric_name,
@@ -179,7 +179,7 @@ class FeedbackCollector(DomainStatsMetricCollector):
)
for metric_name, _ in self._CHANNEL_METRICS:
- registry.gauge_delete_matching(metric_name, match_labels={})
+ registry.delete_matching(metric_name, match_labels={})
for channel, channel_stats in channels.items():
labels = {"channel": str(channel), "valid": str(valid)}
for metric_name, key in self._CHANNEL_METRICS:
diff --git a/openviking/metrics/collectors/manager.py b/openviking/metrics/collectors/manager.py
index 82214bb13..ec1ab964e 100644
--- a/openviking/metrics/collectors/manager.py
+++ b/openviking/metrics/collectors/manager.py
@@ -68,11 +68,11 @@ class CollectorManager:
If the collector has `config.ttl_seconds`, a gate is created to enforce TTL + SWR.
"""
- if not isinstance(collector, Refreshable):
- raise TypeError("CollectorManager only accepts Refreshable collectors")
+ if not isinstance(collector, MetricCollector) or not isinstance(collector, Refreshable):
+ raise TypeError("CollectorManager only accepts refreshable MetricCollector instances")
self._collectors.append(collector)
- collector_name = self._collector_name(collector)
- ttl_seconds = getattr(cast(Refreshable, collector).config, "ttl_seconds", None)
+ collector_name = collector.collector_name()
+ ttl_seconds = cast(Refreshable, collector).config.ttl_seconds
if ttl_seconds is not None:
self._gates[collector_name] = RefreshGate(ttl_seconds=float(ttl_seconds))
@@ -96,9 +96,9 @@ class CollectorManager:
tasks: list[asyncio.Task[RefreshResult]] = []
for c in self._collectors:
- collector_name = self._collector_name(c)
+ collector_name = c.collector_name()
gate = self._gates.get(collector_name)
- timeout_seconds = float(getattr(cast(Refreshable, c).config, "timeout_seconds", 0.5))
+ timeout_seconds = cast(Refreshable, c).config.timeout_seconds
if gate is None:
tasks.append(
asyncio.create_task(
@@ -222,37 +222,22 @@ class CollectorManager:
asyncio.to_thread(collector.collect, registry), timeout=timeout
)
return RefreshResult(
- collector=self._collector_name(collector),
+ collector=collector.collector_name(),
attempted=collected is not False,
success=collected is not False,
reason="skipped" if collected is False else "ok",
)
except asyncio.TimeoutError:
return RefreshResult(
- collector=self._collector_name(collector),
+ collector=collector.collector_name(),
attempted=True,
success=False,
reason="timeout",
)
except Exception:
return RefreshResult(
- collector=self._collector_name(collector),
+ collector=collector.collector_name(),
attempted=True,
success=False,
reason="exception",
)
-
- def _collector_name(self, collector: MetricCollector) -> str:
- """
- Resolve the display name used for refresh results and refresh-gate bookkeeping.
-
- The manager prefers the collector's explicit `collector_name()` method and falls back to
- a `name` attribute or the concrete class name for compatibility.
- """
- try:
- return str(collector.collector_name())
- except AttributeError:
- name = getattr(collector, "name", None)
- if name is not None:
- return str(name)
- return collector.__class__.__name__
diff --git a/openviking/metrics/collectors/model_usage.py b/openviking/metrics/collectors/model_usage.py
index b21fc1d45..a26d76541 100644
--- a/openviking/metrics/collectors/model_usage.py
+++ b/openviking/metrics/collectors/model_usage.py
@@ -139,7 +139,7 @@ class ModelUsageCollector(DomainStatsMetricCollector):
The replacement deletes any prior series for the same `model_type`, ensuring only one of
`valid="1"` or `valid="0"` remains visible at a time.
"""
- registry.gauge_delete_matching(
+ registry.delete_matching(
self.USAGE_AVAILABLE,
match_labels={"model_type": str(model_type)},
)
diff --git a/openviking/metrics/collectors/queue.py b/openviking/metrics/collectors/queue.py
index e8b1ac417..79f261bbe 100644
--- a/openviking/metrics/collectors/queue.py
+++ b/openviking/metrics/collectors/queue.py
@@ -12,26 +12,6 @@ from openviking.metrics.datasources.queue import QueuePipelineStateDataSource
from .base import CollectorConfig, EventMetricCollector, StateMetricCollector
-QUEUE_DURATION_BUCKETS: tuple[float, ...] = (
- 0.01,
- 0.05,
- 0.1,
- 0.25,
- 0.5,
- 1.0,
- 2.5,
- 5.0,
- 10.0,
- 30.0,
- 60.0,
- 120.0,
- 300.0,
- 600.0,
- 1800.0,
- 3600.0,
- 21600.0,
- 86400.0,
-)
_QUEUE_OUTCOMES = frozenset({"success", "failed", "requeued", "cancelled", "exception"})
@@ -69,7 +49,6 @@ class QueueDurationCollector(EventMetricCollector):
process_duration,
labels=labels,
label_names=("queue", "outcome"),
- buckets=QUEUE_DURATION_BUCKETS,
)
raw_end_to_end = payload.get("end_to_end_duration_seconds")
@@ -86,7 +65,6 @@ class QueueDurationCollector(EventMetricCollector):
end_to_end_duration,
labels=labels,
label_names=("queue", "outcome"),
- buckets=QUEUE_DURATION_BUCKETS,
)
diff --git a/openviking/metrics/collectors/ragfs.py b/openviking/metrics/collectors/ragfs.py
index b0bf3d6b3..4c802ad9c 100644
--- a/openviking/metrics/collectors/ragfs.py
+++ b/openviking/metrics/collectors/ragfs.py
@@ -21,7 +21,7 @@ class RagfsMetricCollector(DomainStatsMetricCollector):
data_source: RagfsMetricDataSource
config: CollectorConfig = CollectorConfig(ttl_seconds=None, timeout_seconds=1.0)
_collect_lock: Lock = field(default_factory=Lock, init=False, repr=False)
- _tracked_series: set[tuple[str, str, tuple[tuple[str, str], ...]]] = field(
+ _tracked_series: set[tuple[str, tuple[tuple[str, str], ...]]] = field(
default_factory=set, init=False, repr=False
)
@@ -72,10 +72,10 @@ class RagfsMetricCollector(DomainStatsMetricCollector):
labels=labels,
label_names=label_names,
)
- identity = (kind, name, tuple(sorted(labels.items())))
+ identity = (name, tuple(sorted(labels.items())))
current_series.add(identity)
self._tracked_series.add(identity)
for identity in sorted(self._tracked_series - current_series):
- kind, name, labels = identity
- getattr(registry, f"{kind}_delete_matching")(name, match_labels=dict(labels))
+ name, labels = identity
+ registry.delete_matching(name, match_labels=dict(labels))
self._tracked_series.remove(identity)
diff --git a/openviking/metrics/collectors/task_tracker.py b/openviking/metrics/collectors/task_tracker.py
index b55049963..e7a86886e 100644
--- a/openviking/metrics/collectors/task_tracker.py
+++ b/openviking/metrics/collectors/task_tracker.py
@@ -57,7 +57,7 @@ class TaskTrackerCollector(StateMetricCollector):
self.FAILED,
self.CANCELLED,
):
- registry.gauge_delete_matching(metric_name, match_labels={})
+ registry.delete_matching(metric_name, match_labels={})
for task_type, counts in counts_by_type.items():
labels = {"task_type": str(task_type)}
registry.set_gauge(
diff --git a/openviking/metrics/core/registry.py b/openviking/metrics/core/registry.py
index 8512d2b0c..11aec8523 100644
--- a/openviking/metrics/core/registry.py
+++ b/openviking/metrics/core/registry.py
@@ -1,21 +1,10 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""
-In-process metric registry used by OpenViking.
-
-Design goals:
-1) Keep the registry as a "current state store" for metrics (no snapshot/view layer).
-2) Provide a strict label contract per metric name to prevent accidental high-cardinality exports.
-3) Be safe under concurrent updates from request threads and exporter refresh threads.
-4) Allow bounded memory usage via per-metric series limits, while exposing drop counts for debugging.
+"""Bind OpenViking metric names and tags to OpenTelemetry SDK instruments.
-Important behaviors:
-- Label normalization: label dicts are normalized into a sorted tuple of (key, value) pairs.
-- Label contract: once a metric name is registered with a label key set, subsequent writes must
- use the same label keys (same keys, same order). Violations raise ValueError.
-- Series limit: when a metric family reaches `max_series_per_metric`, new series are dropped
- and `openviking_metrics_dropped_series_total{metric="<name>"}` will reflect the drops.
+The SDK owns event aggregation. Already accumulated native/state values are kept
+as SDK data points and exported as snapshots, never replayed as observations.
"""
from __future__ import annotations
@@ -23,835 +12,256 @@ from __future__ import annotations
import math
import threading
import time
-from bisect import bisect_left
-from typing import Mapping, Sequence
-
-from .types import normalize_labels
-
-DEFAULT_LATENCY_BUCKETS: tuple[float, ...] = (0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
-
-
-def _canonicalize_label_names(label_names: Sequence[str]) -> tuple[str, ...]:
- """
- Return the canonical family label order used by the in-process registry.
-
- The registry normalizes concrete label mappings into key-sorted tuples, so family-level
- `label_names` must follow the same stable order to avoid false mismatches between writers
- that provide the same key set with different input ordering.
- """
- return tuple(sorted(str(name) for name in label_names))
-
-
-def _validate_label_keys(
- name: str,
- label_names: tuple[str, ...],
- normalized: tuple[tuple[str, str], ...],
-) -> None:
- """Check normalized labels against the named metric's declared keys; return None."""
- if not label_names and normalized:
- raise ValueError(f"metric {name} does not accept labels")
- if label_names and tuple(k for k, _ in normalized) != label_names:
- raise ValueError(f"metric {name} label keys mismatch: expected {label_names}")
-
-
-def _validate_counter_value(value: float) -> float:
- """Return value as a finite, nonnegative float or raise ValueError."""
- value = float(value)
- if not math.isfinite(value) or value < 0:
- raise ValueError("counter value must be finite and nonnegative")
- return value
-
-
-def _validate_histogram_values(
- bounds: tuple[float, ...],
- bucket_counts: Sequence[int],
- count: int,
- value_sum: float,
-) -> tuple[tuple[int, ...], float]:
- """Validate bounds and totals; return bucket_counts as a tuple and value_sum as a float."""
- bucket_counts = tuple(bucket_counts)
- value_sum = float(value_sum)
- if any(not math.isfinite(bound) or bound < 0 for bound in bounds) or any(
- left >= right for left, right in zip(bounds, bounds[1:], strict=False)
- ):
- raise ValueError("histogram bounds must be finite, nonnegative and increasing")
- if len(bucket_counts) != len(bounds) + 1 or any(
- type(value) is not int or value < 0 for value in (*bucket_counts, count)
- ):
- raise ValueError("histogram counts must be nonnegative integers matching bounds")
- if sum(bucket_counts) != count or not math.isfinite(value_sum) or value_sum < 0:
- raise ValueError("histogram count or sum is invalid")
- return bucket_counts, value_sum
-
-
-def _labels_contains(
- labels: tuple[tuple[str, str], ...],
- match: tuple[tuple[str, str], ...],
-) -> bool:
- """
- Return whether a normalized label set contains all requested label pairs.
-
- This helper is used by partial-delete operations where callers provide only a subset
- of labels and expect every matching series to be removed.
- """
- if not match:
- return True
- label_set = set(labels)
- for item in match:
- if item not in label_set:
- return False
- return True
+from dataclasses import dataclass, field, replace
+from typing import Mapping
+
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+ AggregationTemporality,
+ Gauge,
+ Histogram,
+ HistogramDataPoint,
+ InMemoryMetricReader,
+ Metric,
+ MetricsData,
+ NumberDataPoint,
+ ResourceMetrics,
+ ScopeMetrics,
+ Sum,
+)
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.util.instrumentation import InstrumentationScope
+
+from .types import histogram_schema, normalize_labels
+
+# SDK collection policy: a compact 1/2/5 scale from milliseconds to multi-day work.
+# Business collectors record actual seconds and do not manage these boundaries.
+DEFAULT_LATENCY_BUCKETS = tuple(
+ factor * 10.0**power for power in range(-2, 7) for factor in (1, 2, 5)
+)
+
+
+@dataclass
+class _MetricBinding:
+ kind: str
+ label_names: tuple[str, ...]
+ mode: str
+ bounds: tuple[float, ...] = ()
+ instrument: object | None = None
+ points: dict[tuple[tuple[str, str], ...], NumberDataPoint | HistogramDataPoint | None] = field(
+ default_factory=dict
+ )
class MetricRegistry:
- """
- MetricRegistry holds Counter/Gauge/Histogram families in memory.
-
- The exporter reads current values via `iter_counters/iter_gauges/iter_histograms`.
- Collectors should use the unified write helpers (`inc_counter/set_gauge/observe_histogram`)
- to keep call sites consistent and reduce accidental label misuse.
- """
+ """Keep metric contracts and adapt state snapshots; delegate accumulation to the SDK."""
def __init__(self, *, max_series_per_metric: int = 2048) -> None:
- """
- Initialize an empty in-process metrics registry.
-
- Args:
- max_series_per_metric: Hard cap for the number of distinct label series allowed
- inside one metric family before additional series are dropped.
- """
- self._max_series_per_metric = max_series_per_metric
- self._lock = threading.Lock()
- self._counters: dict[str, _CounterFamily] = {}
- self._gauges: dict[str, _GaugeFamily] = {}
- self._histograms: dict[str, _HistogramFamily] = {}
- self._dropped_series_total: dict[str, int] = {}
-
- def inc_counter(
- self,
- name: str,
- *,
- amount: float = 1.0,
- labels: Mapping[str, str] | None = None,
- label_names: Sequence[str] = (),
- ) -> None:
- """
- Increment a Counter metric.
-
- Args:
- name: Prometheus metric name.
- amount: Increment delta (float, will be rendered as int when possible).
- labels: Optional label dict. Keys must exactly match `label_names`.
- label_names: Ordered label key tuple for this metric name.
-
- Notes:
- This helper enforces the per-metric label contract by delegating to the underlying
- Counter family, which validates label keys.
- """
- self.counter(name, label_names=label_names).inc(labels=labels, amount=float(amount))
-
- def set_counter(
- self,
- name: str,
- value: float,
- *,
- labels: Mapping[str, str] | None = None,
- label_names: Sequence[str] = (),
- ) -> None:
- """Replace the named counter's labelled value, including zero or reset; return None."""
- value = _validate_counter_value(value)
- ln = _canonicalize_label_names(label_names)
- normalized = normalize_labels(labels)
- _validate_label_keys(name, ln, normalized)
- self.counter(name, label_names=ln).set(value, labels=dict(normalized))
-
- def set_histogram(
- self,
- name: str,
- *,
- bucket_bounds: Sequence[float],
- bucket_counts: Sequence[int],
- count: int,
- value_sum: float,
- labels: Mapping[str, str] | None = None,
- label_names: Sequence[str] = (),
- ) -> None:
- """Replace a named histogram from disjoint bucket counts and totals; return None."""
- bounds = tuple(float(bound) for bound in bucket_bounds)
- bucket_counts, value_sum = _validate_histogram_values(
- bounds, bucket_counts, count, value_sum
- )
- ln = _canonicalize_label_names(label_names)
- normalized = normalize_labels(labels)
- _validate_label_keys(name, ln, normalized)
- self.histogram(name, label_names=ln, buckets=bounds).set(
- bucket_counts, count, value_sum, labels=dict(normalized)
+ self._lock = threading.RLock()
+ self._max_series = max_series_per_metric
+ self._bindings: dict[str, _MetricBinding] = {}
+ self._reader = InMemoryMetricReader()
+ self._provider = MeterProvider(
+ metric_readers=[self._reader], resource=Resource({}), shutdown_on_exit=False
)
+ self._meter = self._provider.get_meter("openviking.metrics")
+ self._dropped = self._meter.create_counter("openviking_metrics_dropped_series_total")
- def set_gauge(
- self,
- name: str,
- value: float,
- *,
- labels: Mapping[str, str] | None = None,
- label_names: Sequence[str] = (),
- ) -> None:
- """
- Set a Gauge metric to an absolute value.
-
- Args:
- name: Prometheus metric name.
- value: Gauge value (float).
- labels: Optional label dict. Keys must exactly match `label_names`.
- label_names: Ordered label key tuple for this metric name.
- """
- self.gauge(name, label_names=label_names).set(float(value), labels=labels)
+ def _bind(self, name, kind, label_names, labels, *, mode, bounds=()):
+ names = tuple(sorted(str(key) for key in label_names))
+ normalized = normalize_labels(labels)
+ if tuple(key for key, _ in normalized) != names:
+ raise ValueError(f"metric {name} label keys mismatch: expected {names}")
+ if kind == "histogram" and "bucket_schema" in names:
+ raise ValueError("bucket_schema is reserved for the histogram's actual bucket layout")
+ binding = self._bindings.get(name)
+ if binding is None:
+ instrument = None
+ if mode == "event":
+ if kind == "counter":
+ instrument = self._meter.create_counter(name)
+ else:
+ instrument = self._meter.create_histogram(
+ name, explicit_bucket_boundaries_advisory=bounds
+ )
+ binding = _MetricBinding(kind, names, mode, bounds, instrument)
+ self._bindings[name] = binding
+ elif (binding.kind, binding.label_names, binding.mode, binding.bounds) != (
+ kind,
+ names,
+ mode,
+ bounds,
+ ):
+ raise ValueError(f"metric {name} type, labels, collection mode or buckets mismatch")
+ if normalized not in binding.points and len(binding.points) >= self._max_series:
+ self._dropped.add(1, {"metric": name})
+ return None, normalized
+ return binding, normalized
+
+ def inc_counter(self, name, *, amount=1.0, labels=None, label_names=()) -> None:
+ amount = float(amount)
+ if not math.isfinite(amount) or amount <= 0:
+ raise ValueError("counter increment must be finite and positive")
+ with self._lock:
+ binding, key = self._bind(name, "counter", label_names, labels, mode="event")
+ if binding is not None:
+ binding.instrument.add(amount, dict(key))
+ binding.points[key] = None
def observe_histogram(
- self,
- name: str,
- value: float,
- *,
- labels: Mapping[str, str] | None = None,
- label_names: Sequence[str] = (),
- buckets: Sequence[float] = DEFAULT_LATENCY_BUCKETS,
+ self, name, value, *, labels=None, label_names=(), buckets=DEFAULT_LATENCY_BUCKETS
) -> None:
- """
- Observe a value into a Histogram metric.
-
- Args:
- name: Prometheus metric name (base name; exporter renders _bucket/_count/_sum).
- value: Observation value.
- labels: Optional label dict. Keys must exactly match `label_names`.
- label_names: Ordered label key tuple for this metric name.
- buckets: Histogram bucket boundaries (ascending). "+Inf" bucket is implicit.
- """
- self.histogram(name, label_names=label_names, buckets=buckets).observe(
- float(value), labels=labels
- )
-
- def counter(self, name: str, *, label_names: Sequence[str] = ()) -> "_Counter":
- """
- Get or create a Counter family handle.
-
- Args:
- name: Prometheus metric name.
- label_names: Canonical ordered label keys for the family.
-
- Returns:
- A lightweight `_Counter` wrapper bound to the requested family.
- """
- ln = _canonicalize_label_names(label_names)
- with self._lock:
- if name in self._gauges or name in self._histograms:
- raise ValueError(f"metric {name} already registered with a different type")
- family = self._counters.get(name)
- if family is None:
- family = _CounterFamily(
- name=name,
- label_names=ln,
- max_series=self._max_series_per_metric,
- on_drop=self._on_drop_series,
- )
- self._counters[name] = family
- else:
- family._validate_label_names(ln)
- return _Counter(family)
-
- def gauge(self, name: str, *, label_names: Sequence[str] = ()) -> "_Gauge":
- """
- Get or create a Gauge family handle.
-
- Args:
- name: Prometheus metric name.
- label_names: Canonical ordered label keys for the family.
-
- Returns:
- A lightweight `_Gauge` wrapper bound to the requested family.
- """
- ln = _canonicalize_label_names(label_names)
- with self._lock:
- if name in self._counters or name in self._histograms:
- raise ValueError(f"metric {name} already registered with a different type")
- family = self._gauges.get(name)
- if family is None:
- family = _GaugeFamily(
- name=name,
- label_names=ln,
- max_series=self._max_series_per_metric,
- on_drop=self._on_drop_series,
- )
- self._gauges[name] = family
- else:
- family._validate_label_names(ln)
- return _Gauge(family)
-
- def histogram(
- self,
- name: str,
- *,
- label_names: Sequence[str] = (),
- buckets: Sequence[float] = DEFAULT_LATENCY_BUCKETS,
- ) -> "_Histogram":
- """
- Get or create a Histogram family handle.
-
- Args:
- name: Prometheus metric base name.
- label_names: Canonical ordered label keys for the family.
- buckets: Histogram bucket upper bounds.
-
- Returns:
- A lightweight `_Histogram` wrapper bound to the requested family.
- """
- ln = _canonicalize_label_names(label_names)
- b = tuple(float(x) for x in buckets)
- with self._lock:
- if name in self._counters or name in self._gauges:
- raise ValueError(f"metric {name} already registered with a different type")
- family = self._histograms.get(name)
- if family is None:
- family = _HistogramFamily(
- name=name,
- label_names=ln,
- bucket_bounds=b,
- max_series=self._max_series_per_metric,
- on_drop=self._on_drop_series,
- )
- self._histograms[name] = family
- else:
- family._validate_label_names(ln)
- family._validate_buckets(b)
- return _Histogram(family)
-
- def iter_counters(self, *, include_start_time: bool = False):
- """
- Iterate over all counter families and their current series values.
-
- The returned family payload is a detached snapshot so exporters can iterate without
- holding the registry lock during rendering. When include_start_time is True,
- each series also includes its cumulative start time in Unix nanoseconds.
- """
- with self._lock:
- families = dict(self._counters)
- for name, family in families.items():
- values = family.copy_values()
- if values is not None:
- yield (
- name,
- [
- (labels, *value) if include_start_time else (labels, value[0])
- for labels, value in values.items()
- ],
- )
-
- def counter_label_names(self, name: str) -> tuple[str, ...]:
- """Return the registered label key tuple for a counter family, if present."""
- with self._lock:
- family = self._counters.get(name)
- return family.label_names if family is not None else ()
-
- def iter_gauges(self):
- """
- Iterate over all gauge families and their current series values.
-
- As with counters, the family payload is copied before iteration so rendering stays
- independent from concurrent writers.
- """
- with self._lock:
- families = dict(self._gauges)
- for name, family in families.items():
- values = family.copy_values()
- if values is not None:
- yield name, list(values.items())
-
- def gauge_label_names(self, name: str) -> tuple[str, ...]:
- """Return the registered label key tuple for a gauge family, if present."""
- with self._lock:
- family = self._gauges.get(name)
- return family.label_names if family is not None else ()
-
- def gauge_get(self, name: str, *, labels: Mapping[str, str] | None = None) -> float | None:
- """
- Read the current value of a gauge series.
-
- Args:
- name: Gauge metric name.
- labels: Optional label dict identifying a single series.
-
- Returns:
- The current gauge value, or `None` when the family or series does not exist.
- """
+ value = float(value)
+ bounds = self._bounds(buckets)
+ if not math.isfinite(value) or value < 0:
+ raise ValueError("histogram observation must be finite and nonnegative")
with self._lock:
- family = self._gauges.get(name)
- if family is None:
- return None
- return family.get_value(labels=labels)
-
- def gauge_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:
- """
- Delete every gauge series whose labels contain the provided subset.
-
- Args:
- name: Gauge metric name.
- match_labels: Partial label set used for matching series to remove.
- """
- with self._lock:
- family = self._gauges.get(name)
- if family is None:
- return
- family.delete_matching(match_labels=match_labels)
-
- def counter_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:
- """Delete counter series matching the supplied label subset; return None."""
- with self._lock:
- family = self._counters.get(name)
- if family is not None:
- family.delete_matching(match_labels=match_labels)
-
- def histogram_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:
- """Delete histogram series matching the supplied label subset; return None."""
- with self._lock:
- family = self._histograms.get(name)
- if family is not None:
- family.delete_matching(match_labels=match_labels)
-
- def iter_histograms(self, *, include_start_time: bool = False):
- """
- Iterate over all histogram families and their materialized series snapshots.
-
- Each yielded item contains family metadata and a detached list of per-series bucket
- counts, sample counts, and sums ready for exporter rendering. When include_start_time
- is True, each series also includes its cumulative start time in Unix nanoseconds.
- """
- with self._lock:
- families = dict(self._histograms)
- for name, family in families.items():
- snapshot = family.copy_series()
- if snapshot is not None:
- label_names, bucket_bounds, series = snapshot
- yield (
- name,
- label_names,
- bucket_bounds,
- (series if include_start_time else [values[:-1] for values in series]),
- )
-
- def iter_dropped_series(self):
- """
- Iterate over metrics that have dropped series because of series-limit enforcement.
-
- Exporters use this bookkeeping to expose internal pressure signals when a metric family
- hits its cardinality cap and starts rejecting additional series.
- """
- with self._lock:
- for metric_name, dropped in self._dropped_series_total.items():
- yield metric_name, dropped
-
- def _on_drop_series(self, metric_name: str) -> None:
- """Record that a new series was rejected for the given metric family."""
- with self._lock:
- self._dropped_series_total[metric_name] = (
- self._dropped_series_total.get(metric_name, 0) + 1
+ binding, key = self._bind(
+ name, "histogram", label_names, labels, mode="event", bounds=bounds
)
-
-
-class _CounterFamily:
- """Internal mutable storage for one counter metric family and all of its series."""
-
- def __init__(
- self,
- *,
- name: str,
- label_names: tuple[str, ...],
- max_series: int,
- on_drop,
- ) -> None:
- """Create an internal counter family with bounded series cardinality."""
- self.name = name
- self.label_names = label_names
- self._max_series = max_series
- self._on_drop = on_drop
- self._lock = threading.Lock()
- self._values: dict[tuple[tuple[str, str], ...], tuple[float, int]] = {}
- self._had_deletions = False
-
- def inc(self, *, labels: Mapping[str, str] | None, amount: float) -> None:
- """
- Increase one counter series by a positive amount.
-
- New series are rejected once the family has reached the configured series limit, but
- existing series may continue to grow even after the cap is reached.
- """
- if amount <= 0:
- raise ValueError("counter can only be increased by a positive amount")
- key = self._normalize_and_validate(labels)
+ if binding is not None:
+ binding.instrument.record(value, dict(key))
+ binding.points[key] = None
+
+ @staticmethod
+ def _bounds(buckets):
+ bounds = tuple(float(value) for value in buckets)
+ if any(not math.isfinite(value) or value < 0 for value in bounds) or any(
+ left >= right for left, right in zip(bounds, bounds[1:], strict=False)
+ ):
+ raise ValueError("histogram bounds must be finite, nonnegative and increasing")
+ return bounds
+
+ def set_counter(self, name, value, *, labels=None, label_names=()) -> None:
+ value = float(value)
+ if not math.isfinite(value) or value < 0:
+ raise ValueError("counter value must be finite and nonnegative")
+ self._set_number(name, value, "counter", labels, label_names)
+
+ def set_gauge(self, name, value, *, labels=None, label_names=()) -> None:
+ self._set_number(name, float(value), "gauge", labels, label_names)
+
+ def _set_number(self, name, value, kind, labels, label_names):
with self._lock:
- if key not in self._values and len(self._values) >= self._max_series:
- self._on_drop(self.name)
+ binding, key = self._bind(name, kind, label_names, labels, mode="snapshot")
+ if binding is None:
return
- value, start_time_ns = self._values.get(key, (0.0, 0))
- self._values[key] = (value + float(amount), start_time_ns or time.time_ns())
-
- def copy_values(self) -> dict[tuple[tuple[str, str], ...], tuple[float, int]] | None:
- """Return totals with their start times, or None if deletion emptied this family."""
- with self._lock:
- if self._had_deletions and not self._values:
- return None
- return dict(self._values)
-
- def set(self, *, labels: Mapping[str, str] | None, value: float) -> None:
- """Replace labels with value, starting a new cumulative period on decrease; return None."""
- value = _validate_counter_value(value)
- key = self._normalize_and_validate(labels)
- with self._lock:
- if key not in self._values and len(self._values) >= self._max_series:
- self._on_drop(self.name)
- return
- previous = self._values.get(key)
- start_time_ns = (
- previous[1] if previous is not None and value >= previous[0] else time.time_ns()
- )
- self._values[key] = (value, start_time_ns)
-
- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:
- """Remove counter entries containing the supplied labels; return None."""
- normalized = normalize_labels(match_labels)
- with self._lock:
- for key in list(self._values):
- if _labels_contains(key, normalized):
- del self._values[key]
- self._had_deletions = True
-
- def _normalize_and_validate(
- self, labels: Mapping[str, str] | None
- ) -> tuple[tuple[str, str], ...]:
- """Normalize label input and enforce the family label contract."""
- normalized = normalize_labels(labels)
- self._validate_label_names_against(normalized)
- return normalized
-
- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:
- """Ensure subsequent family lookups use the original declared label keys."""
- if self.label_names != label_names:
- raise ValueError(
- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"
- )
-
- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:
- """Ensure a concrete series write uses exactly the configured label keys."""
- _validate_label_keys(self.name, self.label_names, normalized)
-
-
-class _GaugeFamily:
- """Internal mutable storage for one gauge metric family and all of its series."""
+ now = time.time_ns()
+ previous = binding.points.get(key)
+ start = now
+ if previous is not None and (kind == "gauge" or value >= previous.value):
+ start = previous.start_time_unix_nano
+ binding.points[key] = NumberDataPoint(dict(key), start, now, value)
- def __init__(
- self,
- *,
- name: str,
- label_names: tuple[str, ...],
- max_series: int,
- on_drop,
+ def set_histogram(
+ self, name, *, bucket_bounds, bucket_counts, count, value_sum, labels=None, label_names=()
) -> None:
- """Create an internal gauge family with bounded series cardinality."""
- self.name = name
- self.label_names = label_names
- self._max_series = max_series
- self._on_drop = on_drop
- self._lock = threading.Lock()
- self._values: dict[tuple[tuple[str, str], ...], float] = {}
- self._had_deletions = False
-
- def set(self, *, labels: Mapping[str, str] | None, value: float) -> None:
- """
- Set one gauge series to an absolute value.
-
- As with counters, creating a brand-new series is still subject to the family-level
- cardinality cap.
- """
- key = self._normalize_and_validate(labels)
+ """Accept a native histogram validated at the RagfsMetricDataSource boundary."""
+ bounds = tuple(bucket_bounds)
+ counts = tuple(bucket_counts)
with self._lock:
- if key not in self._values and len(self._values) >= self._max_series:
- self._on_drop(self.name)
- return
- self._values[key] = float(value)
-
- def add(self, *, labels: Mapping[str, str] | None, delta: float) -> None:
- """
- Add a signed delta to one gauge series.
-
- This is the internal primitive behind the public gauge `inc(...)` and `dec(...)`
- helpers.
- """
- key = self._normalize_and_validate(labels)
- with self._lock:
- if key not in self._values and len(self._values) >= self._max_series:
- self._on_drop(self.name)
+ binding, key = self._bind(
+ name, "histogram", label_names, labels, mode="snapshot", bounds=bounds
+ )
+ if binding is None:
return
- self._values[key] = self._values.get(key, 0.0) + float(delta)
-
- def copy_values(self) -> dict[tuple[tuple[str, str], ...], float] | None:
- """Return a gauge value snapshot, or None if deletion emptied this family."""
- with self._lock:
- if self._had_deletions and not self._values:
- return None
- return dict(self._values)
-
- def get_value(self, *, labels: Mapping[str, str] | None) -> float | None:
- """Return the current value for one normalized gauge series, if that series exists."""
- key = self._normalize_and_validate(labels)
- with self._lock:
- return self._values.get(key)
-
- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:
- """Delete every stored gauge series that contains the requested label subset."""
- normalized = normalize_labels(match_labels)
- with self._lock:
- keys = list(self._values.keys())
- for k in keys:
- if _labels_contains(k, normalized):
- self._values.pop(k, None)
- self._had_deletions = True
-
- def _normalize_and_validate(
- self, labels: Mapping[str, str] | None
- ) -> tuple[tuple[str, str], ...]:
- """Normalize label input and enforce the family label contract."""
- normalized = normalize_labels(labels)
- self._validate_label_names_against(normalized)
- return normalized
-
- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:
- """Ensure subsequent family lookups use the original declared label keys."""
- if self.label_names != label_names:
- raise ValueError(
- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"
+ now = time.time_ns()
+ previous = binding.points.get(key)
+ start = now
+ if (
+ previous is not None
+ and count >= previous.count
+ and value_sum >= previous.sum
+ and all(
+ value >= old for value, old in zip(counts, previous.bucket_counts, strict=True)
+ )
+ ):
+ start = previous.start_time_unix_nano
+ # Native snapshots do not contain min/max; None omits these optional OTLP fields.
+ binding.points[key] = HistogramDataPoint(
+ attributes=dict(key),
+ start_time_unix_nano=start,
+ time_unix_nano=now,
+ count=count,
+ sum=value_sum,
+ bucket_counts=counts,
+ explicit_bounds=bounds,
+ min=None,
+ max=None,
)
- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:
- """Ensure a concrete series write uses exactly the configured label keys."""
- if not self.label_names and normalized:
- raise ValueError(f"metric {self.name} does not accept labels")
- if self.label_names and tuple(k for k, _ in normalized) != self.label_names:
- raise ValueError(f"metric {self.name} label keys mismatch: expected {self.label_names}")
-
-
-class _HistogramFamily:
- """Internal mutable storage for one histogram metric family and all of its label series."""
-
- def __init__(
- self,
- *,
- name: str,
- label_names: tuple[str, ...],
- bucket_bounds: tuple[float, ...],
- max_series: int,
- on_drop,
- ) -> None:
- """Create an internal histogram family with fixed buckets and bounded series count."""
- self.name = name
- self.label_names = label_names
- self.bucket_bounds = bucket_bounds
- self._max_series = max_series
- self._on_drop = on_drop
- self._lock = threading.Lock()
- self._series: dict[tuple[tuple[str, str], ...], _HistogramSeries] = {}
- self._had_deletions = False
-
- def observe(self, *, labels: Mapping[str, str] | None, value: float) -> None:
- """
- Record one histogram observation for the selected label series.
-
- A new series is materialized lazily on first use, subject to the same family-level
- series cap used by counters and gauges.
- """
- key = self._normalize_and_validate(labels)
- with self._lock:
- series = self._series.get(key)
- if series is None:
- if len(self._series) >= self._max_series:
- self._on_drop(self.name)
- return
- series = _HistogramSeries(bucket_bounds=self.bucket_bounds)
- self._series[key] = series
- series.observe(float(value))
-
- def set(
- self,
- *,
- labels: Mapping[str, str] | None,
- bucket_counts: Sequence[int],
- count: int,
- value_sum: float,
- ) -> None:
- """Validate bucket_counts, count and value_sum before replacing labels; return None."""
- bucket_counts, value_sum = _validate_histogram_values(
- self.bucket_bounds, bucket_counts, count, value_sum
- )
- key = self._normalize_and_validate(labels)
+ def gauge_get(self, name, *, labels=None):
with self._lock:
- series = self._series.get(key)
- if series is None:
- if len(self._series) >= self._max_series:
- self._on_drop(self.name)
- return
- series = _HistogramSeries(bucket_bounds=self.bucket_bounds)
- self._series[key] = series
- series.set_values(bucket_counts, count, value_sum)
-
- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:
- """Remove histogram entries containing the supplied labels; return None."""
- normalized = normalize_labels(match_labels)
- with self._lock:
- for key in list(self._series):
- if _labels_contains(key, normalized):
- del self._series[key]
- self._had_deletions = True
-
- def copy_series(
- self,
- ) -> (
- tuple[
- tuple[str, ...],
- tuple[float, ...],
- list[tuple[tuple[tuple[str, str], ...], tuple[int, ...], int, float, int]],
- ]
- | None
- ):
- """Return histogram values and start times, or None if deletion emptied this family."""
- with self._lock:
- if self._had_deletions and not self._series:
+ binding = self._bindings.get(name)
+ if binding is None or binding.kind != "gauge":
return None
- series: list[tuple[tuple[tuple[str, str], ...], tuple[int, ...], int, float, int]] = []
- for labels, s in self._series.items():
- series.append((labels, *s.copy_values()))
- return self.label_names, self.bucket_bounds, series
-
- def _normalize_and_validate(
- self, labels: Mapping[str, str] | None
- ) -> tuple[tuple[str, str], ...]:
- """Normalize label input and enforce the family label contract."""
- normalized = normalize_labels(labels)
- self._validate_label_names_against(normalized)
- return normalized
+ point = binding.points.get(normalize_labels(labels))
+ return None if point is None else point.value
- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:
- """Ensure subsequent family lookups use the original declared label keys."""
- if self.label_names != label_names:
- raise ValueError(
- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"
- )
-
- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:
- """Ensure a concrete series write uses exactly the configured label keys."""
- _validate_label_keys(self.name, self.label_names, normalized)
-
- def _validate_buckets(self, bucket_bounds: tuple[float, ...]) -> None:
- """Ensure a histogram family is never reopened with a different bucket layout."""
- if self.bucket_bounds != bucket_bounds:
- raise ValueError(f"metric {self.name} buckets mismatch")
-
-
-class _HistogramSeries:
- """Internal bucket/count/sum accumulator for one concrete histogram label series."""
-
- def __init__(self, *, bucket_bounds: tuple[float, ...]) -> None:
- """Create one histogram series with counters for all finite buckets plus `+Inf`."""
- self._bucket_bounds = bucket_bounds
- self._lock = threading.Lock()
- self._bucket_counts: list[int] = [0] * (len(bucket_bounds) + 1)
- self._count: int = 0
- self._sum: float = 0.0
- self._start_time_ns = time.time_ns()
-
- def observe(self, value: float) -> None:
- """
- Record a single observation into the appropriate bucket and aggregate summary totals.
-
- Bucket selection uses `bisect_left`, so each observation increments the first bucket
- whose upper bound is greater than or equal to the recorded value.
- """
- idx = bisect_left(self._bucket_bounds, value)
+ def delete_matching(self, name, *, match_labels: Mapping[str, str]) -> None:
+ """Remove matching state snapshots; the registered metric owns its type."""
+ match = set(normalize_labels(match_labels))
with self._lock:
- self._bucket_counts[idx] += 1
- self._count += 1
- self._sum += value
-
- def copy_values(self) -> tuple[tuple[int, ...], int, float, int]:
- """Return bucket counts, sample count, sum and cumulative start time under one lock."""
- with self._lock:
- return tuple(self._bucket_counts), self._count, self._sum, self._start_time_ns
-
- def set_values(self, bucket_counts: tuple[int, ...], count: int, value_sum: float) -> None:
- """Replace validated totals, starting a new period if any total decreases; return None."""
+ binding = self._bindings.get(name)
+ if binding is None:
+ return
+ if binding.mode != "snapshot":
+ raise ValueError(f"metric {name} is not a state snapshot")
+ for key in list(binding.points):
+ if match.issubset(key):
+ del binding.points[key]
+
+ def collect(self, *, service_name="openviking-server") -> MetricsData:
+ """Return SDK event aggregates and current native/state snapshots in one standard batch."""
with self._lock:
- if (
- count < self._count
- or value_sum < self._sum
- or any(
- current < previous
- for current, previous in zip(bucket_counts, self._bucket_counts, strict=True)
+ data = self._reader.get_metrics_data()
+ metrics = [
+ metric
+ for resource in (() if data is None else data.resource_metrics)
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ ]
+ now = time.time_ns()
+ for name, binding in self._bindings.items():
+ if binding.mode != "snapshot" or not binding.points:
+ continue
+ points = tuple(
+ replace(point, time_unix_nano=now) for point in binding.points.values()
)
- ):
- self._start_time_ns = time.time_ns()
- self._bucket_counts = list(bucket_counts)
- self._count = count
- self._sum = value_sum
-
-
-class _Counter:
- """Public lightweight handle used by callers to mutate one counter family."""
-
- def __init__(self, family: _CounterFamily) -> None:
- """Bind a public counter handle to one internal counter family."""
- self._family = family
-
- def inc(self, amount: float = 1.0, *, labels: Mapping[str, str] | None = None) -> None:
- """Increment one series in the bound counter family using the public wrapper API."""
- self._family.inc(labels=labels, amount=amount)
-
- def set(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:
- """Replace the labelled series with a validated absolute value; return None."""
- self._family.set(labels=labels, value=value)
-
-
-class _Gauge:
- """Public lightweight handle used by callers to mutate one gauge family."""
-
- def __init__(self, family: _GaugeFamily) -> None:
- """Bind a public gauge handle to one internal gauge family."""
- self._family = family
-
- def set(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:
- """Set one series in the bound gauge family through the public wrapper API."""
- self._family.set(labels=labels, value=value)
-
- def inc(self, amount: float = 1.0, *, labels: Mapping[str, str] | None = None) -> None:
- """Increase one series in the bound gauge family by a positive delta."""
- self._family.add(labels=labels, delta=amount)
-
-
-class _Histogram:
- """Public lightweight handle used by callers to mutate one histogram family."""
-
- def __init__(self, family: _HistogramFamily) -> None:
- """Bind a public histogram handle to one internal histogram family."""
- self._family = family
-
- def observe(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:
- """Record one observation for a series in the bound histogram family wrapper."""
- self._family.observe(labels=labels, value=value)
-
- def set(
- self,
- bucket_counts: Sequence[int],
- count: int,
- value_sum: float,
- *,
- labels: Mapping[str, str] | None = None,
- ) -> None:
- """Replace the labelled series using validated disjoint counts and totals; return None."""
- self._family.set(
- labels=labels, bucket_counts=bucket_counts, count=count, value_sum=value_sum
+ if binding.kind == "counter":
+ aggregate = Sum(points, AggregationTemporality.CUMULATIVE, True)
+ elif binding.kind == "gauge":
+ aggregate = Gauge(points)
+ else:
+ aggregate = Histogram(points, AggregationTemporality.CUMULATIVE)
+ metrics.append(Metric(name, "OpenViking metric.", "", aggregate))
+ # Use the actual SDK boundaries, including boundaries from the Rust SDK.
+ for index, metric in enumerate(metrics):
+ if isinstance(metric.data, Histogram) and metric.data.data_points:
+ schema = histogram_schema(tuple(metric.data.data_points[0].explicit_bounds))
+ points = tuple(
+ replace(
+ point,
+ attributes={
+ **point.attributes,
+ "bucket_schema": schema,
+ },
+ )
+ for point in metric.data.data_points
+ )
+ metrics[index] = replace(metric, data=replace(metric.data, data_points=points))
+ return MetricsData(
+ (
+ ResourceMetrics(
+ resource=Resource({"service.name": service_name}),
+ scope_metrics=(
+ ScopeMetrics(
+ scope=InstrumentationScope("openviking.metrics"),
+ metrics=tuple(sorted(metrics, key=lambda metric: metric.name)),
+ schema_url="",
+ ),
+ ),
+ schema_url="",
+ ),
+ )
)
diff --git a/openviking/metrics/core/types.py b/openviking/metrics/core/types.py
index a2b48c8c2..adba96595 100644
--- a/openviking/metrics/core/types.py
+++ b/openviking/metrics/core/types.py
@@ -16,8 +16,9 @@ All labels are normalized into a sorted tuple of (key, value) pairs to provide:
from __future__ import annotations
+import hashlib
import re
-from dataclasses import dataclass
+import struct
from typing import Mapping
_LABEL_KEY_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
@@ -34,6 +35,11 @@ FORBIDDEN_LABEL_KEYS: set[str] = {
}
+def histogram_schema(bounds: tuple[float, ...]) -> str:
+ """Identify the actual bucket layout so rolling upgrades cannot mix layouts."""
+ return hashlib.sha256(struct.pack(f"!{len(bounds)}d", *bounds)).hexdigest()[:16]
+
+
def normalize_labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str], ...]:
"""
Normalize a label mapping into the canonical internal representation.
@@ -61,40 +67,3 @@ def normalize_labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str],
items.append((key, str(v)))
items.sort(key=lambda x: x[0])
return tuple(items)
-
-
-def render_labels(normalized: tuple[tuple[str, str], ...]) -> str:
- """
- Render normalized labels into Prometheus exposition syntax.
-
- Args:
- normalized: Canonical normalized label tuple.
-
- Returns:
- An empty string when no labels are present, or a `{k="v",...}` block ready for text
- exposition output.
- """
- if not normalized:
- return ""
- parts = [f'{k}="{_escape_label_value(v)}"' for k, v in normalized]
- return "{" + ",".join(parts) + "}"
-
-
-def _escape_label_value(value: str) -> str:
- """Escape a label value according to Prometheus text exposition rules."""
- return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
-
-
-@dataclass(frozen=True, slots=True)
-class MetricSeriesId:
- """
- Immutable identifier for a single metric series in the in-process registry.
-
- Attributes:
- name: Metric family name.
- labels: Canonical normalized label tuple uniquely identifying one concrete series within
- that metric family.
- """
-
- name: str
- labels: tuple[tuple[str, str], ...]
diff --git a/openviking/metrics/datasources/ragfs.py b/openviking/metrics/datasources/ragfs.py
index 1501b18e6..6182f0cf1 100644
--- a/openviking/metrics/datasources/ragfs.py
+++ b/openviking/metrics/datasources/ragfs.py
@@ -102,7 +102,7 @@ class RagfsMetricDataSource(DomainStatsMetricDataSource):
or not _LABEL.fullmatch(key)
or key.startswith("__")
or key in _FORBIDDEN_LABELS
- or (kind == "histogram" and key == "le")
+ or (kind == "histogram" and key in {"le", "bucket_schema"})
for key, value in labels.items()
):
raise ValueError("native metric labels are invalid or forbidden")
diff --git a/openviking/metrics/exporters/otel.py b/openviking/metrics/exporters/otel.py
index 936e2e023..e6ef811fc 100644
--- a/openviking/metrics/exporters/otel.py
+++ b/openviking/metrics/exporters/otel.py
@@ -1,728 +1,129 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""
-OpenTelemetry OTLP metrics exporter implementation.
-
-This exporter serializes the in-process MetricRegistry directly into OTLP protobuf
-messages and sends them to an OTLP endpoint over HTTP or gRPC.
-
-Key design goals:
-- Export the same MetricRegistry snapshot semantics that PrometheusExporter renders
-- Preserve histogram bucket/count/sum semantics without re-aggregation
-- Keep exporter lifecycle self-contained through start()/shutdown()
-- Refresh collectors before export so snapshot-based metrics stay aligned
-"""
+"""Refresh OpenViking state and send SDK metric data through official OTLP exporters."""
from __future__ import annotations
import asyncio
-import time
-from typing import Any, Iterable, Optional, Tuple
-import requests
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+ OTLPMetricExporter as GrpcExporter,
+)
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+ OTLPMetricExporter as HttpExporter,
+)
from openviking.metrics.core.base import MetricExporter
from openviking_cli.utils import get_logger
logger = get_logger(__name__)
-try:
- import grpc
-except ImportError:
- grpc = None
-
-try:
- from requests import RequestException
-except Exception:
- RequestException = Exception
-
-try:
- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
- ExportMetricsServiceRequest,
- )
- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2_grpc import (
- MetricsServiceStub,
- )
- from opentelemetry.proto.common.v1.common_pb2 import AnyValue, InstrumentationScope, KeyValue
- from opentelemetry.proto.metrics.v1.metrics_pb2 import (
- AGGREGATION_TEMPORALITY_CUMULATIVE,
- Gauge,
- Histogram,
- HistogramDataPoint,
- Metric,
- NumberDataPoint,
- Sum,
- )
- from opentelemetry.proto.resource.v1.resource_pb2 import Resource
-except ImportError:
- ExportMetricsServiceRequest = None
- MetricsServiceStub = None
- AnyValue = None
- InstrumentationScope = None
- KeyValue = None
- Gauge = None
- Histogram = None
- HistogramDataPoint = None
- Metric = None
- NumberDataPoint = None
- Sum = None
- Resource = None
- AGGREGATION_TEMPORALITY_CUMULATIVE = None
-
-
-def _compute_next_tick_and_sleep(
- *,
- now: float,
- interval_s: float,
- next_tick: float,
-) -> tuple[float, float]:
- """
- Compute next tick time and sleep duration for a fixed-interval loop.
-
- This is a pure function to keep scheduling behavior testable without adding
- production-only hooks.
-
- Rules:
- - If the loop overruns (now >= next_tick), advance next_tick by interval until it is in the future.
- - Sleep only until the next_tick.
-
- Args:
- now: Current monotonic time in seconds.
- interval_s: Target loop interval in seconds.
- next_tick: Previously scheduled tick time in seconds.
-
- Returns:
- (next_tick, sleep_seconds)
- """
- while next_tick <= now:
- next_tick += interval_s
- return next_tick, max(0.0, next_tick - now)
-
class OTelMetricExporter(MetricExporter):
- """
- Export MetricRegistry snapshots to OTLP metrics.
-
- This exporter intentionally avoids OTel SDK metric aggregation. Instead, it
- periodically snapshots MetricRegistry, converts the snapshot into OTLP protobuf
- payloads, and pushes them to the configured collector endpoint.
- """
+ """Own the application refresh cycle; the SDK owns encoding, transport and retries."""
def __init__(
self,
*,
registry,
collector_manager=None,
- refresh_deadline_seconds: float = 1.0,
- protocol: str = "grpc",
- insecure: bool = False,
- endpoint: str = "localhost:4317",
- service_name: str = "openviking-server",
- export_interval_ms: int = 10000,
- export_timeout_seconds: Optional[float] = None,
- headers: Optional[dict[str, str]] = None,
- enabled: bool = True,
- ) -> None:
- """
- Initialize the OTLP metrics exporter.
-
- Args:
- registry: The in-process metric registry backing all exporters.
- collector_manager: Optional collector refresh orchestrator.
- refresh_deadline_seconds: Best-effort refresh timeout before each export.
- protocol: Transport protocol, either "grpc" or "http".
- insecure: Whether gRPC transport should use plaintext.
- endpoint: OTLP collector endpoint.
- service_name: Service name written into OTLP resource attributes.
- export_interval_ms: Periodic export interval in milliseconds.
- export_timeout_seconds: Transport timeout for one OTLP export request.
- headers: Additional OTLP exporter headers for vendor-specific auth.
- enabled: Whether the exporter is enabled.
- """
+ refresh_deadline_seconds=1.0,
+ protocol="grpc",
+ insecure=False,
+ endpoint="localhost:4317",
+ service_name="openviking-server",
+ export_interval_ms=10000,
+ export_timeout_seconds=None,
+ headers=None,
+ ):
self._registry = registry
self._collector_manager = collector_manager
self._refresh_deadline_seconds = float(refresh_deadline_seconds)
- self._protocol = protocol.lower()
- self._insecure = bool(insecure)
- self._endpoint = endpoint
self._service_name = service_name
- self._headers = {str(key): str(value) for key, value in (headers or {}).items()}
- self._export_interval_ms = max(1000, int(export_interval_ms))
- self._export_timeout_seconds = float(
+ self._interval = max(1000, int(export_interval_ms)) / 1000
+ self._timeout = float(
export_timeout_seconds
if export_timeout_seconds is not None
else max(5.0, self._refresh_deadline_seconds)
)
- self._enabled = bool(enabled)
- self._export_task: Optional[asyncio.Task[Any]] = None
- self._http_session: Optional[requests.Session] = None
- self._grpc_channel: Optional[Any] = None
- self._grpc_stub: Optional[Any] = None
- self._start_time_ns = time.time_ns()
-
- if self._protocol not in {"grpc", "http"}:
- logger.warning(
- "[OTelMetricExporter] invalid protocol=%s, defaulting to grpc",
- self._protocol,
- )
- self._protocol = "grpc"
-
- if self._enabled:
- self._init_transport()
-
- def _init_transport(self) -> None:
- """
- Initialize the transport client for the configured OTLP protocol.
-
- The exporter keeps the transport client alive across export cycles so the
- periodic loop only pays the cost of serializing current registry snapshots.
- """
- if not self._protobuf_available():
- logger.warning(
- "[OTelMetricExporter] initialization failed: OTLP metrics protobuf dependencies are not available"
- )
- self._enabled = False
- return
+ self._closed = False
+ self._export_task = None
+ self._export_lock = asyncio.Lock()
+ if protocol not in ("grpc", "http"):
+ raise ValueError(f"unsupported OTLP protocol: {protocol}")
+ options = {"endpoint": endpoint, "headers": headers, "timeout": self._timeout}
+ if protocol == "http":
+ self._sdk = HttpExporter(**options)
+ else:
+ self._sdk = GrpcExporter(**options, insecure=insecure)
- try:
- if self._protocol == "http":
- self._validate_http_endpoint(self._endpoint)
- self._http_session = requests.Session()
- else:
- if grpc is None or MetricsServiceStub is None:
- raise ImportError("gRPC OTLP metrics dependencies are not available")
- if self._insecure:
- self._grpc_channel = grpc.insecure_channel(self._endpoint)
- else:
- self._grpc_channel = grpc.secure_channel(
- self._endpoint,
- grpc.ssl_channel_credentials(),
+ async def export(self) -> str:
+ async with self._export_lock:
+ if self._closed:
+ return ""
+ if self._collector_manager is not None:
+ try:
+ await asyncio.wait_for(
+ self._collector_manager.refresh_all(
+ self._registry, deadline_seconds=self._refresh_deadline_seconds
+ ),
+ timeout=self._refresh_deadline_seconds,
)
- self._grpc_stub = MetricsServiceStub(self._grpc_channel)
-
- logger.info(
- "[OTelMetricExporter] initialized: phase=init_transport protocol=%s endpoint=%s service_name=%s export_interval_ms=%s export_timeout_seconds=%s",
- self._protocol,
- self._endpoint,
- self._service_name,
- self._export_interval_ms,
- self._export_timeout_seconds,
- )
- except Exception as exc:
- logger.warning(
- "[OTelMetricExporter] initialization failed: phase=init_transport protocol=%s endpoint=%s service_name=%s export_timeout_seconds=%s error=%s",
- self._protocol,
- self._endpoint,
- self._service_name,
- self._export_timeout_seconds,
- exc,
- )
- self._enabled = False
-
- def _protobuf_available(self) -> bool:
- """Return whether all required OTLP protobuf symbols are available."""
- return all(
- symbol is not None
- for symbol in (
- ExportMetricsServiceRequest,
- AnyValue,
- InstrumentationScope,
- KeyValue,
- Metric,
- NumberDataPoint,
- HistogramDataPoint,
- Gauge,
- Sum,
- Histogram,
- Resource,
- AGGREGATION_TEMPORALITY_CUMULATIVE,
- )
- )
-
- def _validate_http_endpoint(self, endpoint: str) -> None:
- """
- Validate that an OTLP/HTTP endpoint includes a URL scheme and path.
-
- Args:
- endpoint: OTLP/HTTP metrics endpoint.
- """
- if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
- raise ValueError(
- "OTLP/HTTP endpoint must include scheme, e.g. 'http://localhost:4318/v1/metrics'"
+ except Exception as exc:
+ logger.warning("OTLP metric source refresh failed: %s", exc)
+ data = self._registry.collect(service_name=self._service_name)
+ if not any(
+ scope.metrics
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ ):
+ return ""
+ sending = asyncio.create_task(
+ asyncio.to_thread(self._sdk.export, data, timeout_millis=self._timeout * 1000)
)
-
- async def export(self) -> str:
- """
- Refresh collectors and export the current MetricRegistry snapshot.
-
- Returns:
- An empty string to preserve the MetricExporter interface contract.
- """
- if not self._enabled:
- return ""
-
- await self._refresh_collectors()
- request = self._build_export_request()
- if request is None:
- return ""
-
- await self._send_request(request)
+ try:
+ await asyncio.shield(sending)
+ except asyncio.CancelledError:
+ # Closing the SDK transport must wait for the in-flight blocking export.
+ await sending
+ raise
return ""
- async def _refresh_collectors(self) -> None:
- """
- Best-effort refresh of snapshot-driven collectors before serialization.
-
- Export must never break application flow, so refresh failures are swallowed.
- """
- if self._collector_manager is None:
- return
- started = time.monotonic()
- try:
- await asyncio.wait_for(
- self._collector_manager.refresh_all(
- self._registry,
- deadline_seconds=self._refresh_deadline_seconds,
- ),
- timeout=self._refresh_deadline_seconds,
- )
- except asyncio.TimeoutError as exc:
- elapsed_ms = int((time.monotonic() - started) * 1000)
- logger.warning(
- "[OTelMetricExporter] collector refresh failed: phase=refresh_collectors reason=timeout protocol=%s endpoint=%s deadline_seconds=%s elapsed_ms=%s error=%s",
- self._protocol,
- self._endpoint,
- self._refresh_deadline_seconds,
- elapsed_ms,
- exc,
- )
- except Exception as exc:
- elapsed_ms = int((time.monotonic() - started) * 1000)
- logger.warning(
- "[OTelMetricExporter] collector refresh failed: phase=refresh_collectors reason=exception protocol=%s endpoint=%s deadline_seconds=%s elapsed_ms=%s error=%s",
- self._protocol,
- self._endpoint,
- self._refresh_deadline_seconds,
- elapsed_ms,
- exc,
- )
-
- def _build_export_request(self) -> Optional[ExportMetricsServiceRequest]:
- """
- Build an OTLP ExportMetricsServiceRequest from the current registry snapshot.
-
- Returns:
- A populated OTLP export request, or None when there is no metric family to send.
- """
- if not self._protobuf_available():
- return None
-
- now_ns = time.time_ns()
- request = ExportMetricsServiceRequest()
- resource_metrics = request.resource_metrics.add()
- resource_metrics.resource.CopyFrom(self._build_resource())
- scope_metrics = resource_metrics.scope_metrics.add()
- scope_metrics.scope.CopyFrom(
- InstrumentationScope(name="openviking.metrics.exporter.otel", version="1")
- )
-
- self._append_counter_metrics(scope_metrics.metrics, now_ns)
- self._append_gauge_metrics(scope_metrics.metrics, now_ns)
- self._append_histogram_metrics(scope_metrics.metrics, now_ns)
- self._append_dropped_series_metrics(scope_metrics.metrics, now_ns)
-
- if not scope_metrics.metrics:
- return None
- return request
-
- def _build_resource(self) -> Resource:
- """
- Build the OTLP resource envelope for this exporter.
-
- Returns:
- A Resource message containing the service name.
- """
- resource = Resource()
- resource.attributes.append(
- KeyValue(key="service.name", value=AnyValue(string_value=self._service_name))
- )
- return resource
-
- def _append_counter_metrics(self, metrics, now_ns: int) -> None:
- """
- Append all counter families from MetricRegistry as cumulative OTLP sums.
-
- Empty unlabeled counters emit a single zero data point to match PrometheusExporter.
- """
- for name, counter_series in self._registry.iter_counters(include_start_time=True):
- metric = metrics.add()
- metric.name = name
- metric.description = "OpenViking metric."
- metric.sum.CopyFrom(self._build_sum(counter_series, now_ns, name=name))
-
- def _append_gauge_metrics(self, metrics, now_ns: int) -> None:
- """
- Append all gauge families from MetricRegistry as OTLP gauges.
-
- Empty unlabeled gauges emit a single zero data point to match PrometheusExporter.
- """
- for name, gauge_series in self._registry.iter_gauges():
- metric = metrics.add()
- metric.name = name
- metric.description = "OpenViking metric."
- metric.gauge.CopyFrom(self._build_gauge(gauge_series, now_ns, name=name))
-
- def _append_histogram_metrics(self, metrics, now_ns: int) -> None:
- """
- Append all histogram families from MetricRegistry as cumulative OTLP histograms.
-
- Bucket counts and explicit bounds are copied directly from the registry snapshot.
- """
- for name, label_names, bucket_bounds, series_iter in self._registry.iter_histograms(
- include_start_time=True
- ):
- series_list = list(series_iter)
- metric = metrics.add()
- metric.name = name
- metric.description = "OpenViking metric."
- metric.histogram.CopyFrom(
- self._build_histogram(
- label_names=label_names,
- bucket_bounds=bucket_bounds,
- series_list=series_list,
- now_ns=now_ns,
- )
- )
-
- def _append_dropped_series_metrics(self, metrics, now_ns: int) -> None:
- """
- Append dropped-series diagnostics as a cumulative OTLP sum family.
-
- This mirrors the `openviking_metrics_dropped_series_total` metric emitted by
- PrometheusExporter.
- """
- dropped_points: list[tuple[tuple[tuple[str, str], ...], float, int]] = []
- for metric_name, dropped in self._registry.iter_dropped_series():
- dropped_points.append(((("metric", metric_name),), float(dropped), self._start_time_ns))
-
- if not dropped_points:
- return
-
- metric = metrics.add()
- metric.name = "openviking_metrics_dropped_series_total"
- metric.description = "OpenViking metric."
- metric.sum.CopyFrom(self._build_sum(dropped_points, now_ns, name=metric.name))
-
- def _build_sum(self, series, now_ns: int, *, name: str) -> Sum:
- """
- Build an OTLP Sum message from a counter-like registry family.
-
- Args:
- series: List of `(labels, value, start_time_ns)` tuples from MetricRegistry.
- now_ns: Export timestamp in nanoseconds.
- name: Metric family name for zero-series handling.
- """
- sum_metric = Sum(
- aggregation_temporality=AGGREGATION_TEMPORALITY_CUMULATIVE,
- is_monotonic=True,
- )
-
- if not series and not self._registry.counter_label_names(name):
- point = NumberDataPoint(
- start_time_unix_nano=self._start_time_ns,
- time_unix_nano=now_ns,
- as_int=0,
- )
- sum_metric.data_points.append(point)
- return sum_metric
-
- for labels, value, start_time_ns in series:
- point = NumberDataPoint(
- start_time_unix_nano=start_time_ns,
- time_unix_nano=max(now_ns, start_time_ns),
- )
- point.attributes.extend(self._build_attributes(labels))
- if float(value).is_integer() and -(2**63) <= value < 2**63:
- point.as_int = int(value)
- else:
- point.as_double = float(value)
- sum_metric.data_points.append(point)
- return sum_metric
-
- def _build_gauge(self, series, now_ns: int, *, name: str) -> Gauge:
- """
- Build an OTLP Gauge message from a registry gauge family.
-
- Args:
- series: List of `(labels, value)` tuples from MetricRegistry.
- now_ns: Export timestamp in nanoseconds.
- name: Metric family name for zero-series handling.
- """
- gauge_metric = Gauge()
-
- if not series and not self._registry.gauge_label_names(name):
- point = NumberDataPoint(time_unix_nano=now_ns, as_int=0)
- gauge_metric.data_points.append(point)
- return gauge_metric
-
- for labels, value in series:
- point = NumberDataPoint(time_unix_nano=now_ns)
- point.attributes.extend(self._build_attributes(labels))
- if float(value).is_integer() and -(2**63) <= value < 2**63:
- point.as_int = int(value)
- else:
- point.as_double = float(value)
- gauge_metric.data_points.append(point)
- return gauge_metric
-
- def _build_histogram(
- self,
- *,
- label_names: tuple[str, ...],
- bucket_bounds: tuple[float, ...],
- series_list,
- now_ns: int,
- ) -> Histogram:
- """
- Build an OTLP Histogram message from a registry histogram family.
-
- Args:
- label_names: Registered label keys for the family.
- bucket_bounds: Explicit bucket upper bounds from the registry.
- series_list: Registry series with bucket counts, totals and start_time_ns.
- now_ns: Export timestamp in nanoseconds.
- """
- histogram_metric = Histogram(
- aggregation_temporality=AGGREGATION_TEMPORALITY_CUMULATIVE,
- )
-
- if not series_list and not label_names:
- point = HistogramDataPoint(
- start_time_unix_nano=self._start_time_ns,
- time_unix_nano=now_ns,
- count=0,
- sum=0.0,
- )
- point.bucket_counts.extend((0,) * (len(bucket_bounds) + 1))
- point.explicit_bounds.extend(bucket_bounds)
- histogram_metric.data_points.append(point)
- return histogram_metric
-
- for labels, bucket_counts, count, value_sum, start_time_ns in series_list:
- point = HistogramDataPoint(
- start_time_unix_nano=start_time_ns,
- time_unix_nano=max(now_ns, start_time_ns),
- count=int(count),
- sum=float(value_sum),
- )
- point.attributes.extend(self._build_attributes(labels))
- point.bucket_counts.extend(int(v) for v in bucket_counts)
- point.explicit_bounds.extend(float(v) for v in bucket_bounds)
- histogram_metric.data_points.append(point)
- return histogram_metric
-
- def _build_attributes(self, labels: Iterable[Tuple[str, str]]) -> list[KeyValue]:
- """
- Convert MetricRegistry labels into OTLP KeyValue attributes.
-
- Args:
- labels: Iterable of sorted `(key, value)` label pairs.
- """
- return [KeyValue(key=key, value=AnyValue(string_value=value)) for key, value in labels]
-
- async def _send_request(self, request: ExportMetricsServiceRequest) -> None:
- """
- Send one OTLP metrics export request using the configured transport.
-
- Args:
- request: The populated OTLP metrics export payload.
- """
- metric_count = self._count_otlp_metrics(request)
- request_bytes = request.ByteSize()
- try:
- if self._protocol == "http":
- await asyncio.to_thread(self._send_http_request, request)
- else:
- await asyncio.to_thread(self._send_grpc_request, request)
- except (RequestException, OSError, TimeoutError) as exc:
- logger.warning(
- "[OTelMetricExporter] export failed: phase=send_request protocol=%s endpoint=%s timeout_seconds=%s metric_count=%s request_bytes=%s error_type=%s error=%s",
- self._protocol,
- self._endpoint,
- self._export_timeout_seconds,
- metric_count,
- request_bytes,
- type(exc).__name__,
- exc,
- )
- except Exception as exc:
- # Keep exporter failure best-effort. Log unexpected failures at warning level.
- logger.warning(
- "[OTelMetricExporter] export failed: phase=send_request protocol=%s endpoint=%s timeout_seconds=%s metric_count=%s request_bytes=%s error_type=%s error=%s",
- self._protocol,
- self._endpoint,
- self._export_timeout_seconds,
- metric_count,
- request_bytes,
- type(exc).__name__,
- exc,
- )
-
- def _count_otlp_metrics(self, request: ExportMetricsServiceRequest) -> int:
- """
- Count OTLP metric families in one export request for logging purposes.
-
- Args:
- request: The populated OTLP metrics export payload.
-
- Returns:
- Number of OTLP Metric messages in the request.
- """
- count = 0
- for resource_metrics in request.resource_metrics:
- for scope_metrics in resource_metrics.scope_metrics:
- count += len(scope_metrics.metrics)
- return count
-
- def _send_http_request(self, request: ExportMetricsServiceRequest) -> None:
- """
- Send one OTLP/HTTP metrics request.
-
- Args:
- request: The populated OTLP metrics export payload.
- """
- if self._http_session is None:
- return
- payload = request.SerializeToString()
- request_headers = {"Content-Type": "application/x-protobuf"}
- if self._headers:
- request_headers.update(self._headers)
- self._http_session.post(
- self._endpoint,
- data=payload,
- headers=request_headers,
- timeout=self._export_timeout_seconds,
- ).raise_for_status()
-
- def _send_grpc_request(self, request: ExportMetricsServiceRequest) -> None:
- """
- Send one OTLP/gRPC metrics request.
-
- Args:
- request: The populated OTLP metrics export payload.
- """
- if self._grpc_stub is None:
- return
- metadata = list(self._headers.items()) if self._headers else None
- if metadata:
- self._grpc_stub.Export(
- request,
- timeout=self._export_timeout_seconds,
- metadata=metadata,
- )
- else:
- self._grpc_stub.Export(request, timeout=self._export_timeout_seconds)
-
def start(self) -> None:
- """
- Start the exporter-owned periodic background loop.
-
- The exporter keeps its own lifecycle so application startup can create the
- exporter once and let it handle periodic exports internally.
- """
- if not self._enabled:
- return
- if self._export_task is not None and not self._export_task.done():
+ if self._closed or self._export_task is not None:
return
try:
- self._export_task = asyncio.create_task(
- self._run_export_loop(),
- name="openviking-otel-metrics-export-loop",
- )
+ loop = asyncio.get_running_loop()
except RuntimeError:
- logger.warning(
- "[OTelMetricExporter] periodic export loop not started: phase=start protocol=%s endpoint=%s reason=no running event loop",
- self._protocol,
- self._endpoint,
- )
- self._export_task = None
+ logger.warning("OTLP periodic export requires a running application event loop")
+ return
+ self._export_task = loop.create_task(self._run(), name="openviking-otel-metrics")
- async def _run_export_loop(self) -> None:
- """
- Run the periodic export loop until shutdown is requested.
- """
- interval_s = float(self._export_interval_ms) / 1000.0
- next_tick = time.monotonic() + interval_s
- while self._enabled:
+ async def _run(self) -> None:
+ loop = asyncio.get_running_loop()
+ next_tick = loop.time()
+ while not self._closed:
try:
await self.export()
- except asyncio.CancelledError:
- raise
except Exception as exc:
- logger.warning(
- "[OTelMetricExporter] periodic export loop iteration failed: phase=run_export_loop protocol=%s endpoint=%s error=%s",
- self._protocol,
- self._endpoint,
- exc,
- )
- now = time.monotonic()
- next_tick, sleep_s = _compute_next_tick_and_sleep(
- now=now,
- interval_s=interval_s,
- next_tick=next_tick,
- )
- await asyncio.sleep(sleep_s)
+ logger.warning("OTLP metrics export failed: %s", exc)
+ next_tick += self._interval
+ now = loop.time()
+ if next_tick < now:
+ next_tick = now + self._interval
+ await asyncio.sleep(next_tick - now)
async def shutdown(self) -> None:
- """
- Stop the exporter loop and release transport resources.
- """
- self._enabled = False
-
+ self._closed = True
if self._export_task is not None:
self._export_task.cancel()
try:
await self._export_task
except asyncio.CancelledError:
- logger.info(
- "[OTelMetricExporter] periodic export loop cancelled: phase=shutdown protocol=%s endpoint=%s",
- self._protocol,
- self._endpoint,
- )
- except Exception as exc:
- logger.warning(
- "[OTelMetricExporter] periodic export loop shutdown failed: phase=shutdown protocol=%s endpoint=%s error=%s",
- self._protocol,
- self._endpoint,
- exc,
- )
+ pass
self._export_task = None
-
- if self._http_session is not None:
- try:
- self._http_session.close()
- except Exception as exc:
- logger.warning(
- "[OTelMetricExporter] http session close failed: phase=shutdown protocol=%s endpoint=%s error_type=%s error=%s",
- self._protocol,
- self._endpoint,
- type(exc).__name__,
- exc,
- )
- self._http_session = None
-
- if self._grpc_channel is not None:
- try:
- # grpc.Channel.close() is synchronous and returns None.
- self._grpc_channel.close()
- except Exception as exc:
- logger.warning(
- "[OTelMetricExporter] grpc channel close failed: phase=shutdown protocol=%s endpoint=%s error_type=%s error=%s",
- self._protocol,
- self._endpoint,
- type(exc).__name__,
- exc,
- )
- self._grpc_channel = None
- self._grpc_stub = None
+ async with self._export_lock:
+ if self._sdk is not None:
+ await asyncio.to_thread(self._sdk.shutdown)
+ self._sdk = None
diff --git a/openviking/metrics/exporters/prometheus.py b/openviking/metrics/exporters/prometheus.py
index 38a97616e..d99af5ab1 100644
--- a/openviking/metrics/exporters/prometheus.py
+++ b/openviking/metrics/exporters/prometheus.py
@@ -1,203 +1,79 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""
-Prometheus exporter implementation.
+"""Expose SDK metric data using the official Prometheus client serializer."""
-Key behaviors:
-- On each export, attempts to refresh all registered collectors (best-effort).
-- Renders the current registry state into Prometheus text exposition format.
+from itertools import accumulate, chain
-The exporter is intentionally tolerant:
-- Collector refresh errors are swallowed (metrics must not break `/metrics`).
-- Families with no samples are still exported as a single zero sample when possible,
- to keep dashboards stable.
-"""
-
-from __future__ import annotations
-
-from typing import Iterable
+from opentelemetry.sdk.metrics.export import Gauge, Histogram, Sum
+from prometheus_client import CollectorRegistry, generate_latest
+from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily
from openviking.metrics.core.base import MetricExporter
-from openviking.metrics.core.types import render_labels
class PrometheusExporter(MetricExporter):
- """
- Render the in-process registry into Prometheus text exposition format.
-
- This exporter optionally refreshes refresh-managed collectors before rendering, then walks
- the registry families and serializes counters, gauges, histograms, and dropped-series
- diagnostics into a single text payload suitable for `/metrics`.
- """
+ """Adapt SDK data to Prometheus metric families, preserving OpenViking names and tags."""
- def __init__(
- self, *, registry, collector_manager=None, refresh_deadline_seconds: float = 1.0
- ) -> None:
- """
- Initialize the Prometheus exporter.
-
- Args:
- registry: The in-process metric registry that stores current metric values.
- collector_manager: Optional refresh orchestrator invoked before each export.
- refresh_deadline_seconds: Best-effort time budget for the pre-render refresh stage.
- """
+ def __init__(self, *, registry, collector_manager=None, refresh_deadline_seconds=1.0):
self._registry = registry
self._collector_manager = collector_manager
self._refresh_deadline_seconds = float(refresh_deadline_seconds)
+ self._prometheus = CollectorRegistry()
+ self._prometheus.register(self)
async def export(self) -> str:
- """
- Refresh collectors if configured and return the Prometheus exposition payload.
-
- Collector refresh remains best-effort: refresh failures are swallowed so observability
- does not break the serving of the metrics endpoint itself.
- """
if self._collector_manager is not None:
try:
await self._collector_manager.refresh_all(
- self._registry,
- deadline_seconds=self._refresh_deadline_seconds,
+ self._registry, deadline_seconds=self._refresh_deadline_seconds
)
except Exception:
pass
return self.render()
def render(self) -> str:
- """
- Serialize the registry into Prometheus text exposition format.
-
- Returns:
- A newline-terminated text payload containing every registered family in the
- Prometheus exposition layout expected by scrapers and tooling.
- """
- lines: list[str] = []
-
- # Process counters
- for name, counter in self._registry.iter_counters():
- self._add_help_and_type(lines, name, "counter")
- self._process_counter_series(lines, name, counter)
-
- # Process gauges
- for name, gauge in self._registry.iter_gauges():
- self._add_help_and_type(lines, name, "gauge")
- self._process_gauge_series(lines, name, gauge)
-
- # Process histograms
- for name, label_names, bucket_bounds, series_iter in self._registry.iter_histograms():
- self._add_help_and_type(lines, name, "histogram")
- series_list = list(series_iter)
- if not series_list and not label_names:
- self._render_empty_histogram(lines, name, bucket_bounds)
- else:
- for labels, bucket_counts, count, value_sum in series_list:
- self._render_histogram_series(
- lines, name, bucket_bounds, labels, bucket_counts, count, value_sum
- )
-
- # Process dropped series
- for metric_name, dropped in self._registry.iter_dropped_series():
- lines.append(
- f'openviking_metrics_dropped_series_total{{metric="{metric_name}"}} {dropped}'
- )
-
- return "\n".join(lines) + "\n"
-
- def _add_help_and_type(self, lines: list[str], name: str, metric_type: str) -> None:
- """Append the Prometheus `HELP` and `TYPE` prelude for one metric family."""
- lines.append(f"# HELP {name} OpenViking metric.")
- lines.append(f"# TYPE {name} {metric_type}")
-
- def _process_counter_series(self, lines: list[str], name: str, counter) -> None:
- """
- Render all counter series for one family into the output buffer.
-
- Empty unlabeled counters are rendered as a single zero sample to keep dashboards and
- alert queries stable before the first real increment occurs.
- """
- if not counter and not self._registry.counter_label_names(name):
- lines.append(f"{name} 0")
- for labels, value in counter:
- lines.append(
- f"{name}{render_labels(labels)} {int(value) if value.is_integer() else value}"
- )
-
- def _process_gauge_series(self, lines: list[str], name: str, gauge) -> None:
- """
- Render all gauge series for one family into the output buffer.
-
- Empty unlabeled gauges also emit one zero sample so consumers do not need to special-case
- missing series before initialization.
- """
- if not gauge and not self._registry.gauge_label_names(name):
- lines.append(f"{name} 0")
- for labels, value in gauge:
- lines.append(f"{name}{render_labels(labels)} {value}")
-
- def _render_empty_histogram(
- self, lines: list[str], name: str, bucket_bounds: tuple[float, ...]
- ) -> None:
- """Render an unlabeled histogram family with zero samples across every bucket."""
- zero_counts = (0,) * (len(bucket_bounds) + 1)
- cumulative = 0
- for bound, c in _iter_histogram_buckets(bucket_bounds, zero_counts):
- cumulative += c
- lines.append(f"{name}_bucket{render_labels((('le', bound),))} {cumulative}")
- lines.append(f"{name}_count 0")
- lines.append(f"{name}_sum 0")
-
- def _render_histogram_series(
- self,
- lines: list[str],
- name: str,
- bucket_bounds: tuple[float, ...],
- labels: tuple[tuple[str, str], ...],
- bucket_counts: tuple[int, ...],
- count: int,
- value_sum: float,
- ) -> None:
- """
- Render one histogram series with stable label ordering and cumulative bucket counts.
-
- Histogram buckets are exported cumulatively per the Prometheus exposition rules, while
- `_count` and `_sum` reuse the same sorted label tuple for deterministic output.
- """
- # Sort labels once per series to avoid repeated sorting
- sorted_labels = sorted(labels, key=lambda x: x[0])
- cumulative = 0
- for bound, c in _iter_histogram_buckets(bucket_bounds, bucket_counts):
- cumulative += c
- # Insert 'le' label in correct position without full sort
- merged_labels = self._merge_le_label(sorted_labels, str(bound))
- lines.append(f"{name}_bucket{render_labels(merged_labels)} {cumulative}")
- # Use sorted labels for count and sum to ensure consistent order
- lines.append(f"{name}_count{render_labels(tuple(sorted_labels))} {count}")
- lines.append(f"{name}_sum{render_labels(tuple(sorted_labels))} {value_sum}")
-
- def _merge_le_label(
- self, sorted_labels: list[tuple[str, str]], le: str
- ) -> tuple[tuple[str, str], ...]:
- """Insert the histogram `le` label into an already sorted label list without resorting."""
- # Find insertion point for 'le' label
- pos = len(sorted_labels)
- for i, (key, _) in enumerate(sorted_labels):
- if key > "le":
- pos = i
- break
- # Create new list with 'le' inserted
- merged = sorted_labels.copy()
- merged.insert(pos, ("le", le))
- return tuple(merged)
-
-
-def _iter_histogram_buckets(
- bounds: tuple[float, ...], counts: tuple[int, ...]
-) -> Iterable[tuple[str, int]]:
- """
- Yield histogram bucket upper bounds paired with their per-bucket sample counts.
-
- The final synthetic `+Inf` bucket is always yielded after the configured finite bounds.
- """
- for i, upper in enumerate(bounds):
- yield str(upper), counts[i]
- yield "+Inf", counts[len(bounds)]
+ return generate_latest(self._prometheus).decode("utf-8")
+
+ def collect(self):
+ """Implement the Prometheus custom-collector boundary for SDK and native snapshots."""
+ data = self._registry.collect()
+ for resource in data.resource_metrics:
+ for scope in resource.scope_metrics:
+ for metric in scope.metrics:
+ points = metric.data.data_points
+ if not points:
+ continue
+ keys = sorted(points[0].attributes)
+ if isinstance(metric.data, Sum):
+ family = CounterMetricFamily(metric.name, "OpenViking metric.", labels=keys)
+ elif isinstance(metric.data, Gauge):
+ family = GaugeMetricFamily(metric.name, "OpenViking metric.", labels=keys)
+ elif isinstance(metric.data, Histogram):
+ family = HistogramMetricFamily(
+ metric.name, "OpenViking metric.", labels=keys
+ )
+ else:
+ raise TypeError(
+ f"unsupported SDK metric data: {type(metric.data).__name__}"
+ )
+ for point in points:
+ labels = [str(point.attributes[key]) for key in keys]
+ if isinstance(metric.data, Histogram):
+ family.add_metric(
+ labels,
+ buckets=list(
+ zip(
+ chain(
+ (str(bound) for bound in point.explicit_bounds),
+ ("+Inf",),
+ ),
+ accumulate(point.bucket_counts),
+ strict=True,
+ )
+ ),
+ sum_value=point.sum,
+ )
+ else:
+ family.add_metric(labels, point.value)
+ yield family
diff --git a/openviking/metrics/global_api.py b/openviking/metrics/global_api.py
index f7156347f..ef756b6bc 100644
--- a/openviking/metrics/global_api.py
+++ b/openviking/metrics/global_api.py
@@ -27,18 +27,12 @@ from typing import Optional
from openviking.observability.events import (
make_payload_subscriber,
register_event_subscriber,
- try_publish_event,
unregister_event_subscriber,
)
from openviking.server.config import ServerConfig
from openviking_cli.utils import get_logger
-from .account_dimension import (
- configure_metric_account_dimension as _configure_metric_account_dimension_runtime,
-)
-from .account_dimension import (
- reset_metric_account_dimension,
-)
+from .account_dimension import configure_metric_account_dimension, reset_metric_account_dimension
from .bootstrap import create_default_collector_manager
from .collectors.cache import CacheCollector
from .collectors.embedding import EmbeddingCollector
@@ -99,40 +93,6 @@ def _shutdown_exporters_best_effort(exporters: list) -> None:
)
-def configure_metric_account_dimension(
- *,
- enabled: bool | None = None,
- metric_allowlist: set[str] | list[str] | tuple[str, ...] | None = None,
- max_active_accounts: int | None = None,
- policy=None,
- resolver=None,
-) -> None:
- """
- Configure the process-global account-dimension runtime used by collector write helpers.
-
- This function is a thin façade over `account_dimension.configure_metric_account_dimension`
- so callers do not need to import policy internals directly from the metrics package.
- """
- _configure_metric_account_dimension_runtime(
- enabled=enabled,
- metric_allowlist=metric_allowlist,
- max_active_accounts=max_active_accounts,
- policy=policy,
- resolver=resolver,
- )
-
-
-def _create_metric_registry() -> MetricRegistry:
- """
- Create a new `MetricRegistry` instance.
-
- This helper centralizes the registry instantiation to avoid security issues
- introduced by bridging classes (e.g., an external registry implementation
- bypassing internal safety checks).
- """
- return MetricRegistry()
-
-
def init_metrics_from_server_config(
config: ServerConfig, *, app=None, service=None, registry: MetricRegistry | None = None
) -> None:
@@ -172,7 +132,7 @@ def init_metrics_from_server_config(
metric_allowlist=list(account_dimension.metric_allowlist or []),
max_active_accounts=int(account_dimension.max_active_accounts),
)
- _registry = registry or _create_metric_registry()
+ _registry = registry or MetricRegistry()
collector_manager = create_default_collector_manager(
app=app, service=service, config=config
)
@@ -203,7 +163,6 @@ def init_metrics_from_server_config(
service_name=exporters_config.otel.service_name,
export_interval_ms=exporters_config.otel.export_interval_ms,
headers=exporters_config.otel.headers,
- enabled=True,
)
otel_exporter.start()
_exporters.append(otel_exporter)
@@ -268,27 +227,6 @@ def get_metrics_registry() -> MetricRegistry:
return reg
-def try_get_metrics_registry() -> MetricRegistry | None:
- """
- Return the global MetricRegistry if metrics are initialized, otherwise `None`.
-
- This helper is intended for best-effort call sites that want to avoid raising an exception
- when metrics are disabled.
- """
- return _registry
-
-
-def try_dispatch_event(event_name: str, payload: dict) -> None:
- """
- Publish an in-process observability event.
-
- Kept for older call sites that imported the metrics API directly. The event is now
- published to the shared bus, so non-metrics subscribers can consume it even when
- Prometheus metrics are disabled.
- """
- try_publish_event(event_name, payload)
-
-
def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:
"""
Build the default in-process event router for event-driven metrics.
diff --git a/pyproject.toml b/pyproject.toml
index 494082049..a2e6980ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -77,10 +77,11 @@ dependencies = [
"tree-sitter-php==0.24.1",
"tree-sitter-lua==0.5.0",
# OpenTelemetry
- "opentelemetry-api>=1.14",
- "opentelemetry-sdk>=1.14",
- "opentelemetry-exporter-otlp-proto-grpc>=1.14",
- "opentelemetry-exporter-otlp-proto-http>=1.14",
+ "prometheus-client>=0.24,<1",
+ "opentelemetry-api>=1.40,<2",
+ "opentelemetry-sdk>=1.40,<2",
+ "opentelemetry-exporter-otlp-proto-grpc>=1.40,<2",
+ "opentelemetry-exporter-otlp-proto-http>=1.40,<2",
"opentelemetry-instrumentation-asyncio>=0.61b0",
"loguru>=0.7.3",
"cryptography>=48.0.1",
diff --git a/tests/agfs/test_fs_binding.py b/tests/agfs/test_fs_binding.py
index 6ef1e5256..ac55b5973 100644
--- a/tests/agfs/test_fs_binding.py
+++ b/tests/agfs/test_fs_binding.py
@@ -49,7 +49,20 @@ class TestVikingFSBindingLocal:
async def test_native_metrics_and_legacy_stats(self, viking_fs_binding_instance):
"""Use the binding fixture to check flat metrics and legacy stats; return None."""
client = viking_fs_binding_instance._async_agfs
- metrics = await client.run("metrics")
+ lease = await client.pathlock_acquire_tree(
+ f"/local/default/temp/metrics_scan_{uuid.uuid4().hex}"
+ )
+ try:
+ metrics = await client.run("metrics")
+ finally:
+ await client.pathlock_release(lease)
+ scan = next(m for m in metrics if m["name"] == "lock_descendant_scan_duration_seconds")
+ legacy_count = next(m for m in metrics if m["name"] == "lock_descendant_scans_total")
+ legacy_sum = next(
+ m for m in metrics if m["name"] == "lock_descendant_scan_duration_seconds_total"
+ )
+ assert scan["count"] == legacy_count["value"] > 0
+ assert scan["sum"] == legacy_sum["value"]
shapes = {
"counter": {"value", "scale"},
"gauge": {"value"},
@@ -70,7 +83,9 @@ class TestVikingFSBindingLocal:
"route",
}
if metric["type"] == "histogram":
- assert len(metric["bucket_counts"]) == len(metric["bucket_bounds"]) + 1 == 12
+ assert metric["count"] > 0
+ assert len(metric["bucket_counts"]) == len(metric["bucket_bounds"]) + 1
+ assert metric["bucket_bounds"][-1] > 404261.58497 * 1e9
assert sum(metric["bucket_counts"]) == metric["count"]
assert metric["scale"] == 1e-9 and type(metric["sum"]) is int
else:
diff --git a/tests/metrics/collectors/test_collector_contracts.py b/tests/metrics/collectors/test_collector_contracts.py
index 68f82705e..ff5b22237 100644
--- a/tests/metrics/collectors/test_collector_contracts.py
+++ b/tests/metrics/collectors/test_collector_contracts.py
@@ -194,9 +194,13 @@ def test_domain_stats_delta_helper_applies_non_negative_deltas_and_handles_reset
collector.collect(registry)
- counters = dict(registry.iter_counters())
- series = dict(counters["openviking_test_cumulative_total"])
- assert series[()] == 18
+ metrics = {
+ m.name: m
+ for r in registry.collect().resource_metrics
+ for s in r.scope_metrics
+ for m in s.metrics
+ }
+ assert metrics["openviking_test_cumulative_total"].data.data_points[0].value == 18
def test_service_probe_collector_emits_valid_on_success_and_invalid_on_failure(
diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.py
index d8ee26aa6..1f316010e 100644
--- a/tests/metrics/collectors/test_event_collectors.py
+++ b/tests/metrics/collectors/test_event_collectors.py
@@ -5,7 +5,6 @@ from __future__ import annotations
import re
-import openviking.metrics.global_api as global_api
from openviking.metrics.collectors.base import EventMetricCollector
from openviking.metrics.collectors.cache import CacheCollector
from openviking.metrics.collectors.embedding import EmbeddingCollector
@@ -69,21 +68,22 @@ def test_queue_duration_collector_records_processing_and_end_to_end_histograms(
{
"queue": "Semantic",
"outcome": "success",
- "process_duration_seconds": 2.5,
- "end_to_end_duration_seconds": 8.0,
+ "process_duration_seconds": 60.0,
+ "end_to_end_duration_seconds": 404261.58497,
},
registry,
)
text = render_prometheus(registry)
- labels = 'outcome="success",queue="Semantic"'
+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)
+ labels = f'bucket_schema="{schema}",outcome="success",queue="Semantic"'
assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text
- assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 2.5" in text
+ assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 60.0" in text
assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text
- assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 8.0" in text
+ assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 404261.58497" in text
assert (
- "openviking_queue_end_to_end_duration_seconds_bucket{"
- 'le="86400.0",outcome="success",queue="Semantic"} 1' in text
+ f'openviking_queue_end_to_end_duration_seconds_bucket{{bucket_schema="{schema}",'
+ 'le="500000.0",outcome="success",queue="Semantic"} 1' in text
)
@@ -121,9 +121,10 @@ def test_queue_duration_collector_allows_processing_only_and_ignores_bad_payload
)
text = render_prometheus(registry)
- assert (
- "openviking_queue_process_duration_seconds_count{"
- 'outcome="requeued",queue="Embedding"} 1' in text
+ assert re.search(
+ r'openviking_queue_process_duration_seconds_count\{bucket_schema="[0-9a-f]{16}",'
+ r'outcome="requeued",queue="Embedding"\} 1',
+ text,
)
assert "openviking_queue_end_to_end_duration_seconds" not in text
assert 'outcome="unknown"' not in text
@@ -270,17 +271,6 @@ def test_telemetry_bridge_collector_ignores_malformed_payloads_instead_of_raisin
assert "openviking_operation_requests_total" not in text
-def test_http_collector_importable() -> None:
- from openviking.metrics.collectors.http import HTTPCollector
-
- assert HTTPCollector is not None
-
-
-def test_global_api_no_longer_exports_metrics_enabled_helper() -> None:
- """Avoid redundant exports: metrics enablement is read via direct config access."""
- assert not hasattr(global_api, "is_metrics_enabled_from_server_config")
-
-
def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prometheus):
"""`embedding.call` must produce calls/duration/tokens metrics keyed by provider+model."""
EmbeddingCollector().receive(
@@ -422,8 +412,10 @@ def test_embedding_collector_records_success_volume_and_latency(registry, render
'openviking_embedding_requests_total{account_id="__unknown__",status="ok"} 1' in text
or 'openviking_embedding_requests_total{account_id="__unknown__",status="ok"} 1.0' in text
)
- assert (
- 'openviking_embedding_latency_seconds_count{account_id="__unknown__",status="ok"} 1' in text
+ assert re.search(
+ r'openviking_embedding_latency_seconds_count\{account_id="__unknown__",'
+ r'bucket_schema="[0-9a-f]{16}",status="ok"\} 1',
+ text,
)
diff --git a/tests/metrics/collectors/test_state_collectors.py b/tests/metrics/collectors/test_state_collectors.py
index 57b5795cd..ddcac0fd2 100644
--- a/tests/metrics/collectors/test_state_collectors.py
+++ b/tests/metrics/collectors/test_state_collectors.py
@@ -394,12 +394,18 @@ def test_ragfs_collector_replaces_recovers_and_deletes(monkeypatch):
},
]
collector.collect(registry)
- assert dict(registry.iter_counters())["openviking_ragfs_calls_total"] == [((), count)]
+ metrics = {
+ m.name: m
+ for r in registry.collect().resource_metrics
+ for s in r.scope_metrics
+ for m in s.metrics
+ }
+ assert metrics["openviking_ragfs_calls_total"].data.data_points[0].value == count
assert registry.gauge_get("openviking_ragfs_tasks") == count
- hist = list(registry.iter_histograms())[0]
- assert hist[2] == pytest.approx((1e-6,))
- assert hist[3][0][1:3] == ((count, 0), count)
- assert hist[3][0][3] == pytest.approx(123e-9 * count)
+ hist = metrics["openviking_ragfs_latency_seconds"].data.data_points[0]
+ assert hist.explicit_bounds == pytest.approx((1e-6,))
+ assert hist.bucket_counts == (count, 0) and hist.count == count
+ assert hist.sum == pytest.approx(123e-9 * count)
assert "account_id" not in PrometheusExporter(registry=registry).render()
previous = PrometheusExporter(registry=registry).render()
records.append({"broken": True})
@@ -425,16 +431,14 @@ def test_ragfs_collector_replaces_recovers_and_deletes(monkeypatch):
patch.setattr(registry, "set_gauge", fail)
with pytest.raises(RuntimeError, match="injected"):
collector.collect(registry)
- assert dict(registry.iter_counters())["openviking_ragfs_partial_total"] == [((), 9)]
+ assert "openviking_ragfs_partial_total 9.0" in PrometheusExporter(registry=registry).render()
records.clear()
with monkeypatch.context() as patch:
- patch.setattr(registry, "counter_delete_matching", fail)
+ patch.setattr(registry, "delete_matching", fail)
with pytest.raises(RuntimeError, match="injected"):
collector.collect(registry)
collector.collect(registry)
- assert list(registry.iter_counters()) == []
- assert list(registry.iter_gauges()) == []
- assert list(registry.iter_histograms()) == []
+ assert not registry.collect().resource_metrics[0].scope_metrics[0].metrics
assert "openviking_ragfs_" not in PrometheusExporter(registry=registry).render()
diff --git a/tests/metrics/core/test_exporter.py b/tests/metrics/core/test_exporter.py
index c879ccede..724eaa704 100644
--- a/tests/metrics/core/test_exporter.py
+++ b/tests/metrics/core/test_exporter.py
@@ -2,166 +2,93 @@
# SPDX-License-Identifier: AGPL-3.0
import asyncio
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from threading import Event, Thread
import pytest
+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
-from openviking.metrics.exporters.otel import OTelMetricExporter, _compute_next_tick_and_sleep
+from openviking.metrics.exporters.otel import OTelMetricExporter
def test_exporter_outputs_help_type_and_empty_metrics(registry, render_prometheus):
- registry.counter("openviking_empty_counter_total")
- registry.gauge("openviking_empty_gauge")
- registry.histogram("openviking_empty_histogram_seconds")
+ registry.set_counter("openviking_empty_counter_total", 0)
+ registry.set_gauge("openviking_empty_gauge", 0)
+ registry.set_histogram(
+ "openviking_empty_histogram_seconds",
+ bucket_bounds=(1,),
+ bucket_counts=(0, 0),
+ count=0,
+ value_sum=0,
+ )
text = render_prometheus(registry)
assert "# TYPE openviking_empty_counter_total counter" in text
- assert "openviking_empty_counter_total 0" in text
+ assert "openviking_empty_counter_total 0.0" in text
assert "# TYPE openviking_empty_histogram_seconds histogram" in text
- assert "openviking_empty_histogram_seconds_count 0" in text
- assert 'openviking_empty_histogram_seconds_bucket{le="+Inf"} 0' in text
+ assert 'le="+Inf"} 0.0' in text
@pytest.mark.asyncio
-async def test_otel_exporter_start_and_shutdown_manage_background_loop(registry, monkeypatch):
- """OTel exporter should own its periodic export loop lifecycle."""
- exporter = OTelMetricExporter(registry=registry, enabled=False, export_interval_ms=10)
- exporter._enabled = True
-
- calls = 0
- tick = asyncio.Event()
-
- async def _fake_export():
- nonlocal calls
- calls += 1
- tick.set()
- return ""
-
- monkeypatch.setattr(exporter, "export", _fake_export)
-
- exporter.start()
- await asyncio.wait_for(tick.wait(), timeout=0.2)
- assert calls >= 1
- assert exporter._export_task is not None
-
- task_before = exporter._export_task
- exporter.start()
- assert exporter._export_task is task_before
-
- await exporter.shutdown()
- assert exporter._export_task is None
-
-
-async def test_otel_export_loop_aligns_to_next_fixed_tick_after_overrun():
- """Scheduling helper should advance next tick and compute sleep after one interval overrun."""
- next_tick, sleep_s = _compute_next_tick_and_sleep(now=1.15, interval_s=1.0, next_tick=1.0)
- assert next_tick == pytest.approx(2.0)
- assert sleep_s == pytest.approx(0.85)
-
-
-def test_otel_http_send_uses_export_timeout_not_refresh_deadline(registry):
- """HTTP transport timeout should be decoupled from collector refresh timeout."""
+async def test_otel_exporter_start_and_shutdown_manage_background_loop(registry):
+ """Exercise the real SDK HTTP exporter and wait for its in-flight send at shutdown."""
+ received, release = Event(), Event()
+ requests = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers["Content-Length"]))
+ requests.append((self.path, self.headers, ExportMetricsServiceRequest.FromString(body)))
+ received.set()
+ release.wait(5)
+ self.send_response(200)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+
+ def log_message(self, *args):
+ pass
+
+ server = HTTPServer(("127.0.0.1", 0), Handler)
+ thread = Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ registry.inc_counter("openviking_operation_requests_total", amount=2)
+ registry.observe_histogram("openviking_operation_duration_seconds", 60.123)
exporter = OTelMetricExporter(
registry=registry,
- enabled=False,
protocol="http",
- endpoint="http://localhost:4318/v1/metrics",
- refresh_deadline_seconds=1.0,
+ endpoint=f"http://127.0.0.1:{server.server_port}/v1/metrics",
+ service_name="metrics-contract",
+ headers={"X-ByteAPM-AppKey": "test-app-key"},
)
-
- class _FakeSession:
- def __init__(self):
- self.timeout = None
-
- def post(self, _url, **kwargs):
- self.timeout = kwargs.get("timeout")
-
- class _Resp:
- @staticmethod
- def raise_for_status():
- return None
-
- return _Resp()
-
- fake_session = _FakeSession()
- exporter._http_session = fake_session
-
- request = exporter._build_export_request()
- assert request is None
-
- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
- ExportMetricsServiceRequest,
- )
-
- exporter._send_http_request(ExportMetricsServiceRequest())
-
- assert fake_session.timeout == 5.0
-
-
-def test_otel_http_send_forwards_custom_headers(registry):
- """HTTP transport should merge vendor auth headers with protobuf content type."""
- exporter = OTelMetricExporter(
- registry=registry,
- enabled=False,
- protocol="http",
- endpoint="https://apmplus-cn-beijing.ivolces.com/api/otlp/v1/metrics",
- headers={"X-ByteAPM-AppKey": "metric-appkey"},
- )
-
- class _FakeSession:
- def __init__(self):
- self.headers = None
-
- def post(self, _url, **kwargs):
- self.headers = kwargs.get("headers")
-
- class _Resp:
- @staticmethod
- def raise_for_status():
- return None
-
- return _Resp()
-
- exporter._http_session = _FakeSession()
-
- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
- ExportMetricsServiceRequest,
- )
-
- exporter._send_http_request(ExportMetricsServiceRequest())
-
- assert exporter._http_session.headers == {
- "Content-Type": "application/x-protobuf",
- "X-ByteAPM-AppKey": "metric-appkey",
- }
-
-
-def test_otel_grpc_send_forwards_custom_metadata(registry):
- """gRPC transport should forward vendor auth headers as metadata pairs."""
- exporter = OTelMetricExporter(
- registry=registry,
- enabled=False,
- protocol="grpc",
- endpoint="apmplus-cn-beijing.ivolces.com:4317",
- headers={"x-byteapm-appkey": "metric-appkey"},
- )
-
- class _FakeStub:
- def __init__(self):
- self.metadata = None
- self.timeout = None
-
- def Export(self, _request, **kwargs):
- self.metadata = kwargs.get("metadata")
- self.timeout = kwargs.get("timeout")
- return None
-
- exporter._grpc_stub = _FakeStub()
-
- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
- ExportMetricsServiceRequest,
- )
-
- exporter._send_grpc_request(ExportMetricsServiceRequest())
-
- assert exporter._grpc_stub.metadata == [("x-byteapm-appkey", "metric-appkey")]
- assert exporter._grpc_stub.timeout == 5.0
+ stopping = None
+ try:
+ exporter.start()
+ exporter.start()
+ assert await asyncio.to_thread(received.wait, 2)
+ stopping = asyncio.create_task(exporter.shutdown())
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(asyncio.shield(stopping), 0.05)
+ release.set()
+ await asyncio.wait_for(stopping, 2)
+ await exporter.export()
+ assert len(requests) == 1
+ path, headers, request = requests[0]
+ assert path == "/v1/metrics"
+ assert headers["X-ByteAPM-AppKey"] == "test-app-key"
+ assert headers["Content-Type"] == "application/x-protobuf"
+ resource = request.resource_metrics[0]
+ assert {item.key: item.value.string_value for item in resource.resource.attributes} == {
+ "service.name": "metrics-contract"
+ }
+ metrics = {metric.name: metric for metric in resource.scope_metrics[0].metrics}
+ assert metrics["openviking_operation_requests_total"].sum.data_points[0].as_double == 2
+ point = metrics["openviking_operation_duration_seconds"].histogram.data_points[0]
+ assert point.count == 1 and point.sum == pytest.approx(60.123)
+ assert point.bucket_counts[-1] == 0
+ finally:
+ release.set()
+ if stopping is not None:
+ await stopping
+ await exporter.shutdown()
+ await asyncio.to_thread(server.shutdown)
+ server.server_close()
+ thread.join(timeout=2)
diff --git a/tests/metrics/core/test_refresh.py b/tests/metrics/core/test_refresh.py
index 077f0a456..25fdcf4e1 100644
--- a/tests/metrics/core/test_refresh.py
+++ b/tests/metrics/core/test_refresh.py
@@ -7,10 +7,20 @@ import pytest
from openviking.metrics.collectors.base import CollectorConfig, Refreshable
from openviking.metrics.collectors.manager import CollectorManager, RefreshResult
+from openviking.metrics.core.base import MetricCollector
from openviking.metrics.core.refresh import RefreshGate
from openviking.metrics.core.registry import MetricRegistry
+class _RefreshableCollector(MetricCollector, Refreshable):
+ @classmethod
+ def kind(cls):
+ return "state"
+
+ def collector_name(self):
+ return "dummy"
+
+
def test_refresh_gate_ttl():
gate = RefreshGate(ttl_seconds=1.0)
d0 = gate.decide(now=0.0)
@@ -42,7 +52,7 @@ async def test_collector_manager_requires_refreshable():
def collect(self, registry) -> None:
self.calls += 1
- registry.counter("openviking_dummy_refresh_total").inc()
+ registry.inc_counter("openviking_dummy_refresh_total")
mgr = CollectorManager()
c = DummyCollector()
@@ -53,8 +63,7 @@ async def test_collector_manager_requires_refreshable():
@pytest.mark.asyncio
async def test_collector_manager_ttl_and_timeout():
- class DummyCollector(Refreshable):
- name = "dummy"
+ class DummyCollector(_RefreshableCollector):
config = CollectorConfig(ttl_seconds=10.0, timeout_seconds=0.01)
def __init__(self) -> None:
@@ -62,7 +71,7 @@ async def test_collector_manager_ttl_and_timeout():
def collect(self, registry) -> None:
self.calls += 1
- registry.counter("openviking_dummy_refresh_total").inc()
+ registry.inc_counter("openviking_dummy_refresh_total")
registry = MetricRegistry()
mgr = CollectorManager()
@@ -82,8 +91,7 @@ async def test_collector_manager_ttl_and_timeout():
@pytest.mark.asyncio
async def test_collector_manager_swr_when_ttl_expired_and_has_last_success():
- class DummyCollector(Refreshable):
- name = "dummy"
+ class DummyCollector(_RefreshableCollector):
config = CollectorConfig(ttl_seconds=1.0, timeout_seconds=0.5)
def __init__(self) -> None:
@@ -113,8 +121,7 @@ async def test_collector_manager_swr_when_ttl_expired_and_has_last_success():
@pytest.mark.asyncio
async def test_collector_manager_deadline_exceeded_does_not_stick_inflight(monkeypatch):
- class DummyCollector(Refreshable):
- name = "dummy"
+ class DummyCollector(_RefreshableCollector):
config = CollectorConfig(ttl_seconds=10.0, timeout_seconds=10.0)
def collect(self, registry) -> None:
diff --git a/tests/metrics/core/test_registry.py b/tests/metrics/core/test_registry.py
index 3ba9304e1..2044be617 100644
--- a/tests/metrics/core/test_registry.py
+++ b/tests/metrics/core/test_registry.py
@@ -1,93 +1,77 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-from pathlib import Path
-
import pytest
+from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics
+from prometheus_client.parser import text_string_to_metric_families
-from openviking.metrics.core.base import MetricCollector
-from openviking.metrics.core.registry import MetricRegistry
-from openviking.metrics.exporters.otel import OTelMetricExporter
-
-
-def test_metric_collector_metric_name_includes_namespace_and_optional_unit():
- assert (
- MetricCollector.metric_name("cache", "hits", unit="total") == "openviking_cache_hits_total"
- )
- assert (
- MetricCollector.metric_name("resource", "stage_duration", unit="seconds")
- == "openviking_resource_stage_duration_seconds"
- )
- assert MetricCollector.metric_name("lock", "active") == "openviking_lock_active"
+from openviking.metrics.core.registry import DEFAULT_LATENCY_BUCKETS, MetricRegistry
def test_registry_rejects_type_conflict():
registry = MetricRegistry()
- registry.counter("openviking_conflict_total")
+ registry.inc_counter("calls_total")
+ with pytest.raises(ValueError):
+ registry.set_gauge("calls_total", 1)
with pytest.raises(ValueError):
- registry.gauge("openviking_conflict_total")
+ registry.set_counter("calls_total", 1)
def test_registry_label_keys_must_match_definition():
registry = MetricRegistry()
- c = registry.counter("openviking_labeled_total", label_names=("a", "b"))
- with pytest.raises(ValueError):
- c.inc(labels={"a": "1"})
- with pytest.raises(ValueError):
- c.inc(labels={"a": "1", "b": "2", "c": "3"})
+ for labels in ({"a": "1"}, {"a": "1", "b": "2", "c": "3"}):
+ with pytest.raises(ValueError):
+ registry.inc_counter("calls_total", labels=labels, label_names=("a", "b"))
+ with pytest.raises(ValueError, match="bucket_schema is reserved"):
+ registry.observe_histogram(
+ "latency", 1, labels={"bucket_schema": "custom"}, label_names=("bucket_schema",)
+ )
def test_registry_canonicalizes_label_name_order_for_same_family(render_prometheus):
registry = MetricRegistry()
-
- registry.inc_counter(
- "openviking_ordered_total",
- labels={"status": "ok", "provider": "local"},
- label_names=("status", "provider"),
- )
- registry.inc_counter(
- "openviking_ordered_total",
- labels={"provider": "local", "status": "ok"},
- label_names=("provider", "status"),
- )
-
- text = render_prometheus(registry)
- assert 'openviking_ordered_total{provider="local",status="ok"} 2' in text
+ for names in (("status", "provider"), ("provider", "status")):
+ registry.inc_counter(
+ "calls_total", labels={"status": "ok", "provider": "local"}, label_names=names
+ )
+ assert 'calls_total{provider="local",status="ok"} 2.0' in render_prometheus(registry)
def test_counter_only_increases():
registry = MetricRegistry()
- c = registry.counter("openviking_counter_total")
- c.inc()
- with pytest.raises(ValueError):
- c.inc(amount=0)
- with pytest.raises(ValueError):
- c.inc(amount=-1)
+ registry.inc_counter("calls_total")
+ for amount in (0, -1, float("inf")):
+ with pytest.raises(ValueError):
+ registry.inc_counter("calls_total", amount=amount)
def test_histogram_boundary_bucket(registry, render_prometheus):
- h = registry.histogram("openviking_latency_seconds", buckets=(0.05, 0.1))
- h.observe(0.05)
- text = render_prometheus(registry)
- assert 'openviking_latency_seconds_bucket{le="0.05"} 1' in text
- assert 'openviking_latency_seconds_bucket{le="0.1"} 1' in text
- assert 'openviking_latency_seconds_bucket{le="+Inf"} 1' in text
-
-
-def test_collectors_do_not_embed_openviking_metric_name_literals():
- root = Path(__file__).resolve().parents[3] / "openviking" / "metrics" / "collectors"
- offenders = []
- for path in root.glob("*.py"):
- text = path.read_text(encoding="utf-8")
- if '"openviking_' in text or "'openviking_" in text:
- offenders.append(path.name)
- assert offenders == []
-
-
-def test_registry_absolute_values_and_deletion():
- """Use real families with fixed inputs to verify replacement and deletion; return None."""
+ durations = (0.05, 20.0, 60.0, 60.001, 120.0, 3600.0, 404261.58497)
+ for duration in durations:
+ registry.observe_histogram("latency_seconds", duration)
+ family = next(text_string_to_metric_families(render_prometheus(registry)))
+ buckets = {float(s.labels["le"]): s.value for s in family.samples if s.name.endswith("_bucket")}
+ for bound in DEFAULT_LATENCY_BUCKETS:
+ assert buckets[bound] == sum(value <= bound for value in durations)
+ assert buckets[float("inf")] == len(durations)
+ schema = family.samples[0].labels["bucket_schema"]
+ request = encode_metrics(registry.collect())
+ point = request.resource_metrics[0].scope_metrics[0].metrics[0].histogram.data_points[0]
+ assert tuple(point.explicit_bounds) == DEFAULT_LATENCY_BUCKETS
+ assert point.count == sum(point.bucket_counts) == len(durations)
+ assert point.bucket_counts[-1] == 0
+ assert point.sum == pytest.approx(sum(durations))
+ assert {a.key: a.value.string_value for a in point.attributes} == {"bucket_schema": schema}
+ legacy = MetricRegistry()
+ legacy.observe_histogram("latency_seconds", 60, buckets=(0.05, 0.1, 10.0))
+ legacy_family = next(text_string_to_metric_families(render_prometheus(legacy)))
+ assert legacy_family.samples[0].labels["bucket_schema"] != schema
+
+
+def test_registry_absolute_values_and_deletion(render_prometheus):
registry = MetricRegistry(max_series_per_metric=1)
labels = {"plugin": "memfs"}
+ started = None
for value in (10, 12, 3, 0):
registry.set_counter("calls_total", value, labels=labels, label_names=("plugin",))
registry.set_histogram(
@@ -99,200 +83,64 @@ def test_registry_absolute_values_and_deletion():
labels=labels,
label_names=("plugin",),
)
- assert dict(registry.iter_counters())["calls_total"] == [(tuple(labels.items()), value)]
- assert list(registry.iter_histograms())[0][3] == [
- (tuple(labels.items()), (value, 0), value, value * 0.001)
- ]
- registry.inc_counter("calls_total", labels=labels, label_names=("plugin",))
- registry.observe_histogram(
- "latency_seconds",
- 0.002,
- labels=labels,
- label_names=("plugin",),
- buckets=(0.001,),
- )
- assert list(registry.iter_histograms())[0][3][0][1:] == ((0, 1), 1, 0.002)
+ request = encode_metrics(registry.collect())
+ metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}
+ counter = metrics["calls_total"].sum.data_points[0]
+ histogram = metrics["latency_seconds"].histogram.data_points[0]
+ assert counter.as_double == histogram.count == value
+ assert histogram.sum == pytest.approx(value * 0.001)
+ assert not histogram.HasField("min") and not histogram.HasField("max")
+ if value == 10:
+ started = counter.start_time_unix_nano
+ elif value == 12:
+ assert counter.start_time_unix_nano == started
+ else:
+ assert counter.start_time_unix_nano > started
+ assert histogram.start_time_unix_nano <= histogram.time_unix_nano
registry.set_counter("calls_total", 5, labels={"plugin": "other"}, label_names=("plugin",))
- assert dict(registry.iter_counters())["calls_total"][0][1] == 1
- assert dict(registry.iter_dropped_series()) == {"calls_total": 1}
- for name, method in (
- ("calls_total", registry.counter_delete_matching),
- ("latency_seconds", registry.histogram_delete_matching),
- ):
- method(name, match_labels={"plugin": "other"})
- method(name, match_labels=labels)
- method(name, match_labels=labels)
- assert list(registry.iter_counters()) == []
- assert list(registry.iter_histograms()) == []
+ text = render_prometheus(registry)
+ assert 'plugin="other"' not in text
+ assert 'openviking_metrics_dropped_series_total{metric="calls_total"} 1.0' in text
+ registry.delete_matching("calls_total", match_labels=labels)
+ registry.delete_matching("latency_seconds", match_labels=labels)
+ assert not any(
+ line.startswith(("calls_total", "latency_seconds"))
+ for line in render_prometheus(registry).splitlines()
+ )
@pytest.mark.parametrize("value", [-1, float("inf"), float("nan")])
-def test_registry_rejects_invalid_absolute_counter(value):
- """Reject invalid counter input without replacing the previous value; return None."""
+def test_registry_rejects_invalid_absolute_counter(value, render_prometheus):
registry = MetricRegistry()
registry.set_counter("calls_total", 4)
with pytest.raises(ValueError):
registry.set_counter("calls_total", value)
- assert dict(registry.iter_counters())["calls_total"] == [((), 4)]
- empty = MetricRegistry()
- with pytest.raises(ValueError):
- empty.set_counter("calls_total", value)
- assert list(empty.iter_counters()) == []
-
-
- "changes",
- [
- {"bucket_bounds": (1.0, 0.0)},
- {"bucket_bounds": (float("inf"),)},
- {"bucket_counts": (1,)},
- {"bucket_counts": (-1, 2)},
- {"count": 2},
- {"count": True},
- {"bucket_counts": (True, 0)},
- {"value_sum": float("nan")},
- ],
-)
-def test_registry_rejects_invalid_absolute_histogram(changes):
- """Reject malformed histogram input without corrupting an existing series; return None."""
- registry = MetricRegistry()
- values = {
- "bucket_bounds": (0.001,),
- "bucket_counts": (1, 0),
- "count": 1,
- "value_sum": 0.001,
- }
- registry.set_histogram("latency_seconds", **values)
- before = list(registry.iter_histograms())
- with pytest.raises(ValueError):
- registry.set_histogram("latency_seconds", **(values | changes))
- assert list(registry.iter_histograms()) == before
- empty = MetricRegistry()
- with pytest.raises(ValueError):
- empty.set_histogram("latency_seconds", **(values | changes))
- assert list(empty.iter_histograms()) == []
- empty.set_histogram("latency_seconds", **values)
- assert list(empty.iter_histograms()) == before
-
-
-def test_registry_deletion_hides_empty_families_but_keeps_contract(render_prometheus):
- """Delete all series for each type and verify export absence and schema retention."""
- registry = MetricRegistry()
- registry.counter("untouched_total")
- registry.gauge("untouched_gauge")
- registry.histogram("untouched_histogram")
- registry.counter_delete_matching("untouched_total", match_labels={})
- registry.gauge_delete_matching("untouched_gauge", match_labels={})
- registry.histogram_delete_matching("untouched_histogram", match_labels={})
- assert "untouched_total 0" in render_prometheus(registry)
- registry.set_counter("removed_total", 1)
- registry.set_gauge("removed_gauge", 1)
- registry.set_histogram(
- "removed_histogram",
- bucket_bounds=[1],
- bucket_counts=[1, 0],
- count=1,
- value_sum=1,
- )
- for kind, name in (
- ("counter", "removed_total"),
- ("gauge", "removed_gauge"),
- ("histogram", "removed_histogram"),
- ):
- getattr(registry, f"{kind}_delete_matching")(name, match_labels={})
- text = render_prometheus(registry)
- assert "removed_" not in text
- assert "untouched_total 0" in text and "untouched_gauge 0" in text
- assert "untouched_histogram_count 0" in text
- exporter = OTelMetricExporter(registry=registry, enabled=False)
- request = exporter._build_export_request()
- metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}
- assert set(metrics) == {"untouched_total", "untouched_gauge", "untouched_histogram"}
- assert metrics["untouched_total"].sum.data_points[0].as_int == 0
- assert metrics["untouched_gauge"].gauge.data_points[0].as_int == 0
- assert metrics["untouched_histogram"].histogram.data_points[0].count == 0
- with pytest.raises(ValueError):
- registry.gauge("removed_total")
- with pytest.raises(ValueError):
- registry.histogram("removed_histogram", buckets=[2])
- with pytest.raises(ValueError):
- registry.counter("removed_total").set(-1)
- with pytest.raises(ValueError):
- registry.histogram("removed_histogram", buckets=[1]).set((1,), 1, 1)
- assert "removed_" not in render_prometheus(registry)
- registry.set_counter("removed_total", 2)
- registry.gauge("removed_gauge").inc(2)
- registry.histogram("removed_histogram", buckets=[1]).observe(0.5)
- text = render_prometheus(registry)
- assert "removed_total 2" in text
- assert "removed_gauge 2.0" in text
- assert "removed_histogram_count 1" in text
- request = exporter._build_export_request()
- metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}
- assert metrics["removed_total"].sum.data_points[0].as_int == 2
- assert metrics["removed_gauge"].gauge.data_points[0].as_int == 2
- assert metrics["removed_histogram"].histogram.data_points[0].count == 1
+ assert "calls_total 4.0" in render_prometheus(registry)
@pytest.mark.parametrize("kind", ("counter", "gauge", "histogram"))
def test_registry_partial_deletion_keeps_active_series(kind, render_prometheus):
- """Delete selected labels for kind; retain active values and the label contract."""
registry = MetricRegistry()
- options = {"buckets": [1]} if kind == "histogram" else {}
- handle = getattr(registry, kind)("active", label_names=("plugin",), **options)
+ name = "active_total" if kind == "counter" else "active"
for plugin in ("removed", "kept"):
+ labels = {"plugin": plugin}
if kind == "histogram":
- handle.observe(0.5, labels={"plugin": plugin})
+ registry.set_histogram(
+ name,
+ bucket_bounds=[1],
+ bucket_counts=[1, 0],
+ count=1,
+ value_sum=0.5,
+ labels=labels,
+ label_names=("plugin",),
+ )
else:
- handle.inc(labels={"plugin": plugin})
- delete = getattr(registry, f"{kind}_delete_matching")
- delete("active", match_labels={"plugin": "removed"})
+ getattr(registry, f"set_{kind}")(name, 1, labels=labels, label_names=("plugin",))
+ delete = registry.delete_matching
+ delete(name, match_labels={"plugin": "removed"})
text = render_prometheus(registry)
- assert 'plugin="removed"' not in text
- sample = "active_count" if kind == "histogram" else "active"
- assert f'{sample}{{plugin="kept"}} 1' in text
- delete("active", match_labels={"plugin": "kept"})
- assert "active" not in render_prometheus(registry)
+ assert 'plugin="removed"' not in text and 'plugin="kept"' in text
+ delete(name, match_labels={"plugin": "kept"})
+ assert name not in render_prometheus(registry)
with pytest.raises(ValueError):
- getattr(registry, kind)("active", label_names=("other",), **options)
- if kind == "histogram":
- handle.set((0, 0), 0, 0, labels={"plugin": "kept"})
- else:
- handle.set(0, labels={"plugin": "kept"})
- assert f'{sample}{{plugin="kept"}} 0' in render_prometheus(registry)
-
-
-def test_registry_handles_validate_absolute_values(render_prometheus):
- """Call public handles with integer and invalid inputs and verify validated storage."""
- registry = MetricRegistry()
- counter = registry.counter("calls_total")
- counter.set(1)
- assert "calls_total 1" in render_prometheus(registry)
- for value in (-1, float("inf"), float("-inf"), float("nan")):
- with pytest.raises(ValueError):
- counter.set(value)
- assert dict(registry.iter_counters())["calls_total"] == [((), 1)]
- histogram = registry.histogram("latency", buckets=[1])
- histogram.set([1, 0], 1, 1)
- assert "latency_sum 1.0" in render_prometheus(registry)
- before = list(registry.iter_histograms())
- for counts, count, value_sum in (
- ((1,), 1, 1),
- ((-1, 2), 1, 1),
- ((True, 0), 1, 1),
- ((1.0, 0), 1, 1),
- ((1, 0), True, 1),
- ((1, 0), 2, 1),
- ((1, 0), 1, -1),
- ((1, 0), 1, float("inf")),
- ((1, 0), 1, float("nan")),
- ):
- with pytest.raises(ValueError):
- histogram.set(counts, count, value_sum)
- assert list(registry.iter_histograms()) == before
- for bounds in ((-1,), (float("inf"),), (float("nan"),), (1, 1), (2, 1)):
- invalid = MetricRegistry()
- handle = invalid.histogram("invalid", buckets=bounds)
- with pytest.raises(ValueError):
- handle.set((0,) * (len(bounds) + 1), 0, 0)
- assert list(invalid.iter_histograms())[0][3] == []
+ registry.set_gauge(name, 2, labels={"other": "value"}, label_names=("other",))
diff --git a/tests/metrics/datasources/test_state_datasources.py b/tests/metrics/datasources/test_state_datasources.py
index 76f133bdf..58e6e33c3 100644
--- a/tests/metrics/datasources/test_state_datasources.py
+++ b/tests/metrics/datasources/test_state_datasources.py
@@ -293,6 +293,7 @@ def test_ragfs_datasource_validates_records(changes):
{},
{"bucket_counts": [1]},
{"bucket_counts": [True, 0]},
+ {"labels": {"bucket_schema": "caller-supplied"}},
{"count": 3},
{"bucket_bounds": [2, 1]},
{"sum": -1},
diff --git a/tests/metrics/integration/test_bootstrap.py b/tests/metrics/integration/test_bootstrap.py
index 98b2e3d1c..1c8ffddd1 100644
--- a/tests/metrics/integration/test_bootstrap.py
+++ b/tests/metrics/integration/test_bootstrap.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import re
from pathlib import Path
import pytest
@@ -168,17 +169,19 @@ def test_resource_ingestion_event_datasource_can_drive_resource_ingestion_collec
'openviking_resource_stage_total{account_id="__unknown__",stage="parse",status="error"} 1'
in text
)
- assert (
- 'openviking_resource_wait_duration_seconds_count{account_id="__unknown__",operation="queue_processing"} 1'
- in text
+ assert re.search(
+ r'openviking_resource_wait_duration_seconds_count\{account_id="__unknown__",'
+ r'bucket_schema="[0-9a-f]{16}",operation="queue_processing"\} 1',
+ text,
)
assert (
'openviking_resource_stage_total{account_id="acct-resource",stage="parse",status="ok"} 1'
in text
)
- assert (
- 'openviking_resource_wait_duration_seconds_count{account_id="acct-resource",operation="queue_processing"} 1'
- in text
+ assert re.search(
+ r'openviking_resource_wait_duration_seconds_count\{account_id="acct-resource",'
+ r'bucket_schema="[0-9a-f]{16}",operation="queue_processing"\} 1',
+ text,
)
reset_metric_account_dimension()
diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.py
index 29118905f..898744c4e 100644
--- a/tests/metrics/integration/test_telemetry_bridge.py
+++ b/tests/metrics/integration/test_telemetry_bridge.py
@@ -1,6 +1,8 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+import re
+
from openviking.metrics.datasources.telemetry_bridge import TelemetryBridgeEventDataSource
from openviking.metrics.global_api import init_metrics_from_server_config, shutdown_metrics
from openviking.observability.events import try_publish_event
@@ -24,7 +26,8 @@ def test_queue_duration_event_is_wired_to_metrics_registry(registry, render_prom
},
)
text = render_prometheus(registry)
- labels = 'outcome="success",queue="SessionCommit"'
+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)
+ labels = f'bucket_schema="{schema}",outcome="success",queue="SessionCommit"'
assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text
assert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text
finally:
@@ -87,9 +90,10 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende
'openviking_operation_requests_total{account_id="__unknown__",operation="resource.process",status="ok"} 1'
in text
)
- assert (
- 'openviking_operation_duration_seconds_count{account_id="__unknown__",operation="resource.process",status="ok"} 1'
- in text
+ assert re.search(
+ r'openviking_operation_duration_seconds_count\{account_id="__unknown__",'
+ r'bucket_schema="[0-9a-f]{16}",operation="resource.process",status="ok"\} 1',
+ text,
)
assert (
'openviking_operation_tokens_total{account_id="__unknown__",operation="resource.process",stage="rerank",token_type="rerank"} 4'
@@ -132,9 +136,10 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende
'openviking_resource_stage_total{account_id="__unknown__",stage="parse",status="warning"} 1'
in text
)
- assert (
- 'openviking_resource_wait_duration_seconds_count{account_id="__unknown__",operation="resource.process"} 1'
- in text
+ assert re.search(
+ r'openviking_resource_wait_duration_seconds_count\{account_id="__unknown__",'
+ r'bucket_schema="[0-9a-f]{16}",operation="resource.process"\} 1',
+ text,
)
finally:
shutdown_metrics(app=None)
diff --git a/tests/server/test_prometheus_metrics.py b/tests/server/test_prometheus_metrics.py
index 77ed66598..e4eb66f3d 100644
--- a/tests/server/test_prometheus_metrics.py
+++ b/tests/server/test_prometheus_metrics.py
@@ -4,6 +4,7 @@
"""Tests for the Prometheus metrics endpoint and exposition output."""
import json
+import re
from types import SimpleNamespace
import httpx
@@ -33,39 +34,45 @@ class TestPrometheusExposition:
def test_counter_histogram_and_labels(self):
registry = MetricRegistry()
labels = {"context_type": "memory"}
- registry.counter(
- "openviking_retrieval_requests_total",
- label_names=("context_type",),
- ).inc(labels=labels)
- registry.counter(
- "openviking_retrieval_requests_total",
- label_names=("context_type",),
- ).inc(labels=labels)
- registry.histogram(
+ registry.inc_counter(
+ "openviking_retrieval_requests_total", label_names=("context_type",), labels=labels
+ )
+ registry.inc_counter(
+ "openviking_retrieval_requests_total", label_names=("context_type",), labels=labels
+ )
+ registry.observe_histogram(
"openviking_retrieval_latency_seconds",
+ 0.02,
label_names=("context_type",),
- ).observe(0.02, labels=labels)
+ labels=labels,
+ )
- registry.counter("openviking_cache_hits_total", label_names=("level",)).inc(
- labels={"level": "L0"}
+ registry.inc_counter(
+ "openviking_cache_hits_total", label_names=("level",), labels={"level": "L0"}
)
- registry.counter("openviking_cache_hits_total", label_names=("level",)).inc(
- labels={"level": "L0"}
+ registry.inc_counter(
+ "openviking_cache_hits_total", label_names=("level",), labels={"level": "L0"}
)
- registry.counter("openviking_cache_misses_total", label_names=("level",)).inc(
- labels={"level": "L1"}
+ registry.inc_counter(
+ "openviking_cache_misses_total", label_names=("level",), labels={"level": "L1"}
)
exporter = PrometheusExporter(registry=registry)
text = exporter.render()
+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)
assert 'openviking_retrieval_requests_total{context_type="memory"} 2' in text
- assert 'openviking_retrieval_latency_seconds_count{context_type="memory"} 1' in text
assert (
- 'openviking_retrieval_latency_seconds_bucket{context_type="memory",le="0.05"} 1' in text
+ f'openviking_retrieval_latency_seconds_count{{bucket_schema="{schema}",context_type="memory"}} 1'
+ in text
)
assert (
- 'openviking_retrieval_latency_seconds_bucket{context_type="memory",le="+Inf"} 1' in text
+ f'openviking_retrieval_latency_seconds_bucket{{bucket_schema="{schema}",context_type="memory",le="0.05"}} 1'
+ in text
+ )
+ assert (
+ f'openviking_retrieval_latency_seconds_bucket{{bucket_schema="{schema}",context_type="memory",le="+Inf"}} 1'
+ in text
)
assert 'openviking_cache_hits_total{level="L0"} 2' in text
assert 'openviking_cache_misses_total{level="L1"} 1' in text
@@ -109,9 +116,10 @@ class TestRetrievalStatsMetricsIntegration:
'openviking_retrieval_results_total{account_id="__unknown__",context_type="memory"} 3'
in text
)
- assert (
- 'openviking_retrieval_latency_seconds_count{account_id="__unknown__",context_type="memory"} 1'
- in text
+ assert re.search(
+ r'openviking_retrieval_latency_seconds_count\{account_id="__unknown__",'
+ r'bucket_schema="[0-9a-f]{16}",context_type="memory"\} 1',
+ text,
)
finally:
shutdown_metrics(app=None)
@@ -172,10 +180,11 @@ class TestMetricsEndpoint:
transport = httpx.ASGITransport(app=app)
try:
registry = get_metrics_registry()
- registry.counter(
+ registry.inc_counter(
"openviking_retrieval_requests_total",
label_names=("context_type",),
- ).inc(labels={"context_type": "memory"})
+ labels={"context_type": "memory"},
+ )
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get("/metrics")
assert resp.status_code == 200
diff --git a/uv.lock b/uv.lock
index 981f11393..3bc66e331 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3548,6 +3548,7 @@ dependencies = [
{ name = "pathspec" },
{ name = "pdfminer-six" },
{ name = "pdfplumber" },
+ { name = "prometheus-client" },
{ name = "protobuf" },
{ name = "pydantic" },
{ name = "python-multipart" },
@@ -3770,11 +3771,11 @@ requires-dist = [
{ name = "opencode-ai", marker = "extra == 'bot'", specifier = ">=0.1.0a0" },
{ name = "opensandbox", marker = "extra == 'bot'", specifier = ">=0.1.5" },
{ name = "opensandbox-server", marker = "extra == 'bot'", specifier = ">=0.1.6" },
- { name = "opentelemetry-api", specifier = ">=1.14" },
- { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.14" },
- { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.14" },
+ { name = "opentelemetry-api", specifier = ">=1.40,<2" },
+ { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.40,<2" },
+ { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.40,<2" },
{ name = "opentelemetry-instrumentation-asyncio", specifier = ">=0.61b0" },
- { name = "opentelemetry-sdk", specifier = ">=1.14" },
+ { name = "opentelemetry-sdk", specifier = ">=1.40,<2" },
{ name = "openviking-sdk", specifier = ">=0.1.17" },
{ name = "orjson", specifier = ">=3.11.7" },
{ name = "packaging", specifier = ">=24.0" },
@@ -3784,6 +3785,7 @@ requires-dist = [
{ name = "pathspec", specifier = ">=1.1.1" },
{ name = "pdfminer-six", specifier = ">=20251230" },
{ name = "pdfplumber", specifier = ">=0.10.0" },
+ { name = "prometheus-client", specifier = ">=0.24,<1" },
{ name = "prompt-toolkit", marker = "extra == 'bot'", specifier = ">=3.0.0" },
{ name = "protobuf", specifier = ">=6.33.5" },
{ name = "psycopg2-binary", marker = "extra == 'opengauss'", specifier = ">=2.9,<3" },
@@ -4355,6 +4357,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9c/92/e18be996a01c7fd0e7dd7d198edefe42813cdfe1637bbbc80370ce656f62/primp-1.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:efadef0dfd10e733a254a949abf9ed05c668c28a68aa6513d811c0c6acd54cdb", size = 3611571, upload-time = "2026-03-11T06:43:31.249Z" },
]
+[[package]]
+name = "prometheus-client"
+version = "0.26.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" },
+]
+
[[package]]
name = "prompt-toolkit"
version = "3.0.52"