Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(metrics): 复用标准 SDK 并修复长耗时统计 (#6474)
Qin Haojie07b5181d61c1
diff --git a/Cargo.lock b/Cargo.lockindex 585b8b6e4..bde98e932 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3564,6 +3564,29 @@ dependencies = ["vcpkg",]+[[package]]+name = "opentelemetry"+version = "0.33.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee"+dependencies = [+ "js-sys",+]++[[package]]+name = "opentelemetry_sdk"+version = "0.33.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061"+dependencies = [+ "futures-channel",+ "futures-executor",+ "futures-util",+ "opentelemetry",+ "portable-atomic",+ "thiserror 2.0.18",+]+[[package]]name = "option-ext"version = "0.2.0"@@ -4094,6 +4117,8 @@ dependencies = ["lru","mime_guess","native-tls",+ "opentelemetry",+ "opentelemetry_sdk","path-clean","radix_trie","rand 0.8.5",diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.tomlindex c39fb87d7..32b8f7129 100644--- a/crates/ragfs/Cargo.toml+++ b/crates/ragfs/Cargo.toml@@ -45,6 +45,10 @@ serde_json = "1.0"# Loggingtracing = "0.1"+# Metrics aggregation; the binding reads cumulative SDK snapshots on demand.+opentelemetry = { version = "0.33", default-features = false, features = ["metrics"] }+opentelemetry_sdk = { version = "0.33", default-features = false, features = ["metrics", "experimental_metrics_custom_reader"] }+# Git (gitoxide)gix-hash = "0.14"gix-object = "0.42"diff --git a/crates/ragfs/src/core/stats.rs b/crates/ragfs/src/core/stats.rsindex 6907babee..b0eedfe80 100644--- a/crates/ragfs/src/core/stats.rs+++ b/crates/ragfs/src/core/stats.rs@@ -3,12 +3,15 @@//! This module provides functionality to track filesystem operations,//! including operation counts and latency statistics.-use serde::{Deserialize, Serialize};use std::collections::HashMap;use std::sync::Arc;use std::time::{Duration, Instant};+use opentelemetry::KeyValue;+use serde::{Deserialize, Serialize};use tokio::sync::RwLock;+use crate::metrics::duration::DurationMetrics;+/// Type of filesystem operation#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]pub enum FsOperation {@@ -92,12 +95,6 @@ impl FsOperation {}}-/// Inclusive upper bounds for operation duration buckets, in nanoseconds.-pub const DURATION_BUCKET_BOUNDS_NS: [u64; 11] = [- 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000,- 100_000_000, 500_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000,-];-/// Statistics for a single operation type#[derive(Debug, Clone, Serialize, Deserialize)]pub struct OperationStats {@@ -113,8 +110,10 @@ pub struct OperationStats {pub min_time_ns: u64,/// Maximum time spent (nanoseconds).pub max_time_ns: u64,+ /// Inclusive duration boundaries returned by the SDK, in nanoseconds.+ pub duration_bucket_bounds_ns: Vec<u64>,/// Noncumulative duration counts; the last bucket holds overflow samples.- pub duration_bucket_counts: [u64; 12],+ pub duration_bucket_counts: Vec<u64>,}impl Default for OperationStats {@@ -127,26 +126,13 @@ impl Default for OperationStats {total_time_ns: 0,min_time_ns: u64::MAX,max_time_ns: 0,- duration_bucket_counts: [0; 12],+ duration_bucket_bounds_ns: Vec::new(),+ duration_bucket_counts: Vec::new(),}}}impl OperationStats {- /// Record the supplied duration and success flag, updating totals in place.- pub fn record(&mut self, duration: Duration, success: bool) {- let ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);- self.count = self.count.saturating_add(1);- let outcome = if success { &mut self.success_count } else { &mut self.error_count };- *outcome = outcome.saturating_add(1);- self.total_time_ns = self.total_time_ns.saturating_add(ns);- self.min_time_ns = self.min_time_ns.min(ns);- self.max_time_ns = self.max_time_ns.max(ns);- let bucket = DURATION_BUCKET_BOUNDS_NS.partition_point(|bound| *bound < ns);- self.duration_bucket_counts[bucket] =- self.duration_bucket_counts[bucket].saturating_add(1);- }-/// Return this operation's average nanoseconds, or zero when no samples exist.pub fn avg_time_ns(&self) -> f64 {if self.count == 0 {@@ -185,15 +171,11 @@ impl FilesystemStats {self.operations.values().map(|s| s.count).sum()}- /// Reset all statistics- pub fn reset(&mut self) {- *self = Self::default();- }}/// Thread-safe statistics collectorpub struct StatsCollector {- stats: Arc<RwLock<FilesystemStats>>,+ durations: RwLock<DurationMetrics>,}impl Default for StatsCollector {@@ -206,25 +188,65 @@ impl StatsCollector {/// Create a new statistics collectorpub fn new() -> Self {Self {- stats: Arc::new(RwLock::new(FilesystemStats::default())),+ durations: RwLock::new(DurationMetrics::default()),}}/// Record the supplied operation, duration, and success flag; return unit.pub async fn record(&self, op: FsOperation, duration: Duration, success: bool) {- let mut stats = self.stats.write().await;- stats.operations.entry(op).or_default().record(duration, success);+ self.durations.read().await.record(+ duration,+ &[+ KeyValue::new("operation", op.as_str()),+ KeyValue::new("status", if success { "success" } else { "error" }),+ ],+ );}/// Get a snapshot of current statisticspub async fn snapshot(&self) -> FilesystemStats {- self.stats.read().await.clone()+ let mut snapshot = FilesystemStats::default();+ for point in self.durations.read().await.snapshot() {+ let operation = point+ .attributes()+ .find(|attribute| attribute.key.as_str() == "operation")+ .expect("filesystem duration has an operation attribute");+ let operation = FsOperation::all()+ .iter()+ .find(|op| op.as_str() == operation.value.as_str())+ .expect("filesystem duration uses a known operation");+ let success = point.attributes().any(|attribute| {+ attribute.key.as_str() == "status" && attribute.value.as_str() == "success"+ });+ let stats = snapshot.operations.get_mut(operation).unwrap();+ stats.count = stats.count.saturating_add(point.count());+ if success {+ stats.success_count = point.count();+ } else {+ stats.error_count = point.count();+ }+ stats.total_time_ns = stats.total_time_ns.saturating_add(point.sum());+ stats.min_time_ns = stats.min_time_ns.min(point.min().unwrap_or(u64::MAX));+ stats.max_time_ns = stats.max_time_ns.max(point.max().unwrap_or(0));+ if stats.duration_bucket_counts.is_empty() {+ stats.duration_bucket_bounds_ns = point.bounds().map(|bound| bound as u64).collect();+ stats.duration_bucket_counts = point.bucket_counts().collect();+ } else {+ for (count, incoming) in stats+ .duration_bucket_counts+ .iter_mut()+ .zip(point.bucket_counts())+ {+ *count = count.saturating_add(incoming);+ }+ }+ }+ snapshot}/// Reset all statisticspub async fn reset(&self) {- let mut stats = self.stats.write().await;- stats.reset();+ *self.durations.write().await = DurationMetrics::default();}}diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rsindex d7fcbdb19..ece5885c4 100644--- a/crates/ragfs/src/lock/manager.rs+++ b/crates/ragfs/src/lock/manager.rs@@ -1293,11 +1293,8 @@ impl PathLockManager {let descendants = self.provider.scan_descendant_locks(path).await?;let now_ns = Self::now_ns();- let mut metrics = self.metrics.write().await;- metrics.descendant_scan_count += 1;- metrics.descendant_scan_duration_ns = metrics- .descendant_scan_duration_ns- .saturating_add(u64::try_from(scan_start.elapsed().as_nanos()).unwrap_or(u64::MAX));+ let metrics = self.metrics.read().await;+ metrics.descendant_scan.record(scan_start.elapsed(), &[]);drop(metrics);debug!(path = %path, descendant_count = descendants.len(), scan_ms = scan_start.elapsed().as_millis() as u64, "scanned descendant pathlock tokens");diff --git a/crates/ragfs/src/lock/metrics.rs b/crates/ragfs/src/lock/metrics.rsindex ca01d4ec3..b768e6436 100644--- a/crates/ragfs/src/lock/metrics.rs+++ b/crates/ragfs/src/lock/metrics.rs@@ -1,6 +1,7 @@//! Lock observability metrics.use crate::lock::types::PathLockConflict;+use crate::metrics::duration::DurationMetrics;/// Max number of recent conflicts retained for observability.const MAX_RECENT_CONFLICTS: usize = 32;@@ -18,10 +19,8 @@ pub struct LockMetrics {pub stale_leases_released: usize,/// Cumulative count of stale cleanup failures.pub stale_cleanup_failures: usize,- /// Cumulative count of descendant scan operations.- pub descendant_scan_count: usize,- /// Total nanoseconds spent in descendant scans.- pub descendant_scan_duration_ns: u64,+ /// SDK-owned descendant scan duration distribution.+ pub(crate) descendant_scan: DurationMetrics,/// Total nanoseconds spent waiting for successfully acquired locks.pub wait_duration_ns: u64,/// Count of conflicts by type (exact/exact, tree/exact, etc.).diff --git a/crates/ragfs/src/metrics/duration.rs b/crates/ragfs/src/metrics/duration.rsnew file mode 100644index 000000000..5e9b7bf20--- /dev/null+++ b/crates/ragfs/src/metrics/duration.rs@@ -0,0 +1,102 @@+//! SDK-owned duration aggregation shared by filesystem and path-lock instrumentation.++use std::sync::{Arc, Weak};+use std::time::Duration;++use opentelemetry::metrics::{Histogram, MeterProvider};+use opentelemetry::KeyValue;+use opentelemetry_sdk::error::OTelSdkResult;+use opentelemetry_sdk::metrics::data::{+ AggregatedMetrics, HistogramDataPoint, MetricData, ResourceMetrics,+};+use opentelemetry_sdk::metrics::reader::MetricReader;+use opentelemetry_sdk::metrics::{+ InstrumentKind, ManualReader, Pipeline, SdkMeterProvider, Temporality,+};++/// Retain the same reader in the provider and the native snapshot owner.+#[derive(Clone, Debug)]+struct SharedReader(Arc<ManualReader>);++impl MetricReader for SharedReader {+ fn register_pipeline(&self, pipeline: Weak<Pipeline>) {+ self.0.register_pipeline(pipeline);+ }++ fn collect(&self, metrics: &mut ResourceMetrics) -> OTelSdkResult {+ self.0.collect(metrics)+ }++ fn force_flush(&self) -> OTelSdkResult {+ self.0.force_flush()+ }++ fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {+ self.0.shutdown_with_timeout(timeout)+ }++ fn temporality(&self, kind: InstrumentKind) -> Temporality {+ self.0.temporality(kind)+ }+}++/// Own one SDK provider for the lifetime of the measured subsystem.+#[derive(Clone, Debug)]+pub(crate) struct DurationMetrics {+ // Dropping the final provider handle shuts down its reader.+ _provider: SdkMeterProvider,+ reader: SharedReader,+ histogram: Histogram<u64>,+}++impl Default for DurationMetrics {+ fn default() -> Self {+ let reader = SharedReader(Arc::new(ManualReader::default()));+ let provider = SdkMeterProvider::builder()+ .with_reader(reader.clone())+ .build();+ // A bounded 1/2/5 scale covers short I/O through multi-day background work.+ // This is SDK collection policy, not a timeout or a per-operation setting.+ let boundaries = (5..=15)+ .flat_map(|power| [1, 2, 5].map(|factor| (factor * 10_u64.pow(power)) as f64))+ .collect();+ let histogram = provider+ .meter("ragfs")+ .u64_histogram("duration")+ .with_unit("ns")+ .with_boundaries(boundaries)+ .build();+ Self {+ _provider: provider,+ reader,+ histogram,+ }+ }+}++impl DurationMetrics {+ pub(crate) fn record(&self, duration: Duration, attributes: &[KeyValue]) {+ self.histogram.record(+ u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX),+ attributes,+ );+ }++ pub(crate) fn snapshot(&self) -> Vec<HistogramDataPoint<u64>> {+ let mut metrics = ResourceMetrics::default();+ // The provider and reader have the same owner; neither is shut down independently.+ self.reader+ .collect(&mut metrics)+ .expect("live duration metrics provider");+ metrics+ .scope_metrics()+ .flat_map(|scope| scope.metrics())+ .flat_map(|metric| match metric.data() {+ AggregatedMetrics::U64(MetricData::Histogram(histogram)) => {+ histogram.data_points().cloned().collect::<Vec<_>>()+ }+ _ => unreachable!("duration provider contains only a u64 histogram"),+ })+ .collect()+ }+}diff --git a/crates/ragfs/src/metrics/mod.rs b/crates/ragfs/src/metrics/mod.rsindex 7f6f1ec47..97602e028 100644--- a/crates/ragfs/src/metrics/mod.rs+++ b/crates/ragfs/src/metrics/mod.rs@@ -2,10 +2,11 @@use std::collections::BTreeMap;-use crate::core::stats::DURATION_BUCKET_BOUNDS_NS;use crate::core::{Error, FilesystemStats, FsOperation, Result};use crate::lock::metrics::LockMetrics;+pub(crate) mod duration;+/// One metric series, identified by its name and bounded labels.#[derive(Debug, Clone, PartialEq)]pub struct RagfsMetric {@@ -68,6 +69,9 @@ pub(crate) fn operation_metrics(plugin: &str, stats: &FilesystemStats) -> Vec<Ra1.0,));}+ if stats.count == 0 {+ continue;+ }metrics.push(RagfsMetric {name: "ragfs_operation_duration_seconds".into(),labels: [@@ -75,7 +79,7 @@ pub(crate) fn operation_metrics(plugin: &str, stats: &FilesystemStats) -> Vec<Ra("operation".into(), operation.as_str().into()),].into(),value: RagfsMetricValue::Histogram {- bucket_bounds: DURATION_BUCKET_BOUNDS_NS.to_vec(),+ bucket_bounds: stats.duration_bucket_bounds_ns.clone(),bucket_counts: stats.duration_bucket_counts.to_vec(),count: stats.count,sum: stats.total_time_ns,@@ -128,6 +132,7 @@ pub(crate) fn cache_metrics(stats: crate::cache::CacheMetricsSnapshot) -> Vec<Ra/// Convert the single lock manager's snapshot into unlabeled metric records.pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {let mut metrics = Vec::new();+ let scan = stats.descendant_scan.snapshot().into_iter().next();for (name, value) in [("lock_active", stats.active_lock_count),("lock_waiting", stats.waiting_lock_count),@@ -142,13 +147,28 @@ pub(crate) fn lock_metrics(stats: LockMetrics) -> Vec<RagfsMetric> {for (name, value) in [("lock_conflicts_total", stats.conflict_count),("lock_stale_leases_released_total", stats.stale_leases_released),- ("lock_descendant_scans_total", stats.descendant_scan_count),] {metrics.push(RagfsMetric::counter(name, &[], value as u64, 1.0));}metrics.push(RagfsMetric::counter(- "lock_descendant_scan_duration_seconds_total", &[], stats.descendant_scan_duration_ns, 1e-9,+ "lock_descendant_scans_total", &[], scan.as_ref().map_or(0, |point| point.count()), 1.0,));+ metrics.push(RagfsMetric::counter(+ "lock_descendant_scan_duration_seconds_total", &[], scan.as_ref().map_or(0, |point| point.sum()), 1e-9,+ ));+ if let Some(scan) = scan {+ metrics.push(RagfsMetric {+ name: "lock_descendant_scan_duration_seconds".into(),+ labels: BTreeMap::new(),+ value: RagfsMetricValue::Histogram {+ bucket_bounds: scan.bounds().map(|bound| bound as u64).collect(),+ bucket_counts: scan.bucket_counts().collect(),+ count: scan.count(),+ sum: scan.sum(),+ scale: 1e-9,+ },+ });+ }metrics}diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.mdindex 8f0451a40..e52b98590 100644--- a/docs/en/concepts/12-metrics.md+++ b/docs/en/concepts/12-metrics.md@@ -84,11 +84,11 @@ Collectors turn inputs into metric semantics:### MetricRegistry-The MetricRegistry is the in-process metric store that keeps the current metric values and serves them to the exporter.+MetricRegistry keeps OpenViking metric names, label contracts, and account-dimension rules. Event counts and durations are accumulated by the OpenTelemetry SDK. Cumulative values from RAGFS and state collectors enter as snapshots, without being replayed as new observations.### Exporter-The first exporter implementation is the Prometheus exporter, which renders registry contents into Prometheus exposition text.+`/metrics` uses the official Prometheus Python client for text exposition. OTLP HTTP/gRPC uses the official OpenTelemetry exporters for encoding, delivery, and retries. Existing endpoint, headers, TLS, and export-interval settings remain unchanged.## Usage@@ -139,6 +139,36 @@ Notes:The tables list representative metrics defined by the collectors. A series appears only when its feature is enabled, the relevant event occurs, and collection succeeds. An absent series does not necessarily mean zero.+### Duration distributions and percentiles++Instrumentation records actual durations and labels; Python and RAGFS delegate duration+aggregation to their respective OpenTelemetry SDKs. The integration layer supplies the default+collection policy, so business collectors do not maintain bucket boundaries. Python starts at+0.01 seconds and RAGFS at 0.0001 seconds, using a 1/2/5 scale per decade through 5,000,000 seconds+(about 58 days). Larger observations remain in `+Inf`. These boundaries are not timeouts;+dashboards must not filter slow operations with `< 10` or a fixed axis maximum.++Prometheus `_bucket`, `_count`, `_sum` series and OTLP histograms carry `bucket_schema`,+derived from their actual boundaries; collectors cannot set it themselves.+**Keep `bucket_schema` when aggregating percentiles**+so rolling upgrades cannot mix incompatible bucket layouts. Older series without this label+form a separate group; their lost overflow distribution cannot be reconstructed.++```promql+histogram_quantile(0.99,+ sum by (le, bucket_schema, operation) (+ rate(openviking_operation_duration_seconds_bucket[5m])+ )+)+```++Use `0.95` for P95. Compute the mean from `rate(_sum) / rate(_count)` with the same+`operation, bucket_schema` grouping, showing it only when the count is positive.+No completed observations must remain a gap, not zero latency. Percentiles are still+bucket-interpolated estimates, particularly with low traffic or coarse buckets.+Each default Python duration histogram label set exports 30 Prometheus series; RAGFS exports 36.+Account for the extra buckets in scrape and storage capacity planning.+### Requests and Operations| Metric Family | Type | Common Labels | Meaning |@@ -255,6 +285,7 @@ Notes:| `openviking_lock_stale_leases_released_total` | Counter | none | stale leases released || `openviking_lock_descendant_scans_total` | Counter | none | completed descendant scans || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | none | cumulative descendant scan duration |+| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | completed descendant scan duration distribution, supporting mean, P95, and P99 |The queue duration `outcome` label is one of `success`, `failed`, `requeued`, `cancelled`, or `exception`. `failed` means the handler returned a settled failure that can be acknowledged, while `exception` means processing raised and the current message is left unacknowledged.@@ -302,7 +333,10 @@ seconds, for example `123 ns = 0.000000123 seconds`.These metrics have no `mount` or `account_id` label. Cache and multi-backendfamilies are absent when those capabilities are not enabled. `status` is`success` or `error`; `exists=false` is successful and `replace` uses `rename`.-Histogram buckets cover all outcomes, with finite bounds from 0.0001 to 10 seconds.+Histograms cover all outcomes and are accumulated by the Rust OpenTelemetry SDK;+exported boundaries and counts come directly from SDK snapshots.+Histograms are emitted only after an operation has observations, avoiding empty long-tail+buckets for every plugin/operation pair; existing operation counters remain available.Python `get_stats()` retains its microsecond fields.The protocol family always contains both closed-label series. One unreadablediff --git a/docs/en/guides/11-grafana-prometheus.md b/docs/en/guides/11-grafana-prometheus.mdindex 67bd83e8e..ee1f0b74d 100644--- a/docs/en/guides/11-grafana-prometheus.md+++ b/docs/en/guides/11-grafana-prometheus.md@@ -333,7 +333,7 @@ sum by (route, status) (rate(openviking_http_requests_total[5m]))P95 latency:```promql-histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[5m])))+histogram_quantile(0.95, sum by (le, route, bucket_schema) (rate(openviking_http_request_duration_seconds_bucket[5m])))```Queue backlog:diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.mdindex 283fba384..d07ee5cbd 100644--- a/docs/zh/concepts/12-metrics.md+++ b/docs/zh/concepts/12-metrics.md@@ -84,11 +84,11 @@ Collector 负责把输入转成指标语义:### MetricRegistry-MetricRegistry 是进程内的指标注册中心,用于保存当前指标值,并在导出时统一读取。+MetricRegistry 保留 OpenViking 的指标名、标签约束和账户维度规则。单次事件的计数与耗时直接交给 OpenTelemetry SDK 累计;RAGFS 和状态采集器返回的累计值作为快照接入,不会重新作为事件计数。### Exporter-当前首个落地导出器是 Prometheus Exporter,用于把 registry 中的指标渲染成 Prometheus exposition 文本。+`/metrics` 使用官方 Prometheus Python 客户端输出文本;OTLP HTTP/gRPC 使用 OpenTelemetry 官方导出器处理编码、发送和重试。现有 endpoint、headers、TLS 和上报周期配置保持不变。## 使用方式@@ -139,6 +139,31 @@ scrape_configs:下面列出 Collector 定义的代表性指标。某个指标是否出现,取决于功能是否启用、是否发生对应事件,以及 Collector 是否采集成功;未出现不能直接解释为数值为零。+### 耗时分布与分位数++业务打点只记录实际耗时和标签;Python 与 RAGFS 的耗时累计由各自的 OpenTelemetry SDK 完成。+接入层统一配置默认统计范围,业务采集器无需维护桶边界。Python 从 0.01 秒、RAGFS 从 0.0001 秒起,+按每个十倍数量级的 1、2、5 档覆盖到 5,000,000 秒(约 58 天),超出范围仍计入 `+Inf`。+这里的范围不是任务超时;看板不应通过 `< 10` 或固定纵轴上限过滤慢请求。++Prometheus 的 `_bucket`、`_count`、`_sum` 和 OTLP Histogram 都带有 `bucket_schema`,+该标签由实际桶边界计算,采集器不能自行设置。**聚合分位数时必须保留 `bucket_schema`**,避免滚动升级时把+新旧分桶混算。没有该标签的旧数据会形成独立的一组;其已经合并到最后一桶的细节无法恢复。++```promql+histogram_quantile(0.99,+ sum by (le, bucket_schema, operation) (+ rate(openviking_operation_duration_seconds_bucket[5m])+ )+)+```++P95 将 `0.99` 改为 `0.95`。平均耗时使用同一组 `operation, bucket_schema` 的+`rate(_sum) / rate(_count)`,仅在次数大于零时显示;没有完成记录时不能用零耗时代替。+分位数仍是桶内插值估算,低流量或粗粒度桶内的结果不等于精确的单次请求耗时。+默认每组 Python 耗时直方图输出 30 条 Prometheus 时序,RAGFS 输出 36 条;+应将新增桶的采集和存储成本计入容量规划。+### 请求与操作| 指标族 | 类型 | 常见标签 | 含义 |@@ -256,6 +281,7 @@ Telemetry bridge 还识别旧摘要键,对应 `request`、`process`、`parse`| `openviking_lock_stale_leases_released_total` | Counter | 无 | 已释放的过期租约数 || `openviking_lock_descendant_scans_total` | Counter | 无 | 已完成的后代锁扫描次数 || `openviking_lock_descendant_scan_duration_seconds_total` | Counter | 无 | 后代锁扫描累计耗时 |+| `openviking_lock_descendant_scan_duration_seconds` | Histogram | `bucket_schema` | 已完成后代锁扫描的耗时分布,可计算平均、P95 和 P99 |队列耗时指标中的 `outcome` 可取 `success`、`failed`、`requeued`、`cancelled`、`exception`。其中 `failed` 表示 handler 返回了可确认的失败结果,`exception` 表示处理过程抛出异常,当前消息不会 ACK。@@ -301,7 +327,8 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba这些指标不带 `mount` 和 `account_id` 标签。未启用 Cache 或 multi-backend时,不生成对应指标族。`status` 为 `success` 或 `error`。`exists=false` 计为成功;`replace` 使用 `rename` 操作标签。-Histogram 包含全部操作结果,有限桶边界范围为 0.0001 至 10 秒。+Histogram 包含全部操作结果,使用 Rust OpenTelemetry SDK 统计;输出边界和计数直接来自 SDK。+尚未发生的操作不导出空直方图,避免为每个插件和操作持续发布无样本的长尾桶;原有操作计数器保留。Python `get_stats()` 保留原有微秒字段。protocol 指标始终包含闭集中的两条 series。任何 protocol、account 或 segmentdiff --git a/docs/zh/guides/11-grafana-prometheus.md b/docs/zh/guides/11-grafana-prometheus.mdindex 5d84f7fd8..fdf636f45 100644--- a/docs/zh/guides/11-grafana-prometheus.md+++ b/docs/zh/guides/11-grafana-prometheus.md@@ -333,7 +333,7 @@ sum by (route, status) (rate(openviking_http_requests_total[5m]))P95 延迟:```promql-histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[5m])))+histogram_quantile(0.95, sum by (le, route, bucket_schema) (rate(openviking_http_request_duration_seconds_bucket[5m])))```队列积压:diff --git a/examples/grafana/openviking_demo_dashboard.json b/examples/grafana/openviking_demo_dashboard.jsonindex da863a8c7..71b1cca12 100644--- a/examples/grafana/openviking_demo_dashboard.json+++ b/examples/grafana/openviking_demo_dashboard.json@@ -324,8 +324,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "HTTP P95",+ "expr": "histogram_quantile(0.95, sum by (le, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "HTTP P95 / {{bucket_schema}}","range": true,"refId": "A"}@@ -1586,8 +1586,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, route) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{route}}",+ "expr": "histogram_quantile(0.95, sum by (le, route, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{route}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -1687,8 +1687,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_http_request_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_http_request_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -2219,8 +2219,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, context_type) (rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{context_type}}",+ "expr": "histogram_quantile(0.95, sum by (le, context_type, bucket_schema) (label_replace(rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{context_type}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -2320,8 +2320,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_retrieval_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -2623,8 +2623,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(openviking_operation_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{operation}}",+ "expr": "histogram_quantile(0.95, sum by (le, operation, bucket_schema) (label_replace(rate(openviking_operation_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{operation}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -2724,8 +2724,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_operation_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_operation_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -3041,8 +3041,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{stage}}",+ "expr": "histogram_quantile(0.95, sum by (le, stage, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{stage}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -3142,8 +3142,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -3858,8 +3858,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, provider, model_name) (rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{provider}} / {{model_name}}",+ "expr": "histogram_quantile(0.95, sum by (le, provider, model_name, bucket_schema) (label_replace(rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{provider}} / {{model_name}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -3959,8 +3959,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_vlm_call_duration_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -4262,8 +4262,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, status) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{status}}",+ "expr": "histogram_quantile(0.95, sum by (le, status, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{status}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -4363,8 +4363,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval])))",- "legendFormat": "{{account_id}}",+ "expr": "histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{account_id}} / {{bucket_schema}}","range": true,"refId": "A"}@@ -4895,7 +4895,24 @@},"unit": "short"},- "overrides": []+ "overrides": [+ {+ "matcher": {+ "id": "byFrameRefID",+ "options": "D"+ },+ "properties": [+ {+ "id": "unit",+ "value": "s"+ },+ {+ "id": "custom.axisPlacement",+ "value": "right"+ }+ ]+ }+ ]},"gridPos": {"h": 8,@@ -4953,8 +4970,8 @@},{"editorMode": "code",- "expr": "topk(10, histogram_quantile(0.95, sum by (le, account_id) (rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]))))",- "legendFormat": "embedding p95 {{account_id}}",+ "expr": "(histogram_quantile(0.95, sum by (le, account_id, bucket_schema) (label_replace(rate(openviking_embedding_latency_seconds_bucket[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))) and on (account_id) (topk(10, sum by (account_id) (rate(openviking_vlm_calls_total[$__rate_interval]))))",+ "legendFormat": "embedding p95 {{account_id}} / {{bucket_schema}}","range": true,"refId": "D"}@@ -5054,7 +5071,7 @@"x": 0,"y": 196},- "id": 59,+ "id": 64,"panels": [],"title": "Memory Extraction","type": "row"@@ -5089,7 +5106,7 @@"x": 0,"y": 197},- "id": 60,+ "id": 65,"options": {"legend": {"calcs": [],@@ -5245,8 +5262,8 @@"targets": [{"editorMode": "code",- "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(openviking_resource_stage_duration_seconds_bucket{stage=~\"semantic_queue_wait|semantic_execute|embedding_queue_wait|embedding_execute\"}[$__rate_interval])))",- "legendFormat": "{{stage}}",+ "expr": "histogram_quantile(0.95, sum by (le, stage, bucket_schema) (label_replace(rate(openviking_resource_stage_duration_seconds_bucket{stage=~\"semantic_queue_wait|semantic_execute|embedding_queue_wait|embedding_execute\"}[$__rate_interval]), \"bucket_schema\", \"legacy\", \"bucket_schema\", \"^$\")))",+ "legendFormat": "{{stage}} / {{bucket_schema}}","range": true,"refId": "A"}diff --git a/openviking/metrics/collectors/base.py b/openviking/metrics/collectors/base.pyindex 936a1ed98..443115275 100644--- a/openviking/metrics/collectors/base.py+++ b/openviking/metrics/collectors/base.py@@ -170,7 +170,7 @@ class CollectorMetricWriter:kwargs["buckets"] = bucketsself._registry.observe_histogram(name, value, **kwargs)- def gauge_delete_matching(+ def delete_matching(self,name: str,*,@@ -179,7 +179,7 @@ class CollectorMetricWriter:) -> None:"""Delete matching gauge series while preserving the current account partition."""if not metric_supports_account_dimension(name):- self._registry.gauge_delete_matching(name, match_labels=dict(match_labels))+ self._registry.delete_matching(name, match_labels=dict(match_labels))returnfinal_account_id = resolve_metric_account_label(metric_name=name,@@ -188,7 +188,7 @@ class CollectorMetricWriter:)labels = dict(match_labels)labels["account_id"] = final_account_id- self._registry.gauge_delete_matching(name, match_labels=labels)+ self._registry.delete_matching(name, match_labels=labels)def _with_account_labels(self,@@ -341,7 +341,7 @@ class StateMetricCollector(MetricCollector, Refreshable, ABC):label_names: tuple[str, ...] = (),) -> None:"""Replace the full gauge series selected by `match_labels` with one fresh value."""- registry.gauge_delete_matching(str(metric_name), match_labels=match_labels)+ registry.delete_matching(str(metric_name), match_labels=match_labels)if labels is None:registry.set_gauge(str(metric_name), float(value))return@@ -421,7 +421,7 @@ class DomainStatsMetricCollector(MetricCollector, Refreshable, ABC):) -> None:"""Replace all existing global gauge series for the provided metric names in one pass."""for metric_name, value in zip(metric_names, values, strict=True):- registry.gauge_delete_matching(str(metric_name), match_labels={})+ registry.delete_matching(str(metric_name), match_labels={})registry.set_gauge(str(metric_name),float(value),@@ -534,7 +534,7 @@ class ProbeMetricCollector(MetricCollector, Refreshable, ABC):label_names: tuple[str, ...] = (),) -> None:"""Replace the selected gauge series with one fresh readiness-style value."""- registry.gauge_delete_matching(str(metric_name), match_labels=match_labels)+ registry.delete_matching(str(metric_name), match_labels=match_labels)if labels is None:registry.set_gauge(str(metric_name), float(value))return@@ -601,4 +601,4 @@ class ProbeMetricCollector(MetricCollector, Refreshable, ABC):) -> None:"""Delete all gauge series whose label values are listed in `values`."""for value in values:- registry.gauge_delete_matching(str(metric_name), match_labels={label_key: str(value)})+ registry.delete_matching(str(metric_name), match_labels={label_key: str(value)})diff --git a/openviking/metrics/collectors/feedback.py b/openviking/metrics/collectors/feedback.pyindex 285c0cc22..5cf6cc859 100644--- a/openviking/metrics/collectors/feedback.py+++ b/openviking/metrics/collectors/feedback.py@@ -169,7 +169,7 @@ class FeedbackCollector(DomainStatsMetricCollector):channels: dict[str, dict[str, Any]],) -> None:for metric_name, _ in self._SUMMARY_METRICS:- registry.gauge_delete_matching(metric_name, match_labels={})+ registry.delete_matching(metric_name, match_labels={})for metric_name, key in self._SUMMARY_METRICS:registry.set_gauge(metric_name,@@ -179,7 +179,7 @@ class FeedbackCollector(DomainStatsMetricCollector):)for metric_name, _ in self._CHANNEL_METRICS:- registry.gauge_delete_matching(metric_name, match_labels={})+ registry.delete_matching(metric_name, match_labels={})for channel, channel_stats in channels.items():labels = {"channel": str(channel), "valid": str(valid)}for metric_name, key in self._CHANNEL_METRICS:diff --git a/openviking/metrics/collectors/manager.py b/openviking/metrics/collectors/manager.pyindex 82214bb13..ec1ab964e 100644--- a/openviking/metrics/collectors/manager.py+++ b/openviking/metrics/collectors/manager.py@@ -68,11 +68,11 @@ class CollectorManager:If the collector has `config.ttl_seconds`, a gate is created to enforce TTL + SWR."""- if not isinstance(collector, Refreshable):- raise TypeError("CollectorManager only accepts Refreshable collectors")+ if not isinstance(collector, MetricCollector) or not isinstance(collector, Refreshable):+ raise TypeError("CollectorManager only accepts refreshable MetricCollector instances")self._collectors.append(collector)- collector_name = self._collector_name(collector)- ttl_seconds = getattr(cast(Refreshable, collector).config, "ttl_seconds", None)+ collector_name = collector.collector_name()+ ttl_seconds = cast(Refreshable, collector).config.ttl_secondsif ttl_seconds is not None:self._gates[collector_name] = RefreshGate(ttl_seconds=float(ttl_seconds))@@ -96,9 +96,9 @@ class CollectorManager:tasks: list[asyncio.Task[RefreshResult]] = []for c in self._collectors:- collector_name = self._collector_name(c)+ collector_name = c.collector_name()gate = self._gates.get(collector_name)- timeout_seconds = float(getattr(cast(Refreshable, c).config, "timeout_seconds", 0.5))+ timeout_seconds = cast(Refreshable, c).config.timeout_secondsif gate is None:tasks.append(asyncio.create_task(@@ -222,37 +222,22 @@ class CollectorManager:asyncio.to_thread(collector.collect, registry), timeout=timeout)return RefreshResult(- collector=self._collector_name(collector),+ collector=collector.collector_name(),attempted=collected is not False,success=collected is not False,reason="skipped" if collected is False else "ok",)except asyncio.TimeoutError:return RefreshResult(- collector=self._collector_name(collector),+ collector=collector.collector_name(),attempted=True,success=False,reason="timeout",)except Exception:return RefreshResult(- collector=self._collector_name(collector),+ collector=collector.collector_name(),attempted=True,success=False,reason="exception",)-- def _collector_name(self, collector: MetricCollector) -> str:- """- Resolve the display name used for refresh results and refresh-gate bookkeeping.-- The manager prefers the collector's explicit `collector_name()` method and falls back to- a `name` attribute or the concrete class name for compatibility.- """- try:- return str(collector.collector_name())- except AttributeError:- name = getattr(collector, "name", None)- if name is not None:- return str(name)- return collector.__class__.__name__diff --git a/openviking/metrics/collectors/model_usage.py b/openviking/metrics/collectors/model_usage.pyindex b21fc1d45..a26d76541 100644--- a/openviking/metrics/collectors/model_usage.py+++ b/openviking/metrics/collectors/model_usage.py@@ -139,7 +139,7 @@ class ModelUsageCollector(DomainStatsMetricCollector):The replacement deletes any prior series for the same `model_type`, ensuring only one of`valid="1"` or `valid="0"` remains visible at a time."""- registry.gauge_delete_matching(+ registry.delete_matching(self.USAGE_AVAILABLE,match_labels={"model_type": str(model_type)},)diff --git a/openviking/metrics/collectors/queue.py b/openviking/metrics/collectors/queue.pyindex e8b1ac417..79f261bbe 100644--- a/openviking/metrics/collectors/queue.py+++ b/openviking/metrics/collectors/queue.py@@ -12,26 +12,6 @@ from openviking.metrics.datasources.queue import QueuePipelineStateDataSourcefrom .base import CollectorConfig, EventMetricCollector, StateMetricCollector-QUEUE_DURATION_BUCKETS: tuple[float, ...] = (- 0.01,- 0.05,- 0.1,- 0.25,- 0.5,- 1.0,- 2.5,- 5.0,- 10.0,- 30.0,- 60.0,- 120.0,- 300.0,- 600.0,- 1800.0,- 3600.0,- 21600.0,- 86400.0,-)_QUEUE_OUTCOMES = frozenset({"success", "failed", "requeued", "cancelled", "exception"})@@ -69,7 +49,6 @@ class QueueDurationCollector(EventMetricCollector):process_duration,labels=labels,label_names=("queue", "outcome"),- buckets=QUEUE_DURATION_BUCKETS,)raw_end_to_end = payload.get("end_to_end_duration_seconds")@@ -86,7 +65,6 @@ class QueueDurationCollector(EventMetricCollector):end_to_end_duration,labels=labels,label_names=("queue", "outcome"),- buckets=QUEUE_DURATION_BUCKETS,)diff --git a/openviking/metrics/collectors/ragfs.py b/openviking/metrics/collectors/ragfs.pyindex b0bf3d6b3..4c802ad9c 100644--- a/openviking/metrics/collectors/ragfs.py+++ b/openviking/metrics/collectors/ragfs.py@@ -21,7 +21,7 @@ class RagfsMetricCollector(DomainStatsMetricCollector):data_source: RagfsMetricDataSourceconfig: CollectorConfig = CollectorConfig(ttl_seconds=None, timeout_seconds=1.0)_collect_lock: Lock = field(default_factory=Lock, init=False, repr=False)- _tracked_series: set[tuple[str, str, tuple[tuple[str, str], ...]]] = field(+ _tracked_series: set[tuple[str, tuple[tuple[str, str], ...]]] = field(default_factory=set, init=False, repr=False)@@ -72,10 +72,10 @@ class RagfsMetricCollector(DomainStatsMetricCollector):labels=labels,label_names=label_names,)- identity = (kind, name, tuple(sorted(labels.items())))+ identity = (name, tuple(sorted(labels.items())))current_series.add(identity)self._tracked_series.add(identity)for identity in sorted(self._tracked_series - current_series):- kind, name, labels = identity- getattr(registry, f"{kind}_delete_matching")(name, match_labels=dict(labels))+ name, labels = identity+ registry.delete_matching(name, match_labels=dict(labels))self._tracked_series.remove(identity)diff --git a/openviking/metrics/collectors/task_tracker.py b/openviking/metrics/collectors/task_tracker.pyindex b55049963..e7a86886e 100644--- a/openviking/metrics/collectors/task_tracker.py+++ b/openviking/metrics/collectors/task_tracker.py@@ -57,7 +57,7 @@ class TaskTrackerCollector(StateMetricCollector):self.FAILED,self.CANCELLED,):- registry.gauge_delete_matching(metric_name, match_labels={})+ registry.delete_matching(metric_name, match_labels={})for task_type, counts in counts_by_type.items():labels = {"task_type": str(task_type)}registry.set_gauge(diff --git a/openviking/metrics/core/registry.py b/openviking/metrics/core/registry.pyindex 8512d2b0c..11aec8523 100644--- a/openviking/metrics/core/registry.py+++ b/openviking/metrics/core/registry.py@@ -1,21 +1,10 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""-In-process metric registry used by OpenViking.--Design goals:-1) Keep the registry as a "current state store" for metrics (no snapshot/view layer).-2) Provide a strict label contract per metric name to prevent accidental high-cardinality exports.-3) Be safe under concurrent updates from request threads and exporter refresh threads.-4) Allow bounded memory usage via per-metric series limits, while exposing drop counts for debugging.+"""Bind OpenViking metric names and tags to OpenTelemetry SDK instruments.-Important behaviors:-- Label normalization: label dicts are normalized into a sorted tuple of (key, value) pairs.-- Label contract: once a metric name is registered with a label key set, subsequent writes must- use the same label keys (same keys, same order). Violations raise ValueError.-- Series limit: when a metric family reaches `max_series_per_metric`, new series are dropped- and `openviking_metrics_dropped_series_total{metric="<name>"}` will reflect the drops.+The SDK owns event aggregation. Already accumulated native/state values are kept+as SDK data points and exported as snapshots, never replayed as observations."""from __future__ import annotations@@ -23,835 +12,256 @@ from __future__ import annotationsimport mathimport threadingimport time-from bisect import bisect_left-from typing import Mapping, Sequence--from .types import normalize_labels--DEFAULT_LATENCY_BUCKETS: tuple[float, ...] = (0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)---def _canonicalize_label_names(label_names: Sequence[str]) -> tuple[str, ...]:- """- Return the canonical family label order used by the in-process registry.-- The registry normalizes concrete label mappings into key-sorted tuples, so family-level- `label_names` must follow the same stable order to avoid false mismatches between writers- that provide the same key set with different input ordering.- """- return tuple(sorted(str(name) for name in label_names))---def _validate_label_keys(- name: str,- label_names: tuple[str, ...],- normalized: tuple[tuple[str, str], ...],-) -> None:- """Check normalized labels against the named metric's declared keys; return None."""- if not label_names and normalized:- raise ValueError(f"metric {name} does not accept labels")- if label_names and tuple(k for k, _ in normalized) != label_names:- raise ValueError(f"metric {name} label keys mismatch: expected {label_names}")---def _validate_counter_value(value: float) -> float:- """Return value as a finite, nonnegative float or raise ValueError."""- value = float(value)- if not math.isfinite(value) or value < 0:- raise ValueError("counter value must be finite and nonnegative")- return value---def _validate_histogram_values(- bounds: tuple[float, ...],- bucket_counts: Sequence[int],- count: int,- value_sum: float,-) -> tuple[tuple[int, ...], float]:- """Validate bounds and totals; return bucket_counts as a tuple and value_sum as a float."""- bucket_counts = tuple(bucket_counts)- value_sum = float(value_sum)- if any(not math.isfinite(bound) or bound < 0 for bound in bounds) or any(- left >= right for left, right in zip(bounds, bounds[1:], strict=False)- ):- raise ValueError("histogram bounds must be finite, nonnegative and increasing")- if len(bucket_counts) != len(bounds) + 1 or any(- type(value) is not int or value < 0 for value in (*bucket_counts, count)- ):- raise ValueError("histogram counts must be nonnegative integers matching bounds")- if sum(bucket_counts) != count or not math.isfinite(value_sum) or value_sum < 0:- raise ValueError("histogram count or sum is invalid")- return bucket_counts, value_sum---def _labels_contains(- labels: tuple[tuple[str, str], ...],- match: tuple[tuple[str, str], ...],-) -> bool:- """- Return whether a normalized label set contains all requested label pairs.-- This helper is used by partial-delete operations where callers provide only a subset- of labels and expect every matching series to be removed.- """- if not match:- return True- label_set = set(labels)- for item in match:- if item not in label_set:- return False- return True+from dataclasses import dataclass, field, replace+from typing import Mapping++from opentelemetry.sdk.metrics import MeterProvider+from opentelemetry.sdk.metrics.export import (+ AggregationTemporality,+ Gauge,+ Histogram,+ HistogramDataPoint,+ InMemoryMetricReader,+ Metric,+ MetricsData,+ NumberDataPoint,+ ResourceMetrics,+ ScopeMetrics,+ Sum,+)+from opentelemetry.sdk.resources import Resource+from opentelemetry.sdk.util.instrumentation import InstrumentationScope++from .types import histogram_schema, normalize_labels++# SDK collection policy: a compact 1/2/5 scale from milliseconds to multi-day work.+# Business collectors record actual seconds and do not manage these boundaries.+DEFAULT_LATENCY_BUCKETS = tuple(+ factor * 10.0**power for power in range(-2, 7) for factor in (1, 2, 5)+)+++@dataclass+class _MetricBinding:+ kind: str+ label_names: tuple[str, ...]+ mode: str+ bounds: tuple[float, ...] = ()+ instrument: object | None = None+ points: dict[tuple[tuple[str, str], ...], NumberDataPoint | HistogramDataPoint | None] = field(+ default_factory=dict+ )class MetricRegistry:- """- MetricRegistry holds Counter/Gauge/Histogram families in memory.-- The exporter reads current values via `iter_counters/iter_gauges/iter_histograms`.- Collectors should use the unified write helpers (`inc_counter/set_gauge/observe_histogram`)- to keep call sites consistent and reduce accidental label misuse.- """+ """Keep metric contracts and adapt state snapshots; delegate accumulation to the SDK."""def __init__(self, *, max_series_per_metric: int = 2048) -> None:- """- Initialize an empty in-process metrics registry.-- Args:- max_series_per_metric: Hard cap for the number of distinct label series allowed- inside one metric family before additional series are dropped.- """- self._max_series_per_metric = max_series_per_metric- self._lock = threading.Lock()- self._counters: dict[str, _CounterFamily] = {}- self._gauges: dict[str, _GaugeFamily] = {}- self._histograms: dict[str, _HistogramFamily] = {}- self._dropped_series_total: dict[str, int] = {}-- def inc_counter(- self,- name: str,- *,- amount: float = 1.0,- labels: Mapping[str, str] | None = None,- label_names: Sequence[str] = (),- ) -> None:- """- Increment a Counter metric.-- Args:- name: Prometheus metric name.- amount: Increment delta (float, will be rendered as int when possible).- labels: Optional label dict. Keys must exactly match `label_names`.- label_names: Ordered label key tuple for this metric name.-- Notes:- This helper enforces the per-metric label contract by delegating to the underlying- Counter family, which validates label keys.- """- self.counter(name, label_names=label_names).inc(labels=labels, amount=float(amount))-- def set_counter(- self,- name: str,- value: float,- *,- labels: Mapping[str, str] | None = None,- label_names: Sequence[str] = (),- ) -> None:- """Replace the named counter's labelled value, including zero or reset; return None."""- value = _validate_counter_value(value)- ln = _canonicalize_label_names(label_names)- normalized = normalize_labels(labels)- _validate_label_keys(name, ln, normalized)- self.counter(name, label_names=ln).set(value, labels=dict(normalized))-- def set_histogram(- self,- name: str,- *,- bucket_bounds: Sequence[float],- bucket_counts: Sequence[int],- count: int,- value_sum: float,- labels: Mapping[str, str] | None = None,- label_names: Sequence[str] = (),- ) -> None:- """Replace a named histogram from disjoint bucket counts and totals; return None."""- bounds = tuple(float(bound) for bound in bucket_bounds)- bucket_counts, value_sum = _validate_histogram_values(- bounds, bucket_counts, count, value_sum- )- ln = _canonicalize_label_names(label_names)- normalized = normalize_labels(labels)- _validate_label_keys(name, ln, normalized)- self.histogram(name, label_names=ln, buckets=bounds).set(- bucket_counts, count, value_sum, labels=dict(normalized)+ self._lock = threading.RLock()+ self._max_series = max_series_per_metric+ self._bindings: dict[str, _MetricBinding] = {}+ self._reader = InMemoryMetricReader()+ self._provider = MeterProvider(+ metric_readers=[self._reader], resource=Resource({}), shutdown_on_exit=False)+ self._meter = self._provider.get_meter("openviking.metrics")+ self._dropped = self._meter.create_counter("openviking_metrics_dropped_series_total")- def set_gauge(- self,- name: str,- value: float,- *,- labels: Mapping[str, str] | None = None,- label_names: Sequence[str] = (),- ) -> None:- """- Set a Gauge metric to an absolute value.-- Args:- name: Prometheus metric name.- value: Gauge value (float).- labels: Optional label dict. Keys must exactly match `label_names`.- label_names: Ordered label key tuple for this metric name.- """- self.gauge(name, label_names=label_names).set(float(value), labels=labels)+ def _bind(self, name, kind, label_names, labels, *, mode, bounds=()):+ names = tuple(sorted(str(key) for key in label_names))+ normalized = normalize_labels(labels)+ if tuple(key for key, _ in normalized) != names:+ raise ValueError(f"metric {name} label keys mismatch: expected {names}")+ if kind == "histogram" and "bucket_schema" in names:+ raise ValueError("bucket_schema is reserved for the histogram's actual bucket layout")+ binding = self._bindings.get(name)+ if binding is None:+ instrument = None+ if mode == "event":+ if kind == "counter":+ instrument = self._meter.create_counter(name)+ else:+ instrument = self._meter.create_histogram(+ name, explicit_bucket_boundaries_advisory=bounds+ )+ binding = _MetricBinding(kind, names, mode, bounds, instrument)+ self._bindings[name] = binding+ elif (binding.kind, binding.label_names, binding.mode, binding.bounds) != (+ kind,+ names,+ mode,+ bounds,+ ):+ raise ValueError(f"metric {name} type, labels, collection mode or buckets mismatch")+ if normalized not in binding.points and len(binding.points) >= self._max_series:+ self._dropped.add(1, {"metric": name})+ return None, normalized+ return binding, normalized++ def inc_counter(self, name, *, amount=1.0, labels=None, label_names=()) -> None:+ amount = float(amount)+ if not math.isfinite(amount) or amount <= 0:+ raise ValueError("counter increment must be finite and positive")+ with self._lock:+ binding, key = self._bind(name, "counter", label_names, labels, mode="event")+ if binding is not None:+ binding.instrument.add(amount, dict(key))+ binding.points[key] = Nonedef observe_histogram(- self,- name: str,- value: float,- *,- labels: Mapping[str, str] | None = None,- label_names: Sequence[str] = (),- buckets: Sequence[float] = DEFAULT_LATENCY_BUCKETS,+ self, name, value, *, labels=None, label_names=(), buckets=DEFAULT_LATENCY_BUCKETS) -> None:- """- Observe a value into a Histogram metric.-- Args:- name: Prometheus metric name (base name; exporter renders _bucket/_count/_sum).- value: Observation value.- labels: Optional label dict. Keys must exactly match `label_names`.- label_names: Ordered label key tuple for this metric name.- buckets: Histogram bucket boundaries (ascending). "+Inf" bucket is implicit.- """- self.histogram(name, label_names=label_names, buckets=buckets).observe(- float(value), labels=labels- )-- def counter(self, name: str, *, label_names: Sequence[str] = ()) -> "_Counter":- """- Get or create a Counter family handle.-- Args:- name: Prometheus metric name.- label_names: Canonical ordered label keys for the family.-- Returns:- A lightweight `_Counter` wrapper bound to the requested family.- """- ln = _canonicalize_label_names(label_names)- with self._lock:- if name in self._gauges or name in self._histograms:- raise ValueError(f"metric {name} already registered with a different type")- family = self._counters.get(name)- if family is None:- family = _CounterFamily(- name=name,- label_names=ln,- max_series=self._max_series_per_metric,- on_drop=self._on_drop_series,- )- self._counters[name] = family- else:- family._validate_label_names(ln)- return _Counter(family)-- def gauge(self, name: str, *, label_names: Sequence[str] = ()) -> "_Gauge":- """- Get or create a Gauge family handle.-- Args:- name: Prometheus metric name.- label_names: Canonical ordered label keys for the family.-- Returns:- A lightweight `_Gauge` wrapper bound to the requested family.- """- ln = _canonicalize_label_names(label_names)- with self._lock:- if name in self._counters or name in self._histograms:- raise ValueError(f"metric {name} already registered with a different type")- family = self._gauges.get(name)- if family is None:- family = _GaugeFamily(- name=name,- label_names=ln,- max_series=self._max_series_per_metric,- on_drop=self._on_drop_series,- )- self._gauges[name] = family- else:- family._validate_label_names(ln)- return _Gauge(family)-- def histogram(- self,- name: str,- *,- label_names: Sequence[str] = (),- buckets: Sequence[float] = DEFAULT_LATENCY_BUCKETS,- ) -> "_Histogram":- """- Get or create a Histogram family handle.-- Args:- name: Prometheus metric base name.- label_names: Canonical ordered label keys for the family.- buckets: Histogram bucket upper bounds.-- Returns:- A lightweight `_Histogram` wrapper bound to the requested family.- """- ln = _canonicalize_label_names(label_names)- b = tuple(float(x) for x in buckets)- with self._lock:- if name in self._counters or name in self._gauges:- raise ValueError(f"metric {name} already registered with a different type")- family = self._histograms.get(name)- if family is None:- family = _HistogramFamily(- name=name,- label_names=ln,- bucket_bounds=b,- max_series=self._max_series_per_metric,- on_drop=self._on_drop_series,- )- self._histograms[name] = family- else:- family._validate_label_names(ln)- family._validate_buckets(b)- return _Histogram(family)-- def iter_counters(self, *, include_start_time: bool = False):- """- Iterate over all counter families and their current series values.-- The returned family payload is a detached snapshot so exporters can iterate without- holding the registry lock during rendering. When include_start_time is True,- each series also includes its cumulative start time in Unix nanoseconds.- """- with self._lock:- families = dict(self._counters)- for name, family in families.items():- values = family.copy_values()- if values is not None:- yield (- name,- [- (labels, *value) if include_start_time else (labels, value[0])- for labels, value in values.items()- ],- )-- def counter_label_names(self, name: str) -> tuple[str, ...]:- """Return the registered label key tuple for a counter family, if present."""- with self._lock:- family = self._counters.get(name)- return family.label_names if family is not None else ()-- def iter_gauges(self):- """- Iterate over all gauge families and their current series values.-- As with counters, the family payload is copied before iteration so rendering stays- independent from concurrent writers.- """- with self._lock:- families = dict(self._gauges)- for name, family in families.items():- values = family.copy_values()- if values is not None:- yield name, list(values.items())-- def gauge_label_names(self, name: str) -> tuple[str, ...]:- """Return the registered label key tuple for a gauge family, if present."""- with self._lock:- family = self._gauges.get(name)- return family.label_names if family is not None else ()-- def gauge_get(self, name: str, *, labels: Mapping[str, str] | None = None) -> float | None:- """- Read the current value of a gauge series.-- Args:- name: Gauge metric name.- labels: Optional label dict identifying a single series.-- Returns:- The current gauge value, or `None` when the family or series does not exist.- """+ value = float(value)+ bounds = self._bounds(buckets)+ if not math.isfinite(value) or value < 0:+ raise ValueError("histogram observation must be finite and nonnegative")with self._lock:- family = self._gauges.get(name)- if family is None:- return None- return family.get_value(labels=labels)-- def gauge_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:- """- Delete every gauge series whose labels contain the provided subset.-- Args:- name: Gauge metric name.- match_labels: Partial label set used for matching series to remove.- """- with self._lock:- family = self._gauges.get(name)- if family is None:- return- family.delete_matching(match_labels=match_labels)-- def counter_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:- """Delete counter series matching the supplied label subset; return None."""- with self._lock:- family = self._counters.get(name)- if family is not None:- family.delete_matching(match_labels=match_labels)-- def histogram_delete_matching(self, name: str, *, match_labels: Mapping[str, str]) -> None:- """Delete histogram series matching the supplied label subset; return None."""- with self._lock:- family = self._histograms.get(name)- if family is not None:- family.delete_matching(match_labels=match_labels)-- def iter_histograms(self, *, include_start_time: bool = False):- """- Iterate over all histogram families and their materialized series snapshots.-- Each yielded item contains family metadata and a detached list of per-series bucket- counts, sample counts, and sums ready for exporter rendering. When include_start_time- is True, each series also includes its cumulative start time in Unix nanoseconds.- """- with self._lock:- families = dict(self._histograms)- for name, family in families.items():- snapshot = family.copy_series()- if snapshot is not None:- label_names, bucket_bounds, series = snapshot- yield (- name,- label_names,- bucket_bounds,- (series if include_start_time else [values[:-1] for values in series]),- )-- def iter_dropped_series(self):- """- Iterate over metrics that have dropped series because of series-limit enforcement.-- Exporters use this bookkeeping to expose internal pressure signals when a metric family- hits its cardinality cap and starts rejecting additional series.- """- with self._lock:- for metric_name, dropped in self._dropped_series_total.items():- yield metric_name, dropped-- def _on_drop_series(self, metric_name: str) -> None:- """Record that a new series was rejected for the given metric family."""- with self._lock:- self._dropped_series_total[metric_name] = (- self._dropped_series_total.get(metric_name, 0) + 1+ binding, key = self._bind(+ name, "histogram", label_names, labels, mode="event", bounds=bounds)---class _CounterFamily:- """Internal mutable storage for one counter metric family and all of its series."""-- def __init__(- self,- *,- name: str,- label_names: tuple[str, ...],- max_series: int,- on_drop,- ) -> None:- """Create an internal counter family with bounded series cardinality."""- self.name = name- self.label_names = label_names- self._max_series = max_series- self._on_drop = on_drop- self._lock = threading.Lock()- self._values: dict[tuple[tuple[str, str], ...], tuple[float, int]] = {}- self._had_deletions = False-- def inc(self, *, labels: Mapping[str, str] | None, amount: float) -> None:- """- Increase one counter series by a positive amount.-- New series are rejected once the family has reached the configured series limit, but- existing series may continue to grow even after the cap is reached.- """- if amount <= 0:- raise ValueError("counter can only be increased by a positive amount")- key = self._normalize_and_validate(labels)+ if binding is not None:+ binding.instrument.record(value, dict(key))+ binding.points[key] = None++ @staticmethod+ def _bounds(buckets):+ bounds = tuple(float(value) for value in buckets)+ if any(not math.isfinite(value) or value < 0 for value in bounds) or any(+ left >= right for left, right in zip(bounds, bounds[1:], strict=False)+ ):+ raise ValueError("histogram bounds must be finite, nonnegative and increasing")+ return bounds++ def set_counter(self, name, value, *, labels=None, label_names=()) -> None:+ value = float(value)+ if not math.isfinite(value) or value < 0:+ raise ValueError("counter value must be finite and nonnegative")+ self._set_number(name, value, "counter", labels, label_names)++ def set_gauge(self, name, value, *, labels=None, label_names=()) -> None:+ self._set_number(name, float(value), "gauge", labels, label_names)++ def _set_number(self, name, value, kind, labels, label_names):with self._lock:- if key not in self._values and len(self._values) >= self._max_series:- self._on_drop(self.name)+ binding, key = self._bind(name, kind, label_names, labels, mode="snapshot")+ if binding is None:return- value, start_time_ns = self._values.get(key, (0.0, 0))- self._values[key] = (value + float(amount), start_time_ns or time.time_ns())-- def copy_values(self) -> dict[tuple[tuple[str, str], ...], tuple[float, int]] | None:- """Return totals with their start times, or None if deletion emptied this family."""- with self._lock:- if self._had_deletions and not self._values:- return None- return dict(self._values)-- def set(self, *, labels: Mapping[str, str] | None, value: float) -> None:- """Replace labels with value, starting a new cumulative period on decrease; return None."""- value = _validate_counter_value(value)- key = self._normalize_and_validate(labels)- with self._lock:- if key not in self._values and len(self._values) >= self._max_series:- self._on_drop(self.name)- return- previous = self._values.get(key)- start_time_ns = (- previous[1] if previous is not None and value >= previous[0] else time.time_ns()- )- self._values[key] = (value, start_time_ns)-- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:- """Remove counter entries containing the supplied labels; return None."""- normalized = normalize_labels(match_labels)- with self._lock:- for key in list(self._values):- if _labels_contains(key, normalized):- del self._values[key]- self._had_deletions = True-- def _normalize_and_validate(- self, labels: Mapping[str, str] | None- ) -> tuple[tuple[str, str], ...]:- """Normalize label input and enforce the family label contract."""- normalized = normalize_labels(labels)- self._validate_label_names_against(normalized)- return normalized-- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:- """Ensure subsequent family lookups use the original declared label keys."""- if self.label_names != label_names:- raise ValueError(- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"- )-- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:- """Ensure a concrete series write uses exactly the configured label keys."""- _validate_label_keys(self.name, self.label_names, normalized)---class _GaugeFamily:- """Internal mutable storage for one gauge metric family and all of its series."""+ now = time.time_ns()+ previous = binding.points.get(key)+ start = now+ if previous is not None and (kind == "gauge" or value >= previous.value):+ start = previous.start_time_unix_nano+ binding.points[key] = NumberDataPoint(dict(key), start, now, value)- def __init__(- self,- *,- name: str,- label_names: tuple[str, ...],- max_series: int,- on_drop,+ def set_histogram(+ self, name, *, bucket_bounds, bucket_counts, count, value_sum, labels=None, label_names=()) -> None:- """Create an internal gauge family with bounded series cardinality."""- self.name = name- self.label_names = label_names- self._max_series = max_series- self._on_drop = on_drop- self._lock = threading.Lock()- self._values: dict[tuple[tuple[str, str], ...], float] = {}- self._had_deletions = False-- def set(self, *, labels: Mapping[str, str] | None, value: float) -> None:- """- Set one gauge series to an absolute value.-- As with counters, creating a brand-new series is still subject to the family-level- cardinality cap.- """- key = self._normalize_and_validate(labels)+ """Accept a native histogram validated at the RagfsMetricDataSource boundary."""+ bounds = tuple(bucket_bounds)+ counts = tuple(bucket_counts)with self._lock:- if key not in self._values and len(self._values) >= self._max_series:- self._on_drop(self.name)- return- self._values[key] = float(value)-- def add(self, *, labels: Mapping[str, str] | None, delta: float) -> None:- """- Add a signed delta to one gauge series.-- This is the internal primitive behind the public gauge `inc(...)` and `dec(...)`- helpers.- """- key = self._normalize_and_validate(labels)- with self._lock:- if key not in self._values and len(self._values) >= self._max_series:- self._on_drop(self.name)+ binding, key = self._bind(+ name, "histogram", label_names, labels, mode="snapshot", bounds=bounds+ )+ if binding is None:return- self._values[key] = self._values.get(key, 0.0) + float(delta)-- def copy_values(self) -> dict[tuple[tuple[str, str], ...], float] | None:- """Return a gauge value snapshot, or None if deletion emptied this family."""- with self._lock:- if self._had_deletions and not self._values:- return None- return dict(self._values)-- def get_value(self, *, labels: Mapping[str, str] | None) -> float | None:- """Return the current value for one normalized gauge series, if that series exists."""- key = self._normalize_and_validate(labels)- with self._lock:- return self._values.get(key)-- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:- """Delete every stored gauge series that contains the requested label subset."""- normalized = normalize_labels(match_labels)- with self._lock:- keys = list(self._values.keys())- for k in keys:- if _labels_contains(k, normalized):- self._values.pop(k, None)- self._had_deletions = True-- def _normalize_and_validate(- self, labels: Mapping[str, str] | None- ) -> tuple[tuple[str, str], ...]:- """Normalize label input and enforce the family label contract."""- normalized = normalize_labels(labels)- self._validate_label_names_against(normalized)- return normalized-- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:- """Ensure subsequent family lookups use the original declared label keys."""- if self.label_names != label_names:- raise ValueError(- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"+ now = time.time_ns()+ previous = binding.points.get(key)+ start = now+ if (+ previous is not None+ and count >= previous.count+ and value_sum >= previous.sum+ and all(+ value >= old for value, old in zip(counts, previous.bucket_counts, strict=True)+ )+ ):+ start = previous.start_time_unix_nano+ # Native snapshots do not contain min/max; None omits these optional OTLP fields.+ binding.points[key] = HistogramDataPoint(+ attributes=dict(key),+ start_time_unix_nano=start,+ time_unix_nano=now,+ count=count,+ sum=value_sum,+ bucket_counts=counts,+ explicit_bounds=bounds,+ min=None,+ max=None,)- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:- """Ensure a concrete series write uses exactly the configured label keys."""- if not self.label_names and normalized:- raise ValueError(f"metric {self.name} does not accept labels")- if self.label_names and tuple(k for k, _ in normalized) != self.label_names:- raise ValueError(f"metric {self.name} label keys mismatch: expected {self.label_names}")---class _HistogramFamily:- """Internal mutable storage for one histogram metric family and all of its label series."""-- def __init__(- self,- *,- name: str,- label_names: tuple[str, ...],- bucket_bounds: tuple[float, ...],- max_series: int,- on_drop,- ) -> None:- """Create an internal histogram family with fixed buckets and bounded series count."""- self.name = name- self.label_names = label_names- self.bucket_bounds = bucket_bounds- self._max_series = max_series- self._on_drop = on_drop- self._lock = threading.Lock()- self._series: dict[tuple[tuple[str, str], ...], _HistogramSeries] = {}- self._had_deletions = False-- def observe(self, *, labels: Mapping[str, str] | None, value: float) -> None:- """- Record one histogram observation for the selected label series.-- A new series is materialized lazily on first use, subject to the same family-level- series cap used by counters and gauges.- """- key = self._normalize_and_validate(labels)- with self._lock:- series = self._series.get(key)- if series is None:- if len(self._series) >= self._max_series:- self._on_drop(self.name)- return- series = _HistogramSeries(bucket_bounds=self.bucket_bounds)- self._series[key] = series- series.observe(float(value))-- def set(- self,- *,- labels: Mapping[str, str] | None,- bucket_counts: Sequence[int],- count: int,- value_sum: float,- ) -> None:- """Validate bucket_counts, count and value_sum before replacing labels; return None."""- bucket_counts, value_sum = _validate_histogram_values(- self.bucket_bounds, bucket_counts, count, value_sum- )- key = self._normalize_and_validate(labels)+ def gauge_get(self, name, *, labels=None):with self._lock:- series = self._series.get(key)- if series is None:- if len(self._series) >= self._max_series:- self._on_drop(self.name)- return- series = _HistogramSeries(bucket_bounds=self.bucket_bounds)- self._series[key] = series- series.set_values(bucket_counts, count, value_sum)-- def delete_matching(self, *, match_labels: Mapping[str, str]) -> None:- """Remove histogram entries containing the supplied labels; return None."""- normalized = normalize_labels(match_labels)- with self._lock:- for key in list(self._series):- if _labels_contains(key, normalized):- del self._series[key]- self._had_deletions = True-- def copy_series(- self,- ) -> (- tuple[- tuple[str, ...],- tuple[float, ...],- list[tuple[tuple[tuple[str, str], ...], tuple[int, ...], int, float, int]],- ]- | None- ):- """Return histogram values and start times, or None if deletion emptied this family."""- with self._lock:- if self._had_deletions and not self._series:+ binding = self._bindings.get(name)+ if binding is None or binding.kind != "gauge":return None- series: list[tuple[tuple[tuple[str, str], ...], tuple[int, ...], int, float, int]] = []- for labels, s in self._series.items():- series.append((labels, *s.copy_values()))- return self.label_names, self.bucket_bounds, series-- def _normalize_and_validate(- self, labels: Mapping[str, str] | None- ) -> tuple[tuple[str, str], ...]:- """Normalize label input and enforce the family label contract."""- normalized = normalize_labels(labels)- self._validate_label_names_against(normalized)- return normalized+ point = binding.points.get(normalize_labels(labels))+ return None if point is None else point.value- def _validate_label_names(self, label_names: tuple[str, ...]) -> None:- """Ensure subsequent family lookups use the original declared label keys."""- if self.label_names != label_names:- raise ValueError(- f"metric {self.name} label_names mismatch: {self.label_names} vs {label_names}"- )-- def _validate_label_names_against(self, normalized: tuple[tuple[str, str], ...]) -> None:- """Ensure a concrete series write uses exactly the configured label keys."""- _validate_label_keys(self.name, self.label_names, normalized)-- def _validate_buckets(self, bucket_bounds: tuple[float, ...]) -> None:- """Ensure a histogram family is never reopened with a different bucket layout."""- if self.bucket_bounds != bucket_bounds:- raise ValueError(f"metric {self.name} buckets mismatch")---class _HistogramSeries:- """Internal bucket/count/sum accumulator for one concrete histogram label series."""-- def __init__(self, *, bucket_bounds: tuple[float, ...]) -> None:- """Create one histogram series with counters for all finite buckets plus `+Inf`."""- self._bucket_bounds = bucket_bounds- self._lock = threading.Lock()- self._bucket_counts: list[int] = [0] * (len(bucket_bounds) + 1)- self._count: int = 0- self._sum: float = 0.0- self._start_time_ns = time.time_ns()-- def observe(self, value: float) -> None:- """- Record a single observation into the appropriate bucket and aggregate summary totals.-- Bucket selection uses `bisect_left`, so each observation increments the first bucket- whose upper bound is greater than or equal to the recorded value.- """- idx = bisect_left(self._bucket_bounds, value)+ def delete_matching(self, name, *, match_labels: Mapping[str, str]) -> None:+ """Remove matching state snapshots; the registered metric owns its type."""+ match = set(normalize_labels(match_labels))with self._lock:- self._bucket_counts[idx] += 1- self._count += 1- self._sum += value-- def copy_values(self) -> tuple[tuple[int, ...], int, float, int]:- """Return bucket counts, sample count, sum and cumulative start time under one lock."""- with self._lock:- return tuple(self._bucket_counts), self._count, self._sum, self._start_time_ns-- def set_values(self, bucket_counts: tuple[int, ...], count: int, value_sum: float) -> None:- """Replace validated totals, starting a new period if any total decreases; return None."""+ binding = self._bindings.get(name)+ if binding is None:+ return+ if binding.mode != "snapshot":+ raise ValueError(f"metric {name} is not a state snapshot")+ for key in list(binding.points):+ if match.issubset(key):+ del binding.points[key]++ def collect(self, *, service_name="openviking-server") -> MetricsData:+ """Return SDK event aggregates and current native/state snapshots in one standard batch."""with self._lock:- if (- count < self._count- or value_sum < self._sum- or any(- current < previous- for current, previous in zip(bucket_counts, self._bucket_counts, strict=True)+ data = self._reader.get_metrics_data()+ metrics = [+ metric+ for resource in (() if data is None else data.resource_metrics)+ for scope in resource.scope_metrics+ for metric in scope.metrics+ ]+ now = time.time_ns()+ for name, binding in self._bindings.items():+ if binding.mode != "snapshot" or not binding.points:+ continue+ points = tuple(+ replace(point, time_unix_nano=now) for point in binding.points.values())- ):- self._start_time_ns = time.time_ns()- self._bucket_counts = list(bucket_counts)- self._count = count- self._sum = value_sum---class _Counter:- """Public lightweight handle used by callers to mutate one counter family."""-- def __init__(self, family: _CounterFamily) -> None:- """Bind a public counter handle to one internal counter family."""- self._family = family-- def inc(self, amount: float = 1.0, *, labels: Mapping[str, str] | None = None) -> None:- """Increment one series in the bound counter family using the public wrapper API."""- self._family.inc(labels=labels, amount=amount)-- def set(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:- """Replace the labelled series with a validated absolute value; return None."""- self._family.set(labels=labels, value=value)---class _Gauge:- """Public lightweight handle used by callers to mutate one gauge family."""-- def __init__(self, family: _GaugeFamily) -> None:- """Bind a public gauge handle to one internal gauge family."""- self._family = family-- def set(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:- """Set one series in the bound gauge family through the public wrapper API."""- self._family.set(labels=labels, value=value)-- def inc(self, amount: float = 1.0, *, labels: Mapping[str, str] | None = None) -> None:- """Increase one series in the bound gauge family by a positive delta."""- self._family.add(labels=labels, delta=amount)---class _Histogram:- """Public lightweight handle used by callers to mutate one histogram family."""-- def __init__(self, family: _HistogramFamily) -> None:- """Bind a public histogram handle to one internal histogram family."""- self._family = family-- def observe(self, value: float, *, labels: Mapping[str, str] | None = None) -> None:- """Record one observation for a series in the bound histogram family wrapper."""- self._family.observe(labels=labels, value=value)-- def set(- self,- bucket_counts: Sequence[int],- count: int,- value_sum: float,- *,- labels: Mapping[str, str] | None = None,- ) -> None:- """Replace the labelled series using validated disjoint counts and totals; return None."""- self._family.set(- labels=labels, bucket_counts=bucket_counts, count=count, value_sum=value_sum+ if binding.kind == "counter":+ aggregate = Sum(points, AggregationTemporality.CUMULATIVE, True)+ elif binding.kind == "gauge":+ aggregate = Gauge(points)+ else:+ aggregate = Histogram(points, AggregationTemporality.CUMULATIVE)+ metrics.append(Metric(name, "OpenViking metric.", "", aggregate))+ # Use the actual SDK boundaries, including boundaries from the Rust SDK.+ for index, metric in enumerate(metrics):+ if isinstance(metric.data, Histogram) and metric.data.data_points:+ schema = histogram_schema(tuple(metric.data.data_points[0].explicit_bounds))+ points = tuple(+ replace(+ point,+ attributes={+ **point.attributes,+ "bucket_schema": schema,+ },+ )+ for point in metric.data.data_points+ )+ metrics[index] = replace(metric, data=replace(metric.data, data_points=points))+ return MetricsData(+ (+ ResourceMetrics(+ resource=Resource({"service.name": service_name}),+ scope_metrics=(+ ScopeMetrics(+ scope=InstrumentationScope("openviking.metrics"),+ metrics=tuple(sorted(metrics, key=lambda metric: metric.name)),+ schema_url="",+ ),+ ),+ schema_url="",+ ),+ ))diff --git a/openviking/metrics/core/types.py b/openviking/metrics/core/types.pyindex a2b48c8c2..adba96595 100644--- a/openviking/metrics/core/types.py+++ b/openviking/metrics/core/types.py@@ -16,8 +16,9 @@ All labels are normalized into a sorted tuple of (key, value) pairs to provide:from __future__ import annotations+import hashlibimport re-from dataclasses import dataclass+import structfrom typing import Mapping_LABEL_KEY_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")@@ -34,6 +35,11 @@ FORBIDDEN_LABEL_KEYS: set[str] = {}+def histogram_schema(bounds: tuple[float, ...]) -> str:+ """Identify the actual bucket layout so rolling upgrades cannot mix layouts."""+ return hashlib.sha256(struct.pack(f"!{len(bounds)}d", *bounds)).hexdigest()[:16]++def normalize_labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str], ...]:"""Normalize a label mapping into the canonical internal representation.@@ -61,40 +67,3 @@ def normalize_labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str],items.append((key, str(v)))items.sort(key=lambda x: x[0])return tuple(items)---def render_labels(normalized: tuple[tuple[str, str], ...]) -> str:- """- Render normalized labels into Prometheus exposition syntax.-- Args:- normalized: Canonical normalized label tuple.-- Returns:- An empty string when no labels are present, or a `{k="v",...}` block ready for text- exposition output.- """- if not normalized:- return ""- parts = [f'{k}="{_escape_label_value(v)}"' for k, v in normalized]- return "{" + ",".join(parts) + "}"---def _escape_label_value(value: str) -> str:- """Escape a label value according to Prometheus text exposition rules."""- return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')---@dataclass(frozen=True, slots=True)-class MetricSeriesId:- """- Immutable identifier for a single metric series in the in-process registry.-- Attributes:- name: Metric family name.- labels: Canonical normalized label tuple uniquely identifying one concrete series within- that metric family.- """-- name: str- labels: tuple[tuple[str, str], ...]diff --git a/openviking/metrics/datasources/ragfs.py b/openviking/metrics/datasources/ragfs.pyindex 1501b18e6..6182f0cf1 100644--- a/openviking/metrics/datasources/ragfs.py+++ b/openviking/metrics/datasources/ragfs.py@@ -102,7 +102,7 @@ class RagfsMetricDataSource(DomainStatsMetricDataSource):or not _LABEL.fullmatch(key)or key.startswith("__")or key in _FORBIDDEN_LABELS- or (kind == "histogram" and key == "le")+ or (kind == "histogram" and key in {"le", "bucket_schema"})for key, value in labels.items()):raise ValueError("native metric labels are invalid or forbidden")diff --git a/openviking/metrics/exporters/otel.py b/openviking/metrics/exporters/otel.pyindex 936e2e023..e6ef811fc 100644--- a/openviking/metrics/exporters/otel.py+++ b/openviking/metrics/exporters/otel.py@@ -1,728 +1,129 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""-OpenTelemetry OTLP metrics exporter implementation.--This exporter serializes the in-process MetricRegistry directly into OTLP protobuf-messages and sends them to an OTLP endpoint over HTTP or gRPC.--Key design goals:-- Export the same MetricRegistry snapshot semantics that PrometheusExporter renders-- Preserve histogram bucket/count/sum semantics without re-aggregation-- Keep exporter lifecycle self-contained through start()/shutdown()-- Refresh collectors before export so snapshot-based metrics stay aligned-"""+"""Refresh OpenViking state and send SDK metric data through official OTLP exporters."""from __future__ import annotationsimport asyncio-import time-from typing import Any, Iterable, Optional, Tuple-import requests+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (+ OTLPMetricExporter as GrpcExporter,+)+from opentelemetry.exporter.otlp.proto.http.metric_exporter import (+ OTLPMetricExporter as HttpExporter,+)from openviking.metrics.core.base import MetricExporterfrom openviking_cli.utils import get_loggerlogger = get_logger(__name__)-try:- import grpc-except ImportError:- grpc = None--try:- from requests import RequestException-except Exception:- RequestException = Exception--try:- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (- ExportMetricsServiceRequest,- )- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2_grpc import (- MetricsServiceStub,- )- from opentelemetry.proto.common.v1.common_pb2 import AnyValue, InstrumentationScope, KeyValue- from opentelemetry.proto.metrics.v1.metrics_pb2 import (- AGGREGATION_TEMPORALITY_CUMULATIVE,- Gauge,- Histogram,- HistogramDataPoint,- Metric,- NumberDataPoint,- Sum,- )- from opentelemetry.proto.resource.v1.resource_pb2 import Resource-except ImportError:- ExportMetricsServiceRequest = None- MetricsServiceStub = None- AnyValue = None- InstrumentationScope = None- KeyValue = None- Gauge = None- Histogram = None- HistogramDataPoint = None- Metric = None- NumberDataPoint = None- Sum = None- Resource = None- AGGREGATION_TEMPORALITY_CUMULATIVE = None---def _compute_next_tick_and_sleep(- *,- now: float,- interval_s: float,- next_tick: float,-) -> tuple[float, float]:- """- Compute next tick time and sleep duration for a fixed-interval loop.-- This is a pure function to keep scheduling behavior testable without adding- production-only hooks.-- Rules:- - If the loop overruns (now >= next_tick), advance next_tick by interval until it is in the future.- - Sleep only until the next_tick.-- Args:- now: Current monotonic time in seconds.- interval_s: Target loop interval in seconds.- next_tick: Previously scheduled tick time in seconds.-- Returns:- (next_tick, sleep_seconds)- """- while next_tick <= now:- next_tick += interval_s- return next_tick, max(0.0, next_tick - now)-class OTelMetricExporter(MetricExporter):- """- Export MetricRegistry snapshots to OTLP metrics.-- This exporter intentionally avoids OTel SDK metric aggregation. Instead, it- periodically snapshots MetricRegistry, converts the snapshot into OTLP protobuf- payloads, and pushes them to the configured collector endpoint.- """+ """Own the application refresh cycle; the SDK owns encoding, transport and retries."""def __init__(self,*,registry,collector_manager=None,- refresh_deadline_seconds: float = 1.0,- protocol: str = "grpc",- insecure: bool = False,- endpoint: str = "localhost:4317",- service_name: str = "openviking-server",- export_interval_ms: int = 10000,- export_timeout_seconds: Optional[float] = None,- headers: Optional[dict[str, str]] = None,- enabled: bool = True,- ) -> None:- """- Initialize the OTLP metrics exporter.-- Args:- registry: The in-process metric registry backing all exporters.- collector_manager: Optional collector refresh orchestrator.- refresh_deadline_seconds: Best-effort refresh timeout before each export.- protocol: Transport protocol, either "grpc" or "http".- insecure: Whether gRPC transport should use plaintext.- endpoint: OTLP collector endpoint.- service_name: Service name written into OTLP resource attributes.- export_interval_ms: Periodic export interval in milliseconds.- export_timeout_seconds: Transport timeout for one OTLP export request.- headers: Additional OTLP exporter headers for vendor-specific auth.- enabled: Whether the exporter is enabled.- """+ refresh_deadline_seconds=1.0,+ protocol="grpc",+ insecure=False,+ endpoint="localhost:4317",+ service_name="openviking-server",+ export_interval_ms=10000,+ export_timeout_seconds=None,+ headers=None,+ ):self._registry = registryself._collector_manager = collector_managerself._refresh_deadline_seconds = float(refresh_deadline_seconds)- self._protocol = protocol.lower()- self._insecure = bool(insecure)- self._endpoint = endpointself._service_name = service_name- self._headers = {str(key): str(value) for key, value in (headers or {}).items()}- self._export_interval_ms = max(1000, int(export_interval_ms))- self._export_timeout_seconds = float(+ self._interval = max(1000, int(export_interval_ms)) / 1000+ self._timeout = float(export_timeout_secondsif export_timeout_seconds is not Noneelse max(5.0, self._refresh_deadline_seconds))- self._enabled = bool(enabled)- self._export_task: Optional[asyncio.Task[Any]] = None- self._http_session: Optional[requests.Session] = None- self._grpc_channel: Optional[Any] = None- self._grpc_stub: Optional[Any] = None- self._start_time_ns = time.time_ns()-- if self._protocol not in {"grpc", "http"}:- logger.warning(- "[OTelMetricExporter] invalid protocol=%s, defaulting to grpc",- self._protocol,- )- self._protocol = "grpc"-- if self._enabled:- self._init_transport()-- def _init_transport(self) -> None:- """- Initialize the transport client for the configured OTLP protocol.-- The exporter keeps the transport client alive across export cycles so the- periodic loop only pays the cost of serializing current registry snapshots.- """- if not self._protobuf_available():- logger.warning(- "[OTelMetricExporter] initialization failed: OTLP metrics protobuf dependencies are not available"- )- self._enabled = False- return+ self._closed = False+ self._export_task = None+ self._export_lock = asyncio.Lock()+ if protocol not in ("grpc", "http"):+ raise ValueError(f"unsupported OTLP protocol: {protocol}")+ options = {"endpoint": endpoint, "headers": headers, "timeout": self._timeout}+ if protocol == "http":+ self._sdk = HttpExporter(**options)+ else:+ self._sdk = GrpcExporter(**options, insecure=insecure)- try:- if self._protocol == "http":- self._validate_http_endpoint(self._endpoint)- self._http_session = requests.Session()- else:- if grpc is None or MetricsServiceStub is None:- raise ImportError("gRPC OTLP metrics dependencies are not available")- if self._insecure:- self._grpc_channel = grpc.insecure_channel(self._endpoint)- else:- self._grpc_channel = grpc.secure_channel(- self._endpoint,- grpc.ssl_channel_credentials(),+ async def export(self) -> str:+ async with self._export_lock:+ if self._closed:+ return ""+ if self._collector_manager is not None:+ try:+ await asyncio.wait_for(+ self._collector_manager.refresh_all(+ self._registry, deadline_seconds=self._refresh_deadline_seconds+ ),+ timeout=self._refresh_deadline_seconds,)- self._grpc_stub = MetricsServiceStub(self._grpc_channel)-- logger.info(- "[OTelMetricExporter] initialized: phase=init_transport protocol=%s endpoint=%s service_name=%s export_interval_ms=%s export_timeout_seconds=%s",- self._protocol,- self._endpoint,- self._service_name,- self._export_interval_ms,- self._export_timeout_seconds,- )- except Exception as exc:- logger.warning(- "[OTelMetricExporter] initialization failed: phase=init_transport protocol=%s endpoint=%s service_name=%s export_timeout_seconds=%s error=%s",- self._protocol,- self._endpoint,- self._service_name,- self._export_timeout_seconds,- exc,- )- self._enabled = False-- def _protobuf_available(self) -> bool:- """Return whether all required OTLP protobuf symbols are available."""- return all(- symbol is not None- for symbol in (- ExportMetricsServiceRequest,- AnyValue,- InstrumentationScope,- KeyValue,- Metric,- NumberDataPoint,- HistogramDataPoint,- Gauge,- Sum,- Histogram,- Resource,- AGGREGATION_TEMPORALITY_CUMULATIVE,- )- )-- def _validate_http_endpoint(self, endpoint: str) -> None:- """- Validate that an OTLP/HTTP endpoint includes a URL scheme and path.-- Args:- endpoint: OTLP/HTTP metrics endpoint.- """- if not endpoint.startswith("http://") and not endpoint.startswith("https://"):- raise ValueError(- "OTLP/HTTP endpoint must include scheme, e.g. 'http://localhost:4318/v1/metrics'"+ except Exception as exc:+ logger.warning("OTLP metric source refresh failed: %s", exc)+ data = self._registry.collect(service_name=self._service_name)+ if not any(+ scope.metrics+ for resource in data.resource_metrics+ for scope in resource.scope_metrics+ ):+ return ""+ sending = asyncio.create_task(+ asyncio.to_thread(self._sdk.export, data, timeout_millis=self._timeout * 1000))-- async def export(self) -> str:- """- Refresh collectors and export the current MetricRegistry snapshot.-- Returns:- An empty string to preserve the MetricExporter interface contract.- """- if not self._enabled:- return ""-- await self._refresh_collectors()- request = self._build_export_request()- if request is None:- return ""-- await self._send_request(request)+ try:+ await asyncio.shield(sending)+ except asyncio.CancelledError:+ # Closing the SDK transport must wait for the in-flight blocking export.+ await sending+ raisereturn ""- async def _refresh_collectors(self) -> None:- """- Best-effort refresh of snapshot-driven collectors before serialization.-- Export must never break application flow, so refresh failures are swallowed.- """- if self._collector_manager is None:- return- started = time.monotonic()- try:- await asyncio.wait_for(- self._collector_manager.refresh_all(- self._registry,- deadline_seconds=self._refresh_deadline_seconds,- ),- timeout=self._refresh_deadline_seconds,- )- except asyncio.TimeoutError as exc:- elapsed_ms = int((time.monotonic() - started) * 1000)- logger.warning(- "[OTelMetricExporter] collector refresh failed: phase=refresh_collectors reason=timeout protocol=%s endpoint=%s deadline_seconds=%s elapsed_ms=%s error=%s",- self._protocol,- self._endpoint,- self._refresh_deadline_seconds,- elapsed_ms,- exc,- )- except Exception as exc:- elapsed_ms = int((time.monotonic() - started) * 1000)- logger.warning(- "[OTelMetricExporter] collector refresh failed: phase=refresh_collectors reason=exception protocol=%s endpoint=%s deadline_seconds=%s elapsed_ms=%s error=%s",- self._protocol,- self._endpoint,- self._refresh_deadline_seconds,- elapsed_ms,- exc,- )-- def _build_export_request(self) -> Optional[ExportMetricsServiceRequest]:- """- Build an OTLP ExportMetricsServiceRequest from the current registry snapshot.-- Returns:- A populated OTLP export request, or None when there is no metric family to send.- """- if not self._protobuf_available():- return None-- now_ns = time.time_ns()- request = ExportMetricsServiceRequest()- resource_metrics = request.resource_metrics.add()- resource_metrics.resource.CopyFrom(self._build_resource())- scope_metrics = resource_metrics.scope_metrics.add()- scope_metrics.scope.CopyFrom(- InstrumentationScope(name="openviking.metrics.exporter.otel", version="1")- )-- self._append_counter_metrics(scope_metrics.metrics, now_ns)- self._append_gauge_metrics(scope_metrics.metrics, now_ns)- self._append_histogram_metrics(scope_metrics.metrics, now_ns)- self._append_dropped_series_metrics(scope_metrics.metrics, now_ns)-- if not scope_metrics.metrics:- return None- return request-- def _build_resource(self) -> Resource:- """- Build the OTLP resource envelope for this exporter.-- Returns:- A Resource message containing the service name.- """- resource = Resource()- resource.attributes.append(- KeyValue(key="service.name", value=AnyValue(string_value=self._service_name))- )- return resource-- def _append_counter_metrics(self, metrics, now_ns: int) -> None:- """- Append all counter families from MetricRegistry as cumulative OTLP sums.-- Empty unlabeled counters emit a single zero data point to match PrometheusExporter.- """- for name, counter_series in self._registry.iter_counters(include_start_time=True):- metric = metrics.add()- metric.name = name- metric.description = "OpenViking metric."- metric.sum.CopyFrom(self._build_sum(counter_series, now_ns, name=name))-- def _append_gauge_metrics(self, metrics, now_ns: int) -> None:- """- Append all gauge families from MetricRegistry as OTLP gauges.-- Empty unlabeled gauges emit a single zero data point to match PrometheusExporter.- """- for name, gauge_series in self._registry.iter_gauges():- metric = metrics.add()- metric.name = name- metric.description = "OpenViking metric."- metric.gauge.CopyFrom(self._build_gauge(gauge_series, now_ns, name=name))-- def _append_histogram_metrics(self, metrics, now_ns: int) -> None:- """- Append all histogram families from MetricRegistry as cumulative OTLP histograms.-- Bucket counts and explicit bounds are copied directly from the registry snapshot.- """- for name, label_names, bucket_bounds, series_iter in self._registry.iter_histograms(- include_start_time=True- ):- series_list = list(series_iter)- metric = metrics.add()- metric.name = name- metric.description = "OpenViking metric."- metric.histogram.CopyFrom(- self._build_histogram(- label_names=label_names,- bucket_bounds=bucket_bounds,- series_list=series_list,- now_ns=now_ns,- )- )-- def _append_dropped_series_metrics(self, metrics, now_ns: int) -> None:- """- Append dropped-series diagnostics as a cumulative OTLP sum family.-- This mirrors the `openviking_metrics_dropped_series_total` metric emitted by- PrometheusExporter.- """- dropped_points: list[tuple[tuple[tuple[str, str], ...], float, int]] = []- for metric_name, dropped in self._registry.iter_dropped_series():- dropped_points.append(((("metric", metric_name),), float(dropped), self._start_time_ns))-- if not dropped_points:- return-- metric = metrics.add()- metric.name = "openviking_metrics_dropped_series_total"- metric.description = "OpenViking metric."- metric.sum.CopyFrom(self._build_sum(dropped_points, now_ns, name=metric.name))-- def _build_sum(self, series, now_ns: int, *, name: str) -> Sum:- """- Build an OTLP Sum message from a counter-like registry family.-- Args:- series: List of `(labels, value, start_time_ns)` tuples from MetricRegistry.- now_ns: Export timestamp in nanoseconds.- name: Metric family name for zero-series handling.- """- sum_metric = Sum(- aggregation_temporality=AGGREGATION_TEMPORALITY_CUMULATIVE,- is_monotonic=True,- )-- if not series and not self._registry.counter_label_names(name):- point = NumberDataPoint(- start_time_unix_nano=self._start_time_ns,- time_unix_nano=now_ns,- as_int=0,- )- sum_metric.data_points.append(point)- return sum_metric-- for labels, value, start_time_ns in series:- point = NumberDataPoint(- start_time_unix_nano=start_time_ns,- time_unix_nano=max(now_ns, start_time_ns),- )- point.attributes.extend(self._build_attributes(labels))- if float(value).is_integer() and -(2**63) <= value < 2**63:- point.as_int = int(value)- else:- point.as_double = float(value)- sum_metric.data_points.append(point)- return sum_metric-- def _build_gauge(self, series, now_ns: int, *, name: str) -> Gauge:- """- Build an OTLP Gauge message from a registry gauge family.-- Args:- series: List of `(labels, value)` tuples from MetricRegistry.- now_ns: Export timestamp in nanoseconds.- name: Metric family name for zero-series handling.- """- gauge_metric = Gauge()-- if not series and not self._registry.gauge_label_names(name):- point = NumberDataPoint(time_unix_nano=now_ns, as_int=0)- gauge_metric.data_points.append(point)- return gauge_metric-- for labels, value in series:- point = NumberDataPoint(time_unix_nano=now_ns)- point.attributes.extend(self._build_attributes(labels))- if float(value).is_integer() and -(2**63) <= value < 2**63:- point.as_int = int(value)- else:- point.as_double = float(value)- gauge_metric.data_points.append(point)- return gauge_metric-- def _build_histogram(- self,- *,- label_names: tuple[str, ...],- bucket_bounds: tuple[float, ...],- series_list,- now_ns: int,- ) -> Histogram:- """- Build an OTLP Histogram message from a registry histogram family.-- Args:- label_names: Registered label keys for the family.- bucket_bounds: Explicit bucket upper bounds from the registry.- series_list: Registry series with bucket counts, totals and start_time_ns.- now_ns: Export timestamp in nanoseconds.- """- histogram_metric = Histogram(- aggregation_temporality=AGGREGATION_TEMPORALITY_CUMULATIVE,- )-- if not series_list and not label_names:- point = HistogramDataPoint(- start_time_unix_nano=self._start_time_ns,- time_unix_nano=now_ns,- count=0,- sum=0.0,- )- point.bucket_counts.extend((0,) * (len(bucket_bounds) + 1))- point.explicit_bounds.extend(bucket_bounds)- histogram_metric.data_points.append(point)- return histogram_metric-- for labels, bucket_counts, count, value_sum, start_time_ns in series_list:- point = HistogramDataPoint(- start_time_unix_nano=start_time_ns,- time_unix_nano=max(now_ns, start_time_ns),- count=int(count),- sum=float(value_sum),- )- point.attributes.extend(self._build_attributes(labels))- point.bucket_counts.extend(int(v) for v in bucket_counts)- point.explicit_bounds.extend(float(v) for v in bucket_bounds)- histogram_metric.data_points.append(point)- return histogram_metric-- def _build_attributes(self, labels: Iterable[Tuple[str, str]]) -> list[KeyValue]:- """- Convert MetricRegistry labels into OTLP KeyValue attributes.-- Args:- labels: Iterable of sorted `(key, value)` label pairs.- """- return [KeyValue(key=key, value=AnyValue(string_value=value)) for key, value in labels]-- async def _send_request(self, request: ExportMetricsServiceRequest) -> None:- """- Send one OTLP metrics export request using the configured transport.-- Args:- request: The populated OTLP metrics export payload.- """- metric_count = self._count_otlp_metrics(request)- request_bytes = request.ByteSize()- try:- if self._protocol == "http":- await asyncio.to_thread(self._send_http_request, request)- else:- await asyncio.to_thread(self._send_grpc_request, request)- except (RequestException, OSError, TimeoutError) as exc:- logger.warning(- "[OTelMetricExporter] export failed: phase=send_request protocol=%s endpoint=%s timeout_seconds=%s metric_count=%s request_bytes=%s error_type=%s error=%s",- self._protocol,- self._endpoint,- self._export_timeout_seconds,- metric_count,- request_bytes,- type(exc).__name__,- exc,- )- except Exception as exc:- # Keep exporter failure best-effort. Log unexpected failures at warning level.- logger.warning(- "[OTelMetricExporter] export failed: phase=send_request protocol=%s endpoint=%s timeout_seconds=%s metric_count=%s request_bytes=%s error_type=%s error=%s",- self._protocol,- self._endpoint,- self._export_timeout_seconds,- metric_count,- request_bytes,- type(exc).__name__,- exc,- )-- def _count_otlp_metrics(self, request: ExportMetricsServiceRequest) -> int:- """- Count OTLP metric families in one export request for logging purposes.-- Args:- request: The populated OTLP metrics export payload.-- Returns:- Number of OTLP Metric messages in the request.- """- count = 0- for resource_metrics in request.resource_metrics:- for scope_metrics in resource_metrics.scope_metrics:- count += len(scope_metrics.metrics)- return count-- def _send_http_request(self, request: ExportMetricsServiceRequest) -> None:- """- Send one OTLP/HTTP metrics request.-- Args:- request: The populated OTLP metrics export payload.- """- if self._http_session is None:- return- payload = request.SerializeToString()- request_headers = {"Content-Type": "application/x-protobuf"}- if self._headers:- request_headers.update(self._headers)- self._http_session.post(- self._endpoint,- data=payload,- headers=request_headers,- timeout=self._export_timeout_seconds,- ).raise_for_status()-- def _send_grpc_request(self, request: ExportMetricsServiceRequest) -> None:- """- Send one OTLP/gRPC metrics request.-- Args:- request: The populated OTLP metrics export payload.- """- if self._grpc_stub is None:- return- metadata = list(self._headers.items()) if self._headers else None- if metadata:- self._grpc_stub.Export(- request,- timeout=self._export_timeout_seconds,- metadata=metadata,- )- else:- self._grpc_stub.Export(request, timeout=self._export_timeout_seconds)-def start(self) -> None:- """- Start the exporter-owned periodic background loop.-- The exporter keeps its own lifecycle so application startup can create the- exporter once and let it handle periodic exports internally.- """- if not self._enabled:- return- if self._export_task is not None and not self._export_task.done():+ if self._closed or self._export_task is not None:returntry:- self._export_task = asyncio.create_task(- self._run_export_loop(),- name="openviking-otel-metrics-export-loop",- )+ loop = asyncio.get_running_loop()except RuntimeError:- logger.warning(- "[OTelMetricExporter] periodic export loop not started: phase=start protocol=%s endpoint=%s reason=no running event loop",- self._protocol,- self._endpoint,- )- self._export_task = None+ logger.warning("OTLP periodic export requires a running application event loop")+ return+ self._export_task = loop.create_task(self._run(), name="openviking-otel-metrics")- async def _run_export_loop(self) -> None:- """- Run the periodic export loop until shutdown is requested.- """- interval_s = float(self._export_interval_ms) / 1000.0- next_tick = time.monotonic() + interval_s- while self._enabled:+ async def _run(self) -> None:+ loop = asyncio.get_running_loop()+ next_tick = loop.time()+ while not self._closed:try:await self.export()- except asyncio.CancelledError:- raiseexcept Exception as exc:- logger.warning(- "[OTelMetricExporter] periodic export loop iteration failed: phase=run_export_loop protocol=%s endpoint=%s error=%s",- self._protocol,- self._endpoint,- exc,- )- now = time.monotonic()- next_tick, sleep_s = _compute_next_tick_and_sleep(- now=now,- interval_s=interval_s,- next_tick=next_tick,- )- await asyncio.sleep(sleep_s)+ logger.warning("OTLP metrics export failed: %s", exc)+ next_tick += self._interval+ now = loop.time()+ if next_tick < now:+ next_tick = now + self._interval+ await asyncio.sleep(next_tick - now)async def shutdown(self) -> None:- """- Stop the exporter loop and release transport resources.- """- self._enabled = False-+ self._closed = Trueif self._export_task is not None:self._export_task.cancel()try:await self._export_taskexcept asyncio.CancelledError:- logger.info(- "[OTelMetricExporter] periodic export loop cancelled: phase=shutdown protocol=%s endpoint=%s",- self._protocol,- self._endpoint,- )- except Exception as exc:- logger.warning(- "[OTelMetricExporter] periodic export loop shutdown failed: phase=shutdown protocol=%s endpoint=%s error=%s",- self._protocol,- self._endpoint,- exc,- )+ passself._export_task = None-- if self._http_session is not None:- try:- self._http_session.close()- except Exception as exc:- logger.warning(- "[OTelMetricExporter] http session close failed: phase=shutdown protocol=%s endpoint=%s error_type=%s error=%s",- self._protocol,- self._endpoint,- type(exc).__name__,- exc,- )- self._http_session = None-- if self._grpc_channel is not None:- try:- # grpc.Channel.close() is synchronous and returns None.- self._grpc_channel.close()- except Exception as exc:- logger.warning(- "[OTelMetricExporter] grpc channel close failed: phase=shutdown protocol=%s endpoint=%s error_type=%s error=%s",- self._protocol,- self._endpoint,- type(exc).__name__,- exc,- )- self._grpc_channel = None- self._grpc_stub = None+ async with self._export_lock:+ if self._sdk is not None:+ await asyncio.to_thread(self._sdk.shutdown)+ self._sdk = Nonediff --git a/openviking/metrics/exporters/prometheus.py b/openviking/metrics/exporters/prometheus.pyindex 38a97616e..d99af5ab1 100644--- a/openviking/metrics/exporters/prometheus.py+++ b/openviking/metrics/exporters/prometheus.py@@ -1,203 +1,79 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""-Prometheus exporter implementation.+"""Expose SDK metric data using the official Prometheus client serializer."""-Key behaviors:-- On each export, attempts to refresh all registered collectors (best-effort).-- Renders the current registry state into Prometheus text exposition format.+from itertools import accumulate, chain-The exporter is intentionally tolerant:-- Collector refresh errors are swallowed (metrics must not break `/metrics`).-- Families with no samples are still exported as a single zero sample when possible,- to keep dashboards stable.-"""--from __future__ import annotations--from typing import Iterable+from opentelemetry.sdk.metrics.export import Gauge, Histogram, Sum+from prometheus_client import CollectorRegistry, generate_latest+from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamilyfrom openviking.metrics.core.base import MetricExporter-from openviking.metrics.core.types import render_labelsclass PrometheusExporter(MetricExporter):- """- Render the in-process registry into Prometheus text exposition format.-- This exporter optionally refreshes refresh-managed collectors before rendering, then walks- the registry families and serializes counters, gauges, histograms, and dropped-series- diagnostics into a single text payload suitable for `/metrics`.- """+ """Adapt SDK data to Prometheus metric families, preserving OpenViking names and tags."""- def __init__(- self, *, registry, collector_manager=None, refresh_deadline_seconds: float = 1.0- ) -> None:- """- Initialize the Prometheus exporter.-- Args:- registry: The in-process metric registry that stores current metric values.- collector_manager: Optional refresh orchestrator invoked before each export.- refresh_deadline_seconds: Best-effort time budget for the pre-render refresh stage.- """+ def __init__(self, *, registry, collector_manager=None, refresh_deadline_seconds=1.0):self._registry = registryself._collector_manager = collector_managerself._refresh_deadline_seconds = float(refresh_deadline_seconds)+ self._prometheus = CollectorRegistry()+ self._prometheus.register(self)async def export(self) -> str:- """- Refresh collectors if configured and return the Prometheus exposition payload.-- Collector refresh remains best-effort: refresh failures are swallowed so observability- does not break the serving of the metrics endpoint itself.- """if self._collector_manager is not None:try:await self._collector_manager.refresh_all(- self._registry,- deadline_seconds=self._refresh_deadline_seconds,+ self._registry, deadline_seconds=self._refresh_deadline_seconds)except Exception:passreturn self.render()def render(self) -> str:- """- Serialize the registry into Prometheus text exposition format.-- Returns:- A newline-terminated text payload containing every registered family in the- Prometheus exposition layout expected by scrapers and tooling.- """- lines: list[str] = []-- # Process counters- for name, counter in self._registry.iter_counters():- self._add_help_and_type(lines, name, "counter")- self._process_counter_series(lines, name, counter)-- # Process gauges- for name, gauge in self._registry.iter_gauges():- self._add_help_and_type(lines, name, "gauge")- self._process_gauge_series(lines, name, gauge)-- # Process histograms- for name, label_names, bucket_bounds, series_iter in self._registry.iter_histograms():- self._add_help_and_type(lines, name, "histogram")- series_list = list(series_iter)- if not series_list and not label_names:- self._render_empty_histogram(lines, name, bucket_bounds)- else:- for labels, bucket_counts, count, value_sum in series_list:- self._render_histogram_series(- lines, name, bucket_bounds, labels, bucket_counts, count, value_sum- )-- # Process dropped series- for metric_name, dropped in self._registry.iter_dropped_series():- lines.append(- f'openviking_metrics_dropped_series_total{{metric="{metric_name}"}} {dropped}'- )-- return "\n".join(lines) + "\n"-- def _add_help_and_type(self, lines: list[str], name: str, metric_type: str) -> None:- """Append the Prometheus `HELP` and `TYPE` prelude for one metric family."""- lines.append(f"# HELP {name} OpenViking metric.")- lines.append(f"# TYPE {name} {metric_type}")-- def _process_counter_series(self, lines: list[str], name: str, counter) -> None:- """- Render all counter series for one family into the output buffer.-- Empty unlabeled counters are rendered as a single zero sample to keep dashboards and- alert queries stable before the first real increment occurs.- """- if not counter and not self._registry.counter_label_names(name):- lines.append(f"{name} 0")- for labels, value in counter:- lines.append(- f"{name}{render_labels(labels)} {int(value) if value.is_integer() else value}"- )-- def _process_gauge_series(self, lines: list[str], name: str, gauge) -> None:- """- Render all gauge series for one family into the output buffer.-- Empty unlabeled gauges also emit one zero sample so consumers do not need to special-case- missing series before initialization.- """- if not gauge and not self._registry.gauge_label_names(name):- lines.append(f"{name} 0")- for labels, value in gauge:- lines.append(f"{name}{render_labels(labels)} {value}")-- def _render_empty_histogram(- self, lines: list[str], name: str, bucket_bounds: tuple[float, ...]- ) -> None:- """Render an unlabeled histogram family with zero samples across every bucket."""- zero_counts = (0,) * (len(bucket_bounds) + 1)- cumulative = 0- for bound, c in _iter_histogram_buckets(bucket_bounds, zero_counts):- cumulative += c- lines.append(f"{name}_bucket{render_labels((('le', bound),))} {cumulative}")- lines.append(f"{name}_count 0")- lines.append(f"{name}_sum 0")-- def _render_histogram_series(- self,- lines: list[str],- name: str,- bucket_bounds: tuple[float, ...],- labels: tuple[tuple[str, str], ...],- bucket_counts: tuple[int, ...],- count: int,- value_sum: float,- ) -> None:- """- Render one histogram series with stable label ordering and cumulative bucket counts.-- Histogram buckets are exported cumulatively per the Prometheus exposition rules, while- `_count` and `_sum` reuse the same sorted label tuple for deterministic output.- """- # Sort labels once per series to avoid repeated sorting- sorted_labels = sorted(labels, key=lambda x: x[0])- cumulative = 0- for bound, c in _iter_histogram_buckets(bucket_bounds, bucket_counts):- cumulative += c- # Insert 'le' label in correct position without full sort- merged_labels = self._merge_le_label(sorted_labels, str(bound))- lines.append(f"{name}_bucket{render_labels(merged_labels)} {cumulative}")- # Use sorted labels for count and sum to ensure consistent order- lines.append(f"{name}_count{render_labels(tuple(sorted_labels))} {count}")- lines.append(f"{name}_sum{render_labels(tuple(sorted_labels))} {value_sum}")-- def _merge_le_label(- self, sorted_labels: list[tuple[str, str]], le: str- ) -> tuple[tuple[str, str], ...]:- """Insert the histogram `le` label into an already sorted label list without resorting."""- # Find insertion point for 'le' label- pos = len(sorted_labels)- for i, (key, _) in enumerate(sorted_labels):- if key > "le":- pos = i- break- # Create new list with 'le' inserted- merged = sorted_labels.copy()- merged.insert(pos, ("le", le))- return tuple(merged)---def _iter_histogram_buckets(- bounds: tuple[float, ...], counts: tuple[int, ...]-) -> Iterable[tuple[str, int]]:- """- Yield histogram bucket upper bounds paired with their per-bucket sample counts.-- The final synthetic `+Inf` bucket is always yielded after the configured finite bounds.- """- for i, upper in enumerate(bounds):- yield str(upper), counts[i]- yield "+Inf", counts[len(bounds)]+ return generate_latest(self._prometheus).decode("utf-8")++ def collect(self):+ """Implement the Prometheus custom-collector boundary for SDK and native snapshots."""+ data = self._registry.collect()+ for resource in data.resource_metrics:+ for scope in resource.scope_metrics:+ for metric in scope.metrics:+ points = metric.data.data_points+ if not points:+ continue+ keys = sorted(points[0].attributes)+ if isinstance(metric.data, Sum):+ family = CounterMetricFamily(metric.name, "OpenViking metric.", labels=keys)+ elif isinstance(metric.data, Gauge):+ family = GaugeMetricFamily(metric.name, "OpenViking metric.", labels=keys)+ elif isinstance(metric.data, Histogram):+ family = HistogramMetricFamily(+ metric.name, "OpenViking metric.", labels=keys+ )+ else:+ raise TypeError(+ f"unsupported SDK metric data: {type(metric.data).__name__}"+ )+ for point in points:+ labels = [str(point.attributes[key]) for key in keys]+ if isinstance(metric.data, Histogram):+ family.add_metric(+ labels,+ buckets=list(+ zip(+ chain(+ (str(bound) for bound in point.explicit_bounds),+ ("+Inf",),+ ),+ accumulate(point.bucket_counts),+ strict=True,+ )+ ),+ sum_value=point.sum,+ )+ else:+ family.add_metric(labels, point.value)+ yield familydiff --git a/openviking/metrics/global_api.py b/openviking/metrics/global_api.pyindex f7156347f..ef756b6bc 100644--- a/openviking/metrics/global_api.py+++ b/openviking/metrics/global_api.py@@ -27,18 +27,12 @@ from typing import Optionalfrom openviking.observability.events import (make_payload_subscriber,register_event_subscriber,- try_publish_event,unregister_event_subscriber,)from openviking.server.config import ServerConfigfrom openviking_cli.utils import get_logger-from .account_dimension import (- configure_metric_account_dimension as _configure_metric_account_dimension_runtime,-)-from .account_dimension import (- reset_metric_account_dimension,-)+from .account_dimension import configure_metric_account_dimension, reset_metric_account_dimensionfrom .bootstrap import create_default_collector_managerfrom .collectors.cache import CacheCollectorfrom .collectors.embedding import EmbeddingCollector@@ -99,40 +93,6 @@ def _shutdown_exporters_best_effort(exporters: list) -> None:)-def configure_metric_account_dimension(- *,- enabled: bool | None = None,- metric_allowlist: set[str] | list[str] | tuple[str, ...] | None = None,- max_active_accounts: int | None = None,- policy=None,- resolver=None,-) -> None:- """- Configure the process-global account-dimension runtime used by collector write helpers.-- This function is a thin façade over `account_dimension.configure_metric_account_dimension`- so callers do not need to import policy internals directly from the metrics package.- """- _configure_metric_account_dimension_runtime(- enabled=enabled,- metric_allowlist=metric_allowlist,- max_active_accounts=max_active_accounts,- policy=policy,- resolver=resolver,- )---def _create_metric_registry() -> MetricRegistry:- """- Create a new `MetricRegistry` instance.-- This helper centralizes the registry instantiation to avoid security issues- introduced by bridging classes (e.g., an external registry implementation- bypassing internal safety checks).- """- return MetricRegistry()--def init_metrics_from_server_config(config: ServerConfig, *, app=None, service=None, registry: MetricRegistry | None = None) -> None:@@ -172,7 +132,7 @@ def init_metrics_from_server_config(metric_allowlist=list(account_dimension.metric_allowlist or []),max_active_accounts=int(account_dimension.max_active_accounts),)- _registry = registry or _create_metric_registry()+ _registry = registry or MetricRegistry()collector_manager = create_default_collector_manager(app=app, service=service, config=config)@@ -203,7 +163,6 @@ def init_metrics_from_server_config(service_name=exporters_config.otel.service_name,export_interval_ms=exporters_config.otel.export_interval_ms,headers=exporters_config.otel.headers,- enabled=True,)otel_exporter.start()_exporters.append(otel_exporter)@@ -268,27 +227,6 @@ def get_metrics_registry() -> MetricRegistry:return reg-def try_get_metrics_registry() -> MetricRegistry | None:- """- Return the global MetricRegistry if metrics are initialized, otherwise `None`.-- This helper is intended for best-effort call sites that want to avoid raising an exception- when metrics are disabled.- """- return _registry---def try_dispatch_event(event_name: str, payload: dict) -> None:- """- Publish an in-process observability event.-- Kept for older call sites that imported the metrics API directly. The event is now- published to the shared bus, so non-metrics subscribers can consume it even when- Prometheus metrics are disabled.- """- try_publish_event(event_name, payload)--def _build_event_router(registry: MetricRegistry) -> EventCollectorRouter:"""Build the default in-process event router for event-driven metrics.diff --git a/pyproject.toml b/pyproject.tomlindex 494082049..a2e6980ee 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -77,10 +77,11 @@ dependencies = ["tree-sitter-php==0.24.1","tree-sitter-lua==0.5.0",# OpenTelemetry- "opentelemetry-api>=1.14",- "opentelemetry-sdk>=1.14",- "opentelemetry-exporter-otlp-proto-grpc>=1.14",- "opentelemetry-exporter-otlp-proto-http>=1.14",+ "prometheus-client>=0.24,<1",+ "opentelemetry-api>=1.40,<2",+ "opentelemetry-sdk>=1.40,<2",+ "opentelemetry-exporter-otlp-proto-grpc>=1.40,<2",+ "opentelemetry-exporter-otlp-proto-http>=1.40,<2","opentelemetry-instrumentation-asyncio>=0.61b0","loguru>=0.7.3","cryptography>=48.0.1",diff --git a/tests/agfs/test_fs_binding.py b/tests/agfs/test_fs_binding.pyindex 6ef1e5256..ac55b5973 100644--- a/tests/agfs/test_fs_binding.py+++ b/tests/agfs/test_fs_binding.py@@ -49,7 +49,20 @@ class TestVikingFSBindingLocal:async def test_native_metrics_and_legacy_stats(self, viking_fs_binding_instance):"""Use the binding fixture to check flat metrics and legacy stats; return None."""client = viking_fs_binding_instance._async_agfs- metrics = await client.run("metrics")+ lease = await client.pathlock_acquire_tree(+ f"/local/default/temp/metrics_scan_{uuid.uuid4().hex}"+ )+ try:+ metrics = await client.run("metrics")+ finally:+ await client.pathlock_release(lease)+ scan = next(m for m in metrics if m["name"] == "lock_descendant_scan_duration_seconds")+ legacy_count = next(m for m in metrics if m["name"] == "lock_descendant_scans_total")+ legacy_sum = next(+ m for m in metrics if m["name"] == "lock_descendant_scan_duration_seconds_total"+ )+ assert scan["count"] == legacy_count["value"] > 0+ assert scan["sum"] == legacy_sum["value"]shapes = {"counter": {"value", "scale"},"gauge": {"value"},@@ -70,7 +83,9 @@ class TestVikingFSBindingLocal:"route",}if metric["type"] == "histogram":- assert len(metric["bucket_counts"]) == len(metric["bucket_bounds"]) + 1 == 12+ assert metric["count"] > 0+ assert len(metric["bucket_counts"]) == len(metric["bucket_bounds"]) + 1+ assert metric["bucket_bounds"][-1] > 404261.58497 * 1e9assert sum(metric["bucket_counts"]) == metric["count"]assert metric["scale"] == 1e-9 and type(metric["sum"]) is intelse:diff --git a/tests/metrics/collectors/test_collector_contracts.py b/tests/metrics/collectors/test_collector_contracts.pyindex 68f82705e..ff5b22237 100644--- a/tests/metrics/collectors/test_collector_contracts.py+++ b/tests/metrics/collectors/test_collector_contracts.py@@ -194,9 +194,13 @@ def test_domain_stats_delta_helper_applies_non_negative_deltas_and_handles_resetcollector.collect(registry)- counters = dict(registry.iter_counters())- series = dict(counters["openviking_test_cumulative_total"])- assert series[()] == 18+ metrics = {+ m.name: m+ for r in registry.collect().resource_metrics+ for s in r.scope_metrics+ for m in s.metrics+ }+ assert metrics["openviking_test_cumulative_total"].data.data_points[0].value == 18def test_service_probe_collector_emits_valid_on_success_and_invalid_on_failure(diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.pyindex d8ee26aa6..1f316010e 100644--- a/tests/metrics/collectors/test_event_collectors.py+++ b/tests/metrics/collectors/test_event_collectors.py@@ -5,7 +5,6 @@ from __future__ import annotationsimport re-import openviking.metrics.global_api as global_apifrom openviking.metrics.collectors.base import EventMetricCollectorfrom openviking.metrics.collectors.cache import CacheCollectorfrom openviking.metrics.collectors.embedding import EmbeddingCollector@@ -69,21 +68,22 @@ def test_queue_duration_collector_records_processing_and_end_to_end_histograms({"queue": "Semantic","outcome": "success",- "process_duration_seconds": 2.5,- "end_to_end_duration_seconds": 8.0,+ "process_duration_seconds": 60.0,+ "end_to_end_duration_seconds": 404261.58497,},registry,)text = render_prometheus(registry)- labels = 'outcome="success",queue="Semantic"'+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)+ labels = f'bucket_schema="{schema}",outcome="success",queue="Semantic"'assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in text- assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 2.5" in text+ assert f"openviking_queue_process_duration_seconds_sum{{{labels}}} 60.0" in textassert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in text- assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 8.0" in text+ assert f"openviking_queue_end_to_end_duration_seconds_sum{{{labels}}} 404261.58497" in textassert (- "openviking_queue_end_to_end_duration_seconds_bucket{"- 'le="86400.0",outcome="success",queue="Semantic"} 1' in text+ f'openviking_queue_end_to_end_duration_seconds_bucket{{bucket_schema="{schema}",'+ 'le="500000.0",outcome="success",queue="Semantic"} 1' in text)@@ -121,9 +121,10 @@ def test_queue_duration_collector_allows_processing_only_and_ignores_bad_payload)text = render_prometheus(registry)- assert (- "openviking_queue_process_duration_seconds_count{"- 'outcome="requeued",queue="Embedding"} 1' in text+ assert re.search(+ r'openviking_queue_process_duration_seconds_count\{bucket_schema="[0-9a-f]{16}",'+ r'outcome="requeued",queue="Embedding"\} 1',+ text,)assert "openviking_queue_end_to_end_duration_seconds" not in textassert 'outcome="unknown"' not in text@@ -270,17 +271,6 @@ def test_telemetry_bridge_collector_ignores_malformed_payloads_instead_of_raisinassert "openviking_operation_requests_total" not in text-def test_http_collector_importable() -> None:- from openviking.metrics.collectors.http import HTTPCollector-- assert HTTPCollector is not None---def test_global_api_no_longer_exports_metrics_enabled_helper() -> None:- """Avoid redundant exports: metrics enablement is read via direct config access."""- assert not hasattr(global_api, "is_metrics_enabled_from_server_config")--def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prometheus):"""`embedding.call` must produce calls/duration/tokens metrics keyed by provider+model."""EmbeddingCollector().receive(@@ -422,8 +412,10 @@ def test_embedding_collector_records_success_volume_and_latency(registry, render'openviking_embedding_requests_total{account_id="__unknown__",status="ok"} 1' in textor 'openviking_embedding_requests_total{account_id="__unknown__",status="ok"} 1.0' in text)- assert (- 'openviking_embedding_latency_seconds_count{account_id="__unknown__",status="ok"} 1' in text+ assert re.search(+ r'openviking_embedding_latency_seconds_count\{account_id="__unknown__",'+ r'bucket_schema="[0-9a-f]{16}",status="ok"\} 1',+ text,)diff --git a/tests/metrics/collectors/test_state_collectors.py b/tests/metrics/collectors/test_state_collectors.pyindex 57b5795cd..ddcac0fd2 100644--- a/tests/metrics/collectors/test_state_collectors.py+++ b/tests/metrics/collectors/test_state_collectors.py@@ -394,12 +394,18 @@ def test_ragfs_collector_replaces_recovers_and_deletes(monkeypatch):},]collector.collect(registry)- assert dict(registry.iter_counters())["openviking_ragfs_calls_total"] == [((), count)]+ metrics = {+ m.name: m+ for r in registry.collect().resource_metrics+ for s in r.scope_metrics+ for m in s.metrics+ }+ assert metrics["openviking_ragfs_calls_total"].data.data_points[0].value == countassert registry.gauge_get("openviking_ragfs_tasks") == count- hist = list(registry.iter_histograms())[0]- assert hist[2] == pytest.approx((1e-6,))- assert hist[3][0][1:3] == ((count, 0), count)- assert hist[3][0][3] == pytest.approx(123e-9 * count)+ hist = metrics["openviking_ragfs_latency_seconds"].data.data_points[0]+ assert hist.explicit_bounds == pytest.approx((1e-6,))+ assert hist.bucket_counts == (count, 0) and hist.count == count+ assert hist.sum == pytest.approx(123e-9 * count)assert "account_id" not in PrometheusExporter(registry=registry).render()previous = PrometheusExporter(registry=registry).render()records.append({"broken": True})@@ -425,16 +431,14 @@ def test_ragfs_collector_replaces_recovers_and_deletes(monkeypatch):patch.setattr(registry, "set_gauge", fail)with pytest.raises(RuntimeError, match="injected"):collector.collect(registry)- assert dict(registry.iter_counters())["openviking_ragfs_partial_total"] == [((), 9)]+ assert "openviking_ragfs_partial_total 9.0" in PrometheusExporter(registry=registry).render()records.clear()with monkeypatch.context() as patch:- patch.setattr(registry, "counter_delete_matching", fail)+ patch.setattr(registry, "delete_matching", fail)with pytest.raises(RuntimeError, match="injected"):collector.collect(registry)collector.collect(registry)- assert list(registry.iter_counters()) == []- assert list(registry.iter_gauges()) == []- assert list(registry.iter_histograms()) == []+ assert not registry.collect().resource_metrics[0].scope_metrics[0].metricsassert "openviking_ragfs_" not in PrometheusExporter(registry=registry).render()diff --git a/tests/metrics/core/test_exporter.py b/tests/metrics/core/test_exporter.pyindex c879ccede..724eaa704 100644--- a/tests/metrics/core/test_exporter.py+++ b/tests/metrics/core/test_exporter.py@@ -2,166 +2,93 @@# SPDX-License-Identifier: AGPL-3.0import asyncio+from http.server import BaseHTTPRequestHandler, HTTPServer+from threading import Event, Threadimport pytest+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest-from openviking.metrics.exporters.otel import OTelMetricExporter, _compute_next_tick_and_sleep+from openviking.metrics.exporters.otel import OTelMetricExporterdef test_exporter_outputs_help_type_and_empty_metrics(registry, render_prometheus):- registry.counter("openviking_empty_counter_total")- registry.gauge("openviking_empty_gauge")- registry.histogram("openviking_empty_histogram_seconds")+ registry.set_counter("openviking_empty_counter_total", 0)+ registry.set_gauge("openviking_empty_gauge", 0)+ registry.set_histogram(+ "openviking_empty_histogram_seconds",+ bucket_bounds=(1,),+ bucket_counts=(0, 0),+ count=0,+ value_sum=0,+ )text = render_prometheus(registry)assert "# TYPE openviking_empty_counter_total counter" in text- assert "openviking_empty_counter_total 0" in text+ assert "openviking_empty_counter_total 0.0" in textassert "# TYPE openviking_empty_histogram_seconds histogram" in text- assert "openviking_empty_histogram_seconds_count 0" in text- assert 'openviking_empty_histogram_seconds_bucket{le="+Inf"} 0' in text+ assert 'le="+Inf"} 0.0' in text@pytest.mark.asyncio-async def test_otel_exporter_start_and_shutdown_manage_background_loop(registry, monkeypatch):- """OTel exporter should own its periodic export loop lifecycle."""- exporter = OTelMetricExporter(registry=registry, enabled=False, export_interval_ms=10)- exporter._enabled = True-- calls = 0- tick = asyncio.Event()-- async def _fake_export():- nonlocal calls- calls += 1- tick.set()- return ""-- monkeypatch.setattr(exporter, "export", _fake_export)-- exporter.start()- await asyncio.wait_for(tick.wait(), timeout=0.2)- assert calls >= 1- assert exporter._export_task is not None-- task_before = exporter._export_task- exporter.start()- assert exporter._export_task is task_before-- await exporter.shutdown()- assert exporter._export_task is None---async def test_otel_export_loop_aligns_to_next_fixed_tick_after_overrun():- """Scheduling helper should advance next tick and compute sleep after one interval overrun."""- next_tick, sleep_s = _compute_next_tick_and_sleep(now=1.15, interval_s=1.0, next_tick=1.0)- assert next_tick == pytest.approx(2.0)- assert sleep_s == pytest.approx(0.85)---def test_otel_http_send_uses_export_timeout_not_refresh_deadline(registry):- """HTTP transport timeout should be decoupled from collector refresh timeout."""+async def test_otel_exporter_start_and_shutdown_manage_background_loop(registry):+ """Exercise the real SDK HTTP exporter and wait for its in-flight send at shutdown."""+ received, release = Event(), Event()+ requests = []++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ body = self.rfile.read(int(self.headers["Content-Length"]))+ requests.append((self.path, self.headers, ExportMetricsServiceRequest.FromString(body)))+ received.set()+ release.wait(5)+ self.send_response(200)+ self.send_header("Content-Length", "0")+ self.end_headers()++ def log_message(self, *args):+ pass++ server = HTTPServer(("127.0.0.1", 0), Handler)+ thread = Thread(target=server.serve_forever, daemon=True)+ thread.start()+ registry.inc_counter("openviking_operation_requests_total", amount=2)+ registry.observe_histogram("openviking_operation_duration_seconds", 60.123)exporter = OTelMetricExporter(registry=registry,- enabled=False,protocol="http",- endpoint="http://localhost:4318/v1/metrics",- refresh_deadline_seconds=1.0,+ endpoint=f"http://127.0.0.1:{server.server_port}/v1/metrics",+ service_name="metrics-contract",+ headers={"X-ByteAPM-AppKey": "test-app-key"},)-- class _FakeSession:- def __init__(self):- self.timeout = None-- def post(self, _url, **kwargs):- self.timeout = kwargs.get("timeout")-- class _Resp:- @staticmethod- def raise_for_status():- return None-- return _Resp()-- fake_session = _FakeSession()- exporter._http_session = fake_session-- request = exporter._build_export_request()- assert request is None-- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (- ExportMetricsServiceRequest,- )-- exporter._send_http_request(ExportMetricsServiceRequest())-- assert fake_session.timeout == 5.0---def test_otel_http_send_forwards_custom_headers(registry):- """HTTP transport should merge vendor auth headers with protobuf content type."""- exporter = OTelMetricExporter(- registry=registry,- enabled=False,- protocol="http",- endpoint="https://apmplus-cn-beijing.ivolces.com/api/otlp/v1/metrics",- headers={"X-ByteAPM-AppKey": "metric-appkey"},- )-- class _FakeSession:- def __init__(self):- self.headers = None-- def post(self, _url, **kwargs):- self.headers = kwargs.get("headers")-- class _Resp:- @staticmethod- def raise_for_status():- return None-- return _Resp()-- exporter._http_session = _FakeSession()-- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (- ExportMetricsServiceRequest,- )-- exporter._send_http_request(ExportMetricsServiceRequest())-- assert exporter._http_session.headers == {- "Content-Type": "application/x-protobuf",- "X-ByteAPM-AppKey": "metric-appkey",- }---def test_otel_grpc_send_forwards_custom_metadata(registry):- """gRPC transport should forward vendor auth headers as metadata pairs."""- exporter = OTelMetricExporter(- registry=registry,- enabled=False,- protocol="grpc",- endpoint="apmplus-cn-beijing.ivolces.com:4317",- headers={"x-byteapm-appkey": "metric-appkey"},- )-- class _FakeStub:- def __init__(self):- self.metadata = None- self.timeout = None-- def Export(self, _request, **kwargs):- self.metadata = kwargs.get("metadata")- self.timeout = kwargs.get("timeout")- return None-- exporter._grpc_stub = _FakeStub()-- from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (- ExportMetricsServiceRequest,- )-- exporter._send_grpc_request(ExportMetricsServiceRequest())-- assert exporter._grpc_stub.metadata == [("x-byteapm-appkey", "metric-appkey")]- assert exporter._grpc_stub.timeout == 5.0+ stopping = None+ try:+ exporter.start()+ exporter.start()+ assert await asyncio.to_thread(received.wait, 2)+ stopping = asyncio.create_task(exporter.shutdown())+ with pytest.raises(asyncio.TimeoutError):+ await asyncio.wait_for(asyncio.shield(stopping), 0.05)+ release.set()+ await asyncio.wait_for(stopping, 2)+ await exporter.export()+ assert len(requests) == 1+ path, headers, request = requests[0]+ assert path == "/v1/metrics"+ assert headers["X-ByteAPM-AppKey"] == "test-app-key"+ assert headers["Content-Type"] == "application/x-protobuf"+ resource = request.resource_metrics[0]+ assert {item.key: item.value.string_value for item in resource.resource.attributes} == {+ "service.name": "metrics-contract"+ }+ metrics = {metric.name: metric for metric in resource.scope_metrics[0].metrics}+ assert metrics["openviking_operation_requests_total"].sum.data_points[0].as_double == 2+ point = metrics["openviking_operation_duration_seconds"].histogram.data_points[0]+ assert point.count == 1 and point.sum == pytest.approx(60.123)+ assert point.bucket_counts[-1] == 0+ finally:+ release.set()+ if stopping is not None:+ await stopping+ await exporter.shutdown()+ await asyncio.to_thread(server.shutdown)+ server.server_close()+ thread.join(timeout=2)diff --git a/tests/metrics/core/test_refresh.py b/tests/metrics/core/test_refresh.pyindex 077f0a456..25fdcf4e1 100644--- a/tests/metrics/core/test_refresh.py+++ b/tests/metrics/core/test_refresh.py@@ -7,10 +7,20 @@ import pytestfrom openviking.metrics.collectors.base import CollectorConfig, Refreshablefrom openviking.metrics.collectors.manager import CollectorManager, RefreshResult+from openviking.metrics.core.base import MetricCollectorfrom openviking.metrics.core.refresh import RefreshGatefrom openviking.metrics.core.registry import MetricRegistry+class _RefreshableCollector(MetricCollector, Refreshable):+ @classmethod+ def kind(cls):+ return "state"++ def collector_name(self):+ return "dummy"++def test_refresh_gate_ttl():gate = RefreshGate(ttl_seconds=1.0)d0 = gate.decide(now=0.0)@@ -42,7 +52,7 @@ async def test_collector_manager_requires_refreshable():def collect(self, registry) -> None:self.calls += 1- registry.counter("openviking_dummy_refresh_total").inc()+ registry.inc_counter("openviking_dummy_refresh_total")mgr = CollectorManager()c = DummyCollector()@@ -53,8 +63,7 @@ async def test_collector_manager_requires_refreshable():@pytest.mark.asyncioasync def test_collector_manager_ttl_and_timeout():- class DummyCollector(Refreshable):- name = "dummy"+ class DummyCollector(_RefreshableCollector):config = CollectorConfig(ttl_seconds=10.0, timeout_seconds=0.01)def __init__(self) -> None:@@ -62,7 +71,7 @@ async def test_collector_manager_ttl_and_timeout():def collect(self, registry) -> None:self.calls += 1- registry.counter("openviking_dummy_refresh_total").inc()+ registry.inc_counter("openviking_dummy_refresh_total")registry = MetricRegistry()mgr = CollectorManager()@@ -82,8 +91,7 @@ async def test_collector_manager_ttl_and_timeout():@pytest.mark.asyncioasync def test_collector_manager_swr_when_ttl_expired_and_has_last_success():- class DummyCollector(Refreshable):- name = "dummy"+ class DummyCollector(_RefreshableCollector):config = CollectorConfig(ttl_seconds=1.0, timeout_seconds=0.5)def __init__(self) -> None:@@ -113,8 +121,7 @@ async def test_collector_manager_swr_when_ttl_expired_and_has_last_success():@pytest.mark.asyncioasync def test_collector_manager_deadline_exceeded_does_not_stick_inflight(monkeypatch):- class DummyCollector(Refreshable):- name = "dummy"+ class DummyCollector(_RefreshableCollector):config = CollectorConfig(ttl_seconds=10.0, timeout_seconds=10.0)def collect(self, registry) -> None:diff --git a/tests/metrics/core/test_registry.py b/tests/metrics/core/test_registry.pyindex 3ba9304e1..2044be617 100644--- a/tests/metrics/core/test_registry.py+++ b/tests/metrics/core/test_registry.py@@ -1,93 +1,77 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-from pathlib import Path-import pytest+from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics+from prometheus_client.parser import text_string_to_metric_families-from openviking.metrics.core.base import MetricCollector-from openviking.metrics.core.registry import MetricRegistry-from openviking.metrics.exporters.otel import OTelMetricExporter---def test_metric_collector_metric_name_includes_namespace_and_optional_unit():- assert (- MetricCollector.metric_name("cache", "hits", unit="total") == "openviking_cache_hits_total"- )- assert (- MetricCollector.metric_name("resource", "stage_duration", unit="seconds")- == "openviking_resource_stage_duration_seconds"- )- assert MetricCollector.metric_name("lock", "active") == "openviking_lock_active"+from openviking.metrics.core.registry import DEFAULT_LATENCY_BUCKETS, MetricRegistrydef test_registry_rejects_type_conflict():registry = MetricRegistry()- registry.counter("openviking_conflict_total")+ registry.inc_counter("calls_total")+ with pytest.raises(ValueError):+ registry.set_gauge("calls_total", 1)with pytest.raises(ValueError):- registry.gauge("openviking_conflict_total")+ registry.set_counter("calls_total", 1)def test_registry_label_keys_must_match_definition():registry = MetricRegistry()- c = registry.counter("openviking_labeled_total", label_names=("a", "b"))- with pytest.raises(ValueError):- c.inc(labels={"a": "1"})- with pytest.raises(ValueError):- c.inc(labels={"a": "1", "b": "2", "c": "3"})+ for labels in ({"a": "1"}, {"a": "1", "b": "2", "c": "3"}):+ with pytest.raises(ValueError):+ registry.inc_counter("calls_total", labels=labels, label_names=("a", "b"))+ with pytest.raises(ValueError, match="bucket_schema is reserved"):+ registry.observe_histogram(+ "latency", 1, labels={"bucket_schema": "custom"}, label_names=("bucket_schema",)+ )def test_registry_canonicalizes_label_name_order_for_same_family(render_prometheus):registry = MetricRegistry()-- registry.inc_counter(- "openviking_ordered_total",- labels={"status": "ok", "provider": "local"},- label_names=("status", "provider"),- )- registry.inc_counter(- "openviking_ordered_total",- labels={"provider": "local", "status": "ok"},- label_names=("provider", "status"),- )-- text = render_prometheus(registry)- assert 'openviking_ordered_total{provider="local",status="ok"} 2' in text+ for names in (("status", "provider"), ("provider", "status")):+ registry.inc_counter(+ "calls_total", labels={"status": "ok", "provider": "local"}, label_names=names+ )+ assert 'calls_total{provider="local",status="ok"} 2.0' in render_prometheus(registry)def test_counter_only_increases():registry = MetricRegistry()- c = registry.counter("openviking_counter_total")- c.inc()- with pytest.raises(ValueError):- c.inc(amount=0)- with pytest.raises(ValueError):- c.inc(amount=-1)+ registry.inc_counter("calls_total")+ for amount in (0, -1, float("inf")):+ with pytest.raises(ValueError):+ registry.inc_counter("calls_total", amount=amount)def test_histogram_boundary_bucket(registry, render_prometheus):- h = registry.histogram("openviking_latency_seconds", buckets=(0.05, 0.1))- h.observe(0.05)- text = render_prometheus(registry)- assert 'openviking_latency_seconds_bucket{le="0.05"} 1' in text- assert 'openviking_latency_seconds_bucket{le="0.1"} 1' in text- assert 'openviking_latency_seconds_bucket{le="+Inf"} 1' in text---def test_collectors_do_not_embed_openviking_metric_name_literals():- root = Path(__file__).resolve().parents[3] / "openviking" / "metrics" / "collectors"- offenders = []- for path in root.glob("*.py"):- text = path.read_text(encoding="utf-8")- if '"openviking_' in text or "'openviking_" in text:- offenders.append(path.name)- assert offenders == []---def test_registry_absolute_values_and_deletion():- """Use real families with fixed inputs to verify replacement and deletion; return None."""+ durations = (0.05, 20.0, 60.0, 60.001, 120.0, 3600.0, 404261.58497)+ for duration in durations:+ registry.observe_histogram("latency_seconds", duration)+ family = next(text_string_to_metric_families(render_prometheus(registry)))+ buckets = {float(s.labels["le"]): s.value for s in family.samples if s.name.endswith("_bucket")}+ for bound in DEFAULT_LATENCY_BUCKETS:+ assert buckets[bound] == sum(value <= bound for value in durations)+ assert buckets[float("inf")] == len(durations)+ schema = family.samples[0].labels["bucket_schema"]+ request = encode_metrics(registry.collect())+ point = request.resource_metrics[0].scope_metrics[0].metrics[0].histogram.data_points[0]+ assert tuple(point.explicit_bounds) == DEFAULT_LATENCY_BUCKETS+ assert point.count == sum(point.bucket_counts) == len(durations)+ assert point.bucket_counts[-1] == 0+ assert point.sum == pytest.approx(sum(durations))+ assert {a.key: a.value.string_value for a in point.attributes} == {"bucket_schema": schema}+ legacy = MetricRegistry()+ legacy.observe_histogram("latency_seconds", 60, buckets=(0.05, 0.1, 10.0))+ legacy_family = next(text_string_to_metric_families(render_prometheus(legacy)))+ assert legacy_family.samples[0].labels["bucket_schema"] != schema+++def test_registry_absolute_values_and_deletion(render_prometheus):registry = MetricRegistry(max_series_per_metric=1)labels = {"plugin": "memfs"}+ started = Nonefor value in (10, 12, 3, 0):registry.set_counter("calls_total", value, labels=labels, label_names=("plugin",))registry.set_histogram(@@ -99,200 +83,64 @@ def test_registry_absolute_values_and_deletion():labels=labels,label_names=("plugin",),)- assert dict(registry.iter_counters())["calls_total"] == [(tuple(labels.items()), value)]- assert list(registry.iter_histograms())[0][3] == [- (tuple(labels.items()), (value, 0), value, value * 0.001)- ]- registry.inc_counter("calls_total", labels=labels, label_names=("plugin",))- registry.observe_histogram(- "latency_seconds",- 0.002,- labels=labels,- label_names=("plugin",),- buckets=(0.001,),- )- assert list(registry.iter_histograms())[0][3][0][1:] == ((0, 1), 1, 0.002)+ request = encode_metrics(registry.collect())+ metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}+ counter = metrics["calls_total"].sum.data_points[0]+ histogram = metrics["latency_seconds"].histogram.data_points[0]+ assert counter.as_double == histogram.count == value+ assert histogram.sum == pytest.approx(value * 0.001)+ assert not histogram.HasField("min") and not histogram.HasField("max")+ if value == 10:+ started = counter.start_time_unix_nano+ elif value == 12:+ assert counter.start_time_unix_nano == started+ else:+ assert counter.start_time_unix_nano > started+ assert histogram.start_time_unix_nano <= histogram.time_unix_nanoregistry.set_counter("calls_total", 5, labels={"plugin": "other"}, label_names=("plugin",))- assert dict(registry.iter_counters())["calls_total"][0][1] == 1- assert dict(registry.iter_dropped_series()) == {"calls_total": 1}- for name, method in (- ("calls_total", registry.counter_delete_matching),- ("latency_seconds", registry.histogram_delete_matching),- ):- method(name, match_labels={"plugin": "other"})- method(name, match_labels=labels)- method(name, match_labels=labels)- assert list(registry.iter_counters()) == []- assert list(registry.iter_histograms()) == []+ text = render_prometheus(registry)+ assert 'plugin="other"' not in text+ assert 'openviking_metrics_dropped_series_total{metric="calls_total"} 1.0' in text+ registry.delete_matching("calls_total", match_labels=labels)+ registry.delete_matching("latency_seconds", match_labels=labels)+ assert not any(+ line.startswith(("calls_total", "latency_seconds"))+ for line in render_prometheus(registry).splitlines()+ )@pytest.mark.parametrize("value", [-1, float("inf"), float("nan")])-def test_registry_rejects_invalid_absolute_counter(value):- """Reject invalid counter input without replacing the previous value; return None."""+def test_registry_rejects_invalid_absolute_counter(value, render_prometheus):registry = MetricRegistry()registry.set_counter("calls_total", 4)with pytest.raises(ValueError):registry.set_counter("calls_total", value)- assert dict(registry.iter_counters())["calls_total"] == [((), 4)]- empty = MetricRegistry()- with pytest.raises(ValueError):- empty.set_counter("calls_total", value)- assert list(empty.iter_counters()) == []--- "changes",- [- {"bucket_bounds": (1.0, 0.0)},- {"bucket_bounds": (float("inf"),)},- {"bucket_counts": (1,)},- {"bucket_counts": (-1, 2)},- {"count": 2},- {"count": True},- {"bucket_counts": (True, 0)},- {"value_sum": float("nan")},- ],-)-def test_registry_rejects_invalid_absolute_histogram(changes):- """Reject malformed histogram input without corrupting an existing series; return None."""- registry = MetricRegistry()- values = {- "bucket_bounds": (0.001,),- "bucket_counts": (1, 0),- "count": 1,- "value_sum": 0.001,- }- registry.set_histogram("latency_seconds", **values)- before = list(registry.iter_histograms())- with pytest.raises(ValueError):- registry.set_histogram("latency_seconds", **(values | changes))- assert list(registry.iter_histograms()) == before- empty = MetricRegistry()- with pytest.raises(ValueError):- empty.set_histogram("latency_seconds", **(values | changes))- assert list(empty.iter_histograms()) == []- empty.set_histogram("latency_seconds", **values)- assert list(empty.iter_histograms()) == before---def test_registry_deletion_hides_empty_families_but_keeps_contract(render_prometheus):- """Delete all series for each type and verify export absence and schema retention."""- registry = MetricRegistry()- registry.counter("untouched_total")- registry.gauge("untouched_gauge")- registry.histogram("untouched_histogram")- registry.counter_delete_matching("untouched_total", match_labels={})- registry.gauge_delete_matching("untouched_gauge", match_labels={})- registry.histogram_delete_matching("untouched_histogram", match_labels={})- assert "untouched_total 0" in render_prometheus(registry)- registry.set_counter("removed_total", 1)- registry.set_gauge("removed_gauge", 1)- registry.set_histogram(- "removed_histogram",- bucket_bounds=[1],- bucket_counts=[1, 0],- count=1,- value_sum=1,- )- for kind, name in (- ("counter", "removed_total"),- ("gauge", "removed_gauge"),- ("histogram", "removed_histogram"),- ):- getattr(registry, f"{kind}_delete_matching")(name, match_labels={})- text = render_prometheus(registry)- assert "removed_" not in text- assert "untouched_total 0" in text and "untouched_gauge 0" in text- assert "untouched_histogram_count 0" in text- exporter = OTelMetricExporter(registry=registry, enabled=False)- request = exporter._build_export_request()- metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}- assert set(metrics) == {"untouched_total", "untouched_gauge", "untouched_histogram"}- assert metrics["untouched_total"].sum.data_points[0].as_int == 0- assert metrics["untouched_gauge"].gauge.data_points[0].as_int == 0- assert metrics["untouched_histogram"].histogram.data_points[0].count == 0- with pytest.raises(ValueError):- registry.gauge("removed_total")- with pytest.raises(ValueError):- registry.histogram("removed_histogram", buckets=[2])- with pytest.raises(ValueError):- registry.counter("removed_total").set(-1)- with pytest.raises(ValueError):- registry.histogram("removed_histogram", buckets=[1]).set((1,), 1, 1)- assert "removed_" not in render_prometheus(registry)- registry.set_counter("removed_total", 2)- registry.gauge("removed_gauge").inc(2)- registry.histogram("removed_histogram", buckets=[1]).observe(0.5)- text = render_prometheus(registry)- assert "removed_total 2" in text- assert "removed_gauge 2.0" in text- assert "removed_histogram_count 1" in text- request = exporter._build_export_request()- metrics = {m.name: m for m in request.resource_metrics[0].scope_metrics[0].metrics}- assert metrics["removed_total"].sum.data_points[0].as_int == 2- assert metrics["removed_gauge"].gauge.data_points[0].as_int == 2- assert metrics["removed_histogram"].histogram.data_points[0].count == 1+ assert "calls_total 4.0" in render_prometheus(registry)@pytest.mark.parametrize("kind", ("counter", "gauge", "histogram"))def test_registry_partial_deletion_keeps_active_series(kind, render_prometheus):- """Delete selected labels for kind; retain active values and the label contract."""registry = MetricRegistry()- options = {"buckets": [1]} if kind == "histogram" else {}- handle = getattr(registry, kind)("active", label_names=("plugin",), **options)+ name = "active_total" if kind == "counter" else "active"for plugin in ("removed", "kept"):+ labels = {"plugin": plugin}if kind == "histogram":- handle.observe(0.5, labels={"plugin": plugin})+ registry.set_histogram(+ name,+ bucket_bounds=[1],+ bucket_counts=[1, 0],+ count=1,+ value_sum=0.5,+ labels=labels,+ label_names=("plugin",),+ )else:- handle.inc(labels={"plugin": plugin})- delete = getattr(registry, f"{kind}_delete_matching")- delete("active", match_labels={"plugin": "removed"})+ getattr(registry, f"set_{kind}")(name, 1, labels=labels, label_names=("plugin",))+ delete = registry.delete_matching+ delete(name, match_labels={"plugin": "removed"})text = render_prometheus(registry)- assert 'plugin="removed"' not in text- sample = "active_count" if kind == "histogram" else "active"- assert f'{sample}{{plugin="kept"}} 1' in text- delete("active", match_labels={"plugin": "kept"})- assert "active" not in render_prometheus(registry)+ assert 'plugin="removed"' not in text and 'plugin="kept"' in text+ delete(name, match_labels={"plugin": "kept"})+ assert name not in render_prometheus(registry)with pytest.raises(ValueError):- getattr(registry, kind)("active", label_names=("other",), **options)- if kind == "histogram":- handle.set((0, 0), 0, 0, labels={"plugin": "kept"})- else:- handle.set(0, labels={"plugin": "kept"})- assert f'{sample}{{plugin="kept"}} 0' in render_prometheus(registry)---def test_registry_handles_validate_absolute_values(render_prometheus):- """Call public handles with integer and invalid inputs and verify validated storage."""- registry = MetricRegistry()- counter = registry.counter("calls_total")- counter.set(1)- assert "calls_total 1" in render_prometheus(registry)- for value in (-1, float("inf"), float("-inf"), float("nan")):- with pytest.raises(ValueError):- counter.set(value)- assert dict(registry.iter_counters())["calls_total"] == [((), 1)]- histogram = registry.histogram("latency", buckets=[1])- histogram.set([1, 0], 1, 1)- assert "latency_sum 1.0" in render_prometheus(registry)- before = list(registry.iter_histograms())- for counts, count, value_sum in (- ((1,), 1, 1),- ((-1, 2), 1, 1),- ((True, 0), 1, 1),- ((1.0, 0), 1, 1),- ((1, 0), True, 1),- ((1, 0), 2, 1),- ((1, 0), 1, -1),- ((1, 0), 1, float("inf")),- ((1, 0), 1, float("nan")),- ):- with pytest.raises(ValueError):- histogram.set(counts, count, value_sum)- assert list(registry.iter_histograms()) == before- for bounds in ((-1,), (float("inf"),), (float("nan"),), (1, 1), (2, 1)):- invalid = MetricRegistry()- handle = invalid.histogram("invalid", buckets=bounds)- with pytest.raises(ValueError):- handle.set((0,) * (len(bounds) + 1), 0, 0)- assert list(invalid.iter_histograms())[0][3] == []+ registry.set_gauge(name, 2, labels={"other": "value"}, label_names=("other",))diff --git a/tests/metrics/datasources/test_state_datasources.py b/tests/metrics/datasources/test_state_datasources.pyindex 76f133bdf..58e6e33c3 100644--- a/tests/metrics/datasources/test_state_datasources.py+++ b/tests/metrics/datasources/test_state_datasources.py@@ -293,6 +293,7 @@ def test_ragfs_datasource_validates_records(changes):{},{"bucket_counts": [1]},{"bucket_counts": [True, 0]},+ {"labels": {"bucket_schema": "caller-supplied"}},{"count": 3},{"bucket_bounds": [2, 1]},{"sum": -1},diff --git a/tests/metrics/integration/test_bootstrap.py b/tests/metrics/integration/test_bootstrap.pyindex 98b2e3d1c..1c8ffddd1 100644--- a/tests/metrics/integration/test_bootstrap.py+++ b/tests/metrics/integration/test_bootstrap.py@@ -3,6 +3,7 @@from __future__ import annotations+import refrom pathlib import Pathimport pytest@@ -168,17 +169,19 @@ def test_resource_ingestion_event_datasource_can_drive_resource_ingestion_collec'openviking_resource_stage_total{account_id="__unknown__",stage="parse",status="error"} 1'in text)- assert (- 'openviking_resource_wait_duration_seconds_count{account_id="__unknown__",operation="queue_processing"} 1'- in text+ assert re.search(+ r'openviking_resource_wait_duration_seconds_count\{account_id="__unknown__",'+ r'bucket_schema="[0-9a-f]{16}",operation="queue_processing"\} 1',+ text,)assert ('openviking_resource_stage_total{account_id="acct-resource",stage="parse",status="ok"} 1'in text)- assert (- 'openviking_resource_wait_duration_seconds_count{account_id="acct-resource",operation="queue_processing"} 1'- in text+ assert re.search(+ r'openviking_resource_wait_duration_seconds_count\{account_id="acct-resource",'+ r'bucket_schema="[0-9a-f]{16}",operation="queue_processing"\} 1',+ text,)reset_metric_account_dimension()diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.pyindex 29118905f..898744c4e 100644--- a/tests/metrics/integration/test_telemetry_bridge.py+++ b/tests/metrics/integration/test_telemetry_bridge.py@@ -1,6 +1,8 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+import re+from openviking.metrics.datasources.telemetry_bridge import TelemetryBridgeEventDataSourcefrom openviking.metrics.global_api import init_metrics_from_server_config, shutdown_metricsfrom openviking.observability.events import try_publish_event@@ -24,7 +26,8 @@ def test_queue_duration_event_is_wired_to_metrics_registry(registry, render_prom},)text = render_prometheus(registry)- labels = 'outcome="success",queue="SessionCommit"'+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)+ labels = f'bucket_schema="{schema}",outcome="success",queue="SessionCommit"'assert f"openviking_queue_process_duration_seconds_count{{{labels}}} 1" in textassert f"openviking_queue_end_to_end_duration_seconds_count{{{labels}}} 1" in textfinally:@@ -87,9 +90,10 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende'openviking_operation_requests_total{account_id="__unknown__",operation="resource.process",status="ok"} 1'in text)- assert (- 'openviking_operation_duration_seconds_count{account_id="__unknown__",operation="resource.process",status="ok"} 1'- in text+ assert re.search(+ r'openviking_operation_duration_seconds_count\{account_id="__unknown__",'+ r'bucket_schema="[0-9a-f]{16}",operation="resource.process",status="ok"\} 1',+ text,)assert ('openviking_operation_tokens_total{account_id="__unknown__",operation="resource.process",stage="rerank",token_type="rerank"} 4'@@ -132,9 +136,10 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende'openviking_resource_stage_total{account_id="__unknown__",stage="parse",status="warning"} 1'in text)- assert (- 'openviking_resource_wait_duration_seconds_count{account_id="__unknown__",operation="resource.process"} 1'- in text+ assert re.search(+ r'openviking_resource_wait_duration_seconds_count\{account_id="__unknown__",'+ r'bucket_schema="[0-9a-f]{16}",operation="resource.process"\} 1',+ text,)finally:shutdown_metrics(app=None)diff --git a/tests/server/test_prometheus_metrics.py b/tests/server/test_prometheus_metrics.pyindex 77ed66598..e4eb66f3d 100644--- a/tests/server/test_prometheus_metrics.py+++ b/tests/server/test_prometheus_metrics.py@@ -4,6 +4,7 @@"""Tests for the Prometheus metrics endpoint and exposition output."""import json+import refrom types import SimpleNamespaceimport httpx@@ -33,39 +34,45 @@ class TestPrometheusExposition:def test_counter_histogram_and_labels(self):registry = MetricRegistry()labels = {"context_type": "memory"}- registry.counter(- "openviking_retrieval_requests_total",- label_names=("context_type",),- ).inc(labels=labels)- registry.counter(- "openviking_retrieval_requests_total",- label_names=("context_type",),- ).inc(labels=labels)- registry.histogram(+ registry.inc_counter(+ "openviking_retrieval_requests_total", label_names=("context_type",), labels=labels+ )+ registry.inc_counter(+ "openviking_retrieval_requests_total", label_names=("context_type",), labels=labels+ )+ registry.observe_histogram("openviking_retrieval_latency_seconds",+ 0.02,label_names=("context_type",),- ).observe(0.02, labels=labels)+ labels=labels,+ )- registry.counter("openviking_cache_hits_total", label_names=("level",)).inc(- labels={"level": "L0"}+ registry.inc_counter(+ "openviking_cache_hits_total", label_names=("level",), labels={"level": "L0"})- registry.counter("openviking_cache_hits_total", label_names=("level",)).inc(- labels={"level": "L0"}+ registry.inc_counter(+ "openviking_cache_hits_total", label_names=("level",), labels={"level": "L0"})- registry.counter("openviking_cache_misses_total", label_names=("level",)).inc(- labels={"level": "L1"}+ registry.inc_counter(+ "openviking_cache_misses_total", label_names=("level",), labels={"level": "L1"})exporter = PrometheusExporter(registry=registry)text = exporter.render()+ schema = re.search(r'bucket_schema="([0-9a-f]{16})"', text).group(1)assert 'openviking_retrieval_requests_total{context_type="memory"} 2' in text- assert 'openviking_retrieval_latency_seconds_count{context_type="memory"} 1' in textassert (- 'openviking_retrieval_latency_seconds_bucket{context_type="memory",le="0.05"} 1' in text+ f'openviking_retrieval_latency_seconds_count{{bucket_schema="{schema}",context_type="memory"}} 1'+ in text)assert (- 'openviking_retrieval_latency_seconds_bucket{context_type="memory",le="+Inf"} 1' in text+ f'openviking_retrieval_latency_seconds_bucket{{bucket_schema="{schema}",context_type="memory",le="0.05"}} 1'+ in text+ )+ assert (+ f'openviking_retrieval_latency_seconds_bucket{{bucket_schema="{schema}",context_type="memory",le="+Inf"}} 1'+ in text)assert 'openviking_cache_hits_total{level="L0"} 2' in textassert 'openviking_cache_misses_total{level="L1"} 1' in text@@ -109,9 +116,10 @@ class TestRetrievalStatsMetricsIntegration:'openviking_retrieval_results_total{account_id="__unknown__",context_type="memory"} 3'in text)- assert (- 'openviking_retrieval_latency_seconds_count{account_id="__unknown__",context_type="memory"} 1'- in text+ assert re.search(+ r'openviking_retrieval_latency_seconds_count\{account_id="__unknown__",'+ r'bucket_schema="[0-9a-f]{16}",context_type="memory"\} 1',+ text,)finally:shutdown_metrics(app=None)@@ -172,10 +180,11 @@ class TestMetricsEndpoint:transport = httpx.ASGITransport(app=app)try:registry = get_metrics_registry()- registry.counter(+ registry.inc_counter("openviking_retrieval_requests_total",label_names=("context_type",),- ).inc(labels={"context_type": "memory"})+ labels={"context_type": "memory"},+ )async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:resp = await client.get("/metrics")assert resp.status_code == 200diff --git a/uv.lock b/uv.lockindex 981f11393..3bc66e331 100644--- a/uv.lock+++ b/uv.lock@@ -3548,6 +3548,7 @@ dependencies = [{ name = "pathspec" },{ name = "pdfminer-six" },{ name = "pdfplumber" },+ { name = "prometheus-client" },{ name = "protobuf" },{ name = "pydantic" },{ name = "python-multipart" },@@ -3770,11 +3771,11 @@ requires-dist = [{ name = "opencode-ai", marker = "extra == 'bot'", specifier = ">=0.1.0a0" },{ name = "opensandbox", marker = "extra == 'bot'", specifier = ">=0.1.5" },{ name = "opensandbox-server", marker = "extra == 'bot'", specifier = ">=0.1.6" },- { name = "opentelemetry-api", specifier = ">=1.14" },- { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.14" },- { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.14" },+ { name = "opentelemetry-api", specifier = ">=1.40,<2" },+ { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.40,<2" },+ { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.40,<2" },{ name = "opentelemetry-instrumentation-asyncio", specifier = ">=0.61b0" },- { name = "opentelemetry-sdk", specifier = ">=1.14" },+ { name = "opentelemetry-sdk", specifier = ">=1.40,<2" },{ name = "openviking-sdk", specifier = ">=0.1.17" },{ name = "orjson", specifier = ">=3.11.7" },{ name = "packaging", specifier = ">=24.0" },@@ -3784,6 +3785,7 @@ requires-dist = [{ name = "pathspec", specifier = ">=1.1.1" },{ name = "pdfminer-six", specifier = ">=20251230" },{ name = "pdfplumber", specifier = ">=0.10.0" },+ { name = "prometheus-client", specifier = ">=0.24,<1" },{ name = "prompt-toolkit", marker = "extra == 'bot'", specifier = ">=3.0.0" },{ name = "protobuf", specifier = ">=6.33.5" },{ name = "psycopg2-binary", marker = "extra == 'opengauss'", specifier = ">=2.9,<3" },@@ -4355,6 +4357,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/9c/92/e18be996a01c7fd0e7dd7d198edefe42813cdfe1637bbbc80370ce656f62/primp-1.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:efadef0dfd10e733a254a949abf9ed05c668c28a68aa6513d811c0c6acd54cdb", size = 3611571, upload-time = "2026-03-11T06:43:31.249Z" },]+[[package]]+name = "prometheus-client"+version = "0.26.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" },+]+[[package]]name = "prompt-toolkit"version = "3.0.52"