Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Feat/memory extraction operation metrics (#4410)

Jiahui Zhou805633d5b83d
diff --git a/docs/design/metric-design.md b/docs/design/metric-design.md
index a5daaf8b5..ea8d212e3 100644
--- a/docs/design/metric-design.md
+++ b/docs/design/metric-design.md
@@ -785,7 +785,7 @@ sequenceDiagram
| `summary.vector.returned` | `openviking_vector_returned_total` |
| `summary.vector.scanned` | `openviking_vector_scanned_total` |
| `summary.semantic_nodes.{total|done|pending|running}` | `openviking_semantic_nodes_total{status=...}` |
-| `summary.memory.extracted` | `openviking_memory_extracted_total` |
+| `summary.memory.extracted` | `openviking_memory_extracted_total{memory_type=...}` |
### 3.8 多租户支持范围
diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md
index b286e04ab..a7068c455 100644
--- a/docs/en/concepts/12-metrics.md
+++ b/docs/en/concepts/12-metrics.md
@@ -186,7 +186,7 @@ Typical `stage` values include:
| `openviking_vector_passed_total` | Counter | `operation` | total passed candidates |
| `openviking_vector_returned_total` | Counter | `operation` | total returned candidates |
| `openviking_vector_scanned_total` | Counter | `operation` | total scanned candidates |
-| `openviking_memory_extracted_total` | Counter | `operation` | total extracted memory items |
+| `openviking_memory_extracted_total` | Counter | `operation`, `memory_type` | total extracted memory items, split by memory schema type |
| `openviking_semantic_nodes_total` | Counter | `status` | total semantic nodes |
### Model Calls and Tokens
diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md
index 40dbeac93..42569d09a 100644
--- a/docs/zh/concepts/12-metrics.md
+++ b/docs/zh/concepts/12-metrics.md
@@ -186,7 +186,7 @@ scrape_configs:
| `openviking_vector_passed_total` | Counter | `operation` | 向量候选通过数量累计 |
| `openviking_vector_returned_total` | Counter | `operation` | 向量候选返回数量累计 |
| `openviking_vector_scanned_total` | Counter | `operation` | 向量候选扫描数量累计 |
-| `openviking_memory_extracted_total` | Counter | `operation` | memory extracted 数量累计 |
+| `openviking_memory_extracted_total` | Counter | `operation`, `memory_type` | memory extracted 数量累计,按记忆 schema 类型拆分 |
| `openviking_semantic_nodes_total` | Counter | `status` | semantic nodes 数量累计 |
### 模型调用与 Token
diff --git a/examples/grafana/openviking_demo_dashboard.json b/examples/grafana/openviking_demo_dashboard.json
index 77da8533e..024762161 100644
--- a/examples/grafana/openviking_demo_dashboard.json
+++ b/examples/grafana/openviking_demo_dashboard.json
@@ -5045,6 +5045,131 @@
],
"title": "Tenant Pipeline Hotspots",
"type": "bargauge"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 196
+ },
+ "id": 59,
+ "panels": [],
+ "title": "Memory Extraction",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 12,
+ "lineWidth": 2,
+ "showPoints": "never",
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 197
+ },
+ "id": 60,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.4.1",
+ "targets": [
+ {
+ "editorMode": "code",
+ "expr": "sum by (memory_type) (rate(openviking_memory_extracted_total[$__rate_interval]))",
+ "legendFormat": "{{memory_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Extracted Memories / By Type",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "bars",
+ "fillOpacity": 50,
+ "lineWidth": 1,
+ "showPoints": "never",
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 197
+ },
+ "id": 61,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.4.1",
+ "targets": [
+ {
+ "editorMode": "code",
+ "expr": "sum by (memory_type, action, result) (rate(openviking_memory_operations_total[$__rate_interval]))",
+ "legendFormat": "{{memory_type}} / {{action}} / {{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Memory Operations / Type + Action",
+ "type": "timeseries"
}
],
"preload": false,
@@ -5114,4 +5239,4 @@
"uid": "openviking-demo",
"version": 39,
"weekStart": ""
-}
\ No newline at end of file
+}
diff --git a/openviking/metrics/collectors/embedding.py b/openviking/metrics/collectors/embedding.py
index 003eb8bf9..46723d192 100644
--- a/openviking/metrics/collectors/embedding.py
+++ b/openviking/metrics/collectors/embedding.py
@@ -91,6 +91,7 @@ class EmbeddingCollector(EventMetricCollector):
duration_seconds=float(payload["duration_seconds"]),
prompt_tokens=int(payload["prompt_tokens"]),
completion_tokens=int(payload["completion_tokens"]),
+ error_code=str(payload.get("error_code") or "OK"),
account_id=(
None if payload.get("account_id") is None else str(payload.get("account_id"))
),
@@ -123,44 +124,50 @@ class EmbeddingCollector(EventMetricCollector):
duration_seconds: float,
prompt_tokens: int,
completion_tokens: int,
+ error_code: str = "OK",
account_id: str | None = None,
) -> None:
"""Record one embedding provider call as calls/tokens counters and a latency sample."""
- labels = {"provider": str(provider), "model_name": str(model_name)}
+ labels = {
+ "provider": str(provider),
+ "model_name": str(model_name),
+ "error_code": str(error_code or "unknown"),
+ }
registry.inc_counter(
self.CALLS_TOTAL,
labels=labels,
- label_names=("provider", "model_name"),
+ label_names=("provider", "model_name", "error_code"),
account_id=account_id,
)
registry.observe_histogram(
self.CALL_DURATION_SECONDS,
float(duration_seconds),
labels=labels,
- label_names=("provider", "model_name"),
+ label_names=("provider", "model_name", "error_code"),
account_id=account_id,
)
- if int(prompt_tokens) > 0:
+ token_labels = {"provider": str(provider), "model_name": str(model_name)}
+ if labels["error_code"] == "OK" and int(prompt_tokens) > 0:
registry.inc_counter(
self.TOKENS_INPUT_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=int(prompt_tokens),
account_id=account_id,
)
- if int(completion_tokens) > 0:
+ if labels["error_code"] == "OK" and int(completion_tokens) > 0:
registry.inc_counter(
self.TOKENS_OUTPUT_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=int(completion_tokens),
account_id=account_id,
)
total_tokens = int(prompt_tokens) + int(completion_tokens)
- if total_tokens > 0:
+ if labels["error_code"] == "OK" and total_tokens > 0:
registry.inc_counter(
self.TOKENS_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=total_tokens,
account_id=account_id,
diff --git a/openviking/metrics/collectors/telemetry_bridge.py b/openviking/metrics/collectors/telemetry_bridge.py
index ae2296fe1..6546eaf66 100644
--- a/openviking/metrics/collectors/telemetry_bridge.py
+++ b/openviking/metrics/collectors/telemetry_bridge.py
@@ -257,17 +257,62 @@ class TelemetryBridgeCollector(EventMetricCollector):
memory = summary.get("memory") or {}
extracted = int(memory.get("extracted", 0) or 0)
- if extracted > 0:
+ extract = memory.get("extract") or {}
+ actions_by_type = extract.get("actions_by_type") or {}
+ if isinstance(actions_by_type, Mapping) and actions_by_type:
+ for memory_type, type_actions in actions_by_type.items():
+ if not isinstance(type_actions, Mapping):
+ continue
+ created = int(type_actions.get("created", 0) or 0)
+ merged = int(type_actions.get("merged", 0) or 0)
+ deleted = int(type_actions.get("deleted", 0) or 0)
+ # ``memory.extracted`` historically counts every returned
+ # memory context, including successful deletions. Keep the
+ # type breakdown additive with that established total.
+ extracted_for_type = created + merged + deleted
+ if extracted_for_type > 0:
+ registry.inc_counter(
+ self.MEMORY_EXTRACTED_TOTAL,
+ labels={"operation": operation, "memory_type": str(memory_type)},
+ label_names=("operation", "memory_type"),
+ amount=extracted_for_type,
+ )
+ elif extracted > 0:
registry.inc_counter(
self.MEMORY_EXTRACTED_TOTAL,
- labels={"operation": operation},
- label_names=("operation",),
+ labels={"operation": operation, "memory_type": "unknown"},
+ label_names=("operation", "memory_type"),
amount=extracted,
)
- extract = memory.get("extract") or {}
actions = extract.get("actions") or {}
- if isinstance(actions, Mapping):
+ if isinstance(actions_by_type, Mapping) and actions_by_type:
+ for memory_type, type_actions in actions_by_type.items():
+ if not isinstance(type_actions, Mapping):
+ continue
+ for action, result in (
+ ("created", "success"),
+ ("merged", "success"),
+ ("deleted", "success"),
+ ("skipped", "skipped"),
+ ("failed", "failed"),
+ ):
+ value = int(type_actions.get(action, 0) or 0)
+ if value <= 0:
+ continue
+ metric_action = "updated" if action == "merged" else action
+ registry.inc_counter(
+ self.MEMORY_OPERATIONS_TOTAL,
+ labels={
+ "operation": operation,
+ "memory_type": str(memory_type),
+ "action": metric_action,
+ "result": result,
+ },
+ label_names=("operation", "memory_type", "action", "result"),
+ amount=value,
+ )
+ elif isinstance(actions, Mapping):
for action, result in (
("created", "success"),
("merged", "success"),
@@ -283,10 +328,11 @@ class TelemetryBridgeCollector(EventMetricCollector):
self.MEMORY_OPERATIONS_TOTAL,
labels={
"operation": operation,
+ "memory_type": "unknown",
"action": metric_action,
"result": result,
},
- label_names=("operation", "action", "result"),
+ label_names=("operation", "memory_type", "action", "result"),
amount=value,
)
diff --git a/openviking/metrics/collectors/vlm.py b/openviking/metrics/collectors/vlm.py
index e62036629..8686e991b 100644
--- a/openviking/metrics/collectors/vlm.py
+++ b/openviking/metrics/collectors/vlm.py
@@ -79,6 +79,7 @@ class VLMCollector(EventMetricCollector):
duration_seconds=float(payload["duration_seconds"]),
prompt_tokens=int(payload["prompt_tokens"]),
completion_tokens=int(payload["completion_tokens"]),
+ error_code=str(payload.get("error_code") or "OK"),
account_id=(
None if payload.get("account_id") is None else str(payload.get("account_id"))
),
@@ -93,6 +94,7 @@ class VLMCollector(EventMetricCollector):
duration_seconds: float,
prompt_tokens: int,
completion_tokens: int,
+ error_code: str = "OK",
account_id: str | None = None,
) -> None:
"""
@@ -101,37 +103,46 @@ class VLMCollector(EventMetricCollector):
Input, output, and total token counters are all emitted because different dashboards care
about cost attribution at different levels of granularity.
"""
- labels = {"provider": str(provider), "model_name": str(model_name)}
+ labels = {
+ "provider": str(provider),
+ "model_name": str(model_name),
+ "error_code": str(error_code or "unknown"),
+ }
registry.inc_counter(
self.CALLS_TOTAL,
labels=labels,
- label_names=("provider", "model_name"),
+ label_names=("provider", "model_name", "error_code"),
account_id=account_id,
)
registry.observe_histogram(
self.CALL_DURATION_SECONDS,
float(duration_seconds),
labels=labels,
- label_names=("provider", "model_name"),
+ label_names=("provider", "model_name", "error_code"),
account_id=account_id,
)
+ # A failed attempt has no reliable usage payload. Keep token families
+ # success-only and avoid multiplying their cardinality by ``result``.
+ if labels["error_code"] != "OK":
+ return
+ token_labels = {"provider": str(provider), "model_name": str(model_name)}
registry.inc_counter(
self.TOKENS_INPUT_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=int(prompt_tokens),
account_id=account_id,
)
registry.inc_counter(
self.TOKENS_OUTPUT_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=int(completion_tokens),
account_id=account_id,
)
registry.inc_counter(
self.TOKENS_TOTAL,
- labels=labels,
+ labels=token_labels,
label_names=("provider", "model_name"),
amount=int(prompt_tokens) + int(completion_tokens),
account_id=account_id,
diff --git a/openviking/metrics/datasources/model_usage.py b/openviking/metrics/datasources/model_usage.py
index cdff63b09..3e16d4695 100644
--- a/openviking/metrics/datasources/model_usage.py
+++ b/openviking/metrics/datasources/model_usage.py
@@ -131,10 +131,11 @@ class VLMEventDataSource(EventMetricDataSource):
duration_seconds: float,
prompt_tokens: int,
completion_tokens: int,
+ error_code: str = "OK",
account_id: str | None = None,
) -> None:
"""
- Emit one VLM call event with model identity, latency, token usage, and account context.
+ Emit one VLM call event with model identity, latency, token usage, error code, and account context.
The caller is expected to provide already-normalized provider/model identifiers and the
final token counts that should be reflected in Prometheus usage metrics.
@@ -147,6 +148,7 @@ class VLMEventDataSource(EventMetricDataSource):
"duration_seconds": float(duration_seconds),
"prompt_tokens": int(prompt_tokens),
"completion_tokens": int(completion_tokens),
+ "error_code": str(error_code or "unknown"),
"account_id": None if account_id is None else str(account_id),
},
)
@@ -168,6 +170,7 @@ class EmbeddingEventDataSource(EventMetricDataSource):
duration_seconds: float,
prompt_tokens: int,
completion_tokens: int,
+ error_code: str = "OK",
account_id: str | None = None,
) -> None:
"""Emit one embedding provider call with tokens, latency, and optional account context."""
@@ -179,6 +182,7 @@ class EmbeddingEventDataSource(EventMetricDataSource):
"duration_seconds": float(duration_seconds),
"prompt_tokens": int(prompt_tokens),
"completion_tokens": int(completion_tokens),
+ "error_code": str(error_code or "unknown"),
"account_id": None if account_id is None else str(account_id),
},
)
diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.py
index bbec5e4ef..c874fd054 100644
--- a/openviking/models/embedder/volcengine_embedders.py
+++ b/openviking/models/embedder/volcengine_embedders.py
@@ -2,7 +2,9 @@
# SPDX-License-Identifier: AGPL-3.0
"""Volcengine Embedder Implementation"""
-from typing import Any, Dict, List, Optional
+import time
+
+from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar
import volcenginesdkarkruntime
@@ -16,11 +18,61 @@ from openviking.models.embedder.base import (
truncate_and_normalize,
)
from openviking.telemetry import get_current_telemetry
+from openviking.metrics.datasources import EmbeddingEventDataSource
+from openviking.observability.context import get_root_observability_context
+from openviking.utils.model_retry import extract_metric_error_code
from openviking.utils.async_client_cache import LoopScopedAsyncClientCache
from openviking_cli.utils.logger import default_logger as logger
VOLCENGINE_CLIENT_REQUEST_ID_HEADER = "X-Client-Request-Id"
VOLCENGINE_CLIENT_REQUEST_ID = "ToB-direct,OpenViking_Service,openviking-service_cn-beijing"
+T = TypeVar("T")
+
+
+def _record_failed_embedding_call(
+ embedder, *, duration_seconds: float, error: Exception
+) -> None:
+ """Emit a failed Ark request attempt with the same model labels as a success."""
+ try:
+ root_context = get_root_observability_context()
+ EmbeddingEventDataSource.record_call(
+ provider="volcengine",
+ model_name=str(embedder.model_name),
+ duration_seconds=max(float(duration_seconds), 0.0),
+ prompt_tokens=0,
+ completion_tokens=0,
+ error_code=extract_metric_error_code(error),
+ account_id=root_context.account_id if root_context is not None else None,
+ )
+ except Exception:
+ # Metrics must never change the provider failure behavior.
+ return
+
+
+def _measure_embedding_call(embedder, call: Callable[[], T]) -> tuple[T, float]:
+ """Measure one Ark SDK request and emit its failed-attempt metric when needed."""
+ started = time.perf_counter()
+ try:
+ return call(), time.perf_counter() - started
+ except Exception as error:
+ _record_failed_embedding_call(
+ embedder, duration_seconds=time.perf_counter() - started, error=error
+ )
+ raise
+
+
+async def _measure_embedding_call_async(
+ embedder, call: Callable[[], Awaitable[T]]
+) -> tuple[T, float]:
+ """Async equivalent of :func:`_measure_embedding_call`."""
+ started = time.perf_counter()
+ try:
+ return await call(), time.perf_counter() - started
+ except Exception as error:
+ _record_failed_embedding_call(
+ embedder, duration_seconds=time.perf_counter() - started, error=error
+ )
+ raise
def _build_volcengine_headers(extra_headers: Optional[Dict[str, str]]) -> Dict[str, str]:
@@ -144,7 +196,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
"""Multimodal inputs are supported when using the multimodal endpoint."""
return self.input_type == "multimodal"
- def _update_telemetry_token_usage(self, response) -> None:
+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:
usage = getattr(response, "usage", None)
if not usage:
return
@@ -171,6 +223,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
provider="volcengine",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
+ duration_seconds=duration_seconds,
)
def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
@@ -191,22 +244,32 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
def _embed_call():
if self.input_type == "multimodal":
# Use multimodal embeddings API
- response = self.client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- extra_headers=self.extra_headers,
+ response, duration_seconds = _measure_embedding_call(
+ self,
+ lambda: self.client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
vector = response.data.embedding
else:
# Use text embeddings API (text-only)
text = extract_text_from_content(content)
- response = self.client.embeddings.create(
- input=text,
- model=self.model_name,
- extra_headers=self.extra_headers,
+ response, duration_seconds = _measure_embedding_call(
+ self,
+ lambda: self.client.embeddings.create(
+ input=text,
+ model=self.model_name,
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
vector = response.data[0].embedding
vector = truncate_and_normalize(vector, self.dimension)
@@ -231,21 +294,31 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
async def _embed_call() -> EmbedResult:
if self.input_type == "multimodal":
- response = await client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- extra_headers=self.extra_headers,
+ response, duration_seconds = await _measure_embedding_call_async(
+ self,
+ lambda: client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
vector = response.data.embedding
else:
text = extract_text_from_content(content)
- response = await client.embeddings.create(
- input=text,
- model=self.model_name,
- extra_headers=self.extra_headers,
+ response, duration_seconds = await _measure_embedding_call_async(
+ self,
+ lambda: client.embeddings.create(
+ input=text,
+ model=self.model_name,
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
vector = response.data[0].embedding
return EmbedResult(dense_vector=truncate_and_normalize(vector, self.dimension))
@@ -308,7 +381,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
self._ark_kwargs = ark_kwargs
self._async_client_cache = LoopScopedAsyncClientCache()
- def _update_telemetry_token_usage(self, response) -> None:
+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:
usage = getattr(response, "usage", None)
if not usage:
return
@@ -335,6 +408,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
provider="volcengine",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
+ duration_seconds=duration_seconds,
)
def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
@@ -354,13 +428,18 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
def _embed_call():
# Must use multimodal endpoint for sparse
- response = self.client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
- extra_headers=self.extra_headers,
+ response, duration_seconds = _measure_embedding_call(
+ self,
+ lambda: self.client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ sparse_embedding={"type": "enabled"},
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
item = response.data
sparse_vector = getattr(item, "sparse_embedding", None)
return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))
@@ -383,13 +462,18 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
client = self._get_async_client()
async def _embed_call() -> EmbedResult:
- response = await client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
- extra_headers=self.extra_headers,
+ response, duration_seconds = await _measure_embedding_call_async(
+ self,
+ lambda: client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ sparse_embedding={"type": "enabled"},
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
item = response.data
sparse_vector = getattr(item, "sparse_embedding", None)
return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))
@@ -466,7 +550,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
self._async_client_cache = LoopScopedAsyncClientCache()
self._dimension = dimension or 2048
- def _update_telemetry_token_usage(self, response) -> None:
+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:
usage = getattr(response, "usage", None)
if not usage:
return
@@ -493,6 +577,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
provider="volcengine",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
+ duration_seconds=duration_seconds,
)
@property
@@ -524,13 +609,18 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
def _embed_call():
# Always use multimodal for hybrid to get both
- response = self.client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
- extra_headers=self.extra_headers,
+ response, duration_seconds = _measure_embedding_call(
+ self,
+ lambda: self.client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ sparse_embedding={"type": "enabled"},
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
item = response.data
dense_vector = truncate_and_normalize(item.embedding, self.dimension)
sparse_vector = getattr(item, "sparse_embedding", None)
@@ -557,13 +647,18 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
client = self._get_async_client()
async def _embed_call() -> EmbedResult:
- response = await client.multimodal_embeddings.create(
- input=to_multimodal_input(content),
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
- extra_headers=self.extra_headers,
+ response, duration_seconds = await _measure_embedding_call_async(
+ self,
+ lambda: client.multimodal_embeddings.create(
+ input=to_multimodal_input(content),
+ model=self.model_name,
+ sparse_embedding={"type": "enabled"},
+ extra_headers=self.extra_headers,
+ ),
+ )
+ self._update_telemetry_token_usage(
+ response, duration_seconds=duration_seconds
)
- self._update_telemetry_token_usage(response)
item = response.data
dense_vector = truncate_and_normalize(item.embedding, self.dimension)
sparse_vector = getattr(item, "sparse_embedding", None)
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index 6134faff6..f7f96dc02 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -194,7 +194,11 @@ class VolcEngineVLM(OpenAIVLM):
client = self.get_client()
t0 = time.perf_counter()
- response = client.chat.completions.create(**kwargs)
+ try:
+ response = client.chat.completions.create(**kwargs)
+ except Exception as error:
+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)
+ raise
elapsed = time.perf_counter() - t0
self._update_token_usage_from_response(response, duration_seconds=elapsed)
result = self._build_vlm_response(response, has_tools=bool(tools))
@@ -254,6 +258,7 @@ class VolcEngineVLM(OpenAIVLM):
tracer.info(f"message.content={content}")
return content
except Exception as e:
+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=e)
last_error = e
if attempt < self.max_retries:
await asyncio.sleep(2**attempt)
@@ -413,7 +418,11 @@ class VolcEngineVLM(OpenAIVLM):
client = self.get_client()
t0 = time.perf_counter()
- response = client.chat.completions.create(**kwargs)
+ try:
+ response = client.chat.completions.create(**kwargs)
+ except Exception as error:
+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)
+ raise
elapsed = time.perf_counter() - t0
self._update_token_usage_from_response(response, duration_seconds=elapsed)
result = self._build_vlm_response(response, has_tools=bool(tools))
@@ -457,7 +466,11 @@ class VolcEngineVLM(OpenAIVLM):
client = self.get_async_client()
t0 = time.perf_counter()
- response = await client.chat.completions.create(**kwargs)
+ try:
+ response = await client.chat.completions.create(**kwargs)
+ except Exception as error:
+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)
+ raise
elapsed = time.perf_counter() - t0
self._update_token_usage_from_response(response, duration_seconds=elapsed)
result = self._build_vlm_response(response, has_tools=bool(tools))
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index 7d092632b..5f9e7e931 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -14,6 +14,7 @@ from openviking.utils.model_retry import (
OrderedCredentialSwitcher,
PrimaryBackupSwitcher,
classify_api_error,
+ extract_metric_error_code,
)
from openviking_cli.utils import get_logger
@@ -288,6 +289,32 @@ class VLMBase(ABC):
e,
)
+ def record_failed_call(self, *, duration_seconds: float, error: Exception) -> None:
+ """Record one failed provider request attempt without token usage."""
+ try:
+ from openviking.metrics.datasources import VLMEventDataSource
+ from openviking.observability.context import get_root_observability_context
+
+ root_context = get_root_observability_context()
+ VLMEventDataSource.record_call(
+ provider=str(self.provider),
+ model_name=str(self.model or "unknown"),
+ duration_seconds=max(float(duration_seconds), 0.0),
+ prompt_tokens=0,
+ completion_tokens=0,
+ error_code=extract_metric_error_code(error),
+ account_id=root_context.account_id if root_context is not None else None,
+ )
+ except Exception as metrics_error:
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(
+ "vlm failed-call metrics emit failed provider=%s model_name=%s err=%s: %s",
+ self.provider,
+ self.model,
+ type(metrics_error).__name__,
+ metrics_error,
+ )
+
@property
def token_tracker(self):
"""Public accessor for this instance's token usage tracker."""
diff --git a/openviking/session/compressor_v3.py b/openviking/session/compressor_v3.py
index 5d1cb93df..cfcbfc84f 100644
--- a/openviking/session/compressor_v3.py
+++ b/openviking/session/compressor_v3.py
@@ -130,7 +130,23 @@ def _initialize_extraction_telemetry() -> None:
telemetry.set(name, 0)
-def _report_extraction_telemetry(result: Any) -> None:
+def _memory_type_by_uri(operations: ResolvedOperations) -> dict[str, str]:
+ """Map applied memory URIs to their stable extraction schema names."""
+ types_by_uri: dict[str, str] = {}
+ for operation in getattr(operations, "upsert_operations", []) or []:
+ memory_type = str(getattr(operation, "memory_type", "") or "unknown")
+ for uri in getattr(operation, "uris", []) or []:
+ types_by_uri[str(uri)] = memory_type
+ for file_content in getattr(operations, "delete_file_contents", []) or []:
+ uri = str(getattr(file_content, "uri", "") or "")
+ if uri:
+ types_by_uri[uri] = str(
+ getattr(file_content, "memory_type", "") or "unknown"
+ )
+ return types_by_uri
+
+
+def _report_extraction_telemetry(result: Any, operations: ResolvedOperations) -> None:
telemetry = get_current_telemetry()
telemetry.set(
"memory.extract.candidates.total",
@@ -142,6 +158,29 @@ def _report_extraction_telemetry(result: Any) -> None:
telemetry.set("memory.extract.skipped", len(result.skipped_operations))
telemetry.set("memory.extract.failed", len(result.errors))
+ types_by_uri = _memory_type_by_uri(operations)
+ actions_by_type: dict[str, dict[str, int]] = {}
+
+ def add(memory_type: Any, action: str) -> None:
+ normalized_type = str(memory_type or "unknown")
+ type_actions = actions_by_type.setdefault(normalized_type, {})
+ type_actions[action] = type_actions.get(action, 0) + 1
+
+ for uri in result.written_uris:
+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "created")
+ for uri in result.edited_uris:
+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "merged")
+ for uri in result.deleted_uris:
+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "deleted")
+ for operation in result.skipped_operations:
+ add(getattr(operation, "memory_type", None), "skipped")
+ for uri, _error in result.errors:
+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "failed")
+
+ for memory_type, actions in actions_by_type.items():
+ for action, value in actions.items():
+ telemetry.set(f"memory.extract.by_type.{memory_type}.{action}", value)
+
async def _commit_experience_snapshot(
viking_fs: Any,
@@ -705,7 +744,7 @@ class SessionCompressorV3:
result = update_result.apply_result
patch_operations = update_result.operations
- _report_extraction_telemetry(result)
+ _report_extraction_telemetry(result, patch_operations)
memory_diff = None
if archive_uri and viking_fs and result is not None:
diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.py
index 2c154b99c..a8951bba4 100644
--- a/openviking/telemetry/operation.py
+++ b/openviking/telemetry/operation.py
@@ -359,6 +359,16 @@ class TelemetrySummaryBuilder:
"extracted": memories_extracted,
}
if cls._has_metric_prefix("memory.extract", counters, gauges):
+ actions_by_type: Dict[str, Dict[str, int]] = {}
+ type_action_prefix = "memory.extract.by_type."
+ for key, value in gauges.items():
+ if not key.startswith(type_action_prefix):
+ continue
+ suffix = key[len(type_action_prefix) :]
+ memory_type, separator, action = suffix.rpartition(".")
+ if not separator or not memory_type or not action:
+ continue
+ actions_by_type.setdefault(memory_type, {})[action] = cls._i(value, 0)
memory_summary["extract"] = {
"duration_ms": cls._f(gauges.get("memory.extract.total.duration_ms"), 0.0),
"candidates": {
@@ -373,6 +383,7 @@ class TelemetrySummaryBuilder:
"skipped": cls._i(gauges.get("memory.extract.skipped"), 0),
"failed": cls._i(gauges.get("memory.extract.failed"), 0),
},
+ "actions_by_type": actions_by_type,
"stages": {
public_key: cls._f(gauges.get(metric_key), 0.0)
for public_key, metric_key in cls._MEMORY_EXTRACT_STAGE_KEYS.items()
diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.py
index eac035643..35a089dce 100644
--- a/openviking/utils/model_retry.py
+++ b/openviking/utils/model_retry.py
@@ -23,6 +23,57 @@ ERROR_CLASS_QUOTA_EXCEEDED = "quota_exceeded"
ERROR_CLASS_TRANSIENT = "transient"
ERROR_CLASS_UNKNOWN = "unknown"
+_METRIC_ERROR_CODE_MAX_LENGTH = 64
+
+
+def _normalize_metric_error_code(value: object) -> str | None:
+ """Return a bounded structured error code suitable for a metric label."""
+ if value is None:
+ return None
+ if isinstance(value, int) and 100 <= value <= 599:
+ return str(value)
+ if not isinstance(value, str):
+ return None
+ code = value.strip()
+ if not code or len(code) > _METRIC_ERROR_CODE_MAX_LENGTH:
+ return None
+ if not re.fullmatch(r"[A-Za-z0-9_.-]+", code):
+ return None
+ return code
+
+
+def extract_metric_error_code(error: BaseException) -> str:
+ """Extract a low-cardinality provider error code for model-call metrics.
+
+ Structured HTTP/SDK error codes are preferred over message parsing. Free-form
+ provider messages and request IDs are intentionally never used as metric labels.
+ """
+ error_chain = _iter_exception_chain(error)
+ for exc in error_chain:
+ for attr in ("status_code", "error_code", "code"):
+ normalized = _normalize_metric_error_code(getattr(exc, attr, None))
+ if normalized is not None:
+ return normalized
+
+ body = getattr(exc, "body", None)
+ if isinstance(body, dict):
+ candidates = [body.get("status_code"), body.get("error_code"), body.get("code")]
+ nested = body.get("error")
+ if isinstance(nested, dict):
+ candidates.extend(
+ [nested.get("status_code"), nested.get("error_code"), nested.get("code")]
+ )
+ for value in candidates:
+ normalized = _normalize_metric_error_code(value)
+ if normalized is not None:
+ return normalized
+
+ if any(isinstance(exc, TimeoutError) for exc in error_chain):
+ return "timeout"
+ if any(isinstance(exc, ConnectionError) for exc in error_chain):
+ return "connection_error"
+ return "unknown"
+
INPUT_TOO_LARGE_PATTERNS = (
"413",
"payload too large",
diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.py
index 2579af669..fc4491e50 100644
--- a/tests/metrics/collectors/test_event_collectors.py
+++ b/tests/metrics/collectors/test_event_collectors.py
@@ -13,6 +13,7 @@ from openviking.metrics.collectors.rerank import RerankCollector
from openviking.metrics.collectors.retrieval import RetrievalCollector
from openviking.metrics.collectors.session import SessionCollector
from openviking.metrics.collectors.telemetry_bridge import TelemetryBridgeCollector
+from openviking.metrics.collectors.vlm import VLMCollector
class _DummyEventCollector(EventMetricCollector):
@@ -213,11 +214,11 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome
# Account dimension is supported for embedding families; default runtime resolves to __unknown__.
assert re.search(
- r'openviking_embedding_calls_total\{(?=[^}]*account_id="__unknown__")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',
+ r'openviking_embedding_calls_total\{(?=[^}]*account_id="__unknown__")(?=[^}]*error_code="OK")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',
text,
)
assert re.search(
- r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*account_id="__unknown__")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',
+ r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*account_id="__unknown__")(?=[^}]*error_code="OK")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',
text,
)
assert re.search(
@@ -234,6 +235,58 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome
)
+def test_embedding_collector_records_failed_call_duration_with_error_code(registry, render_prometheus):
+ EmbeddingCollector().receive(
+ "embedding.call",
+ {
+ "provider": "volcengine",
+ "model_name": "doubao-embedding",
+ "duration_seconds": 0.12,
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "error_code": "429",
+ },
+ registry,
+ )
+ text = render_prometheus(registry)
+
+ assert re.search(
+ r'openviking_embedding_calls_total\{(?=[^}]*error_code="429")(?=[^}]*model_name="doubao-embedding")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',
+ text,
+ )
+ assert re.search(
+ r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*error_code="429")(?=[^}]*model_name="doubao-embedding")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',
+ text,
+ )
+ assert "openviking_embedding_tokens_total" not in text
+
+
+def test_vlm_collector_records_error_code_on_calls_and_duration_only(registry, render_prometheus):
+ VLMCollector().receive(
+ "vlm.call",
+ {
+ "provider": "volcengine",
+ "model_name": "doubao-vlm",
+ "duration_seconds": 0.12,
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "error_code": "timeout",
+ },
+ registry,
+ )
+ text = render_prometheus(registry)
+
+ assert re.search(
+ r'openviking_vlm_calls_total\{(?=[^}]*error_code="timeout")(?=[^}]*model_name="doubao-vlm")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',
+ text,
+ )
+ assert re.search(
+ r'openviking_vlm_call_duration_seconds_count\{(?=[^}]*error_code="timeout")(?=[^}]*model_name="doubao-vlm")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',
+ text,
+ )
+ assert "openviking_vlm_tokens_total" not in text
+
+
def test_embedding_collector_records_success_volume_and_latency(registry, render_prometheus):
"""`embedding.success` must increment request volume and observe latency."""
EmbeddingCollector().receive(
diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.py
index 4db5d8d11..edb0c8b5f 100644
--- a/tests/metrics/integration/test_telemetry_bridge.py
+++ b/tests/metrics/integration/test_telemetry_bridge.py
@@ -37,7 +37,11 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende
"deleted": 1,
"skipped": 3,
"failed": 1,
- }
+ },
+ "actions_by_type": {
+ "preferences": {"created": 2, "skipped": 3},
+ "events": {"merged": 1, "deleted": 1, "failed": 1},
+ },
},
},
"semantic_nodes": {"OK": 12},
@@ -67,25 +71,32 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende
in text
)
assert 'openviking_vector_searches_total{operation="resource.process"} 1' in text
- assert 'openviking_memory_extracted_total{operation="resource.process"} 4' in text
assert (
- 'openviking_memory_operations_total{action="created",operation="resource.process",result="success"} 2'
+ 'openviking_memory_extracted_total{memory_type="preferences",operation="resource.process"} 2'
in text
)
assert (
- 'openviking_memory_operations_total{action="updated",operation="resource.process",result="success"} 1'
+ 'openviking_memory_extracted_total{memory_type="events",operation="resource.process"} 2'
in text
)
assert (
- 'openviking_memory_operations_total{action="deleted",operation="resource.process",result="success"} 1'
+ 'openviking_memory_operations_total{action="created",memory_type="preferences",operation="resource.process",result="success"} 2'
in text
)
assert (
- 'openviking_memory_operations_total{action="skipped",operation="resource.process",result="skipped"} 3'
+ 'openviking_memory_operations_total{action="updated",memory_type="events",operation="resource.process",result="success"} 1'
in text
)
assert (
- 'openviking_memory_operations_total{action="failed",operation="resource.process",result="failed"} 1'
+ 'openviking_memory_operations_total{action="deleted",memory_type="events",operation="resource.process",result="success"} 1'
+ in text
+ )
+ assert (
+ 'openviking_memory_operations_total{action="skipped",memory_type="preferences",operation="resource.process",result="skipped"} 3'
+ in text
+ )
+ assert (
+ 'openviking_memory_operations_total{action="failed",memory_type="events",operation="resource.process",result="failed"} 1'
in text
)
assert (
diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.py
index 66246eddf..4d246cd1e 100644
--- a/tests/unit/test_model_retry.py
+++ b/tests/unit/test_model_retry.py
@@ -13,6 +13,7 @@ from openviking.utils.model_retry import (
ERROR_CLASS_QUOTA_EXCEEDED,
ERROR_CLASS_TRANSIENT,
classify_api_error,
+ extract_metric_error_code,
retry_async,
retry_sync,
)
@@ -22,6 +23,31 @@ def test_classify_api_error_recognizes_request_burst_too_fast():
assert classify_api_error(RuntimeError("RequestBurstTooFast")) == ERROR_CLASS_TRANSIENT
+class _ProviderError(RuntimeError):
+ def __init__(self, *, status_code=None, error_code=None, code=None, body=None):
+ super().__init__("provider request failed")
+ self.status_code = status_code
+ self.error_code = error_code
+ self.code = code
+ self.body = body
+
+
+def test_extract_metric_error_code_prefers_structured_provider_code():
+ assert extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded")) == "429"
+ assert extract_metric_error_code(_ProviderError(body={"error": {"code": "InvalidParameter"}})) == (
+ "InvalidParameter"
+ )
+
+
+def test_extract_metric_error_code_uses_safe_fallbacks_only():
+ assert extract_metric_error_code(TimeoutError("request timed out")) == "timeout"
+ assert extract_metric_error_code(ConnectionError("connection reset")) == "connection_error"
+ wrapped_timeout = RuntimeError("request failed")
+ wrapped_timeout.__cause__ = TimeoutError("request timed out")
+ assert extract_metric_error_code(wrapped_timeout) == "timeout"
+ assert extract_metric_error_code(RuntimeError("request_id=not-a-metric-label")) == "unknown"
+
+
def test_classify_all_credentials_failed_prefers_transient_over_auth():
err = AllCredentialsFailedError(
[
diff --git a/tests/unit/test_volcengine_embedder_input.py b/tests/unit/test_volcengine_embedder_input.py
index e0f74b19d..9cd5aad6b 100644
--- a/tests/unit/test_volcengine_embedder_input.py
+++ b/tests/unit/test_volcengine_embedder_input.py
@@ -2,9 +2,11 @@
# SPDX-License-Identifier: AGPL-3.0
"""Input-guard tests for Volcengine sparse-capable embedding modes."""
-from unittest.mock import patch
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
from openviking.models.embedder.volcengine_embedders import (
+ VolcengineDenseEmbedder,
VolcengineHybridEmbedder,
VolcengineSparseEmbedder,
)
@@ -30,3 +32,26 @@ def test_hybrid_embedder_downgrades_multimodal_input_to_text(_mock_ark):
assert embedder.supports_multimodal is False
assert embedder.prepare_embedding_input(_MULTIMODAL_INPUT) == "diagram of a heat exchanger"
+
+
+@patch("openviking.models.embedder.volcengine_embedders.volcenginesdkarkruntime.Ark")
+@patch("openviking.models.embedder.volcengine_embedders.time.perf_counter")
+def test_dense_embedder_passes_local_provider_duration_to_metrics(mock_perf_counter, mock_ark):
+ """Embedding call duration must come from this SDK request, not shared embedder state."""
+ response = SimpleNamespace(
+ usage=SimpleNamespace(prompt_tokens=3, total_tokens=3),
+ data=[SimpleNamespace(embedding=[1.0, 0.0])],
+ )
+ client = MagicMock()
+ client.embeddings.create.return_value = response
+ mock_ark.return_value = client
+ mock_perf_counter.side_effect = [100.0, 100.25]
+
+ embedder = VolcengineDenseEmbedder(
+ model_name="test", api_key="test-key", input_type="text", dimension=2
+ )
+ embedder.update_token_usage = MagicMock()
+
+ embedder.embed("hello")
+
+ assert embedder.update_token_usage.call_args.kwargs["duration_seconds"] == 0.25