Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feat/memory extraction operation metrics (#4410)
Jiahui Zhou805633d5b83d
diff --git a/docs/design/metric-design.md b/docs/design/metric-design.mdindex a5daaf8b5..ea8d212e3 100644--- a/docs/design/metric-design.md+++ b/docs/design/metric-design.md@@ -785,7 +785,7 @@ sequenceDiagram| `summary.vector.returned` | `openviking_vector_returned_total` || `summary.vector.scanned` | `openviking_vector_scanned_total` || `summary.semantic_nodes.{total|done|pending|running}` | `openviking_semantic_nodes_total{status=...}` |-| `summary.memory.extracted` | `openviking_memory_extracted_total` |+| `summary.memory.extracted` | `openviking_memory_extracted_total{memory_type=...}` |### 3.8 多租户支持范围diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.mdindex b286e04ab..a7068c455 100644--- a/docs/en/concepts/12-metrics.md+++ b/docs/en/concepts/12-metrics.md@@ -186,7 +186,7 @@ Typical `stage` values include:| `openviking_vector_passed_total` | Counter | `operation` | total passed candidates || `openviking_vector_returned_total` | Counter | `operation` | total returned candidates || `openviking_vector_scanned_total` | Counter | `operation` | total scanned candidates |-| `openviking_memory_extracted_total` | Counter | `operation` | total extracted memory items |+| `openviking_memory_extracted_total` | Counter | `operation`, `memory_type` | total extracted memory items, split by memory schema type || `openviking_semantic_nodes_total` | Counter | `status` | total semantic nodes |### Model Calls and Tokensdiff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.mdindex 40dbeac93..42569d09a 100644--- a/docs/zh/concepts/12-metrics.md+++ b/docs/zh/concepts/12-metrics.md@@ -186,7 +186,7 @@ scrape_configs:| `openviking_vector_passed_total` | Counter | `operation` | 向量候选通过数量累计 || `openviking_vector_returned_total` | Counter | `operation` | 向量候选返回数量累计 || `openviking_vector_scanned_total` | Counter | `operation` | 向量候选扫描数量累计 |-| `openviking_memory_extracted_total` | Counter | `operation` | memory extracted 数量累计 |+| `openviking_memory_extracted_total` | Counter | `operation`, `memory_type` | memory extracted 数量累计,按记忆 schema 类型拆分 || `openviking_semantic_nodes_total` | Counter | `status` | semantic nodes 数量累计 |### 模型调用与 Tokendiff --git a/examples/grafana/openviking_demo_dashboard.json b/examples/grafana/openviking_demo_dashboard.jsonindex 77da8533e..024762161 100644--- a/examples/grafana/openviking_demo_dashboard.json+++ b/examples/grafana/openviking_demo_dashboard.json@@ -5045,6 +5045,131 @@],"title": "Tenant Pipeline Hotspots","type": "bargauge"+ },+ {+ "collapsed": false,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 196+ },+ "id": 59,+ "panels": [],+ "title": "Memory Extraction",+ "type": "row"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "drawStyle": "line",+ "fillOpacity": 12,+ "lineWidth": 2,+ "showPoints": "never",+ "stacking": {+ "group": "A",+ "mode": "normal"+ }+ },+ "unit": "ops"+ },+ "overrides": []+ },+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 197+ },+ "id": 60,+ "options": {+ "legend": {+ "calcs": [],+ "displayMode": "table",+ "placement": "right",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "pluginVersion": "12.4.1",+ "targets": [+ {+ "editorMode": "code",+ "expr": "sum by (memory_type) (rate(openviking_memory_extracted_total[$__rate_interval]))",+ "legendFormat": "{{memory_type}}",+ "range": true,+ "refId": "A"+ }+ ],+ "title": "Extracted Memories / By Type",+ "type": "timeseries"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "drawStyle": "bars",+ "fillOpacity": 50,+ "lineWidth": 1,+ "showPoints": "never",+ "stacking": {+ "group": "A",+ "mode": "normal"+ }+ },+ "unit": "ops"+ },+ "overrides": []+ },+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 197+ },+ "id": 61,+ "options": {+ "legend": {+ "calcs": [],+ "displayMode": "table",+ "placement": "right",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "pluginVersion": "12.4.1",+ "targets": [+ {+ "editorMode": "code",+ "expr": "sum by (memory_type, action, result) (rate(openviking_memory_operations_total[$__rate_interval]))",+ "legendFormat": "{{memory_type}} / {{action}} / {{result}}",+ "range": true,+ "refId": "A"+ }+ ],+ "title": "Memory Operations / Type + Action",+ "type": "timeseries"}],"preload": false,@@ -5114,4 +5239,4 @@"uid": "openviking-demo","version": 39,"weekStart": ""-}\ No newline at end of file+}diff --git a/openviking/metrics/collectors/embedding.py b/openviking/metrics/collectors/embedding.pyindex 003eb8bf9..46723d192 100644--- a/openviking/metrics/collectors/embedding.py+++ b/openviking/metrics/collectors/embedding.py@@ -91,6 +91,7 @@ class EmbeddingCollector(EventMetricCollector):duration_seconds=float(payload["duration_seconds"]),prompt_tokens=int(payload["prompt_tokens"]),completion_tokens=int(payload["completion_tokens"]),+ error_code=str(payload.get("error_code") or "OK"),account_id=(None if payload.get("account_id") is None else str(payload.get("account_id"))),@@ -123,44 +124,50 @@ class EmbeddingCollector(EventMetricCollector):duration_seconds: float,prompt_tokens: int,completion_tokens: int,+ error_code: str = "OK",account_id: str | None = None,) -> None:"""Record one embedding provider call as calls/tokens counters and a latency sample."""- labels = {"provider": str(provider), "model_name": str(model_name)}+ labels = {+ "provider": str(provider),+ "model_name": str(model_name),+ "error_code": str(error_code or "unknown"),+ }registry.inc_counter(self.CALLS_TOTAL,labels=labels,- label_names=("provider", "model_name"),+ label_names=("provider", "model_name", "error_code"),account_id=account_id,)registry.observe_histogram(self.CALL_DURATION_SECONDS,float(duration_seconds),labels=labels,- label_names=("provider", "model_name"),+ label_names=("provider", "model_name", "error_code"),account_id=account_id,)- if int(prompt_tokens) > 0:+ token_labels = {"provider": str(provider), "model_name": str(model_name)}+ if labels["error_code"] == "OK" and int(prompt_tokens) > 0:registry.inc_counter(self.TOKENS_INPUT_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=int(prompt_tokens),account_id=account_id,)- if int(completion_tokens) > 0:+ if labels["error_code"] == "OK" and int(completion_tokens) > 0:registry.inc_counter(self.TOKENS_OUTPUT_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=int(completion_tokens),account_id=account_id,)total_tokens = int(prompt_tokens) + int(completion_tokens)- if total_tokens > 0:+ if labels["error_code"] == "OK" and total_tokens > 0:registry.inc_counter(self.TOKENS_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=total_tokens,account_id=account_id,diff --git a/openviking/metrics/collectors/telemetry_bridge.py b/openviking/metrics/collectors/telemetry_bridge.pyindex ae2296fe1..6546eaf66 100644--- a/openviking/metrics/collectors/telemetry_bridge.py+++ b/openviking/metrics/collectors/telemetry_bridge.py@@ -257,17 +257,62 @@ class TelemetryBridgeCollector(EventMetricCollector):memory = summary.get("memory") or {}extracted = int(memory.get("extracted", 0) or 0)- if extracted > 0:+ extract = memory.get("extract") or {}+ actions_by_type = extract.get("actions_by_type") or {}+ if isinstance(actions_by_type, Mapping) and actions_by_type:+ for memory_type, type_actions in actions_by_type.items():+ if not isinstance(type_actions, Mapping):+ continue+ created = int(type_actions.get("created", 0) or 0)+ merged = int(type_actions.get("merged", 0) or 0)+ deleted = int(type_actions.get("deleted", 0) or 0)+ # ``memory.extracted`` historically counts every returned+ # memory context, including successful deletions. Keep the+ # type breakdown additive with that established total.+ extracted_for_type = created + merged + deleted+ if extracted_for_type > 0:+ registry.inc_counter(+ self.MEMORY_EXTRACTED_TOTAL,+ labels={"operation": operation, "memory_type": str(memory_type)},+ label_names=("operation", "memory_type"),+ amount=extracted_for_type,+ )+ elif extracted > 0:registry.inc_counter(self.MEMORY_EXTRACTED_TOTAL,- labels={"operation": operation},- label_names=("operation",),+ labels={"operation": operation, "memory_type": "unknown"},+ label_names=("operation", "memory_type"),amount=extracted,)- extract = memory.get("extract") or {}actions = extract.get("actions") or {}- if isinstance(actions, Mapping):+ if isinstance(actions_by_type, Mapping) and actions_by_type:+ for memory_type, type_actions in actions_by_type.items():+ if not isinstance(type_actions, Mapping):+ continue+ for action, result in (+ ("created", "success"),+ ("merged", "success"),+ ("deleted", "success"),+ ("skipped", "skipped"),+ ("failed", "failed"),+ ):+ value = int(type_actions.get(action, 0) or 0)+ if value <= 0:+ continue+ metric_action = "updated" if action == "merged" else action+ registry.inc_counter(+ self.MEMORY_OPERATIONS_TOTAL,+ labels={+ "operation": operation,+ "memory_type": str(memory_type),+ "action": metric_action,+ "result": result,+ },+ label_names=("operation", "memory_type", "action", "result"),+ amount=value,+ )+ elif isinstance(actions, Mapping):for action, result in (("created", "success"),("merged", "success"),@@ -283,10 +328,11 @@ class TelemetryBridgeCollector(EventMetricCollector):self.MEMORY_OPERATIONS_TOTAL,labels={"operation": operation,+ "memory_type": "unknown","action": metric_action,"result": result,},- label_names=("operation", "action", "result"),+ label_names=("operation", "memory_type", "action", "result"),amount=value,)diff --git a/openviking/metrics/collectors/vlm.py b/openviking/metrics/collectors/vlm.pyindex e62036629..8686e991b 100644--- a/openviking/metrics/collectors/vlm.py+++ b/openviking/metrics/collectors/vlm.py@@ -79,6 +79,7 @@ class VLMCollector(EventMetricCollector):duration_seconds=float(payload["duration_seconds"]),prompt_tokens=int(payload["prompt_tokens"]),completion_tokens=int(payload["completion_tokens"]),+ error_code=str(payload.get("error_code") or "OK"),account_id=(None if payload.get("account_id") is None else str(payload.get("account_id"))),@@ -93,6 +94,7 @@ class VLMCollector(EventMetricCollector):duration_seconds: float,prompt_tokens: int,completion_tokens: int,+ error_code: str = "OK",account_id: str | None = None,) -> None:"""@@ -101,37 +103,46 @@ class VLMCollector(EventMetricCollector):Input, output, and total token counters are all emitted because different dashboards careabout cost attribution at different levels of granularity."""- labels = {"provider": str(provider), "model_name": str(model_name)}+ labels = {+ "provider": str(provider),+ "model_name": str(model_name),+ "error_code": str(error_code or "unknown"),+ }registry.inc_counter(self.CALLS_TOTAL,labels=labels,- label_names=("provider", "model_name"),+ label_names=("provider", "model_name", "error_code"),account_id=account_id,)registry.observe_histogram(self.CALL_DURATION_SECONDS,float(duration_seconds),labels=labels,- label_names=("provider", "model_name"),+ label_names=("provider", "model_name", "error_code"),account_id=account_id,)+ # A failed attempt has no reliable usage payload. Keep token families+ # success-only and avoid multiplying their cardinality by ``result``.+ if labels["error_code"] != "OK":+ return+ token_labels = {"provider": str(provider), "model_name": str(model_name)}registry.inc_counter(self.TOKENS_INPUT_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=int(prompt_tokens),account_id=account_id,)registry.inc_counter(self.TOKENS_OUTPUT_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=int(completion_tokens),account_id=account_id,)registry.inc_counter(self.TOKENS_TOTAL,- labels=labels,+ labels=token_labels,label_names=("provider", "model_name"),amount=int(prompt_tokens) + int(completion_tokens),account_id=account_id,diff --git a/openviking/metrics/datasources/model_usage.py b/openviking/metrics/datasources/model_usage.pyindex cdff63b09..3e16d4695 100644--- a/openviking/metrics/datasources/model_usage.py+++ b/openviking/metrics/datasources/model_usage.py@@ -131,10 +131,11 @@ class VLMEventDataSource(EventMetricDataSource):duration_seconds: float,prompt_tokens: int,completion_tokens: int,+ error_code: str = "OK",account_id: str | None = None,) -> None:"""- Emit one VLM call event with model identity, latency, token usage, and account context.+ Emit one VLM call event with model identity, latency, token usage, error code, and account context.The caller is expected to provide already-normalized provider/model identifiers and thefinal token counts that should be reflected in Prometheus usage metrics.@@ -147,6 +148,7 @@ class VLMEventDataSource(EventMetricDataSource):"duration_seconds": float(duration_seconds),"prompt_tokens": int(prompt_tokens),"completion_tokens": int(completion_tokens),+ "error_code": str(error_code or "unknown"),"account_id": None if account_id is None else str(account_id),},)@@ -168,6 +170,7 @@ class EmbeddingEventDataSource(EventMetricDataSource):duration_seconds: float,prompt_tokens: int,completion_tokens: int,+ error_code: str = "OK",account_id: str | None = None,) -> None:"""Emit one embedding provider call with tokens, latency, and optional account context."""@@ -179,6 +182,7 @@ class EmbeddingEventDataSource(EventMetricDataSource):"duration_seconds": float(duration_seconds),"prompt_tokens": int(prompt_tokens),"completion_tokens": int(completion_tokens),+ "error_code": str(error_code or "unknown"),"account_id": None if account_id is None else str(account_id),},)diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.pyindex bbec5e4ef..c874fd054 100644--- a/openviking/models/embedder/volcengine_embedders.py+++ b/openviking/models/embedder/volcengine_embedders.py@@ -2,7 +2,9 @@# SPDX-License-Identifier: AGPL-3.0"""Volcengine Embedder Implementation"""-from typing import Any, Dict, List, Optional+import time++from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVarimport volcenginesdkarkruntime@@ -16,11 +18,61 @@ from openviking.models.embedder.base import (truncate_and_normalize,)from openviking.telemetry import get_current_telemetry+from openviking.metrics.datasources import EmbeddingEventDataSource+from openviking.observability.context import get_root_observability_context+from openviking.utils.model_retry import extract_metric_error_codefrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils.logger import default_logger as loggerVOLCENGINE_CLIENT_REQUEST_ID_HEADER = "X-Client-Request-Id"VOLCENGINE_CLIENT_REQUEST_ID = "ToB-direct,OpenViking_Service,openviking-service_cn-beijing"+T = TypeVar("T")+++def _record_failed_embedding_call(+ embedder, *, duration_seconds: float, error: Exception+) -> None:+ """Emit a failed Ark request attempt with the same model labels as a success."""+ try:+ root_context = get_root_observability_context()+ EmbeddingEventDataSource.record_call(+ provider="volcengine",+ model_name=str(embedder.model_name),+ duration_seconds=max(float(duration_seconds), 0.0),+ prompt_tokens=0,+ completion_tokens=0,+ error_code=extract_metric_error_code(error),+ account_id=root_context.account_id if root_context is not None else None,+ )+ except Exception:+ # Metrics must never change the provider failure behavior.+ return+++def _measure_embedding_call(embedder, call: Callable[[], T]) -> tuple[T, float]:+ """Measure one Ark SDK request and emit its failed-attempt metric when needed."""+ started = time.perf_counter()+ try:+ return call(), time.perf_counter() - started+ except Exception as error:+ _record_failed_embedding_call(+ embedder, duration_seconds=time.perf_counter() - started, error=error+ )+ raise+++async def _measure_embedding_call_async(+ embedder, call: Callable[[], Awaitable[T]]+) -> tuple[T, float]:+ """Async equivalent of :func:`_measure_embedding_call`."""+ started = time.perf_counter()+ try:+ return await call(), time.perf_counter() - started+ except Exception as error:+ _record_failed_embedding_call(+ embedder, duration_seconds=time.perf_counter() - started, error=error+ )+ raisedef _build_volcengine_headers(extra_headers: Optional[Dict[str, str]]) -> Dict[str, str]:@@ -144,7 +196,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):"""Multimodal inputs are supported when using the multimodal endpoint."""return self.input_type == "multimodal"- def _update_telemetry_token_usage(self, response) -> None:+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:usage = getattr(response, "usage", None)if not usage:return@@ -171,6 +223,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):provider="volcengine",prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,+ duration_seconds=duration_seconds,)def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:@@ -191,22 +244,32 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):def _embed_call():if self.input_type == "multimodal":# Use multimodal embeddings API- response = self.client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- extra_headers=self.extra_headers,+ response, duration_seconds = _measure_embedding_call(+ self,+ lambda: self.client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)vector = response.data.embeddingelse:# Use text embeddings API (text-only)text = extract_text_from_content(content)- response = self.client.embeddings.create(- input=text,- model=self.model_name,- extra_headers=self.extra_headers,+ response, duration_seconds = _measure_embedding_call(+ self,+ lambda: self.client.embeddings.create(+ input=text,+ model=self.model_name,+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)vector = response.data[0].embeddingvector = truncate_and_normalize(vector, self.dimension)@@ -231,21 +294,31 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):async def _embed_call() -> EmbedResult:if self.input_type == "multimodal":- response = await client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- extra_headers=self.extra_headers,+ response, duration_seconds = await _measure_embedding_call_async(+ self,+ lambda: client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)vector = response.data.embeddingelse:text = extract_text_from_content(content)- response = await client.embeddings.create(- input=text,- model=self.model_name,- extra_headers=self.extra_headers,+ response, duration_seconds = await _measure_embedding_call_async(+ self,+ lambda: client.embeddings.create(+ input=text,+ model=self.model_name,+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)vector = response.data[0].embeddingreturn EmbedResult(dense_vector=truncate_and_normalize(vector, self.dimension))@@ -308,7 +381,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):self._ark_kwargs = ark_kwargsself._async_client_cache = LoopScopedAsyncClientCache()- def _update_telemetry_token_usage(self, response) -> None:+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:usage = getattr(response, "usage", None)if not usage:return@@ -335,6 +408,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):provider="volcengine",prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,+ duration_seconds=duration_seconds,)def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:@@ -354,13 +428,18 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):def _embed_call():# Must use multimodal endpoint for sparse- response = self.client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- sparse_embedding={"type": "enabled"},- extra_headers=self.extra_headers,+ response, duration_seconds = _measure_embedding_call(+ self,+ lambda: self.client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ sparse_embedding={"type": "enabled"},+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)item = response.datasparse_vector = getattr(item, "sparse_embedding", None)return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))@@ -383,13 +462,18 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):client = self._get_async_client()async def _embed_call() -> EmbedResult:- response = await client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- sparse_embedding={"type": "enabled"},- extra_headers=self.extra_headers,+ response, duration_seconds = await _measure_embedding_call_async(+ self,+ lambda: client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ sparse_embedding={"type": "enabled"},+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)item = response.datasparse_vector = getattr(item, "sparse_embedding", None)return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))@@ -466,7 +550,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):self._async_client_cache = LoopScopedAsyncClientCache()self._dimension = dimension or 2048- def _update_telemetry_token_usage(self, response) -> None:+ def _update_telemetry_token_usage(self, response, *, duration_seconds: float) -> None:usage = getattr(response, "usage", None)if not usage:return@@ -493,6 +577,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):provider="volcengine",prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,+ duration_seconds=duration_seconds,)@property@@ -524,13 +609,18 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):def _embed_call():# Always use multimodal for hybrid to get both- response = self.client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- sparse_embedding={"type": "enabled"},- extra_headers=self.extra_headers,+ response, duration_seconds = _measure_embedding_call(+ self,+ lambda: self.client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ sparse_embedding={"type": "enabled"},+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)item = response.datadense_vector = truncate_and_normalize(item.embedding, self.dimension)sparse_vector = getattr(item, "sparse_embedding", None)@@ -557,13 +647,18 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):client = self._get_async_client()async def _embed_call() -> EmbedResult:- response = await client.multimodal_embeddings.create(- input=to_multimodal_input(content),- model=self.model_name,- sparse_embedding={"type": "enabled"},- extra_headers=self.extra_headers,+ response, duration_seconds = await _measure_embedding_call_async(+ self,+ lambda: client.multimodal_embeddings.create(+ input=to_multimodal_input(content),+ model=self.model_name,+ sparse_embedding={"type": "enabled"},+ extra_headers=self.extra_headers,+ ),+ )+ self._update_telemetry_token_usage(+ response, duration_seconds=duration_seconds)- self._update_telemetry_token_usage(response)item = response.datadense_vector = truncate_and_normalize(item.embedding, self.dimension)sparse_vector = getattr(item, "sparse_embedding", None)diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 6134faff6..f7f96dc02 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -194,7 +194,11 @@ class VolcEngineVLM(OpenAIVLM):client = self.get_client()t0 = time.perf_counter()- response = client.chat.completions.create(**kwargs)+ try:+ response = client.chat.completions.create(**kwargs)+ except Exception as error:+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)+ raiseelapsed = time.perf_counter() - t0self._update_token_usage_from_response(response, duration_seconds=elapsed)result = self._build_vlm_response(response, has_tools=bool(tools))@@ -254,6 +258,7 @@ class VolcEngineVLM(OpenAIVLM):tracer.info(f"message.content={content}")return contentexcept Exception as e:+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=e)last_error = eif attempt < self.max_retries:await asyncio.sleep(2**attempt)@@ -413,7 +418,11 @@ class VolcEngineVLM(OpenAIVLM):client = self.get_client()t0 = time.perf_counter()- response = client.chat.completions.create(**kwargs)+ try:+ response = client.chat.completions.create(**kwargs)+ except Exception as error:+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)+ raiseelapsed = time.perf_counter() - t0self._update_token_usage_from_response(response, duration_seconds=elapsed)result = self._build_vlm_response(response, has_tools=bool(tools))@@ -457,7 +466,11 @@ class VolcEngineVLM(OpenAIVLM):client = self.get_async_client()t0 = time.perf_counter()- response = await client.chat.completions.create(**kwargs)+ try:+ response = await client.chat.completions.create(**kwargs)+ except Exception as error:+ self.record_failed_call(duration_seconds=time.perf_counter() - t0, error=error)+ raiseelapsed = time.perf_counter() - t0self._update_token_usage_from_response(response, duration_seconds=elapsed)result = self._build_vlm_response(response, has_tools=bool(tools))diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 7d092632b..5f9e7e931 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -14,6 +14,7 @@ from openviking.utils.model_retry import (OrderedCredentialSwitcher,PrimaryBackupSwitcher,classify_api_error,+ extract_metric_error_code,)from openviking_cli.utils import get_logger@@ -288,6 +289,32 @@ class VLMBase(ABC):e,)+ def record_failed_call(self, *, duration_seconds: float, error: Exception) -> None:+ """Record one failed provider request attempt without token usage."""+ try:+ from openviking.metrics.datasources import VLMEventDataSource+ from openviking.observability.context import get_root_observability_context++ root_context = get_root_observability_context()+ VLMEventDataSource.record_call(+ provider=str(self.provider),+ model_name=str(self.model or "unknown"),+ duration_seconds=max(float(duration_seconds), 0.0),+ prompt_tokens=0,+ completion_tokens=0,+ error_code=extract_metric_error_code(error),+ account_id=root_context.account_id if root_context is not None else None,+ )+ except Exception as metrics_error:+ if logger.isEnabledFor(logging.DEBUG):+ logger.debug(+ "vlm failed-call metrics emit failed provider=%s model_name=%s err=%s: %s",+ self.provider,+ self.model,+ type(metrics_error).__name__,+ metrics_error,+ )+@propertydef token_tracker(self):"""Public accessor for this instance's token usage tracker."""diff --git a/openviking/session/compressor_v3.py b/openviking/session/compressor_v3.pyindex 5d1cb93df..cfcbfc84f 100644--- a/openviking/session/compressor_v3.py+++ b/openviking/session/compressor_v3.py@@ -130,7 +130,23 @@ def _initialize_extraction_telemetry() -> None:telemetry.set(name, 0)-def _report_extraction_telemetry(result: Any) -> None:+def _memory_type_by_uri(operations: ResolvedOperations) -> dict[str, str]:+ """Map applied memory URIs to their stable extraction schema names."""+ types_by_uri: dict[str, str] = {}+ for operation in getattr(operations, "upsert_operations", []) or []:+ memory_type = str(getattr(operation, "memory_type", "") or "unknown")+ for uri in getattr(operation, "uris", []) or []:+ types_by_uri[str(uri)] = memory_type+ for file_content in getattr(operations, "delete_file_contents", []) or []:+ uri = str(getattr(file_content, "uri", "") or "")+ if uri:+ types_by_uri[uri] = str(+ getattr(file_content, "memory_type", "") or "unknown"+ )+ return types_by_uri+++def _report_extraction_telemetry(result: Any, operations: ResolvedOperations) -> None:telemetry = get_current_telemetry()telemetry.set("memory.extract.candidates.total",@@ -142,6 +158,29 @@ def _report_extraction_telemetry(result: Any) -> None:telemetry.set("memory.extract.skipped", len(result.skipped_operations))telemetry.set("memory.extract.failed", len(result.errors))+ types_by_uri = _memory_type_by_uri(operations)+ actions_by_type: dict[str, dict[str, int]] = {}++ def add(memory_type: Any, action: str) -> None:+ normalized_type = str(memory_type or "unknown")+ type_actions = actions_by_type.setdefault(normalized_type, {})+ type_actions[action] = type_actions.get(action, 0) + 1++ for uri in result.written_uris:+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "created")+ for uri in result.edited_uris:+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "merged")+ for uri in result.deleted_uris:+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "deleted")+ for operation in result.skipped_operations:+ add(getattr(operation, "memory_type", None), "skipped")+ for uri, _error in result.errors:+ add(types_by_uri.get(str(uri), MemoryUpdater.memory_type_from_uri(uri)), "failed")++ for memory_type, actions in actions_by_type.items():+ for action, value in actions.items():+ telemetry.set(f"memory.extract.by_type.{memory_type}.{action}", value)+async def _commit_experience_snapshot(viking_fs: Any,@@ -705,7 +744,7 @@ class SessionCompressorV3:result = update_result.apply_resultpatch_operations = update_result.operations- _report_extraction_telemetry(result)+ _report_extraction_telemetry(result, patch_operations)memory_diff = Noneif archive_uri and viking_fs and result is not None:diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.pyindex 2c154b99c..a8951bba4 100644--- a/openviking/telemetry/operation.py+++ b/openviking/telemetry/operation.py@@ -359,6 +359,16 @@ class TelemetrySummaryBuilder:"extracted": memories_extracted,}if cls._has_metric_prefix("memory.extract", counters, gauges):+ actions_by_type: Dict[str, Dict[str, int]] = {}+ type_action_prefix = "memory.extract.by_type."+ for key, value in gauges.items():+ if not key.startswith(type_action_prefix):+ continue+ suffix = key[len(type_action_prefix) :]+ memory_type, separator, action = suffix.rpartition(".")+ if not separator or not memory_type or not action:+ continue+ actions_by_type.setdefault(memory_type, {})[action] = cls._i(value, 0)memory_summary["extract"] = {"duration_ms": cls._f(gauges.get("memory.extract.total.duration_ms"), 0.0),"candidates": {@@ -373,6 +383,7 @@ class TelemetrySummaryBuilder:"skipped": cls._i(gauges.get("memory.extract.skipped"), 0),"failed": cls._i(gauges.get("memory.extract.failed"), 0),},+ "actions_by_type": actions_by_type,"stages": {public_key: cls._f(gauges.get(metric_key), 0.0)for public_key, metric_key in cls._MEMORY_EXTRACT_STAGE_KEYS.items()diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.pyindex eac035643..35a089dce 100644--- a/openviking/utils/model_retry.py+++ b/openviking/utils/model_retry.py@@ -23,6 +23,57 @@ ERROR_CLASS_QUOTA_EXCEEDED = "quota_exceeded"ERROR_CLASS_TRANSIENT = "transient"ERROR_CLASS_UNKNOWN = "unknown"+_METRIC_ERROR_CODE_MAX_LENGTH = 64+++def _normalize_metric_error_code(value: object) -> str | None:+ """Return a bounded structured error code suitable for a metric label."""+ if value is None:+ return None+ if isinstance(value, int) and 100 <= value <= 599:+ return str(value)+ if not isinstance(value, str):+ return None+ code = value.strip()+ if not code or len(code) > _METRIC_ERROR_CODE_MAX_LENGTH:+ return None+ if not re.fullmatch(r"[A-Za-z0-9_.-]+", code):+ return None+ return code+++def extract_metric_error_code(error: BaseException) -> str:+ """Extract a low-cardinality provider error code for model-call metrics.++ Structured HTTP/SDK error codes are preferred over message parsing. Free-form+ provider messages and request IDs are intentionally never used as metric labels.+ """+ error_chain = _iter_exception_chain(error)+ for exc in error_chain:+ for attr in ("status_code", "error_code", "code"):+ normalized = _normalize_metric_error_code(getattr(exc, attr, None))+ if normalized is not None:+ return normalized++ body = getattr(exc, "body", None)+ if isinstance(body, dict):+ candidates = [body.get("status_code"), body.get("error_code"), body.get("code")]+ nested = body.get("error")+ if isinstance(nested, dict):+ candidates.extend(+ [nested.get("status_code"), nested.get("error_code"), nested.get("code")]+ )+ for value in candidates:+ normalized = _normalize_metric_error_code(value)+ if normalized is not None:+ return normalized++ if any(isinstance(exc, TimeoutError) for exc in error_chain):+ return "timeout"+ if any(isinstance(exc, ConnectionError) for exc in error_chain):+ return "connection_error"+ return "unknown"+INPUT_TOO_LARGE_PATTERNS = ("413","payload too large",diff --git a/tests/metrics/collectors/test_event_collectors.py b/tests/metrics/collectors/test_event_collectors.pyindex 2579af669..fc4491e50 100644--- a/tests/metrics/collectors/test_event_collectors.py+++ b/tests/metrics/collectors/test_event_collectors.py@@ -13,6 +13,7 @@ from openviking.metrics.collectors.rerank import RerankCollectorfrom openviking.metrics.collectors.retrieval import RetrievalCollectorfrom openviking.metrics.collectors.session import SessionCollectorfrom openviking.metrics.collectors.telemetry_bridge import TelemetryBridgeCollector+from openviking.metrics.collectors.vlm import VLMCollectorclass _DummyEventCollector(EventMetricCollector):@@ -213,11 +214,11 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome# Account dimension is supported for embedding families; default runtime resolves to __unknown__.assert re.search(- r'openviking_embedding_calls_total\{(?=[^}]*account_id="__unknown__")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',+ r'openviking_embedding_calls_total\{(?=[^}]*account_id="__unknown__")(?=[^}]*error_code="OK")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',text,)assert re.search(- r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*account_id="__unknown__")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',+ r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*account_id="__unknown__")(?=[^}]*error_code="OK")(?=[^}]*model_name="text-embedding-3-large")(?=[^}]*provider="openai")[^}]*\} 1(?:\.0)?',text,)assert re.search(@@ -234,6 +235,58 @@ def test_embedding_collector_maps_call_metrics_and_tokens(registry, render_prome)+def test_embedding_collector_records_failed_call_duration_with_error_code(registry, render_prometheus):+ EmbeddingCollector().receive(+ "embedding.call",+ {+ "provider": "volcengine",+ "model_name": "doubao-embedding",+ "duration_seconds": 0.12,+ "prompt_tokens": 0,+ "completion_tokens": 0,+ "error_code": "429",+ },+ registry,+ )+ text = render_prometheus(registry)++ assert re.search(+ r'openviking_embedding_calls_total\{(?=[^}]*error_code="429")(?=[^}]*model_name="doubao-embedding")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',+ text,+ )+ assert re.search(+ r'openviking_embedding_call_duration_seconds_count\{(?=[^}]*error_code="429")(?=[^}]*model_name="doubao-embedding")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',+ text,+ )+ assert "openviking_embedding_tokens_total" not in text+++def test_vlm_collector_records_error_code_on_calls_and_duration_only(registry, render_prometheus):+ VLMCollector().receive(+ "vlm.call",+ {+ "provider": "volcengine",+ "model_name": "doubao-vlm",+ "duration_seconds": 0.12,+ "prompt_tokens": 0,+ "completion_tokens": 0,+ "error_code": "timeout",+ },+ registry,+ )+ text = render_prometheus(registry)++ assert re.search(+ r'openviking_vlm_calls_total\{(?=[^}]*error_code="timeout")(?=[^}]*model_name="doubao-vlm")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',+ text,+ )+ assert re.search(+ r'openviking_vlm_call_duration_seconds_count\{(?=[^}]*error_code="timeout")(?=[^}]*model_name="doubao-vlm")(?=[^}]*provider="volcengine")[^}]*\} 1(?:\.0)?',+ text,+ )+ assert "openviking_vlm_tokens_total" not in text++def test_embedding_collector_records_success_volume_and_latency(registry, render_prometheus):"""`embedding.success` must increment request volume and observe latency."""EmbeddingCollector().receive(diff --git a/tests/metrics/integration/test_telemetry_bridge.py b/tests/metrics/integration/test_telemetry_bridge.pyindex 4db5d8d11..edb0c8b5f 100644--- a/tests/metrics/integration/test_telemetry_bridge.py+++ b/tests/metrics/integration/test_telemetry_bridge.py@@ -37,7 +37,11 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rende"deleted": 1,"skipped": 3,"failed": 1,- }+ },+ "actions_by_type": {+ "preferences": {"created": 2, "skipped": 3},+ "events": {"merged": 1, "deleted": 1, "failed": 1},+ },},},"semantic_nodes": {"OK": 12},@@ -67,25 +71,32 @@ def test_telemetry_bridge_records_operation_and_resource_metrics(registry, rendein text)assert 'openviking_vector_searches_total{operation="resource.process"} 1' in text- assert 'openviking_memory_extracted_total{operation="resource.process"} 4' in textassert (- 'openviking_memory_operations_total{action="created",operation="resource.process",result="success"} 2'+ 'openviking_memory_extracted_total{memory_type="preferences",operation="resource.process"} 2'in text)assert (- 'openviking_memory_operations_total{action="updated",operation="resource.process",result="success"} 1'+ 'openviking_memory_extracted_total{memory_type="events",operation="resource.process"} 2'in text)assert (- 'openviking_memory_operations_total{action="deleted",operation="resource.process",result="success"} 1'+ 'openviking_memory_operations_total{action="created",memory_type="preferences",operation="resource.process",result="success"} 2'in text)assert (- 'openviking_memory_operations_total{action="skipped",operation="resource.process",result="skipped"} 3'+ 'openviking_memory_operations_total{action="updated",memory_type="events",operation="resource.process",result="success"} 1'in text)assert (- 'openviking_memory_operations_total{action="failed",operation="resource.process",result="failed"} 1'+ 'openviking_memory_operations_total{action="deleted",memory_type="events",operation="resource.process",result="success"} 1'+ in text+ )+ assert (+ 'openviking_memory_operations_total{action="skipped",memory_type="preferences",operation="resource.process",result="skipped"} 3'+ in text+ )+ assert (+ 'openviking_memory_operations_total{action="failed",memory_type="events",operation="resource.process",result="failed"} 1'in text)assert (diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex 66246eddf..4d246cd1e 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -13,6 +13,7 @@ from openviking.utils.model_retry import (ERROR_CLASS_QUOTA_EXCEEDED,ERROR_CLASS_TRANSIENT,classify_api_error,+ extract_metric_error_code,retry_async,retry_sync,)@@ -22,6 +23,31 @@ def test_classify_api_error_recognizes_request_burst_too_fast():assert classify_api_error(RuntimeError("RequestBurstTooFast")) == ERROR_CLASS_TRANSIENT+class _ProviderError(RuntimeError):+ def __init__(self, *, status_code=None, error_code=None, code=None, body=None):+ super().__init__("provider request failed")+ self.status_code = status_code+ self.error_code = error_code+ self.code = code+ self.body = body+++def test_extract_metric_error_code_prefers_structured_provider_code():+ assert extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded")) == "429"+ assert extract_metric_error_code(_ProviderError(body={"error": {"code": "InvalidParameter"}})) == (+ "InvalidParameter"+ )+++def test_extract_metric_error_code_uses_safe_fallbacks_only():+ assert extract_metric_error_code(TimeoutError("request timed out")) == "timeout"+ assert extract_metric_error_code(ConnectionError("connection reset")) == "connection_error"+ wrapped_timeout = RuntimeError("request failed")+ wrapped_timeout.__cause__ = TimeoutError("request timed out")+ assert extract_metric_error_code(wrapped_timeout) == "timeout"+ assert extract_metric_error_code(RuntimeError("request_id=not-a-metric-label")) == "unknown"++def test_classify_all_credentials_failed_prefers_transient_over_auth():err = AllCredentialsFailedError([diff --git a/tests/unit/test_volcengine_embedder_input.py b/tests/unit/test_volcengine_embedder_input.pyindex e0f74b19d..9cd5aad6b 100644--- a/tests/unit/test_volcengine_embedder_input.py+++ b/tests/unit/test_volcengine_embedder_input.py@@ -2,9 +2,11 @@# SPDX-License-Identifier: AGPL-3.0"""Input-guard tests for Volcengine sparse-capable embedding modes."""-from unittest.mock import patch+from types import SimpleNamespace+from unittest.mock import MagicMock, patchfrom openviking.models.embedder.volcengine_embedders import (+ VolcengineDenseEmbedder,VolcengineHybridEmbedder,VolcengineSparseEmbedder,)@@ -30,3 +32,26 @@ def test_hybrid_embedder_downgrades_multimodal_input_to_text(_mock_ark):assert embedder.supports_multimodal is Falseassert embedder.prepare_embedding_input(_MULTIMODAL_INPUT) == "diagram of a heat exchanger"+++@patch("openviking.models.embedder.volcengine_embedders.volcenginesdkarkruntime.Ark")+@patch("openviking.models.embedder.volcengine_embedders.time.perf_counter")+def test_dense_embedder_passes_local_provider_duration_to_metrics(mock_perf_counter, mock_ark):+ """Embedding call duration must come from this SDK request, not shared embedder state."""+ response = SimpleNamespace(+ usage=SimpleNamespace(prompt_tokens=3, total_tokens=3),+ data=[SimpleNamespace(embedding=[1.0, 0.0])],+ )+ client = MagicMock()+ client.embeddings.create.return_value = response+ mock_ark.return_value = client+ mock_perf_counter.side_effect = [100.0, 100.25]++ embedder = VolcengineDenseEmbedder(+ model_name="test", api_key="test-key", input_type="text", dimension=2+ )+ embedder.update_token_usage = MagicMock()++ embedder.embed("hello")++ assert embedder.update_token_usage.call_args.kwargs["duration_seconds"] == 0.25