Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(models): 统一模型错误分类与重试边界 (#6047)
hezhang-1216acd7caf0d9fc
diff --git a/bot/vikingbot/compile/ops/common.py b/bot/vikingbot/compile/ops/common.pyindex 360ec2363..6c49864df 100644--- a/bot/vikingbot/compile/ops/common.py+++ b/bot/vikingbot/compile/ops/common.py@@ -11,7 +11,7 @@ from dataclasses import asdictfrom typing import TYPE_CHECKINGfrom openviking.core.namespace import relative_uri_path-from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, classify_api_error+from openviking.models.model_error import ModelErrorKind, model_error_kindfrom vikingbot.compile import file_opsfrom vikingbot.compile.hashing import content_hash, digestfrom vikingbot.compile.pipeline_io import MAX_JOB_ATTEMPTS@@ -347,7 +347,7 @@ async def job(runtime: Pipeline, name, inputs, work):except OSError as exc:if (attempt == MAX_JOB_ATTEMPTS - 1- or classify_api_error(exc) == ERROR_CLASS_INPUT_TOO_LARGE+ or model_error_kind(exc) is ModelErrorKind.INPUT_TOO_LARGE):raiseruntime.metrics["job_retries"] += 1diff --git a/bot/vikingbot/compile/ops/reduce.py b/bot/vikingbot/compile/ops/reduce.pyindex 1c4b54832..eb7af8d55 100644--- a/bot/vikingbot/compile/ops/reduce.py+++ b/bot/vikingbot/compile/ops/reduce.py@@ -11,7 +11,7 @@ from functools import partialfrom typing import TYPE_CHECKINGfrom openviking.core.namespace import relative_uri_path-from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, classify_api_error+from openviking.models.model_error import ModelErrorKind, model_error_kindfrom vikingbot.compile import file_opsfrom vikingbot.compile.hashing import content_hash, digestfrom vikingbot.compile.ops import common@@ -317,7 +317,7 @@ async def reduce_group(breakexcept ModelCallError as exc:if (- classify_api_error(exc) != ERROR_CLASS_INPUT_TOO_LARGE+ model_error_kind(exc) is not ModelErrorKind.INPUT_TOO_LARGEor depth == 3or stage == "map"):diff --git a/bot/vikingbot/compile/pipeline_io.py b/bot/vikingbot/compile/pipeline_io.pyindex 3aef9ea87..c51d185b6 100644--- a/bot/vikingbot/compile/pipeline_io.py+++ b/bot/vikingbot/compile/pipeline_io.py@@ -16,6 +16,7 @@ import json_repairfrom loguru import loggerfrom pydantic import BaseModel, ValidationError+from openviking.models.model_error import ModelErrorfrom openviking.utils.token_estimation import estimate_text_tokensfrom openviking_cli.exceptions import NotFoundErrorfrom vikingbot.compile.hashing import digest@@ -318,6 +319,12 @@ class JsonModel:self.usage[name] = self.usage.get(name, 0) + valueself.metrics[f"{stage}_{name}"] += valueif response.finish_reason == "error":+ error_kind = getattr(response, "error_kind", None)+ if error_kind is not None:+ raise ModelError(+ error_kind,+ response.content or "Model transport failure; see provider diagnostics",+ )raise OSError(response.content or "Model transport failure; see provider diagnostics")diff --git a/bot/vikingbot/providers/base.py b/bot/vikingbot/providers/base.pyindex 159b1b1c9..17365eb94 100644--- a/bot/vikingbot/providers/base.py+++ b/bot/vikingbot/providers/base.py@@ -7,6 +7,8 @@ from typing import Any, AsyncIterator, Callable, Literalimport json_repair+from openviking.models.model_error import ModelErrorKind+@dataclassclass ToolCallRequest:@@ -27,6 +29,7 @@ class LLMResponse:finish_reason: str = "stop"usage: dict[str, int] = field(default_factory=dict)reasoning_content: str | None = None # Kimi, DeepSeek-R1 etc.+ error_kind: ModelErrorKind | None = None@propertydef has_tool_calls(self) -> bool:diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.pyindex bd5d0fe49..f47575a32 100644--- a/bot/vikingbot/providers/vlm_adapter.py+++ b/bot/vikingbot/providers/vlm_adapter.py@@ -15,6 +15,7 @@ from typing import Anyfrom loguru import logger+from openviking.models.model_error import model_error_kindfrom openviking.models.vlm.backends.volcengine_vlm import build_volcengine_request_headersfrom openviking.utils.model_retry import is_retryable_rate_limit_error, rate_limit_retry_delayfrom openviking.utils.multimodal import redact_image_data_urls@@ -336,6 +337,7 @@ class VLMProviderAdapter(LLMProvider):return LLMResponse(content=f"Error calling LLM in VLM Adapter: {str(e)}",finish_reason="error",+ error_kind=model_error_kind(e),)async def chat_stream(@@ -389,6 +391,7 @@ class VLMProviderAdapter(LLMProvider):response=LLMResponse(content=f"Error calling LLM in VLM Adapter stream: {str(exc)}",finish_reason="error",+ error_kind=model_error_kind(exc),),)diff --git a/openviking/config/embedding.py b/openviking/config/embedding.pyindex 656bfaa73..29ca34af2 100644--- a/openviking/config/embedding.py+++ b/openviking/config/embedding.py@@ -17,12 +17,6 @@ from openviking.models.embedder.retry_embedder import RetryEmbedderfrom openviking.models.vlm.token_usage import TokenUsageTrackerfrom openviking.service.task_tracker_concurrency import KeyedAsyncLockPoolfrom openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen-from openviking.utils.model_retry import (- ERROR_CLASS_AUTH,- ERROR_CLASS_INPUT_TOO_LARGE,- ERROR_CLASS_PERMANENT,- classify_api_error,-)from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -260,16 +254,7 @@ class AccountEmbeddingProvider:f"got {len(result.dense_vector)}")except Exception as exc:- # The breaker is shared by every call on this account, so only- # provider-health failures may trip it. Request-level rejections- # (400 / input too large) are specific to one input, and auth- # errors are surfaced per call instead.- if classify_api_error(exc) not in {- ERROR_CLASS_AUTH,- ERROR_CLASS_INPUT_TOO_LARGE,- ERROR_CLASS_PERMANENT,- }:- resource.breaker.record_failure(exc)+ resource.breaker.record_failure(exc)raiseresource.breaker.record_success()return resultdiff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex 329ede0b8..a8e21c235 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -12,16 +12,14 @@ from dataclasses import dataclassfrom threading import Lockfrom typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Optional, TypeVar, Union+from openviking.models.model_error import model_error_kindfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_input import (resolve_embedding_max_input_tokens,truncate_embedding_input,)from openviking.utils.exceptions import AllCredentialsFailedError-from openviking.utils.model_retry import (- OrderedCredentialSwitcher,- classify_api_error,-)+from openviking.utils.model_retry import OrderedCredentialSwitcherfrom openviking_cli.utils import get_loggerT = TypeVar("T")@@ -856,16 +854,18 @@ class FailoverEmbedder(EmbedderBase):self._switcher.commit_success(idx)return resultexcept Exception as exc:- error_class = classify_api_error(exc)- aggregated_errors.append((credential_id, error_class, exc, idx))+ kind = model_error_kind(exc)- if self._switcher.is_fail_fast(error_class):+ if self._switcher.is_fail_fast(kind):# Request-level failure: re-raise the original exception;# trying other credentials is useless.raise+ aggregated_errors.append((credential_id, kind, exc, idx))logger.warning(- f"Credential {credential_id} failed with {error_class}, trying next credential"+ "Credential %s failed with %s, trying next credential",+ credential_id,+ kind.value,)raise AllCredentialsFailedError(aggregated_errors)@@ -901,16 +901,18 @@ class FailoverEmbedder(EmbedderBase):self._switcher.commit_success(idx)return resultexcept Exception as exc:- error_class = classify_api_error(exc)- aggregated_errors.append((credential_id, error_class, exc, idx))+ kind = model_error_kind(exc)- if self._switcher.is_fail_fast(error_class):+ if self._switcher.is_fail_fast(kind):# Request-level failure: re-raise the original exception;# trying other credentials is useless.raise+ aggregated_errors.append((credential_id, kind, exc, idx))logger.warning(- f"Credential {credential_id} failed with {error_class}, trying next credential"+ "Credential %s failed with %s, trying next credential",+ credential_id,+ kind.value,)raise AllCredentialsFailedError(aggregated_errors)diff --git a/openviking/models/embedder/cohere_embedders.py b/openviking/models/embedder/cohere_embedders.pyindex 43bdabe86..b3f502fae 100644--- a/openviking/models/embedder/cohere_embedders.py+++ b/openviking/models/embedder/cohere_embedders.py@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optionalimport httpxfrom openviking.models.embedder.base import DenseEmbedderBase, EmbedResult, truncate_and_normalize+from openviking.models.model_error import model_error_boundaryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils import get_logger@@ -132,6 +133,7 @@ class CohereDenseEmbedder(DenseEmbedderBase):return truncate_and_normalize(vector, self._dimension)return vector+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:input_type = "search_query" if is_query else "search_document"try:@@ -157,6 +159,7 @@ class CohereDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Cohere embedding failed: {e}") from e+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:input_type = "search_query" if is_query else "search_document"diff --git a/openviking/models/embedder/dashscope_embedders.py b/openviking/models/embedder/dashscope_embedders.pyindex 44e3d3c95..b29b94080 100644--- a/openviking/models/embedder/dashscope_embedders.py+++ b/openviking/models/embedder/dashscope_embedders.py@@ -17,6 +17,7 @@ from openviking.models.embedder.base import (EmbedResult,truncate_and_normalize,)+from openviking.models.model_error import model_error_boundaryfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils.logger import default_logger as logger@@ -225,6 +226,7 @@ class DashScopeDenseEmbedder(DenseEmbedderBase):# embed# ------------------------------------------------------------------+ @model_error_boundarydef embed(self, text: EmbeddingInput, is_query: bool = False) -> EmbedResult:if isinstance(text, list):contents = self._to_dashscope_contents(text)@@ -256,6 +258,7 @@ class DashScopeDenseEmbedder(DenseEmbedderBase):# embed_async# ------------------------------------------------------------------+ @model_error_boundaryasync def embed_async(self, text: EmbeddingInput, is_query: bool = False) -> EmbedResult:if isinstance(text, list):contents = self._to_dashscope_contents(text)@@ -291,6 +294,7 @@ class DashScopeDenseEmbedder(DenseEmbedderBase):# embed_content — multimodal content (text + image URLs)# ------------------------------------------------------------------+ @model_error_boundarydef embed_content(self, contents: List[Dict[str, str]], *, enable_fusion: Optional[bool] = None) -> EmbedResult:@@ -326,6 +330,7 @@ class DashScopeDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"DashScope content embedding failed: {e}") from e+ @model_error_boundaryasync def embed_content_async(self, contents: List[Dict[str, str]], *, enable_fusion: Optional[bool] = None) -> EmbedResult:diff --git a/openviking/models/embedder/gemini_embedders.py b/openviking/models/embedder/gemini_embedders.pyindex a81dbf665..1963edb58 100644--- a/openviking/models/embedder/gemini_embedders.py+++ b/openviking/models/embedder/gemini_embedders.py@@ -20,6 +20,7 @@ from openviking.models.embedder.base import (EmbedResult,truncate_and_normalize,)+from openviking.models.model_error import model_error_boundaryfrom openviking_cli.utils import get_loggerlogger = get_logger(__name__)@@ -184,6 +185,7 @@ class GeminiDenseEmbedder(DenseEmbedderBase):f"task_type={self.task_type!r})")+ @model_error_boundarydef embed(self,text: str,@@ -225,6 +227,7 @@ class GeminiDenseEmbedder(DenseEmbedderBase):except (APIError, ClientError) as e:_raise_api_error(e, self.model_name)+ @model_error_boundaryasync def embed_async(self,text: str,diff --git a/openviking/models/embedder/jina_embedders.py b/openviking/models/embedder/jina_embedders.pyindex 74629dd81..2a00ec491 100644--- a/openviking/models/embedder/jina_embedders.py+++ b/openviking/models/embedder/jina_embedders.py@@ -10,6 +10,7 @@ from openviking.models.embedder.base import (DenseEmbedderBase,EmbedResult,)+from openviking.models.model_error import model_error_boundaryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils import get_logger@@ -183,6 +184,7 @@ class JinaDenseEmbedder(DenseEmbedderBase):f"to a valid task type for this model. API details: {error.message}") from error+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text@@ -224,6 +226,7 @@ class JinaDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:client = self._get_async_client()diff --git a/openviking/models/embedder/litellm_embedders.py b/openviking/models/embedder/litellm_embedders.pyindex 2c0365ed8..88c039c87 100644--- a/openviking/models/embedder/litellm_embedders.py+++ b/openviking/models/embedder/litellm_embedders.py@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optionalimport litellmfrom openviking.models.embedder.base import DenseEmbedderBase, EmbedResult+from openviking.models.model_error import model_error_boundaryfrom openviking.telemetry import get_current_telemetryfrom openviking_cli.utils import get_logger@@ -166,6 +167,7 @@ class LiteLLMDenseEmbedder(DenseEmbedderBase):completion_tokens=output_tokens,)+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text via litellm.@@ -199,6 +201,7 @@ class LiteLLMDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"LiteLLM embedding failed: {e}") from e+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:async def _call() -> EmbedResult:kwargs = self._build_kwargs(is_query=is_query)diff --git a/openviking/models/embedder/minimax_embedders.py b/openviking/models/embedder/minimax_embedders.pyindex 9c46f38ea..c397124d1 100644--- a/openviking/models/embedder/minimax_embedders.py+++ b/openviking/models/embedder/minimax_embedders.py@@ -8,6 +8,7 @@ import httpximport requestsfrom openviking.models.embedder.base import DenseEmbedderBase, EmbedResult+from openviking.models.model_error import ModelError, ModelErrorKind, model_error_boundaryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils.logger import default_logger as logger@@ -112,7 +113,12 @@ class MinimaxDenseEmbedder(DenseEmbedderBase):# Check for business error codebase_resp = data.get("base_resp", {})if base_resp.get("status_code") != 0:- raise RuntimeError(f"MiniMax API error: {base_resp.get('status_msg')}")+ raise ModelError(+ ModelErrorKind.UNKNOWN,+ f"MiniMax API error: {base_resp.get('status_msg')}",+ provider="minimax",+ code=str(base_resp.get("status_code")),+ )vectors = data.get("vectors", [])if not vectors:@@ -120,6 +126,8 @@ class MinimaxDenseEmbedder(DenseEmbedderBase):return vectors+ except ModelError:+ raiseexcept requests.exceptions.RequestException as e:raise RuntimeError(f"MiniMax network error: {str(e)}") from eexcept Exception as e:@@ -169,18 +177,26 @@ class MinimaxDenseEmbedder(DenseEmbedderBase):base_resp = data.get("base_resp", {})if base_resp.get("status_code") != 0:- raise RuntimeError(f"MiniMax API error: {base_resp.get('status_msg')}")+ raise ModelError(+ ModelErrorKind.UNKNOWN,+ f"MiniMax API error: {base_resp.get('status_msg')}",+ provider="minimax",+ code=str(base_resp.get("status_code")),+ )vectors = data.get("vectors", [])if not vectors:raise RuntimeError("MiniMax API returned empty vectors")return vectors+ except ModelError:+ raiseexcept httpx.HTTPError as e:raise RuntimeError(f"MiniMax network error: {str(e)}") from eexcept Exception as e:raise RuntimeError(f"MiniMax embedding failed: {str(e)}") from e+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text"""vectors = self._run_once(@@ -199,6 +215,7 @@ class MinimaxDenseEmbedder(DenseEmbedderBase):)return result+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:async def _call() -> EmbedResult:vectors = await self._call_api_async([text], is_query=is_query)diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex eec981220..a0a76e9e8 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optionalimport httpximport openai-from openviking_cli.utils import get_loggerfrom openviking.models.embedder.base import (DenseEmbedderBase,@@ -15,6 +14,7 @@ from openviking.models.embedder.base import (HybridEmbedderBase,SparseEmbedderBase,)+from openviking.models.model_error import model_error_boundaryfrom openviking.models.network import (create_optional_async_httpx_client,create_optional_sync_httpx_client,@@ -22,6 +22,7 @@ from openviking.models.network import (from openviking.models.vlm.registry import DEFAULT_AZURE_API_VERSIONfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCache+from openviking_cli.utils import get_loggerlogger = get_logger(__name__)@@ -362,6 +363,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):return self._async_client_cache.get(_build_async_client)+ @model_error_boundarydef embed(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text, text batches, or multimodal content@@ -400,6 +402,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e+ @model_error_boundaryasync def embed_async(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:client = self._get_async_client()embedding_input = self._prepare_embedding_input(text)diff --git a/openviking/models/embedder/retry_embedder.py b/openviking/models/embedder/retry_embedder.pyindex b7b04aa43..b176c5214 100644--- a/openviking/models/embedder/retry_embedder.py+++ b/openviking/models/embedder/retry_embedder.py@@ -5,7 +5,7 @@from typing import Any, Dictfrom openviking.models.embedder.base import EmbedderBase, EmbeddingInput, EmbedResult-from openviking.utils.model_retry import retry_async, retry_sync+from openviking.utils.model_retry import effective_max_retries, retry_async, retry_syncfrom openviking_cli.utils import get_loggerlogger = get_logger(__name__)@@ -37,9 +37,10 @@ class RetryEmbedder(EmbedderBase):) -> EmbedResult:return retry_sync(lambda: self.inner.embed(content, is_query=is_query, **kwargs),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} embedding",+ model_errors_only=True,)async def embed_async(@@ -50,9 +51,10 @@ class RetryEmbedder(EmbedderBase):) -> EmbedResult:return await retry_async(lambda: self.inner.embed_async(content, is_query=is_query, **kwargs),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} async embedding",+ model_errors_only=True,)def prepare_embedding_input(self, content: EmbeddingInput) -> EmbeddingInput:diff --git a/openviking/models/embedder/vikingdb_embedders.py b/openviking/models/embedder/vikingdb_embedders.pyindex c98b41efa..631c728e2 100644--- a/openviking/models/embedder/vikingdb_embedders.py+++ b/openviking/models/embedder/vikingdb_embedders.py@@ -12,6 +12,7 @@ from openviking.models.embedder.base import (HybridEmbedderBase,SparseEmbedderBase,)+from openviking.models.model_error import model_error_boundary, model_error_from_statusfrom openviking.storage.vectordb.collection.volcengine_clients import (DEFAULT_TIMEOUT,ClientForDataApi,@@ -65,10 +66,12 @@ class VikingDBClientMixin:try:response = self.client.do_req("POST", path, req_body=req_body)if response.status_code != 200:- logger.warning(- f"VikingDB API returned bad code: {response.status_code}, message: {response.text}"+ raise model_error_from_status(+ response.status_code,+ f"VikingDB API returned bad code: {response.status_code}, "+ f"message: {response.text}",+ provider="vikingdb",)- return []result = response.json()return result.get("result", {}).get("data", [])@@ -102,12 +105,11 @@ class VikingDBClientMixin:content=req.body,)if response.status_code != 200:- logger.warning(- "VikingDB API returned bad code: %s, message: %s",+ raise model_error_from_status(response.status_code,- response.text,+ f"VikingDB API returned bad code: {response.status_code}, message: {response.text}",+ provider="vikingdb",)- return []result = response.json()return result.get("result", {}).get("data", [])@@ -182,6 +184,7 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):return self.document_paramreturn None+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:input_type = self._resolve_input_type(is_query)@@ -212,6 +215,7 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):)return result+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:async def _call() -> EmbedResult:results = await self._call_api_async([text], dense_model=self.dense_model)@@ -264,6 +268,7 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):"version": model_version,}+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:def _call() -> EmbedResult:results = self._call_api([text], sparse_model=self.sparse_model)@@ -292,6 +297,7 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):)return result+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:async def _call() -> EmbedResult:results = await self._call_api_async([text], sparse_model=self.sparse_model)@@ -360,6 +366,7 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):return self.document_paramreturn None+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:input_type = self._resolve_input_type(is_query)@@ -398,6 +405,7 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):)return result+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:async def _call() -> EmbedResult:results = await self._call_api_async(diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.pyindex 86b9c0fc0..b13b36072 100644--- a/openviking/models/embedder/volcengine_embedders.py+++ b/openviking/models/embedder/volcengine_embedders.py@@ -3,12 +3,11 @@"""Volcengine Embedder Implementation"""import time-from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVarimport volcenginesdkarkruntime-from openviking_cli.utils.logger import default_logger as logger+from openviking.metrics.datasources import EmbeddingEventDataSourcefrom openviking.models.embedder.base import (DenseEmbedderBase,EmbeddingInput,@@ -18,15 +17,16 @@ from openviking.models.embedder.base import (extract_text_from_content,truncate_and_normalize,)+from openviking.models.model_error import model_error_boundaryfrom openviking.models.network import (create_optional_async_httpx_client,create_optional_sync_httpx_client,)-from openviking.telemetry import get_current_telemetry-from openviking.metrics.datasources import EmbeddingEventDataSourcefrom openviking.observability.context import get_root_observability_context-from openviking.utils.model_retry import extract_metric_error_code+from openviking.telemetry import get_current_telemetryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCache+from openviking.utils.model_retry import extract_metric_error_code+from openviking_cli.utils.logger import default_logger as loggerVOLCENGINE_CLIENT_REQUEST_ID_HEADER = "X-Client-Request-Id"VOLCENGINE_CLIENT_REQUEST_ID = "ToB-direct,OpenViking_Service,openviking-service_cn-beijing"@@ -254,6 +254,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):duration_seconds=duration_seconds,)+ @model_error_boundarydef embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text or multimodal content@@ -317,6 +318,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):lambda: _create_async_ark(self._ark_kwargs))+ @model_error_boundaryasync def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:client = self._get_async_client()@@ -439,6 +441,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):duration_seconds=duration_seconds,)+ @model_error_boundarydef embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Perform sparse embedding on text or multimodal content@@ -486,6 +489,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):lambda: _create_async_ark(self._ark_kwargs))+ @model_error_boundaryasync def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:client = self._get_async_client()@@ -619,6 +623,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):"""return False+ @model_error_boundarydef embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Perform hybrid embedding on text or multimodal content@@ -671,6 +676,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):lambda: _create_async_ark(self._ark_kwargs))+ @model_error_boundaryasync def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:client = self._get_async_client()diff --git a/openviking/models/embedder/voyage_embedders.py b/openviking/models/embedder/voyage_embedders.pyindex 67708efd3..2c856a275 100644--- a/openviking/models/embedder/voyage_embedders.py+++ b/openviking/models/embedder/voyage_embedders.py@@ -7,6 +7,7 @@ from typing import Any, Dict, List, Optionalimport openaifrom openviking.models.embedder.base import DenseEmbedderBase, EmbedResult+from openviking.models.model_error import model_error_boundaryfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils import get_logger@@ -103,6 +104,7 @@ class VoyageDenseEmbedder(DenseEmbedderBase):))+ @model_error_boundarydef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Perform dense embedding on text."""@@ -131,6 +133,7 @@ class VoyageDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e+ @model_error_boundaryasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:client = self._get_async_client()diff --git a/openviking/models/model_error.py b/openviking/models/model_error.pynew file mode 100644index 000000000..dccbe3647--- /dev/null+++ b/openviking/models/model_error.py@@ -0,0 +1,388 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Provider-neutral model API errors."""++from __future__ import annotations++import functools+import inspect+import re+from enum import Enum+from typing import Any, Callable, TypeVar, cast+++class ModelErrorKind(str, Enum):+ TRANSIENT_SERVICE_ERROR = "transient_service_error"+ TRANSIENT_NETWORK_ERROR = "transient_network_error"+ RATE_LIMIT_ERROR = "rate_limit_error"+ AUTHENTICATION_ERROR = "authentication_error"+ PERMISSION_DENIED = "permission_denied"+ QUOTA_EXHAUSTED = "quota_exhausted"+ MODEL_NOT_FOUND = "model_not_found"+ UNSUPPORTED = "unsupported"+ INVALID_REQUEST = "invalid_request"+ INPUT_TOO_LARGE = "input_too_large"+ CONTENT_SAFETY = "content_safety"+ UNKNOWN = "unknown"+++class ModelError(Exception):+ """A structured model API failure reported by a provider boundary."""++ def __init__(+ self,+ kind: ModelErrorKind,+ message: str,+ *,+ provider: str | None = None,+ status_code: int | None = None,+ code: str | None = None,+ ) -> None:+ super().__init__(message)+ self.kind = kind+ self.provider = provider+ self.status_code = status_code+ self.code = code+++_CODE_KINDS = {+ # Authentication and authorization.+ "APIKEYINVALID": ModelErrorKind.AUTHENTICATION_ERROR,+ "INVALIDAPIKEY": ModelErrorKind.AUTHENTICATION_ERROR,+ "UNAUTHENTICATED": ModelErrorKind.AUTHENTICATION_ERROR,+ "AUTHENTICATIONERROR": ModelErrorKind.AUTHENTICATION_ERROR,+ "PERMISSIONDENIED": ModelErrorKind.PERMISSION_DENIED,+ "ACCESSDENIED": ModelErrorKind.PERMISSION_DENIED,+ # Quota and billing exhaustion.+ "INSUFFICIENTQUOTA": ModelErrorKind.QUOTA_EXHAUSTED,+ "ACCOUNTQUOTAEXCEEDED": ModelErrorKind.QUOTA_EXHAUSTED,+ "MODELQUOTAEXCEEDED": ModelErrorKind.QUOTA_EXHAUSTED,+ "QUOTAEXCEEDED": ModelErrorKind.QUOTA_EXHAUSTED,+ "BILLINGHARDLIMITREACHED": ModelErrorKind.QUOTA_EXHAUSTED,+ "ACCOUNTOVERDUE": ModelErrorKind.QUOTA_EXHAUSTED,+ # Request rate limits.+ "RATELIMITEXCEEDED": ModelErrorKind.RATE_LIMIT_ERROR,+ "REQUESTLIMITEXCEEDED": ModelErrorKind.RATE_LIMIT_ERROR,+ "REQUESTBURSTTOOFAST": ModelErrorKind.RATE_LIMIT_ERROR,+ "TOOMANYREQUESTS": ModelErrorKind.RATE_LIMIT_ERROR,+ "MODELACCOUNTRPMRATELIMITEXCEEDED": ModelErrorKind.RATE_LIMIT_ERROR,+ "MODELACCOUNTTPMRATELIMITEXCEEDED": ModelErrorKind.RATE_LIMIT_ERROR,+ "MODELACCOUNTTOTALTOKENRATELIMITEXCEEDED": ModelErrorKind.RATE_LIMIT_ERROR,+ # Request shape and model capabilities.+ "CONTEXTLENGTHEXCEEDED": ModelErrorKind.INPUT_TOO_LARGE,+ "INPUTTEXTEXCEEDSLIMIT": ModelErrorKind.INPUT_TOO_LARGE,+ "CONTENTTOOLARGE": ModelErrorKind.INPUT_TOO_LARGE,+ "CONTENTFILTER": ModelErrorKind.CONTENT_SAFETY,+ "CONTENTPOLICYVIOLATION": ModelErrorKind.CONTENT_SAFETY,+ "SAFETY": ModelErrorKind.CONTENT_SAFETY,+ "MODELNOTFOUND": ModelErrorKind.MODEL_NOT_FOUND,+ "INVALIDMODEL": ModelErrorKind.MODEL_NOT_FOUND,+ "INVALIDENDPOINTORMODELNOTFOUND": ModelErrorKind.MODEL_NOT_FOUND,+ "UNSUPPORTEDPARAMETER": ModelErrorKind.UNSUPPORTED,+ "UNSUPPORTEDVALUE": ModelErrorKind.UNSUPPORTED,+}++_TRANSIENT_SERVICE_STATUS = {409, 500, 502, 503}+_TRANSIENT_NETWORK_STATUS = {408, 504}+++def _exception_chain(error: BaseException) -> list[BaseException]:+ chain: list[BaseException] = []+ seen: set[int] = set()+ current: BaseException | None = error+ while current is not None and id(current) not in seen:+ chain.append(current)+ seen.add(id(current))+ current = current.__cause__ or current.__context__+ return chain+++def _normalized_code(value: object) -> str | None:+ if not isinstance(value, str) or not value:+ return None+ return "".join(character for character in value.upper() if character.isalnum())+++def _body(error: BaseException) -> dict[str, Any]:+ value = getattr(error, "body", None)+ if not isinstance(value, dict):+ response = getattr(error, "response", None)+ try:+ value = response.json() if response is not None else None+ except Exception:+ value = None+ if isinstance(value, dict):+ nested = value.get("error")+ if isinstance(nested, dict):+ return {**value, **nested}+ return value+ return {}+++def _error_code(error: BaseException) -> str | None:+ body = _body(error)+ for value in (+ getattr(error, "error_code", None),+ getattr(error, "code", None),+ body.get("error_code"),+ body.get("code"),+ body.get("type"),+ ):+ if isinstance(value, str) and value:+ return value+ return None+++def _status_code(error: BaseException) -> int | None:+ for value in (+ getattr(error, "status_code", None),+ getattr(getattr(error, "response", None), "status_code", None),+ getattr(error, "code", None),+ ):+ if isinstance(value, int) and 100 <= value <= 599:+ return value+ return None+++def _structured_reasons(value: object) -> set[str]:+ reasons: set[str] = set()+ if isinstance(value, dict):+ for key, item in value.items():+ if key in {"reason", "status"} and isinstance(item, str):+ normalized = _normalized_code(item)+ if normalized:+ reasons.add(normalized)+ reasons.update(_structured_reasons(item))+ elif isinstance(value, (list, tuple)):+ for item in value:+ reasons.update(_structured_reasons(item))+ elif hasattr(value, "model_dump"):+ reasons.update(_structured_reasons(value.model_dump()))+ return reasons+++def _kind_from_code(error: BaseException) -> ModelErrorKind | None:+ normalized = _normalized_code(_error_code(error))+ if normalized in _CODE_KINDS:+ return _CODE_KINDS[normalized]++ details = getattr(error, "details", None)+ reasons = _structured_reasons(details)+ if reasons & {"QUOTAEXCEEDED", "BILLINGDISABLED"}:+ return ModelErrorKind.QUOTA_EXHAUSTED+ if reasons & {"RATELIMITEXCEEDED", "SERVINGCAPACITYEXCEEDED"}:+ return ModelErrorKind.RATE_LIMIT_ERROR+ if reasons & {"APIKEYINVALID", "UNAUTHENTICATED"}:+ return ModelErrorKind.AUTHENTICATION_ERROR+ return None+++def _kind_from_status(status_code: int | None) -> ModelErrorKind | None:+ if status_code in _TRANSIENT_SERVICE_STATUS:+ return ModelErrorKind.TRANSIENT_SERVICE_ERROR+ if status_code in _TRANSIENT_NETWORK_STATUS:+ return ModelErrorKind.TRANSIENT_NETWORK_ERROR+ if status_code == 401:+ return ModelErrorKind.AUTHENTICATION_ERROR+ if status_code == 403:+ return ModelErrorKind.PERMISSION_DENIED+ if status_code == 405 or status_code == 501:+ return ModelErrorKind.UNSUPPORTED+ if status_code == 413:+ return ModelErrorKind.INPUT_TOO_LARGE+ if status_code == 429:+ return ModelErrorKind.RATE_LIMIT_ERROR+ if status_code in {400, 422}:+ return ModelErrorKind.INVALID_REQUEST+ if status_code is not None and status_code >= 500:+ return ModelErrorKind.TRANSIENT_SERVICE_ERROR+ return None+++def _sdk_kind(error: BaseException) -> tuple[ModelErrorKind, str] | None:+ module = type(error).__module__+ name = type(error).__name__+ mro_names = {base.__name__ for base in type(error).__mro__}++ if isinstance(error, TimeoutError):+ return ModelErrorKind.TRANSIENT_NETWORK_ERROR, "network"+ if isinstance(error, ConnectionError):+ return ModelErrorKind.TRANSIENT_NETWORK_ERROR, "network"++ if module.startswith("openai"):+ provider = "openai"+ class_kinds = {+ "APITimeoutError": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "APIConnectionError": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "AuthenticationError": ModelErrorKind.AUTHENTICATION_ERROR,+ "PermissionDeniedError": ModelErrorKind.PERMISSION_DENIED,+ "NotFoundError": ModelErrorKind.MODEL_NOT_FOUND,+ "InternalServerError": ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ }+ kind = _kind_from_code(error)+ if kind is None and _status_code(error) == 500:+ # llama.cpp reports inputs above its physical batch capacity as+ # HTTP 500 without a business code. Retrying the same input cannot help.+ message = _body(error).get("message")+ if isinstance(message, str) and re.fullmatch(+ r"input \(\d+ tokens\) is too large to process\. "+ r"increase the physical batch size \(current batch size: \d+\)",+ message,+ ):+ kind = ModelErrorKind.INPUT_TOO_LARGE+ kind = kind or class_kinds.get(name) or _kind_from_status(_status_code(error))+ return kind or ModelErrorKind.UNKNOWN, provider++ if module.startswith("volcenginesdkarkruntime"):+ provider = "volcengine"+ class_kinds = {+ "ArkAPITimeoutError": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "ArkAPIConnectionError": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "ArkAuthenticationError": ModelErrorKind.AUTHENTICATION_ERROR,+ "ArkPermissionDeniedError": ModelErrorKind.PERMISSION_DENIED,+ "ArkNotFoundError": ModelErrorKind.MODEL_NOT_FOUND,+ "ArkInternalServerError": ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ "ArkContentFilterFinishReasonError": ModelErrorKind.CONTENT_SAFETY,+ }+ kind = (+ _kind_from_code(error)+ or class_kinds.get(name)+ or _kind_from_status(_status_code(error))+ )+ return kind or ModelErrorKind.UNKNOWN, provider++ if module.startswith("litellm"):+ provider = "litellm"+ class_kinds = {+ "Timeout": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "APIConnectionError": ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ "RateLimitError": ModelErrorKind.RATE_LIMIT_ERROR,+ "AuthenticationError": ModelErrorKind.AUTHENTICATION_ERROR,+ "PermissionDeniedError": ModelErrorKind.PERMISSION_DENIED,+ "NotFoundError": ModelErrorKind.MODEL_NOT_FOUND,+ "BadRequestError": ModelErrorKind.INVALID_REQUEST,+ "ContentPolicyViolationError": ModelErrorKind.CONTENT_SAFETY,+ "ContextWindowExceededError": ModelErrorKind.INPUT_TOO_LARGE,+ "ServiceUnavailableError": ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ "InternalServerError": ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ }+ return _kind_from_code(error) or class_kinds.get(name, ModelErrorKind.UNKNOWN), provider++ if module.startswith("google.genai"):+ kind = _kind_from_code(error)+ status_code = _status_code(error)+ # RESOURCE_EXHAUSTED is ambiguous unless Google supplies a reason.+ if status_code == 429 and kind is None:+ kind = ModelErrorKind.UNKNOWN+ return kind or _kind_from_status(status_code) or ModelErrorKind.UNKNOWN, "gemini"++ if module.startswith("httpx"):+ if name == "HTTPStatusError":+ kind = _kind_from_code(error) or _kind_from_status(_status_code(error))+ elif "TimeoutException" in mro_names:+ kind = ModelErrorKind.TRANSIENT_NETWORK_ERROR+ elif mro_names & {"NetworkError", "TransportError", "ProtocolError", "ProxyError"}:+ kind = ModelErrorKind.TRANSIENT_NETWORK_ERROR+ else:+ kind = None+ return kind or ModelErrorKind.UNKNOWN, "http"++ if module.startswith("requests"):+ if name == "HTTPError":+ kind = _kind_from_code(error) or _kind_from_status(_status_code(error))+ elif mro_names & {"Timeout", "ConnectTimeout", "ReadTimeout", "ConnectionError"}:+ kind = ModelErrorKind.TRANSIENT_NETWORK_ERROR+ else:+ kind = None+ return kind or ModelErrorKind.UNKNOWN, "http"++ return None+++def model_error_from_status(+ status_code: int,+ message: str,+ *,+ provider: str,+ code: str | None = None,+) -> ModelError:+ """Build a model error from a structured HTTP or provider status."""+ normalized = _normalized_code(code)+ kind = _CODE_KINDS.get(normalized) if normalized else None+ return ModelError(+ kind or _kind_from_status(status_code) or ModelErrorKind.UNKNOWN,+ message,+ provider=provider,+ status_code=status_code,+ code=code,+ )+++def model_error_from_exception(error: BaseException) -> ModelError | None:+ """Convert a recognized SDK/API exception at the provider boundary."""+ for item in _exception_chain(error):+ if isinstance(item, ModelError):+ return item+ classified = _sdk_kind(item)+ if classified is None:+ continue+ kind, provider = classified+ return ModelError(+ kind,+ str(error),+ provider=provider,+ status_code=_status_code(item),+ code=_error_code(item),+ )+ return None+++def model_error_kind(error: BaseException) -> ModelErrorKind | None:+ """Return the structured model kind carried by an exception chain."""+ for item in _exception_chain(error):+ kind = getattr(item, "kind", None)+ if isinstance(kind, ModelErrorKind):+ return kind+ converted = model_error_from_exception(error)+ return converted.kind if converted is not None else None+++F = TypeVar("F", bound=Callable[..., Any])+++def model_error_boundary(func: F) -> F:+ """Translate recognized provider failures at a public provider boundary."""++ if inspect.iscoroutinefunction(func):++ @functools.wraps(func)+ async def _async_wrapper(*args: Any, **kwargs: Any) -> Any:+ try:+ return await func(*args, **kwargs)+ except Exception as error:+ converted = model_error_from_exception(error)+ if converted is None or converted is error:+ raise+ if args and isinstance(getattr(args[0], "provider", None), str):+ converted.provider = args[0].provider+ if any(item is converted for item in _exception_chain(error)):+ raise converted+ raise converted from error++ return cast(F, _async_wrapper)++ @functools.wraps(func)+ def _sync_wrapper(*args: Any, **kwargs: Any) -> Any:+ try:+ return func(*args, **kwargs)+ except Exception as error:+ converted = model_error_from_exception(error)+ if converted is None or converted is error:+ raise+ if args and isinstance(getattr(args[0], "provider", None), str):+ converted.provider = args[0].provider+ if any(item is converted for item in _exception_chain(error)):+ raise converted+ raise converted from error++ return cast(F, _sync_wrapper)diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex 699ad343f..917046e6e 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -16,6 +16,7 @@ import litellmfrom litellm import acompletion, completionfrom litellm.types.utils import all_litellm_params+from openviking.models.model_error import model_error_boundaryfrom openviking.telemetry import tracerfrom openviking.utils.message_format import format_messages, sanitize_openai_messagesfrom openviking.utils.multimodal import redact_image_data_urls@@ -454,6 +455,7 @@ class LiteLLMVLMProvider(VLMBase):kwargs_messages = [{"role": "user", "content": content}]return self._build_kwargs(model, kwargs_messages, tools, tool_choice, thinking=thinking)+ @model_error_boundarydef get_completion(self,prompt: str = "",@@ -477,6 +479,7 @@ class LiteLLMVLMProvider(VLMBase):return _call()+ @model_error_boundaryasync def get_completion_async(self,prompt: str = "",@@ -508,6 +511,7 @@ class LiteLLMVLMProvider(VLMBase):return await _call()+ @model_error_boundarydef get_vision_completion(self,prompt: str = "",@@ -531,6 +535,7 @@ class LiteLLMVLMProvider(VLMBase):return _call()+ @model_error_boundaryasync def get_vision_completion_async(self,prompt: str = "",diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 7f6023ead..fec0d336f 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -12,6 +12,7 @@ from urllib.parse import urlparseimport httpx+from openviking.models.model_error import model_error_boundaryfrom openviking.models.network import (create_optional_async_httpx_client,create_optional_sync_httpx_client,@@ -334,6 +335,7 @@ class OpenAIVLM(VLMBase):content = self._extract_content_from_response(response)return self._clean_response(content)+ @model_error_boundarydef get_completion(self,prompt: str = "",@@ -358,6 +360,7 @@ class OpenAIVLM(VLMBase):return _call()+ @model_error_boundaryasync def get_completion_async(self,prompt: str = "",@@ -442,6 +445,7 @@ class OpenAIVLM(VLMBase):}return {"type": "image_url", "image_url": {"url": image}}+ @model_error_boundarydef get_vision_completion(self,prompt: str = "",@@ -469,6 +473,7 @@ class OpenAIVLM(VLMBase):return _call()+ @model_error_boundaryasync def get_vision_completion_async(self,prompt: str = "",diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 7ec667c78..c57c8568d 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -9,6 +9,7 @@ import uuidfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union+from openviking.models.model_error import model_error_boundaryfrom openviking.models.network import (create_optional_async_httpx_client,create_optional_sync_httpx_client,@@ -120,12 +121,12 @@ class VolcEngineVLM(OpenAIVLM):raise ImportError("Please install volcenginesdkarkruntime: pip install volcenginesdkarkruntime")- kwargs = dict(- api_key=self.api_key,- base_url=self.api_base,- timeout=self.timeout,- max_retries=0,- )+ kwargs = {+ "api_key": self.api_key,+ "base_url": self.api_base,+ "timeout": self.timeout,+ "max_retries": 0,+ }http_client = create_optional_sync_httpx_client(self.api_base,timeout=self.timeout,@@ -143,12 +144,12 @@ class VolcEngineVLM(OpenAIVLM):raise ImportError("Please install volcenginesdkarkruntime: pip install volcenginesdkarkruntime")- kwargs = dict(- api_key=self.api_key,- base_url=self.api_base,- timeout=self.timeout,- max_retries=0,- )+ kwargs = {+ "api_key": self.api_key,+ "base_url": self.api_base,+ "timeout": self.timeout,+ "max_retries": 0,+ }http_client = create_optional_async_httpx_client(self.api_base,timeout=self.timeout,@@ -190,6 +191,7 @@ class VolcEngineVLM(OpenAIVLM):media_type=media_type,)+ @model_error_boundarydef get_completion(self,prompt: str = "",@@ -232,6 +234,7 @@ class VolcEngineVLM(OpenAIVLM):return resultreturn self._clean_response(str(result))+ @model_error_boundaryasync def get_completion_async(self,prompt: str = "",@@ -409,6 +412,7 @@ class VolcEngineVLM(OpenAIVLM):else:return {"type": "image_url", "image_url": {"url": image}}+ @model_error_boundarydef get_vision_completion(self,prompt: str = "",@@ -459,6 +463,7 @@ class VolcEngineVLM(OpenAIVLM):return resultreturn self._clean_response(str(result))+ @model_error_boundaryasync def get_vision_completion_async(self,prompt: str = "",diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 930b41cee..deb07cb63 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -9,11 +9,11 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, AsyncIterator, Callable, Dict, List, Optional, Union+from openviking.models.model_error import model_error_kindfrom openviking.utils.exceptions import AllCredentialsFailedErrorfrom openviking.utils.model_retry import (OrderedCredentialSwitcher,PrimaryBackupSwitcher,- classify_api_error,extract_metric_error_code,)from openviking_cli.utils import get_logger@@ -880,17 +880,19 @@ class MultiCredentialVLM(VLMBase):return resultexcept Exception as exc:_annotate_vlm_error(exc, vlm_instance)- error_class = classify_api_error(exc)- aggregated_errors.append((credential_id, error_class, exc, idx))+ kind = model_error_kind(exc)- if self._switcher.is_fail_fast(error_class):+ if self._switcher.is_fail_fast(kind):# Request-level failure (400 / input too large / content# safety): re-raise the original exception so callers can# react to its type; trying other credentials is useless.raise+ aggregated_errors.append((credential_id, kind, exc, idx))self._logger.warning(- f"Credential {credential_id} failed with {error_class}, trying next credential"+ "Credential %s failed with %s, trying next credential",+ credential_id,+ kind.value,)raise AllCredentialsFailedError(aggregated_errors)@@ -927,16 +929,18 @@ class MultiCredentialVLM(VLMBase):return resultexcept Exception as exc:_annotate_vlm_error(exc, vlm_instance)- error_class = classify_api_error(exc)- aggregated_errors.append((credential_id, error_class, exc, idx))+ kind = model_error_kind(exc)- if self._switcher.is_fail_fast(error_class):+ if self._switcher.is_fail_fast(kind):# Request-level failure: re-raise the original exception;# trying other credentials is useless.raise+ aggregated_errors.append((credential_id, kind, exc, idx))self._logger.warning(- f"Credential {credential_id} failed with {error_class}, trying next credential"+ "Credential %s failed with %s, trying next credential",+ credential_id,+ kind.value,)raise AllCredentialsFailedError(aggregated_errors)@@ -972,13 +976,14 @@ class MultiCredentialVLM(VLMBase):if emitted:raise- error_class = classify_api_error(exc)- aggregated_errors.append((credential_id, error_class, exc, idx))- if self._switcher.is_fail_fast(error_class):+ kind = model_error_kind(exc)+ if self._switcher.is_fail_fast(kind):raise-+ aggregated_errors.append((credential_id, kind, exc, idx))self._logger.warning(- f"Credential {credential_id} failed with {error_class}, trying next credential"+ "Credential %s failed with %s, trying next credential",+ credential_id,+ kind.value,)raise AllCredentialsFailedError(aggregated_errors)@@ -1117,16 +1122,16 @@ class MultiCredentialVLM(VLMBase):return resultexcept Exception as error:_annotate_vlm_error(error, vlm)- error_class = classify_api_error(error)- errors.append((credential_id, error_class, error, idx))+ kind = model_error_kind(error)if idx == start:active_credential_failed = True- if self._switcher.is_fail_fast(error_class):+ if self._switcher.is_fail_fast(kind):raise+ errors.append((credential_id, kind, error, idx))self._logger.warning("Credential %s failed media understanding with %s, trying next credential",credential_id,- error_class,+ kind.value,)if attempted:diff --git a/openviking/models/vlm/retry_vlm.py b/openviking/models/vlm/retry_vlm.pyindex d8b01d708..89a0598ea 100644--- a/openviking/models/vlm/retry_vlm.py+++ b/openviking/models/vlm/retry_vlm.py@@ -5,7 +5,7 @@from pathlib import Pathfrom typing import Any, Dict, List, Optional, Union-from openviking.utils.model_retry import retry_async, retry_sync+from openviking.utils.model_retry import effective_max_retries, retry_async, retry_syncfrom openviking_cli.utils import get_loggerfrom .base import VLMBase, VLMResponse@@ -59,9 +59,10 @@ class RetryVLM(VLMBase):messages=messages,**kwargs,),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} VLM completion",+ model_errors_only=True,)async def get_completion_async(@@ -84,9 +85,10 @@ class RetryVLM(VLMBase):max_tokens=max_tokens,**kwargs,),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} VLM async completion",+ model_errors_only=True,)def get_vision_completion(@@ -109,9 +111,10 @@ class RetryVLM(VLMBase):messages=messages,**kwargs,),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} VLM vision completion",+ model_errors_only=True,)async def get_vision_completion_async(@@ -134,9 +137,10 @@ class RetryVLM(VLMBase):messages=messages,**kwargs,),- max_retries=self.max_retries,+ max_retries=effective_max_retries(self.max_retries),logger=logger,operation_name=f"{self.provider} VLM async vision completion",+ model_errors_only=True,)def supports_media(diff --git a/openviking/service/search_service.py b/openviking/service/search_service.pyindex 840d72bc6..59e7e3d50 100644--- a/openviking/service/search_service.py+++ b/openviking/service/search_service.py@@ -17,6 +17,7 @@ from openviking.utils.image_search import (is_http_url,is_viking_uri,)+from openviking.utils.model_retry import model_retry_scopefrom openviking_cli.exceptions import InvalidArgumentError, NotInitializedErrorfrom openviking_cli.retrieve import ContextTypefrom openviking_cli.utils import get_logger@@ -109,6 +110,7 @@ class SearchService:"image_url must be a data:image base64 URI, http(s) URL, or viking:// URI.")+ @model_retry_scope(0)async def search(self,query: str,@@ -167,6 +169,7 @@ class SearchService:)return result+ @model_retry_scope(0)async def find(self,query: str,diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex a71a37eed..73b9b2d78 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optionalfrom openviking.core.context import ContextType, ResourceContentTypefrom openviking.models.embedder.base import embed_compat+from openviking.models.model_error import ModelErrorKind, model_error_kindfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.storage.acl import ACL_GRANT_FIELDS, ACL_MODE_FIELD, AclMode@@ -35,7 +36,6 @@ from openviking.storage.queuefs.embedding_msg import ()from openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult-from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.viking_vector_index_backend import (VIKINGDB_CONTENT_MAX_SIZE,@@ -44,16 +44,8 @@ from openviking.storage.viking_vector_index_backend import ()from openviking.telemetry import bind_telemetry, resolve_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker-from openviking.utils.circuit_breaker import (- CircuitBreakerOpen,- classify_api_error,-)+from openviking.utils.circuit_breaker import CircuitBreakerOpenfrom openviking.utils.log_correlation import log_correlation-from openviking.utils.model_retry import (- ERROR_CLASS_AUTH,- ERROR_CLASS_INPUT_TOO_LARGE,- ERROR_CLASS_PERMANENT,-)from openviking.utils.tags import merge_search_tagsfrom openviking.utils.time_utils import get_current_timestampfrom openviking_cli.session.user_id import UserIdentifier@@ -783,32 +775,40 @@ class TextEmbeddingHandler(DequeueHandlerBase):embedding_msg,f"Failed to generate embedding: {embed_err}",)- error_class = classify_api_error(embed_err)+ kind = model_error_kind(embed_err)try:from openviking.metrics.datasources import EmbeddingEventDataSourceEmbeddingEventDataSource.record_error(- error_code=str(error_class or "unknown"),+ error_code=kind.value if kind is not None else "unknown",account_id=embedding_msg.context_data.get("account_id"),)except Exception:pass- if error_class == ERROR_CLASS_INPUT_TOO_LARGE:+ if kind is ModelErrorKind.INPUT_TOO_LARGE:execute_status = "error"self._log_embedding_error(logging.ERROR, error_msg, embedding_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msgreturn ProcessResult.failed(error_msg)- if error_class == ERROR_CLASS_PERMANENT:+ if kind in {+ ModelErrorKind.MODEL_NOT_FOUND,+ ModelErrorKind.UNSUPPORTED,+ ModelErrorKind.INVALID_REQUEST,+ ModelErrorKind.CONTENT_SAFETY,+ }:execute_status = "error"self._log_embedding_error(logging.CRITICAL, error_msg, embedding_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msgreturn ProcessResult.failed(error_msg)- if error_class == ERROR_CLASS_AUTH:+ if kind in {+ ModelErrorKind.AUTHENTICATION_ERROR,+ ModelErrorKind.PERMISSION_DENIED,+ }:execute_status = "error"# Bad/expired credential: retrying cannot succeed. Fail# terminally instead of re-enqueueing, which would cycle@@ -821,47 +821,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):request_failed_message = error_msgreturn ProcessResult.failed(error_msg)- # Transient or unknown — re-enqueue for retry- self._log_embedding_error(logging.WARNING, error_msg, embedding_msg)- execute_status = "requeued"- if self._vikingdb.has_queue_manager:- if embedding_msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:- terminal_error = (- f"{error_msg} (failure requeue limit reached: "- f"{MAX_FAILURE_REQUEUE_COUNT})"- )- self._log_embedding_error(- logging.ERROR, terminal_error, embedding_msg- )- self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)- execute_status = "error"- request_failed_message = terminal_error- return ProcessResult.failed(terminal_error)- embedding_msg.failure_requeue_count += 1- try:- await self._reenqueue_embedding_msg(embedding_msg)- self._merge_request_stats(- embedding_msg.telemetry_id,- requeue_count=1,- )- get_request_wait_tracker().record_embedding_requeue(- embedding_msg.telemetry_id- )- logger.info(- "Re-enqueued embedding message after transient error: %s",- self._embedding_delivery_log_context(embedding_msg),- )- return ProcessResult.requeued()- except Exception as requeue_err:- logger.error(- self._embedding_error_msg(- embedding_msg,- f"Failed to re-enqueue message: {requeue_err}",- )- )-self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)execute_status = "error"+ self._log_embedding_error(logging.ERROR, error_msg, embedding_msg)request_failed_message = error_msgreturn ProcessResult.failed(error_msg)diff --git a/openviking/storage/queuefs/retry_policy.py b/openviking/storage/queuefs/retry_policy.pydeleted file mode 100644index 921483091..000000000--- a/openviking/storage/queuefs/retry_policy.py+++ /dev/null@@ -1,5 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""Shared limits for durable queue retry policies."""--MAX_FAILURE_REQUEUE_COUNT = 2diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 58930f4b6..4163c3216 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -10,6 +10,7 @@ from contextlib import nullcontextfrom typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuplefrom openviking.core.namespace import classify_uri+from openviking.models.model_error import ModelErrorKind, model_error_kindfrom openviking.observability.context import (bind_root_observability_context,reset_root_observability_context,@@ -51,7 +52,6 @@ from openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.index_action import FieldPatchfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult-from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStatsfrom openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key@@ -65,13 +65,8 @@ from openviking.telemetry import ()from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.span_models import create_root_span_attributes-from openviking.utils.circuit_breaker import (- CircuitBreaker,- CircuitBreakerOpen,- classify_api_error,-)+from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpenfrom openviking.utils.ingest_options import IngestOptions-from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENTfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.config import get_openviking_config@@ -82,6 +77,18 @@ if TYPE_CHECKING:logger = get_logger(__name__)+_PERMANENT_STORAGE_ERRORS = (+ FileNotFoundError,+ PermissionError,+ IsADirectoryError,+ NotADirectoryError,+ AGFSNotADirectoryError,+)+++def _is_permanent_storage_error(error: BaseException) -> bool:+ return any(isinstance(item, _PERMANENT_STORAGE_ERRORS) for item in (error, error.__cause__))+class RequestQueueStats:processed: int = 0@@ -327,20 +334,7 @@ class SemanticProcessor(DequeueHandlerBase):error: Exception,*,work: SemanticMessageWork,- consume_failure_budget: bool = True,) -> ProcessResult:- if consume_failure_budget and msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:- terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"- logger.error("Semantic message reached failure requeue limit: %s", msg.uri)- self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, terminal_error- )- await self._cleanup_local_artifact(msg)- return ProcessResult.failed(terminal_error)-- if consume_failure_budget:- msg.failure_requeue_count += 1try:await work.reenqueue_after_error()self._merge_request_stats(msg.telemetry_id, requeue_count=1)@@ -793,15 +787,14 @@ class SemanticProcessor(DequeueHandlerBase):msg,e,work=work,- consume_failure_budget=False,)if result.error is not None:execute_status = "error"return resultreturn ProcessResult.failed(str(e))- error_class = classify_api_error(e)- if error_class == ERROR_CLASS_INPUT_TOO_LARGE:+ kind = model_error_kind(e)+ if kind is ModelErrorKind.INPUT_TOO_LARGE:execute_status = "error"logger.error(f"Input too large processing semantic message, dropping: {e}",@@ -815,16 +808,18 @@ class SemanticProcessor(DequeueHandlerBase):if msg is not None:await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))- elif error_class == ERROR_CLASS_PERMANENT:+ elif _is_permanent_storage_error(e) or kind in {+ ModelErrorKind.MODEL_NOT_FOUND,+ ModelErrorKind.UNSUPPORTED,+ ModelErrorKind.INVALID_REQUEST,+ ModelErrorKind.CONTENT_SAFETY,+ }:execute_status = "error"logger.critical(f"Permanent error processing semantic message, dropping: {e}",exc_info=True,)- # A malformed filesystem target does not indicate an API outage.- if breaker is not None and not any(- isinstance(exc, AGFSNotADirectoryError) for exc in (e, e.__cause__)- ):+ if breaker is not None and kind is not None:breaker.record_failure(e)if msg is not None:self._merge_request_stats(msg.telemetry_id, error_count=1)@@ -835,23 +830,19 @@ class SemanticProcessor(DequeueHandlerBase):await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))else:- # Transient or unknown — re-enqueue for retry- execute_status = "requeued"- logger.warning(- f"Transient API error processing semantic message, re-enqueueing: {e}",+ execute_status = "error"+ if breaker is not None and kind is not None:+ breaker.record_failure(e)+ logger.error(+ f"Error processing semantic message, not re-enqueueing: {e}",exc_info=True,)- if breaker is not None:- breaker.record_failure(e)- if msg is not None and work is not None:- result = await self._requeue_semantic_msg_after_error(- msg,- e,- work=work,+ if msg is not None:+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ get_request_wait_tracker().mark_semantic_failed(+ msg.telemetry_id, msg.id, str(e))- if result.error is not None:- execute_status = "error"- return result+ await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))finally:if msg is not None and execute_started_at is not None:diff --git a/openviking/utils/circuit_breaker.py b/openviking/utils/circuit_breaker.pyindex 9af434a07..8305aa22a 100644--- a/openviking/utils/circuit_breaker.py+++ b/openviking/utils/circuit_breaker.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport threadingimport time+from openviking.models.model_error import ModelErrorKind, model_error_kindfrom openviking.utils.model_retry import (ERROR_CLASS_AUTH,ERROR_CLASS_INPUT_TOO_LARGE,@@ -95,9 +96,19 @@ class CircuitBreaker:def record_failure(self, error: Exception) -> None:"""Record a failed API call. May trip the breaker."""- error_class = classify_api_error(error)- if error_class == ERROR_CLASS_INPUT_TOO_LARGE:- logger.info(f"Circuit breaker ignoring row-specific input error: {error}")+ kind = model_error_kind(error)+ legacy_class = classify_api_error(error) if kind is None else None+ if kind in {+ ModelErrorKind.AUTHENTICATION_ERROR,+ ModelErrorKind.PERMISSION_DENIED,+ ModelErrorKind.MODEL_NOT_FOUND,+ ModelErrorKind.UNSUPPORTED,+ ModelErrorKind.INVALID_REQUEST,+ ModelErrorKind.INPUT_TOO_LARGE,+ ModelErrorKind.CONTENT_SAFETY,+ } or legacy_class == ERROR_CLASS_INPUT_TOO_LARGE:+ label = kind.value if kind is not None else legacy_class+ logger.info("Circuit breaker ignoring %s error: %s", label, error)returnwith self._lock:@@ -115,14 +126,15 @@ class CircuitBreaker:)return- if error_class in (+ if kind is ModelErrorKind.QUOTA_EXHAUSTED or legacy_class in {ERROR_CLASS_PERMANENT,ERROR_CLASS_AUTH,ERROR_CLASS_QUOTA_EXCEEDED,- ):+ }:self._state = _STATE_OPENself._current_reset_timeout = self._base_reset_timeout- logger.info(f"Circuit breaker tripped immediately on {error_class} error: {error}")+ label = kind.value if kind is not None else legacy_class+ logger.info("Circuit breaker tripped immediately on %s: %s", label, error)returnif self._failure_count >= self._failure_threshold:diff --git a/openviking/utils/exceptions.py b/openviking/utils/exceptions.pyindex d3b8444b2..0bf111976 100644--- a/openviking/utils/exceptions.py+++ b/openviking/utils/exceptions.py@@ -2,6 +2,8 @@# SPDX-License-Identifier: AGPL-3.0"""Common exception helpers."""+from openviking.models.model_error import ModelErrorKind+HTTP_STATUS_TO_ERROR_CODE = {400: "INVALID_ARGUMENT",401: "UNAUTHENTICATED",@@ -34,15 +36,42 @@ def error_code_from_http_status(status: int | None, default: str = "INVALID_ARGUclass AllCredentialsFailedError(Exception):"""Raised when all credentials in the chain have failed."""- def __init__(self, errors: list[tuple[str, str, Exception, int]]):+ def __init__(+ self,+ errors: list[tuple[str, ModelErrorKind | str, Exception, int]],+ ):"""Initialize the error with a list of credential failures.Args:- errors: List of tuples containing (credential_id, error_class, exception, attempts)+ errors: List of tuples containing+ (credential_id, model_error_kind, exception, credential_index)"""- self.errors = errors+ self.errors = [+ (+ credential_id,+ kind if isinstance(kind, ModelErrorKind) else ModelErrorKind.UNKNOWN,+ error,+ credential_index,+ )+ for credential_id, kind, error, credential_index in errors+ ]+ kinds = [kind for _, kind, _, _ in self.errors]+ self.kind = self._aggregate_kind(kinds)message = "All credentials failed:\n" + "\n".join(- f" - {cred_id}: {error_class} - {exc}"- for cred_id, error_class, exc, attempts in errors+ f" - {cred_id}: {kind.value} - {exc}" for cred_id, kind, exc, _ in self.errors)super().__init__(message)++ @staticmethod+ def _aggregate_kind(kinds: list[ModelErrorKind]) -> ModelErrorKind:+ for kind in (+ ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ ModelErrorKind.RATE_LIMIT_ERROR,+ ModelErrorKind.QUOTA_EXHAUSTED,+ ):+ if kind in kinds:+ return kind+ if kinds and all(kind is kinds[0] for kind in kinds):+ return kinds[0]+ return ModelErrorKind.UNKNOWNdiff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.pyindex 2caf88c82..5fe9a9a18 100644--- a/openviking/utils/model_retry.py+++ b/openviking/utils/model_retry.py@@ -6,15 +6,45 @@ import randomimport reimport threadingimport time-from typing import Awaitable, Callable, TypeVar-+from contextlib import asynccontextmanager+from contextvars import ContextVar+from typing import AsyncIterator, Awaitable, Callable, TypeVar++from openviking.models.model_error import (+ ModelErrorKind,+ model_error_from_exception,+ model_error_kind,+)from openviking.pyagfs.exceptions import AGFSNotADirectoryError-from openviking.utils.exceptions import AllCredentialsFailedErrorlogger = logging.getLogger(__name__)T = TypeVar("T")+_max_retries_override: ContextVar[int | None] = ContextVar(+ "model_max_retries_override",+ default=None,+)+++@asynccontextmanager+async def model_retry_scope(max_retries: int) -> AsyncIterator[None]:+ """Temporarily override model retries for the current async context."""+ if max_retries < 0:+ raise ValueError("max_retries must be non-negative")+ token = _max_retries_override.set(max_retries)+ try:+ yield+ finally:+ _max_retries_override.reset(token)+++def effective_max_retries(configured_max_retries: int) -> int:+ """Return the request override when set, otherwise the configured value."""+ override = _max_retries_override.get()+ return configured_max_retries if override is None else override++# Error classification categories returned by classify_api_error()ERROR_CLASS_PERMANENT = "permanent" # request-level 4xx (e.g. 400 invalid parameter)ERROR_CLASS_AUTH = "auth" # credential-level 401/403 (key invalid / no permission / overdue)@@ -217,19 +247,9 @@ def classify_api_error(error: Exception) -> str:- ``quota_exceeded`` is checked before ``transient`` because quota errorstypically include "429" / "TooManyRequests" which would otherwise matchthe transient category.- - an aggregated ``AllCredentialsFailedError`` is classified from its- per-credential classes, not its concatenated message.+ Structured model errors are intentionally handled by their callers using+ ``ModelErrorKind`` and are not collapsed into this legacy string taxonomy."""- if isinstance(error, AllCredentialsFailedError):- classes = [ec for (_cid, ec, _exc, _idx) in error.errors if ec]- if ERROR_CLASS_TRANSIENT in classes:- return ERROR_CLASS_TRANSIENT- if ERROR_CLASS_QUOTA_EXCEEDED in classes:- return ERROR_CLASS_QUOTA_EXCEEDED- if classes and all(ec == ERROR_CLASS_AUTH for ec in classes):- return ERROR_CLASS_AUTH- return ERROR_CLASS_UNKNOWN-for exc in (error, getattr(error, "__cause__", None)):if exc is not None and isinstance(exc, _PERMANENT_IO_ERRORS):return ERROR_CLASS_PERMANENT@@ -288,6 +308,14 @@ def classify_api_error(error: Exception) -> str:def is_retryable_api_error(error: Exception) -> bool:"""Return True if the error should be retried."""+ kind = model_error_kind(error)+ if kind is not None:+ return kind in {+ ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ ModelErrorKind.RATE_LIMIT_ERROR,+ ModelErrorKind.UNKNOWN,+ }return classify_api_error(error) == ERROR_CLASS_TRANSIENT@@ -393,6 +421,28 @@ def _compute_delay(return delay+def _retry_limit(+ error: Exception,+ *,+ max_retries: int,+ is_retryable: Callable[[Exception], bool],+ model_errors_only: bool,+) -> int:+ model_error = model_error_from_exception(error)+ if model_error is not None:+ if model_error.kind in {+ ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ ModelErrorKind.RATE_LIMIT_ERROR,+ ModelErrorKind.UNKNOWN,+ }:+ return max_retries+ return 0+ if model_errors_only:+ return 0+ return max_retries if is_retryable(error) else 0++def retry_sync(func: Callable[[], T],*,@@ -403,6 +453,7 @@ def retry_sync(is_retryable: Callable[[Exception], bool] = is_retryable_api_error,logger=None,operation_name: str = "operation",+ model_errors_only: bool = False,) -> T:"""Retry a sync function on known transient errors."""attempt = 0@@ -411,7 +462,13 @@ def retry_sync(try:return func()except Exception as e:- if max_retries <= 0 or attempt >= max_retries or not is_retryable(e):+ retry_limit = _retry_limit(+ e,+ max_retries=max_retries,+ is_retryable=is_retryable,+ model_errors_only=model_errors_only,+ )+ if attempt >= retry_limit:raisedelay = _compute_delay(@@ -425,7 +482,7 @@ def retry_sync("%s failed with retryable error (retry %d/%d): %s; retrying in %.2fs",operation_name,attempt + 1,- max_retries,+ retry_limit,e,delay,)@@ -443,6 +500,7 @@ async def retry_async(is_retryable: Callable[[Exception], bool] = is_retryable_api_error,logger=None,operation_name: str = "operation",+ model_errors_only: bool = False,) -> T:"""Retry an async function on known transient errors."""attempt = 0@@ -451,7 +509,13 @@ async def retry_async(try:return await func()except Exception as e:- if max_retries <= 0 or attempt >= max_retries or not is_retryable(e):+ retry_limit = _retry_limit(+ e,+ max_retries=max_retries,+ is_retryable=is_retryable,+ model_errors_only=model_errors_only,+ )+ if attempt >= retry_limit:raisedelay = _compute_delay(@@ -465,7 +529,7 @@ async def retry_async("%s failed with retryable error (retry %d/%d): %s; retrying in %.2fs",operation_name,attempt + 1,- max_retries,+ retry_limit,e,delay,)@@ -476,8 +540,8 @@ async def retry_async(class PrimaryBackupSwitcher:"""Thread-safe primary/backup switcher with automatic failback logic.- When an error of type ERROR_CLASS_PERMANENT or ERROR_CLASS_QUOTA_EXCEEDED occurs,- switches to backup immediately. Then, after either:+ When a credential- or provider-specific model error occurs, switches to+ backup immediately. Then, after either:- 10 minutes have passed, OR- 200 requests have been made to backupit will attempt to failback to primary. If failback fails, it switches back@@ -535,18 +599,29 @@ class PrimaryBackupSwitcher:def record_primary_failure(self, error: Exception) -> bool:"""Record a primary failure. Returns True if should switch to backup.- Switches to backup immediately for ERROR_CLASS_PERMANENT,- ERROR_CLASS_AUTH or ERROR_CLASS_QUOTA_EXCEEDED.+ Request-specific errors and unknown failures do not switch models."""- error_class = classify_api_error(error)- if error_class in (- ERROR_CLASS_PERMANENT,- ERROR_CLASS_AUTH,- ERROR_CLASS_QUOTA_EXCEEDED,- ):+ kind = model_error_kind(error)+ should_switch = kind in {+ ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ ModelErrorKind.TRANSIENT_NETWORK_ERROR,+ ModelErrorKind.RATE_LIMIT_ERROR,+ ModelErrorKind.AUTHENTICATION_ERROR,+ ModelErrorKind.PERMISSION_DENIED,+ ModelErrorKind.QUOTA_EXHAUSTED,+ ModelErrorKind.MODEL_NOT_FOUND,+ }+ if kind is None:+ should_switch = classify_api_error(error) in {+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_AUTH,+ ERROR_CLASS_QUOTA_EXCEEDED,+ }+ if should_switch:with self._lock:if not self._using_backup:- logger.warning(f"Primary failed with {error_class}, switching to backup")+ label = kind.value if kind is not None else classify_api_error(error)+ logger.warning("Primary failed with %s, switching to backup", label)self._using_backup = True# Always reset timer and counter when we fail (whether initial fail or failback fail)self._switch_to_backup_time = time.monotonic()@@ -570,8 +645,7 @@ class PrimaryBackupSwitcher:class OrderedCredentialSwitcher:"""Thread-safe ordered N-credential switcher with hierarchical failback.- Supports ordered failover across multiple credentials. When a credential fails- with quota_exceeded or permanent error, it advances to the next credential.+ Supports ordered failover across multiple credentials.After failback thresholds are met, it attempts to move back to a higher-prioritycredential (one step at a time, not all the way back to index 0).@@ -592,17 +666,8 @@ class OrderedCredentialSwitcher:failback_request_count: Number of requests after which to attempt failbackNote:- Failure handling is driven by the error class (see- ``classify_api_error``):-- - request-level errors (``permanent`` 400 / ``input_too_large`` /- ``content_safety``) fail fast: the same request fails on every- credential of the same model, so switching is useless.- - credential-level ``auth`` errors (401/403) advance to the next- credential in multi-credential mode; the last (or single)- credential fails fast.- - ``quota_exceeded`` (and ``transient`` once its retries are- exhausted) and ``unknown`` advance to the next credential.+ Request-specific and unknown model errors fail fast. Credential,+ model-routing, quota, and exhausted transient failures advance."""if n < 1:raise ValueError("Number of credentials must be >= 1")@@ -666,18 +731,19 @@ class OrderedCredentialSwitcher:self._active_request_count += 1@staticmethod- def is_fail_fast(error_class: str) -> bool:+ def is_fail_fast(kind: ModelErrorKind | None) -> bool:"""Whether an error is request-level and must not try other credentials.- Request-level errors (400 parameter error, input too large, content- safety) fail on every credential of the same model, so the caller should- re-raise immediately instead of cycling through credentials.+ Unknown failures remain within the retry layer and never expand into a+ credential or model failover."""- return error_class in (- ERROR_CLASS_PERMANENT,- ERROR_CLASS_INPUT_TOO_LARGE,- ERROR_CLASS_CONTENT_SAFETY,- )+ return kind is None or kind in {+ ModelErrorKind.UNSUPPORTED,+ ModelErrorKind.INVALID_REQUEST,+ ModelErrorKind.INPUT_TOO_LARGE,+ ModelErrorKind.CONTENT_SAFETY,+ ModelErrorKind.UNKNOWN,+ }def commit_success(self, idx: int) -> None:"""Record that credential ``idx`` successfully served a request.diff --git a/tests/api_test/filesystem/test_fs_cp.py b/tests/api_test/filesystem/test_fs_cp.pyindex cb5aaffed..ef3e19487 100644--- a/tests/api_test/filesystem/test_fs_cp.py+++ b/tests/api_test/filesystem/test_fs_cp.py@@ -153,10 +153,6 @@ class TestFsCp:child_content = f"directory child remains {suffix}"try:assert api_client.fs_mkdir(directory_uri).status_code == 200- # mkdir queues a parent refresh; finish it before preparing files so- # this test exercises type validation, not a transient refresh lock.- settled = api_client.system_wait()- assert settled.status_code == 200, settled.textfor uri, content in ((file_uri, file_content), (child_uri, child_content)):write = api_client.fs_write(uri, content, mode="create", wait=True)assert write.status_code == 200, write.textdiff --git a/tests/cli/conftest.py b/tests/cli/conftest.pyindex 1a5480c94..cd2f69fad 100644--- a/tests/cli/conftest.py+++ b/tests/cli/conftest.py@@ -298,9 +298,7 @@ def _write_cli_config():CLI_CONFIG_PATH = _write_cli_config()-ADD_RESOURCE_WAIT_TIMEOUT = "300"ADD_RESOURCE_COMMAND_TIMEOUT = 360-WRITE_WAIT_TIMEOUT = "300"WRITE_COMMAND_TIMEOUT = 360@@ -570,8 +568,6 @@ def ov_write(uri, content, *extra_args):content,*extra_args,"--wait",- "--timeout",- WRITE_WAIT_TIMEOUT,"-o","json",],diff --git a/tests/cli/test_cli_filesystem.py b/tests/cli/test_cli_filesystem.pyindex 1a9cee45b..9649ba9c4 100644--- a/tests/cli/test_cli_filesystem.py+++ b/tests/cli/test_cli_filesystem.py@@ -127,7 +127,10 @@ class TestFsMv:f.write("move me")temp_path = f.nametry:- ov_add_resource(temp_path, src_uri)+ add_result = ov_add_resource(temp_path, src_uri)+ assert add_result["exit_code"] == 0, (+ f"add-resource for mv failed: {add_result['stderr'][:300]}"+ )finally:os.unlink(temp_path)r = ov_mv(src_uri, dst_uri)diff --git a/tests/cli/test_cli_system.py b/tests/cli/test_cli_system.pyindex 46d6bc43f..029716cc0 100644--- a/tests/cli/test_cli_system.py+++ b/tests/cli/test_cli_system.py@@ -31,16 +31,3 @@ class TestSystemStatus:assert data is not None, f"status should return JSON, got stdout: {r['stdout'][:200]}"assert data.get("ok") is True, f"Expected ok=true, got {data.get('ok')}"assert "result" in data, "'result' field should exist in status response"---class TestSystemWait:- @pytest.mark.skip(- reason="CLI --timeout passed as query param but server only reads from request body, "- "causing infinite wait when queue is not empty. "- "Re-enable after CLI/Server timeout parameter mismatch is fixed."- )- def test_wait_with_timeout(self):- r = ov(["wait", "--timeout", "30", "-o", "json"], timeout=120)- assert r["exit_code"] == 0, (- f"ov wait should exit 0, got {r['exit_code']}: {r['stderr'][:300]}"- )diff --git a/tests/service/test_search_service_image.py b/tests/service/test_search_service_image.pyindex 50bead176..be3ff50c7 100644--- a/tests/service/test_search_service_image.py+++ b/tests/service/test_search_service_image.py@@ -1,13 +1,17 @@+import asyncio+import pytestfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.search_service import SearchService+from openviking.utils.model_retry import effective_max_retriesfrom openviking_cli.session.user_id import UserIdentifierclass FakeVikingFS:def __init__(self):self.image_url = None+ self.retry_values = {}async def read_file_bytes(self, uri, ctx=None):assert uri == "viking://resources/cat.png"@@ -15,20 +19,31 @@ class FakeVikingFS:async def find(self, **kwargs):self.image_url = kwargs["image_url"]+ await asyncio.sleep(0) # Keep the search scope active during the concurrent call.+ self.retry_values["find"] = effective_max_retries(3)+ return {"ok": True}++ async def search(self, **kwargs):+ self.retry_values["search"] = effective_max_retries(3)return {"ok": True}@pytest.mark.asyncio-async def test_search_service_resolves_viking_image_url_to_data_uri():+async def test_image_search_resolves_uri_and_isolates_retry_scope():fs = FakeVikingFS()service = SearchService(fs)ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.USER)- result = await service.find(- query="",- image_url="viking://resources/cat.png",- ctx=ctx,+ result, _ = await asyncio.gather(+ service.find(query="", image_url="viking://resources/cat.png", ctx=ctx),+ fs.search(query="concurrent background work"),)assert result == {"ok": True}assert fs.image_url.startswith("data:image/png;base64,")+ assert fs.retry_values == {"find": 0, "search": 3}++ await service.search(query="search", ctx=ctx)++ assert fs.retry_values == {"find": 0, "search": 0}+ assert effective_max_retries(3) == 3diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 630fe96dc..c0f149c01 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -13,6 +13,7 @@ import pytestimport requestsfrom openviking.models.embedder.base import DenseEmbedderBase, EmbedResult+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.server.identity import RequestContext, Role, UserIdentifierfrom openviking.service.resource_service import ResourceServicefrom openviking.storage.collection_schemas import (@@ -32,7 +33,6 @@ from openviking.storage.expr import Eqfrom openviking.storage.index_action import IndexActionfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.queuefs.process_result import ProcessOutcome-from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.vectordb import engine as vectordb_enginefrom openviking.storage.vectordb.collection.collection import Collection@@ -861,16 +861,7 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w@pytest.mark.asyncio- ("failure_requeue_count", "expected_outcome"),- [- (0, ProcessOutcome.REQUEUED),- (MAX_FAILURE_REQUEUE_COUNT, ProcessOutcome.FAILED),- ],-)-async def test_embedding_failure_requeue_limit(- monkeypatch, failure_requeue_count, expected_outcome-):+async def test_embedding_model_error_fails_without_reenqueue(monkeypatch):class _QueueingVikingDB:is_closing = Falsehas_queue_manager = True@@ -881,88 +872,23 @@ async def test_embedding_failure_requeue_limit(async def enqueue_embedding_msg(self, msg):self.enqueued.append(msg)- class _TransientErrorEmbedder(_DummyEmbedder):+ class _ModelErrorEmbedder(_DummyEmbedder):async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- raise RuntimeError("500 Internal Server Error")+ raise ModelError(ModelErrorKind.TRANSIENT_SERVICE_ERROR, "model request failed")vikingdb = _QueueingVikingDB()monkeypatch.setattr("openviking_cli.utils.config.get_openviking_config",- lambda: _DummyConfig(_TransientErrorEmbedder()),+ lambda: _DummyConfig(_ModelErrorEmbedder()),)handler = TextEmbeddingHandler(vikingdb)- msg = EmbeddingMsg(- message="hello",- context_data={- "id": "id-1",- "uri": "viking://resources/sample",- "account_id": "default",- "abstract": "sample",- },- failure_requeue_count=failure_requeue_count,- )-- result = await handler.on_dequeue({"data": msg.to_json()})-- assert result.outcome is expected_outcome- if expected_outcome is ProcessOutcome.REQUEUED:- assert len(vikingdb.enqueued) == 1- assert vikingdb.enqueued[0].failure_requeue_count == 1- else:- assert vikingdb.enqueued == []- assert result.error is not None- assert "failure requeue limit reached" in result.error+ result = await handler.on_dequeue(_build_queue_payload())-async def test_embedding_failure_requeue_stops_after_three_attempts(monkeypatch):- class _QueueingVikingDB:- is_closing = False- has_queue_manager = True-- def __init__(self):- self.enqueued = []-- async def enqueue_embedding_msg(self, msg):- self.enqueued.append(msg)-- class _TransientErrorEmbedder(_DummyEmbedder):- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- self.calls += 1- raise RuntimeError("500 Internal Server Error")-- embedder = _TransientErrorEmbedder()- vikingdb = _QueueingVikingDB()- monkeypatch.setattr(- "openviking_cli.utils.config.get_openviking_config",- lambda: _DummyConfig(embedder),- )- handler = TextEmbeddingHandler(vikingdb)- msg = EmbeddingMsg(- message="hello",- context_data={- "id": "id-1",- "uri": "viking://resources/sample",- "account_id": "default",- "abstract": "sample",- },- )-- results = []- for _ in range(3):- results.append(await handler.on_dequeue({"data": msg.to_json()}))- if results[-1].outcome is ProcessOutcome.REQUEUED:- msg = vikingdb.enqueued[-1]-- assert [result.outcome for result in results] == [- ProcessOutcome.REQUEUED,- ProcessOutcome.REQUEUED,- ProcessOutcome.FAILED,- ]- assert [msg.failure_requeue_count for msg in vikingdb.enqueued] == [1, 2]- assert embedder.calls == 3- assert results[-1].error is not None- assert "failure requeue limit reached" in results[-1].error+ assert result.outcome is ProcessOutcome.FAILED+ assert result.error is not None+ assert "model request failed" in result.error+ assert vikingdb.enqueued == []@pytest.mark.asyncio@@ -984,7 +910,11 @@ async def test_embedding_auth_error_fails_terminally_without_reenqueue(monkeypatclass _AuthErrorEmbedder(_DummyEmbedder):async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- raise RuntimeError("Error code: 401 - {'code': 'AuthenticationError'} Unauthorized")+ raise ModelError(+ ModelErrorKind.AUTHENTICATION_ERROR,+ "Error code: 401 - {'code': 'AuthenticationError'} Unauthorized",+ status_code=401,+ )vikingdb = _QueueingVikingDB()monkeypatch.setattr(@@ -1016,9 +946,11 @@ async def test_request_level_400_does_not_trip_account_breaker(monkeypatch, is_qasync def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:if self.calls == 0:self.calls += 1- raise RuntimeError(+ raise ModelError(+ ModelErrorKind.INVALID_REQUEST,"Error code: 400 - {'error': {'code': 'InvalidParameter', "- "'message': 'Invalid base64 image url'}}"+ "'message': 'Invalid base64 image url'}}",+ status_code=400,)return self.embed(text, is_query=is_query)@@ -1030,7 +962,7 @@ async def test_request_level_400_does_not_trip_account_breaker(monkeypatch, is_qhandler = TextEmbeddingHandler(SimpleNamespace(is_closing=False, has_queue_manager=True))provider = handler._embedding_provider- with pytest.raises(RuntimeError, match="400"):+ with pytest.raises(ModelError, match="400"):await provider.embed("default", "bad input", is_query=is_query)(await handler.breaker()).check() # would raise CircuitBreakerOpen if tripped@@ -1205,10 +1137,6 @@ async def test_embedding_handler_propagates_account_id_on_error(monkeypatch):"openviking.metrics.datasources.EmbeddingEventDataSource.record_error",staticmethod(lambda **kwargs: captured.update(kwargs)),)- monkeypatch.setattr(- "openviking.storage.collection_schemas.classify_api_error",- lambda _err: "unknown",- )handler = TextEmbeddingHandler(_DummyVikingDB())await handler.on_dequeue(_build_queue_payload_for_account("acct-embed-error"))diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 99979cf56..1e682013b 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -54,9 +54,6 @@ def _build_data(msg: SemanticMsg) -> dict:@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, context_type):processor = _processor()- processor._circuit_breaker = SimpleNamespace(- check=MagicMock(side_effect=CircuitBreakerOpen), retry_after=0- )entered, release = asyncio.Event(), asyncio.Event()written = []@@ -71,6 +68,9 @@ async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, contextlambda: SimpleNamespace(SEMANTIC="Semantic", get_queue=lambda _: queue),)msg = _make_msg(context_type=context_type, telemetry_id=str(uuid4()))+ processor._circuit_breakers[msg.account_id] = SimpleNamespace(+ check=MagicMock(side_effect=CircuitBreakerOpen), retry_after=0+ )worker = asyncio.create_task(processor.on_dequeue(msg.to_dict()))try:await asyncio.wait_for(entered.wait(), 1)@@ -167,14 +167,8 @@ async def test_memory_ls_error_returns_failed():@pytest.mark.asyncio-async def test_memory_ls_transient_error_requeues():- """Transient errors during ls() re-enqueue the msg and increment requeue count.-- A 500-class error wrapped by the processor's `raise RuntimeError(...) from e`- is classified as `transient`. The outer on_dequeue() path must call- _reenqueue_semantic_msg(), bump requeue_count, and return REQUEUED without- a terminal error.- """+async def test_memory_ls_transient_error_returns_failed():+ """Queue processing does not retry an error after the owning layer exhausts it."""processor = _processor()fake_fs = MagicMock()@@ -184,8 +178,6 @@ async def test_memory_ls_transient_error_requeues():msg = _make_msg(telemetry_id="tel-1")data = _build_data(msg)- reenqueue_mock = AsyncMock()-with (patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",@@ -195,14 +187,14 @@ async def test_memory_ls_transient_error_requeues():"openviking.storage.queuefs.semantic_work.resolve_telemetry",return_value=None,),- patch.object(processor, "_reenqueue_semantic_msg", new=reenqueue_mock),):result = await processor.on_dequeue(data)- assert result.outcome is ProcessOutcome.REQUEUED+ assert result.outcome is ProcessOutcome.FAILEDassert result.value is None- assert result.error is None- reenqueue_mock.assert_awaited_once()+ assert result.error == (+ f"Failed to list memory directory {msg.uri}: 500 Internal Server Error"+ )@pytest.mark.asynciodiff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pyindex 999f0e12e..af31296ad 100644--- a/tests/storage/test_semantic_processor_permanent_storage_error.py+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -10,13 +10,13 @@ from uuid import uuid4import pytest+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.service.task_queue_middleware import TaskWorkQueueMiddlewarefrom openviking.service.task_work_index import TaskWorkIndexfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import NamedQueuefrom openviking.storage.queuefs.process_result import ProcessOutcome-from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor@@ -217,20 +217,14 @@ async def test_malformed_resources_do_not_pause_following_valid_work(semantic_en@pytest.mark.asyncio- ("error", "trips_breaker"),- [(TimeoutError("API timeout"), True), (LockAcquisitionError("tree is locked"), False)],-)-async def test_retryable_errors_keep_existing_behavior(- semantic_env, monkeypatch, error, trips_breaker-):+async def test_lock_error_requeues_without_consuming_failure_budget(semantic_env, monkeypatch):env = semantic_envsleep = AsyncMock()monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)msg = env.message()breaker = CircuitBreaker(failure_threshold=1)env.processor._circuit_breakers[msg.account_id] = breaker- env.dag.run.side_effect = error+ env.dag.run.side_effect = LockAcquisitionError("tree is locked")result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)@@ -238,72 +232,34 @@ async def test_retryable_errors_keep_existing_behavior(env.retry_queue.enqueue.assert_awaited_once()requeued = env.retry_queue.enqueue.call_args.args[0]assert requeued.id == msg.id- assert requeued.failure_requeue_count == (0 if isinstance(error, LockAcquisitionError) else 1)+ assert requeued.failure_requeue_count == 0assert not env.tracker.is_complete(msg.telemetry_id)assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1- if trips_breaker:- with pytest.raises(CircuitBreakerOpen):- breaker.check()- sleep.assert_awaited_once()- else:- breaker.check()- sleep.assert_not_awaited()+ breaker.check()+ sleep.assert_not_awaited()@pytest.mark.asyncio-async def test_semantic_failure_requeue_limit_is_terminal(semantic_env):+async def test_semantic_model_error_fails_without_requeue(semantic_env):env = semantic_env- msg = env.message(failure_requeue_count=MAX_FAILURE_REQUEUE_COUNT)- error = TimeoutError("API timeout")+ msg = env.message()+ error = ModelError(ModelErrorKind.TRANSIENT_SERVICE_ERROR, "model request failed")env.dag.run.side_effect = errorresult = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)- terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"assert result.outcome is ProcessOutcome.FAILED- assert result.error == terminal_error+ assert result.error == str(error)env.retry_queue.enqueue.assert_not_awaited()- await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)- assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {- "processed": 0,- "requeue_count": 0,- "error_count": 1,- "errors": [{"message": terminal_error}],- }---async def test_semantic_failure_requeue_stops_after_three_attempts(semantic_env):- env = semantic_env- msg = env.message()- work = SimpleNamespace(reenqueue_after_error=AsyncMock())- error = TimeoutError("API timeout")-- results = []- for _ in range(3):- msg = SemanticMsg.from_dict(msg.to_dict())- results.append(- await env.processor._requeue_semantic_msg_after_error(- msg,- error,- work=work,- )- )-- assert [result.outcome for result in results] == [- ProcessOutcome.REQUEUED,- ProcessOutcome.REQUEUED,- ProcessOutcome.FAILED,- ]- assert msg.failure_requeue_count == MAX_FAILURE_REQUEUE_COUNT- assert work.reenqueue_after_error.await_count == MAX_FAILURE_REQUEUE_COUNT+ await _assert_request_failed(env, msg, error)+ env.processor._account_breaker(msg.account_id).check()@pytest.mark.asyncio-async def test_permanent_api_error_still_opens_breaker(semantic_env):+async def test_invalid_request_does_not_open_breaker(semantic_env):env = semantic_envmsg = env.message()- error = RuntimeError("400 invalid model parameter")+ error = ModelError(ModelErrorKind.INVALID_REQUEST, "400 invalid model parameter")env.dag.run.side_effect = errorresult = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)@@ -312,8 +268,9 @@ async def test_permanent_api_error_still_opens_breaker(semantic_env):assert result.error == str(error)env.retry_queue.enqueue.assert_not_awaited()await _assert_request_failed(env, msg, error)- with pytest.raises(CircuitBreakerOpen):- env.processor._account_breaker(msg.account_id).check()+ breaker = env.processor._account_breaker(msg.account_id)+ breaker.check()+ assert breaker._failure_count == 0@pytest.mark.asynciodiff --git a/tests/unit/test_cohere_embedder.py b/tests/unit/test_cohere_embedder.pyindex 27aee11fc..82dbc2c09 100644--- a/tests/unit/test_cohere_embedder.py+++ b/tests/unit/test_cohere_embedder.py@@ -12,6 +12,7 @@ from openviking.models.embedder.cohere_embedders import (COHERE_MODEL_DIMENSIONS,get_cohere_model_default_dimension,)+from openviking.models.model_error import ModelErrorclass TestCohereDenseEmbedder:@@ -167,7 +168,7 @@ class TestCohereDenseEmbedder:api_key="bad-key",)- with pytest.raises(RuntimeError, match="Cohere API error"):+ with pytest.raises(ModelError, match="Cohere API error"):embedder.embed("Hello world")@patch("openviking.models.embedder.cohere_embedders.httpx.Client")diff --git a/tests/unit/test_extra_headers_vlm.py b/tests/unit/test_extra_headers_vlm.pyindex d8c86108e..62b2ede2f 100644--- a/tests/unit/test_extra_headers_vlm.py+++ b/tests/unit/test_extra_headers_vlm.py@@ -14,6 +14,7 @@ from openai import AsyncOpenAI, OpenAIfrom volcenginesdkarkruntime import Ark, AsyncArkfrom volcenginesdkarkruntime._exceptions import ArkAPIStatusError+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.models.vlm.backends.litellm_vlm import (LiteLLMVLMProvider,detect_provider_by_model,@@ -349,17 +350,21 @@ class TestOpenAIVLMClientRetries:assert call_kwargs["max_retries"] == 0@pytest.mark.parametrize(- ("status_code", "message"),+ ("status_code", "message", "expected_kind"),[- (400, "Total tokens of multi-modal content and text exceed max message tokens."),- (400, "The parameter model is invalid."),- (401, "Unauthorized"),- (503, "Service Unavailable"),+ (+ 400,+ "Total tokens of multi-modal content and text exceed max message tokens.",+ ModelErrorKind.INVALID_REQUEST,+ ),+ (400, "The parameter model is invalid.", ModelErrorKind.INVALID_REQUEST),+ (401, "Unauthorized", ModelErrorKind.AUTHENTICATION_ERROR),+ (503, "Service Unavailable", ModelErrorKind.TRANSIENT_SERVICE_ERROR),],)@patch("volcenginesdkarkruntime.AsyncArk")def test_volcengine_async_client_performs_single_request(- self, mock_async_ark_class, status_code, message+ self, mock_async_ark_class, status_code, message, expected_kind):error = ArkAPIStatusError(f"Error code: {status_code} - {message}",@@ -382,10 +387,11 @@ class TestOpenAIVLMClientRetries:})- with pytest.raises(ArkAPIStatusError) as raised:+ with pytest.raises(ModelError) as raised:asyncio.run(vlm.get_completion_async("hello"))- assert raised.value is error+ assert raised.value.kind == expected_kind+ assert raised.value.__cause__ is errorassert mock_client.chat.completions.create.await_count == 1call_kwargs = mock_async_ark_class.call_args.kwargsassert call_kwargs["timeout"] == 12.0diff --git a/tests/unit/test_failover_embedder_permanent_advance.py b/tests/unit/test_failover_embedder_permanent_advance.pyindex c423d1c20..664862ad2 100644--- a/tests/unit/test_failover_embedder_permanent_advance.py+++ b/tests/unit/test_failover_embedder_permanent_advance.py@@ -17,6 +17,7 @@ import timeimport pytestfrom openviking.models.embedder.base import EmbedderBase, EmbedResult, FailoverEmbedder+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.utils.exceptions import AllCredentialsFailedError@@ -47,16 +48,22 @@ class _StubEmbedder(EmbedderBase):def _make_400_error() -> Exception:- """Construct a request-level 400 parameter error (PERMANENT, fail-fast)."""- return RuntimeError(+ """Construct a request-level invalid request error."""+ return ModelError(+ ModelErrorKind.INVALID_REQUEST,"Error code: 400 - {'error': {'message': "- "'The parameter `model` specified in the request are not valid'}}"+ "'The parameter `model` specified in the request are not valid'}}",+ status_code=400,)def _make_401_error() -> Exception:- """Construct a credential-level 401 auth error (AUTH, advances in multi-credential)."""- return RuntimeError("Error code: 401 - {'error': {'message': 'Incorrect API key provided'}}")+ """Construct a credential-level auth error."""+ return ModelError(+ ModelErrorKind.AUTHENTICATION_ERROR,+ "Error code: 401 - {'error': {'message': 'Incorrect API key provided'}}",+ status_code=401,+ )def test_auth_error_on_primary_advances_to_backup():@@ -106,7 +113,7 @@ def test_permanent_400_fails_fast_without_trying_backup():credential_ids=["primary", "backup"],)- with pytest.raises(RuntimeError, match="400"):+ with pytest.raises(ModelError, match="400"):fe.embed("hello")assert primary.calls == 1@@ -202,12 +209,18 @@ def test_fast_failover_commits_new_active_index():def test_content_safety_does_not_cycle_the_ring():"""Request-level content-safety errors fail fast and must not try others."""- cs = _StubEmbedder("cs", error=RuntimeError("content_filter: sensitive content detected"))+ cs = _StubEmbedder(+ "cs",+ error=ModelError(+ ModelErrorKind.CONTENT_SAFETY,+ "content_filter: sensitive content detected",+ ),+ )backup = _StubEmbedder("backup")fe = FailoverEmbedder(embedders=[cs, backup], credential_ids=["cs", "backup"])- with pytest.raises(RuntimeError, match="content_filter"):+ with pytest.raises(ModelError, match="content_filter"):fe.embed("hello")assert cs.calls == 1diff --git a/tests/unit/test_gemini_embedder.py b/tests/unit/test_gemini_embedder.pyindex ff33eefb9..e5a5154fe 100644--- a/tests/unit/test_gemini_embedder.py+++ b/tests/unit/test_gemini_embedder.py@@ -9,6 +9,8 @@ from unittest.mock import MagicMock, patchimport pytest+from openviking.models.model_error import ModelError, ModelErrorKind+def _make_mock_embedding(values):emb = MagicMock()@@ -130,7 +132,7 @@ class TestGeminiDenseEmbedderEmbed:assert kwargs["config"].task_type == "RETRIEVAL_QUERY"@patch("openviking.models.embedder.gemini_embedders.genai.Client")- def test_embed_raises_runtime_error_on_api_error(self, mock_client_class):+ def test_embed_raises_model_error_on_api_error(self, mock_client_class):from google.genai.errors import APIErrorfrom openviking.models.embedder.gemini_embedders import GeminiDenseEmbedder@@ -140,7 +142,7 @@ class TestGeminiDenseEmbedderEmbed:mock_response.status_code = 401mock_client.models.embed_content.side_effect = APIError(401, {}, response=mock_response)embedder = GeminiDenseEmbedder("gemini-embedding-2-preview", api_key="key")- with pytest.raises(RuntimeError, match="Gemini embedding failed"):+ with pytest.raises(ModelError, match="Gemini embedding failed"):embedder.embed("hello")@patch("openviking.models.embedder.gemini_embedders.genai.Client")@@ -224,29 +226,33 @@ class TestGeminiValidation:class TestGeminiErrorMessages:@pytest.mark.parametrize(- "code,match",+ "reason,expected_kind",[- (401, "Invalid API key"),- (403, "Permission denied"),- (404, "Model not found"),- (429, "Resource exhausted"),- (500, "service error"),+ ("QUOTA_EXCEEDED", ModelErrorKind.QUOTA_EXHAUSTED),+ ("RATE_LIMIT_EXCEEDED", ModelErrorKind.RATE_LIMIT_ERROR),+ (None, ModelErrorKind.UNKNOWN),],)@patch("openviking.models.embedder.gemini_embedders.genai.Client")- def test_error_code_hint(self, mock_client, code, match):- from google.genai.errors import APIError+ def test_resource_exhausted_uses_structured_reason(self, mock_client, reason, expected_kind):+ from google.genai.errors import ClientErrorfrom openviking.models.embedder.gemini_embedders import GeminiDenseEmbeddermock = mock_client.return_value- mock_resp = MagicMock()- mock_resp.status_code = code- mock.models.embed_content.side_effect = APIError(code, {}, response=mock_resp)+ error = ClientError(+ 429,+ {"error": {"details": [] if reason is None else [{"reason": reason}]}},+ )+ mock.models.embed_content.side_effect = errorembedder = GeminiDenseEmbedder("gemini-embedding-2-preview", api_key="k")- with pytest.raises(RuntimeError, match=match):+ with pytest.raises(ModelError, match="Resource exhausted") as raised:embedder.embed("hello")+ assert raised.value.kind is expected_kind+ assert raised.value.status_code == 429+ assert raised.value.__cause__.__cause__ is error+@patch("openviking.models.embedder.gemini_embedders.genai.Client")def test_error_message_includes_http_code(self, mock_client):from google.genai.errors import APIError@@ -258,7 +264,7 @@ class TestGeminiErrorMessages:mock_resp.status_code = 404mock.models.embed_content.side_effect = APIError(404, {}, response=mock_resp)embedder = GeminiDenseEmbedder("gemini-embedding-2-preview", api_key="k")- with pytest.raises(RuntimeError, match="HTTP 404"):+ with pytest.raises(ModelError, match="HTTP 404"):embedder.embed("hello")diff --git a/tests/unit/test_jina_embedder.py b/tests/unit/test_jina_embedder.pyindex 0f2266482..c7ebe64d7 100644--- a/tests/unit/test_jina_embedder.py+++ b/tests/unit/test_jina_embedder.py@@ -10,6 +10,7 @@ from openviking.models.embedder import JinaDenseEmbedderfrom openviking.models.embedder.jina_embedders import (JINA_MODEL_DIMENSIONS,)+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking_cli.utils.config.embedding_config import EmbeddingModelConfig@@ -229,20 +230,25 @@ class TestJinaDenseEmbedder:mock_client = MagicMock()mock_openai_class.return_value = mock_client- mock_client.embeddings.create.side_effect = openai.APIError(+ error = openai.APIError(message="Test API error",request=MagicMock(),body=None,)+ mock_client.embeddings.create.side_effect = errorembedder = JinaDenseEmbedder(model_name="jina-embeddings-v5-text-small",api_key="test-api-key",)- with pytest.raises(RuntimeError, match="Jina API error"):+ with pytest.raises(ModelError, match="Jina API error") as raised:embedder.embed("Hello world")+ assert raised.value.kind is ModelErrorKind.UNKNOWN+ assert raised.value.provider == "jina"+ assert raised.value.__cause__.__cause__ is error+def test_build_extra_body_none(self):"""Test _build_extra_body returns None when no params set"""embedder = JinaDenseEmbedder(@@ -290,7 +296,7 @@ class TestJinaDenseEmbedder:@patch("openviking.models.embedder.jina_embedders.openai.OpenAI")def test_422_task_error_actionable_message(self, mock_openai_class):- """422 error mentioning 'task' should produce actionable RuntimeError."""+ """422 error mentioning 'task' should preserve the actionable provider message."""import openaimock_client = MagicMock()@@ -308,30 +314,7 @@ class TestJinaDenseEmbedder:api_key="test-key",)- with pytest.raises(RuntimeError, match="query_param.*document_param"):- embedder.embed("hello")-- @patch("openviking.models.embedder.jina_embedders.openai.OpenAI")- def test_non_422_error_passthrough(self, mock_openai_class):- """Non-422 API errors should use the generic 'Jina API error' message."""- import openai-- mock_client = MagicMock()- mock_openai_class.return_value = mock_client-- error = openai.APIError(- message="Internal server error",- request=MagicMock(),- body=None,- )- mock_client.embeddings.create.side_effect = error-- embedder = JinaDenseEmbedder(- model_name="jina-embeddings-v5-text-small",- api_key="test-key",- )-- with pytest.raises(RuntimeError, match="Jina API error"):+ with pytest.raises(ModelError, match="query_param.*document_param"):embedder.embed("hello")def test_code_model_dimensions(self):diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex cedf5a2d8..35945c47c 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -2,8 +2,11 @@# SPDX-License-Identifier: Apache-2.0"""Tests for shared model retry helpers."""+import httpx+import openaiimport pytest+from openviking.models.model_error import ModelError, ModelErrorKind, model_error_boundaryfrom openviking.pyagfs.exceptions import (AGFSClientError,AGFSConnectionError,@@ -96,40 +99,60 @@ def test_extract_metric_error_code_uses_safe_fallbacks_only():assert extract_metric_error_code(RuntimeError("request_id=not-a-metric-label")) == "unknown"-def test_classify_all_credentials_failed_prefers_transient_over_auth():+def test_all_credentials_failed_prefers_transient_over_auth():err = AllCredentialsFailedError([- ("cred-a", ERROR_CLASS_AUTH, RuntimeError("401 Unauthorized"), 0),- ("cred-b", ERROR_CLASS_TRANSIENT, RuntimeError("500 server error"), 1),+ (+ "cred-a",+ ModelErrorKind.AUTHENTICATION_ERROR,+ ModelError(ModelErrorKind.AUTHENTICATION_ERROR, "unauthorized"),+ 0,+ ),+ (+ "cred-b",+ ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ ModelError(ModelErrorKind.TRANSIENT_SERVICE_ERROR, "server error"),+ 1,+ ),])- assert classify_api_error(err) == ERROR_CLASS_TRANSIENT+ assert err.kind is ModelErrorKind.TRANSIENT_SERVICE_ERROR-def test_retry_sync_retries_transient_error_until_success():+def test_retry_sync_retries_transient_error_until_success(monkeypatch):attempts = {"count": 0}+ monkeypatch.setattr("openviking.utils.model_retry.time.sleep", lambda _delay: None)+ sdk_error = openai.InternalServerError(+ "Internal server error",+ response=httpx.Response(500, request=httpx.Request("POST", "https://example.test")),+ body={"error": {"code": 500, "type": "server_error", "message": "Internal server error"}},+ )+ @model_error_boundarydef _call():attempts["count"] += 1if attempts["count"] < 3:- raise RuntimeError("429 TooManyRequests")+ raise sdk_errorreturn "ok"- assert retry_sync(_call, max_retries=3) == "ok"+ assert retry_sync(_call, max_retries=3, model_errors_only=True) == "ok"assert attempts["count"] == 3@pytest.mark.asyncio-async def test_retry_async_does_not_retry_unknown_error():+async def test_retry_async_preserves_local_error_without_parsing_message():attempts = {"count": 0}+ error = RuntimeError("HTTP 503 is input data, not an API failure")+ @model_error_boundaryasync def _call():attempts["count"] += 1- raise RuntimeError("some unexpected validation failure")+ raise error- with pytest.raises(RuntimeError):- await retry_async(_call, max_retries=3)+ with pytest.raises(RuntimeError) as raised:+ await retry_async(_call, max_retries=3, model_errors_only=True)+ assert raised.value is errorassert attempts["count"] == 1@@ -223,16 +246,24 @@ def test_content_safety_takes_precedence_over_400():def test_retry_sync_does_not_retry_quota_exceeded():- """Quota-exceeded errors should NOT be retried."""+ """Structured quota exhaustion takes precedence over the retryable HTTP 429."""attempts = {"count": 0}+ sdk_error = openai.RateLimitError(+ "rate limited",+ response=httpx.Response(429, request=httpx.Request("POST", "https://example.test")),+ body={"error": {"code": "insufficient_quota"}},+ )+ @model_error_boundarydef _call():attempts["count"] += 1- raise RuntimeError("AccountQuotaExceeded")+ raise sdk_error- with pytest.raises(RuntimeError, match="AccountQuotaExceeded"):- retry_sync(_call, max_retries=5)+ with pytest.raises(ModelError) as raised:+ retry_sync(_call, max_retries=5, model_errors_only=True)+ assert raised.value.kind is ModelErrorKind.QUOTA_EXHAUSTED+ assert raised.value.__cause__ is sdk_errorassert attempts["count"] == 1@@ -305,30 +336,44 @@ def test_20015_inside_request_id_is_not_input_too_large():@pytest.mark.parametrize(- "message",+ "body",[- "expected maxLength: 50000, actual: 75000",- "Error code: 400 - InvalidParameter: Total tokens of multi-modal content "- "and text exceed max message tokens.",+ {"code": "context_length_exceeded", "message": "Maximum context length exceeded"},+ {+ "code": 500,+ "type": "server_error",+ "message": (+ "input (8525 tokens) is too large to process. increase the physical batch size "+ "(current batch size: 2048)"+ ),+ },],)-def test_input_too_large_does_not_retry_or_trip_breaker(message):+def test_input_too_large_does_not_retry_or_trip_breaker(body, monkeypatch):from openviking.utils.circuit_breaker import CircuitBreakerbreaker = CircuitBreaker()attempts = {"count": 0}+ monkeypatch.setattr("openviking.utils.model_retry.time.sleep", lambda _delay: None)+ sdk_error = openai.InternalServerError(+ "Input rejected",+ response=httpx.Response(500, request=httpx.Request("POST", "https://example.test")),+ body={"error": body},+ )+ @model_error_boundarydef _call():attempts["count"] += 1- error = RuntimeError(message)- breaker.record_failure(error)- raise error+ raise sdk_error- with pytest.raises(RuntimeError) as raised:- retry_sync(_call, max_retries=5)+ # Repeated bad inputs must not block a subsequent valid request.+ for _ in range(5):+ with pytest.raises(ModelError) as raised:+ retry_sync(_call, max_retries=3, model_errors_only=True)+ assert raised.value.kind is ModelErrorKind.INPUT_TOO_LARGE+ breaker.record_failure(raised.value)- assert str(raised.value) == message- assert attempts["count"] == 1+ assert attempts["count"] == 5breaker.check()diff --git a/tests/unit/test_ordered_credential_switcher.py b/tests/unit/test_ordered_credential_switcher.pyindex 0e05dae94..05c55618d 100644--- a/tests/unit/test_ordered_credential_switcher.py+++ b/tests/unit/test_ordered_credential_switcher.py@@ -5,16 +5,8 @@import pytest-from openviking.utils.model_retry import (- ERROR_CLASS_AUTH,- ERROR_CLASS_CONTENT_SAFETY,- ERROR_CLASS_INPUT_TOO_LARGE,- ERROR_CLASS_PERMANENT,- ERROR_CLASS_QUOTA_EXCEEDED,- ERROR_CLASS_TRANSIENT,- ERROR_CLASS_UNKNOWN,- OrderedCredentialSwitcher,-)+from openviking.models.model_error import ModelErrorKind+from openviking.utils.model_retry import OrderedCredentialSwitcherclass TestOrderedCredentialSwitcher:@@ -41,14 +33,18 @@ class TestOrderedCredentialSwitcher:OrderedCredentialSwitcher(n=-1)def test_is_fail_fast_classification(self):- """Request-level errors are fail-fast; credential/quota/transient are not."""- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_PERMANENT) is True- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_INPUT_TOO_LARGE) is True- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_CONTENT_SAFETY) is True- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_AUTH) is False- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_QUOTA_EXCEEDED) is False- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_TRANSIENT) is False- assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_UNKNOWN) is False+ """Request/unknown errors fail fast; model-routing failures may advance."""+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.INVALID_REQUEST) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.INPUT_TOO_LARGE) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.CONTENT_SAFETY) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.UNKNOWN) is True+ assert OrderedCredentialSwitcher.is_fail_fast(None) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.AUTHENTICATION_ERROR) is False+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.QUOTA_EXHAUSTED) is False+ assert (+ OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.TRANSIENT_SERVICE_ERROR) is False+ )+ assert OrderedCredentialSwitcher.is_fail_fast(ModelErrorKind.MODEL_NOT_FOUND) is Falsedef test_commit_success_different_index_fast_failover(self):"""commit_success on a different index commits it as the new active one."""diff --git a/tests/unit/test_retry_embedder.py b/tests/unit/test_retry_embedder.pyindex 6ce38860b..9851e7138 100644--- a/tests/unit/test_retry_embedder.py+++ b/tests/unit/test_retry_embedder.py@@ -15,7 +15,9 @@ from openviking.models.embedder.base import (SparseEmbedderBase,)from openviking.models.embedder.retry_embedder import RetryEmbedder+from openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.models.vlm.token_usage import TokenUsageTracker+from openviking.utils.model_retry import model_retry_scopefrom openviking_cli.utils.config.embedding_config import (EmbeddingConfig,EmbeddingCredential,@@ -23,10 +25,14 @@ from openviking_cli.utils.config.embedding_config import ()-class ProviderError(RuntimeError):+class ProviderError(ModelError):def __init__(self, status_code: int):- super().__init__(f"HTTP {status_code}")- self.status_code = status_code+ kinds = {+ 400: ModelErrorKind.INVALID_REQUEST,+ 401: ModelErrorKind.AUTHENTICATION_ERROR,+ 503: ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ }+ super().__init__(kinds[status_code], f"HTTP {status_code}", status_code=status_code)class FakeDenseEmbedder(DenseEmbedderBase):@@ -124,15 +130,20 @@ def test_retry_embedder_does_not_retry_permanent_or_auth_errors(status_code):assert inner.calls == 1-def test_retry_embedder_uses_max_retries_as_additional_attempts():- errors = [ProviderError(503) for _ in range(3)]- inner = FakeDenseEmbedder(errors, max_retries=2)+def test_unknown_api_error_exhausts_retry_budget_without_credential_failover():+ errors = [ModelError(ModelErrorKind.UNKNOWN, "unclassified API error") for _ in range(3)]+ primary = FakeDenseEmbedder(errors, max_retries=2)+ backup = FakeDenseEmbedder([[0.3, 0.4]], max_retries=2)+ failover = FailoverEmbedder(+ [RetryEmbedder(primary), RetryEmbedder(backup)], ["primary", "backup"]+ )- with pytest.raises(ProviderError) as raised:- RetryEmbedder(inner).embed("query")+ with pytest.raises(ModelError) as raised:+ failover.embed("query")assert raised.value is errors[-1]- assert inner.calls == 3+ assert primary.calls == 3+ assert backup.calls == 0def test_retry_embedder_sparse_dimension_remains_unsupported():@@ -144,8 +155,10 @@ def test_retry_embedder_sparse_dimension_remains_unsupported():@pytest.mark.asyncio-async def test_retry_releases_semaphore_before_backoff(monkeypatch):- inner = FakeDenseEmbedder([ProviderError(503), [0.1, 0.2]], max_retries=1)+async def test_scoped_failure_restores_retries_and_releases_semaphore(monkeypatch):+ error = ProviderError(503)+ inner = FakeDenseEmbedder([error, ProviderError(503), [0.1, 0.2]], max_retries=1)+ embedder = RetryEmbedder(inner)semaphore = AsyncSemaphore(1)inner._account_semaphore = semaphoreobserved_values = []@@ -155,10 +168,18 @@ async def test_retry_releases_semaphore_before_backoff(monkeypatch):monkeypatch.setattr("openviking.utils.model_retry.asyncio.sleep", observe_backoff)- result = await RetryEmbedder(inner).embed_async("query")+ async with model_retry_scope(0):+ with pytest.raises(ProviderError) as raised:+ await embedder.embed_async("query")+ assert raised.value is error+ assert inner.calls == 1+ assert observed_values == []+ assert semaphore._value == 1++ result = await embedder.embed_async("query")assert result.dense_vector == [0.1, 0.2]- assert inner.calls == 2+ assert inner.calls == 3assert observed_values == [1]assert semaphore._value == 1diff --git a/tests/unit/test_retry_vlm.py b/tests/unit/test_retry_vlm.pyindex c14a21e02..5b43d8149 100644--- a/tests/unit/test_retry_vlm.py+++ b/tests/unit/test_retry_vlm.py@@ -4,22 +4,30 @@import asynciofrom pathlib import Path+import httpx+import openaiimport pytest-from openviking.models.vlm.base import MultiCredentialVLM, VLMBase, VLMFactory+from openviking.models.model_error import ModelError, ModelErrorKind+from openviking.models.vlm.base import VLMBasefrom openviking.models.vlm.retry_vlm import RetryVLM+from openviking.utils.model_retry import model_retry_scopefrom openviking_cli.utils.config.vlm_config import VLMConfig-class ProviderError(RuntimeError):+class ProviderError(ModelError):def __init__(self, status_code: int):messages = {400: "400 invalid parameter",401: "401 unauthorized",503: "503 service unavailable",}- super().__init__(messages[status_code])- self.status_code = status_code+ kinds = {+ 400: ModelErrorKind.INVALID_REQUEST,+ 401: ModelErrorKind.AUTHENTICATION_ERROR,+ 503: ModelErrorKind.TRANSIENT_SERVICE_ERROR,+ }+ super().__init__(kinds[status_code], messages[status_code], status_code=status_code)class FakeVLM(VLMBase):@@ -97,14 +105,21 @@ def test_retry_vlm_retries_sync_transient_failures(method, expected_method):("get_vision_completion_async", "async_vision"),],)-async def test_retry_vlm_retries_async_transient_failures(method, expected_method):- inner = FakeVLM([ProviderError(503), "ok"])+async def test_retry_vlm_scoped_failure_restores_async_retries(method, expected_method):+ error = ProviderError(503)+ inner = FakeVLM([error, ProviderError(503), "ok"])vlm = RetryVLM(inner)+ async with model_retry_scope(0):+ with pytest.raises(ProviderError) as raised:+ await getattr(vlm, method)(prompt="hello")+ assert raised.value is error+ assert len(inner.calls) == 1+result = await getattr(vlm, method)(prompt="hello")assert result == "ok"- assert [name for name, _kwargs in inner.calls] == [expected_method, expected_method]+ assert [name for name, _kwargs in inner.calls] == [expected_method] * 3@pytest.mark.parametrize("status_code", [400, 401])@@ -195,38 +210,60 @@ def test_retry_vlm_delegates_token_tracker_and_close():assert inner.closed is True-def test_vlm_factory_wraps_provider_with_standard_retry():- config = {"provider": "kimi", "api_key": "key", "model": "kimi-code"}-- standard = VLMFactory.create(config)-- assert isinstance(standard, RetryVLM)- assert standard.inner.__class__.__name__ == "KimiVLM"-+def test_vlm_config_switches_model_after_primary_not_found():+ calls = []++ def handler(request):+ calls.append(request.url.host)+ if request.url.host == "primary.example.test":+ return httpx.Response(+ 404,+ json={"error": {"code": "model_not_found", "message": "Model does not exist"}},+ )+ assert request.url.host == "backup.example.test"+ return httpx.Response(+ 200,+ json={+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "backup response"},+ "finish_reason": "stop",+ }+ ],+ },+ )-def test_vlm_config_wraps_each_credential_inside_failover():config = VLMConfig(- model="kimi-code",+ model="missing-model",+ provider="openai",+ max_retries=2,credentials=[{"id": "primary",- "provider": "kimi","api_key": "primary-key",+ "api_base": "https://primary.example.test/v1",},{"id": "backup",- "provider": "glm","api_key": "backup-key",- "model": "glm-4.6v",+ "api_base": "https://backup.example.test/v1",+ "model": "backup-model",},],)-instance = config.get_vlm_instance()+ for credential in instance._vlm_instances:+ credential.inner._sync_client = openai.OpenAI(+ api_key="test-key",+ base_url=credential.inner.api_base,+ max_retries=0,+ http_client=httpx.Client(transport=httpx.MockTransport(handler)),+ )- assert isinstance(instance, MultiCredentialVLM)- assert all(isinstance(child, RetryVLM) for child in instance._vlm_instances)- assert [child.inner.__class__.__name__ for child in instance._vlm_instances] == [- "KimiVLM",- "GLMVLM",- ]+ try:+ assert instance.get_completion(prompt="hello") == "backup response"+ assert calls == ["primary.example.test", "backup.example.test"]+ assert instance.active_credential_id == "backup"+ finally:+ instance.close()diff --git a/tests/unit/test_vikingbot_vlm_adapter_retry.py b/tests/unit/test_vikingbot_vlm_adapter_retry.pyindex 4bf3c9790..b05a39932 100644--- a/tests/unit/test_vikingbot_vlm_adapter_retry.py+++ b/tests/unit/test_vikingbot_vlm_adapter_retry.py@@ -1,8 +1,11 @@+from collections import Counterfrom types import SimpleNamespace+from unittest.mock import AsyncMockimport httpximport pytestimport vikingbot.providers.vlm_adapter as vlm_adapter+from vikingbot.compile.pipeline_io import JsonModel, ModelCallErrorfrom vikingbot.providers.vlm_adapter import VLMProviderAdapterfrom volcenginesdkarkruntime._exceptions import (ArkAPITimeoutError,@@ -10,6 +13,7 @@ from volcenginesdkarkruntime._exceptions import (ArkRateLimitError,)+from openviking.models.model_error import ModelError, ModelErrorKind, model_error_kindfrom openviking.models.vlm.backends.openai_vlm import OpenAIVLMfrom openviking.models.vlm.backends.volcengine_vlm import (VOLCENGINE_CLIENT_REQUEST_ID,@@ -121,19 +125,28 @@ async def test_chat_retries_rate_limit_until_success(monkeypatch):@pytest.mark.asyncio-async def test_chat_does_not_retry_errors_without_rate_limit_markers(monkeypatch):+async def test_chat_preserves_input_error_for_compile_without_retry(monkeypatch):async def _sleep(_delay: float):raise AssertionError("non-retryable errors must not sleep/retry")monkeypatch.setattr(vlm_adapter.asyncio, "sleep", _sleep)- fake_vlm = _FakeVLM([RuntimeError("AuthenticationError Unauthorized")])+ fake_vlm = _FakeVLM([ModelError(ModelErrorKind.INPUT_TOO_LARGE, "Input rejected")])adapter = VLMProviderAdapter(fake_vlm, "test-model", langfuse_client=_DisabledLangfuse())+ model = JsonModel(+ adapter,+ "test-model",+ 0.0,+ SimpleNamespace(put=AsyncMock()),+ SimpleNamespace(merge_input_chars=16_000, pipeline_call_seconds=5),+ {},+ Counter(),+ )- response = await adapter.chat(messages=[{"role": "user", "content": "hello"}])+ with pytest.raises(ModelCallError) as raised:+ await model.call("reduce", [{"role": "user", "content": "hello"}], [], max_tokens=128)- assert response.finish_reason == "error"- assert "AuthenticationError" in response.content+ assert model_error_kind(raised.value) is ModelErrorKind.INPUT_TOO_LARGEassert fake_vlm.calls == 1diff --git a/tests/unit/test_voyage_embedder.py b/tests/unit/test_voyage_embedder.pyindex fffcbad89..72b68586d 100644--- a/tests/unit/test_voyage_embedder.py+++ b/tests/unit/test_voyage_embedder.py@@ -8,6 +8,7 @@ import pytestfrom openviking.models.embedder import VoyageDenseEmbedderfrom openviking.models.embedder.voyage_embedders import VOYAGE_MODEL_DIMENSIONS+from openviking.models.model_error import ModelErrorclass TestVoyageDenseEmbedder:@@ -114,5 +115,5 @@ class TestVoyageDenseEmbedder:api_key="voyage-key",)- with pytest.raises(RuntimeError, match="Voyage API error"):+ with pytest.raises(ModelError, match="Voyage API error"):embedder.embed("Hello world")