Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: implement multi-credential priority call design (#2468)
dingbenbe77b2dac1dd
diff --git a/openviking/models/embedder/__init__.py b/openviking/models/embedder/__init__.pyindex 8aae64390..229c35b94 100644--- a/openviking/models/embedder/__init__.py+++ b/openviking/models/embedder/__init__.py@@ -19,10 +19,12 @@ Supported providers:"""from openviking.models.embedder.base import (+ AllCredentialsFailedError,CompositeHybridEmbedder,DenseEmbedderBase,EmbedderBase,EmbedResult,+ FailoverEmbedder,HybridEmbedderBase,SparseEmbedderBase,)@@ -66,6 +68,8 @@ __all__ = ["SparseEmbedderBase","HybridEmbedderBase","CompositeHybridEmbedder",+ "FailoverEmbedder",+ "AllCredentialsFailedError",# Google Gemini implementations"GeminiDenseEmbedder",# Jina AI implementationsdiff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex 7fa429e89..61efef88f 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -15,7 +15,13 @@ from openviking.utils.embedding_input import (resolve_embedding_max_input_tokens,truncate_embedding_input,)-from openviking.utils.model_retry import retry_async, retry_sync+from openviking.utils.exceptions import AllCredentialsFailedError+from openviking.utils.model_retry import (+ OrderedCredentialSwitcher,+ classify_api_error,+ retry_async,+ retry_sync,+)from openviking_cli.utils import get_loggerT = TypeVar("T")@@ -202,9 +208,7 @@ class EmbedderBase(ABC):return contentreturn truncate_embedding_input(content, self.max_input_tokens)- def prepare_embedding_inputs(- self, contents: List["EmbeddingInput"]- ) -> List["EmbeddingInput"]:+ def prepare_embedding_inputs(self, contents: List["EmbeddingInput"]) -> List["EmbeddingInput"]:"""Apply this embedder's input guard to a batch."""return [self.prepare_embedding_input(content) for content in contents]@@ -261,9 +265,7 @@ class EmbedderBase(ABC):"""Batch embed document texts."""return self.embed_batch(texts, is_query=False)- async def embed_async(- self, content: "EmbeddingInput", is_query: bool = False- ) -> EmbedResult:+ async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Async embed text or multimodal content.Subclasses should override this with a non-blocking implementation.@@ -586,10 +588,7 @@ class CompositeHybridEmbedder(HybridEmbedderBase):@propertydef supports_multimodal(self) -> bool:"""Supports multimodal input only if both sub-embedders do."""- return (- self.dense_embedder.supports_multimodal- and self.sparse_embedder.supports_multimodal- )+ return self.dense_embedder.supports_multimodal and self.sparse_embedder.supports_multimodaldef embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Combine results from both embedders"""@@ -616,9 +615,7 @@ class CompositeHybridEmbedder(HybridEmbedderBase):for d, s in zip(dense_results, sparse_results, strict=True)]- async def embed_async(- self, content: "EmbeddingInput", is_query: bool = False- ) -> EmbedResult:+ async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:dense_input = self.dense_embedder.prepare_embedding_input(content)sparse_input = self.sparse_embedder.prepare_embedding_input(content)dense_res, sparse_res = await asyncio.gather(@@ -714,3 +711,246 @@ def exponential_backoff_retry()time.sleep(delay)+++class FailoverEmbedder(EmbedderBase):+ """Embedder wrapper that provides failover across multiple ordered credentials.++ When a credential fails with a credential-level (auth) or quota/transient+ error, this wrapper advances to the next credential in the list. After+ failback thresholds are met, it attempts to move back to a higher-priority+ credential.++ Credentials are tried in order (index 0 is highest priority).+ """++ def __init__(+ self,+ embedders: List[EmbedderBase],+ credential_ids: List[str],+ failback_timeout_seconds: float = 600.0,+ failback_request_count: int = 50,+ ):+ """Initialize FailoverEmbedder with multiple embedder instances.++ Args:+ embedders: List of embedder instances in priority order (0 is highest)+ credential_ids: List of credential IDs corresponding to the embedder instances+ failback_timeout_seconds: Time after which to attempt failback+ failback_request_count: Number of requests after which to attempt failback++ Note:+ Request-level errors (e.g. HTTP 400 parameter error, input too+ large, content safety) fail fast and re-raise the original+ exception, since the same request fails on every credential of the+ same model. Credential-level auth errors (401/403) advance to the+ next credential; only the last credential fails fast. Per-credential+ retry counts are handled by each underlying embedder; there is no+ global retry cap.+ """+ if not embedders:+ raise ValueError("At least one embedder instance is required")+ if len(embedders) != len(credential_ids):+ raise ValueError("embedders and credential_ids must have the same length")++ # Use the first embedder's config as base+ first = embedders[0]+ super().__init__(+ model_name=first.model_name,+ config=first.config,+ )++ self._embedders = embedders+ self._credential_ids = credential_ids+ self._switcher = OrderedCredentialSwitcher(+ n=len(embedders),+ failback_timeout_seconds=failback_timeout_seconds,+ failback_request_count=failback_request_count,+ )++ def _embed_with_failover(self, method_name: str, *args, **kwargs) -> EmbedResult:+ """Execute an embedder method with multi-credential failover support.++ Args:+ method_name: Name of the method to call on embedder instances+ *args: Positional arguments to pass to the method+ **kwargs: Keyword arguments to pass to the method++ Returns:+ The result from the embedder method++ Raises:+ AllCredentialsFailedError if all credentials fail+ """+ aggregated_errors = []++ # Start from the current (possibly failed-back) active credential, then+ # cycle through the whole ring once so an unavailable active credential+ # does not block the request. A credential that succeeds this cycle+ # becomes the new active one (fast failover); slow sticky failback to+ # higher priority is still handled by maybe_failback() across requests.+ start = self._switcher.maybe_failback()+ n = self._switcher.n++ for offset in range(n):+ idx = (start + offset) % n+ credential_id = self._credential_ids[idx]+ embedder = self._embedders[idx]++ try:+ method = getattr(embedder, method_name)+ result = method(*args, **kwargs)+ self._switcher.commit_success(idx)+ return result+ except Exception as exc:+ error_class = classify_api_error(exc)+ aggregated_errors.append((credential_id, error_class, exc, idx))++ if self._switcher.is_fail_fast(error_class):+ # Request-level failure: re-raise the original exception;+ # trying other credentials is useless.+ raise++ logger.warning(+ f"Credential {credential_id} failed with {error_class}, trying next credential"+ )++ raise AllCredentialsFailedError(aggregated_errors)++ async def _embed_with_failover_async(self, method_name: str, *args, **kwargs) -> EmbedResult:+ """Execute an async embedder method with multi-credential failover support.++ Args:+ method_name: Name of the async method to call on embedder instances+ *args: Positional arguments to pass to the method+ **kwargs: Keyword arguments to pass to the method++ Returns:+ The result from the async embedder method++ Raises:+ AllCredentialsFailedError if all credentials fail+ """+ aggregated_errors = []++ # See the sync variant for the ring-traversal rationale.+ start = self._switcher.maybe_failback()+ n = self._switcher.n++ for offset in range(n):+ idx = (start + offset) % n+ credential_id = self._credential_ids[idx]+ embedder = self._embedders[idx]++ try:+ method = getattr(embedder, method_name)+ result = await method(*args, **kwargs)+ self._switcher.commit_success(idx)+ return result+ except Exception as exc:+ error_class = classify_api_error(exc)+ aggregated_errors.append((credential_id, error_class, exc, idx))++ if self._switcher.is_fail_fast(error_class):+ # Request-level failure: re-raise the original exception;+ # trying other credentials is useless.+ raise++ logger.warning(+ f"Credential {credential_id} failed with {error_class}, trying next credential"+ )++ raise AllCredentialsFailedError(aggregated_errors)++ @property+ def supports_multimodal(self) -> bool:+ """Whether all embedders support multimodal input."""+ return all(embedder.supports_multimodal for embedder in self._embedders)++ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Embed text or multimodal content with multi-credential failover support."""+ return self._embed_with_failover("embed", content, is_query=is_query)++ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:+ """Batch embed with multi-credential failover support."""+ return self._embed_with_failover("embed_batch", contents, is_query=is_query)++ async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Async embed with multi-credential failover support."""+ return await self._embed_with_failover_async("embed_async", content, is_query=is_query)++ async def embed_batch_async(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:+ """Async batch embed with multi-credential failover support."""+ return await self._embed_with_failover_async(+ "embed_batch_async", contents, is_query=is_query+ )++ def get_dimension(self) -> int:+ """Get the dense dimension from the first embedder.++ Raises:+ AttributeError: if the wrapped embedders are sparse; sparse+ embedders have no fixed dense dimension, so a dimension should+ never be requested from them.+ """+ first = self._embedders[0]+ if not hasattr(first, "get_dimension"):+ raise AttributeError(+ f"{type(first).__name__} has no get_dimension "+ "(sparse embedders have no fixed dimension)"+ )+ return first.get_dimension()++ @property+ def active_credential_index(self) -> int:+ """Get the index of the currently active credential."""+ return self._switcher.get_active_index()++ @property+ def active_credential_id(self) -> str:+ """Get the ID of the currently active credential."""+ idx = self._switcher.get_active_index()+ if idx < len(self._credential_ids):+ return self._credential_ids[idx]+ return "exhausted"++ @property+ def is_dense(self) -> bool:+ """Check if the first embedder is dense."""+ return self._embedders[0].is_dense++ @property+ def is_sparse(self) -> bool:+ """Check if the first embedder is sparse."""+ return self._embedders[0].is_sparse++ @property+ def is_hybrid(self) -> bool:+ """Check if the first embedder is hybrid."""+ return self._embedders[0].is_hybrid++ def close(self):+ """Close all embedder instances."""+ for embedder in self._embedders:+ embedder.close()++ def get_token_usage(self) -> Dict[str, Any]:+ """Get combined token usage across credential instances.++ Embedders share a process-wide singleton token tracker (see+ ``_get_token_tracker``), so every wrapped embedder reports the same+ usage. Return a single instance's usage directly; merging would+ double-count.+ """+ if not self._embedders:+ return {}+ return self._embedders[0].get_token_usage()++ def reset_token_usage(self) -> None:+ """Reset token usage for all credential instances."""+ for embedder in self._embedders:+ embedder.reset_token_usage()diff --git a/openviking/models/vlm/__init__.py b/openviking/models/vlm/__init__.pyindex ae04683cd..8d761ec5e 100644--- a/openviking/models/vlm/__init__.py+++ b/openviking/models/vlm/__init__.py@@ -8,13 +8,21 @@ from .backends.kimi_vlm import KimiVLMfrom .backends.litellm_vlm import LiteLLMVLMProviderfrom .backends.openai_vlm import OpenAIVLMfrom .backends.volcengine_vlm import VolcEngineVLM-from .base import VLMBase, VLMFactory, FailoverVLM+from .base import (+ AllCredentialsFailedError,+ FailoverVLM,+ MultiCredentialVLM,+ VLMBase,+ VLMFactory,+)from .registry import get_all_provider_names, is_valid_provider__all__ = ["VLMBase","VLMFactory","FailoverVLM",+ "MultiCredentialVLM",+ "AllCredentialsFailedError","OpenAIVLM","CodexVLM","KimiVLM",diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex c05510cf1..70117fe87 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -9,8 +9,11 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union+from openviking.utils.exceptions import AllCredentialsFailedErrorfrom openviking.utils.model_retry import (+ OrderedCredentialSwitcher,PrimaryBackupSwitcher,+ classify_api_error,)from openviking_cli.utils import get_logger@@ -256,6 +259,11 @@ class VLMBase(ABC):e,)+ @property+ def token_tracker(self):+ """Public accessor for this instance's token usage tracker."""+ return self._token_tracker+def get_token_usage(self) -> Dict[str, Any]:"""Get token usage@@ -561,7 +569,7 @@ class FailoverVLM(VLMBase):from openviking.models.vlm.token_usage import TokenUsageTrackermerged_tracker = TokenUsageTracker.merge(- self.primary._token_tracker, self.backup._token_tracker+ self.primary.token_tracker, self.backup.token_tracker)return merged_tracker.to_dict()@@ -569,3 +577,254 @@ class FailoverVLM(VLMBase):"""Reset token usage for both primary and backup instances."""self.primary.reset_token_usage()self.backup.reset_token_usage()+++class MultiCredentialVLM(VLMBase):+ """VLM wrapper that provides failover across multiple ordered credentials.++ When a credential fails with quota_exceeded or permanent errors, this wrapper+ automatically advances to the next credential in the list. After failback thresholds+ are met, it attempts to move back to a higher-priority credential.++ Credentials are tried in order (index 0 is highest priority).+ """++ def __init__(+ self,+ vlm_instances: List[VLMBase],+ credential_ids: List[str],+ failback_timeout_seconds: float = 600.0, # 10 minutes+ failback_request_count: int = 50,+ ):+ """Initialize MultiCredentialVLM with multiple VLM instances.++ Args:+ vlm_instances: List of VLM instances in priority order (0 is highest)+ credential_ids: List of credential IDs corresponding to the VLM instances+ failback_timeout_seconds: Time after which to attempt failback+ failback_request_count: Number of requests after which to attempt failback+ """+ if not vlm_instances:+ raise ValueError("At least one VLM instance is required")+ if len(vlm_instances) != len(credential_ids):+ raise ValueError("vlm_instances and credential_ids must have the same length")++ # Use the first instance's config as base+ first = vlm_instances[0]+ config = {+ "model": first.model,+ "provider": first.provider,+ }+ super().__init__(config)++ self._vlm_instances = vlm_instances+ self._credential_ids = credential_ids+ self._logger = logging.getLogger(__name__)+ self._switcher = OrderedCredentialSwitcher(+ n=len(vlm_instances),+ failback_timeout_seconds=failback_timeout_seconds,+ failback_request_count=failback_request_count,+ )++ def _get_completion_with_failover(self, method_name: str, *args, **kwargs):+ """Execute a VLM method with multi-credential failover support.++ Args:+ method_name: Name of the method to call on VLM instances+ *args: Positional arguments to pass to the method+ **kwargs: Keyword arguments to pass to the method++ Returns:+ The result from the VLM method++ Raises:+ AllCredentialsFailedError if all credentials fail+ """+ aggregated_errors = []++ # Start from the current (possibly failed-back) active credential, then+ # cycle through the whole ring once. This means an unavailable active+ # credential (e.g. the last one) does not block the request: a working+ # credential found this cycle serves it and becomes the new active one+ # (fast failover). The slow, sticky failback to higher priority is still+ # handled by maybe_failback() across requests.+ start = self._switcher.maybe_failback()+ n = self._switcher.n++ for offset in range(n):+ idx = (start + offset) % n+ credential_id = self._credential_ids[idx]+ vlm_instance = self._vlm_instances[idx]++ try:+ method = getattr(vlm_instance, method_name)+ result = method(*args, **kwargs)+ self._switcher.commit_success(idx)+ return result+ except Exception as exc:+ error_class = classify_api_error(exc)+ aggregated_errors.append((credential_id, error_class, exc, idx))++ if self._switcher.is_fail_fast(error_class):+ # Request-level failure (400 / input too large / content+ # safety): re-raise the original exception so callers can+ # react to its type; trying other credentials is useless.+ raise++ self._logger.warning(+ f"Credential {credential_id} failed with {error_class}, trying next credential"+ )++ raise AllCredentialsFailedError(aggregated_errors)++ async def _get_completion_with_failover_async(self, method_name: str, *args, **kwargs):+ """Execute an async VLM method with multi-credential failover support.++ Args:+ method_name: Name of the async method to call on VLM instances+ *args: Positional arguments to pass to the method+ **kwargs: Keyword arguments to pass to the method++ Returns:+ The result from the async VLM method++ Raises:+ AllCredentialsFailedError if all credentials fail+ """+ aggregated_errors = []++ # See the sync variant for the ring-traversal rationale.+ start = self._switcher.maybe_failback()+ n = self._switcher.n++ for offset in range(n):+ idx = (start + offset) % n+ credential_id = self._credential_ids[idx]+ vlm_instance = self._vlm_instances[idx]++ try:+ method = getattr(vlm_instance, method_name)+ result = await method(*args, **kwargs)+ self._switcher.commit_success(idx)+ return result+ except Exception as exc:+ error_class = classify_api_error(exc)+ aggregated_errors.append((credential_id, error_class, exc, idx))++ if self._switcher.is_fail_fast(error_class):+ # Request-level failure: re-raise the original exception;+ # trying other credentials is useless.+ raise++ self._logger.warning(+ f"Credential {credential_id} failed with {error_class}, trying next credential"+ )++ raise AllCredentialsFailedError(aggregated_errors)++ def get_completion(+ self,+ prompt: str = "",+ thinking: bool = False,+ tools: Optional[List[Dict[str, Any]]] = None,+ tool_choice: Optional[str] = None,+ messages: Optional[List[Dict[str, Any]]] = None,+ ) -> Union[str, VLMResponse]:+ """Get text completion with multi-credential failover support."""+ return self._get_completion_with_failover(+ "get_completion",+ prompt=prompt,+ thinking=thinking,+ tools=tools,+ tool_choice=tool_choice,+ messages=messages,+ )++ async def get_completion_async(+ self,+ prompt: str = "",+ thinking: bool = False,+ tools: Optional[List[Dict[str, Any]]] = None,+ tool_choice: Optional[str] = None,+ messages: Optional[List[Dict[str, Any]]] = None,+ ) -> Union[str, VLMResponse]:+ """Get text completion asynchronously with multi-credential failover support."""+ return await self._get_completion_with_failover_async(+ "get_completion_async",+ prompt=prompt,+ thinking=thinking,+ tools=tools,+ tool_choice=tool_choice,+ messages=messages,+ )++ def get_vision_completion(+ self,+ prompt: str = "",+ images: Optional[List[Union[str, Path, bytes]]] = None,+ thinking: bool = False,+ tools: Optional[List[Dict[str, Any]]] = None,+ tool_choice: Optional[str] = None,+ messages: Optional[List[Dict[str, Any]]] = None,+ ) -> Union[str, VLMResponse]:+ """Get vision completion with multi-credential failover support."""+ return self._get_completion_with_failover(+ "get_vision_completion",+ prompt=prompt,+ images=images,+ thinking=thinking,+ tools=tools,+ tool_choice=tool_choice,+ messages=messages,+ )++ async def get_vision_completion_async(+ self,+ prompt: str = "",+ images: Optional[List[Union[str, Path, bytes]]] = None,+ thinking: bool = False,+ tools: Optional[List[Dict[str, Any]]] = None,+ tool_choice: Optional[str] = None,+ messages: Optional[List[Dict[str, Any]]] = None,+ ) -> Union[str, VLMResponse]:+ """Get vision completion asynchronously with multi-credential failover support."""+ return await self._get_completion_with_failover_async(+ "get_vision_completion_async",+ prompt=prompt,+ images=images,+ thinking=thinking,+ tools=tools,+ tool_choice=tool_choice,+ messages=messages,+ )++ @property+ def active_credential_index(self) -> int:+ """Get the index of the currently active credential."""+ return self._switcher.get_active_index()++ @property+ def active_credential_id(self) -> str:+ """Get the ID of the currently active credential."""+ idx = self._switcher.get_active_index()+ if idx < len(self._credential_ids):+ return self._credential_ids[idx]+ return "exhausted"++ def get_token_usage(self) -> Dict[str, Any]:+ """Get combined token usage from all credential instances."""+ from openviking.models.vlm.token_usage import TokenUsageTracker++ if not self._vlm_instances:+ return {}++ merged_tracker = self._vlm_instances[0].token_tracker+ for instance in self._vlm_instances[1:]:+ merged_tracker = TokenUsageTracker.merge(merged_tracker, instance.token_tracker)++ return merged_tracker.to_dict()++ def reset_token_usage(self) -> None:+ """Reset token usage for all credential instances."""+ for instance in self._vlm_instances:+ instance.reset_token_usage()diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex a322b9223..a0e3b574d 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -145,10 +145,25 @@ def _get_active_embedding_model_config(config: "OpenVikingConfig") -> Any:def _build_embedding_metadata(config: "OpenVikingConfig") -> Dict[str, Any]:model_cfg = _get_active_embedding_model_config(config)+ # When credentials are configured, the first credential drives the actual+ # provider/model used at request time (see EmbeddingConfig._effective_*),+ # so the metadata signature must reflect the credential rather than the+ # parent's possibly-default provider/model. Otherwise a credential-only+ # OpenAI config would still be signed as ``provider=volcengine`` (the parent+ # default), masking real vector-space changes.+ first_cred = None+ creds = getattr(model_cfg, "credentials", None) or []+ if creds:+ first_cred = creds[0]+ cred_provider = getattr(first_cred, "provider", None) if first_cred else None+ cred_model = getattr(first_cred, "model", None) if first_cred else Noneprovider = (- getattr(model_cfg, "provider", None) or getattr(model_cfg, "backend", None) or ""+ cred_provider+ or getattr(model_cfg, "provider", None)+ or getattr(model_cfg, "backend", None)+ or "").lower()- model = getattr(model_cfg, "model", None) or ""+ model = cred_model or getattr(model_cfg, "model", None) or ""dimension = config.embedding.dimensionmodel_path = getattr(model_cfg, "model_path", None)model_identity = model@@ -284,9 +299,53 @@ async def init_context_collection(storage) -> bool:)return False+ # Embedding metadata differs from current config and the collection is+ # non-empty. Decide whether the user has explicitly opted in to keep the+ # existing vectors despite the metadata drift.+ existing_dimension = (+ existing_embedding_meta.get("dimension") if existing_embedding_meta else None+ )+ current_dimension = embedding_meta.get("dimension")+ dimension_changed = (+ existing_dimension is not None+ and current_dimension is not None+ and existing_dimension != current_dimension+ )++ allow_override = bool(getattr(config.embedding, "allow_metadata_override", False))+ if (+ allow_override+ and not dimension_changed+ and hasattr(storage, "update_collection_description")+ ):+ logger.warning(+ "Embedding metadata changed (provider/model) but dimension is "+ "unchanged; embedding.allow_metadata_override=true, so the existing "+ "collection metadata will be rewritten and existing vectors kept. "+ "old=%s new=%s",+ existing_embedding_meta,+ embedding_meta,+ )+ await storage.update_collection_description(+ _encode_collection_description(+ base_description or "Unified context collection",+ embedding_meta,+ )+ )+ return False++ if dimension_changed:+ raise EmbeddingRebuildRequiredError(+ "Existing collection embedding dimension "+ f"({existing_dimension}) does not match current configuration "+ f"({current_dimension}). Vectors are incompatible; rebuild is required."+ )+raise EmbeddingRebuildRequiredError("Existing collection embedding metadata does not match current configuration. "- "Rebuild is required before using the current embedding model."+ "Rebuild is required before using the current embedding model, or set "+ "embedding.allow_metadata_override=true to keep existing vectors when "+ "only provider/model changed (dimension must remain the same).")@@ -446,7 +505,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):# Process string (text) or list (multimodal) messagesif not isinstance(embedding_msg.message, (str, list)):- logger.debug(f"Skipping unsupported message type: {type(embedding_msg.message)}")+ logger.debug(+ f"Skipping unsupported message type: {type(embedding_msg.message)}"+ )self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)report_success = Truediff --git a/openviking/utils/circuit_breaker.py b/openviking/utils/circuit_breaker.pyindex 726f78d12..9af434a07 100644--- a/openviking/utils/circuit_breaker.py+++ b/openviking/utils/circuit_breaker.py@@ -8,6 +8,7 @@ import threadingimport timefrom openviking.utils.model_retry import (+ ERROR_CLASS_AUTH,ERROR_CLASS_INPUT_TOO_LARGE,ERROR_CLASS_PERMANENT,ERROR_CLASS_QUOTA_EXCEEDED,@@ -114,7 +115,11 @@ class CircuitBreaker:)return- if error_class in (ERROR_CLASS_PERMANENT, ERROR_CLASS_QUOTA_EXCEEDED):+ if error_class in (+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_AUTH,+ ERROR_CLASS_QUOTA_EXCEEDED,+ ):self._state = _STATE_OPENself._current_reset_timeout = self._base_reset_timeoutlogger.info(f"Circuit breaker tripped immediately on {error_class} error: {error}")diff --git a/openviking/utils/exceptions.py b/openviking/utils/exceptions.pynew file mode 100644index 000000000..a7bdb037e--- /dev/null+++ b/openviking/utils/exceptions.py@@ -0,0 +1,20 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Common exceptions for credential failover functionality."""+++class AllCredentialsFailedError(Exception):+ """Raised when all credentials in the chain have failed."""++ def __init__(self, errors: list[tuple[str, str, Exception, int]]):+ """Initialize the error with a list of credential failures.++ Args:+ errors: List of tuples containing (credential_id, error_class, exception, attempts)+ """+ self.errors = errors+ message = "All credentials failed:\n" + "\n".join(+ f" - {cred_id}: {error_class} - {exc}"+ for cred_id, error_class, exc, attempts in errors+ )+ super().__init__(message)diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.pyindex 0566269b4..9efe91103 100644--- a/openviking/utils/model_retry.py+++ b/openviking/utils/model_retry.py@@ -13,7 +13,9 @@ logger = logging.getLogger(__name__)T = TypeVar("T")# Error classification categories returned by classify_api_error()-ERROR_CLASS_PERMANENT = "permanent"+ERROR_CLASS_PERMANENT = "permanent" # request-level 4xx (e.g. 400 invalid parameter)+ERROR_CLASS_AUTH = "auth" # credential-level 401/403 (key invalid / no permission / overdue)+ERROR_CLASS_CONTENT_SAFETY = "content_safety" # request content rejected by moderationERROR_CLASS_INPUT_TOO_LARGE = "input_too_large"ERROR_CLASS_QUOTA_EXCEEDED = "quota_exceeded"ERROR_CLASS_TRANSIENT = "transient"@@ -36,8 +38,12 @@ INPUT_TOO_LARGE_PATTERNS = ("expected maxlength",)-PERMANENT_API_ERROR_PATTERNS = (- "400",+PERMANENT_API_ERROR_PATTERNS = ("400",)++# Credential-level errors: in multi-credential mode these advance to the next+# credential (another key may be valid / have permission / have balance); with a+# single credential or on the last credential they fail fast.+AUTH_API_ERROR_PATTERNS = ("401","403","forbidden",@@ -45,6 +51,18 @@ PERMANENT_API_ERROR_PATTERNS = ("accountoverdue",)+# Content moderation rejections. Same request content fails on every credential+# of the same model, so these fail fast (no point switching credentials).+CONTENT_SAFETY_PATTERNS = (+ "content policy",+ "content_filter",+ "contentfilter",+ "moderation",+ "sensitive content",+ "内容安全",+ "敏感",+)+QUOTA_EXCEEDED_PATTERNS = ("quotaexceeded", # also 429"quota limit",@@ -98,12 +116,18 @@ def _pattern_matches(text_lower: str, text_compact: str, pattern: str) -> bool:def classify_api_error(error: Exception) -> str:- """Classify an API error as permanent, quota_exceeded, transient, or unknown.-- ``quota_exceeded`` is checked before ``transient`` because quota errors- typically include "429" / "TooManyRequests" which would otherwise match- the transient category. Quota errors should not be retried; the caller- should fail over to a backup model instead.+ """Classify an API error into one of the ERROR_CLASS_* categories.++ Order matters:+ - ``content_safety`` is checked before ``permanent`` so a moderation+ rejection that happens to embed "400" in its message is not misclassified.+ - ``auth`` (401/403) is separated from ``permanent`` (400): auth errors are+ credential-level and may be resolved by switching credentials, whereas a+ 400 is a request-level error that fails on every credential of the same+ model.+ - ``quota_exceeded`` is checked before ``transient`` because quota errors+ typically include "429" / "TooManyRequests" which would otherwise match+ the transient category."""for exc in (error, getattr(error, "__cause__", None)):if exc is not None and isinstance(exc, _PERMANENT_IO_ERRORS):@@ -120,6 +144,15 @@ def classify_api_error(error: Exception) -> str:if _pattern_matches(text_lower, text_compact, pattern):return ERROR_CLASS_INPUT_TOO_LARGE+ # Content safety before permanent so a moderation message containing "400"+ # is not misclassified as a permanent parameter error.+ for text in texts:+ text_lower = text.lower()+ text_compact = text_lower.replace(" ", "")+ for pattern in CONTENT_SAFETY_PATTERNS:+ if _pattern_matches(text_lower, text_compact, pattern):+ return ERROR_CLASS_CONTENT_SAFETY+for text in texts:text_lower = text.lower()text_compact = text_lower.replace(" ", "")@@ -127,6 +160,13 @@ def classify_api_error(error: Exception) -> str:if _pattern_matches(text_lower, text_compact, pattern):return ERROR_CLASS_PERMANENT+ for text in texts:+ text_lower = text.lower()+ text_compact = text_lower.replace(" ", "")+ for pattern in AUTH_API_ERROR_PATTERNS:+ if _pattern_matches(text_lower, text_compact, pattern):+ return ERROR_CLASS_AUTH+# Check quota_exceeded *before* transient so that "429 … AccountQuotaExceeded"# is classified as quota_exceeded, not transient.for text in texts:@@ -305,10 +345,15 @@ class PrimaryBackupSwitcher:def record_primary_failure(self, error: Exception) -> bool:"""Record a primary failure. Returns True if should switch to backup.- Switches to backup immediately for ERROR_CLASS_PERMANENT or ERROR_CLASS_QUOTA_EXCEEDED.+ Switches to backup immediately for ERROR_CLASS_PERMANENT,+ ERROR_CLASS_AUTH or ERROR_CLASS_QUOTA_EXCEEDED."""error_class = classify_api_error(error)- if error_class in (ERROR_CLASS_PERMANENT, ERROR_CLASS_QUOTA_EXCEEDED):+ if error_class in (+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_AUTH,+ ERROR_CLASS_QUOTA_EXCEEDED,+ ):with self._lock:if not self._using_backup:logger.warning(f"Primary failed with {error_class}, switching to backup")@@ -330,3 +375,210 @@ class PrimaryBackupSwitcher:"""Check if currently using backup."""with self._lock:return self._using_backup+++class OrderedCredentialSwitcher:+ """Thread-safe ordered N-credential switcher with hierarchical failback.++ Supports ordered failover across multiple credentials. When a credential fails+ with quota_exceeded or permanent error, it advances to the next credential.+ After failback thresholds are met, it attempts to move back to a higher-priority+ credential (one step at a time, not all the way back to index 0).++ _active_idx == _n indicates all credentials are exhausted.+ """++ def __init__(+ self,+ n: int,+ failback_timeout_seconds: float = 600.0, # 10 minutes+ failback_request_count: int = 50,+ ):+ """Initialize the switcher.++ Args:+ n: Number of credentials (must be >= 1)+ failback_timeout_seconds: Time after which to attempt failback+ failback_request_count: Number of requests after which to attempt failback++ Note:+ Failure handling is driven by the error class (see+ ``classify_api_error``):++ - request-level errors (``permanent`` 400 / ``input_too_large`` /+ ``content_safety``) fail fast: the same request fails on every+ credential of the same model, so switching is useless.+ - credential-level ``auth`` errors (401/403) advance to the next+ credential in multi-credential mode; the last (or single)+ credential fails fast.+ - ``quota_exceeded`` (and ``transient`` once its retries are+ exhausted) and ``unknown`` advance to the next credential.+ """+ if n < 1:+ raise ValueError("Number of credentials must be >= 1")++ # Configuration (read-only after construction)+ self._n = n+ self._failback_timeout = failback_timeout_seconds+ self._failback_request_count = failback_request_count++ # Runtime state (protected by _lock)+ self._lock = threading.Lock()+ self._active_idx = 0+ self._last_switch_time: float = 0.0+ self._active_request_count = 0++ @property+ def n(self) -> int:+ """Get the number of credentials."""+ return self._n++ def maybe_failback(self) -> int:+ """Attempt a one-step failback toward higher-priority credentials.++ If the active credential is not already the highest priority (index 0)+ and a failback threshold (timeout or request count) is met, move the+ active index back one step. This mutates state and must be called only+ when about to issue a request, not for pure observation.++ Returns the (possibly updated) active credential index.+ """+ with self._lock:+ if self._active_idx > 0:+ timer_hit = (time.monotonic() - self._last_switch_time) >= self._failback_timeout+ count_hit = self._active_request_count >= self._failback_request_count+ if timer_hit or count_hit:+ previous_idx = self._active_idx+ self._active_idx -= 1+ self._last_switch_time = time.monotonic()+ self._active_request_count = 0+ logger.info(+ f"Failback condition met (timer={timer_hit}, count={count_hit}), "+ f"switching active credential from {previous_idx} to {self._active_idx}"+ )+ return self._active_idx++ def get_active_index(self) -> int:+ """Return the current active credential index (pure read, no side effects).++ Use :meth:`maybe_failback` to trigger failback before issuing a request.+ """+ with self._lock:+ return self._active_idx++ def on_success(self, idx: int) -> None:+ """Record a successful call on the given credential index.++ Increments the request counter for active_idx if idx matches.+ """+ with self._lock:+ if idx == self._active_idx and self._active_idx > 0:+ self._active_request_count += 1++ @staticmethod+ def is_fail_fast(error_class: str) -> bool:+ """Whether an error is request-level and must not try other credentials.++ Request-level errors (400 parameter error, input too large, content+ safety) fail on every credential of the same model, so the caller should+ re-raise immediately instead of cycling through credentials.+ """+ return error_class in (+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_INPUT_TOO_LARGE,+ ERROR_CLASS_CONTENT_SAFETY,+ )++ def commit_success(self, idx: int) -> None:+ """Record that credential ``idx`` successfully served a request.++ - If ``idx`` is the current active credential, advance the failback+ request counter (so failback to a higher-priority credential can+ eventually trigger).+ - If ``idx`` differs (a lower/other-priority credential served the+ request after the active one was unavailable), commit it as the new+ active credential (fast failover) and reset failback timers/counters.+ """+ with self._lock:+ if idx == self._active_idx:+ if self._active_idx > 0:+ self._active_request_count += 1+ return+ logger.info(+ f"Fast failover: credential {idx} served the request; "+ f"switching active credential from {self._active_idx} to {idx}"+ )+ self._active_idx = idx+ self._last_switch_time = time.monotonic()+ self._active_request_count = 0++ def on_failure(self, idx: int, error_class: str) -> bool:+ """Record a failure and decide whether to advance to the next credential.++ Args:+ idx: The credential index that failed+ error_class: One of ERROR_CLASS_* constants++ Returns:+ True if the caller should advance to the next credential (idx += 1)+ False if fail-fast (caller should re-raise the original exception)+ """+ # Transient errors that have exhausted retries are treated as quota_exceeded+ if error_class == ERROR_CLASS_TRANSIENT:+ error_class = ERROR_CLASS_QUOTA_EXCEEDED++ with self._lock:+ # Request-level errors fail fast: the same request fails on every+ # credential of the same model, so switching credentials is useless.+ if error_class in (+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_INPUT_TOO_LARGE,+ ERROR_CLASS_CONTENT_SAFETY,+ ):+ logger.warning(+ f"Credential {idx} failed with {error_class} (request-level), fail-fast"+ )+ return False++ if error_class == ERROR_CLASS_AUTH:+ # Credential-level error (key invalid / no permission / overdue).+ # In multi-credential mode, advance to the next credential since+ # another credential may have a valid key / permission / balance.+ # The last credential (or single-credential mode) fails fast.+ if idx == self._active_idx and self._active_idx + 1 < self._n:+ self._active_idx += 1+ self._last_switch_time = time.monotonic()+ self._active_request_count = 0+ logger.warning(+ f"Credential {idx} failed with auth error; "+ f"advancing to {self._active_idx} (multi-credential mode)"+ )+ return True+ logger.warning(f"Credential {idx} failed with auth error, fail-fast")+ return False++ if error_class == ERROR_CLASS_QUOTA_EXCEEDED:+ if idx == self._active_idx:+ self._active_idx = min(self._active_idx + 1, self._n)+ self._last_switch_time = time.monotonic()+ self._active_request_count = 0+ logger.warning(+ f"Credential {idx} failed with quota_exceeded, advancing to {self._active_idx}"+ )+ return True++ # Unknown error class: default to advancing (be conservative)+ if idx == self._active_idx:+ self._active_idx = min(self._active_idx + 1, self._n)+ self._last_switch_time = time.monotonic()+ self._active_request_count = 0+ logger.warning(+ f"Credential {idx} failed with unknown error class: {error_class}, advancing to {self._active_idx}"+ )+ return True++ @property+ def is_exhausted(self) -> bool:+ """Check if all credentials are exhausted."""+ with self._lock:+ return self._active_idx >= self._ndiff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex 5f06e196d..448ab0d00 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -1,10 +1,35 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-from typing import Any, Literal, Optional, cast+from typing import Any, ClassVar, List, Literal, Optional, Tuple, castfrom pydantic import BaseModel, Field, model_validator+class EmbeddingCredential(BaseModel):+ """Single embedding credential configuration for multi-credential failover."""++ id: Optional[str] = Field(default=None, description="Unique identifier for this credential")+ provider: Optional[str] = Field(default=None, description="Provider type")+ model: Optional[str] = Field(+ default=None,+ description=(+ "Model name (or endpoint id) for this credential. "+ "Overrides the parent EmbeddingModelConfig.model when set, allowing "+ "each credential to point to a different deployment / endpoint."+ ),+ )+ api_key: Optional[str] = Field(default=None, description="API key")+ api_base: Optional[str] = Field(default=None, description="API base URL")+ api_version: Optional[str] = Field(default=None, description="API version")+ ak: Optional[str] = Field(default=None, description="Access Key ID for VikingDB API")+ sk: Optional[str] = Field(default=None, description="Access Key Secret for VikingDB API")+ region: Optional[str] = Field(default=None, description="Region for VikingDB API")+ host: Optional[str] = Field(default=None, description="Host for VikingDB API")+ extra_headers: Optional[dict[str, str]] = Field(default=None, description="Extra HTTP headers")++ model_config = {"extra": "forbid"}++class EmbeddingModelConfig(BaseModel):"""Configuration for a specific embedding model"""@@ -94,6 +119,19 @@ class EmbeddingModelConfig(BaseModel):description="Maximum video frames for DashScope multimodal models (multimodal models only).",)+ # New multi-credential configuration+ credentials: List[EmbeddingCredential] = Field(+ default_factory=list,+ description="Ordered list of credentials for failover. Call order matches array index (0 is highest priority).",+ )++ failback_timeout_seconds: float = Field(+ default=600.0, description="Time in seconds after which to attempt failback to primary"+ )+ failback_request_count: int = Field(+ default=50, description="Number of backup requests after which to attempt failback"+ )+model_config = {"extra": "forbid"}@model_validator(mode="before")@@ -111,79 +149,148 @@ class EmbeddingModelConfig(BaseModel):data[key] = value.lower()return data- @model_validator(mode="after")- def validate_config(self):- """Validate configuration completeness and consistency"""- if self.backend and not self.provider:- self.provider = self.backend+ _VALID_PROVIDERS: ClassVar[Tuple[str, ...]] = (+ "openai",+ "azure",+ "volcengine",+ "vikingdb",+ "jina",+ "ollama",+ "gemini",+ "voyage",+ "dashscope",+ "minimax",+ "cohere",+ "litellm",+ "local",+ )- if not self.model:- raise ValueError("Embedding model name is required")+ @classmethod+ def _validate_provider_auth(+ cls,+ *,+ label: str,+ provider: Optional[str],+ api_key: Optional[str],+ api_base: Optional[str],+ ak: Optional[str],+ sk: Optional[str],+ region: Optional[str],+ ) -> None:+ """Validate provider name and provider-specific auth requirements.++ Shared by both the top-level config validator (single-credential mode)+ and the per-credential validator (multi-credential mode).- if not self.provider:- raise ValueError("Embedding provider is required")+ Args:+ label: Prefix used in error messages, e.g. "Embedding" or+ "credentials[ark-primary]".+ provider: Provider name (already lowercased / normalized).+ api_key, api_base, ak, sk, region: Resolved auth fields, with+ credential values taking precedence over parent fallbacks+ in multi-credential mode.+ """+ if not provider:+ raise ValueError(f"{label}: provider is required")+ if provider not in cls._VALID_PROVIDERS:+ raise ValueError(+ f"{label}: invalid provider '{provider}'. Must be one of: "+ + ", ".join(f"'{p}'" for p in cls._VALID_PROVIDERS)+ )- if self.provider not in [- "openai",- "azure",+ if provider == "openai":+ # Allow missing api_key when api_base is set (e.g. local OpenAI-compatible servers)+ if not api_key and not api_base:+ raise ValueError(f"{label}: OpenAI provider requires 'api_key' to be set")+ elif provider == "azure":+ if not api_key:+ raise ValueError(f"{label}: Azure provider requires 'api_key' to be set")+ if not api_base:+ raise ValueError(+ f"{label}: Azure provider requires 'api_base' (Azure endpoint) to be set"+ )+ elif provider in {"volcengine",- "vikingdb","jina",- "ollama","gemini","voyage",- "dashscope","minimax","cohere",- "litellm",- "local",- ]:- raise ValueError(- f"Invalid embedding provider: '{self.provider}'. Must be one of: "- "'openai', 'azure', 'volcengine', 'vikingdb', 'jina', 'ollama', 'gemini', 'voyage', 'dashscope', 'minimax', 'cohere', 'litellm', 'local'"- )-- # Provider-specific validation- if self.provider == "openai":- # Allow missing api_key when api_base is set (e.g. local OpenAI-compatible servers)- if not self.api_key and not self.api_base:- raise ValueError("OpenAI provider requires 'api_key' to be set")-- elif self.provider == "azure":- if not self.api_key:- raise ValueError("Azure provider requires 'api_key' to be set")- if not self.api_base:- raise ValueError("Azure provider requires 'api_base' (Azure endpoint) to be set")-- elif self.provider == "ollama":- # Ollama runs locally, no API key required- pass-- elif self.provider == "volcengine":- if not self.api_key:- raise ValueError("Volcengine provider requires 'api_key' to be set")-- elif self.provider == "vikingdb":- missing = []- if not self.ak:- missing.append("ak")- if not self.sk:- missing.append("sk")- if not self.region:- missing.append("region")-+ "dashscope",+ }:+ if not api_key:+ provider_label = {+ "volcengine": "Volcengine",+ "jina": "Jina",+ "gemini": "Gemini",+ "voyage": "Voyage",+ "minimax": "MiniMax",+ "cohere": "Cohere",+ "dashscope": "DashScope",+ }[provider]+ raise ValueError(f"{label}: {provider_label} provider requires 'api_key' to be set")+ elif provider == "vikingdb":+ missing = [n for n, v in (("ak", ak), ("sk", sk), ("region", region)) if not v]if missing:raise ValueError(- f"VikingDB provider requires the following fields: {', '.join(missing)}"+ f"{label}: VikingDB provider requires the following fields: "+ f"{', '.join(missing)}")+ # ollama / litellm / local: no auth requirement enforced here++ @model_validator(mode="after")+ def validate_config(self):+ """Validate configuration completeness and consistency"""+ if self.backend and not self.provider:+ self.provider = self.backend- elif self.provider == "jina":- if not self.api_key:- raise ValueError("Jina provider requires 'api_key' to be set")+ if not self.model and not any(c.model for c in self.credentials):+ raise ValueError("Embedding model name is required")++ # When credentials are configured, defer provider/api_key validation to+ # per-credential checks; the parent-level provider/api_key may be left+ # blank because each credential carries its own settings.+ if self.credentials:+ self._validate_credentials()+ self._validate_credential_dimensions()+ return self++ self._validate_provider_auth(+ label="Embedding",+ provider=self.provider,+ api_key=self.api_key,+ api_base=self.api_base,+ ak=self.ak,+ sk=self.sk,+ region=self.region,+ )++ self._validate_provider_specific_options(+ label="Embedding",+ provider=self.provider,+ model=self.model,+ )++ return self++ def _validate_provider_specific_options(+ self, *, label: str, provider: Optional[str], model: Optional[str]+ ) -> None:+ """Validate provider-specific options that go beyond auth.++ These rules depend on the provider/model pair and on parent-level+ fields (``query_param``, ``document_param``, ``input``, ``dimension``,+ ``enable_fusion``, etc.) that are shared across all credentials. They+ must run for both the single-credential path (using ``self.provider``)+ and the multi-credential path (using each credential's resolved+ provider/model), otherwise invalid configurations slip past startup+ validation and surface only at runtime — after collection schema and+ embedding metadata may already have been written.+ """+ if not provider:+ return- elif self.provider == "gemini":- if not self.api_key:- raise ValueError("Gemini provider requires 'api_key' to be set")+ if provider == "gemini":_GEMINI_TASK_TYPES = {"RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT",@@ -200,62 +307,207 @@ class EmbeddingModelConfig(BaseModel):]:if value and value.upper() not in _GEMINI_TASK_TYPES:raise ValueError(- f"Invalid {field_name} '{value}' for Gemini. "+ f"{label}: invalid {field_name} '{value}' for Gemini. "f"Valid task_types: {', '.join(sorted(_GEMINI_TASK_TYPES))}")- elif self.provider == "voyage":- if not self.api_key:- raise ValueError("Voyage provider requires 'api_key' to be set")-- elif self.provider == "minimax":- if not self.api_key:- raise ValueError("MiniMax provider requires 'api_key' to be set")-- elif self.provider == "cohere":- if not self.api_key:- raise ValueError("Cohere provider requires 'api_key' to be set")-- elif self.provider == "dashscope":- if not self.api_key:- raise ValueError("DashScope provider requires 'api_key' to be set")+ elif provider == "dashscope":if self.input == "text" and (self.enable_fusion is not Noneor self.res_level is not Noneor self.max_video_frames is not None):raise ValueError(- "Parameters enable_fusion, res_level, and max_video_frames only apply to multimodal input mode"+ f"{label}: parameters enable_fusion, res_level, and max_video_frames "+ "only apply to multimodal input mode")- elif self.provider == "litellm":+ elif provider == "litellm":# litellm handles auth via env vars or explicit api_key; no strict requirementif not self.dimension:raise ValueError(- "LiteLLM provider requires 'dimension' to be set explicitly. "+ f"{label}: LiteLLM provider requires 'dimension' to be set explicitly. ""Check your embedding model's documentation for the correct dimension.")- elif self.provider == "local":+ elif provider == "local":from openviking.models.embedder.local_embedders import get_local_model_spec- get_local_model_spec(self.model)+ get_local_model_spec(model)- return self+ def _validate_credentials(self) -> None:+ """Validate each credential when credentials list is non-empty.++ Each credential must resolve a provider (from itself or the parent) and+ meet that provider's required fields. The parent-level provider/api_key+ are used as fallbacks where appropriate. Provider-specific options that+ depend on parent-level fields (e.g. LiteLLM dimension, Gemini task+ types, DashScope multimodal-only params) are also re-checked per+ resolved credential so the credentials path enforces the same invariants+ as the single-credential path.+ """+ for idx, cred in enumerate(self.credentials):+ cred_id = cred.id or f"credential-{idx}"+ label = f"credentials[{cred_id}]"+ resolved_provider = (cred.provider or self.provider or "").lower() or None+ resolved_model = cred.model or self.model+ self._validate_provider_auth(+ label=label,+ provider=resolved_provider,+ api_key=cred.api_key or self.api_key,+ api_base=cred.api_base or self.api_base,+ ak=cred.ak or self.ak,+ sk=cred.sk or self.sk,+ region=cred.region or self.region,+ )+ self._validate_provider_specific_options(+ label=label,+ provider=resolved_provider,+ model=resolved_model,+ )++ def _validate_credential_dimensions(self) -> None:+ """Ensure all credentials produce vectors of the same dimension.++ Mixing credentials whose embedders return different vector lengths+ causes ``dense vector dimension mismatch`` errors at write time and+ silently breaks retrieval. We therefore reject such configurations at+ startup. Dimensions are resolved using the same heuristics as+ ``get_effective_dimension`` (provider + model), with the explicit+ parent ``self.dimension`` always winning when set.+ """+ if len(self.credentials) <= 1:+ return++ resolved: list[tuple[str, int]] = []+ for idx, cred in enumerate(self.credentials):+ cred_id = cred.id or f"credential-{idx}"+ provider = (cred.provider or self.provider or "").lower()+ model = cred.model or self.model+ try:+ dim = self._resolve_dimension_for(provider, model)+ except ValueError:+ # Unknown dimension (e.g. ollama with no explicit dim and+ # unknown model): skip - the per-credential validation above+ # would have already required `dimension` if needed via+ # parent.dimension, and resolution failures here are non-fatal+ # because the credential may still work at runtime.+ continue+ if dim is None:+ continue+ resolved.append((cred_id, dim))++ distinct = {dim for _, dim in resolved}+ if len(distinct) > 1:+ details = ", ".join(f"{cid}={dim}" for cid, dim in resolved)+ raise ValueError(+ "Embedding credentials have conflicting vector dimensions: "+ f"{details}. All credentials must produce vectors of the same "+ "dimension; mixing different dimensions corrupts the vector "+ "store. Either align provider/model across credentials, or "+ "set 'dimension' explicitly on the parent embedding config "+ "to force a fixed schema dimension."+ )++ def _resolve_dimension_for(self, provider: str, model: Optional[str]) -> Optional[int]:+ """Resolve dimension for a (provider, model) pair using the same+ rules as ``get_effective_dimension``. The explicit parent dimension,+ when set, always wins."""+ if self.dimension is not None:+ return self.dimension++ provider = (provider or "").lower()+ if provider in {"openai", "azure"}:+ mapping = {+ "text-embedding-ada-002": 1536,+ "text-embedding-3-small": 1536,+ "text-embedding-3-large": 3072,+ }+ return mapping.get((model or "").lower())++ if provider == "voyage":+ from openviking.models.embedder.voyage_embedders import (+ get_voyage_model_default_dimension,+ )++ return get_voyage_model_default_dimension(model)++ if provider == "cohere":+ from openviking.models.embedder.cohere_embedders import (+ get_cohere_model_default_dimension,+ )++ return get_cohere_model_default_dimension(model)++ if provider == "gemini":+ from openviking.models.embedder.gemini_embedders import GeminiDenseEmbedder++ return GeminiDenseEmbedder._default_dimension(model)++ if provider == "local":+ from openviking.models.embedder.local_embedders import (+ get_local_model_default_dimension,+ )++ return get_local_model_default_dimension(model)++ if provider == "dashscope":+ try:+ from openviking.models.embedder.dashscope_embedders import (+ get_dashscope_model_default_dimension,+ )++ return get_dashscope_model_default_dimension(model)+ except ImportError:+ return 1024++ # Providers without a known dimension lookup (volcengine, jina,+ # vikingdb, ollama for unlisted models, etc.) cannot be cross-checked+ # here without an explicit dimension. Return None so validation skips+ # them; same-provider credentials are typically dimension-compatible+ # by construction, and any real mismatch will surface at write time.+ return None++ def _effective_provider(self) -> Optional[str]:+ """Resolve the provider that actually drives this config.++ When credentials are configured, the first credential's provider takes+ precedence over the parent's (which may still hold its default value of+ ``volcengine``). Without this fallback, credential-only configurations+ targeting a non-default provider would silently use the parent's+ default for dimension/metadata resolution and produce a vector-space+ mismatch (e.g. parent volcengine -> 2048 dim while the actual OpenAI+ embedder returns 1536).+ """+ if self.credentials and self.credentials[0].provider:+ return self.credentials[0].provider+ return self.provider++ def _effective_model(self) -> Optional[str]:+ """Resolve the model that actually drives this config.++ Same rationale as ``_effective_provider``: credential-level model wins+ when present so dimension/metadata reflect the model the embedder will+ actually call.+ """+ if self.credentials and self.credentials[0].model:+ return self.credentials[0].model+ return self.modeldef get_effective_dimension(self) -> int:"""Resolve the dimension used for schema creation and validation."""if self.dimension is not None:return self.dimension- provider = (self.provider or "").lower()+ provider = (self._effective_provider() or "").lower()+ effective_model = self._effective_model()if provider in {"openai", "azure"}:openai_model_dimensions = {"text-embedding-ada-002": 1536,"text-embedding-3-small": 1536,"text-embedding-3-large": 3072,}- model_lower = (self.model or "").lower()+ model_lower = (effective_model or "").lower()if model_lower in openai_model_dimensions:return openai_model_dimensions[model_lower]@@ -264,19 +516,19 @@ class EmbeddingModelConfig(BaseModel):get_voyage_model_default_dimension,)- return get_voyage_model_default_dimension(self.model)+ return get_voyage_model_default_dimension(effective_model)if provider == "cohere":from openviking.models.embedder.cohere_embedders import (get_cohere_model_default_dimension,)- return get_cohere_model_default_dimension(self.model)+ return get_cohere_model_default_dimension(effective_model)if provider == "gemini":from openviking.models.embedder.gemini_embedders import GeminiDenseEmbedder- return GeminiDenseEmbedder._default_dimension(self.model)+ return GeminiDenseEmbedder._default_dimension(effective_model)if provider == "ollama":# Common Ollama embedding models and their dimensions@@ -298,12 +550,12 @@ class EmbeddingModelConfig(BaseModel):"embeddinggemma": 768,"embeddinggemma:300m": 768,}- model_lower = (self.model or "").lower()+ model_lower = (effective_model or "").lower()if model_lower in ollama_model_dimensions:return ollama_model_dimensions[model_lower]# For unknown Ollama models, require explicit dimensionraise ValueError(- f"Unknown dimension for Ollama model '{self.model}'. "+ f"Unknown dimension for Ollama model '{effective_model}'. "f"Please set 'dimension' explicitly in your embedding config. "f"Known models: {list(ollama_model_dimensions.keys())}")@@ -311,7 +563,7 @@ class EmbeddingModelConfig(BaseModel):if provider == "local":from openviking.models.embedder.local_embedders import get_local_model_default_dimension- return get_local_model_default_dimension(self.model)+ return get_local_model_default_dimension(effective_model)if provider == "dashscope":try:@@ -319,7 +571,7 @@ class EmbeddingModelConfig(BaseModel):get_dashscope_model_default_dimension,)- return get_dashscope_model_default_dimension(self.model)+ return get_dashscope_model_default_dimension(effective_model)except ImportError:# Fallback dimension if dashscope_embedders module doesn't exist yetreturn 1024@@ -381,22 +633,24 @@ class EmbeddingConfig(BaseModel):default="content_only",description="Text source for file vectorization: summary_first|summary_only|content_only",)- image_vectorization: str = Field(- default="summary_only",- description=(- "How IMAGE files are vectorized: "- "'summary_only' embeds only the generated text summary (text embedding); "- "'image_only' sends the image itself to a multimodal embedder; "- "'image_and_summary' sends both the image and its text summary together. "- "'image_only' and 'image_and_summary' require a multimodal-capable embedder "- "(e.g. Volcengine with input='multimodal')."- ),- )max_input_tokens: int = Field(default=4096,ge=100,description="Maximum estimated tokens sent to embeddings when raw text fallback is used",)+ allow_metadata_override: bool = Field(+ default=False,+ description=(+ "When true, allow starting up against an existing collection whose "+ "embedding metadata (provider/model) differs from the current config, "+ "as long as dimension is unchanged. The collection metadata will be "+ "rewritten to the new config and a warning will be logged. "+ "Useful when migrating an existing index to a new model deployment "+ "(e.g. switching ARK endpoint id) while keeping previously indexed "+ "vectors. Note: vector semantics may drift if the underlying model "+ "actually changed; only enable when you understand the implication."+ ),+ )model_config = {"extra": "forbid"}@@ -427,11 +681,6 @@ class EmbeddingConfig(BaseModel):raise ValueError("embedding.text_source must be one of: summary_first, summary_only, content_only")- if self.image_vectorization not in {"summary_only", "image_only", "image_and_summary"}:- raise ValueError(- "embedding.image_vectorization must be one of: "- "summary_only, image_only, image_and_summary"- )return selfdef _create_embedder(@@ -740,27 +989,83 @@ class EmbeddingConfig(BaseModel):from openviking.models.embedder.base import DenseEmbedderBase, SparseEmbedderBaseif self.hybrid:+ if self.hybrid.credentials:+ return self._create_failover_embedder("hybrid", self.hybrid)provider = self._require_provider(self.hybrid.provider)return self._create_embedder(provider, "hybrid", self.hybrid)if self.dense and self.sparse:- dense_provider = self._require_provider(self.dense.provider)- dense_embedder = cast(- DenseEmbedderBase,- self._create_embedder(dense_provider, "dense", self.dense),- )- sparse_embedder = self._create_embedder(- self._require_provider(self.sparse.provider), "sparse", self.sparse+ # Handle failover for both dense and sparse if credentials are configured+ dense_embedder = self._create_single_or_failover_embedder("dense", self.dense)+ sparse_embedder = self._create_single_or_failover_embedder("sparse", self.sparse)+ return CompositeHybridEmbedder(+ cast(DenseEmbedderBase, dense_embedder),+ cast(SparseEmbedderBase, sparse_embedder),)- sparse_embedder = cast(SparseEmbedderBase, sparse_embedder)- return CompositeHybridEmbedder(dense_embedder, sparse_embedder)if self.dense:- provider = self._require_provider(self.dense.provider)- return self._create_embedder(provider, "dense", self.dense)+ return self._create_single_or_failover_embedder("dense", self.dense)raise ValueError("No embedding configuration found (dense, sparse, or hybrid)")+ def _create_single_or_failover_embedder(+ self, embedder_type: str, config: "EmbeddingModelConfig"+ ):+ """Create either a single embedder or a FailoverEmbedder based on credentials config."""+ if config.credentials:+ return self._create_failover_embedder(embedder_type, config)+ provider = self._require_provider(config.provider)+ return self._create_embedder(provider, embedder_type, config)++ def _create_failover_embedder(self, embedder_type: str, config: "EmbeddingModelConfig"):+ """Create a FailoverEmbedder with multiple credentials."""+ from openviking.models.embedder import FailoverEmbedder++ embedders = []+ credential_ids = []++ for cred in config.credentials:+ # Create a temporary config merged from the model config and credential+ merged_config = EmbeddingModelConfig(+ model=cred.model or config.model,+ dimension=config.dimension,+ batch_size=config.batch_size,+ input=config.input,+ query_param=config.query_param,+ document_param=config.document_param,+ provider=cred.provider or config.provider,+ version=config.version,+ ak=cred.ak or config.ak,+ sk=cred.sk or config.sk,+ region=cred.region or config.region,+ host=cred.host or config.host,+ api_key=cred.api_key or config.api_key,+ api_base=cred.api_base or config.api_base,+ api_version=cred.api_version or config.api_version,+ extra_headers=cred.extra_headers or config.extra_headers,+ # Model-behavior fields are shared by all credentials of the+ # same model and live only on the parent config.+ encoding_format=config.encoding_format,+ model_path=config.model_path,+ cache_dir=config.cache_dir,+ enable_fusion=config.enable_fusion,+ res_level=config.res_level,+ max_video_frames=config.max_video_frames,+ )+ provider = self._require_provider(merged_config.provider)+ embedders.append(self._create_embedder(provider, embedder_type, merged_config))+ credential_ids.append(cred.id or f"credential-{len(credential_ids)}")++ if len(embedders) == 1:+ return embedders[0]++ return FailoverEmbedder(+ embedders=embedders,+ credential_ids=credential_ids,+ failback_timeout_seconds=config.failback_timeout_seconds,+ failback_request_count=config.failback_request_count,+ )+@propertydef dimension(self) -> int:"""Get dimension from active config."""diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.pyindex 884ced649..be8b5efb3 100644--- a/openviking_cli/utils/config/vlm_config.py+++ b/openviking_cli/utils/config/vlm_config.py@@ -18,11 +18,39 @@ def _normalize_provider_name(name: Optional[str]) -> Optional[str]:return cleaned or None+class VLMCredential(BaseModel):+ """Single VLM credential configuration for multi-credential failover."""++ id: Optional[str] = Field(default=None, description="Unique identifier for this credential")+ provider: Optional[str] = Field(default=None, description="Provider type")+ model: Optional[str] = Field(+ default=None,+ description=(+ "Model name (or endpoint id) for this credential. "+ "Overrides the parent VLMConfig.model when set, allowing each credential "+ "to point to a different deployment / endpoint."+ ),+ )+ api_key: Optional[str] = Field(default=None, description="API key")+ api_base: Optional[str] = Field(default=None, description="API base URL")+ api_version: Optional[str] = Field(default=None, description="API version")+ forward_api_key: Optional[bool] = Field(+ default=None, description="Whether to pass api_key through to LiteLLM"+ )+ extra_headers: Optional[Dict[str, str]] = Field(default=None, description="Extra HTTP headers")+ extra_request_body: Optional[Dict[str, Any]] = Field(+ default=None, description="Extra JSON body fields"+ )+ stream: Optional[bool] = Field(default=None, description="Enable streaming mode")++ model_config = {"extra": "forbid"}++class VLMConfig(BaseModel):- """VLM configuration, supports multiple provider backends."""+ """VLM configuration, supports multiple provider backends and multi-credential failover."""backup: Optional["VLMConfig"] = Field(- default=None, description="Backup VLM configuration for failover"+ default=None, description="Backup VLM configuration for failover (legacy)")model: Optional[str] = Field(default=None, description="Model name")api_key: Optional[str] = Field(default=None, description="API key")@@ -89,6 +117,19 @@ class VLMConfig(BaseModel):default=False, description="Enable streaming mode for OpenAI-compatible providers")+ # New multi-credential configuration+ credentials: List[VLMCredential] = Field(+ default_factory=list,+ description="Ordered list of credentials for failover. Call order matches array index (0 is highest priority).",+ )++ failback_timeout_seconds: float = Field(+ default=600.0, description="Time in seconds after which to attempt failback to primary"+ )+ failback_request_count: int = Field(+ default=50, description="Number of backup requests after which to attempt failback"+ )+_vlm_instance: Optional[Any] = Nonemodel_config = {"arbitrary_types_allowed": True, "extra": "forbid"}@@ -125,32 +166,53 @@ class VLMConfig(BaseModel):@model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""+ # Validate recursive backup BEFORE normalizing credentials (which clears backup)+ self._validate_no_recursive_backup()+self._migrate_legacy_config()+ self._normalize_credentials()if self._has_any_config():if not self.model:raise ValueError("VLM configuration requires 'model' to be set")- provider_name = self._resolve_provider_name()- if provider_name == "openai-codex":- has_codex_auth_available = _load_codex_auth_module().has_codex_auth_available-- if not self._get_effective_api_key() and not has_codex_auth_available():- raise ValueError(- "VLM configuration requires Codex OAuth credentials in ~/.openviking/codex_auth.json or an importable Codex CLI auth file"- )- elif provider_name != "litellm" and not self._get_effective_api_key():- raise ValueError("VLM configuration requires 'api_key' to be set")+ # When credentials are configured, validate each credential+ if self.credentials:+ for i, cred in enumerate(self.credentials):+ provider_name = cred.provider or self.provider+ if provider_name == "openai-codex":+ has_codex_auth_available = (+ _load_codex_auth_module().has_codex_auth_available+ )+ if not cred.api_key and not has_codex_auth_available():+ raise ValueError(+ f"Credential {i} ({cred.id or 'unnamed'}): requires Codex OAuth credentials in ~/.openviking/codex_auth.json"+ )+ elif provider_name not in ("litellm", None) and not cred.api_key:+ # Also check providers dict for fallback+ if not self._get_credential_api_key(cred):+ raise ValueError(+ f"Credential {i} ({cred.id or 'unnamed'}): requires 'api_key' to be set"+ )+ else:+ # Legacy validation+ provider_name = self._resolve_provider_name()+ if provider_name == "openai-codex":+ has_codex_auth_available = _load_codex_auth_module().has_codex_auth_available+ if not self._get_effective_api_key() and not has_codex_auth_available():+ raise ValueError(+ "VLM configuration requires Codex OAuth credentials in ~/.openviking/codex_auth.json or an importable Codex CLI auth file"+ )+ elif provider_name != "litellm" and not self._get_effective_api_key():+ raise ValueError("VLM configuration requires 'api_key' to be set")return self- @model_validator(mode="after")- def validate_no_recursive_backup(self):+ def _validate_no_recursive_backup(self):"""Prevent recursive backup configurations"""if self.backup is not None:if self.backup.backup is not None:raise ValueError("Backup VLM configuration cannot have its own backup (recursive backups are not allowed)")- return selfdef _migrate_legacy_config(self):"""Migrate legacy config to providers structure."""@@ -183,8 +245,162 @@ class VLMConfig(BaseModel):if self.stream and "stream" not in self.providers[self.provider]:self.providers[self.provider]["stream"] = self.stream+ def _normalize_credentials(self):+ """Normalize credentials configuration:+ 1. Migrate legacy backup config to credentials+ 2. Migrate top-level credentials to credentials list+ 3. Assign default IDs to credentials without IDs+ 4. Apply top-level defaults to credentials+ """+ migrated_credentials = []++ # Step 1: Migrate legacy backup config+ if self.backup is not None and self.backup._has_any_config():+ # Primary credential: resolve via _match_provider() so legacy+ # ``providers: {openai: {...}}`` configs (without top-level+ # provider/api_key) are migrated correctly.+ primary_cfg, primary_provider = self._match_provider()+ primary_cfg = primary_cfg or {}+ primary_cred = VLMCredential(+ id="legacy-primary",+ provider=primary_provider or self.provider,+ model=self.model,+ api_key=primary_cfg.get("api_key") or self.api_key,+ api_base=primary_cfg.get("api_base") or self.api_base,+ api_version=primary_cfg.get("api_version") or self.api_version,+ forward_api_key=(+ primary_cfg.get("forward_api_key")+ if primary_cfg.get("forward_api_key") is not None+ else self.forward_api_key+ ),+ extra_headers=primary_cfg.get("extra_headers") or self.extra_headers,+ extra_request_body=(+ primary_cfg.get("extra_request_body") or self.extra_request_body+ ),+ stream=(+ primary_cfg.get("stream")+ if primary_cfg.get("stream") is not None+ else self.stream+ ),+ )+ migrated_credentials.append(primary_cred)++ # Backup credential: same resolution rules so backup using+ # providers/default_provider also gets a usable api_key/provider.+ backup_cfg, backup_provider = self.backup._match_provider()+ backup_cfg = backup_cfg or {}+ backup_cred = VLMCredential(+ id="legacy-backup",+ provider=backup_provider or self.backup.provider,+ model=self.backup.model,+ api_key=backup_cfg.get("api_key") or self.backup.api_key,+ api_base=backup_cfg.get("api_base") or self.backup.api_base,+ api_version=backup_cfg.get("api_version") or self.backup.api_version,+ forward_api_key=(+ backup_cfg.get("forward_api_key")+ if backup_cfg.get("forward_api_key") is not None+ else self.backup.forward_api_key+ ),+ extra_headers=backup_cfg.get("extra_headers") or self.backup.extra_headers,+ extra_request_body=(+ backup_cfg.get("extra_request_body") or self.backup.extra_request_body+ ),+ stream=(+ backup_cfg.get("stream")+ if backup_cfg.get("stream") is not None+ else self.backup.stream+ ),+ )+ migrated_credentials.append(backup_cred)++ # Clear backup to avoid double processing+ self.backup = None++ # Step 2: If no credentials from backup migration and no explicit credentials,+ # create credentials from top-level config. We resolve the effective provider+ # config first (via _match_provider) so legacy ``providers: {openai: {...}}``+ # configurations - where the api_key/api_base live inside the providers dict+ # rather than at the top level - are migrated correctly. Otherwise the+ # generated credential would be missing provider/api_key and is_available()+ # would return False.+ if not migrated_credentials and not self.credentials:+ if self._has_legacy_provider_config():+ provider_cfg, provider_name = self._match_provider()+ provider_cfg = provider_cfg or {}+ migrated_credentials.append(+ VLMCredential(+ id="default",+ provider=provider_name or self.provider,+ model=self.model,+ api_key=provider_cfg.get("api_key") or self.api_key,+ api_base=provider_cfg.get("api_base") or self.api_base,+ api_version=provider_cfg.get("api_version") or self.api_version,+ forward_api_key=(+ provider_cfg.get("forward_api_key")+ if provider_cfg.get("forward_api_key") is not None+ else self.forward_api_key+ ),+ extra_headers=provider_cfg.get("extra_headers") or self.extra_headers,+ extra_request_body=(+ provider_cfg.get("extra_request_body") or self.extra_request_body+ ),+ stream=(+ provider_cfg.get("stream")+ if provider_cfg.get("stream") is not None+ else self.stream+ ),+ )+ )++ # Step 3: Merge migrated credentials with existing credentials+ if migrated_credentials:+ # Only use migrated if no explicit credentials exist+ if not self.credentials:+ self.credentials = migrated_credentials++ # Step 4: Assign default IDs and apply top-level defaults+ for i, cred in enumerate(self.credentials):+ if not cred.id:+ cred.id = f"credential-{i}"+ # Apply top-level defaults where credential doesn't specify+ if not cred.provider:+ cred.provider = self.provider+ if not cred.api_key:+ cred.api_key = self._get_credential_api_key(cred)+ if not cred.api_base:+ cred.api_base = self.api_base+ if not cred.api_version:+ cred.api_version = self.api_version+ if cred.forward_api_key is None:+ cred.forward_api_key = self.forward_api_key+ if not cred.extra_headers:+ cred.extra_headers = self.extra_headers+ if not cred.extra_request_body:+ cred.extra_request_body = self.extra_request_body+ if cred.stream is None:+ cred.stream = self.stream++ def _has_legacy_provider_config(self) -> bool:+ """Check if there's legacy provider config (not credentials-based)."""+ return (+ self.provider is not None+ or self.api_key is not None+ or self.api_base is not None+ or self.providers+ )++ def _get_credential_api_key(self, cred: VLMCredential) -> str | None:+ """Get effective API key for a credential, checking providers dict."""+ if cred.api_key:+ return cred.api_key+ if cred.provider and cred.provider in self.providers:+ return self.providers[cred.provider].get("api_key")+ return None+def _has_any_config(self) -> bool:"""Check if any config is provided."""+ if self.credentials:+ return Trueif self.api_key or self.model or self.api_base or self.provider or self.default_provider:return Trueif self.providers:@@ -193,6 +409,8 @@ class VLMConfig(BaseModel):def _get_effective_api_key(self) -> str | None:"""Get effective API key."""+ if self.credentials:+ return self.credentials[0].api_key or self._get_credential_api_key(self.credentials[0])if self.api_key:return self.api_keyconfig, _ = self._match_provider()@@ -234,6 +452,22 @@ class VLMConfig(BaseModel):(provider_config_dict, provider_name)"""del model+ # If credentials are configured, use the first one+ if self.credentials:+ cred = self.credentials[0]+ return (+ {+ "api_key": cred.api_key,+ "api_base": cred.api_base,+ "api_version": cred.api_version,+ "forward_api_key": cred.forward_api_key,+ "extra_headers": cred.extra_headers,+ "extra_request_body": cred.extra_request_body,+ "stream": cred.stream,+ },+ cred.provider,+ )+if self.provider:return self._get_provider_config_by_name(self.provider) or {}, self.provider@@ -255,6 +489,9 @@ class VLMConfig(BaseModel):return None, Nonedef _resolve_provider_name(self) -> str | None:+ """Resolve provider name from credentials or legacy config."""+ if self.credentials:+ return self.credentials[0].provider_, name = self._match_provider()return name@@ -269,22 +506,67 @@ class VLMConfig(BaseModel):return self._match_provider(model)def get_vlm_instance(self) -> Any:- """Get VLM instance."""+ """Get VLM instance with multi-credential failover support."""if self._vlm_instance is None:- config_dict = self._build_vlm_config_dict()- from openviking.models.vlm import FailoverVLM, VLMFactory+ from openviking.models.vlm import FailoverVLM, MultiCredentialVLM, VLMFactory++ if self.credentials:+ # Build VLM instances for each credential+ vlm_instances = []+ for cred in self.credentials:+ config_dict = self._build_vlm_config_dict_for_credential(cred)+ vlm_instances.append(VLMFactory.create(config_dict))++ if len(vlm_instances) == 1:+ self._vlm_instance = vlm_instances[0]+ else:+ self._vlm_instance = MultiCredentialVLM(+ vlm_instances,+ credential_ids=[c.id for c in self.credentials],+ failback_timeout_seconds=self.failback_timeout_seconds,+ failback_request_count=self.failback_request_count,+ )+ else:+ # Legacy mode: single VLM with optional backup+ config_dict = self._build_vlm_config_dict()+ primary = VLMFactory.create(config_dict)- primary = VLMFactory.create(config_dict)+ if self.backup is not None and self.backup._has_any_config():+ backup_config_dict = self.backup._build_vlm_config_dict()+ backup = VLMFactory.create(backup_config_dict)+ self._vlm_instance = FailoverVLM(primary, backup)+ else:+ self._vlm_instance = primary- # If backup is configured, wrap with FailoverVLM- if self.backup is not None and self.backup._has_any_config():- backup_config_dict = self.backup._build_vlm_config_dict()- backup = VLMFactory.create(backup_config_dict)- self._vlm_instance = FailoverVLM(primary, backup)- else:- self._vlm_instance = primaryreturn self._vlm_instance+ def _build_vlm_config_dict_for_credential(self, credential: VLMCredential) -> Dict[str, Any]:+ """Build VLM instance config dict for a specific credential."""+ result = {+ "model": credential.model or self.model,+ "temperature": self.temperature,+ "max_retries": self.max_retries,+ "timeout": self.timeout,+ "provider": credential.provider,+ "thinking": self.thinking,+ "max_tokens": self.max_tokens,+ "stream": credential.stream if credential.stream is not None else self.stream,+ "api_version": credential.api_version,+ }++ if credential.api_key:+ result["api_key"] = credential.api_key+ if credential.forward_api_key is not None:+ result["forward_api_key"] = credential.forward_api_key+ if credential.api_base:+ result["api_base"] = credential.api_base+ if credential.extra_headers:+ result["extra_headers"] = credential.extra_headers+ if credential.extra_request_body:+ result["extra_request_body"] = credential.extra_request_body++ return result+def _build_vlm_config_dict(self) -> Dict[str, Any]:"""Build VLM instance config dict."""config, name = self.get_provider_config()diff --git a/tests/unit/test_circuit_breaker.py b/tests/unit/test_circuit_breaker.pyindex 28bed02ab..b68a70336 100644--- a/tests/unit/test_circuit_breaker.py+++ b/tests/unit/test_circuit_breaker.py@@ -13,7 +13,7 @@ from openviking.utils.circuit_breaker import (classify_api_error,)from openviking.utils.model_retry import (- ERROR_CLASS_PERMANENT,+ ERROR_CLASS_AUTH,ERROR_CLASS_TRANSIENT,ERROR_CLASS_UNKNOWN,)@@ -22,30 +22,30 @@ from openviking.utils.model_retry import (class TestClassifyApiError:"""Test API error classification."""- def test_classify_403_as_permanent(self):- """Test 403 Forbidden is classified as permanent."""+ def test_classify_403_as_auth(self):+ """Test 403 Forbidden is classified as auth (credential-level)."""error = Exception("403 Forbidden")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTH- def test_classify_401_as_permanent(self):- """Test 401 Unauthorized is classified as permanent."""+ def test_classify_401_as_auth(self):+ """Test 401 Unauthorized is classified as auth (credential-level)."""error = Exception("401 Unauthorized")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTH- def test_classify_forbidden_as_permanent(self):- """Test 'Forbidden' string is classified as permanent."""+ def test_classify_forbidden_as_auth(self):+ """Test 'Forbidden' string is classified as auth."""error = Exception("Access Forbidden")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTH- def test_classify_unauthorized_as_permanent(self):- """Test 'Unauthorized' string is classified as permanent."""+ def test_classify_unauthorized_as_auth(self):+ """Test 'Unauthorized' string is classified as auth."""error = Exception("Unauthorized access")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTH- def test_classify_account_overdue_as_permanent(self):- """Test 'AccountOverdue' string is classified as permanent."""+ def test_classify_account_overdue_as_auth(self):+ """Test 'AccountOverdue' string is classified as auth."""error = Exception("AccountOverdue error")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTHdef test_classify_429_as_transient(self):"""Test 429 TooManyRequests is classified as transient."""@@ -106,7 +106,7 @@ class TestClassifyApiError:"""Test classification checks error cause."""error = Exception("Primary error")error.__cause__ = Exception("403 Forbidden")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTHdef test_classify_error_with_transient_cause(self):"""Test classification checks error cause for transient."""@@ -114,10 +114,10 @@ class TestClassifyApiError:error.__cause__ = Exception("429 Rate limit")assert classify_api_error(error) == ERROR_CLASS_TRANSIENT- def test_permanent_takes_precedence_over_transient(self):- """Test permanent error patterns are checked first."""+ def test_auth_takes_precedence_over_transient(self):+ """Test auth error patterns are checked before transient."""error = Exception("403 Forbidden 429 timeout")- assert classify_api_error(error) == ERROR_CLASS_PERMANENT+ assert classify_api_error(error) == ERROR_CLASS_AUTHclass TestCircuitBreaker:diff --git a/tests/unit/test_embedding_credential_model_override.py b/tests/unit/test_embedding_credential_model_override.pynew file mode 100644index 000000000..ff126efc3--- /dev/null+++ b/tests/unit/test_embedding_credential_model_override.py@@ -0,0 +1,362 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Tests for per-credential model override in EmbeddingModelConfig."""++from unittest.mock import patch++from openviking_cli.utils.config.embedding_config import (+ EmbeddingConfig,+ EmbeddingCredential,+ EmbeddingModelConfig,+)+++def test_per_credential_model_overrides_parent_model():+ """When a credential specifies its own `model`, the merged config used to+ build that credential's embedder must use the credential's model, not the+ parent EmbeddingModelConfig.model."""+ parent = EmbeddingModelConfig(+ model="parent-model",+ dimension=1024,+ provider="volcengine",+ api_key="parent-key",+ credentials=[+ EmbeddingCredential(+ id="cred-a",+ provider="volcengine",+ model="endpoint-a",+ api_key="key-a",+ api_base="https://example.com/a",+ ),+ EmbeddingCredential(+ id="cred-b",+ provider="volcengine",+ api_key="key-b",+ api_base="https://example.com/b",+ ),+ ],+ )+ cfg = EmbeddingConfig(dense=parent)++ captured_models: list[str | None] = []++ def fake_create_embedder(_self, _provider, _embedder_type, config):+ captured_models.append(config.model)+ return object()++ # Skip wrapping into a FailoverEmbedder; we only care about the merged+ # configs passed into _create_embedder for each credential.+ with (+ patch.object(EmbeddingConfig, "_create_embedder", fake_create_embedder),+ patch(+ "openviking.models.embedder.FailoverEmbedder",+ lambda **kwargs: kwargs,+ ),+ ):+ cfg._create_failover_embedder("dense", parent)++ assert captured_models == ["endpoint-a", "parent-model"]+++def test_credential_inherits_api_key_api_base_api_version_from_parent():+ """Regression: when a credential omits api_key/api_base/api_version, the+ merged config used to build that credential's embedder must fall back to+ the parent EmbeddingModelConfig fields. Validation already accepts this+ inheritance (cred.api_key or self.api_key); _create_failover_embedder+ must use the same fallback so EmbeddingModelConfig re-validation does not+ fail with 'OpenAI provider requires api_key to be set'.+ """+ parent = EmbeddingModelConfig(+ model="text-embedding-3-small",+ dimension=1536,+ provider="openai",+ api_key="parent-key",+ api_base="https://parent.example.com",+ api_version="2024-01-01",+ credentials=[+ # Credential 'a' omits api_key/api_base/api_version entirely;+ # they must be inherited from the parent.+ EmbeddingCredential(id="a"),+ # Credential 'b' overrides api_key but inherits api_base/api_version.+ EmbeddingCredential(id="b", api_key="cred-b-key"),+ ],+ )++ captured_configs: list[EmbeddingModelConfig] = []++ def fake_create_embedder(_self, _provider, _embedder_type, config):+ captured_configs.append(config)+ return object()++ with (+ patch.object(EmbeddingConfig, "_create_embedder", fake_create_embedder),+ patch(+ "openviking.models.embedder.FailoverEmbedder",+ lambda **kwargs: kwargs,+ ),+ ):+ # This call previously raised:+ # ValueError: OpenAI provider requires 'api_key' to be set+ # because merged_config received api_key=None for credential 'a'.+ EmbeddingConfig(dense=parent)._create_failover_embedder("dense", parent)++ assert len(captured_configs) == 2++ # Credential 'a': all auth fields fall back to parent.+ cfg_a = captured_configs[0]+ assert cfg_a.api_key == "parent-key"+ assert cfg_a.api_base == "https://parent.example.com"+ assert cfg_a.api_version == "2024-01-01"++ # Credential 'b': api_key overridden, api_base/api_version still inherited.+ cfg_b = captured_configs[1]+ assert cfg_b.api_key == "cred-b-key"+ assert cfg_b.api_base == "https://parent.example.com"+ assert cfg_b.api_version == "2024-01-01"+++def test_credential_without_inheritance_does_not_raise_at_get_embedder():+ """End-to-end variant: ensure get_embedder() does not re-validate to failure+ when credential omits api_key but parent has it."""+ parent = EmbeddingModelConfig(+ model="text-embedding-3-small",+ dimension=1536,+ provider="openai",+ api_key="parent-key",+ credentials=[EmbeddingCredential(id="a")],+ )+ cfg = EmbeddingConfig(dense=parent)++ captured: list[EmbeddingModelConfig] = []++ def fake_create_embedder(_self, _provider, _embedder_type, config):+ captured.append(config)+ return object()++ with (+ patch.object(EmbeddingConfig, "_create_embedder", fake_create_embedder),+ patch(+ "openviking.models.embedder.FailoverEmbedder",+ lambda **kwargs: kwargs,+ ),+ ):+ cfg._create_failover_embedder("dense", parent)++ # Must not raise; merged config must carry inherited api_key.+ assert len(captured) == 1+ assert captured[0].api_key == "parent-key"+++def test_effective_provider_falls_back_to_first_credential():+ """When parent provider is left at default (volcengine) but credentials+ specify a different provider, _effective_provider must return the+ credential's provider so dimension/metadata reflect reality."""+ cfg = EmbeddingModelConfig(+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk",+ )+ ],+ )++ # Parent provider is the model default ("volcengine"), but the actual+ # request will go to OpenAI.+ assert cfg.provider == "volcengine"+ assert cfg._effective_provider() == "openai"+ assert cfg._effective_model() == "text-embedding-3-small"+++def test_effective_model_prefers_credential_when_set():+ """_effective_model returns the first credential's model when given."""+ cfg = EmbeddingModelConfig(+ provider="openai",+ model="parent-model",+ api_key="sk",+ credentials=[+ EmbeddingCredential(id="a", model="cred-model"),+ EmbeddingCredential(id="b"),+ ],+ )+ assert cfg._effective_model() == "cred-model"+++def test_effective_model_falls_back_to_parent_when_credential_omits_model():+ """When credentials omit model, parent's model is used."""+ cfg = EmbeddingModelConfig(+ provider="openai",+ model="parent-model",+ api_key="sk",+ credentials=[EmbeddingCredential(id="a", api_key="sk-a")],+ )+ assert cfg._effective_model() == "parent-model"+++def test_get_effective_dimension_uses_credential_provider_and_model():+ """Regression: credentials-only OpenAI text-embedding-3-small must yield+ dimension=1536, not the default-provider fallback (2048).+ """+ cfg = EmbeddingModelConfig(+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk",+ )+ ],+ )+ # Was 2048 before the fix because parent provider defaulted to volcengine.+ assert cfg.get_effective_dimension() == 1536+++def test_get_effective_dimension_credential_text_embedding_3_large():+ """text-embedding-3-large via credential => 3072 dim."""+ cfg = EmbeddingModelConfig(+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-large",+ api_key="sk",+ )+ ],+ )+ assert cfg.get_effective_dimension() == 3072+++def test_get_effective_dimension_explicit_dimension_wins():+ """Explicit `dimension` always wins over provider/model heuristics."""+ cfg = EmbeddingModelConfig(+ dimension=4096,+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk",+ )+ ],+ )+ assert cfg.get_effective_dimension() == 4096+++def test_credentials_with_conflicting_dimensions_raise():+ """Multiple credentials whose models map to different dimensions must+ fail at startup to prevent silent vector-store corruption.+ """+ import pytest++ with pytest.raises(ValueError, match="conflicting vector dimensions"):+ EmbeddingModelConfig(+ credentials=[+ EmbeddingCredential(+ id="small",+ provider="openai",+ model="text-embedding-3-small", # 1536+ api_key="sk-1",+ ),+ EmbeddingCredential(+ id="large",+ provider="openai",+ model="text-embedding-3-large", # 3072+ api_key="sk-2",+ ),+ ],+ )+++def test_credentials_same_dimension_via_provider_and_model_pass():+ """Two credentials with identical provider+model resolve to same dimension."""+ cfg = EmbeddingModelConfig(+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk-1",+ ),+ EmbeddingCredential(+ id="b",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk-2",+ ),+ ],+ )+ assert cfg.get_effective_dimension() == 1536+++def test_explicit_parent_dimension_overrides_per_cred_resolution():+ """If parent.dimension is set, all credentials are pinned to that+ dimension and per-cred heuristic resolution is bypassed - so even+ seemingly conflicting providers/models are accepted.+ """+ cfg = EmbeddingModelConfig(+ dimension=1024,+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk-1",+ ),+ EmbeddingCredential(+ id="b",+ provider="openai",+ model="text-embedding-3-large",+ api_key="sk-2",+ ),+ ],+ )+ assert cfg.get_effective_dimension() == 1024+++def test_credentials_with_unknown_dimension_provider_pass():+ """Providers without a known dim lookup (e.g. volcengine) are skipped+ and do not block validation."""+ cfg = EmbeddingModelConfig(+ dimension=1024,+ credentials=[+ EmbeddingCredential(+ id="a",+ provider="volcengine",+ model="ep-A",+ api_key="ark-1",+ ),+ EmbeddingCredential(+ id="b",+ provider="volcengine",+ model="ep-B",+ api_key="ark-2",+ ),+ ],+ )+ assert cfg.get_effective_dimension() == 1024+++def test_cross_provider_credentials_with_compatible_dimensions_pass():+ """Two credentials on different providers but with the same dimension+ (e.g. cohere 1024 + voyage 1024 via explicit parent.dimension) are+ accepted; vector-space compatibility is the user's responsibility."""+ cfg = EmbeddingModelConfig(+ dimension=1024,+ credentials=[+ EmbeddingCredential(+ id="vk",+ provider="volcengine",+ model="ep-X",+ api_key="ark",+ ),+ EmbeddingCredential(+ id="oai",+ provider="openai",+ model="text-embedding-3-small",+ api_key="sk",+ ),+ ],+ )+ # Explicit parent dimension wins; openai's natural 1536 is overridden.+ assert cfg.get_effective_dimension() == 1024diff --git a/tests/unit/test_failover_embedder_permanent_advance.py b/tests/unit/test_failover_embedder_permanent_advance.pynew file mode 100644index 000000000..7f61bbafb--- /dev/null+++ b/tests/unit/test_failover_embedder_permanent_advance.py@@ -0,0 +1,224 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Tests for FailoverEmbedder behavior when credential-level errors occur.++In multi-credential mode, FailoverEmbedder advances on auth errors (HTTP+401/403) by default, since another credential may carry a valid key /+permission / balance. The last credential's auth error re-raises the original+exception.++Request-level errors (e.g. HTTP 400 parameter errors) fail fast and re-raise+immediately without trying other credentials, since the same request fails on+every credential of the same model.+"""++import time+from typing import List++import pytest++from openviking.models.embedder.base import EmbedderBase, EmbedResult, FailoverEmbedder+from openviking.utils.exceptions import AllCredentialsFailedError+++class _StubEmbedder(EmbedderBase):+ """Minimal embedder that returns a canned vector or raises a canned error."""++ def __init__(self, name: str, error: Exception | None = None, dim: int = 4):+ super().__init__(model_name=name, config={"provider": "stub"})+ self._error = error+ self._dim = dim+ self.calls = 0++ @property+ def supports_multimodal(self) -> bool:+ return False++ def get_dimension(self) -> int:+ return self._dim++ def embed(self, content, is_query: bool = False) -> EmbedResult:+ self.calls += 1+ if self._error is not None:+ raise self._error+ return EmbedResult(dense_vector=[0.0] * self._dim)++ def embed_batch(self, contents, is_query: bool = False) -> List[EmbedResult]:+ self.calls += 1+ if self._error is not None:+ raise self._error+ return [EmbedResult(dense_vector=[0.0] * self._dim) for _ in contents]++ async def embed_async(self, content, is_query: bool = False) -> EmbedResult:+ return self.embed(content, is_query=is_query)++ async def embed_batch_async(self, contents, is_query: bool = False) -> List[EmbedResult]:+ return self.embed_batch(contents, is_query=is_query)+++def _make_400_error() -> Exception:+ """Construct a request-level 400 parameter error (PERMANENT, fail-fast)."""+ return RuntimeError(+ "Error code: 400 - {'error': {'message': "+ "'The parameter `model` specified in the request are not valid'}}"+ )+++def _make_401_error() -> Exception:+ """Construct a credential-level 401 auth error (AUTH, advances in multi-credential)."""+ return RuntimeError("Error code: 401 - {'error': {'message': 'Incorrect API key provided'}}")+++def test_auth_error_on_primary_advances_to_backup():+ """In multi-credential mode a 401 on primary advances to backup automatically."""+ primary = _StubEmbedder("primary", error=_make_401_error())+ backup = _StubEmbedder("backup")++ fe = FailoverEmbedder(+ embedders=[primary, backup],+ credential_ids=["primary", "backup"],+ )++ result = fe.embed("hello")++ assert primary.calls == 1+ assert backup.calls == 1+ assert result.dense_vector == [0.0, 0.0, 0.0, 0.0]+++def test_auth_error_on_all_credentials_raises_aggregated():+ """When every credential fails with auth, the whole ring is tried then+ AllCredentialsFailedError is raised aggregating each failure."""+ primary = _StubEmbedder("primary", error=_make_401_error())+ backup = _StubEmbedder("backup", error=_make_401_error())++ fe = FailoverEmbedder(+ embedders=[primary, backup],+ credential_ids=["primary", "backup"],+ )++ with pytest.raises(AllCredentialsFailedError) as excinfo:+ fe.embed("hello")++ assert primary.calls == 1+ assert backup.calls == 1+ # The aggregated error records both failing credentials.+ assert len(excinfo.value.errors) == 2+++def test_permanent_400_fails_fast_without_trying_backup():+ """A request-level 400 fails fast: backup must NOT be tried, original is re-raised."""+ primary = _StubEmbedder("primary", error=_make_400_error())+ backup = _StubEmbedder("backup")++ fe = FailoverEmbedder(+ embedders=[primary, backup],+ credential_ids=["primary", "backup"],+ )++ with pytest.raises(RuntimeError, match="400"):+ fe.embed("hello")++ assert primary.calls == 1+ assert backup.calls == 0+++def test_three_credentials_advance_through_chain():+ """Across 3 credentials, two AUTH errors should land on credential #3."""+ cred0 = _StubEmbedder("cred0", error=_make_401_error())+ cred1 = _StubEmbedder("cred1", error=_make_401_error())+ cred2 = _StubEmbedder("cred2")++ fe = FailoverEmbedder(+ embedders=[cred0, cred1, cred2],+ credential_ids=["c0", "c1", "c2"],+ )++ result = fe.embed("hello")++ assert cred0.calls == 1+ assert cred1.calls == 1+ assert cred2.calls == 1+ assert result.dense_vector == [0.0] * 4+++def test_more_than_ten_credentials_all_tried():+ """With >10 credentials, every one is tried (no global retry cap cuts it short).++ Regression for the removed ``total_max_retries`` cap: exhaustion is decided+ solely by reaching the end of the credential chain.+ """+ n = 15+ failing = [_StubEmbedder(f"cred{i}", error=_make_401_error()) for i in range(n - 1)]+ last = _StubEmbedder(f"cred{n - 1}")+ embedders = failing + [last]++ fe = FailoverEmbedder(+ embedders=embedders,+ credential_ids=[f"c{i}" for i in range(n)],+ )++ result = fe.embed("hello")++ # Every failing credential was attempted exactly once, then the last succeeded.+ assert all(e.calls == 1 for e in failing)+ assert last.calls == 1+ assert result.dense_vector == [0.0] * 4+++def test_ring_wraps_when_active_is_last_and_unavailable():+ """If the active credential is the last one and it is down, the ring wraps+ around to earlier credentials within the same request (fast failover)."""+ good = _StubEmbedder("good") # idx 0, healthy+ down = _StubEmbedder("down", error=_make_401_error()) # idx 1, unavailable++ fe = FailoverEmbedder(embedders=[good, down], credential_ids=["good", "down"])++ # Force the active credential to the last (unavailable) one. Bump the+ # switch timestamp so maybe_failback() does not immediately retreat to 0+ # before we even try idx 1 (which is what we want to exercise here).+ fe._switcher._active_idx = 1+ fe._switcher._last_switch_time = time.monotonic()++ result = fe.embed("hello")++ # Started at idx 1 (down) -> failed -> wrapped to idx 0 (good) -> success.+ assert down.calls == 1+ assert good.calls == 1+ assert result.dense_vector == [0.0] * 4+++def test_fast_failover_commits_new_active_index():+ """A credential that serves the request after the active one was down+ becomes the new active credential (fast failover, choice (a))."""+ good = _StubEmbedder("good") # idx 0+ down = _StubEmbedder("down", error=_make_401_error()) # idx 1++ fe = FailoverEmbedder(embedders=[good, down], credential_ids=["good", "down"])+ fe._switcher._active_idx = 1+ fe._switcher._last_switch_time = time.monotonic()++ fe.embed("hello")++ # active index committed to the credential that actually worked (idx 0).+ assert fe._switcher.get_active_index() == 0+ # A subsequent request now starts directly at idx 0 without touching idx 1.+ good.calls = 0+ down.calls = 0+ fe.embed("hello again")+ assert good.calls == 1+ assert down.calls == 0+++def test_content_safety_does_not_cycle_the_ring():+ """Request-level content-safety errors fail fast and must not try others."""+ cs = _StubEmbedder("cs", error=RuntimeError("content_filter: sensitive content detected"))+ backup = _StubEmbedder("backup")++ fe = FailoverEmbedder(embedders=[cs, backup], credential_ids=["cs", "backup"])++ with pytest.raises(RuntimeError, match="content_filter"):+ fe.embed("hello")++ assert cs.calls == 1+ assert backup.calls == 0diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex 536d57992..016fbf460 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -5,6 +5,8 @@import pytestfrom openviking.utils.model_retry import (+ ERROR_CLASS_AUTH,+ ERROR_CLASS_CONTENT_SAFETY,ERROR_CLASS_INPUT_TOO_LARGE,ERROR_CLASS_PERMANENT,ERROR_CLASS_QUOTA_EXCEEDED,@@ -81,9 +83,58 @@ def test_quota_exceeded_takes_precedence_over_transient():assert classify_api_error(error) == ERROR_CLASS_QUOTA_EXCEEDED-def test_permanent_still_takes_precedence_over_quota():- """Permanent errors (e.g. 403) still take highest precedence."""- assert classify_api_error(RuntimeError("403 AccountQuotaExceeded")) == ERROR_CLASS_PERMANENT+def test_auth_takes_precedence_over_quota():+ """Auth errors (e.g. 403) take precedence over the quota substring."""+ assert classify_api_error(RuntimeError("403 AccountQuotaExceeded")) == ERROR_CLASS_AUTH+++# --- permanent vs auth split (400 vs 401/403) ---+++def test_classify_400_is_permanent():+ """A 400 parameter error is request-level permanent (fail-fast)."""+ error = RuntimeError("Error code: 400 - invalid parameter `model`")+ assert classify_api_error(error) == ERROR_CLASS_PERMANENT+++def test_classify_401_is_auth():+ """A 401 is a credential-level auth error (advances in multi-credential mode)."""+ assert classify_api_error(RuntimeError("Error code: 401 - Incorrect API key")) == (+ ERROR_CLASS_AUTH+ )+++def test_classify_403_is_auth():+ """A 403 forbidden is a credential-level auth error."""+ assert classify_api_error(RuntimeError("403 forbidden")) == ERROR_CLASS_AUTH+++def test_classify_unauthorized_is_auth():+ assert classify_api_error(RuntimeError("Unauthorized")) == ERROR_CLASS_AUTH+++def test_classify_account_overdue_is_auth():+ assert classify_api_error(RuntimeError("AccountOverdue")) == ERROR_CLASS_AUTH+++# --- content safety classification ---+++def test_classify_content_filter_is_content_safety():+ assert classify_api_error(RuntimeError("content_filter triggered")) == (+ ERROR_CLASS_CONTENT_SAFETY+ )+++def test_classify_content_policy_is_content_safety():+ error = RuntimeError("The response was rejected by the content policy")+ assert classify_api_error(error) == ERROR_CLASS_CONTENT_SAFETY+++def test_content_safety_takes_precedence_over_400():+ """A moderation rejection containing '400' is content_safety, not permanent."""+ error = RuntimeError("Error code: 400 - content_filter: sensitive content detected")+ assert classify_api_error(error) == ERROR_CLASS_CONTENT_SAFETYdef test_retry_sync_does_not_retry_quota_exceeded():diff --git a/tests/unit/test_ordered_credential_switcher.py b/tests/unit/test_ordered_credential_switcher.pynew file mode 100644index 000000000..5f3b2baef--- /dev/null+++ b/tests/unit/test_ordered_credential_switcher.py@@ -0,0 +1,347 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Unit tests for OrderedCredentialSwitcher"""++import threading+import time++import pytest++from openviking.utils.model_retry import (+ ERROR_CLASS_AUTH,+ ERROR_CLASS_CONTENT_SAFETY,+ ERROR_CLASS_INPUT_TOO_LARGE,+ ERROR_CLASS_PERMANENT,+ ERROR_CLASS_QUOTA_EXCEEDED,+ ERROR_CLASS_TRANSIENT,+ ERROR_CLASS_UNKNOWN,+ OrderedCredentialSwitcher,+)+++class TestOrderedCredentialSwitcher:+ """Tests for OrderedCredentialSwitcher class."""++ def test_initial_state(self):+ """Test initial state with single credential."""+ switcher = OrderedCredentialSwitcher(n=1)+ assert switcher.n == 1+ assert switcher.get_active_index() == 0+ assert not switcher.is_exhausted++ def test_initial_state_with_multiple_credentials(self):+ """Test initial state with multiple credentials."""+ switcher = OrderedCredentialSwitcher(n=3)+ assert switcher.n == 3+ assert switcher.get_active_index() == 0+ assert not switcher.is_exhausted++ def test_advance_on_quota_exceeded(self):+ """Test advancing to next credential on quota exceeded."""+ switcher = OrderedCredentialSwitcher(n=3)++ # Start at index 0+ assert switcher.get_active_index() == 0++ # Quota exceeded should advance+ result = switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert result is True+ assert switcher.get_active_index() == 1++ # Another quota exceeded should advance again+ result = switcher.on_failure(1, ERROR_CLASS_QUOTA_EXCEEDED)+ assert result is True+ assert switcher.get_active_index() == 2++ # One more should exhaust all+ result = switcher.on_failure(2, ERROR_CLASS_QUOTA_EXCEEDED)+ assert result is True+ assert switcher.is_exhausted++ def test_fail_fast_on_permanent_error(self):+ """PERMANENT (400 parameter error) always fails fast, even with multiple credentials.++ A 400 is request-level: the same request fails on every credential of+ the same model, so switching is useless.+ """+ switcher = OrderedCredentialSwitcher(n=3)++ result = switcher.on_failure(0, ERROR_CLASS_PERMANENT)+ assert result is False+ assert switcher.get_active_index() == 0 # Should stay at current index+ assert not switcher.is_exhausted++ def test_fail_fast_on_input_too_large(self):+ """INPUT_TOO_LARGE fails fast: same oversized input fails on every credential."""+ switcher = OrderedCredentialSwitcher(n=3)++ result = switcher.on_failure(0, ERROR_CLASS_INPUT_TOO_LARGE)+ assert result is False+ assert switcher.get_active_index() == 0+ assert not switcher.is_exhausted++ def test_fail_fast_on_content_safety(self):+ """CONTENT_SAFETY fails fast: moderation rejects the content on every credential."""+ switcher = OrderedCredentialSwitcher(n=3)++ result = switcher.on_failure(0, ERROR_CLASS_CONTENT_SAFETY)+ assert result is False+ assert switcher.get_active_index() == 0+ assert not switcher.is_exhausted++ def test_transient_error_treated_as_quota(self):+ """Test that transient errors are treated as quota exceeded."""+ switcher = OrderedCredentialSwitcher(n=3)++ # Transient should be treated as quota and advance+ result = switcher.on_failure(0, ERROR_CLASS_TRANSIENT)+ assert result is True+ assert switcher.get_active_index() == 1++ def test_unknown_error_advances(self):+ """Test that unknown errors cause advance."""+ switcher = OrderedCredentialSwitcher(n=3)++ result = switcher.on_failure(0, ERROR_CLASS_UNKNOWN)+ assert result is True+ assert switcher.get_active_index() == 1++ def test_on_success_increments_counter(self):+ """Test that success increments request counter when not at index 0."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=3)++ # Advance to index 1 first+ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert switcher.get_active_index() == 1++ # Success should increment counter+ switcher.on_success(1)+ switcher.on_success(1)+ switcher.on_success(1)++ # get_active_index is a pure read and must NOT trigger failback+ assert switcher.get_active_index() == 1+ # After 3 successes, maybe_failback should move back to index 0+ assert switcher.maybe_failback() == 0++ def test_failback_on_request_count(self):+ """Test failback based on request count threshold."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=2)++ # Advance to index 1+ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert switcher.get_active_index() == 1++ # Two successes should trigger failback+ switcher.on_success(1)+ switcher.on_success(1)++ # Next maybe_failback should move back+ assert switcher.maybe_failback() == 0++ def test_failback_on_timeout(self):+ """Test failback based on timeout threshold."""+ switcher = OrderedCredentialSwitcher(n=3, failback_timeout_seconds=0.1)++ # Advance to index 1+ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert switcher.get_active_index() == 1++ # Wait for timeout+ time.sleep(0.2)++ # Next maybe_failback should move back+ assert switcher.maybe_failback() == 0++ def test_get_active_index_is_pure_read(self):+ """get_active_index must never trigger failback, even when thresholds are met.++ Observers (logging/metrics reading active_credential_id/index) must not+ accidentally advance the failback state machine.+ """+ switcher = OrderedCredentialSwitcher(+ n=3, failback_request_count=1, failback_timeout_seconds=0.05+ )++ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert switcher.get_active_index() == 1++ # Meet BOTH thresholds (count + timeout)+ switcher.on_success(1)+ time.sleep(0.1)++ # Repeated pure reads must stay at index 1 (no failback side effect)+ assert switcher.get_active_index() == 1+ assert switcher.get_active_index() == 1++ # Only maybe_failback actually moves back+ assert switcher.maybe_failback() == 0++ def test_hierarchical_failback_one_step(self):+ """Test that failback moves one step at a time, not all the way."""+ switcher = OrderedCredentialSwitcher(n=4, failback_request_count=2)++ # Advance through indexes+ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED) # 0 -> 1+ switcher.on_failure(1, ERROR_CLASS_QUOTA_EXCEEDED) # 1 -> 2+ switcher.on_failure(2, ERROR_CLASS_QUOTA_EXCEEDED) # 2 -> 3+ assert switcher.get_active_index() == 3++ # Two successes should move back to 2+ switcher.on_success(3)+ switcher.on_success(3)+ assert switcher.maybe_failback() == 2++ # Two more successes should move back to 1+ switcher.on_success(2)+ switcher.on_success(2)+ assert switcher.maybe_failback() == 1++ def test_is_exhausted_when_all_credentials_used(self):+ """Test is_exhausted property."""+ switcher = OrderedCredentialSwitcher(n=2)++ assert not switcher.is_exhausted++ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED) # 0 -> 1+ assert not switcher.is_exhausted++ switcher.on_failure(1, ERROR_CLASS_QUOTA_EXCEEDED) # 1 -> 2 (exhausted)+ assert switcher.is_exhausted++ def test_success_does_nothing_at_index_zero(self):+ """Test that success at index 0 doesn't increment counter."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=2)++ # Success at index 0 should not affect anything+ switcher.on_success(0)+ switcher.on_success(0)+ switcher.on_success(0)++ # Should still be at index 0+ assert switcher.get_active_index() == 0++ def test_on_success_for_different_index_ignored(self):+ """Test that success for a different index is ignored."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=2)++ # Advance to index 1+ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED)+ assert switcher.get_active_index() == 1++ # Success for wrong index should be ignored+ switcher.on_success(0) # Wrong index+ switcher.on_success(0) # Wrong index++ # Should still be at index 1 (counter not incremented)+ assert switcher.get_active_index() == 1++ def test_thread_safety(self):+ """Test thread safety of the switcher."""+ switcher = OrderedCredentialSwitcher(n=10)+ results = []++ def worker():+ for _ in range(100):+ idx = switcher.get_active_index()+ if idx < switcher.n - 1:+ switcher.on_failure(idx, ERROR_CLASS_QUOTA_EXCEEDED)+ switcher.on_success(idx)+ results.append(idx)++ threads = []+ for _ in range(10):+ t = threading.Thread(target=worker)+ threads.append(t)+ t.start()++ for t in threads:+ t.join()++ # All operations should have completed without errors+ assert len(results) == 1000++ def test_invalid_initialization(self):+ """Test that n < 1 raises ValueError."""+ with pytest.raises(ValueError, match="Number of credentials must be >= 1"):+ OrderedCredentialSwitcher(n=0)++ with pytest.raises(ValueError, match="Number of credentials must be >= 1"):+ OrderedCredentialSwitcher(n=-1)++ def test_advance_on_auth_error_when_multi_credential(self):+ """Multi-credential mode: AUTH (401/403) advances to next credential by default."""+ switcher = OrderedCredentialSwitcher(n=3)++ # First credential's auth error should advance, not fail-fast+ result = switcher.on_failure(0, ERROR_CLASS_AUTH)+ assert result is True+ assert switcher.get_active_index() == 1++ # Middle credential's auth error should also advance+ result = switcher.on_failure(1, ERROR_CLASS_AUTH)+ assert result is True+ assert switcher.get_active_index() == 2++ def test_last_credential_auth_error_still_fails_fast(self):+ """In multi-credential mode the last credential still fails fast on AUTH."""+ switcher = OrderedCredentialSwitcher(n=2)++ # Advance from 0 -> 1 on auth error+ assert switcher.on_failure(0, ERROR_CLASS_AUTH) is True+ assert switcher.get_active_index() == 1++ # On the last credential (idx 1, n=2), still fail-fast+ result = switcher.on_failure(1, ERROR_CLASS_AUTH)+ assert result is False+ # active_idx stays at 1 (not bumped to n=2/exhausted) so caller can raise normally+ assert switcher.get_active_index() == 1+ assert not switcher.is_exhausted++ def test_single_credential_auth_error_fails_fast(self):+ """With only one credential, AUTH must fail-fast (it's the last)."""+ switcher = OrderedCredentialSwitcher(n=1)++ result = switcher.on_failure(0, ERROR_CLASS_AUTH)+ assert result is False+ assert switcher.get_active_index() == 0++ def test_is_fail_fast_classification(self):+ """Request-level errors are fail-fast; credential/quota/transient are not."""+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_PERMANENT) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_INPUT_TOO_LARGE) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_CONTENT_SAFETY) is True+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_AUTH) is False+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_QUOTA_EXCEEDED) is False+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_TRANSIENT) is False+ assert OrderedCredentialSwitcher.is_fail_fast(ERROR_CLASS_UNKNOWN) is False++ def test_commit_success_same_index_increments_failback_counter(self):+ """commit_success on the active (non-zero) index advances failback counter."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=2)++ switcher.on_failure(0, ERROR_CLASS_QUOTA_EXCEEDED) # active -> 1+ assert switcher.get_active_index() == 1++ switcher.commit_success(1)+ switcher.commit_success(1)+ # Two successes at the active index satisfy the failback threshold.+ assert switcher.maybe_failback() == 0++ def test_commit_success_different_index_fast_failover(self):+ """commit_success on a different index commits it as the new active one."""+ switcher = OrderedCredentialSwitcher(n=3)++ # active starts at 0; a later credential (idx 2) served the request.+ switcher.commit_success(2)+ assert switcher.get_active_index() == 2++ def test_commit_success_at_index_zero_no_counter(self):+ """commit_success at index 0 keeps active at 0 and does not count."""+ switcher = OrderedCredentialSwitcher(n=3, failback_request_count=1)++ switcher.commit_success(0)+ switcher.commit_success(0)+ assert switcher.get_active_index() == 0+ # Still at 0, nothing to fail back to.+ assert switcher.maybe_failback() == 0diff --git a/tests/unit/test_vlm_failover.py b/tests/unit/test_vlm_failover.pyindex fc662607b..d98302de5 100644--- a/tests/unit/test_vlm_failover.py+++ b/tests/unit/test_vlm_failover.py@@ -3,18 +3,19 @@"""Tests for VLM failover/backup configuration functionality."""import time+from unittest.mock import AsyncMock, Mock+import pytest-from unittest.mock import Mock, AsyncMock, patch-from openviking.models.vlm.base import FailoverVLM, VLMResponse, PrimaryBackupSwitcher+from openviking.models.vlm.base import FailoverVLM, PrimaryBackupSwitcherfrom openviking_cli.utils.config.vlm_config import VLMConfigclass TestVLMBackupConfig:- """Tests for VLMConfig backup field validation."""+ """Tests for VLMConfig backup field validation and migration."""- def test_backup_config_allowed(self):- """Test that backup configuration is allowed when not recursive."""+ def test_backup_config_migrated_to_credentials(self):+ """Test that backup configuration is migrated to credentials list."""backup_config = VLMConfig(model="backup-model",api_key="backup-key",@@ -26,11 +27,25 @@ class TestVLMBackupConfig:provider="volcengine",backup=backup_config,)- assert config.backup is not None- assert config.backup.model == "backup-model"-- def test_recursive_backup_config_rejected(self):- """Test that recursive backup configurations are rejected."""+ # After migration, backup should be None but credentials should have 2 entries+ assert config.backup is None+ assert len(config.credentials) == 2+ assert config.credentials[0].id == "legacy-primary"+ assert config.credentials[0].provider == "volcengine"+ assert config.credentials[0].api_key == "primary-key"+ assert config.credentials[0].model == "primary-model"+ assert config.credentials[1].id == "legacy-backup"+ assert config.credentials[1].provider == "openai"+ assert config.credentials[1].api_key == "backup-key"+ assert config.credentials[1].model == "backup-model"++ def test_recursive_backup_config_not_possible(self):+ """Test that recursive backup configurations are automatically prevented by migration.++ With the new multi-credential architecture, when a config with backup is created,+ it gets migrated to credentials format and backup is set to None. This means+ recursive backups are automatically prevented.+ """nested_backup = VLMConfig(model="nested-model",api_key="nested-key",@@ -42,13 +57,22 @@ class TestVLMBackupConfig:provider="openai",backup=nested_backup,)- with pytest.raises(ValueError, match="recursive backups are not allowed"):- VLMConfig(- model="primary-model",- api_key="primary-key",- provider="volcengine",- backup=backup_config,- )+ # After migration, backup_config.backup should be None+ assert backup_config.backup is None+ # And it should have 2 credentials (primary + backup)+ assert len(backup_config.credentials) == 2++ # Now creating a config with backup=backup_config works (no recursion)+ # because backup_config.backup is already None+ config = VLMConfig(+ model="primary-model",+ api_key="primary-key",+ provider="volcengine",+ backup=backup_config,+ )+ # The outer config gets its own 2 credentials (primary + backup_config's top-level)+ assert len(config.credentials) == 2+ assert config.backup is Nonedef test_backup_without_own_backup_allowed(self):"""Test that backup config without its own backup is allowed."""@@ -63,8 +87,267 @@ class TestVLMBackupConfig:provider="volcengine",backup=backup_config,)- # Should not raise- config.validate_no_recursive_backup()+ # Should not raise and should migrate to credentials+ assert len(config.credentials) == 2++ def test_per_credential_model_overrides_parent_model(self):+ """Each credential's `model` field should override parent VLMConfig.model+ when building the per-credential VLM config dict."""+ from openviking_cli.utils.config.vlm_config import VLMCredential++ config = VLMConfig(+ model="parent-model",+ credentials=[+ VLMCredential(+ id="cred-a",+ provider="volcengine",+ model="endpoint-a",+ api_key="key-a",+ api_base="https://example.com/a",+ ),+ VLMCredential(+ id="cred-b",+ provider="volcengine",+ api_key="key-b",+ api_base="https://example.com/b",+ ),+ ],+ )++ dict_a = config._build_vlm_config_dict_for_credential(config.credentials[0])+ dict_b = config._build_vlm_config_dict_for_credential(config.credentials[1])++ assert dict_a["model"] == "endpoint-a"+ # Falls back to parent model when credential.model is not set.+ assert dict_b["model"] == "parent-model"+++class TestLegacyProvidersDictMigration:+ """Tests for migrating legacy ``providers: {name: {...}}`` configs into credentials.++ Regression coverage for: when only ``providers`` was set (without top-level+ ``provider``/``api_key``), ``_normalize_credentials`` previously produced a+ credential with provider=None and api_key=None, which made ``is_available()``+ return False even though the config was valid in the legacy code path.+ """++ def test_providers_only_single_provider_migrates_provider_and_api_key(self):+ """``providers={openai: {api_key: sk}}`` should yield a usable default credential."""+ cfg = VLMConfig(+ model="gpt-4o-mini",+ providers={"openai": {"api_key": "sk-test", "api_base": "https://api.example.com"}},+ )++ assert len(cfg.credentials) == 1+ cred = cfg.credentials[0]+ assert cred.id == "default"+ assert cred.provider == "openai"+ assert cred.api_key == "sk-test"+ assert cred.api_base == "https://api.example.com"+ assert cred.model == "gpt-4o-mini"++ # is_available() must return True - this was the original blocking bug+ assert cfg.is_available() is True++ def test_providers_only_default_provider_picks_correct_one(self):+ """When ``default_provider`` is set, the matching providers entry is used."""+ cfg = VLMConfig(+ model="gpt-4o-mini",+ default_provider="openai",+ providers={+ "openai": {"api_key": "sk-openai"},+ "azure": {"api_key": "sk-azure", "api_base": "https://azure.example.com"},+ },+ )++ assert len(cfg.credentials) == 1+ cred = cfg.credentials[0]+ assert cred.provider == "openai"+ assert cred.api_key == "sk-openai"+ assert cfg.is_available() is True++ def test_top_level_provider_with_providers_dict_merges_correctly(self):+ """``provider=openai`` plus ``providers={openai: {api_key: ...}}`` works."""+ cfg = VLMConfig(+ model="gpt-4o",+ provider="openai",+ providers={"openai": {"api_key": "sk-merged"}},+ )++ assert len(cfg.credentials) == 1+ cred = cfg.credentials[0]+ assert cred.provider == "openai"+ assert cred.api_key == "sk-merged"+ assert cfg.is_available() is True++ def test_legacy_top_level_api_key_still_works(self):+ """Pure top-level legacy config (provider + api_key) keeps working."""+ cfg = VLMConfig(+ model="gpt-4o",+ provider="openai",+ api_key="sk-top-level",+ )++ assert len(cfg.credentials) == 1+ cred = cfg.credentials[0]+ assert cred.provider == "openai"+ assert cred.api_key == "sk-top-level"+ assert cfg.is_available() is True++ def test_providers_only_propagates_extra_fields(self):+ """extra_headers / extra_request_body / api_version / stream all propagate."""+ cfg = VLMConfig(+ model="gpt-4o",+ providers={+ "openai": {+ "api_key": "sk",+ "api_base": "https://api.example.com",+ "api_version": "2024-01-01",+ "extra_headers": {"X-Trace": "1"},+ "extra_request_body": {"foo": "bar"},+ "stream": True,+ }+ },+ )++ cred = cfg.credentials[0]+ assert cred.provider == "openai"+ assert cred.api_key == "sk"+ assert cred.api_base == "https://api.example.com"+ assert cred.api_version == "2024-01-01"+ assert cred.extra_headers == {"X-Trace": "1"}+ assert cred.extra_request_body == {"foo": "bar"}+ assert cred.stream is True++ def test_explicit_credentials_take_precedence_over_legacy(self):+ """When ``credentials`` is set, legacy providers dict is not migrated again."""+ cfg = VLMConfig(+ model="gpt-4o",+ credentials=[+ {"id": "explicit", "provider": "openai", "api_key": "sk-explicit"},+ ],+ providers={"openai": {"api_key": "sk-legacy"}},+ )++ assert len(cfg.credentials) == 1+ assert cfg.credentials[0].id == "explicit"+ assert cfg.credentials[0].api_key == "sk-explicit"++ def test_legacy_backup_with_primary_providers_dict_resolves_via_match_provider(self):+ """Primary side: ``providers={openai: {...}}`` + backup must yield a usable legacy-primary.++ Regression: previously, backup migration only read top-level+ ``self.provider/self.api_key`` and produced a credential with+ provider=None / api_key=None when the primary used the providers dict.+ """+ cfg = VLMConfig(+ model="gpt-4o",+ providers={"openai": {"api_key": "sk-primary"}},+ backup=VLMConfig(model="gpt-4o-mini", provider="openai", api_key="sk-backup"),+ )++ assert len(cfg.credentials) == 2+ primary = cfg.credentials[0]+ assert primary.id == "legacy-primary"+ assert primary.provider == "openai"+ assert primary.api_key == "sk-primary"+ assert primary.model == "gpt-4o"++ backup = cfg.credentials[1]+ assert backup.id == "legacy-backup"+ assert backup.provider == "openai"+ assert backup.api_key == "sk-backup"+ assert backup.model == "gpt-4o-mini"++ # is_available() must return True+ assert cfg.is_available() is True++ def test_legacy_backup_with_backup_providers_dict_resolves_via_match_provider(self):+ """Backup side: backup using ``providers={...}`` must produce usable legacy-backup."""+ cfg = VLMConfig(+ model="gpt-4o",+ provider="openai",+ api_key="sk-primary",+ backup=VLMConfig(+ model="gpt-4o-mini",+ providers={"openai": {"api_key": "sk-backup"}},+ ),+ )++ assert len(cfg.credentials) == 2+ primary = cfg.credentials[0]+ assert primary.provider == "openai"+ assert primary.api_key == "sk-primary"++ backup = cfg.credentials[1]+ assert backup.id == "legacy-backup"+ assert backup.provider == "openai"+ assert backup.api_key == "sk-backup"++ assert cfg.is_available() is True++ def test_legacy_backup_with_default_provider_resolves(self):+ """Backup using ``default_provider`` to disambiguate is migrated correctly."""+ cfg = VLMConfig(+ model="gpt-4o",+ providers={"openai": {"api_key": "sk-primary"}},+ backup=VLMConfig(+ model="gpt-4o-mini",+ default_provider="azure",+ providers={+ "openai": {"api_key": "sk-bk-openai"},+ "azure": {+ "api_key": "sk-bk-azure",+ "api_base": "https://azure.example.com",+ "api_version": "2024-01-01",+ },+ },+ ),+ )++ assert len(cfg.credentials) == 2+ backup = cfg.credentials[1]+ assert backup.provider == "azure"+ assert backup.api_key == "sk-bk-azure"+ assert backup.api_base == "https://azure.example.com"+ assert backup.api_version == "2024-01-01"++ assert cfg.is_available() is True++ def test_legacy_backup_propagates_extra_fields_via_match_provider(self):+ """Backup migration via providers dict propagates extra_headers / stream / etc."""+ cfg = VLMConfig(+ model="gpt-4o",+ providers={+ "openai": {+ "api_key": "sk-primary",+ "extra_headers": {"X-Trace": "p"},+ "stream": True,+ }+ },+ backup=VLMConfig(+ model="gpt-4o-mini",+ providers={+ "openai": {+ "api_key": "sk-backup",+ "extra_headers": {"X-Trace": "b"},+ "extra_request_body": {"foo": "bar"},+ "stream": False,+ }+ },+ ),+ )++ primary = cfg.credentials[0]+ assert primary.api_key == "sk-primary"+ assert primary.extra_headers == {"X-Trace": "p"}+ assert primary.stream is True++ backup = cfg.credentials[1]+ assert backup.api_key == "sk-backup"+ assert backup.extra_headers == {"X-Trace": "b"}+ assert backup.extra_request_body == {"foo": "bar"}+ assert backup.stream is Falseclass TestFailoverVLM:@@ -288,9 +571,7 @@ class TestFailoverVLM:primary.model = "primary-model"primary.provider = "volcengine"primary.get_completion_async = AsyncMock(- side_effect=Exception(- 'API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'- )+ side_effect=Exception('API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'))backup = Mock()@@ -313,23 +594,17 @@ class TestFailoverVLM:primary.model = "primary-model"primary.provider = "volcengine"primary.get_vision_completion_async = AsyncMock(- side_effect=Exception(- 'API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'- )+ side_effect=Exception('API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'))backup = Mock()backup.model = "backup-model"backup.provider = "openai"- backup.get_vision_completion_async = AsyncMock(- return_value="backup async vision response"- )+ backup.get_vision_completion_async = AsyncMock(return_value="backup async vision response")failover = FailoverVLM(primary, backup)- result = await failover.get_vision_completion_async(- prompt="describe", images=["test.jpg"]- )+ result = await failover.get_vision_completion_async(prompt="describe", images=["test.jpg"])assert result == "backup async vision response"@@ -356,8 +631,8 @@ class TestVLMConfigWithBackup:assert instance is mock_vlmmock_factory.create.assert_called_once()- def test_config_with_backup_creates_failover_instance(self, monkeypatch):- """Test that config with backup creates a FailoverVLM instance."""+ def test_config_with_backup_creates_multicredential_instance(self, monkeypatch):+ """Test that config with backup creates a MultiCredentialVLM instance."""mock_factory = Mock()mock_primary = Mock()mock_backup = Mock()@@ -379,9 +654,10 @@ class TestVLMConfigWithBackup:instance = config.get_vlm_instance()- # Should be a FailoverVLM instance- assert hasattr(instance, "primary")- assert hasattr(instance, "backup")+ # Should be a MultiCredentialVLM instance+ assert hasattr(instance, "active_credential_index")+ assert hasattr(instance, "_vlm_instances")+ assert len(instance._vlm_instances) == 2class TestPrimaryBackupSwitcher:@@ -597,10 +873,12 @@ class TestFailoverVLMAutomaticFailback:primary = Mock()primary.model = "primary-model"primary.provider = "volcengine"- primary.get_completion_async = AsyncMock(side_effect=[- Exception("quota exceeded"),- "primary async is back!",- ])+ primary.get_completion_async = AsyncMock(+ side_effect=[+ Exception("quota exceeded"),+ "primary async is back!",+ ]+ )backup = Mock()backup.model = "backup-model"