Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

39.4 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
import asyncio
import importlib
import threading
import weakref
from pathlib import Path
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union

from pydantic import BaseModel, Field, PrivateAttr, ValidationInfo, model_validator

from openviking_cli.utils.config.runtime_field import RuntimeField


def _load_codex_auth_module():
    importlib.import_module("openviking.models.vlm")
    return importlib.import_module("openviking.models.vlm.backends.codex_auth")


def _normalize_provider_name(name: Optional[str]) -> Optional[str]:
    if not isinstance(name, str):
        return name
    cleaned = name.strip().lower()
    return cleaned or None


def _reject_stream_config(data: Any, location: str) -> None:
    if not isinstance(data, dict):
        return
    if "stream" in data:
        raise ValueError(
            f"{location}.stream is not supported; OpenViking VLM calls return complete responses"
        )
    extra_request_body = data.get("extra_request_body")
    if isinstance(extra_request_body, dict) and "stream" in extra_request_body:
        raise ValueError(
            f"{location}.extra_request_body.stream is not supported; "
            "OpenViking VLM calls return complete responses"
        )


def _bind_token_usage_tracker(instance: Any, tracker: Any) -> None:
    """Bind one Account tracker to all concrete VLM instances in a wrapper."""
    if instance is None:
        return
    if hasattr(instance, "_token_tracker"):
        instance._token_tracker = tracker
    inner = getattr(instance, "inner", None)
    if inner is not None:
        _bind_token_usage_tracker(inner, tracker)
        return
    if hasattr(instance, "_vlm_instances"):
        for child in instance._vlm_instances:
            _bind_token_usage_tracker(child, tracker)
    else:
        for name in ("primary", "backup"):
            child = getattr(instance, name, None)
            if child is not None:
                _bind_token_usage_tracker(child, tracker)


class VLMCredential(BaseModel):
    """Single VLM credential configuration for multi-credential failover."""

    id: Optional[str] = RuntimeField(default=None, description="Unique identifier for this credential")
    provider: Optional[str] = RuntimeField(default=None, description="Provider type")
    model: Optional[str] = RuntimeField(
        default=None,
        description=(
            "Model name (or endpoint id) for this credential. "
            "Overrides the parent VLMConfig.model when set, allowing each credential "
            "to point to a different deployment / endpoint."
        ),
    )
    api_key: Optional[str] = RuntimeField(default=None, description="API key")
    api_base: Optional[str] = RuntimeField(default=None, description="API base URL")
    api_version: Optional[str] = RuntimeField(default=None, description="API version")
    forward_api_key: Optional[bool] = RuntimeField(
        default=None, description="Whether to pass api_key through to LiteLLM"
    )
    extra_headers: Optional[Dict[str, str]] = RuntimeField(
        default=None, description="Extra HTTP headers"
    )
    extra_request_body: Optional[Dict[str, Any]] = RuntimeField(
        default=None, description="Extra JSON body fields"
    )
    reasoning_effort: Optional[str] = RuntimeField(
        default=None,
        description="Reasoning effort for OpenAI-compatible reasoning models",
    )
    keepalive_expiry: Optional[float] = RuntimeField(
        default=None,
        ge=0.0,
        description="Idle HTTP connection lifetime for OpenAI-compatible providers",
    )
    max_tokens: Optional[int] = RuntimeField(
        default=None,
        gt=0,
        description=(
            "Maximum completion output tokens for this credential. "
            "Overrides the parent max_tokens when set."
        ),
    )

    @model_validator(mode="before")
    @classmethod
    def reject_stream_config(cls, data: Any) -> Any:
        _reject_stream_config(data, "vlm.credentials[]")
        return data


class VLMMediaConfig(BaseModel):
    """Runtime controls for audio and video inputs handled by the configured VLM."""

    enabled: bool = Field(default=False, description="Enable VLM audio/video understanding")
    max_concurrent: int = Field(
        default=2,
        ge=1,
        description="Maximum concurrent audio/video VLM calls",
    )
    file_processing_timeout: float = Field(
        default=1800.0,
        gt=0.0,
        description="Maximum provider-side media preprocessing wait in seconds",
    )
    file_poll_interval: float = Field(
        default=3.0,
        gt=0.0,
        description="Provider-side media preprocessing poll interval in seconds",
    )
    video_fps: float = Field(
        default=1.0,
        ge=0.2,
        le=5.0,
        description="Video frame sampling rate for providers that support preprocessing",
    )


class VLMConfig(BaseModel):
    """VLM configuration with multi-provider and multi-credential failover.

    Compatibility contract: top-level fields added here must describe common
    runtime behavior that is safe for Account VLM configurations to inherit
    from Cluster configuration. Fields that select a model service, credential,
    endpoint, or provider-specific request identity must instead be represented
    by ``AccountVLMConfig`` (normally ``model`` or ``VLMCredential``) and excluded by
    ``AccountVLMConfig._ACCOUNT_OWNED_MODEL_SERVICE_FIELDS``.
    """

    backup: Optional["VLMConfig"] = Field(
        default=None, description="Backup VLM configuration for failover (legacy)"
    )
    model: Optional[str] = Field(default=None, description="Model name")
    api_key: Optional[str] = Field(default=None, description="API key")
    forward_api_key: Optional[bool] = Field(
        default=None,
        description=(
            "Whether to pass api_key through to LiteLLM. None uses provider-aware defaults."
        ),
    )
    api_base: Optional[str] = Field(default=None, description="API base URL")
    temperature: float = Field(default=0.0, description="Generation temperature")
    max_retries: int = Field(default=3, description="Maximum retry attempts")
    timeout: float = Field(
        default=600.0,
        gt=0.0,
        description=(
            "Per-request HTTP timeout in seconds for VLM API calls. Applied to "
            "the underlying OpenAI/Azure/LiteLLM clients. Increase for slow or "
            "high-latency endpoints (e.g., DashScope, local inference servers)."
        ),
    )
    keepalive_expiry: Optional[float] = Field(
        default=None,
        ge=0.0,
        description=(
            "Idle HTTP connection lifetime in seconds for OpenAI-compatible VLM clients. "
            "Set to 0 to disable connection reuse; None uses the provider SDK default."
        ),
    )

    provider: Optional[str] = Field(default=None, description="Provider type")
    backend: Optional[str] = Field(
        default=None, description="Backend provider (Deprecated, use 'provider' instead)"
    )

    providers: Dict[str, Dict[str, Any]] = Field(
        default_factory=dict,
        description="Multi-provider configuration, e.g. {'openai': {'api_key': 'xxx', 'api_base': 'xxx'}}",
    )

    default_provider: Optional[str] = Field(default=None, description="Default provider name")

    max_tokens: Optional[int] = Field(
        default=None,
        description="Maximum tokens for VLM completion output (None = provider default)",
    )

    thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")

    max_concurrent: int = Field(
        default=32, description="Maximum number of concurrent LLM calls for semantic processing"
    )

    media: VLMMediaConfig = Field(
        default_factory=VLMMediaConfig,
        description="Audio/video runtime controls for the configured VLM",
    )

    api_version: Optional[str] = Field(
        default=None,
        description="API version for Azure OpenAI (e.g., '2025-01-01-preview').",
    )

    extra_headers: Optional[Dict[str, str]] = Field(
        default=None, description="Extra HTTP headers for OpenAI-compatible providers"
    )

    extra_request_body: Optional[Dict[str, Any]] = Field(
        default=None,
        description=(
            "Extra JSON body fields passed to OpenAI-compatible VLM completion requests. "
            "Useful for provider-specific options such as Ollama's {'think': false}."
        ),
    )

    reasoning_effort: Optional[str] = Field(
        default=None,
        description="Reasoning effort for OpenAI-compatible reasoning models",
    )

    # New multi-credential configuration
    credentials: List[VLMCredential] = Field(
        default_factory=list,
        description="Ordered list of credentials for failover. Call order matches array index (0 is highest priority).",
    )

    failback_timeout_seconds: float = Field(
        default=600.0, description="Time in seconds after which to attempt failback to primary"
    )
    failback_request_count: int = Field(
        default=50, description="Number of backup requests after which to attempt failback"
    )

    _vlm_instance: Optional[Any] = None
    _token_usage_tracker: Any = PrivateAttr(default=None)
    _media_semaphores: weakref.WeakKeyDictionary[
        asyncio.AbstractEventLoop,
        weakref.ReferenceType[asyncio.Semaphore],
    ] = PrivateAttr(default_factory=weakref.WeakKeyDictionary)
    _media_semaphore_lock: threading.Lock = PrivateAttr(default_factory=threading.Lock)

    model_config = {"arbitrary_types_allowed": True}

    @model_validator(mode="before")
    @classmethod
    def sync_provider_backend(cls, data: Any) -> Any:
        if isinstance(data, dict):
            _reject_stream_config(data, "vlm")
            provider = data.get("provider")
            backend = data.get("backend")

            if backend is not None and provider is None:
                data["provider"] = backend
            data["provider"] = _normalize_provider_name(data.get("provider"))
            data["backend"] = _normalize_provider_name(data.get("backend"))
            data["default_provider"] = _normalize_provider_name(data.get("default_provider"))
            providers = data.get("providers")
            if isinstance(providers, dict):
                normalized: Dict[str, Dict[str, Any]] = {}
                provider_sources: Dict[str, str] = {}
                for name, config in providers.items():
                    normalized_name = _normalize_provider_name(name) or str(name)
                    _reject_stream_config(config, f"vlm.providers.{normalized_name}")
                    existing_name = provider_sources.get(normalized_name)
                    if existing_name is not None and existing_name != str(name):
                        raise ValueError(
                            "Duplicate VLM provider config after normalization: "
                            f"{existing_name} and {name}"
                        )
                    normalized[normalized_name] = config
                    provider_sources[normalized_name] = str(name)
                data["providers"] = normalized
        return data

    @model_validator(mode="after")
    def validate_config(self, info: ValidationInfo):
        """Validate configuration completeness and consistency"""
        # Validate recursive backup BEFORE normalizing credentials (which clears backup)
        self._validate_no_recursive_backup()

        self._migrate_legacy_config()
        self._normalize_credentials()

        skip_codex_auth_availability = bool(
            info.context and info.context.get("skip_codex_auth_availability")
        )

        if self._has_any_config():
            if not self.model:
                raise ValueError("VLM configuration requires 'model' to be set")
            # When credentials are configured, validate each credential
            if self.credentials:
                for i, cred in enumerate(self.credentials):
                    provider_name = cred.provider or self.provider
                    if provider_name == "openai-codex":
                        has_codex_auth_available = (
                            _load_codex_auth_module().has_codex_auth_available
                        )
                        if (
                            not cred.api_key
                            and not skip_codex_auth_availability
                            and not has_codex_auth_available()
                        ):
                            raise ValueError(
                                f"Credential {i} ({cred.id or 'unnamed'}): requires Codex OAuth credentials in ~/.openviking/codex_auth.json"
                            )
                    elif provider_name not in ("litellm", None) and not cred.api_key:
                        # Also check providers dict for fallback
                        if not self._get_credential_api_key(cred):
                            raise ValueError(
                                f"Credential {i} ({cred.id or 'unnamed'}): requires 'api_key' to be set"
                            )
            else:
                # Legacy validation
                provider_name = self._resolve_provider_name()
                if provider_name == "openai-codex":
                    has_codex_auth_available = _load_codex_auth_module().has_codex_auth_available
                    if (
                        not self._get_effective_api_key()
                        and not skip_codex_auth_availability
                        and not has_codex_auth_available()
                    ):
                        raise ValueError(
                            "VLM configuration requires Codex OAuth credentials in ~/.openviking/codex_auth.json or an importable Codex CLI auth file"
                        )
                elif provider_name != "litellm" and not self._get_effective_api_key():
                    raise ValueError("VLM configuration requires 'api_key' to be set")
        return self

    def _validate_no_recursive_backup(self):
        """Prevent recursive backup configurations"""
        if self.backup is not None:
            if self.backup.backup is not None:
                raise ValueError(
                    "Backup VLM configuration cannot have its own backup (recursive backups are not allowed)"
                )

    def _migrate_legacy_config(self):
        """Migrate legacy config to providers structure."""
        if self.provider and (
            self.api_key
            or self.api_base
            or self.extra_headers
            or self.extra_request_body
            or self.reasoning_effort
            or self.keepalive_expiry is not None
            or self.forward_api_key is not None
        ):
            if self.provider not in self.providers:
                self.providers[self.provider] = {}
            if self.api_key and "api_key" not in self.providers[self.provider]:
                self.providers[self.provider]["api_key"] = self.api_key
            if (
                self.forward_api_key is not None
                and "forward_api_key" not in self.providers[self.provider]
            ):
                self.providers[self.provider]["forward_api_key"] = self.forward_api_key
            if self.api_base and "api_base" not in self.providers[self.provider]:
                self.providers[self.provider]["api_base"] = self.api_base
            if self.extra_headers and "extra_headers" not in self.providers[self.provider]:
                self.providers[self.provider]["extra_headers"] = self.extra_headers
            if (
                self.extra_request_body
                and "extra_request_body" not in self.providers[self.provider]
            ):
                self.providers[self.provider]["extra_request_body"] = self.extra_request_body
            if self.reasoning_effort and "reasoning_effort" not in self.providers[self.provider]:
                self.providers[self.provider]["reasoning_effort"] = self.reasoning_effort
            if (
                self.keepalive_expiry is not None
                and "keepalive_expiry" not in self.providers[self.provider]
            ):
                self.providers[self.provider]["keepalive_expiry"] = self.keepalive_expiry

    def _normalize_credentials(self):
        """Normalize credentials configuration:
        1. Migrate legacy backup config to credentials
        2. Migrate top-level credentials to credentials list
        3. Assign default IDs to credentials without IDs
        4. Apply top-level defaults to credentials
        """
        migrated_credentials = []

        # Step 1: Migrate legacy backup config
        if self.backup is not None and self.backup._has_any_config():
            # Primary credential: resolve via _match_provider() so legacy
            # ``providers: {openai: {...}}`` configs (without top-level
            # provider/api_key) are migrated correctly.
            primary_cfg, primary_provider = self._match_provider()
            primary_cfg = primary_cfg or {}
            primary_cred = VLMCredential(
                id="legacy-primary",
                provider=primary_provider or self.provider,
                model=self.model,
                api_key=primary_cfg.get("api_key") or self.api_key,
                api_base=primary_cfg.get("api_base") or self.api_base,
                api_version=primary_cfg.get("api_version") or self.api_version,
                forward_api_key=(
                    primary_cfg.get("forward_api_key")
                    if primary_cfg.get("forward_api_key") is not None
                    else self.forward_api_key
                ),
                extra_headers=primary_cfg.get("extra_headers") or self.extra_headers,
                extra_request_body=(
                    primary_cfg.get("extra_request_body") or self.extra_request_body
                ),
                reasoning_effort=(primary_cfg.get("reasoning_effort") or self.reasoning_effort),
                keepalive_expiry=(
                    primary_cfg.get("keepalive_expiry")
                    if primary_cfg.get("keepalive_expiry") is not None
                    else self.keepalive_expiry
                ),
                max_tokens=self.max_tokens,
            )
            migrated_credentials.append(primary_cred)

            # Backup credential: same resolution rules so backup using
            # providers/default_provider also gets a usable api_key/provider.
            backup_cfg, backup_provider = self.backup._match_provider()
            backup_cfg = backup_cfg or {}
            backup_cred = VLMCredential(
                id="legacy-backup",
                provider=backup_provider or self.backup.provider,
                model=self.backup.model,
                api_key=backup_cfg.get("api_key") or self.backup.api_key,
                api_base=backup_cfg.get("api_base") or self.backup.api_base,
                api_version=backup_cfg.get("api_version") or self.backup.api_version,
                forward_api_key=(
                    backup_cfg.get("forward_api_key")
                    if backup_cfg.get("forward_api_key") is not None
                    else self.backup.forward_api_key
                ),
                extra_headers=backup_cfg.get("extra_headers") or self.backup.extra_headers,
                extra_request_body=(
                    backup_cfg.get("extra_request_body") or self.backup.extra_request_body
                ),
                reasoning_effort=(
                    backup_cfg.get("reasoning_effort") or self.backup.reasoning_effort
                ),
                keepalive_expiry=(
                    backup_cfg.get("keepalive_expiry")
                    if backup_cfg.get("keepalive_expiry") is not None
                    else self.backup.keepalive_expiry
                ),
                max_tokens=self.backup.max_tokens,
            )
            migrated_credentials.append(backup_cred)

            # Clear backup to avoid double processing
            self.backup = None

        # Step 2: If no credentials from backup migration and no explicit credentials,
        # create credentials from top-level config. We resolve the effective provider
        # config first (via _match_provider) so legacy ``providers: {openai: {...}}``
        # configurations - where the api_key/api_base live inside the providers dict
        # rather than at the top level - are migrated correctly. Otherwise the
        # generated credential would be missing provider/api_key and is_available()
        # would return False.
        if not migrated_credentials and not self.credentials:
            if self._has_legacy_provider_config():
                provider_cfg, provider_name = self._match_provider()
                provider_cfg = provider_cfg or {}
                migrated_credentials.append(
                    VLMCredential(
                        id="default",
                        provider=provider_name or self.provider,
                        model=self.model,
                        api_key=provider_cfg.get("api_key") or self.api_key,
                        api_base=provider_cfg.get("api_base") or self.api_base,
                        api_version=provider_cfg.get("api_version") or self.api_version,
                        forward_api_key=(
                            provider_cfg.get("forward_api_key")
                            if provider_cfg.get("forward_api_key") is not None
                            else self.forward_api_key
                        ),
                        extra_headers=provider_cfg.get("extra_headers") or self.extra_headers,
                        extra_request_body=(
                            provider_cfg.get("extra_request_body") or self.extra_request_body
                        ),
                        reasoning_effort=(
                            provider_cfg.get("reasoning_effort") or self.reasoning_effort
                        ),
                        keepalive_expiry=(
                            provider_cfg.get("keepalive_expiry")
                            if provider_cfg.get("keepalive_expiry") is not None
                            else self.keepalive_expiry
                        ),
                    )
                )

        # Step 3: Merge migrated credentials with existing credentials
        if migrated_credentials:
            # Only use migrated if no explicit credentials exist
            if not self.credentials:
                self.credentials = migrated_credentials

        # Step 4: Assign default IDs and apply top-level defaults
        for i, cred in enumerate(self.credentials):
            if not cred.id:
                cred.id = f"credential-{i}"
            # Apply top-level defaults where credential doesn't specify
            if not cred.provider:
                cred.provider = self.provider
            if not cred.api_key:
                cred.api_key = self._get_credential_api_key(cred)
            if not cred.api_base:
                cred.api_base = self.api_base
            if not cred.api_version:
                cred.api_version = self.api_version
            if cred.forward_api_key is None:
                cred.forward_api_key = self.forward_api_key
            if not cred.extra_headers:
                cred.extra_headers = self.extra_headers
            if not cred.extra_request_body:
                cred.extra_request_body = self.extra_request_body
            if not cred.reasoning_effort:
                cred.reasoning_effort = self.reasoning_effort
            if cred.keepalive_expiry is None:
                cred.keepalive_expiry = self.keepalive_expiry

    def _has_legacy_provider_config(self) -> bool:
        """Check if there's legacy provider config (not credentials-based)."""
        return (
            self.provider is not None
            or self.api_key is not None
            or self.api_base is not None
            or self.providers
        )

    def _get_credential_api_key(self, cred: VLMCredential) -> str | None:
        """Get effective API key for a credential, checking providers dict."""
        if cred.api_key:
            return cred.api_key
        if cred.provider and cred.provider in self.providers:
            return self.providers[cred.provider].get("api_key")
        return None

    def _has_any_config(self) -> bool:
        """Check if any config is provided."""
        if self.credentials:
            return True
        if self.api_key or self.model or self.api_base or self.provider or self.default_provider:
            return True
        if self.providers:
            return True
        return False

    def _get_effective_api_key(self) -> str | None:
        """Get effective API key."""
        if self.credentials:
            return self.credentials[0].api_key or self._get_credential_api_key(self.credentials[0])
        if self.api_key:
            return self.api_key
        config, _ = self._match_provider()
        if config and config.get("api_key"):
            return config["api_key"]
        return None

    def _get_provider_config_by_name(self, provider_name: str) -> Dict[str, Any]:
        config = dict(self.providers.get(provider_name) or {})
        if self.api_key and "api_key" not in config:
            config["api_key"] = self.api_key
        if self.forward_api_key is not None and "forward_api_key" not in config:
            config["forward_api_key"] = self.forward_api_key
        if self.api_base and "api_base" not in config:
            config["api_base"] = self.api_base
        if self.extra_headers and "extra_headers" not in config:
            config["extra_headers"] = self.extra_headers
        if self.extra_request_body and "extra_request_body" not in config:
            config["extra_request_body"] = self.extra_request_body
        if self.reasoning_effort and "reasoning_effort" not in config:
            config["reasoning_effort"] = self.reasoning_effort
        if self.keepalive_expiry is not None and "keepalive_expiry" not in config:
            config["keepalive_expiry"] = self.keepalive_expiry
        return config

    def _provider_has_usable_credentials(self, provider_name: str, config: Dict[str, Any]) -> bool:
        if config.get("api_key"):
            return True
        if provider_name == "litellm":
            return True
        if provider_name == "openai-codex":
            has_codex_auth_available = _load_codex_auth_module().has_codex_auth_available

            return has_codex_auth_available()
        return False

    def _get_provider_config_from_credential(self, cred: VLMCredential) -> Dict[str, Any]:
        config: Dict[str, Any] = {}
        if cred.api_key:
            config["api_key"] = cred.api_key
        if cred.api_base:
            config["api_base"] = cred.api_base
        if cred.api_version:
            config["api_version"] = cred.api_version
        if cred.forward_api_key is not None:
            config["forward_api_key"] = cred.forward_api_key
        if cred.extra_headers:
            config["extra_headers"] = cred.extra_headers
        if cred.extra_request_body:
            config["extra_request_body"] = cred.extra_request_body
        if cred.reasoning_effort:
            config["reasoning_effort"] = cred.reasoning_effort
        if cred.keepalive_expiry is not None:
            config["keepalive_expiry"] = cred.keepalive_expiry
        return config

    def _match_provider(self, model: str | None = None) -> tuple[Dict[str, Any] | None, str | None]:
        """Match provider config.

        Returns:
            (provider_config_dict, provider_name)
        """
        del model
        # If credentials are configured, use the first one
        if self.credentials:
            cred = self.credentials[0]
            return self._get_provider_config_from_credential(cred), cred.provider

        if self.provider:
            return self._get_provider_config_by_name(self.provider) or {}, self.provider

        if self.default_provider:
            return (
                self._get_provider_config_by_name(self.default_provider) or {},
                self.default_provider,
            )

        if len(self.providers) == 1:
            provider_name = next(iter(self.providers))
            return self._get_provider_config_by_name(provider_name), provider_name

        for provider_name in self.providers:
            config = self._get_provider_config_by_name(provider_name)
            if self._provider_has_usable_credentials(provider_name, config):
                return config, provider_name

        return None, None

    def _resolve_provider_name(self) -> str | None:
        """Resolve provider name from credentials or legacy config."""
        if self.credentials:
            return self.credentials[0].provider
        _, name = self._match_provider()
        return name

    def get_provider_config(
        self, model: str | None = None
    ) -> tuple[Dict[str, Any] | None, str | None]:
        """Get provider config.

        Returns:
            (provider_config_dict, provider_name)
        """
        return self._match_provider(model)

    def get_vlm_instance(self) -> Any:
        """Get VLM instance with multi-credential failover support."""
        if self._vlm_instance is None:
            from openviking.models.vlm import FailoverVLM, MultiCredentialVLM, VLMFactory

            if self.credentials:
                # Build VLM instances for each credential
                vlm_instances = []
                for cred in self.credentials:
                    config_dict = self._build_vlm_config_dict_for_credential(cred)
                    vlm_instances.append(VLMFactory.create(config_dict))

                if len(vlm_instances) == 1:
                    self._vlm_instance = vlm_instances[0]
                else:
                    self._vlm_instance = MultiCredentialVLM(
                        vlm_instances,
                        credential_ids=[c.id for c in self.credentials],
                        failback_timeout_seconds=self.failback_timeout_seconds,
                        failback_request_count=self.failback_request_count,
                    )
            else:
                # Legacy mode: single VLM with optional backup
                config_dict = self._build_vlm_config_dict()
                primary = VLMFactory.create(config_dict)

                if self.backup is not None and self.backup._has_any_config():
                    backup_config_dict = self.backup._build_vlm_config_dict()
                    backup = VLMFactory.create(backup_config_dict)
                    self._vlm_instance = FailoverVLM(primary, backup)
                else:
                    self._vlm_instance = primary

            if self._token_usage_tracker is not None:
                _bind_token_usage_tracker(self._vlm_instance, self._token_usage_tracker)

        return self._vlm_instance

    def set_token_usage_tracker(self, tracker: Any) -> None:
        """Bind an externally owned tracker without eagerly creating a client."""
        if self._token_usage_tracker is tracker:
            return
        self._token_usage_tracker = tracker
        if self._vlm_instance is not None:
            _bind_token_usage_tracker(self._vlm_instance, tracker)

    def close(self) -> None:
        """Close and clear the cached VLM instance."""
        instance = self._vlm_instance
        self._vlm_instance = None
        if instance is not None:
            instance.close()

    def __del__(self) -> None:
        """Release a lazily-created client when the config is no longer referenced."""
        try:
            self.close()
        except Exception:
            pass

    def build_vlm_config_dict_for_credential(self, credential: VLMCredential) -> Dict[str, Any]:
        """Return the effective backend settings for one normalized credential."""
        return self._build_vlm_config_dict_for_credential(credential)

    def _build_vlm_config_dict_for_credential(self, credential: VLMCredential) -> Dict[str, Any]:
        """Build VLM instance config dict for a specific credential."""
        result = {
            "model": credential.model or self.model,
            "temperature": self.temperature,
            "max_retries": self.max_retries,
            "timeout": self.timeout,
            "keepalive_expiry": credential.keepalive_expiry,
            "provider": credential.provider,
            "thinking": self.thinking,
            "max_tokens": (
                credential.max_tokens if credential.max_tokens is not None else self.max_tokens
            ),
            "api_version": credential.api_version,
            "media": self.media.model_dump(),
        }

        if credential.api_key:
            result["api_key"] = credential.api_key
        if credential.forward_api_key is not None:
            result["forward_api_key"] = credential.forward_api_key
        if credential.api_base:
            result["api_base"] = credential.api_base
        if credential.extra_headers:
            result["extra_headers"] = credential.extra_headers
        if credential.extra_request_body:
            result["extra_request_body"] = credential.extra_request_body
        if credential.reasoning_effort:
            result["reasoning_effort"] = credential.reasoning_effort

        return result

    def _build_vlm_config_dict(self) -> Dict[str, Any]:
        """Build VLM instance config dict."""
        config, name = self.get_provider_config()

        result = {
            "model": self.model,
            "temperature": self.temperature,
            "max_retries": self.max_retries,
            "timeout": self.timeout,
            "keepalive_expiry": (
                config.get("keepalive_expiry")
                if config and config.get("keepalive_expiry") is not None
                else self.keepalive_expiry
            ),
            "provider": name,
            "thinking": self.thinking,
            "max_tokens": self.max_tokens,
            "api_version": self.api_version,
            "media": self.media.model_dump(),
        }

        if config:
            if config.get("api_key"):
                result["api_key"] = config.get("api_key")
            if config.get("forward_api_key") is not None:
                result["forward_api_key"] = config.get("forward_api_key")
            if config.get("api_base"):
                result["api_base"] = config.get("api_base")
            if config.get("extra_headers"):
                result["extra_headers"] = config.get("extra_headers")
            if config.get("extra_request_body"):
                result["extra_request_body"] = config.get("extra_request_body")
            if config.get("reasoning_effort"):
                result["reasoning_effort"] = config.get("reasoning_effort")

        return result

    def get_completion(
        self,
        prompt: str = "",
        thinking: Optional[bool] = None,
        tools: Optional[List[Dict[str, Any]]] = None,
        tool_choice: Optional[Any] = None,
        messages: Optional[List[Dict[str, Any]]] = None,
    ) -> Union[str, Any]:
        """Get LLM completion."""
        effective_thinking = self.thinking if thinking is None else thinking
        return self.get_vlm_instance().get_completion(
            prompt=prompt,
            thinking=effective_thinking,
            tools=tools,
            tool_choice=tool_choice,
            messages=messages,
        )

    async def get_completion_async(
        self,
        prompt: str = "",
        thinking: Optional[bool] = None,
        tools: Optional[List[Dict[str, Any]]] = None,
        tool_choice: Optional[Any] = None,
        messages: Optional[List[Dict[str, Any]]] = None,
        max_tokens: Optional[int] = None,
    ) -> Union[str, Any]:
        """Get LLM completion asynchronously."""
        effective_thinking = self.thinking if thinking is None else thinking
        return await self.get_vlm_instance().get_completion_async(
            prompt=prompt,
            thinking=effective_thinking,
            tools=tools,
            tool_choice=tool_choice,
            messages=messages,
            max_tokens=max_tokens,
        )

    def is_available(self) -> bool:
        """Check if LLM is configured."""
        if self._resolve_provider_name() == "openai-codex":
            has_codex_auth_available = _load_codex_auth_module().has_codex_auth_available

            return bool(self._get_effective_api_key() or has_codex_auth_available())
        if self._resolve_provider_name() == "litellm":
            return bool(self.model)
        return self._get_effective_api_key() is not None

    def get_vision_completion(
        self,
        prompt: str = "",
        images: Optional[list] = None,
        thinking: Optional[bool] = None,
        tools: Optional[List[Dict[str, Any]]] = None,
        tool_choice: Optional[Any] = None,
        messages: Optional[List[Dict[str, Any]]] = None,
    ) -> Union[str, Any]:
        """Get LLM completion with images."""
        effective_thinking = self.thinking if thinking is None else thinking
        return self.get_vlm_instance().get_vision_completion(
            prompt=prompt,
            images=images,
            thinking=effective_thinking,
            tools=tools,
            tool_choice=tool_choice,
            messages=messages,
        )

    def _get_media_semaphore(self) -> asyncio.Semaphore:
        loop = asyncio.get_running_loop()
        with self._media_semaphore_lock:
            semaphore_ref = self._media_semaphores.get(loop)
            semaphore = semaphore_ref() if semaphore_ref is not None else None
            if semaphore is None:
                semaphore = asyncio.Semaphore(self.media.max_concurrent)
                self._media_semaphores[loop] = weakref.ref(semaphore)
            return semaphore

    def supports_media(
        self,
        *,
        media_type: str,
        filename: str,
        size_bytes: int,
    ) -> bool:
        """Return whether enabled VLM media processing supports this input."""
        if not self.media.enabled or not self.is_available():
            return False
        return bool(
            self.get_vlm_instance().supports_media(
                media_type=media_type,
                filename=filename,
                size_bytes=size_bytes,
            )
        )

    async def get_media_completion_async(
        self,
        *,
        prompt: str,
        media_path: Path,
        filename: str,
        media_type: str,
        prepare_media: Optional[Callable[[], Awaitable[None]]] = None,
    ) -> str:
        """Stage and understand audio/video under the configured media limit."""
        from openviking.models.vlm.base import UnsupportedMediaInputError

        if not self.media.enabled:
            raise UnsupportedMediaInputError("VLM media understanding is disabled")

        async with self._get_media_semaphore():
            if prepare_media is not None:
                await prepare_media()
            return await self.get_vlm_instance().get_media_completion_async(
                prompt=prompt,
                media_path=media_path,
                filename=filename,
                media_type=media_type,
            )

    async def get_vision_completion_async(
        self,
        prompt: str = "",
        images: Optional[list] = None,
        thinking: Optional[bool] = None,
        tools: Optional[List[Dict[str, Any]]] = None,
        tool_choice: Optional[Any] = None,
        messages: Optional[List[Dict[str, Any]]] = None,
    ) -> Union[str, Any]:
        """Get LLM completion with images asynchronously."""
        effective_thinking = self.thinking if thinking is None else thinking
        return await self.get_vlm_instance().get_vision_completion_async(
            prompt=prompt,
            images=images,
            thinking=effective_thinking,
            tools=tools,
            tool_choice=tool_choice,
            messages=messages,
        )


# Resolve forward reference for backup field
VLMConfig.model_rebuild()