OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: enable watsonx vision enabled models in vlm settings (#2459)
Mike Fortman0f4f34356ef3
diff --git a/src/config/model_providers.py b/src/config/model_providers.pyindex bee0584bd..d38bb929c 100644--- a/src/config/model_providers.py+++ b/src/config/model_providers.py@@ -19,6 +19,11 @@ covers the public vendors, but a self-hosted OpenAI-compatible gateway serveswhatever its operator deployed, so those ids can only come from here. Declaredids are added to whatever the table already knows for that provider.+A row may also declare `vision_models`: patterns for ids that accept images+even though LiteLLM's table does not flag them `supports_vision`. The VLM+picker lists only vision-capable models, so without this a provider whose+table entries lag behind its deployment offers no VLM at all.+``OPENRAG_MODEL_PROVIDERS_CONFIG`` points at an alternate YAML file for a singledeployment. If that file is missing or unreadable the shipped default is used;if the shipped default is unreadable too, ``_FALLBACK_PROVIDERS`` below keeps@@ -54,6 +59,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = ("models": (),"embedding_models": (),"exclude_models": (),+ "vision_models": (),},{"name": "ollama",@@ -62,6 +68,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = ("models": (),"embedding_models": (),"exclude_models": (),+ "vision_models": (),},{"name": "watsonx",@@ -70,6 +77,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = ("models": (),"embedding_models": (),"exclude_models": (),+ "vision_models": (),},{"name": "anthropic",@@ -78,6 +86,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = ("models": (),"embedding_models": (),"exclude_models": (),+ "vision_models": (),},)@@ -150,6 +159,8 @@ def _normalize(entry: Any, seen: set[str]) -> dict[str, Any] | None:"embedding_models": _model_ids(entry.get("embedding_models")),# Optional: ids to keep out of the pickers, `*`/`?` wildcards allowed."exclude_models": _model_patterns(entry.get("exclude_models")),+ # Optional: ids that accept images though LiteLLM does not say so.+ "vision_models": _model_patterns(entry.get("vision_models")),}@@ -241,6 +252,7 @@ class ProviderEntry(NamedTuple):models: tuple[str, ...]embedding_models: tuple[str, ...]exclude_models: tuple[str, ...]+ vision_models: tuple[str, ...] = ()def visible_provider_entries(@@ -254,6 +266,7 @@ def visible_provider_entries(entry["models"],entry["embedding_models"],entry["exclude_models"],+ entry["vision_models"],)for entry in visible_providers(run_mode))diff --git a/src/config/model_providers.yaml b/src/config/model_providers.yamlindex 0e9db29eb..fae86a86f 100644--- a/src/config/model_providers.yaml+++ b/src/config/model_providers.yaml@@ -95,6 +95,13 @@ providers:- ibm/granite-13b-instruct-v2- sdaia/allam-1-13b-instruct- bigscience/mt0-xxl+ # Accept images on watsonx.ai (`image_chat` in its foundation_model_specs)+ # but LiteLLM does not flag them `supports_vision`. Every watsonx id it does+ # flag is withdrawn and excluded above, so without these the VLM picker+ # offers no watsonx model at all.+ vision_models:+ - meta-llama/llama-4-maverick-17b-128e-instruct-fp8+ - mistralai/mistral-small-3-1-24b-instruct-2503# watsonx.ai running on a Cloud Pak for Data / IBM Software Hub cluster. It# is a second front door onto the same `/ml/v1` API as the row above, keptdiff --git a/src/services/model_catalog.py b/src/services/model_catalog.pyindex cba943cef..fa3b2e298 100644--- a/src/services/model_catalog.py+++ b/src/services/model_catalog.py@@ -240,10 +240,11 @@ def _model_entry(name: str, info: dict[str, Any]) -> dict[str, Any]:return entry-def _excluded(name: str, patterns: tuple[str, ...]) -> bool:- """Whether a model id is one the config file keeps out of the pickers.+def _matches(name: str, patterns: tuple[str, ...]) -> bool:+ """Whether a model id matches a config pattern list.- Matched on the id as the picker shows it — `gpt-3.5-turbo`, not+ Used for `exclude_models` (ids kept out of the pickers) and `vision_models`+ (ids tagged as accepting images). Matched on the id as the picker shows it — `gpt-3.5-turbo`, not`openai/gpt-3.5-turbo` — because that is the name an operator reads off thescreen when deciding what to suppress. `*` and `?` work, so a wholegeneration goes in one line.@@ -270,6 +271,19 @@ def _declared_entries(names: tuple[str, ...], mode: str) -> list[dict[str, Any]]return [{"model": name, "mode": mode} for name in names]+def _with_vision(entry: dict[str, Any]) -> dict[str, Any]:+ """`entry` with the `vision` capability, for a config `vision_models:` match.++ LiteLLM's table lags the providers: watsonx.ai serves Llama 4 Maverick and+ Mistral Small 3.1 with image input, but the table flags neither, while+ every watsonx id it does flag has been withdrawn.+ """+ capabilities = entry.get("capabilities") or []+ if "vision" in capabilities:+ return entry+ return {**entry, "capabilities": [*capabilities, "vision"]}++@lru_cache(maxsize=8)def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:"""Payload for `providers`, given as `config.model_providers` entries.@@ -319,7 +333,7 @@ def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:bucket.setdefault(provider, []).append(_model_entry(name, info))entries = []- for key, display_name, declared_chat, declared_embed, excluded in providers:+ for key, display_name, declared_chat, declared_embed, excluded, vision in providers:if not is_known_provider(key):# The catalogue would still render, but `split_model_id` cannot# recognise the prefix, so every id would be billed to the default@@ -341,8 +355,11 @@ def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:# Applied last, so `exclude_models` also wins over a `models:` row —# a deployment that suppresses an id means it, wherever the id came# from, and the alternative is two lines that quietly contradict.- chat = [entry for entry in chat if not _excluded(entry["model"], excluded)]- embed = [entry for entry in embed if not _excluded(entry["model"], excluded)]+ chat = [entry for entry in chat if not _matches(entry["model"], excluded)]+ embed = [entry for entry in embed if not _matches(entry["model"], excluded)]+ chat = [+ _with_vision(entry) if _matches(entry["model"], vision) else entry for entry in chat+ ]entries.append({"key": key,@@ -413,7 +430,7 @@ def hide_excluded_live_models(provider: str, payload: Any) -> Any:filtered[key] = [modelfor model in models- if not _excluded(str((model or {}).get("value", "")), patterns)+ if not _matches(str((model or {}).get("value", "")), patterns)if isinstance(model, dict)]return filtereddiff --git a/tests/unit/config/test_model_providers_config.py b/tests/unit/config/test_model_providers_config.pyindex 6482b97af..7453f1275 100644--- a/tests/unit/config/test_model_providers_config.py+++ b/tests/unit/config/test_model_providers_config.py@@ -310,3 +310,19 @@ def test_exclusions_default_to_none_and_ignore_a_non_list(monkeypatch, tmp_path)monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")assert model_providers.visible_provider_entries()[0].exclude_models == ()+++def test_a_row_can_declare_vision_models_by_pattern(monkeypatch, tmp_path):+ monkeypatch.setenv(+ model_providers.CONFIG_PATH_ENV,+ _write(+ tmp_path,+ "providers:\n - name: watsonx\n modes:\n oss: true\n"+ " vision_models:\n - Meta-Llama/llama-4-*\n",+ ),+ )+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")++ entry = model_providers.visible_provider_entries()[0]+ assert entry.vision_models == ("meta-llama/llama-4-*",)+ assert entry.exclude_models == ()diff --git a/tests/unit/services/test_model_catalog.py b/tests/unit/services/test_model_catalog.pyindex 6d7186446..996a8df08 100644--- a/tests/unit/services/test_model_catalog.py+++ b/tests/unit/services/test_model_catalog.py@@ -467,6 +467,51 @@ def test_the_shipped_config_keeps_broken_watsonx_models_out_of_live_inventory(assert not catalogued.intersection(excluded)+def test_a_configured_vision_model_is_tagged_vision(monkeypatch, tmp_path) -> None:+ """`vision_models` adds the capability the VLM picker filters on.++ Applies to LiteLLM's own rows and to declared ones alike, and leaves every+ other model's capabilities as the table had them.+ """+ monkeypatch.setenv(+ model_providers.CONFIG_PATH_ENV,+ _write_providers(+ tmp_path,+ "providers:\n - name: openai\n modes:\n oss: true\n"+ " models:\n - house-tuned-gpt\n"+ " vision_models:\n - gpt-3.5-turbo\n - house-tuned-*\n",+ ),+ )+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")++ models = {entry["model"]: entry for entry in model_catalog.catalog()["providers"][0]["models"]}+ assert "vision" in models["gpt-3.5-turbo"]["capabilities"]+ assert "function_calling" in models["gpt-3.5-turbo"]["capabilities"]+ assert models["house-tuned-gpt"]["capabilities"] == ["vision"]+ assert "vision" not in models["gpt-3.5-turbo-0125"].get("capabilities", [])+++def test_the_shipped_config_offers_watsonx_vision_models(monkeypatch) -> None:+ """The VLM picker shows only `vision` models, so watsonx needs some.++ Every watsonx id LiteLLM flags as vision is withdrawn and excluded; the+ ones watsonx.ai serves with image input are tagged by the config instead.+ """+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")++ vision = {+ entry["model"]+ for provider in model_catalog.catalog()["providers"]+ if provider["key"] == "watsonx"+ for entry in provider["models"]+ if "vision" in entry.get("capabilities", [])+ }+ assert vision == {+ "meta-llama/llama-4-maverick-17b-128e-instruct-fp8",+ "mistralai/mistral-small-3-1-24b-instruct-2503",+ }++def test_a_plain_name_also_excludes_its_regional_listings(monkeypatch, tmp_path) -> None:"""Azure lists the same model per region; suppressing it means all of them."""monkeypatch.setenv(@@ -504,13 +549,13 @@ def test_an_exclusion_without_a_wildcard_leaves_the_dated_snapshots(monkeypatch)because the fix is a single character in a YAML file and reads like a typo."""monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")- exact = model_catalog._excluded("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna",))- wildcard = model_catalog._excluded("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna*",))+ exact = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna",))+ wildcard = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna*",))assert not exactassert wildcard# The wildcard keeps covering the regional forms the bare name covered.- assert model_catalog._excluded("us/gpt-5.6-luna", ("gpt-5.6-luna*",))+ assert model_catalog._matches("us/gpt-5.6-luna", ("gpt-5.6-luna*",))def test_both_luna_generations_are_excluded_from_both_rows(monkeypatch) -> None: