Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: enable watsonx vision enabled models in vlm settings (#2459)

Mike Fortman0f4f34356ef3
diff --git a/src/config/model_providers.py b/src/config/model_providers.py
index bee0584bd..d38bb929c 100644
--- a/src/config/model_providers.py
+++ b/src/config/model_providers.py
@@ -19,6 +19,11 @@ covers the public vendors, but a self-hosted OpenAI-compatible gateway serves
whatever its operator deployed, so those ids can only come from here. Declared
ids are added to whatever the table already knows for that provider.
+A row may also declare `vision_models`: patterns for ids that accept images
+even though LiteLLM's table does not flag them `supports_vision`. The VLM
+picker lists only vision-capable models, so without this a provider whose
+table entries lag behind its deployment offers no VLM at all.
+
``OPENRAG_MODEL_PROVIDERS_CONFIG`` points at an alternate YAML file for a single
deployment. If that file is missing or unreadable the shipped default is used;
if the shipped default is unreadable too, ``_FALLBACK_PROVIDERS`` below keeps
@@ -54,6 +59,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = (
"models": (),
"embedding_models": (),
"exclude_models": (),
+ "vision_models": (),
},
{
"name": "ollama",
@@ -62,6 +68,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = (
"models": (),
"embedding_models": (),
"exclude_models": (),
+ "vision_models": (),
},
{
"name": "watsonx",
@@ -70,6 +77,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = (
"models": (),
"embedding_models": (),
"exclude_models": (),
+ "vision_models": (),
},
{
"name": "anthropic",
@@ -78,6 +86,7 @@ _FALLBACK_PROVIDERS: tuple[dict[str, Any], ...] = (
"models": (),
"embedding_models": (),
"exclude_models": (),
+ "vision_models": (),
},
)
@@ -150,6 +159,8 @@ def _normalize(entry: Any, seen: set[str]) -> dict[str, Any] | None:
"embedding_models": _model_ids(entry.get("embedding_models")),
# Optional: ids to keep out of the pickers, `*`/`?` wildcards allowed.
"exclude_models": _model_patterns(entry.get("exclude_models")),
+ # Optional: ids that accept images though LiteLLM does not say so.
+ "vision_models": _model_patterns(entry.get("vision_models")),
}
@@ -241,6 +252,7 @@ class ProviderEntry(NamedTuple):
models: tuple[str, ...]
embedding_models: tuple[str, ...]
exclude_models: tuple[str, ...]
+ vision_models: tuple[str, ...] = ()
def visible_provider_entries(
@@ -254,6 +266,7 @@ def visible_provider_entries(
entry["models"],
entry["embedding_models"],
entry["exclude_models"],
+ entry["vision_models"],
)
for entry in visible_providers(run_mode)
)
diff --git a/src/config/model_providers.yaml b/src/config/model_providers.yaml
index 0e9db29eb..fae86a86f 100644
--- a/src/config/model_providers.yaml
+++ b/src/config/model_providers.yaml
@@ -95,6 +95,13 @@ providers:
- ibm/granite-13b-instruct-v2
- sdaia/allam-1-13b-instruct
- bigscience/mt0-xxl
+ # Accept images on watsonx.ai (`image_chat` in its foundation_model_specs)
+ # but LiteLLM does not flag them `supports_vision`. Every watsonx id it does
+ # flag is withdrawn and excluded above, so without these the VLM picker
+ # offers no watsonx model at all.
+ vision_models:
+ - meta-llama/llama-4-maverick-17b-128e-instruct-fp8
+ - mistralai/mistral-small-3-1-24b-instruct-2503
# watsonx.ai running on a Cloud Pak for Data / IBM Software Hub cluster. It
# is a second front door onto the same `/ml/v1` API as the row above, kept
diff --git a/src/services/model_catalog.py b/src/services/model_catalog.py
index cba943cef..fa3b2e298 100644
--- a/src/services/model_catalog.py
+++ b/src/services/model_catalog.py
@@ -240,10 +240,11 @@ def _model_entry(name: str, info: dict[str, Any]) -> dict[str, Any]:
return entry
-def _excluded(name: str, patterns: tuple[str, ...]) -> bool:
- """Whether a model id is one the config file keeps out of the pickers.
+def _matches(name: str, patterns: tuple[str, ...]) -> bool:
+ """Whether a model id matches a config pattern list.
- Matched on the id as the picker shows it — `gpt-3.5-turbo`, not
+ Used for `exclude_models` (ids kept out of the pickers) and `vision_models`
+ (ids tagged as accepting images). Matched on the id as the picker shows it — `gpt-3.5-turbo`, not
`openai/gpt-3.5-turbo` — because that is the name an operator reads off the
screen when deciding what to suppress. `*` and `?` work, so a whole
generation goes in one line.
@@ -270,6 +271,19 @@ def _declared_entries(names: tuple[str, ...], mode: str) -> list[dict[str, Any]]
return [{"model": name, "mode": mode} for name in names]
+def _with_vision(entry: dict[str, Any]) -> dict[str, Any]:
+ """`entry` with the `vision` capability, for a config `vision_models:` match.
+
+ LiteLLM's table lags the providers: watsonx.ai serves Llama 4 Maverick and
+ Mistral Small 3.1 with image input, but the table flags neither, while
+ every watsonx id it does flag has been withdrawn.
+ """
+ capabilities = entry.get("capabilities") or []
+ if "vision" in capabilities:
+ return entry
+ return {**entry, "capabilities": [*capabilities, "vision"]}
+
+
@lru_cache(maxsize=8)
def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:
"""Payload for `providers`, given as `config.model_providers` entries.
@@ -319,7 +333,7 @@ def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:
bucket.setdefault(provider, []).append(_model_entry(name, info))
entries = []
- for key, display_name, declared_chat, declared_embed, excluded in providers:
+ for key, display_name, declared_chat, declared_embed, excluded, vision in providers:
if not is_known_provider(key):
# The catalogue would still render, but `split_model_id` cannot
# recognise the prefix, so every id would be billed to the default
@@ -341,8 +355,11 @@ def _catalog(providers: tuple[ProviderEntry, ...]) -> dict[str, Any]:
# Applied last, so `exclude_models` also wins over a `models:` row —
# a deployment that suppresses an id means it, wherever the id came
# from, and the alternative is two lines that quietly contradict.
- chat = [entry for entry in chat if not _excluded(entry["model"], excluded)]
- embed = [entry for entry in embed if not _excluded(entry["model"], excluded)]
+ chat = [entry for entry in chat if not _matches(entry["model"], excluded)]
+ embed = [entry for entry in embed if not _matches(entry["model"], excluded)]
+ chat = [
+ _with_vision(entry) if _matches(entry["model"], vision) else entry for entry in chat
+ ]
entries.append(
{
"key": key,
@@ -413,7 +430,7 @@ def hide_excluded_live_models(provider: str, payload: Any) -> Any:
filtered[key] = [
model
for model in models
- if not _excluded(str((model or {}).get("value", "")), patterns)
+ if not _matches(str((model or {}).get("value", "")), patterns)
if isinstance(model, dict)
]
return filtered
diff --git a/tests/unit/config/test_model_providers_config.py b/tests/unit/config/test_model_providers_config.py
index 6482b97af..7453f1275 100644
--- a/tests/unit/config/test_model_providers_config.py
+++ b/tests/unit/config/test_model_providers_config.py
@@ -310,3 +310,19 @@ def test_exclusions_default_to_none_and_ignore_a_non_list(monkeypatch, tmp_path)
monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
assert model_providers.visible_provider_entries()[0].exclude_models == ()
+
+
+def test_a_row_can_declare_vision_models_by_pattern(monkeypatch, tmp_path):
+ monkeypatch.setenv(
+ model_providers.CONFIG_PATH_ENV,
+ _write(
+ tmp_path,
+ "providers:\n - name: watsonx\n modes:\n oss: true\n"
+ " vision_models:\n - Meta-Llama/llama-4-*\n",
+ ),
+ )
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
+
+ entry = model_providers.visible_provider_entries()[0]
+ assert entry.vision_models == ("meta-llama/llama-4-*",)
+ assert entry.exclude_models == ()
diff --git a/tests/unit/services/test_model_catalog.py b/tests/unit/services/test_model_catalog.py
index 6d7186446..996a8df08 100644
--- a/tests/unit/services/test_model_catalog.py
+++ b/tests/unit/services/test_model_catalog.py
@@ -467,6 +467,51 @@ def test_the_shipped_config_keeps_broken_watsonx_models_out_of_live_inventory(
assert not catalogued.intersection(excluded)
+def test_a_configured_vision_model_is_tagged_vision(monkeypatch, tmp_path) -> None:
+ """`vision_models` adds the capability the VLM picker filters on.
+
+ Applies to LiteLLM's own rows and to declared ones alike, and leaves every
+ other model's capabilities as the table had them.
+ """
+ monkeypatch.setenv(
+ model_providers.CONFIG_PATH_ENV,
+ _write_providers(
+ tmp_path,
+ "providers:\n - name: openai\n modes:\n oss: true\n"
+ " models:\n - house-tuned-gpt\n"
+ " vision_models:\n - gpt-3.5-turbo\n - house-tuned-*\n",
+ ),
+ )
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
+
+ models = {entry["model"]: entry for entry in model_catalog.catalog()["providers"][0]["models"]}
+ assert "vision" in models["gpt-3.5-turbo"]["capabilities"]
+ assert "function_calling" in models["gpt-3.5-turbo"]["capabilities"]
+ assert models["house-tuned-gpt"]["capabilities"] == ["vision"]
+ assert "vision" not in models["gpt-3.5-turbo-0125"].get("capabilities", [])
+
+
+def test_the_shipped_config_offers_watsonx_vision_models(monkeypatch) -> None:
+ """The VLM picker shows only `vision` models, so watsonx needs some.
+
+ Every watsonx id LiteLLM flags as vision is withdrawn and excluded; the
+ ones watsonx.ai serves with image input are tagged by the config instead.
+ """
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
+
+ vision = {
+ entry["model"]
+ for provider in model_catalog.catalog()["providers"]
+ if provider["key"] == "watsonx"
+ for entry in provider["models"]
+ if "vision" in entry.get("capabilities", [])
+ }
+ assert vision == {
+ "meta-llama/llama-4-maverick-17b-128e-instruct-fp8",
+ "mistralai/mistral-small-3-1-24b-instruct-2503",
+ }
+
+
def test_a_plain_name_also_excludes_its_regional_listings(monkeypatch, tmp_path) -> None:
"""Azure lists the same model per region; suppressing it means all of them."""
monkeypatch.setenv(
@@ -504,13 +549,13 @@ def test_an_exclusion_without_a_wildcard_leaves_the_dated_snapshots(monkeypatch)
because the fix is a single character in a YAML file and reads like a typo.
"""
monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
- exact = model_catalog._excluded("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna",))
- wildcard = model_catalog._excluded("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna*",))
+ exact = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna",))
+ wildcard = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna*",))
assert not exact
assert wildcard
# The wildcard keeps covering the regional forms the bare name covered.
- assert model_catalog._excluded("us/gpt-5.6-luna", ("gpt-5.6-luna*",))
+ assert model_catalog._matches("us/gpt-5.6-luna", ("gpt-5.6-luna*",))
def test_both_luna_generations_are_excluded_from_both_rows(monkeypatch) -> None: