OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Docling VLM endpoints and separators for both Azure providers (#2446)
Rico Furtadoab7a46b5708f
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.pyindex 8f732d733..a14e19a69 100644--- a/src/api/settings/endpoints.py+++ b/src/api/settings/endpoints.py@@ -874,13 +874,15 @@ async def update_settings(effective_vlm_provider)credentials = current_config.providers.credential_values(effective_vlm_provider)- api_key = getattr(vlm_provider_config, "api_key", "") or credentials.get(- "api_key", ""- )- vlm_provider_missing = not api_key or not vlm_provider_config.configured++ # Whether the provider is usable at all is the provider's own+ # question, and `configured` already answers it against that+ # provider's required fields.+ vlm_provider_missing = not vlm_provider_config.configuredif effective_vlm_provider in ("azure", "azure_ai"):- endpoint = credentials.get("api_base", "")- vlm_provider_missing = vlm_provider_missing or not endpoint+ vlm_provider_missing = vlm_provider_missing or not credentials.get(+ "api_base", ""+ )if effective_vlm_provider == "watsonx":vlm_provider_missing = (vlm_provider_missing@@ -897,6 +899,37 @@ async def update_settings(},status_code=400,)++ # Separately: docling-serve calls the model endpoint itself,+ # with headers OpenRAG builds for it, and every remote branch+ # in `docling_service` builds an API-key header. So VLM needs a+ # key even where the provider itself can authenticate other+ # ways — Azure OpenAI already accepts a Microsoft Entra token+ # or a service principal for inference.+ #+ # This is a limit of the VLM integration, not of the provider,+ # and it is reported as one. Rolling it into "provider is not+ # configured" above sent operators to re-enter credentials that+ # were already valid, and wrote "an API key is the only way to+ # authenticate" into a gate that has no business saying so.+ # Entra for VLM needs token acquisition, refresh and+ # propagation to docling-serve, and is a separate change.+ api_key = getattr(vlm_provider_config, "api_key", "") or credentials.get(+ "api_key", ""+ )+ if not api_key:+ return JSONResponse(+ {+ "error": (+ f"Cannot enable Docling VLM: provider "+ f"'{effective_vlm_provider}' is configured, but image "+ "descriptions are sent to the document converter, which "+ "authenticates with an API key. Add one in "+ "Settings > Providers, or turn off image descriptions."+ )+ },+ status_code=400,+ )elif effective_vlm_provider == "local":# Local provider does not require any credentialspassdiff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex bdf89a0ec..a585d5524 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -114,6 +114,30 @@ OCR_LANGUAGE_CODES: dict[str, dict[str, str]] = {}+def strip_provider_prefix(model: str, provider: str) -> str:+ """The bare model name from a possibly provider-tagged VLM id.++ `knowledge.vlm_provider` and `knowledge.vlm_model` are separate fields, so+ the model is usually already bare. It is not always: a tagged id can reach+ here from config written by hand or carried over from a picker.++ Both separators are accepted. `:` is the canonical one+ (`llm_gateway.PROVIDER_SEPARATOR`, and the one embedding-space ids use);+ `/` is the legacy form kept working elsewhere in the codebase. Only `/`+ used to be stripped, so an id stored as `azure_ai:gpt-4o` was sent to the+ provider with the prefix still attached — a model name no deployment has.+ """+ name = (model or "").strip()+ key = (provider or "").strip().lower()+ if not name or not key:+ return name+ for separator in (":", "/"):+ prefix = f"{key}{separator}"+ if name.lower().startswith(prefix):+ return name[len(prefix) :]+ return name++def resolve_ocr_languages(engine: str, languages: list[str]) -> list[str]:"""Translate OpenRAG's neutral OCR language codes into engine-specific ones.@@ -311,6 +335,23 @@ class DoclingService:options["include_images"] = True# If picture descriptions are enabled, configure custom/local VLM model+ #+ # Trust boundary: every `picture_description_api` block below hands the+ # provider's own credential to docling-serve, which then calls the model+ # endpoint directly — OpenRAG is not in the path of that request and+ # does not proxy it. This is how remote VLM has always worked here and+ # is not specific to Azure, but it means docling-serve is inside the+ # blast radius of any provider key configured for VLM. Two consequences+ # worth keeping in mind:+ #+ # - these options must never be logged. The credential is a value in+ # `headers`, so logging the options dict, or echoing it into an error,+ # leaks the key. `DoclingServeError` messages below are written to+ # name the *field* that is wrong, never its value.+ # - a short-lived credential would add an expiry problem on top of the+ # trust one, since docling-serve holds whatever it is given for the+ # life of the conversion. That is part of why Microsoft Entra+ # authentication is not wired through here; see the Azure branches.if is_pic_desc_enabled and knowledge_config.vlm_enabled:provider = knowledge_config.vlm_providervlm_model = knowledge_config.vlm_model@@ -395,12 +436,12 @@ class DoclingService:if not (api_key and endpoint):raise DoclingServeError("Docling VLM is enabled but the Azure provider is not configured "- "(api key and endpoint are required)"+ "(an API key and endpoint are required)")parsed = urlparse(endpoint)if parsed.scheme.lower() != "https" or not parsed.netloc:raise DoclingServeError("Azure VLM endpoint must use HTTPS")- deployment = vlm_model.removeprefix("azure/")+ deployment = strip_provider_prefix(vlm_model, "azure")url = (f"{endpoint.rstrip('/')}/openai/deployments/{deployment}/chat/completions"f"?api-version={api_version}"@@ -415,27 +456,62 @@ class DoclingService:"prompt": prompt,}elif provider == "azure_ai":- # Azure AI Foundry, not the Azure OpenAI Service above: its- # api_base already points at the `/models` route and the chat- # endpoint hangs directly off it. Without this branch a Foundry- # VLM fell through to the OpenAI default and was called with- # OpenAI credentials it does not have.+ # Azure AI Foundry, not the Azure OpenAI Service above. Which+ # path the chat endpoint hangs off depends on which surface the+ # operator configured, so the endpoint is classified rather+ # than assumed — see `enhancements/providers/azure/foundry.py`.+ # Without this branch a Foundry VLM fell through to the OpenAI+ # default and was called with OpenAI credentials it does not+ # have.+ from enhancements.providers.azure import foundry+creds = config.providers.credential_values("azure_ai")api_key = creds.get("api_key")- endpoint = creds.get("api_base")+ endpoint = foundry.normalized_api_base(creds.get("api_base"))api_version = creds.get("api_version")if not (api_key and endpoint):raise DoclingServeError("Docling VLM is enabled but the Azure AI Foundry provider is not "- "configured (api key and endpoint are required)"+ "configured (an API key and endpoint are required)")parsed = urlparse(endpoint)if parsed.scheme.lower() != "https" or not parsed.netloc:raise DoclingServeError("Azure AI Foundry VLM endpoint must use HTTPS")- model_id = vlm_model.removeprefix("azure_ai/")- url = f"{endpoint.rstrip('/')}/chat/completions"- if api_version:++ profile = foundry.endpoint_profile(endpoint)+ if profile == "unknown":+ raise DoclingServeError(+ f"Docling VLM cannot use this Azure AI Foundry endpoint. "+ f"{foundry.UNSUPPORTED_ENDPOINT_MESSAGE}"+ )+ if profile == "legacy_deployment_uri":+ # A per-deployment Target URI. The gateway still routes this+ # shape, for installs configured before it was refused at+ # save time, but VLM cannot use it: the URI pins the call to+ # the one deployment baked into it, so the model chosen in+ # the picker is silently ignored. The version this API dates+ # itself with is dropped by normalization too, so the URL+ # built from it would be rejected upstream regardless —+ # refused here, where it can still be read as a+ # configuration error rather than an ingestion failure.+ raise DoclingServeError(+ f"Docling VLM cannot use this Azure AI Foundry endpoint. "+ f"{foundry.DEPLOYMENT_URI_MESSAGE}"+ )+ chat_base = endpoint+ if profile == "legacy_models" and not endpoint.rstrip("/").endswith("/models"):+ # The resource root on the legacy surface: inference hangs+ # off `/models`, which is what LiteLLM appends for the same+ # configuration. Appending it here keeps the VLM call and+ # the gateway call pointing at the same place.+ chat_base = f"{endpoint.rstrip('/')}/models"+ url = f"{chat_base.rstrip('/')}/chat/completions"+ # `api-version` dates the legacy API only; the v1 surface is+ # versionless and ignores it at best.+ if api_version and profile == "legacy_models":url = f"{url}?api-version={api_version}"++ model_id = strip_provider_prefix(vlm_model, "azure_ai")options["picture_description_api"] = {"url": url,"headers": {"api-key": api_key},diff --git a/tests/unit/api/test_settings_endpoints.py b/tests/unit/api/test_settings_endpoints.pyindex 8acb27f34..5b80bdc2d 100644--- a/tests/unit/api/test_settings_endpoints.py+++ b/tests/unit/api/test_settings_endpoints.py@@ -140,6 +140,111 @@ def test_custom_provider_payload_keeps_legacy_openai_secret():assert payload["openai"].credential_values["organization"] == "org-1"+async def _vlm_gate_response(config, provider: str):+ """Run `update_settings` with VLM on, returning whatever the gate answers."""+ from unittest.mock import patch++ from api.settings.endpoints import update_settings++ config.edited = True+ body = SettingsUpdateBody(+ vlm_enabled=True,+ vlm_provider=provider,+ vlm_model=f"{provider}/gpt-4.1",+ )+ models_service = MagicMock()+ models_service.validate_model_configured = AsyncMock(return_value=True)+ rbac = MagicMock()+ rbac.has_permission = AsyncMock(return_value=True)++ with (+ patch("api.settings.endpoints.get_openrag_config", return_value=config),+ patch("api.settings.endpoints.config_manager.save_config_file", return_value=True),+ patch("api.settings.endpoints.clients.refresh_patched_client", new_callable=AsyncMock),+ ):+ return await update_settings(+ body=body,+ session_manager=AsyncMock(),+ user=MagicMock(spec=User),+ models_service=models_service,+ rbac=rbac,+ )+++def _error_text(response) -> str:+ import json++ return json.loads(bytes(response.body).decode())["error"]+++async def test_update_settings_vlm_configured_without_a_key_says_so():+ """A provider can be fully configured and still have no API key.++ Azure OpenAI requires only `api_base`, so a service-principal or Entra+ configuration is `configured` with no key at all. docling-serve calls the+ model endpoint itself and only ever builds an API-key header, so VLM still+ cannot run — but telling the operator the provider "is not configured"+ sends them to re-enter credentials that are already valid. The two gates+ are separate so the message can name the real limitation.+ """+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict({})+ config.providers.set_credentials(+ "azure",+ {+ "api_base": "https://example.openai.azure.com",+ "tenant_id": "t",+ "client_id": "c",+ "client_secret": "s",+ },+ auth_method="service_principal",+ )+ assert config.providers.custom["azure"].configured, "precondition: configured without a key"++ response = await _vlm_gate_response(config, "azure")++ assert response.status_code == 400+ message = _error_text(response)+ assert "authenticates with an API key" in message+ assert "is not configured" not in message+++async def test_update_settings_vlm_unconfigured_provider_says_not_configured():+ """The other half of the split: nothing is set, so the first gate answers."""+ from config.config_manager import OpenRAGConfig++ response = await _vlm_gate_response(OpenRAGConfig.from_dict({}), "azure_ai")++ assert response.status_code == 400+ message = _error_text(response)+ assert "is not configured" in message+ assert "authenticates with an API key" not in message++[email protected]("provider", ["azure", "azure_ai"])+async def test_update_settings_vlm_azure_without_an_endpoint_is_not_configured(provider):+ """Both Azure providers need an endpoint, whatever `configured` claims.++ A config can carry `configured: true` with no `api_base` — the flag is+ read from the file rather than recomputed — and neither provider can be+ called without one.+ """+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict(+ {"providers": {"custom": {provider: {"configured": True, "credentials": {"api_key": "k"}}}}}+ )++ response = await _vlm_gate_response(config, provider)++ assert response.status_code == 400+ assert "is not configured" in _error_text(response)++@pytest.mark.asyncioasync def test_update_settings_vlm_azure_configured():"""Test that update_settings allows enabling VLM with azure when azure credentials are configured."""diff --git a/tests/unit/test_docling_vlm_azure_endpoints.py b/tests/unit/test_docling_vlm_azure_endpoints.pynew file mode 100644index 000000000..a04eec8b3--- /dev/null+++ b/tests/unit/test_docling_vlm_azure_endpoints.py@@ -0,0 +1,317 @@+"""Azure and Azure AI Foundry VLM endpoints handed to docling-serve.++docling-serve calls the model endpoint itself using the `picture_description_api`+block built here, so the URL has to be right for the surface the operator+configured and the credential must not escape into logs or error text.+"""++from __future__ import annotations++import sys+from pathlib import Path+from types import SimpleNamespace++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from services.docling_service import ( # noqa: E402+ DoclingServeError,+ DoclingService,+ strip_provider_prefix,+)++RESOURCE = "https://contoso.services.ai.azure.com"+SECRET = "super-secret-api-key"+++class TestStripProviderPrefix:+ """`:` is canonical; `/` is the legacy form. Only `/` used to be stripped."""++ @pytest.mark.parametrize(+ "model, provider, expected",+ [+ # The bug: a canonical id kept its prefix and was sent as a model+ # name no deployment has.+ ("azure_ai:gpt-4o", "azure_ai", "gpt-4o"),+ ("azure:my-vision", "azure", "my-vision"),+ # The legacy form kept working.+ ("azure_ai/gpt-4o", "azure_ai", "gpt-4o"),+ ("azure/my-vision", "azure", "my-vision"),+ # Already bare: the usual case, since provider and model are+ # separate config fields.+ ("gpt-4o", "azure_ai", "gpt-4o"),+ # A different provider's tag is part of the name, not a prefix.+ ("openai:gpt-4o", "azure_ai", "openai:gpt-4o"),+ # Case-insensitive on the tag, never on the model.+ ("Azure_AI:GPT-4o", "azure_ai", "GPT-4o"),+ ("", "azure_ai", ""),+ ],+ )+ def test_both_separators_are_stripped(self, model, provider, expected) -> None:+ assert strip_provider_prefix(model, provider) == expected++ def test_only_the_first_prefix_goes(self) -> None:+ """A deployment may legitimately contain a colon."""+ assert strip_provider_prefix("azure_ai:team:vision-v2", "azure_ai") == "team:vision-v2"+++def _service_with(provider: str, credentials: dict, vlm_model: str) -> tuple:+ """A DoclingService and the config its VLM branch will read."""+ knowledge = SimpleNamespace(+ ocr=False,+ table_structure=False,+ picture_descriptions=True,+ vlm_enabled=True,+ vlm_provider=provider,+ vlm_model=vlm_model,+ vlm_prompt="describe",+ vlm_max_tokens=100,+ vlm_watsonx_api_version="2024-01-01",+ ocr_languages=[],+ )+ providers = SimpleNamespace(credential_values=lambda _key, **_kw: dict(credentials))+ return DoclingService(), SimpleNamespace(knowledge=knowledge, providers=providers)+++async def _options(monkeypatch, provider: str, credentials: dict, vlm_model: str) -> dict:+ service, config = _service_with(provider, credentials, vlm_model)+ monkeypatch.setattr("services.docling_service.get_openrag_config", lambda: config)+ return await service._build_docling_options_async()+++class TestFoundryEndpointShapes:+ """The chat URL follows the surface, rather than assuming one."""++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "api_base, expected_url",+ [+ # OpenAI-compatible surface: inference hangs directly off it.+ (f"{RESOURCE}/openai/v1", f"{RESOURCE}/openai/v1/chat/completions"),+ (f"{RESOURCE}/openai/v1/", f"{RESOURCE}/openai/v1/chat/completions"),+ (+ f"{RESOURCE}/api/projects/p1/openai/v1",+ f"{RESOURCE}/api/projects/p1/openai/v1/chat/completions",+ ),+ # Legacy surface, named explicitly.+ (f"{RESOURCE}/models", f"{RESOURCE}/models/chat/completions"),+ # Legacy surface given as the resource root: `/models` is where+ # inference lives, and is what the gateway targets for the same+ # configuration.+ (RESOURCE, f"{RESOURCE}/models/chat/completions"),+ ],+ )+ async def test_the_url_matches_the_configured_surface(+ self, monkeypatch, api_base, expected_url+ ) -> None:+ options = await _options(+ monkeypatch, "azure_ai", {"api_base": api_base, "api_key": SECRET}, "gpt-4o"+ )+ assert options["picture_description_api"]["url"] == expected_url++ @pytest.mark.asyncio+ async def test_api_version_is_added_only_on_the_dated_legacy_api(self, monkeypatch) -> None:+ legacy = await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/models", "api_key": SECRET, "api_version": "2024-05-01"},+ "gpt-4o",+ )+ assert legacy["picture_description_api"]["url"].endswith("?api-version=2024-05-01")++ v1 = await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/openai/v1", "api_key": SECRET, "api_version": "2024-05-01"},+ "gpt-4o",+ )+ assert "api-version" not in v1["picture_description_api"]["url"]++ @pytest.mark.asyncio+ async def test_a_pasted_call_path_does_not_double_up(self, monkeypatch) -> None:+ options = await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/openai/v1/chat/completions", "api_key": SECRET},+ "gpt-4o",+ )+ assert options["picture_description_api"]["url"] == f"{RESOURCE}/openai/v1/chat/completions"++ @pytest.mark.asyncio+ async def test_an_unrecognised_endpoint_is_refused(self, monkeypatch) -> None:+ with pytest.raises(DoclingServeError, match="/openai/v1"):+ await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/anthropic/v1", "api_key": SECRET},+ "gpt-4o",+ )++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "api_base",+ [+ "https://contoso.openai.azure.com/openai/deployments/gpt-4o",+ # As pasted from the portal, with the call path and the version+ # that makes it valid. Normalization strips both, so what reaches+ # the classifier is the bare deployment URI either way.+ "https://contoso.openai.azure.com/openai/deployments/gpt-4o/chat/completions"+ "?api-version=2024-02-01",+ ],+ )+ async def test_a_single_deployment_target_uri_is_refused(self, monkeypatch, api_base) -> None:+ """It classifies, so the `unknown` check alone let it through.++ The URI pins every call to the deployment baked into it, so the model+ chosen in the picker would be ignored, and the `api-version` this API+ dates itself with does not survive normalization — the URL built from+ it is rejected upstream during ingestion. Refused here instead, where+ it still reads as a configuration error.+ """+ with pytest.raises(DoclingServeError, match="single deployment's Target URI"):+ await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": api_base, "api_key": SECRET, "api_version": "2024-02-01"},+ "gpt-4o",+ )++ @pytest.mark.asyncio+ async def test_a_cleartext_endpoint_is_refused(self, monkeypatch) -> None:+ with pytest.raises(DoclingServeError, match="HTTPS"):+ await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": "http://contoso.services.ai.azure.com/openai/v1", "api_key": SECRET},+ "gpt-4o",+ )+++class TestSeparatorReachesTheWire:+ """The separator fix, asserted where it matters: the model sent upstream."""++ @pytest.mark.asyncio+ @pytest.mark.parametrize("vlm_model", ["azure_ai:gpt-4o", "azure_ai/gpt-4o", "gpt-4o"])+ async def test_foundry_sends_the_bare_deployment_name(self, monkeypatch, vlm_model) -> None:+ options = await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/openai/v1", "api_key": SECRET},+ vlm_model,+ )+ assert options["picture_description_api"]["params"]["model"] == "gpt-4o"++ @pytest.mark.asyncio+ @pytest.mark.parametrize("vlm_model", ["azure:my-vision", "azure/my-vision", "my-vision"])+ async def test_azure_openai_sends_the_bare_deployment_name(+ self, monkeypatch, vlm_model+ ) -> None:+ options = await _options(+ monkeypatch,+ "azure",+ {"api_base": "https://contoso.openai.azure.com", "api_key": SECRET},+ vlm_model,+ )+ api = options["picture_description_api"]+ assert api["params"]["model"] == "my-vision"+ # The deployment is also in the path, so a stale prefix broke the URL too.+ assert "/openai/deployments/my-vision/chat/completions" in api["url"]+++class TestTheCredentialDoesNotEscape:+ """docling-serve is given the key; nothing else may be.++ The options dict carries the credential in `headers`, so an error or log+ line that echoes it leaks the key.+ """++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "provider, credentials",+ [+ ("azure_ai", {"api_base": f"{RESOURCE}/anthropic/v1", "api_key": SECRET}),+ (+ "azure_ai",+ {"api_base": "http://contoso.services.ai.azure.com/openai/v1", "api_key": SECRET},+ ),+ ("azure", {"api_base": "http://contoso.openai.azure.com", "api_key": SECRET}),+ ("azure_ai", {"api_base": "", "api_key": SECRET}),+ (+ "azure_ai",+ {+ "api_base": "https://contoso.openai.azure.com/openai/deployments/gpt-4o",+ "api_key": SECRET,+ },+ ),+ ],+ )+ async def test_no_rejection_message_contains_the_key(+ self, monkeypatch, provider, credentials+ ) -> None:+ with pytest.raises(DoclingServeError) as exc:+ await _options(monkeypatch, provider, credentials, "gpt-4o")++ assert SECRET not in str(exc.value)+ assert SECRET not in repr(exc.value)++ @pytest.mark.asyncio+ @pytest.mark.parametrize("provider", ["azure_ai", "azure"])+ async def test_building_the_options_logs_nothing_containing_the_key(+ self, monkeypatch, provider+ ) -> None:+ """Including the normalization debug line the Foundry branch triggers."""+ records: list[str] = []++ class _Recorder:+ def __getattr__(self, _name):+ def _log(message="", **fields):+ records.append(f"{message} {fields}")++ return _log++ monkeypatch.setattr("services.docling_service.logger", _Recorder())+ from enhancements.providers.azure import foundry++ monkeypatch.setattr(foundry, "logger", _Recorder())++ api_base = (+ f"{RESOURCE}/openai/v1/chat/completions?api-version=2024-05-01"+ if provider == "azure_ai"+ else "https://contoso.openai.azure.com"+ )+ options = await _options(+ monkeypatch, provider, {"api_base": api_base, "api_key": SECRET}, "gpt-4o"+ )++ # The key is in the options, which is the whole point of the block...+ assert options["picture_description_api"]["headers"]["api-key"] == SECRET+ # ...and nowhere in anything that was logged.+ assert SECRET not in " ".join(records)++ @pytest.mark.asyncio+ async def test_the_credential_is_passed_to_docling_serve_deliberately(+ self, monkeypatch+ ) -> None:+ """Pinning the trust boundary rather than leaving it implicit.++ docling-serve calls the model endpoint directly with these headers, so+ it holds the provider key for the life of the conversion. Any change+ that stops sending it, or starts sending something longer-lived, is a+ change to that boundary and should fail here first.+ """+ options = await _options(+ monkeypatch,+ "azure_ai",+ {"api_base": f"{RESOURCE}/openai/v1", "api_key": SECRET},+ "gpt-4o",+ )+ api = options["picture_description_api"]++ assert api["headers"] == {"api-key": SECRET}+ # API key only for now: no bearer plumbing until Entra is wired+ # end to end, with acquisition, refresh and propagation.+ assert "Authorization" not in api["headers"]