Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: isolate watsonx.ai on-prem configuration and ingestion (#2422)

Edwin Jose7a8b90dd131a
diff --git a/.env.example b/.env.example
index f1d640f8a..cdbc80318 100644
--- a/.env.example
+++ b/.env.example
@@ -260,17 +260,17 @@ WATSONX_ENDPOINT=
WATSONX_PROJECT_ID=
# --- IBM watsonx.ai on-prem (Cloud Pak for Data / Software Hub) ----------
-# WATSONX_ONPREM_ENDPOINT=https://cpd-cluster.example.com
-# WATSONX_ONPREM_USERNAME=
-# WATSONX_ONPREM_API_KEY=
+# WATSONX_ENDPOINT_ONPREM=https://cpd-cluster.example.com
+# WATSONX_USERNAME_ONPREM=
+# WATSONX_API_KEY_ONPREM=
# Alternatively provide base64(username:api-key) instead of the two fields above.
-# WATSONX_ONPREM_ZEN_API_KEY=
-# WATSONX_ONPREM_SPACE_ID=
-# WATSONX_ONPREM_PROJECT_ID=
+# WATSONX_ZEN_API_KEY_ONPREM=
+# WATSONX_SPACE_ID_ONPREM=
+# WATSONX_PROJECT_ID_ONPREM=
# Provider-scoped TLS policy: false (default), true, or a mounted CA bundle path.
# SSL_VERIFY no longer controls this provider. SSL_CERT_FILE remains available
# as the process trust store when verification is enabled.
-# WATSONX_ONPREM_TLS_VERIFY=false
+# WATSONX_TLS_VERIFY_ONPREM=false
# Tune the ibm-watsonx-ai SDK's native 429 retry used during ingestion embedding.
# OPTIONAL: Max retries = retry attempts (default 10).
diff --git a/docs/docs/core-components/ingestion-configure.mdx b/docs/docs/core-components/ingestion-configure.mdx
index 8f7252e6e..db60db383 100644
--- a/docs/docs/core-components/ingestion-configure.mdx
+++ b/docs/docs/core-components/ingestion-configure.mdx
@@ -38,6 +38,7 @@ When you start OpenRAG, a local `docling serve` process starts, and then OpenRAG
* **Docling processor ingestion**: Instead of using a separate Docling Serve service and the Docling Serve API, you can use the Docling processor directly.
To do this, set `DISABLE_INGEST_WITH_LANGFLOW=true` in your [OpenRAG `.env` file](/reference/configuration#document-processing-settings), and then [restart the OpenRAG services](/manage-services).
+This mode uses the embedding provider selected in **Settings**, including IBM watsonx.ai on-prem.
For the underlying functionality for this option, see [`processors.py`](https://github.com/langflow-ai/openrag/blob/main/src/models/processors.py#L58) in the OpenRAG repository.
## Set the embedding model and dimensions {#set-the-embedding-model-and-dimensions}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index 4faa93974..36ed10847 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -63,13 +63,13 @@ These variables are initially set during the [application onboarding process](/i
| `WATSONX_API_KEY` | Not set | API key for the IBM watsonx.ai model provider. |
| `WATSONX_ENDPOINT` | Not set | Custom provider endpoint for the IBM watsonx.ai model provider. |
| `WATSONX_PROJECT_ID` | Not set | Project ID for the IBM watsonx.ai model provider. |
-| `WATSONX_ONPREM_ENDPOINT` | Not set | HTTPS base URL of the IBM Cloud Pak for Data or Software Hub cluster. |
-| `WATSONX_ONPREM_USERNAME` | Not set | Cluster username used with `WATSONX_ONPREM_API_KEY`. |
-| `WATSONX_ONPREM_API_KEY` | Not set | Cluster API key exchanged with the username for CPD authorization. |
-| `WATSONX_ONPREM_ZEN_API_KEY` | Not set | Pre-encoded `base64(username:api-key)` alternative to the username and API-key variables. |
-| `WATSONX_ONPREM_SPACE_ID` | Not set | Deployment space used for inference. |
-| `WATSONX_ONPREM_PROJECT_ID` | Not set | Optional project alternative to a deployment space. |
-| `WATSONX_ONPREM_TLS_VERIFY` | `false` | Provider-scoped TLS policy: `false`, `true`, or a mounted CA bundle path. `SSL_VERIFY` no longer controls this provider; `SSL_CERT_FILE` supplies the process trust store when verification is enabled. |
+| `WATSONX_ENDPOINT_ONPREM` | Not set | HTTPS base URL of the IBM Cloud Pak for Data or Software Hub cluster. |
+| `WATSONX_USERNAME_ONPREM` | Not set | Cluster username used with `WATSONX_API_KEY_ONPREM`. |
+| `WATSONX_API_KEY_ONPREM` | Not set | Cluster API key exchanged with the username for CPD authorization. |
+| `WATSONX_ZEN_API_KEY_ONPREM` | Not set | Pre-encoded `base64(username:api-key)` alternative to the username and API-key variables. |
+| `WATSONX_SPACE_ID_ONPREM` | Not set | Deployment space used for inference. |
+| `WATSONX_PROJECT_ID_ONPREM` | Not set | Optional project alternative to a deployment space. Kept separate from the IBM Cloud `WATSONX_PROJECT_ID`. |
+| `WATSONX_TLS_VERIFY_ONPREM` | `false` | Provider-scoped TLS policy: `false`, `true`, or a mounted CA bundle path. `SSL_VERIFY` no longer controls this provider; `SSL_CERT_FILE` supplies the process trust store when verification is enabled. |
| `RHOAI_ENDPOINT` | Not set | Chat endpoint for the Red Hat OpenShift AI model provider: the predictor Service of the chat `InferenceService`, ending in `/v1`. |
| `RHOAI_EMBEDDINGS_ENDPOINT` | Not set | Embeddings endpoint for the Red Hat OpenShift AI model provider, a second `InferenceService`. Leave unset only if one endpoint serves both chat and embeddings. |
| `RHOAI_API_KEY` | Not set | Kubernetes bearer token for the Red Hat OpenShift AI model provider, sent to both endpoints. Its subject needs `get` on `inferenceservices` in the model namespace. |
diff --git a/enhancements/providers/watsonx/onprem.py b/enhancements/providers/watsonx/onprem.py
index 1808495cd..d687ebb94 100644
--- a/enhancements/providers/watsonx/onprem.py
+++ b/enhancements/providers/watsonx/onprem.py
@@ -852,8 +852,10 @@ def install_litellm_compatibility() -> None:
watsonx_error = common_utils.WatsonXAIError
def _get_api_params_allowing_no_scope(params: dict, model: str | None = None):
+ explicit_space = bool(params.get("space_id"))
+ explicit_project = bool(params.get("project_id") or params.get("watsonx_project"))
try:
- return original_get_api_params(params=params, model=model)
+ api_params = original_get_api_params(params=params, model=model)
except watsonx_error as exc:
if getattr(exc, "status_code", None) != 401 or "space_id" not in str(exc):
raise
@@ -861,6 +863,13 @@ def install_litellm_compatibility() -> None:
# and both were absent — which is exactly the lightweight-engine
# shape. Carry on with an empty scope.
return WatsonXAPIParams(project_id=None, space_id=None, region_name=None)
+ if explicit_space and not explicit_project:
+ # The cloud and on-prem providers share LiteLLM's watsonx adapter.
+ # An explicit on-prem space must win over the cloud provider's
+ # ambient WATSONX_PROJECT_ID, otherwise LiteLLM sends that project
+ # instead and the cluster rejects the request.
+ api_params["project_id"] = None
+ return api_params
setattr(_get_api_params_allowing_no_scope, _PATCH_MARKER, True)
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 2ddc12d15..b2d565c3d 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -825,13 +825,13 @@ class ConfigManager:
# IBM watsonx.ai on-prem (Cloud Pak for Data / Software Hub).
onprem_credentials = {
- "api_base": os.getenv("WATSONX_ONPREM_ENDPOINT"),
- "username": os.getenv("WATSONX_ONPREM_USERNAME"),
- "api_key": os.getenv("WATSONX_ONPREM_API_KEY"),
- "zen_api_key": os.getenv("WATSONX_ONPREM_ZEN_API_KEY"),
- "space_id": os.getenv("WATSONX_ONPREM_SPACE_ID"),
- "project_id": os.getenv("WATSONX_ONPREM_PROJECT_ID"),
- "ssl_verify": os.getenv("WATSONX_ONPREM_TLS_VERIFY"),
+ "api_base": os.getenv("WATSONX_ENDPOINT_ONPREM"),
+ "username": os.getenv("WATSONX_USERNAME_ONPREM"),
+ "api_key": os.getenv("WATSONX_API_KEY_ONPREM"),
+ "zen_api_key": os.getenv("WATSONX_ZEN_API_KEY_ONPREM"),
+ "space_id": os.getenv("WATSONX_SPACE_ID_ONPREM"),
+ "project_id": os.getenv("WATSONX_PROJECT_ID_ONPREM"),
+ "ssl_verify": os.getenv("WATSONX_TLS_VERIFY_ONPREM"),
}
if any(onprem_credentials.values()):
self._seed_custom_provider_credentials(
diff --git a/src/models/processors.py b/src/models/processors.py
index 99eb44f5f..77feb1a8f 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -579,13 +579,25 @@ class TaskProcessor:
text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)
embeddings = []
+ from services.model_catalog import litellm_provider_key, public_model_id
+
+ gateway_model = None
+ if litellm_provider_key(embedding_provider) != embedding_provider:
+ from services.llm_gateway import embeddings as gateway_embeddings
+
+ gateway_model = public_model_id(embedding_provider, embedding_model)
for batch in text_batches:
- resp = await clients.patched_embedding_client.embeddings.create(
- model=litellm_embedding_model, input=batch
- )
+ if gateway_model is not None:
+ response = await gateway_embeddings({"model": gateway_model, "input": batch})
+ data = response.get("data", [])
+ else:
+ response = await clients.patched_embedding_client.embeddings.create(
+ model=litellm_embedding_model, input=batch
+ )
+ data = response.data
embeddings.extend(
- [d["embedding"] if isinstance(d, dict) else d.embedding for d in resp.data]
+ [item["embedding"] if isinstance(item, dict) else item.embedding for item in data]
)
if not embeddings or len(embeddings) == 0:
diff --git a/tests/unit/config/test_generic_provider_config.py b/tests/unit/config/test_generic_provider_config.py
index b95396702..50743217d 100644
--- a/tests/unit/config/test_generic_provider_config.py
+++ b/tests/unit/config/test_generic_provider_config.py
@@ -232,13 +232,13 @@ def test_rhoai_tls_env_alone_reaches_a_stored_config(monkeypatch, tmp_path):
def _clear_watsonx_onprem_env(monkeypatch):
for name in (
- "WATSONX_ONPREM_ENDPOINT",
- "WATSONX_ONPREM_USERNAME",
- "WATSONX_ONPREM_API_KEY",
- "WATSONX_ONPREM_ZEN_API_KEY",
- "WATSONX_ONPREM_SPACE_ID",
- "WATSONX_ONPREM_PROJECT_ID",
- "WATSONX_ONPREM_TLS_VERIFY",
+ "WATSONX_ENDPOINT_ONPREM",
+ "WATSONX_USERNAME_ONPREM",
+ "WATSONX_API_KEY_ONPREM",
+ "WATSONX_ZEN_API_KEY_ONPREM",
+ "WATSONX_SPACE_ID_ONPREM",
+ "WATSONX_PROJECT_ID_ONPREM",
+ "WATSONX_TLS_VERIFY_ONPREM",
):
monkeypatch.delenv(name, raising=False)
@@ -248,11 +248,12 @@ def test_watsonx_onprem_env_seeds_credentials_and_tls(monkeypatch, tmp_path):
_clear_rhoai_env(monkeypatch)
_clear_watsonx_onprem_env(monkeypatch)
- monkeypatch.setenv("WATSONX_ONPREM_ENDPOINT", "https://cpd.example.com")
- monkeypatch.setenv("WATSONX_ONPREM_USERNAME", "cpduser")
- monkeypatch.setenv("WATSONX_ONPREM_API_KEY", "secret")
- monkeypatch.setenv("WATSONX_ONPREM_SPACE_ID", "space-1")
- monkeypatch.setenv("WATSONX_ONPREM_TLS_VERIFY", "/etc/openrag/cpd-ca.pem")
+ monkeypatch.setenv("WATSONX_ENDPOINT_ONPREM", "https://cpd.example.com")
+ monkeypatch.setenv("WATSONX_USERNAME_ONPREM", "cpduser")
+ monkeypatch.setenv("WATSONX_API_KEY_ONPREM", "secret")
+ monkeypatch.setenv("WATSONX_SPACE_ID_ONPREM", "space-1")
+ monkeypatch.setenv("WATSONX_PROJECT_ID_ONPREM", "project-1")
+ monkeypatch.setenv("WATSONX_TLS_VERIFY_ONPREM", "/etc/openrag/cpd-ca.pem")
config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
provider = config.providers.custom["watsonx_onprem"]
@@ -264,6 +265,7 @@ def test_watsonx_onprem_env_seeds_credentials_and_tls(monkeypatch, tmp_path):
"username": "cpduser",
"api_key": "secret",
"space_id": "space-1",
+ "project_id": "project-1",
"ssl_verify": "/etc/openrag/cpd-ca.pem",
}
@@ -273,8 +275,8 @@ def test_watsonx_onprem_env_without_complete_auth_stays_unconfigured(monkeypatch
_clear_rhoai_env(monkeypatch)
_clear_watsonx_onprem_env(monkeypatch)
- monkeypatch.setenv("WATSONX_ONPREM_ENDPOINT", "https://cpd.example.com")
- monkeypatch.setenv("WATSONX_ONPREM_TLS_VERIFY", "false")
+ monkeypatch.setenv("WATSONX_ENDPOINT_ONPREM", "https://cpd.example.com")
+ monkeypatch.setenv("WATSONX_TLS_VERIFY_ONPREM", "false")
config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
diff --git a/tests/unit/enhancements/providers/watsonx/test_onprem.py b/tests/unit/enhancements/providers/watsonx/test_onprem.py
index b5dd44d09..520d7bf30 100644
--- a/tests/unit/enhancements/providers/watsonx/test_onprem.py
+++ b/tests/unit/enhancements/providers/watsonx/test_onprem.py
@@ -281,6 +281,27 @@ def test_a_scope_that_is_set_still_reaches_the_cluster() -> None:
assert payload == {"model_id": "ibm/granite-3-3-8b-instruct", "space_id": "space-1"}
+def test_an_explicit_onprem_space_ignores_the_cloud_project_environment(monkeypatch) -> None:
+ """Cloud and on-prem watsonx scopes must not leak into each other."""
+ monkeypatch.setenv("WATSONX_PROJECT_ID", "cloud-project")
+ watsonx_onprem.install_litellm_compatibility()
+
+ from litellm.llms.watsonx.chat.handler import _get_api_params
+ from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
+
+ api_params = _get_api_params(
+ params={"space_id": "onprem-space"}, model="ibm/granite-3-3-8b-instruct"
+ )
+ payload = IBMWatsonXChatConfig()._prepare_payload(
+ model="ibm/granite-3-3-8b-instruct", api_params=api_params
+ )
+
+ assert payload == {
+ "model_id": "ibm/granite-3-3-8b-instruct",
+ "space_id": "onprem-space",
+ }
+
+
def test_a_self_signed_cluster_cert_is_reported_as_a_trust_problem() -> None:
"""The request never leaves the process, so there is no provider error to quote.
diff --git a/tests/unit/test_processor_mapping_client.py b/tests/unit/test_processor_mapping_client.py
index 247a48df4..4e690b38f 100644
--- a/tests/unit/test_processor_mapping_client.py
+++ b/tests/unit/test_processor_mapping_client.py
@@ -147,3 +147,95 @@ async def test_standard_processor_uses_shared_writer_for_embedding_mapping_and_w
assert bulk_body[1]["chunk_embedding_openai_text_embedding_3_small"] == [0.1, 0.2, 0.3]
assert bulk_body[1]["embedding_provider"] == "openai"
assert bulk_body[1]["embedding_space_id"] == "openai:text-embedding-3-small"
+
+
+async def test_standard_processor_routes_onprem_embeddings_through_gateway(
+ tmp_path,
+ monkeypatch,
+):
+ model = "ibm/slate-30m-english-rtrvr"
+ gateway_calls = []
+ indexed = []
+
+ async def gateway_embeddings(body):
+ gateway_calls.append(body)
+ return {
+ "data": [
+ {"embedding": [0.1, 0.2, 0.3], "index": index}
+ for index, _ in enumerate(body["input"])
+ ]
+ }
+
+ class SessionManager:
+ def get_user_opensearch_client(self, user_id, jwt_token):
+ return SimpleNamespace()
+
+ class ModelsService:
+ async def get_litellm_model_name(self, embedding_model, provider=None):
+ assert embedding_model == model
+ assert provider == "watsonx_onprem"
+ return f"watsonx_onprem/{embedding_model}"
+
+ class FailingEmbeddingClient:
+ class Embeddings:
+ async def create(self, **kwargs):
+ raise AssertionError(
+ "watsonx.ai on-prem must not use the environment-backed embedding client"
+ )
+
+ embeddings = Embeddings()
+
+ class Writer:
+ async def index_chunks(self, context, chunks, *, final=False):
+ indexed.append((context, chunks, final))
+
+ monkeypatch.setattr(
+ "models.processors.get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(
+ embedding_model=model,
+ embedding_provider="watsonx_onprem",
+ chunk_size=None,
+ chunk_overlap=None,
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ "models.processors.clients",
+ SimpleNamespace(opensearch=None, patched_embedding_client=FailingEmbeddingClient()),
+ )
+ monkeypatch.setattr("services.llm_gateway.embeddings", gateway_embeddings)
+
+ file_path = tmp_path / "doc.md"
+ file_path.write_text("# Test\n\nhello world", encoding="utf-8")
+ processor = TaskProcessor(
+ document_service=SimpleNamespace(
+ session_manager=SessionManager(),
+ document_index_writer=Writer(),
+ ),
+ models_service=ModelsService(),
+ docling_service=None,
+ )
+
+ async def document_missing(*args, **kwargs):
+ return False
+
+ monkeypatch.setattr(processor, "check_document_exists", document_missing)
+
+ result = await processor.process_document_standard(
+ file_path=str(file_path),
+ file_hash="file-1",
+ owner_user_id="user-1",
+ jwt_token="Bearer user-token",
+ )
+
+ assert result == {"status": "indexed", "id": "file-1"}
+ assert gateway_calls == [
+ {
+ "model": "watsonx_onprem:ibm/slate-30m-english-rtrvr",
+ "input": ["# Test\n\nhello world"],
+ }
+ ]
+ assert indexed[0][0].embedding_provider == "watsonx_onprem"
+ assert indexed[0][1][0].vector == [0.1, 0.2, 0.3]
diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.py
index ee33e6ab6..d3170d962 100644
--- a/tests/unit/test_processors_clear_stale_chunks.py
+++ b/tests/unit/test_processors_clear_stale_chunks.py
@@ -72,6 +72,7 @@ def _patch_embedding_pipeline(monkeypatch, chunk_count: int, write_client=None):
# Embedding model resolution path (config + fallback).
fake_config = MagicMock()
fake_config.knowledge.embedding_model = "text-embedding-3-small"
+ fake_config.knowledge.embedding_provider = "openai"
monkeypatch.setattr(processors_mod, "get_openrag_config", lambda: fake_config)
monkeypatch.setattr(processors_mod, "get_embedding_model", lambda: "text-embedding-3-small")
monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")