Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: added docling authentication when ibm auth is enabled (#1551)

Lucas Oliveira2b275363e408
diff --git a/.env.example b/.env.example
index 0f933bc48..02fb7451c 100644
--- a/.env.example
+++ b/.env.example
@@ -107,6 +107,11 @@ IBM_SESSION_COOKIE_NAME=ibm-openrag-session
# Header injected by Traefik on every forwarded request containing the OpenSearch Basic credentials
# (base64-encoded username:password). Override if your proxy uses a different header name.
IBM_CREDENTIALS_HEADER=X-IBM-LH-Credentials
+# Optional: Use these to override authentication for local development/calls when IBM_AUTH_ENABLED=true
+IBM_USERNAME=
+IBM_PASSWORD=
+# Set to true to disable OpenSearch security roles/mapping setup during onboarding/startup
+IBM_AUTH_DEV_MODE=false
# make here https://console.cloud.google.com/apis/credentials
GOOGLE_OAUTH_CLIENT_ID=
@@ -168,6 +173,7 @@ LANGFLOW_PUBLIC_URL=
# OPTIONAL: Override the full docling-serve URL (e.g., for remote instances)
# If not set, auto-detects host and uses port 5001
# DOCLING_SERVE_URL=http://my-docling-server:5001
+# DOCLING_SERVE_VERIFY_SSL=true
# OPTIONAL: OCR engine used by docling for scanned documents. Options: "easyocr" (default), "tesseract", "tesseract_cli"
# DOCLING_OCR_ENGINE=easyocr
diff --git a/.github/workflows/autofix.ci.yml b/.github/workflows/autofix.ci.yml
index 9c09c03f5..6c3cf9a94 100644
--- a/.github/workflows/autofix.ci.yml
+++ b/.github/workflows/autofix.ci.yml
@@ -54,7 +54,7 @@ jobs:
base="${{ github.event.pull_request.base.sha }}"
head="${{ github.event.pull_request.head.sha }}"
merge_base=$(git merge-base "$base" "$head")
- mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py')
+ mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py' ':!flows/components/*')
if [ "${#files[@]}" -eq 0 ]; then
echo "No Python files changed."
echo "files=" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/lint-backend.yml b/.github/workflows/lint-backend.yml
index 7e447ad10..24f6fc8b5 100644
--- a/.github/workflows/lint-backend.yml
+++ b/.github/workflows/lint-backend.yml
@@ -45,7 +45,7 @@ jobs:
head="${{ github.event.pull_request.head.sha }}"
# Diff against the merge base so we only see files touched in the PR.
merge_base=$(git merge-base "$base" "$head")
- mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py')
+ mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py' ':!flows/components/*')
if [ "${#files[@]}" -eq 0 ]; then
echo "No Python files changed."
echo "files=" >> "$GITHUB_OUTPUT"
diff --git a/docker-compose.yml b/docker-compose.yml
index 3b8d82bbe..004e96f0a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -112,7 +112,12 @@ services:
- INGESTION_TIMEOUT=${INGESTION_TIMEOUT:-3600}
- UPLOAD_BATCH_SIZE=${UPLOAD_BATCH_SIZE:-25}
- MAX_WORKERS=${MAX_WORKERS:-4}
+ - IBM_AUTH_ENABLED=${IBM_AUTH_ENABLED:-false}
+ - PLATFORM_AUTH_DEV_MODE=${PLATFORM_AUTH_DEV_MODE:-false}
+ - PLATFORM_USERNAME=${PLATFORM_USERNAME}
+ - PLATFORM_PASSWORD=${PLATFORM_PASSWORD}
- DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001}
+ - DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true}
volumes:
- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:U,z
- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z
diff --git a/flows/components/docling_remote.py b/flows/components/docling_remote.py
new file mode 100644
index 000000000..177f3d455
--- /dev/null
+++ b/flows/components/docling_remote.py
@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+import base64
+import json
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import httpx
+from docling_core.types.doc import DoclingDocument
+from pydantic import ValidationError
+
+from lfx.base.data import BaseFileComponent
+from lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput
+from lfx.inputs.inputs import FloatInput
+from lfx.schema import Data
+from lfx.schema.dotdict import dotdict
+from lfx.schema.message import Message
+from lfx.utils.util import transform_localhost_url
+
+
+class DoclingRemoteComponent(BaseFileComponent):
+ display_name = "Docling Serve"
+ description = (
+ "Uses Docling to process input documents connecting to your instance of Docling Serve."
+ )
+ documentation = "https://docling-project.github.io/docling/"
+ trace_type = "tool"
+ icon = "Docling"
+ name = "DoclingRemote"
+
+ MAX_500_RETRIES = 5
+
+ # https://docling-project.github.io/docling/usage/supported_formats/
+ VALID_EXTENSIONS = [
+ "adoc",
+ "asciidoc",
+ "asc",
+ "bmp",
+ "csv",
+ "dotx",
+ "dotm",
+ "docm",
+ "docx",
+ "htm",
+ "html",
+ "jpeg",
+ "jpg",
+ "json",
+ "md",
+ "pdf",
+ "png",
+ "potx",
+ "ppsx",
+ "pptm",
+ "potm",
+ "ppsm",
+ "pptx",
+ "tiff",
+ "txt",
+ "xls",
+ "xlsx",
+ "xhtml",
+ "xml",
+ "webp",
+ ]
+
+ inputs = [
+ *BaseFileComponent.get_base_inputs(),
+ StrInput(
+ name="api_url",
+ display_name="Server address",
+ info="URL of the Docling Serve instance.",
+ required=True,
+ ),
+ StrInput(
+ name="task_id",
+ display_name="Task ID",
+ info=(
+ "Optional task ID from a previous Docling Serve upload. "
+ "If provided, file input is ignored and the component polls for this task's results."
+ ),
+ required=False,
+ ),
+ IntInput(
+ name="max_concurrency",
+ display_name="Concurrency",
+ info="Maximum number of concurrent requests for the server.",
+ advanced=True,
+ value=2,
+ input_types=["Message"],
+ ),
+ FloatInput(
+ name="max_poll_timeout",
+ display_name="Maximum poll time",
+ info="Maximum waiting time for the document conversion to complete.",
+ advanced=True,
+ value=3600,
+ input_types=["Message"],
+ ),
+ TableInput(
+ name="api_headers",
+ display_name="HTTP headers",
+ advanced=True,
+ required=False,
+ info=("Optional headers required for connecting to Docling Serve."),
+ table_schema=[
+ {
+ "name": "key",
+ "display_name": "Key",
+ "type": "string",
+ "description": "Key name",
+ },
+ {
+ "name": "value",
+ "display_name": "Value",
+ "load_from_db": True,
+ "type": "string",
+ "description": "Value of the header",
+ },
+ ],
+ value=[],
+ real_time_refresh=True,
+ input_types=["Data", "JSON"],
+ ),
+ NestedDictInput(
+ name="docling_serve_opts",
+ display_name="Docling options",
+ advanced=True,
+ required=False,
+ info=(
+ "Optional dictionary of additional options. "
+ "See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information."
+ ),
+ input_types=["Message"],
+ ),
+ ]
+
+ outputs = [
+ *BaseFileComponent.get_base_outputs(),
+ ]
+
+ def _process_headers(self) -> dict[str, str]:
+ """Process the headers input into a valid dictionary."""
+ if not self.api_headers:
+ return {}
+
+ component_headers_dict = {}
+ # TableInput normalizes to list
+ items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]
+
+ for item in items:
+ if not item:
+ continue
+
+ # Case 1: Data object
+ if hasattr(item, "data") and isinstance(item.data, dict):
+ data = item.data
+ if "key" in data and "value" in data:
+ component_headers_dict[str(data["key"])] = str(data["value"])
+ else:
+ # Fallback: merge all keys from Data object
+ for k, v in data.items():
+ if k not in ("text_key", "default_value"):
+ component_headers_dict[str(k)] = str(v)
+
+ # Case 2: Dictionary (Table row)
+ elif isinstance(item, dict):
+ if "key" in item and "value" in item:
+ component_headers_dict[str(item["key"])] = str(item["value"])
+ else:
+ # Fallback: merge all keys
+ for k, v in item.items():
+ component_headers_dict[str(k)] = str(v)
+
+ # Case 3: Message object
+ elif hasattr(item, "text") and isinstance(item.text, str):
+ try:
+ parsed = json.loads(item.text)
+ if isinstance(parsed, dict):
+ for k, v in parsed.items():
+ component_headers_dict[str(k)] = str(v)
+ except json.JSONDecodeError:
+ pass
+
+ return component_headers_dict
+
+ def update_build_config(
+ self, build_config: dotdict, field_value: Any, field_name: str | None = None
+ ) -> dotdict:
+ if field_name == "api_headers":
+ if isinstance(field_value, dict):
+ # If it's a dict, convert to list of {key, value} pairs for TableInput
+ # This handles migration from NestedDictInput to TableInput
+ new_value = [{"key": k, "value": v} for k, v in field_value.items()]
+ build_config["api_headers"]["value"] = new_value
+ return build_config
+ if field_value is None:
+ build_config["api_headers"]["value"] = []
+ return build_config
+
+ # Default behavior
+ return super().update_build_config(build_config, field_value, field_name)
+
+ def _poll_and_fetch_result(
+ self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None
+ ) -> Data | None:
+ """Poll for task completion and fetch the result.
+
+ Args:
+ client: The HTTP client to use for requests.
+ base_url: The base URL of the Docling Serve API.
+ task_id: The task ID to poll for.
+ file_path: Optional file path to include in the result data.
+
+ Returns:
+ Data object with the DoclingDocument, or None if processing failed.
+ """
+ http_failures = 0
+ retry_status_start = 500
+ retry_status_end = 600
+ start_wait_time = time.monotonic()
+
+ response = client.get(f"{base_url}/status/poll/{task_id}")
+ response.raise_for_status()
+ task = response.json()
+
+ while task["task_status"] not in ("success", "failure"):
+ processing_time = time.monotonic() - start_wait_time
+ if processing_time >= self.max_poll_timeout:
+ msg = (
+ f"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}."
+ "Please increase the max_poll_timeout parameter or review why the processing "
+ "takes long on the server."
+ )
+ self.log(msg)
+ raise RuntimeError(msg)
+
+ time.sleep(2)
+ response = client.get(f"{base_url}/status/poll/{task_id}")
+
+ if retry_status_start <= response.status_code < retry_status_end:
+ http_failures += 1
+ if http_failures > self.MAX_500_RETRIES:
+ self.log(
+ f"The status requests got a http response {response.status_code} too many times."
+ )
+ return None
+ continue
+
+ task = response.json()
+
+ result_resp = client.get(f"{base_url}/result/{task_id}")
+ result_resp.raise_for_status()
+ result = result_resp.json()
+
+ if "json_content" not in result["document"] or result["document"]["json_content"] is None:
+ self.log("No JSON DoclingDocument found in the result.")
+ return None
+
+ try:
+ doc = DoclingDocument.model_validate(result["document"]["json_content"])
+ data_dict: dict[str, Any] = {"doc": doc}
+ if file_path:
+ data_dict["file_path"] = file_path
+ return Data(data=data_dict)
+ except ValidationError as e:
+ self.log(f"Error validating the document. {e}")
+ return None
+
+ def _process_task_id(self) -> list[Data]:
+ """Process an existing task by polling for status and retrieving results.
+
+ Returns:
+ List containing the result Data object, or empty list if processing failed.
+ """
+ transformed_url = transform_localhost_url(self.api_url)
+ base_url = f"{transformed_url}/v1"
+
+ with httpx.Client(headers=self._process_headers()) as client:
+ result = self._poll_and_fetch_result(client, base_url, self.task_id)
+ return [result] if result else []
+
+ def load_files_base(self) -> list[Data]:
+ """Load and process files, or poll an existing task if task_id is provided.
+
+ Returns:
+ list[Data]: Parsed data from the processed files or task.
+ """
+ if self.task_id:
+ return self._process_task_id()
+ return super().load_files_base()
+
+ def process_files(
+ self, file_list: list[BaseFileComponent.BaseFile]
+ ) -> list[BaseFileComponent.BaseFile]:
+ transformed_url = transform_localhost_url(self.api_url)
+ base_url = f"{transformed_url}/v1"
+
+ def _convert_document(
+ client: httpx.Client, file_path: Path, options: dict[str, Any]
+ ) -> Data | None:
+ encoded_doc = base64.b64encode(file_path.read_bytes()).decode()
+ payload = {
+ "options": options,
+ "sources": [
+ {"kind": "file", "base64_string": encoded_doc, "filename": file_path.name}
+ ],
+ }
+
+ response = client.post(f"{base_url}/convert/source/async", json=payload)
+ response.raise_for_status()
+ task = response.json()
+
+ return self._poll_and_fetch_result(client, base_url, task["task_id"], str(file_path))
+
+ docling_options = {
+ "to_formats": ["json"],
+ "image_export_mode": "placeholder",
+ **(self.docling_serve_opts or {}),
+ }
+
+ processed_data: list[Data | None] = []
+ with (
+ httpx.Client(headers=self._process_headers()) as client,
+ ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,
+ ):
+ futures: list[tuple[int, Future]] = []
+ for i, file in enumerate(file_list):
+ if file.path is None:
+ processed_data.append(None)
+ continue
+
+ futures.append(
+ (i, executor.submit(_convert_document, client, file.path, docling_options))
+ )
+
+ for _index, future in futures:
+ try:
+ result_data = future.result()
+ processed_data.append(result_data)
+ except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:
+ self.log(f"Docling remote processing failed: {exc}")
+ raise
+
+ return self.rollup_data(file_list, processed_data)
diff --git a/flows/components/mcp_component.py b/flows/components/mcp_component.py
new file mode 100644
index 000000000..f0bffb1ad
--- /dev/null
+++ b/flows/components/mcp_component.py
@@ -0,0 +1,835 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+
+from langchain_core.tools import StructuredTool # noqa: TC002
+
+from lfx.base.agents.utils import maybe_unflatten_dict, safe_cache_get, safe_cache_set
+from lfx.base.mcp.util import (
+ MCPStdioClient,
+ MCPStreamableHttpClient,
+ create_input_schema_from_json_schema,
+ update_tools,
+)
+from lfx.custom.custom_component.component_with_cache import ComponentWithCache
+from lfx.inputs.inputs import InputTypes # noqa: TC001
+from lfx.io import (
+ BoolInput,
+ DictInput,
+ DropdownInput,
+ McpInput,
+ MessageTextInput,
+ Output,
+ TableInput,
+)
+from lfx.io.schema import flatten_schema, schema_to_langflow_inputs
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.message import Message
+from lfx.services.deps import get_storage_service, session_scope
+
+
+def resolve_mcp_config(
+ server_name: str, # noqa: ARG001
+ server_config_from_value: dict | None,
+ server_config_from_db: dict | None,
+) -> dict | None:
+ """Resolve MCP server config with proper precedence.
+
+ Resolves the configuration for an MCP server with the following precedence:
+ 1. Database config (takes priority) - ensures edits are reflected
+ 2. Config from value/tweaks (fallback) - allows REST API to provide config for new servers
+
+ Args:
+ server_name: Name of the MCP server
+ server_config_from_value: Config provided via value/tweaks (optional)
+ server_config_from_db: Config from database (optional)
+
+ Returns:
+ Final config to use (DB takes priority, falls back to value)
+ Returns None if no config found in either location
+ """
+ if server_config_from_db:
+ return server_config_from_db
+ return server_config_from_value
+
+
+class MCPToolsComponent(ComponentWithCache):
+ schema_inputs: list = []
+ tools: list[StructuredTool] = []
+ _not_load_actions: bool = False
+ _tool_cache: dict = {}
+ _last_selected_server: str | None = None # Cache for the last selected server
+
+ def __init__(self, **data) -> None:
+ super().__init__(**data)
+ # Initialize cache keys to avoid CacheMiss when accessing them
+ self._ensure_cache_structure()
+
+ # Initialize clients with access to the component cache
+ self.stdio_client: MCPStdioClient = MCPStdioClient(
+ component_cache=self._shared_component_cache
+ )
+ self.streamable_http_client: MCPStreamableHttpClient = MCPStreamableHttpClient(
+ component_cache=self._shared_component_cache
+ )
+
+ def _ensure_cache_structure(self):
+ """Ensure the cache has the required structure."""
+ # Check if servers key exists and is not CacheMiss
+ servers_value = safe_cache_get(self._shared_component_cache, "servers")
+ if servers_value is None:
+ safe_cache_set(self._shared_component_cache, "servers", {})
+
+ # Check if last_selected_server key exists and is not CacheMiss
+ last_server_value = safe_cache_get(self._shared_component_cache, "last_selected_server")
+ if last_server_value is None:
+ safe_cache_set(self._shared_component_cache, "last_selected_server", "")
+
+ default_keys: list[str] = [
+ "code",
+ "_type",
+ "tool_mode",
+ "tool_placeholder",
+ "mcp_server",
+ "tool",
+ "use_cache",
+ "verify_ssl",
+ "headers",
+ ]
+
+ display_name = "MCP Tools"
+ description = "Connect to an MCP server to use its tools."
+ documentation: str = "https://docs.langflow.org/mcp-tools"
+ icon = "Mcp"
+ name = "MCPTools"
+
+ inputs = [
+ McpInput(
+ name="mcp_server",
+ display_name="MCP Server",
+ info="Select the MCP Server that will be used by this component",
+ real_time_refresh=True,
+ ),
+ BoolInput(
+ name="use_cache",
+ display_name="Use Cached Server",
+ info=(
+ "Enable caching of MCP Server and tools to improve performance. "
+ "Disable to always fetch fresh tools and server updates."
+ ),
+ value=False,
+ advanced=True,
+ ),
+ BoolInput(
+ name="verify_ssl",
+ display_name="Verify SSL Certificate",
+ info=(
+ "Enable SSL certificate verification for HTTPS connections. "
+ "Disable only for development/testing with self-signed certificates."
+ ),
+ value=True,
+ advanced=True,
+ ),
+ TableInput(
+ name="headers",
+ display_name="Headers",
+ info=(
+ "HTTP headers to include with MCP server requests. "
+ "Useful for authentication (e.g., Authorization header). "
+ "These headers override any headers configured in the MCP server settings."
+ ),
+ advanced=True,
+ table_schema=[
+ {
+ "name": "key",
+ "display_name": "Key",
+ "type": "string",
+ "description": "Key name",
+ },
+ {
+ "name": "value",
+ "display_name": "Value",
+ "load_from_db": True,
+ "type": "string",
+ "description": "Value of the header",
+ },
+ ],
+ value={},
+ input_types=["Data", "JSON"],
+ ),
+ DropdownInput(
+ name="tool",
+ display_name="Tool",
+ options=[],
+ value="",
+ info="Select the tool to execute",
+ show=False,
+ required=True,
+ real_time_refresh=True,
+ refresh_button=True,
+ ),
+ MessageTextInput(
+ name="tool_placeholder",
+ display_name="Tool Placeholder",
+ info="Placeholder for the tool",
+ value="",
+ show=False,
+ tool_mode=False,
+ ),
+ ]
+
+ outputs = [
+ Output(display_name="Response", name="response", method="build_output"),
+ ]
+
+ async def _validate_schema_inputs(self, tool_obj) -> list[InputTypes]:
+ """Validate and process schema inputs for a tool."""
+ try:
+ if not tool_obj or not hasattr(tool_obj, "args_schema"):
+ msg = "Invalid tool object or missing input schema"
+ raise ValueError(msg)
+
+ flat_schema = flatten_schema(tool_obj.args_schema.schema())
+ input_schema = create_input_schema_from_json_schema(flat_schema)
+ if not input_schema:
+ msg = f"Empty input schema for tool '{tool_obj.name}'"
+ raise ValueError(msg)
+
+ schema_inputs = schema_to_langflow_inputs(input_schema)
+ if not schema_inputs:
+ msg = f"No input parameters defined for tool '{tool_obj.name}'"
+ await logger.awarning(msg)
+ return []
+
+ except Exception as e:
+ msg = f"Error validating schema inputs: {e!s}"
+ await logger.aexception(msg)
+ raise ValueError(msg) from e
+ else:
+ return schema_inputs
+
+ async def update_tool_list(self, mcp_server_value=None):
+ # Accepts mcp_server_value as dict {name, config} or uses self.mcp_server
+ mcp_server = (
+ mcp_server_value if mcp_server_value is not None else getattr(self, "mcp_server", None)
+ )
+ server_name = None
+ server_config_from_value = None
+ if isinstance(mcp_server, dict):
+ server_name = mcp_server.get("name")
+ server_config_from_value = mcp_server.get("config")
+ else:
+ server_name = mcp_server
+ if not server_name:
+ self.tools = []
+ return [], {"name": server_name, "config": server_config_from_value}
+
+ # Check if caching is enabled, default to False
+ use_cache = getattr(self, "use_cache", False)
+
+ # Use shared cache if available and caching is enabled
+ cached = None
+ if use_cache:
+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ cached = servers_cache.get(server_name) if isinstance(servers_cache, dict) else None
+
+ if cached is not None:
+ try:
+ self.tools = cached["tools"]
+ self.tool_names = cached["tool_names"]
+ self._tool_cache = cached["tool_cache"]
+ server_config_from_value = cached["config"]
+ except (TypeError, KeyError, AttributeError) as e:
+ # Handle corrupted cache data by clearing it and continuing to fetch fresh tools
+ msg = f"Unable to use cached data for MCP Server{server_name}: {e}"
+ await logger.awarning(msg)
+ # Clear the corrupted cache entry
+ current_servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ if isinstance(current_servers_cache, dict) and server_name in current_servers_cache:
+ current_servers_cache.pop(server_name)
+ safe_cache_set(self._shared_component_cache, "servers", current_servers_cache)
+ else:
+ return self.tools, {"name": server_name, "config": server_config_from_value}
+
+ try:
+ # Try to fetch from database first to ensure we have the latest config
+ # This ensures database updates (like editing a server) take effect
+ try:
+ from langflow.api.v2.mcp import get_server
+ from langflow.services.database.models.user.crud import get_user_by_id
+
+ from lfx.services.deps import get_settings_service
+ except ImportError as e:
+ msg = (
+ "Langflow MCP server functionality is not available. "
+ "This feature requires the full Langflow installation."
+ )
+ raise ImportError(msg) from e
+
+ server_config_from_db = None
+ async with session_scope() as db:
+ if not self.user_id:
+ msg = "User ID is required for fetching MCP tools."
+ raise ValueError(msg)
+ current_user = await get_user_by_id(db, self.user_id)
+
+ # Try to get server config from DB/API
+ server_config_from_db = await get_server(
+ server_name,
+ current_user,
+ db,
+ storage_service=get_storage_service(),
+ settings_service=get_settings_service(),
+ )
+
+ # Resolve config with proper precedence: DB takes priority, falls back to value
+ server_config = resolve_mcp_config(
+ server_name=server_name,
+ server_config_from_value=server_config_from_value,
+ server_config_from_db=server_config_from_db,
+ )
+
+ if not server_config:
+ self.tools = []
+ return [], {"name": server_name, "config": server_config}
+
+ # Add verify_ssl option to server config if not present
+ if "verify_ssl" not in server_config:
+ verify_ssl = getattr(self, "verify_ssl", True)
+ server_config["verify_ssl"] = verify_ssl
+
+ # Merge headers from component input with server config headers
+ # Component headers take precedence over server config headers
+ component_headers = getattr(self, "headers", None)
+ if component_headers:
+ # Convert list of {"key": k, "value": v}, Data objects, or dicts to dict
+ component_headers_dict = {}
+
+ # TableInput.validate_value normalizes single items to lists
+ items = (
+ component_headers
+ if isinstance(component_headers, list)
+ else [component_headers]
+ )
+
+ for item in items:
+ if not item:
+ continue
+
+ # Case 1: Data object (Langflow standard)
+ if hasattr(item, "data") and isinstance(item.data, dict):
+ data = item.data
+ if "key" in data and "value" in data:
+ component_headers_dict[str(data["key"])] = str(data["value"])
+ else:
+ # Fallback: merge all keys from Data object (excluding internal keys)
+ for k, v in data.items():
+ if k not in ("text_key", "default_value"):
+ component_headers_dict[str(k)] = str(v)
+
+ # Case 2: Dictionary (Table row or direct JSON)
+ elif isinstance(item, dict):
+ if "key" in item and "value" in item:
+ component_headers_dict[str(item["key"])] = str(item["value"])
+ else:
+ # Fallback: merge all keys
+ for k, v in item.items():
+ component_headers_dict[str(k)] = str(v)
+
+ # Case 3: Message object
+ elif hasattr(item, "text") and isinstance(item.text, str):
+ try:
+ parsed = json.loads(item.text)
+ if isinstance(parsed, dict):
+ for k, v in parsed.items():
+ component_headers_dict[str(k)] = str(v)
+ except json.JSONDecodeError:
+ pass
+
+ if component_headers_dict:
+ existing_headers = server_config.get("headers", {}) or {}
+ # Ensure existing_headers is a dict (convert from list if needed)
+ if isinstance(existing_headers, list):
+ existing_dict = {}
+ for item in existing_headers:
+ if isinstance(item, dict) and "key" in item and "value" in item:
+ existing_dict[item["key"]] = item["value"]
+ existing_headers = existing_dict
+ merged_headers = {**existing_headers, **component_headers_dict}
+ server_config["headers"] = merged_headers
+ # Get request_variables from graph context for global variable resolution
+ request_variables = None
+ if hasattr(self, "graph") and self.graph and hasattr(self.graph, "context"):
+ request_variables = self.graph.context.get("request_variables")
+
+ # Only load global variables from database if we have headers that might use them
+ # This avoids unnecessary database queries when headers are empty
+ has_headers = server_config.get("headers") and len(server_config.get("headers", {})) > 0
+ if not request_variables and has_headers:
+ try:
+ from lfx.services.deps import get_variable_service
+
+ variable_service = get_variable_service()
+ if variable_service:
+ async with session_scope() as db:
+ request_variables = await variable_service.get_all_decrypted_variables(
+ user_id=self.user_id, session=db
+ )
+ except Exception as e: # noqa: BLE001
+ await logger.awarning(f"Failed to load global variables for MCP component: {e}")
+
+ _, tool_list, tool_cache = await update_tools(
+ server_name=server_name,
+ server_config=server_config,
+ mcp_stdio_client=self.stdio_client,
+ mcp_streamable_http_client=self.streamable_http_client,
+ request_variables=request_variables,
+ )
+
+ self.tool_names = [tool.name for tool in tool_list if hasattr(tool, "name")]
+ self._tool_cache = tool_cache
+ self.tools = tool_list
+
+ # Cache the result only if caching is enabled
+ if use_cache:
+ cache_data = {
+ "tools": tool_list,
+ "tool_names": self.tool_names,
+ "tool_cache": tool_cache,
+ "config": server_config,
+ }
+
+ # Safely update the servers cache
+ current_servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ if isinstance(current_servers_cache, dict):
+ current_servers_cache[server_name] = cache_data
+ safe_cache_set(self._shared_component_cache, "servers", current_servers_cache)
+
+ except (TimeoutError, asyncio.TimeoutError) as e:
+ msg = f"Timeout updating tool list: {e!s}"
+ await logger.aexception(msg)
+ raise TimeoutError(msg) from e
+ except Exception as e:
+ msg = f"Error updating tool list: {e!s}"
+ await logger.aexception(msg)
+ raise ValueError(msg) from e
+ else:
+ return tool_list, {"name": server_name, "config": server_config}
+
+ async def update_build_config(
+ self, build_config: dict, field_value: str, field_name: str | None = None
+ ) -> dict:
+ """Toggle the visibility of connection-specific fields based on the selected mode."""
+ try:
+ if field_name == "tool":
+ try:
+ # Always refresh tools when cache is disabled, or when tools list is empty
+ # This ensures database edits are reflected immediately when cache is disabled
+ use_cache = getattr(self, "use_cache", False)
+ if len(self.tools) == 0 or not use_cache:
+ try:
+ (
+ self.tools,
+ build_config["mcp_server"]["value"],
+ ) = await self.update_tool_list()
+ build_config["tool"]["options"] = [tool.name for tool in self.tools]
+ build_config["tool"]["placeholder"] = "Select a tool"
+ except (TimeoutError, asyncio.TimeoutError) as e:
+ msg = f"Timeout updating tool list: {e!s}"
+ await logger.aexception(msg)
+ if not build_config["tools_metadata"]["show"]:
+ build_config["tool"]["show"] = True
+ build_config["tool"]["options"] = []
+ build_config["tool"]["value"] = ""
+ build_config["tool"]["placeholder"] = "Timeout on MCP server"
+ else:
+ build_config["tool"]["show"] = False
+ except ValueError:
+ if not build_config["tools_metadata"]["show"]:
+ build_config["tool"]["show"] = True
+ build_config["tool"]["options"] = []
+ build_config["tool"]["value"] = ""
+ build_config["tool"]["placeholder"] = "Error on MCP Server"
+ else:
+ build_config["tool"]["show"] = False
+
+ if field_value == "":
+ return build_config
+ tool_obj = None
+ for tool in self.tools:
+ if tool.name == field_value:
+ tool_obj = tool
+ break
+ if tool_obj is None:
+ msg = f"Tool {field_value} not found in available tools: {self.tools}"
+ await logger.awarning(msg)
+ return build_config
+ await self._update_tool_config(build_config, field_value)
+ except Exception as e:
+ build_config["tool"]["options"] = []
+ msg = f"Failed to update tools: {e!s}"
+ raise ValueError(msg) from e
+ else:
+ return build_config
+ elif field_name == "mcp_server":
+ if not field_value:
+ build_config["tool"]["show"] = False
+ build_config["tool"]["options"] = []
+ build_config["tool"]["value"] = ""
+ build_config["tool"]["placeholder"] = ""
+ build_config["tool_placeholder"]["tool_mode"] = False
+ self.remove_non_default_keys(build_config)
+ return build_config
+
+ build_config["tool_placeholder"]["tool_mode"] = True
+
+ current_server_name = (
+ field_value.get("name") if isinstance(field_value, dict) else field_value
+ )
+ _last_selected_server = safe_cache_get(
+ self._shared_component_cache, "last_selected_server", ""
+ )
+ # Only treat as a server change if there was a previous server selection.
+ # Cold cache (_last_selected_server="") on initial flow load is NOT a server change —
+ # the user didn't switch anything, the backend just hasn't seen this component yet.
+ server_changed = bool(
+ _last_selected_server and current_server_name != _last_selected_server
+ )
+
+ # Determine if "Tool Mode" is active by checking if the tool dropdown is hidden.
+ is_in_tool_mode = build_config["tools_metadata"]["show"]
+
+ # Get use_cache setting to determine if we should use cached data
+ use_cache = getattr(self, "use_cache", False)
+
+ # Fast path: if server didn't change and we already have options, keep them as-is
+ # BUT only if caching is enabled, we're in tool mode, or it's the initial load
+ existing_options = build_config.get("tool", {}).get("options") or []
+ if not server_changed and existing_options:
+ # In non-tool mode with cache disabled, skip the fast path to force refresh
+ # BUT on initial load (cold cache), always preserve saved options from the flow
+ if not is_in_tool_mode and not use_cache and _last_selected_server:
+ pass # Continue to refresh logic below (user-initiated with cache disabled)
+ else:
+ if not is_in_tool_mode:
+ build_config["tool"]["show"] = True
+ safe_cache_set(
+ self._shared_component_cache,
+ "last_selected_server",
+ current_server_name,
+ )
+ return build_config
+
+ # To avoid unnecessary updates, only proceed if the server has actually changed
+ # OR if caching is disabled (to force refresh in non-tool mode)
+ if (
+ (_last_selected_server in (current_server_name, ""))
+ and build_config["tool"]["show"]
+ and use_cache
+ ):
+ if current_server_name:
+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ if isinstance(servers_cache, dict):
+ cached = servers_cache.get(current_server_name)
+ if cached is not None and cached.get("tool_names"):
+ cached_tools = cached["tool_names"]
+ current_tools = build_config["tool"]["options"]
+ if current_tools == cached_tools:
+ return build_config
+ else:
+ return build_config
+ safe_cache_set(
+ self._shared_component_cache, "last_selected_server", current_server_name
+ )
+
+ # When cache is disabled, clear any cached data for this server
+ # This ensures we always fetch fresh data from the database
+ if not use_cache and current_server_name:
+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ if isinstance(servers_cache, dict) and current_server_name in servers_cache:
+ servers_cache.pop(current_server_name)
+ safe_cache_set(self._shared_component_cache, "servers", servers_cache)
+
+ # Check if tools are already cached for this server before clearing
+ cached_tools = None
+ if current_server_name and use_cache:
+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})
+ if isinstance(servers_cache, dict):
+ cached = servers_cache.get(current_server_name)
+ if cached is not None:
+ try:
+ cached_tools = cached["tools"]
+ self.tools = cached_tools
+ self.tool_names = cached["tool_names"]
+ self._tool_cache = cached["tool_cache"]
+ except (TypeError, KeyError, AttributeError) as e:
+ # Handle corrupted cache data by ignoring it
+ msg = f"Unable to use cached data for MCP Server,{current_server_name}: {e}"
+ await logger.awarning(msg)
+ cached_tools = None
+
+ # Clear tools when cache is disabled OR when we don't have cached tools
+ # This ensures fresh tools are fetched after database edits
+ if not cached_tools or not use_cache:
+ self.tools = [] # Clear previous tools to force refresh
+
+ # Clear previous tool inputs if:
+ # 1. Server actually changed
+ # 2. Cache is disabled (meaning tool list will be refreshed)
+ if server_changed or not use_cache:
+ self.remove_non_default_keys(build_config)
+
+ # Only show the tool dropdown if not in tool_mode
+ if not is_in_tool_mode:
+ build_config["tool"]["show"] = True
+ if cached_tools:
+ # Use cached tools to populate options immediately
+ build_config["tool"]["options"] = [tool.name for tool in cached_tools]
+ build_config["tool"]["placeholder"] = "Select a tool"
+ else:
+ # Actually fetch tools now instead of deferring to a frontend callback.
+ # The frontend has no reliable mechanism to trigger a second
+ # update_build_config call for the "tool" field after this response,
+ # so we must populate the options here.
+ try:
+ (
+ self.tools,
+ build_config["mcp_server"]["value"],
+ ) = await self.update_tool_list(mcp_server_value=field_value)
+ build_config["tool"]["options"] = [tool.name for tool in self.tools]
+ build_config["tool"]["placeholder"] = "Select a tool"
+ except (TimeoutError, asyncio.TimeoutError) as e:
+ msg = f"Timeout loading tools for MCP server: {e!s}"
+ await logger.awarning(msg)
+ build_config["tool"]["options"] = []
+ build_config["tool"]["placeholder"] = "Timeout on MCP server"
+ except (
+ ValueError,
+ ImportError,
+ ConnectionError,
+ OSError,
+ RuntimeError,
+ ) as e:
+ msg = f"Error loading tools for MCP server: {e!s}"
+ await logger.awarning(msg)
+ build_config["tool"]["options"] = []
+ build_config["tool"]["placeholder"] = "Error on MCP Server"
+ # Force a value refresh only when the user genuinely switched servers.
+ # server_changed is only True for real user-initiated changes (not initial load).
+ if server_changed:
+ build_config["tool"]["value"] = uuid.uuid4()
+ else:
+ # Keep the tool dropdown hidden if in tool_mode
+ self._not_load_actions = True
+ build_config["tool"]["show"] = False
+
+ elif field_name == "tool_mode":
+ build_config["tool"]["placeholder"] = ""
+ build_config["tool"]["show"] = not bool(field_value) and bool(
+ build_config["mcp_server"]
+ )
+ self.remove_non_default_keys(build_config)
+ self.tool = build_config["tool"]["value"]
+ if field_value:
+ self._not_load_actions = True
+ else:
+ build_config["tool"]["value"] = uuid.uuid4()
+ build_config["tool"]["show"] = True
+ # Fetch tools immediately instead of showing "Loading tools..."
+ try:
+ (
+ self.tools,
+ build_config["mcp_server"]["value"],
+ ) = await self.update_tool_list()
+ build_config["tool"]["options"] = [tool.name for tool in self.tools]
+ build_config["tool"]["placeholder"] = "Select a tool"
+ except (TimeoutError, asyncio.TimeoutError) as e:
+ msg = f"Timeout loading tools when toggling tool mode: {e!s}"
+ await logger.awarning(msg)
+ build_config["tool"]["options"] = []
+ build_config["tool"]["placeholder"] = "Timeout on MCP server"
+ except (ValueError, ImportError, ConnectionError, OSError, RuntimeError) as e:
+ msg = f"Error loading tools when toggling tool mode: {e!s}"
+ await logger.awarning(msg)
+ build_config["tool"]["options"] = []
+ build_config["tool"]["placeholder"] = "Error on MCP Server"
+ elif field_name == "tools_metadata":
+ self._not_load_actions = False
+
+ except Exception as e:
+ msg = f"Error in update_build_config: {e!s}"
+ await logger.aexception(msg)
+ raise ValueError(msg) from e
+ else:
+ return build_config
+
+ def get_inputs_for_all_tools(self, tools: list) -> dict:
+ """Get input schemas for all tools."""
+ inputs = {}
+ for tool in tools:
+ if not tool or not hasattr(tool, "name"):
+ continue
+ try:
+ flat_schema = flatten_schema(tool.args_schema.schema())
+ input_schema = create_input_schema_from_json_schema(flat_schema)
+ langflow_inputs = schema_to_langflow_inputs(input_schema)
+ inputs[tool.name] = langflow_inputs
+ except (AttributeError, ValueError, TypeError, KeyError) as e:
+ msg = f"Error getting inputs for tool {getattr(tool, 'name', 'unknown')}: {e!s}"
+ logger.exception(msg)
+ continue
+ return inputs
+
+ def remove_non_default_keys(self, build_config: dict) -> None:
+ """Remove non-default keys from the build config."""
+ for key in list(build_config.keys()):
+ if key not in self.default_keys:
+ build_config.pop(key)
+
+ async def _update_tool_config(self, build_config: dict, tool_name: str) -> None:
+ """Update tool configuration with proper error handling."""
+ if not self.tools:
+ self.tools, build_config["mcp_server"]["value"] = await self.update_tool_list()
+
+ if not tool_name:
+ return
+
+ tool_obj = next((tool for tool in self.tools if tool.name == tool_name), None)
+ if not tool_obj:
+ msg = f"Tool {tool_name} not found in available tools: {self.tools}"
+ self.remove_non_default_keys(build_config)
+ build_config["tool"]["value"] = ""
+ await logger.awarning(msg)
+ return
+
+ try:
+ # Store current values before removing inputs (only for the current tool)
+ current_values = {}
+ for key, value in build_config.items():
+ if key not in self.default_keys and isinstance(value, dict) and "value" in value:
+ current_values[key] = value["value"]
+
+ # Remove ALL non-default keys (all previous tool inputs)
+ self.remove_non_default_keys(build_config)
+
+ # Get and validate new inputs for the selected tool
+ self.schema_inputs = await self._validate_schema_inputs(tool_obj)
+ if not self.schema_inputs:
+ msg = f"No input parameters to configure for tool '{tool_name}'"
+ await logger.ainfo(msg)
+ return
+
+ # Add new inputs to build config for the selected tool only
+ for schema_input in self.schema_inputs:
+ if not schema_input or not hasattr(schema_input, "name"):
+ msg = "Invalid schema input detected, skipping"
+ await logger.awarning(msg)
+ continue
+
+ try:
+ name = schema_input.name
+ input_dict = schema_input.to_dict()
+ input_dict.setdefault("value", None)
+ input_dict.setdefault("required", True)
+
+ build_config[name] = input_dict
+
+ # Preserve existing value if the parameter name exists in current_values
+ if name in current_values:
+ build_config[name]["value"] = current_values[name]
+
+ except (AttributeError, KeyError, TypeError) as e:
+ msg = f"Error processing schema input {schema_input}: {e!s}"
+ await logger.aexception(msg)
+ continue
+ except ValueError as e:
+ msg = f"Schema validation error for tool {tool_name}: {e!s}"
+ await logger.aexception(msg)
+ self.schema_inputs = []
+ return
+ except (AttributeError, KeyError, TypeError) as e:
+ msg = f"Error updating tool config: {e!s}"
+ await logger.aexception(msg)
+ raise ValueError(msg) from e
+
+ async def build_output(self) -> DataFrame:
+ """Build output with improved error handling and validation."""
+ try:
+ self.tools, _ = await self.update_tool_list()
+ if self.tool != "":
+ # Set session context for persistent MCP sessions using Langflow session ID
+ session_context = self._get_session_context()
+ if session_context:
+ self.stdio_client.set_session_context(session_context)
+ self.streamable_http_client.set_session_context(session_context)
+ exec_tool = self._tool_cache[self.tool]
+ tool_args = self.get_inputs_for_all_tools(self.tools)[self.tool]
+ kwargs = {}
+ for arg in tool_args:
+ value = getattr(self, arg.name, None)
+ if value is not None:
+ if isinstance(value, Message):
+ kwargs[arg.name] = value.text
+ else:
+ kwargs[arg.name] = value
+
+ unflattened_kwargs = maybe_unflatten_dict(kwargs)
+
+ output = await exec_tool.coroutine(**unflattened_kwargs)
+ tool_content = []
+ for item in output.content:
+ item_dict = item.model_dump()
+ item_dict = self.process_output_item(item_dict)
+ tool_content.append(item_dict)
+
+ if isinstance(tool_content, list) and all(
+ isinstance(x, dict) for x in tool_content
+ ):
+ return DataFrame(tool_content)
+ return DataFrame(data=tool_content)
+ return DataFrame(data=[{"error": "You must select a tool"}])
+ except Exception as e:
+ msg = f"Error in build_output: {e!s}"
+ await logger.aexception(msg)
+ raise ValueError(msg) from e
+
+ def process_output_item(self, item_dict):
+ """Process the output of a tool."""
+ if item_dict.get("type") == "text":
+ text = item_dict.get("text")
+ try:
+ parsed = json.loads(text)
+ # Ensure we always return a dictionary for DataFrame compatibility
+ if isinstance(parsed, dict):
+ return parsed
+ # Wrap non-dict parsed values in a dictionary
+ return {"text": text, "parsed_value": parsed, "type": "text"} # noqa: TRY300
+ except json.JSONDecodeError:
+ return item_dict
+ return item_dict
+
+ def _get_session_context(self) -> str | None:
+ """Get the Langflow session ID for MCP session caching."""
+ # Try to get session ID from the component's execution context
+ if hasattr(self, "graph") and hasattr(self.graph, "session_id"):
+ session_id = self.graph.session_id
+ # Include server name to ensure different servers get different sessions
+ server_name = ""
+ mcp_server = getattr(self, "mcp_server", None)
+ if isinstance(mcp_server, dict):
+ server_name = mcp_server.get("name", "")
+ elif mcp_server:
+ server_name = str(mcp_server)
+ return f"{session_id}_{server_name}" if session_id else None
+ return None
+
+ async def _get_tools(self):
+ """Get cached tools or update if necessary."""
+ mcp_server = getattr(self, "mcp_server", None)
+ if not self._not_load_actions:
+ tools, _ = await self.update_tool_list(mcp_server)
+ return tools
+ return []
diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json
index 0ddb6b82c..79e309e64 100644
--- a/flows/ingestion_flow.json
+++ b/flows/ingestion_flow.json
@@ -828,7 +828,7 @@
],
"frozen": false,
"icon": "braces",
- "last_updated": "2026-05-05T13:33:37.431Z",
+ "last_updated": "2026-05-08T19:20:38.334Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {},
@@ -878,7 +878,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "48713154-dcea-4b42-b772-268029c455d7"
+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"
},
"_type": "Component",
"code": {
@@ -1336,9 +1336,10 @@
],
"frozen": false,
"icon": "Docling",
+ "last_updated": "2026-05-08T19:20:27.016Z",
"legacy": false,
"metadata": {
- "code_hash": "72fcef05e057",
+ "code_hash": "3a18033f9e4f",
"dependencies": {
"dependencies": [
{
@@ -1386,30 +1387,67 @@
],
"pinned": false,
"template": {
+ "_frontend_node_flow_id": {
+ "value": "5488df7c-b93f-4f87-a446-b67028bc0813"
+ },
+ "_frontend_node_folder_id": {
+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"
+ },
"_type": "Component",
"api_headers": {
- "_input_type": "NestedDictInput",
+ "_input_type": "TableInput",
"advanced": true,
"display_name": "HTTP headers",
"dynamic": false,
- "info": "Optional dictionary of additional headers required for connecting to Docling Serve.",
+ "info": "Optional headers required for connecting to Docling Serve.",
"input_types": [
- "Message"
+ "Data",
+ "JSON"
],
- "list": false,
+ "is_list": true,
"list_add_label": "Add More",
+ "load_from_db": false,
"name": "api_headers",
"override_skip": false,
"placeholder": "",
+ "real_time_refresh": true,
"required": false,
"show": true,
+ "table_icon": "Table",
+ "table_schema": [
+ {
+ "description": "Key name",
+ "display_name": "Key",
+ "formatter": "text",
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "description": "Value of the header",
+ "display_name": "Value",
+ "formatter": "text",
+ "load_from_db": true,
+ "name": "value",
+ "type": "string"
+ }
+ ],
"title_case": false,
"tool_mode": false,
- "trace_as_input": true,
"trace_as_metadata": true,
"track_in_telemetry": false,
- "type": "NestedDict",
- "value": {}
+ "trigger_icon": "Table",
+ "trigger_text": "Open table",
+ "type": "table",
+ "value": [
+ {
+ "key": "Authorization",
+ "value": "JWT"
+ },
+ {
+ "key": "X-Tenant-Id",
+ "value": "OWNER"
+ }
+ ]
},
"api_url": {
"_input_type": "StrInput",
@@ -1448,7 +1486,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self.api_headers) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"
+ "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self._process_headers()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"
},
"delete_server_file_after_processing": {
"_input_type": "BoolInput",
@@ -1569,6 +1607,7 @@
"type": "bool",
"value": true
},
+ "is_refresh": false,
"max_concurrency": {
"_input_type": "IntInput",
"advanced": true,
@@ -1745,14 +1784,14 @@
"dragging": false,
"id": "DoclingRemote-Dp3PX",
"measured": {
- "height": 397,
- "width": 320
+ "height": 52,
+ "width": 192
},
"position": {
- "x": -158.35302959149863,
- "y": 1766.5193585649456
+ "x": 4.296782621507347,
+ "y": 1764.5830512766954
},
- "selected": true,
+ "selected": false,
"type": "genericNode"
},
{
@@ -4641,7 +4680,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "48713154-dcea-4b42-b772-268029c455d7"
+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"
},
"_type": "Component",
"api_base": {
@@ -6483,7 +6522,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "48713154-dcea-4b42-b772-268029c455d7"
+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"
},
"_type": "Component",
"api_base": {
@@ -7101,7 +7140,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "48713154-dcea-4b42-b772-268029c455d7"
+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"
},
"_type": "Component",
"api_base": {
@@ -7643,9 +7682,9 @@
}
],
"viewport": {
- "x": 267.6557208461858,
- "y": -373.3174538705754,
- "zoom": 0.475706358533402
+ "x": 193.22171349647556,
+ "y": -404.7149184337984,
+ "zoom": 0.5164469534707714
}
},
"description": "Load your data for chat context with Retrieval Augmented Generation.",
diff --git a/pyproject.toml b/pyproject.toml
index 1a473ad2a..e82264c17 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,11 +91,15 @@ override-dependencies = ["python-dotenv>=1.1.0"]
[tool.ruff]
line-length = 100
target-version = "py313"
+exclude = ["flows/components"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
ignore = ["E501"] # line-length handled by formatter
+[tool.ruff.lint.flake8-bugbear]
+extend-immutable-calls = ["fastapi.Depends", "fastapi.params.Depends", "Depends"]
+
# `bootstrap` must be the first import in any module that uses it
# (it loads .env and configures structlog before anything else can
# read env vars or emit logs). Define it as its own isort section
@@ -119,5 +123,6 @@ python_version = "3.13"
ignore_missing_imports = true
no_strict_optional = true
warn_unused_ignores = true
+exclude = ["flows/components"]
diff --git a/sdks/mcp/src/openrag_mcp/config.py b/sdks/mcp/src/openrag_mcp/config.py
index 856ea763b..2ef0d88d6 100644
--- a/sdks/mcp/src/openrag_mcp/config.py
+++ b/sdks/mcp/src/openrag_mcp/config.py
@@ -44,13 +44,13 @@ class Config:
self.openrag_url = os.environ.get("OPENRAG_URL", "http://localhost:3000")
self.api_key = os.environ.get("OPENRAG_API_KEY")
- # IBM auth mode: user provides IBM_USERNAME + IBM_API_KEY instead of OPENRAG_API_KEY.
+ # Platform auth mode: user provides PLATFORM_USERNAME + IBM_API_KEY instead of OPENRAG_API_KEY.
# The MCP forwards these as headers on every SDK request to OpenRAG.
- ibm_username = os.environ.get("IBM_USERNAME")
+ platform_username = os.environ.get("PLATFORM_USERNAME")
ibm_api_key = os.environ.get("IBM_API_KEY")
self.ibm_extra_headers: dict[str, str] = {}
- if ibm_username:
- self.ibm_extra_headers["X-Username"] = ibm_username
+ if platform_username:
+ self.ibm_extra_headers["X-Username"] = platform_username
if ibm_api_key:
self.ibm_extra_headers["X-Api-Key"] = ibm_api_key
@@ -63,9 +63,7 @@ class Config:
# MCP httpx client configuration (OPENRAG_MCP_*)
self.mcp_timeout = _parse_float("OPENRAG_MCP_TIMEOUT", 60.0)
self.mcp_max_connections = _parse_int("OPENRAG_MCP_MAX_CONNECTIONS", 100)
- self.mcp_max_keepalive_connections = _parse_int(
- "OPENRAG_MCP_MAX_KEEPALIVE_CONNECTIONS", 20
- )
+ self.mcp_max_keepalive_connections = _parse_int("OPENRAG_MCP_MAX_KEEPALIVE_CONNECTIONS", 20)
self.mcp_max_retries = _parse_int("OPENRAG_MCP_MAX_RETRIES", 3)
self.mcp_follow_redirects = _parse_bool("OPENRAG_MCP_FOLLOW_REDIRECTS", True)
diff --git a/src/api/docling.py b/src/api/docling.py
index 25d6114a8..70ddf54a5 100644
--- a/src/api/docling.py
+++ b/src/api/docling.py
@@ -1,15 +1,17 @@
-"""Docling service proxy endpoints."""
-
-import os
-import socket
-import struct
-from pathlib import Path
+from typing import Annotated
import httpx
-from fastapi import Request
+from fastapi import Depends, Request
from fastapi.responses import JSONResponse
-from config.settings import DOCLING_SERVE_URL, DOCLING_HOST_IP
+from config.settings import (
+ DOCLING_HOST_IP,
+ DOCLING_SERVE_URL,
+ DOCLING_SERVE_VERIFY_SSL,
+ IBM_AUTH_ENABLED,
+)
+from dependencies import get_optional_user
+from session_manager import User
from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -20,43 +22,53 @@ DOCLING_SERVICE_URL = DOCLING_SERVE_URL
HOST_IP = DOCLING_HOST_IP
-async def health(request: Request) -> JSONResponse:
+async def health(
+ request: Request, user: Annotated[User | None, Depends(get_optional_user)] = None
+) -> JSONResponse:
"""
Proxy health check to docling-serve.
This allows the frontend to check docling status via same-origin request.
"""
health_url = f"{DOCLING_SERVICE_URL}/health"
+ headers = {}
+ if IBM_AUTH_ENABLED and user:
+ if user.jwt_token:
+ headers["Authorization"] = user.jwt_token
+ if user.user_id:
+ headers["X-Tenant-Id"] = user.user_id
+
try:
- async with httpx.AsyncClient() as client:
- response = await client.get(
- health_url,
- timeout=2.0
- )
+ async with httpx.AsyncClient(verify=DOCLING_SERVE_VERIFY_SSL) as client:
+ response = await client.get(health_url, headers=headers, timeout=2.0)
if response.status_code == 200:
- return JSONResponse({
- "status": "healthy",
- "host": HOST_IP
- })
+ return JSONResponse({"status": "healthy", "host": HOST_IP})
else:
- logger.warning("Docling health check failed", url=health_url, status_code=response.status_code)
- return JSONResponse({
- "status": "unhealthy",
- "message": f"Health check failed with status: {response.status_code}",
- "host": HOST_IP
- }, status_code=503)
+ logger.warning(
+ "Docling health check failed", url=health_url, status_code=response.status_code
+ )
+ return JSONResponse(
+ {
+ "status": "unhealthy",
+ "message": f"Health check failed with status: {response.status_code}",
+ "host": HOST_IP,
+ },
+ status_code=503,
+ )
except httpx.TimeoutException:
logger.warning("Docling health check timeout", url=health_url)
- return JSONResponse({
- "status": "unhealthy",
- "message": "Connection timeout",
- "host": HOST_IP
- }, status_code=503)
+ return JSONResponse(
+ {"status": "unhealthy", "message": "Connection timeout", "host": HOST_IP},
+ status_code=503,
+ )
except Exception as e:
logger.error("Docling health check failed", url=health_url, error=str(e))
- return JSONResponse({
- "status": "unhealthy",
- "message": str(e),
- "host": HOST_IP
- }, status_code=503)
+ return JSONResponse(
+ {
+ "status": "unhealthy",
+ "message": "Internal error while checking service health",
+ "host": HOST_IP,
+ },
+ status_code=503,
+ )
diff --git a/src/api/upload.py b/src/api/upload.py
index b36b9b0d4..e2712c8a0 100644
--- a/src/api/upload.py
+++ b/src/api/upload.py
@@ -1,20 +1,20 @@
-from dependencies import get_docling_service
-from dependencies import get_models_service
import os
-from typing import Optional
+from typing import Annotated, Any
from urllib.parse import urlparse
import boto3
from fastapi import Depends, File, Form, UploadFile
-from pydantic import BaseModel
from fastapi.responses import JSONResponse
+from pydantic import BaseModel
from dependencies import (
- get_document_service,
- get_task_service,
get_chat_service,
- get_session_manager,
get_current_user,
+ get_docling_service,
+ get_document_service,
+ get_models_service,
+ get_session_manager,
+ get_task_service,
require_all_permissions,
require_permission,
)
@@ -33,15 +33,15 @@ class UploadBucketBody(BaseModel):
async def upload(
- file: UploadFile = File(...),
- document_service=Depends(get_document_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(require_permission("knowledge:upload")),
+ file: Annotated[UploadFile, File(...)],
+ document_service: Annotated[Any, Depends(get_document_service)],
+ session_manager: Annotated[Any, Depends(get_session_manager)],
+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],
):
"""Upload a single file"""
try:
-
from config.settings import is_no_auth_mode
+
is_no_auth = is_no_auth_mode()
owner_user_id = user.user_id if (user and not is_no_auth) else None
owner_name = user.name if user else None
@@ -57,10 +57,7 @@ async def upload(
return JSONResponse(result, status_code=201)
except Exception as e:
error_msg = str(e)
- if (
- "AuthenticationException" in error_msg
- or "access denied" in error_msg.lower()
- ):
+ if "AuthenticationException" in error_msg or "access denied" in error_msg.lower():
logger.warning("[INGEST] Upload rejected — access denied", error=error_msg)
return JSONResponse({"error": error_msg}, status_code=403)
else:
@@ -70,17 +67,15 @@ async def upload(
async def upload_path(
body: UploadPathBody,
- task_service=Depends(get_task_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(require_permission("knowledge:upload")),
+ task_service: Annotated[Any, Depends(get_task_service)],
+ session_manager: Annotated[Any, Depends(get_session_manager)],
+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],
):
"""Upload all files from a directory path"""
if not body.path or not os.path.isdir(body.path):
return JSONResponse({"error": "Invalid path"}, status_code=400)
- file_paths = [
- os.path.join(root, fn) for root, _, files in os.walk(body.path) for fn in files
- ]
+ file_paths = [os.path.join(root, fn) for root, _, files in os.walk(body.path) for fn in files]
if not file_paths:
return JSONResponse({"error": "No files found in directory"}, status_code=400)
@@ -88,12 +83,14 @@ async def upload_path(
jwt_token = user.jwt_token
from config.settings import is_no_auth_mode
+
is_no_auth = is_no_auth_mode()
owner_user_id = user.user_id if (user and not is_no_auth) else None
owner_name = user.name if user else None
owner_email = user.email if user else None
from api.documents import _ensure_index_exists
+
await _ensure_index_exists(jwt_token)
task_id = await task_service.create_upload_task(
@@ -111,30 +108,32 @@ async def upload_path(
async def upload_context(
- file: UploadFile = File(...),
- previous_response_id: Optional[str] = Form(None),
- endpoint: str = Form("langflow"),
- document_service=Depends(get_document_service),
- chat_service=Depends(get_chat_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(require_all_permissions(("knowledge:upload", "chat:use"))),
+ file: Annotated[UploadFile, File(...)],
+ document_service: Annotated[Any, Depends(get_document_service)],
+ chat_service: Annotated[Any, Depends(get_chat_service)],
+ session_manager: Annotated[Any, Depends(get_session_manager)],
+ user: Annotated[User, Depends(require_all_permissions(("knowledge:upload", "chat:use")))],
+ previous_response_id: Annotated[str | None, Form()] = None,
+ endpoint: Annotated[str, Form()] = "langflow",
):
"""Upload a file and add its content as context to the current conversation"""
filename = file.filename or "uploaded_document"
user_id = user.user_id if user else None
- storage_user_id = (
- (getattr(user, "db_user_id", None) or user.user_id) if user else None
- )
+ storage_user_id = (getattr(user, "db_user_id", None) or user.user_id) if user else None
if previous_response_id and storage_user_id:
from api.chat import _assert_owns
+
await _assert_owns(previous_response_id, storage_user_id)
jwt_token = user.jwt_token
- doc_result = await document_service.process_upload_context(file, filename)
+ doc_result = await document_service.process_upload_context(
+ file, filename, user_id=user_id, jwt_token=jwt_token
+ )
from config.settings import is_no_auth_mode
+
is_no_auth = is_no_auth_mode()
owner_user_id = user.user_id if (user and not is_no_auth) else None
owner_name = user.name if user else None
@@ -166,29 +165,26 @@ async def upload_context(
async def upload_options(
- user: User = Depends(get_current_user),
+ user: Annotated[User, Depends(get_current_user)],
):
"""Return availability of upload features"""
- aws_enabled = bool(
- os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY")
- )
+ aws_enabled = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"))
from config.settings import UPLOAD_BATCH_SIZE
+
return JSONResponse({"aws": aws_enabled, "upload_batch_size": UPLOAD_BATCH_SIZE})
async def upload_bucket(
body: UploadBucketBody,
- task_service=Depends(get_task_service),
- models_service=Depends(get_models_service),
- docling_service=Depends(get_docling_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(require_permission("knowledge:upload")),
+ task_service: Annotated[Any, Depends(get_task_service)],
+ models_service: Annotated[Any, Depends(get_models_service)],
+ docling_service: Annotated[Any, Depends(get_docling_service)],
+ session_manager: Annotated[Any, Depends(get_session_manager)],
+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],
):
"""Process all files from an S3 bucket URL"""
if not os.getenv("AWS_ACCESS_KEY_ID") or not os.getenv("AWS_SECRET_ACCESS_KEY"):
- return JSONResponse(
- {"error": "AWS credentials not configured"}, status_code=400
- )
+ return JSONResponse({"error": "AWS credentials not configured"}, status_code=400)
if not body.s3_url or not body.s3_url.startswith("s3://"):
return JSONResponse({"error": "Invalid S3 URL"}, status_code=400)
@@ -211,9 +207,9 @@ async def upload_bucket(
jwt_token = user.jwt_token
+ from config.settings import is_no_auth_mode
from models.processors import S3FileProcessor
- from config.settings import is_no_auth_mode
is_no_auth = is_no_auth_mode()
owner_user_id = user.user_id if (user and not is_no_auth) else None
owner_name = user.name if user else None
@@ -221,6 +217,7 @@ async def upload_bucket(
task_user_id = user.user_id if (user and not is_no_auth) else None
from api.documents import _ensure_index_exists
+
await _ensure_index_exists(jwt_token)
processor = S3FileProcessor(
diff --git a/src/config/settings.py b/src/config/settings.py
index abdf51e6a..a4bd232a6 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -61,12 +61,24 @@ GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET")
# IBM AMS authentication (Watsonx Data embedded mode)
IBM_AUTH_ENABLED = os.getenv("IBM_AUTH_ENABLED", "false").lower() in ("true", "1", "yes")
+PLATFORM_USERNAME = os.getenv("PLATFORM_USERNAME")
+PLATFORM_PASSWORD = os.getenv("PLATFORM_PASSWORD")
IBM_JWT_PUBLIC_KEY_URL = os.getenv("IBM_JWT_PUBLIC_KEY_URL", "")
IBM_SESSION_COOKIE_NAME = os.getenv("IBM_SESSION_COOKIE_NAME", "ibm-openrag-session")
IBM_CREDENTIALS_HEADER = os.getenv("IBM_CREDENTIALS_HEADER", "X-IBM-LH-Credentials")
DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")
SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY", "")
ENVIRONMENT = os.getenv("ENVIRONMENT", "")
+PLATFORM_AUTH_DEV_MODE = os.getenv("PLATFORM_AUTH_DEV_MODE", "false").lower() in (
+ "true",
+ "1",
+ "yes",
+)
+DOCLING_SERVE_VERIFY_SSL = os.getenv("DOCLING_SERVE_VERIFY_SSL", "true").lower() in (
+ "true",
+ "1",
+ "yes",
+)
# Enable FastAPI's `debug` mode (verbose tracebacks in HTTP error responses
# on the FastAPI app instance). Named explicitly so it isn't confused with
@@ -119,19 +131,21 @@ else:
logger.info("Auto-detected Docling host: %s (URL: %s)", DOCLING_HOST_IP, DOCLING_SERVE_URL)
# Ingestion configuration
-DISABLE_INGEST_WITH_LANGFLOW = os.getenv(
- "DISABLE_INGEST_WITH_LANGFLOW", "false"
-).lower() in ("true", "1", "yes")
+DISABLE_INGEST_WITH_LANGFLOW = os.getenv("DISABLE_INGEST_WITH_LANGFLOW", "false").lower() in (
+ "true",
+ "1",
+ "yes",
+)
# Show the "+" file upload button in the chat input
-OPENRAG_INGEST_VIA_CHAT = os.getenv(
- "OPENRAG_INGEST_VIA_CHAT", "false"
-).lower() in ("true", "1", "yes")
+OPENRAG_INGEST_VIA_CHAT = os.getenv("OPENRAG_INGEST_VIA_CHAT", "false").lower() in (
+ "true",
+ "1",
+ "yes",
+)
# Ingest sample data configuration
-INGEST_SAMPLE_DATA = os.getenv(
- "INGEST_SAMPLE_DATA", "true"
-).lower() in ("true", "1", "yes")
+INGEST_SAMPLE_DATA = os.getenv("INGEST_SAMPLE_DATA", "true").lower() in ("true", "1", "yes")
# Default OpenRAG docs sample ingestion source
# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow
@@ -139,13 +153,15 @@ INGEST_SAMPLE_DATA = os.getenv(
DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()
DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag/")
-#TODO: Enable this when the flow is updated to use the new variables
+# TODO: Enable this when the flow is updated to use the new variables
DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)
-FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(
- "FETCH_OPENRAG_DOCS_AT_STARTUP", "false"
-).lower() in ("true", "1", "yes")
+FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv("FETCH_OPENRAG_DOCS_AT_STARTUP", "false").lower() in (
+ "true",
+ "1",
+ "yes",
+)
# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
UPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)
@@ -161,18 +177,17 @@ LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30
# Default: 3600 seconds (60 minutes)
INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)
+
def is_no_auth_mode():
"""Check if we're running in no-auth mode (OAuth credentials missing)"""
if IBM_AUTH_ENABLED:
- return False # IBM cookie auth is a valid auth mode
+ return False # IBM cookie auth is a valid auth mode (variable name kept for now as per instructions)
result = not (GOOGLE_OAUTH_CLIENT_ID and GOOGLE_OAUTH_CLIENT_SECRET)
return result
# Webhook configuration - must be set to enable webhooks
-WEBHOOK_BASE_URL = os.getenv(
- "WEBHOOK_BASE_URL"
-) # No default - must be explicitly configured
+WEBHOOK_BASE_URL = os.getenv("WEBHOOK_BASE_URL") # No default - must be explicitly configured
# OAuth callback broker URL -- when set, Google (and other providers) redirect
# here instead of directly to the frontend. The broker then forwards to the
@@ -393,23 +408,29 @@ class AppClients:
# The property will handle lazy initialization with probe when first accessed
openai_key = os.getenv("OPENAI_API_KEY")
if openai_key:
- logger.info("OpenAI API key found in environment - will be initialized lazily on first use with HTTP/2 probe")
+ logger.info(
+ "OpenAI API key found in environment - will be initialized lazily on first use with HTTP/2 probe"
+ )
else:
- logger.info("OpenAI API key not found in environment - will be initialized on first use if needed")
+ logger.info(
+ "OpenAI API key not found in environment - will be initialized on first use if needed"
+ )
# Initialize docling-serve HTTP client for document conversion
self.docling_http_client = httpx.AsyncClient(
+ verify=DOCLING_SERVE_VERIFY_SSL,
timeout=httpx.Timeout(
timeout=INGESTION_TIMEOUT,
connect=30.0,
read=INGESTION_TIMEOUT,
write=30.0,
pool=30.0,
- )
+ ),
)
# Eagerly initialize DoclingService to ensure thread-safety
from services.docling_service import DoclingService
+
self._docling_service = DoclingService(httpx_client=self.docling_http_client)
# Initialize Langflow HTTP client with extended timeouts for large documents
@@ -423,7 +444,7 @@ class AppClients:
read=LANGFLOW_TIMEOUT, # Read timeout (most important for large PDFs)
write=LANGFLOW_CONNECT_TIMEOUT, # Write timeout
pool=LANGFLOW_CONNECT_TIMEOUT, # Pool timeout
- )
+ ),
)
logger.info(
"Initialized Langflow HTTP client with extended timeouts",
@@ -433,6 +454,7 @@ class AppClients:
# Wait for Langflow to be healthy before generating API key
from utils.langflow_utils import wait_for_langflow
+
await wait_for_langflow(langflow_http_client=self.langflow_http_client)
# Generate Langflow API key now that Langflow is confirmed ready
@@ -454,9 +476,7 @@ class AppClients:
logger.warning("Failed to initialize Langflow client", error=str(e))
self.langflow_client = None
if self.langflow_client is None:
- logger.warning(
- "No Langflow client initialized yet, will attempt later on first use"
- )
+ logger.warning("No Langflow client initialized yet, will attempt later on first use")
return self
@@ -473,9 +493,7 @@ class AppClients:
)
logger.info("Langflow client initialized on-demand")
except Exception as e:
- logger.error(
- "Failed to initialize Langflow client on-demand", error=str(e)
- )
+ logger.error("Failed to initialize Langflow client on-demand", error=str(e))
self.langflow_client = None
return self.langflow_client
@@ -511,7 +529,7 @@ class AppClients:
os.environ["OPENAI_API_KEY"] = config.providers.openai.api_key
logger.debug("Loaded OpenAI API key from config")
elif not os.environ.get("OPENAI_API_KEY"):
- # Provide dummy key to satisfy AsyncOpenAI constructor;
+ # Provide dummy key to satisfy AsyncOpenAI constructor;
# LiteLLM/MCP will handle routing to other providers if needed.
os.environ["OPENAI_API_KEY"] = "no-key-required"
logger.debug("Using dummy OpenAI API key to satisfy client constructor")
@@ -526,7 +544,9 @@ class AppClients:
os.environ["WATSONX_API_KEY"] = config.providers.watsonx.api_key
if config.providers.watsonx.endpoint:
os.environ["WATSONX_ENDPOINT"] = config.providers.watsonx.endpoint
- os.environ["WATSONX_API_BASE"] = config.providers.watsonx.endpoint # LiteLLM expects this name
+ os.environ["WATSONX_API_BASE"] = (
+ config.providers.watsonx.endpoint
+ ) # LiteLLM expects this name
if config.providers.watsonx.project_id:
os.environ["WATSONX_PROJECT_ID"] = config.providers.watsonx.project_id
if config.providers.watsonx.api_key:
@@ -552,7 +572,6 @@ class AppClients:
os.environ["OPENAI_API_KEY"] = "no-key-required"
logger.debug("Using dummy OpenAI API key fallback (config load failed)")
-
# API key for AsyncOpenAI constructor
api_key = os.environ.get("OPENAI_API_KEY")
@@ -569,16 +588,15 @@ class AppClients:
logger.info(f"Probing client with HTTP/2 using model {model_name}...")
try:
await asyncio.wait_for(
- client.embeddings.create(
- model=model_name,
- input=['test']
- ),
- timeout=5.0
+ client.embeddings.create(model=model_name, input=["test"]), timeout=5.0
)
logger.info(f"HTTP/2 probe successful with {model_name}")
return True
except (TimeoutError, Exception) as probe_error:
- logger.warning(f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1", error=str(probe_error))
+ logger.warning(
+ f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1",
+ error=str(probe_error),
+ )
return False
finally:
# Always close the probe client so its connections are fully
@@ -614,20 +632,27 @@ class AppClients:
if use_http2:
self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI(api_key=api_key))
- logger.info(f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})")
+ logger.info(
+ f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})"
+ )
else:
http_client = httpx.AsyncClient(
- http2=False,
- timeout=httpx.Timeout(60.0, connect=10.0)
+ http2=False, timeout=httpx.Timeout(60.0, connect=10.0)
)
self._patched_async_client = patch_openai_with_mcp(
AsyncOpenAI(api_key=api_key, http_client=http_client)
)
- logger.info(f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})")
+ logger.info(
+ f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})"
+ )
logger.info("Successfully initialized OpenAI client")
except Exception as e:
- logger.error(f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}")
- raise ValueError(f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable.") from e
+ logger.error(
+ f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}"
+ )
+ raise ValueError(
+ f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable."
+ ) from e
return self._patched_async_client
@@ -666,6 +691,7 @@ class AppClients:
return self._docling_service
from services.docling_service import DoclingService
+
self._docling_service = DoclingService(httpx_client=self.docling_http_client)
return self._docling_service
@@ -765,9 +791,7 @@ class AppClients:
return response
- async def _create_langflow_global_variable(
- self, name: str, value: str, modify: bool = False
- ):
+ async def _create_langflow_global_variable(self, name: str, value: str, modify: bool = False):
"""Create a global variable in Langflow via API"""
payload = {
"name": name,
@@ -777,9 +801,7 @@ class AppClients:
}
try:
- response = await self.langflow_request(
- "POST", "/api/v1/variables/", json=payload
- )
+ response = await self.langflow_request("POST", "/api/v1/variables/", json=payload)
if response.status_code in [200, 201]:
logger.info(
@@ -916,18 +938,15 @@ def _component_path(env_var: str, filename: str) -> str:
flows_dir = get_flows_path()
return os.path.join(flows_dir, "components", filename)
-WATSONX_LLM_COMPONENT_PATH = _component_path(
- "WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json"
-)
+
+WATSONX_LLM_COMPONENT_PATH = _component_path("WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json")
WATSONX_LLM_TEXT_COMPONENT_PATH = _component_path(
"WATSONX_LLM_TEXT_COMPONENT_PATH", "watsonx_llm_text.json"
)
WATSONX_EMBEDDING_COMPONENT_PATH = _component_path(
"WATSONX_EMBEDDING_COMPONENT_PATH", "watsonx_embedding.json"
)
-OLLAMA_LLM_COMPONENT_PATH = _component_path(
- "OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json"
-)
+OLLAMA_LLM_COMPONENT_PATH = _component_path("OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json")
OLLAMA_LLM_TEXT_COMPONENT_PATH = _component_path(
"OLLAMA_LLM_TEXT_COMPONENT_PATH", "ollama_llm_text.json"
)
@@ -940,13 +959,9 @@ OLLAMA_EMBEDDING_COMPONENT_PATH = _component_path(
OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME = os.getenv(
"OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME", "Embedding Model"
)
-OPENAI_LLM_COMPONENT_DISPLAY_NAME = os.getenv(
- "OPENAI_LLM_COMPONENT_DISPLAY_NAME", "Language Model"
-)
+OPENAI_LLM_COMPONENT_DISPLAY_NAME = os.getenv("OPENAI_LLM_COMPONENT_DISPLAY_NAME", "Language Model")
-AGENT_COMPONENT_DISPLAY_NAME = os.getenv(
- "AGENT_COMPONENT_DISPLAY_NAME", "Agent"
-)
+AGENT_COMPONENT_DISPLAY_NAME = os.getenv("AGENT_COMPONENT_DISPLAY_NAME", "Agent")
# Provider-specific component IDs
WATSONX_EMBEDDING_COMPONENT_DISPLAY_NAME = os.getenv(
@@ -994,7 +1009,9 @@ def get_agent_config():
def get_embedding_model() -> str:
"""Return the currently configured embedding model."""
- return get_openrag_config().knowledge.embedding_model or (OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else "")
+ return get_openrag_config().knowledge.embedding_model or (
+ OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else ""
+ )
def get_index_name() -> str:
diff --git a/src/dependencies.py b/src/dependencies.py
index e7f943fd3..864a51bf6 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -17,7 +17,8 @@ Usage:
import asyncio
import dataclasses
-from typing import AsyncIterator, Optional, Sequence
+from collections.abc import AsyncIterator, Sequence
+from typing import Optional
from cachetools import TTLCache
from fastapi import Depends, HTTPException, Request
@@ -130,6 +131,8 @@ def get_flows_service(services: dict = Depends(get_services)):
def get_docling_service(services: dict = Depends(get_services)):
return services["docling_service"]
+
+
def get_rbac_service(services: dict = Depends(get_services)):
return services["rbac_service"]
@@ -150,12 +153,13 @@ async def get_db_session() -> AsyncIterator[AsyncSession]:
if SessionLocal is None:
init_engine()
from db.engine import SessionLocal as _SessionLocal
+
assert _SessionLocal is not None
async with _SessionLocal() as session:
yield session
-async def _ensure_db_user(user: User) -> Optional[str]:
+async def _ensure_db_user(user: User) -> str | None:
"""Best-effort DB upsert for the authenticated user. Returns the SQL
`users.id` for this user (so callers can cache the OAuth-sub → DB-id
mapping). Returns None on failure.
@@ -189,6 +193,7 @@ async def _ensure_db_user(user: User) -> Optional[str]:
if SessionLocal is None:
init_engine()
from db.engine import SessionLocal as _SessionLocal
+
if _SessionLocal is None:
return None
async with _SessionLocal() as session:
@@ -216,7 +221,7 @@ async def _resolve_db_user_id(user: User) -> str:
return resolved or user.user_id
-async def _attach_db_user_id(request: Request, user: Optional[User]) -> Optional[User]:
+async def _attach_db_user_id(request: Request, user: User | None) -> User | None:
"""Attach the internal SQL users.id to the request user.
`User.user_id` remains the external auth subject used in JWT/OpenSearch
@@ -235,8 +240,8 @@ async def _attach_db_user_id(request: Request, user: Optional[User]) -> Optional
def invalidate_user_ensured_cache(
- oauth_provider: Optional[str] = None,
- oauth_subject: Optional[str] = None,
+ oauth_provider: str | None = None,
+ oauth_subject: str | None = None,
) -> None:
"""Pop the ensure-cache + lock for a single identity, or clear all
if neither argument is provided.
@@ -353,7 +358,33 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
"""
import auth.ibm_auth as ibm_auth
from auth.ibm_auth import extract_ibm_credentials
- from config.settings import IBM_SESSION_COOKIE_NAME, IBM_CREDENTIALS_HEADER
+ from config.settings import (
+ IBM_CREDENTIALS_HEADER,
+ IBM_SESSION_COOKIE_NAME,
+ PLATFORM_PASSWORD,
+ PLATFORM_USERNAME,
+ )
+
+ # ── Option -1: Environment variable override (local dev/calls) ───────
+
+ if PLATFORM_USERNAME and PLATFORM_PASSWORD:
+ import base64
+
+ logger.debug("[AUTH] Using PLATFORM_USERNAME and PLATFORM_PASSWORD from environment")
+ creds = f"{PLATFORM_USERNAME}:{PLATFORM_PASSWORD}"
+ lh_credentials = base64.b64encode(creds.encode()).decode()
+ user = User(
+ user_id=PLATFORM_USERNAME,
+ email=PLATFORM_USERNAME,
+ name=PLATFORM_USERNAME,
+ picture=None,
+ provider="ibm_ams_env",
+ jwt_token=f"Basic {lh_credentials}",
+ opensearch_username=PLATFORM_USERNAME,
+ opensearch_credentials=lh_credentials,
+ )
+ request.state.user = user
+ return user
# ── Option 0: Configurable credentials header (Traefik production) ───
@@ -369,13 +400,14 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
logger.debug("[AUTH] IBM JWT claims decoded successfully")
sub = claims.get("sub")
if not sub:
- logger.warning("IBM JWT is missing required 'sub' claim; treating as unauthenticated")
+ logger.warning(
+ "IBM JWT is missing required 'sub' claim; treating as unauthenticated"
+ )
else:
user_id = claims.get("username", sub)
email = claims.get("username", sub)
name = claims.get("display_name", claims.get("username", sub))
-
if lh_credentials and lh_credentials.strip() != "":
logger.debug("[AUTH] IBM LH credentials found in request headers")
opensearch_username, _ = extract_ibm_credentials(lh_credentials)
@@ -410,9 +442,7 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
# ibm_token = request.cookies.get(IBM_SESSION_COOKIE_NAME)
if ibm_token and user_id:
logger.debug("[AUTH] IBM JWT cookie present and user_id found")
- logger.debug(
- "[AUTH] LH credentials not available in header, reading from connections.json"
- )
+ logger.debug("[AUTH] LH credentials not available in header, reading from connections.json")
# lh credentials not available in header, read from connections service
connector_service = request.app.state.services.get("connector_service")
if connector_service:
@@ -457,9 +487,7 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
# ── Option 2: ibm-auth-basic cookie (local dev, no Traefik) ─────────
auth_header = request.cookies.get("ibm-auth-basic", "")
if auth_header.startswith("Basic "):
- logger.debug(
- "[AUTH] Debug mode enabled, extracting IBM LH credentials from cookie"
- )
+ logger.debug("[AUTH] Debug mode enabled, extracting IBM LH credentials from cookie")
username, _ = extract_ibm_credentials(auth_header)
logger.debug("[AUTH] IBM LH credentials extracted successfully")
user = User(
@@ -533,7 +561,7 @@ async def get_current_user(
async def get_optional_user(
request: Request,
session_manager=Depends(get_session_manager),
-) -> Optional[User]:
+) -> User | None:
"""
Optionally extract JWT cookie user.
@@ -568,13 +596,9 @@ async def get_optional_user(
user = session_manager.get_user_from_token(auth_token)
# get_effective_jwt_token handles anonymous JWT creation if needed
effective_token = (
- session_manager.get_effective_jwt_token(user.user_id, auth_token)
- if user
- else None
- )
- user_with_token = (
- dataclasses.replace(user, jwt_token=effective_token) if user else None
+ session_manager.get_effective_jwt_token(user.user_id, auth_token) if user else None
)
+ user_with_token = dataclasses.replace(user, jwt_token=effective_token) if user else None
if user_with_token:
return await _attach_db_user_id(request, user_with_token)
@@ -664,8 +688,6 @@ async def get_api_key_user_async(
# Phase 2 will populate api_key_role_ids from the SQL api_keys table.
# In Phase 1 we leave it unset so require_permission falls back to the
# user's live role membership (no privilege escalation possible).
- request.state.api_key_role_ids = getattr(
- request.state, "api_key_role_ids", None
- )
+ request.state.api_key_role_ids = getattr(request.state, "api_key_role_ids", None)
return await _attach_db_user_id(request, user_with_token)
diff --git a/src/models/processors.py b/src/models/processors.py
index 62d3be1e7..51cb655a7 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1,14 +1,19 @@
-from typing import Any, Dict, Optional
-from .tasks import UploadTask, FileTask
-from utils.logging_config import get_logger
+from typing import TYPE_CHECKING, Any
+
from utils.file_utils import (
- get_file_extension,
clean_connector_filename,
+ get_file_extension,
get_filename_aliases,
)
+from utils.logging_config import get_logger
+
+from .tasks import FileTask, UploadTask
logger = get_logger(__name__)
+if TYPE_CHECKING:
+ from connectors.base import DocumentACL
+
class TaskProcessor:
"""Base class for task processors with shared processing logic"""
@@ -27,9 +32,10 @@ class TaskProcessor:
Check if a document with the given hash already exists in OpenSearch.
Consolidated hash checking for all processors.
"""
- from config.settings import get_index_name
import asyncio
+ from config.settings import get_index_name
+
max_retries = 3
retry_delay = 1.0
@@ -37,18 +43,18 @@ class TaskProcessor:
try:
exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)
return exists
- except (asyncio.TimeoutError, Exception) as e:
+ except (TimeoutError, Exception) as e:
if attempt == max_retries - 1:
logger.error(
"OpenSearch exists check failed after retries",
file_hash=file_hash,
error=str(e),
- attempt=attempt + 1
+ attempt=attempt + 1,
)
# On final failure, assume document doesn't exist (safer to reprocess than skip)
logger.warning(
"Assuming document doesn't exist due to connection issues",
- file_hash=file_hash
+ file_hash=file_hash,
)
return False
else:
@@ -57,10 +63,11 @@ class TaskProcessor:
file_hash=file_hash,
error=str(e),
attempt=attempt + 1,
- retry_in=retry_delay
+ retry_in=retry_delay,
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
+ return False
async def check_filename_exists(
self,
@@ -71,9 +78,10 @@ class TaskProcessor:
Check if a document with the given filename already exists in OpenSearch.
Returns True if any chunks with this filename exist.
"""
+ import asyncio
+
from config.settings import get_index_name
from utils.opensearch_queries import build_filename_search_body
- import asyncio
max_retries = 3
retry_delay = 1.0
@@ -94,12 +102,9 @@ class TaskProcessor:
i = 0
while i < len(pending_candidates):
candidate = pending_candidates[i]
- search_body = build_filename_search_body(
- candidate, size=1, source=False
- )
+ search_body = build_filename_search_body(candidate, size=1, source=False)
response = await opensearch_client.search(
- index=get_index_name(),
- body=search_body
+ index=get_index_name(), body=search_body
)
hits = response.get("hits", {}).get("hits", [])
if hits:
@@ -110,18 +115,18 @@ class TaskProcessor:
continue
return False
- except (asyncio.TimeoutError, Exception) as e:
+ except (TimeoutError, Exception) as e:
if attempt == max_retries - 1:
logger.error(
"OpenSearch filename check failed after retries",
filename=filename,
error=str(e),
- attempt=attempt + 1
+ attempt=attempt + 1,
)
# On final failure, assume document doesn't exist (safer to reprocess than skip)
logger.warning(
"Assuming filename doesn't exist due to connection issues",
- filename=filename
+ filename=filename,
)
return False
else:
@@ -130,10 +135,11 @@ class TaskProcessor:
filename=filename,
error=str(e),
attempt=attempt + 1,
- retry_in=retry_delay
+ retry_in=retry_delay,
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
+ return False
async def delete_document_by_filename(
self,
@@ -158,22 +164,15 @@ class TaskProcessor:
for candidate in candidate_filenames:
delete_body = build_filename_delete_body(candidate)
response = await opensearch_client.delete_by_query(
- index=get_index_name(),
- body=delete_body
+ index=get_index_name(), body=delete_body
)
deleted_count += response.get("deleted", 0)
logger.info(
- "Deleted existing document chunks",
- filename=filename,
- deleted_count=deleted_count
+ "Deleted existing document chunks", filename=filename, deleted_count=deleted_count
)
except Exception as e:
- logger.error(
- "Failed to delete existing document",
- filename=filename,
- error=str(e)
- )
+ logger.error("Failed to delete existing document", filename=filename, error=str(e))
raise
async def process_document_standard(
@@ -206,6 +205,7 @@ class TaskProcessor:
acl: DocumentACL instance with access control information
"""
import datetime
+
from config.settings import (
clients,
get_embedding_model,
@@ -223,11 +223,7 @@ class TaskProcessor:
# get_embedding_model() returns empty string when Langflow ingest is enabled,
# but OpenRAG processors still need a concrete embedding model.
configured_embedding_model = get_openrag_config().knowledge.embedding_model
- embedding_model = (
- embedding_model
- or configured_embedding_model
- or get_embedding_model()
- )
+ embedding_model = embedding_model or configured_embedding_model or get_embedding_model()
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
@@ -237,7 +233,6 @@ class TaskProcessor:
# Check if already exists
if await self.check_document_exists(file_hash, opensearch_client):
return {"status": "unchanged", "id": file_hash}
-
logger.info(
"Processing document with embedding model",
@@ -247,11 +242,13 @@ class TaskProcessor:
# Check if this is a .txt or .md file - use simple processing instead of docling
import os
+
file_ext = os.path.splitext(file_path)[1].lower()
-
- if file_ext in ('.txt', '.md'):
+
+ if file_ext in (".txt", ".md"):
# Simple text file processing without docling
from utils.document_processing import process_text_file
+
logger.info(
"Processing as plain text file (bypassing docling)",
file_path=file_path,
@@ -262,7 +259,9 @@ class TaskProcessor:
if original_filename:
slim_doc["filename"] = original_filename
else:
- full_doc = await self.docling_service.convert_file(file_path)
+ full_doc = await self.docling_service.convert_file(
+ file_path, user_id=owner_user_id, auth_header=jwt_token
+ )
slim_doc = extract_relevant(full_doc)
if chunk_size is not None:
@@ -272,11 +271,7 @@ class TaskProcessor:
cs = 0
if cs > 0:
try:
- co = (
- int(chunk_overlap)
- if chunk_overlap is not None
- else 0
- )
+ co = int(chunk_overlap) if chunk_overlap is not None else 0
except (TypeError, ValueError):
co = 0
if co < cs:
@@ -286,7 +281,11 @@ class TaskProcessor:
texts = [c["text"] for c in slim_doc["chunks"]]
- litellm_embedding_model = await self.models_service.get_litellm_model_name(embedding_model) if self.models_service is not None else embedding_model
+ litellm_embedding_model = (
+ await self.models_service.get_litellm_model_name(embedding_model)
+ if self.models_service is not None
+ else embedding_model
+ )
# Split into batches to avoid token limits (8191 limit, use 8000 with buffer or 2000 if it's ollama)
if "ollama" in litellm_embedding_model:
@@ -299,7 +298,9 @@ class TaskProcessor:
resp = await clients.patched_embedding_client.embeddings.create(
model=litellm_embedding_model, input=batch
)
- embeddings.extend([d["embedding"] if isinstance(d, dict) else d.embedding for d in resp.data])
+ embeddings.extend(
+ [d["embedding"] if isinstance(d, dict) else d.embedding for d in resp.data]
+ )
if not embeddings or len(embeddings) == 0:
logger.error(
@@ -317,12 +318,10 @@ class TaskProcessor:
)
# Index each chunk as a separate document
- for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings)):
+ for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True)):
chunk_doc = {
"document_id": file_hash,
- "filename": original_filename
- if original_filename
- else slim_doc["filename"],
+ "filename": original_filename if original_filename else slim_doc["filename"],
"mimetype": slim_doc["mimetype"],
"page": chunk["page"],
"text": chunk["text"],
@@ -359,9 +358,7 @@ class TaskProcessor:
chunk_doc["is_sample_data"] = "true"
chunk_id = f"{file_hash}_{i}"
try:
- await opensearch_client.index(
- index=get_index_name(), id=chunk_id, body=chunk_doc
- )
+ await opensearch_client.index(index=get_index_name(), id=chunk_id, body=chunk_doc)
except Exception as e:
logger.error(
"OpenSearch indexing failed for chunk",
@@ -372,9 +369,7 @@ class TaskProcessor:
raise
return {"status": "indexed", "id": file_hash}
- async def process_item(
- self, upload_task: UploadTask, item: Any, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:
"""
Process a single item in the task.
@@ -421,14 +416,13 @@ class DocumentFileProcessor(TaskProcessor):
self.is_sample_data = is_sample_data
self.connector_type = connector_type
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a regular file path using consolidated methods"""
+ import os
+ import time
+
from models.tasks import TaskStatus
from utils.hash_utils import hash_id
- import time
- import os
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -487,14 +481,12 @@ class ConnectorFileProcessor(TaskProcessor):
owner_email: str = None,
document_service=None,
models_service=None,
- ingest_settings: Optional[Dict[str, Any]] = None,
+ ingest_settings: dict[str, Any] | None = None,
):
super().__init__(
document_service=document_service,
models_service=models_service,
- docling_service=document_service.docling_service
- if document_service
- else None,
+ docling_service=document_service.docling_service if document_service else None,
)
self.connector_service = connector_service
self.connection_id = connection_id
@@ -505,15 +497,12 @@ class ConnectorFileProcessor(TaskProcessor):
self.owner_email = owner_email
self.ingest_settings = ingest_settings
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using consolidated methods"""
+ import time
+
from models.tasks import TaskStatus
from utils.hash_utils import hash_id
- import tempfile
- import time
- import os
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -531,7 +520,7 @@ class ConnectorFileProcessor(TaskProcessor):
# Get file content from connector
document = await connector.get_file_content(file_id)
-
+
# Update filename in task once we have it from the connector
file_task.filename = clean_connector_filename(document.filename, document.mimetype)
@@ -544,13 +533,13 @@ class ConnectorFileProcessor(TaskProcessor):
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
- with open(tmp_path, 'wb') as f:
+ with open(tmp_path, "wb") as f:
f.write(document.content)
# Compute hash
file_hash = hash_id(tmp_path)
- standard_kwargs: Dict[str, Any] = {}
+ standard_kwargs: dict[str, Any] = {}
if isinstance(self.ingest_settings, dict):
s = self.ingest_settings
em = s.get("embeddingModel")
@@ -583,10 +572,12 @@ class ConnectorFileProcessor(TaskProcessor):
)
# Add connector-specific metadata
- result.update({
- "source_url": document.source_url,
- "document_id": document.id,
- })
+ result.update(
+ {
+ "source_url": document.source_url,
+ "document_id": document.id,
+ }
+ )
file_task.status = TaskStatus.COMPLETED
file_task.result = result
@@ -613,7 +604,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
- ingest_settings: Optional[Dict[str, Any]] = None,
+ ingest_settings: dict[str, Any] | None = None,
):
super().__init__(
document_service=langflow_connector_service.task_service.document_service
@@ -633,15 +624,12 @@ class LangflowConnectorFileProcessor(TaskProcessor):
self.owner_email = owner_email
self.ingest_settings = ingest_settings
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using LangflowConnectorService"""
+ import time
+
from models.tasks import TaskStatus
from utils.hash_utils import hash_id
- import tempfile
- import time
- import os
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -650,14 +638,10 @@ class LangflowConnectorFileProcessor(TaskProcessor):
file_id = item # item is the connector file ID
# Get the connector and connection info
- connector = await self.langflow_connector_service.get_connector(
+ connector = await self.langflow_connector_service.get_connector(self.connection_id)
+ connection = await self.langflow_connector_service.connection_manager.get_connection(
self.connection_id
)
- connection = (
- await self.langflow_connector_service.connection_manager.get_connection(
- self.connection_id
- )
- )
if not connector or not connection:
raise ValueError(f"Connection '{self.connection_id}' not found")
@@ -676,15 +660,17 @@ class LangflowConnectorFileProcessor(TaskProcessor):
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
- with open(tmp_path, 'wb') as f:
+ with open(tmp_path, "wb") as f:
f.write(document.content)
# Compute hash and check if already exists
file_hash = hash_id(tmp_path)
# Check if document already exists
- opensearch_client = self.langflow_connector_service.session_manager.get_user_opensearch_client(
- self.user_id, self.jwt_token
+ opensearch_client = (
+ self.langflow_connector_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
+ )
)
if await self.check_document_exists(file_hash, opensearch_client):
file_task.status = TaskStatus.COMPLETED
@@ -746,17 +732,12 @@ class S3FileProcessor(TaskProcessor):
self.owner_name = owner_name
self.owner_email = owner_email
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
- from models.tasks import TaskStatus
- import tempfile
- import os
import time
- import asyncio
- import datetime
- from config.settings import clients, get_embedding_model, get_index_name
+
+ from models.tasks import TaskStatus
+
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -766,7 +747,7 @@ class S3FileProcessor(TaskProcessor):
try:
with auto_cleanup_tempfile() as tmp_path:
# Download object to temporary file
- with open(tmp_path, 'wb') as tmp_file:
+ with open(tmp_path, "wb") as tmp_file:
self.s3_client.download_fileobj(self.bucket, item, tmp_file)
# Compute hash
@@ -835,15 +816,14 @@ class LangflowFileProcessor(TaskProcessor):
self.replace_duplicates = replace_duplicates
self.connector_type = connector_type
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a file path using LangflowFileService upload_and_ingest_file"""
import mimetypes
import os
- from models.tasks import TaskStatus
import time
+ from models.tasks import TaskStatus
+
# Update task status
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -873,21 +853,21 @@ class LangflowFileProcessor(TaskProcessor):
await self.delete_document_by_filename(original_filename, opensearch_client)
# Read file content for processing
- with open(item, 'rb') as f:
+ with open(item, "rb") as f:
content = f.read()
# Create file tuple for upload using ORIGINAL filename
# This ensures the document is indexed with the original name
content_type, _ = mimetypes.guess_type(original_filename)
if not content_type:
- content_type = 'application/octet-stream'
+ content_type = "application/octet-stream"
# Rename .txt to .md for Langflow compatibility
# Langflow has issues processing text/plain files
langflow_filename = original_filename
- if original_filename.lower().endswith('.txt'):
- langflow_filename = original_filename[:-4] + '.md'
- content_type = 'text/markdown'
+ if original_filename.lower().endswith(".txt"):
+ langflow_filename = original_filename[:-4] + ".md"
+ content_type = "text/markdown"
logger.debug(f"Renamed {original_filename} to {langflow_filename} for Langflow")
file_tuple = (langflow_filename, content, content_type)
@@ -897,12 +877,10 @@ class LangflowFileProcessor(TaskProcessor):
effective_jwt = self.jwt_token
if self.session_manager and not effective_jwt:
# Let session manager handle anonymous JWT creation if needed
- self.session_manager.get_user_opensearch_client(
- self.owner_user_id, self.jwt_token
- )
+ self.session_manager.get_user_opensearch_client(self.owner_user_id, self.jwt_token)
# The session manager would have created anonymous JWT if needed
# Get it from the session manager's internal state
- if hasattr(self.session_manager, '_anonymous_jwt'):
+ if hasattr(self.session_manager, "_anonymous_jwt"):
effective_jwt = self.session_manager._anonymous_jwt
# Prepare metadata tweaks similar to API endpoint
@@ -930,10 +908,7 @@ class LangflowFileProcessor(TaskProcessor):
except Exception as e:
# Update task with failure
file_task.status = TaskStatus.FAILED
- file_task.error_message = str(e)
+ file_task.error = str(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
-
-
-from .url import LangflowUrlProcessor
diff --git a/src/services/auth_service.py b/src/services/auth_service.py
index 50ffdda75..6d0676ed7 100644
--- a/src/services/auth_service.py
+++ b/src/services/auth_service.py
@@ -1,24 +1,23 @@
-import uuid
import json
-import httpx
-
import logging
+import os
+import uuid
from datetime import datetime, timedelta
-from typing import Optional
-import os
-from config.settings import OAUTH_BROKER_URL, WEBHOOK_BASE_URL, is_no_auth_mode
+import httpx
from fastapi import HTTPException
-logger = logging.getLogger(__name__)
-from session_manager import SessionManager
-from services.langflow_mcp_service import LangflowMCPService
-from connectors.google_drive.oauth import GoogleDriveOAuth
-from connectors.onedrive.oauth import OneDriveOAuth
-from connectors.sharepoint.oauth import SharePointOAuth
+from config.settings import OAUTH_BROKER_URL, WEBHOOK_BASE_URL, is_no_auth_mode
from connectors.google_drive import GoogleDriveConnector
+from connectors.google_drive.oauth import GoogleDriveOAuth
from connectors.onedrive import OneDriveConnector
+from connectors.onedrive.oauth import OneDriveOAuth
from connectors.sharepoint import SharePointConnector
+from connectors.sharepoint.oauth import SharePointOAuth
+from services.langflow_mcp_service import LangflowMCPService
+from session_manager import SessionManager
+
+logger = logging.getLogger(__name__)
# Connectors that authenticate directly (no OAuth redirect required)
_DIRECT_AUTH_CONNECTORS = {"ibm_cos"}
@@ -34,11 +33,10 @@ class AuthService:
):
self.session_manager = session_manager
self.connector_service = connector_service
- self.used_auth_codes = set() # Track used authorization codes
+ self.used_auth_codes: set[str] = set() # Track used authorization codes
self.flows_service = flows_service
self.langflow_mcp_service = langflow_mcp_service
-
async def init_oauth(
self,
connector_type: str,
@@ -88,6 +86,7 @@ class AuthService:
# Create connection configuration - use data directory for persistence
from config.paths import get_data_file
+
token_file = get_data_file(f"{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json")
effective_redirect_uri = OAUTH_BROKER_URL or redirect_uri
config = {
@@ -99,18 +98,14 @@ class AuthService:
# Only add webhook URL if WEBHOOK_BASE_URL is configured
if WEBHOOK_BASE_URL:
- config["webhook_url"] = (
- f"{WEBHOOK_BASE_URL}/connectors/{connector_type}/webhook"
- )
+ config["webhook_url"] = f"{WEBHOOK_BASE_URL}/connectors/{connector_type}/webhook"
# Create connection in manager
- connection_id = (
- await self.connector_service.connection_manager.create_connection(
- connector_type=connector_type,
- name=connection_name,
- config=config,
- user_id=user_id,
- )
+ connection_id = await self.connector_service.connection_manager.create_connection(
+ connector_type=connector_type,
+ name=connection_name,
+ config=config,
+ user_id=user_id,
)
# Direct-auth connectors (HMAC/API-key based, no OAuth redirect)
@@ -126,22 +121,26 @@ class AuthService:
"sharepoint": (SharePointConnector, SharePointOAuth),
}
- connector_class, oauth_class = connector_class_map.get(
- connector_type, (None, None)
- )
+ connector_class, oauth_class = connector_class_map.get(connector_type, (None, None))
if not connector_class or not oauth_class:
raise ValueError(f"No classes found for connector type: {connector_type}")
+ # Cast to Any to satisfy mypy for class attribute access
+ from typing import Any
+
+ oauth_class_any: Any = oauth_class
+ connector_class_any: Any = connector_class
+
# Get scopes from OAuth class
- scopes = oauth_class.SCOPES
+ scopes = oauth_class_any.SCOPES
# Get endpoints from OAuth class
- auth_endpoint = oauth_class.AUTH_ENDPOINT
- token_endpoint = oauth_class.TOKEN_ENDPOINT
+ auth_endpoint = oauth_class_any.AUTH_ENDPOINT
+ token_endpoint = oauth_class_any.TOKEN_ENDPOINT
# src/services/auth_service.py
- client_key = getattr(connector_class, "CLIENT_ID_ENV_VAR", None)
- secret_key = getattr(connector_class, "CLIENT_SECRET_ENV_VAR", None)
+ client_key = getattr(connector_class_any, "CLIENT_ID_ENV_VAR", None)
+ secret_key = getattr(connector_class_any, "CLIENT_SECRET_ENV_VAR", None)
def _assert_env_key(name, val):
if not isinstance(val, str) or not val.strip():
@@ -173,9 +172,7 @@ class AuthService:
},
}
- async def _init_direct_connection(
- self, connector_type: str, connection_id: str
- ) -> dict:
+ async def _init_direct_connection(self, connector_type: str, connection_id: str) -> dict:
"""Authenticate a non-OAuth connector immediately using env var credentials.
Creates the connection record (already done by the caller) and verifies
@@ -184,10 +181,8 @@ class AuthService:
is needed.
"""
try:
- connection_config = (
- await self.connector_service.connection_manager.get_connection(
- connection_id
- )
+ connection_config = await self.connector_service.connection_manager.get_connection(
+ connection_id
)
if not connection_config:
raise ValueError("Connection not found")
@@ -198,25 +193,19 @@ class AuthService:
authenticated = await connector.authenticate()
if not authenticated:
# Remove the connection so the user can retry after fixing credentials
- await self.connector_service.connection_manager.delete_connection(
- connection_id
- )
+ await self.connector_service.connection_manager.delete_connection(connection_id)
raise ValueError(
f"Could not authenticate with {connector_type}. "
"Check that your credentials and endpoint are correct."
)
# Cache the authenticated connector
- self.connector_service.connection_manager.active_connectors[
- connection_id
- ] = connector
+ self.connector_service.connection_manager.active_connectors[connection_id] = connector
except ValueError:
raise
except Exception as exc:
- await self.connector_service.connection_manager.delete_connection(
- connection_id
- )
+ await self.connector_service.connection_manager.delete_connection(connection_id)
raise ValueError(f"Failed to connect {connector_type}: {exc}") from exc
return {
@@ -236,9 +225,7 @@ class AuthService:
"""Handle OAuth callback - exchange authorization code for tokens"""
logger.info(f"OAuth callback state: {state}")
if not all([connection_id, authorization_code]):
- raise ValueError(
- "Missing required parameters (connection_id, authorization_code)"
- )
+ raise ValueError("Missing required parameters (connection_id, authorization_code)")
# Check if authorization code has already been used
if authorization_code in self.used_auth_codes:
@@ -249,10 +236,8 @@ class AuthService:
try:
# Get connection config
- connection_config = (
- await self.connector_service.connection_manager.get_connection(
- connection_id
- )
+ connection_config = await self.connector_service.connection_manager.get_connection(
+ connection_id
)
if not connection_config:
raise ValueError("Connection not found")
@@ -275,15 +260,14 @@ class AuthService:
"sharepoint": (SharePointConnector, SharePointOAuth),
}
- connector_class, oauth_class = connector_class_map.get(
- connector_type, (None, None)
- )
+ connector_class, oauth_class = connector_class_map.get(connector_type, (None, None))
if not connector_class or not oauth_class:
- raise ValueError(
- f"No classes found for connector type: {connector_type}"
- )
+ raise ValueError(f"No classes found for connector type: {connector_type}")
+
+ from typing import Any
- token_url = oauth_class.TOKEN_ENDPOINT
+ oauth_class_any: Any = oauth_class
+ token_url = oauth_class_any.TOKEN_ENDPOINT
token_payload = {
"code": authorization_code,
@@ -311,9 +295,7 @@ class AuthService:
# OAuth providers typically return scopes as a space-separated string
scopes = (
- granted_scopes.split(" ")
- if isinstance(granted_scopes, str)
- else granted_scopes
+ granted_scopes.split(" ") if isinstance(granted_scopes, str) else granted_scopes
)
token_file_data = {
@@ -324,9 +306,7 @@ class AuthService:
# Add expiry if provided
if token_data.get("expires_in"):
- expiry = datetime.now() + timedelta(
- seconds=int(token_data["expires_in"])
- )
+ expiry = datetime.now() + timedelta(seconds=int(token_data["expires_in"]))
token_file_data["expiry"] = expiry.isoformat()
# Save tokens to file
@@ -343,9 +323,7 @@ class AuthService:
connection_id, connection_config, token_data, request
)
else:
- return await self._handle_data_source_auth(
- connection_id, connection_config
- )
+ return await self._handle_data_source_auth(connection_id, connection_config)
except Exception as e:
# Remove used code from set if we failed
@@ -376,9 +354,6 @@ class AuthService:
token_data["access_token"]
)
-
-
-
response_data = {
"status": "authenticated",
"purpose": "app_auth",
@@ -398,9 +373,7 @@ class AuthService:
"purpose": "data_source",
"user_email": user_info.get("email"),
**(
- {
- "webhook_url": f"{WEBHOOK_BASE_URL}/connectors/google_drive/webhook"
- }
+ {"webhook_url": f"{WEBHOOK_BASE_URL}/connectors/google_drive/webhook"}
if WEBHOOK_BASE_URL
else {}
),
@@ -409,21 +382,15 @@ class AuthService:
response_data["google_drive_connection_id"] = connection_id
else:
# Fallback: delete connection if we can't get user info
- await self.connector_service.connection_manager.delete_connection(
- connection_id
- )
+ await self.connector_service.connection_manager.delete_connection(connection_id)
return response_data
else:
# Clean up connection if session creation failed
- await self.connector_service.connection_manager.delete_connection(
- connection_id
- )
+ await self.connector_service.connection_manager.delete_connection(connection_id)
raise Exception("Failed to create user session")
- async def _handle_data_source_auth(
- self, connection_id: str, connection_config
- ) -> dict:
+ async def _handle_data_source_auth(self, connection_id: str, connection_config) -> dict:
"""Handle data source connection - keep the connection for syncing"""
result = {
"status": "authenticated",
@@ -442,10 +409,8 @@ class AuthService:
logger.info(
f"_handle_data_source_auth: Getting connector for connection_id: {connection_id}"
)
- connector = (
- await self.connector_service.connection_manager.get_connector(
- connection_id
- )
+ connector = await self.connector_service.connection_manager.get_connector(
+ connection_id
)
logger.info(
f"_handle_data_source_auth: Got connector: {connector is not None}, has _detect_base_url: {hasattr(connector, '_detect_base_url') if connector else False}"
@@ -466,47 +431,38 @@ class AuthService:
logger.info(
f"_handle_data_source_auth: Updated connector.base_url to: {connector.base_url}"
)
- await (
- self.connector_service.connection_manager.save_connections()
- )
+ await self.connector_service.connection_manager.save_connections()
result["base_url"] = detected_url
logger.info(
f"_handle_data_source_auth: Auto-detected and saved base URL: {detected_url}"
)
else:
- logger.warning(
- "_handle_data_source_auth: _detect_base_url returned None"
- )
+ logger.warning("_handle_data_source_auth: _detect_base_url returned None")
else:
logger.warning(
- f"_handle_data_source_auth: Connector not available or doesn't have _detect_base_url"
+ "_handle_data_source_auth: Connector not available or doesn't have _detect_base_url"
)
# Clear the cached connector so next get_connector() creates a fresh instance
# with the updated config (including base_url)
- if (
- connection_id
- in self.connector_service.connection_manager.active_connectors
- ):
+ if connection_id in self.connector_service.connection_manager.active_connectors:
logger.info(
f"_handle_data_source_auth: Clearing cached connector for {connection_id}"
)
- del self.connector_service.connection_manager.active_connectors[
- connection_id
- ]
- except Exception as e:
+ del self.connector_service.connection_manager.active_connectors[connection_id]
+ except Exception:
logger.exception("[AUTH] Auto-detect base URL failed")
return result
- async def get_user_info(self, request) -> Optional[dict]:
+ async def get_user_info(self, request) -> dict | None:
"""Get current user information from request"""
from config.settings import IBM_AUTH_ENABLED
# IBM auth mode: user is set by get_optional_user from IBM cookie
if IBM_AUTH_ENABLED:
user = getattr(request.state, "user", None)
- if user and user.provider in ("ibm_ams", "ibm_ams_basic"):
+ if user and user.provider in ("ibm_ams", "ibm_ams_basic", "ibm_ams_env"):
return {
"authenticated": True,
"ibm_auth_mode": True,
@@ -516,9 +472,7 @@ class AuthService:
"name": user.name,
"picture": user.picture,
"provider": user.provider,
- "last_login": user.last_login.isoformat()
- if user.last_login
- else None,
+ "last_login": user.last_login.isoformat() if user.last_login else None,
},
}
return {"authenticated": False, "ibm_auth_mode": True, "user": None}
@@ -539,9 +493,7 @@ class AuthService:
"name": user.name,
"picture": user.picture,
"provider": user.provider,
- "last_login": user.last_login.isoformat()
- if user.last_login
- else None,
+ "last_login": user.last_login.isoformat() if user.last_login else None,
},
}
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index b0befd650..8c971332f 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -1,30 +1,39 @@
import asyncio
-import platform
import json
+import platform
from pathlib import Path
-from typing import Any, Optional, Dict
+from typing import Any
+
import httpx
from pydantic import BaseModel
-from config.settings import get_openrag_config, DOCLING_SERVE_URL
+from config.settings import (
+ DOCLING_SERVE_URL,
+ DOCLING_SERVE_VERIFY_SSL,
+ IBM_AUTH_ENABLED,
+ get_openrag_config,
+)
from utils.logging_config import get_logger
logger = get_logger(__name__)
+
class DoclingConfig(BaseModel):
do_ocr: bool
ocr_engine: str
do_table_structure: bool
do_picture_classification: bool
do_picture_description: bool
- picture_description_local: Optional[dict] = None
+ picture_description_local: dict | None = None
+
class DoclingServeError(Exception):
"""Raised when docling-serve conversion fails."""
+
def get_docling_preset_configs(
table_structure=False, ocr=False, picture_descriptions=False
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
"""Get docling preset configurations based on toggle settings"""
is_macos = platform.system() == "Darwin"
@@ -42,13 +51,16 @@ def get_docling_preset_configs(
return config
+
class DoclingService:
- _default_client: Optional[httpx.AsyncClient] = None
+ _default_client: httpx.AsyncClient | None = None
- def __init__(self, docling_url: Optional[str] = None, httpx_client: Optional[httpx.AsyncClient] = None):
+ def __init__(
+ self, docling_url: str | None = None, httpx_client: httpx.AsyncClient | None = None
+ ):
"""
Initialize the DoclingService.
-
+
Args:
docling_url: Base URL of the Docling Serve instance. If None, auto-detects.
httpx_client: Pre-configured httpx async client.
@@ -57,7 +69,7 @@ class DoclingService:
self.docling_url = docling_url.rstrip("/")
else:
self.docling_url = DOCLING_SERVE_URL
-
+
self.httpx_client = httpx_client
def _get_client(self) -> httpx.AsyncClient:
@@ -65,65 +77,83 @@ class DoclingService:
return self.httpx_client
if DoclingService._default_client is None or DoclingService._default_client.is_closed:
DoclingService._default_client = httpx.AsyncClient(
- timeout=httpx.Timeout(300.0, connect=10.0)
+ timeout=httpx.Timeout(300.0, connect=10.0), verify=DOCLING_SERVE_VERIFY_SSL
)
return DoclingService._default_client
- def _build_docling_options(self) -> Dict[str, Any]:
+ def _build_docling_options(self) -> dict[str, Any]:
"""Build the options payload for docling from OpenRAG configs."""
config = get_openrag_config()
knowledge_config = config.knowledge
-
+
preset = get_docling_preset_configs(
table_structure=knowledge_config.table_structure,
ocr=knowledge_config.ocr,
- picture_descriptions=knowledge_config.picture_descriptions
+ picture_descriptions=knowledge_config.picture_descriptions,
)
-
- options = {
- "to_formats": "json",
- "image_export_mode": "placeholder",
- **preset
- }
+
+ options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}
return options
- async def upload_to_docling_direct_async(self, filename: str, file_content: bytes) -> str:
+ def _get_auth_headers(
+ self, user_id: str | None = None, auth_header: str | None = None
+ ) -> dict[str, str]:
+ """Build authentication headers for Docling Serve if IBM auth is enabled."""
+ headers = {}
+ if IBM_AUTH_ENABLED:
+ if auth_header:
+ headers["Authorization"] = auth_header
+
+ if user_id:
+ headers["X-Tenant-Id"] = user_id
+ return headers
+
+ async def upload_to_docling_direct_async(
+ self,
+ filename: str,
+ file_content: bytes,
+ user_id: str | None = None,
+ auth_header: str | None = None,
+ ) -> str:
"""
Upload a file to Docling Serve asynchronously using direct multipart/form-data upload.
"""
options = self._build_docling_options()
-
+ headers = self._get_auth_headers(user_id, auth_header)
+
# Docling serve async multipart endpoint /v1/convert/file/async
# Options are passed as form data
data = {
k: str(v).lower() if isinstance(v, bool) else v
for k, v in options.items()
if not isinstance(v, dict)
- } # picture_description_local needs to be JSON if it's a dict
+ } # picture_description_local needs to be JSON if it's a dict
if "picture_description_local" in options:
data["picture_description_local"] = json.dumps(options["picture_description_local"])
files = {"files": (filename, file_content)}
-
+
client = self._get_client()
should_close = client != self.httpx_client
-
+
try:
if should_close:
async with client:
response = await client.post(
f"{self.docling_url}/v1/convert/file/async",
files=files,
- data=data
+ data=data,
+ headers=headers,
)
else:
response = await client.post(
f"{self.docling_url}/v1/convert/file/async",
files=files,
- data=data
+ data=data,
+ headers=headers,
)
-
+
response.raise_for_status()
task = response.json()
return task["task_id"]
@@ -135,48 +165,67 @@ class DoclingService:
self,
task_id: str,
poll_interval: float = 1.0,
- timeout: float = 600.0
- ) -> Dict[str, Any]:
+ timeout: float = 600.0,
+ user_id: str | None = None,
+ auth_header: str | None = None,
+ ) -> dict[str, Any]:
"""
Poll Docling Serve for the result of an async conversion task.
"""
client = self._get_client()
should_close = client != self.httpx_client
-
+
try:
if should_close:
async with client:
- return await self._poll_result(client, task_id, poll_interval, timeout)
+ return await self._poll_result(
+ client, task_id, poll_interval, timeout, user_id, auth_header
+ )
else:
- return await self._poll_result(client, task_id, poll_interval, timeout)
+ return await self._poll_result(
+ client, task_id, poll_interval, timeout, user_id, auth_header
+ )
except Exception as e:
logger.error("Docling result retrieval failed", task_id=task_id, error=str(e))
raise
- async def _poll_result(self, client: httpx.AsyncClient, task_id: str, poll_interval: float, timeout: float) -> Dict[str, Any]:
+ async def _poll_result(
+ self,
+ client: httpx.AsyncClient,
+ task_id: str,
+ poll_interval: float,
+ timeout: float,
+ user_id: str | None = None,
+ auth_header: str | None = None,
+ ) -> dict[str, Any]:
"""Internal polling logic."""
elapsed = 0.0
+ headers = self._get_auth_headers(user_id, auth_header)
while elapsed < timeout:
try:
- response = await client.get(f"{self.docling_url}/v1/status/poll/{task_id}")
+ response = await client.get(
+ f"{self.docling_url}/v1/status/poll/{task_id}", headers=headers
+ )
response.raise_for_status()
status_data = response.json()
except Exception as e:
logger.error("Error polling docling status", task_id=task_id, error=str(e))
- raise DoclingServeError(f"Error polling docling status: {str(e)}")
+ raise DoclingServeError(f"Error polling docling status: {str(e)}") from e
status = status_data.get("task_status")
if status == "success":
- result_response = await client.get(f"{self.docling_url}/v1/result/{task_id}")
+ result_response = await client.get(
+ f"{self.docling_url}/v1/result/{task_id}", headers=headers
+ )
result_response.raise_for_status()
result_json = result_response.json()
-
+
# Extract the json_content which matches the old convert_file/bytes return
doc_content = result_json.get("document", {}).get("json_content")
if doc_content is None:
raise DoclingServeError("docling-serve response missing document.json_content")
-
+
return doc_content
elif status == "failure":
raise DoclingServeError(f"Docling conversion failed: {status_data}")
@@ -186,18 +235,34 @@ class DoclingService:
raise TimeoutError(f"Docling task {task_id} did not complete within {timeout} seconds")
- async def convert_file(self, file_path: str) -> Dict[str, Any]:
+ async def convert_file(
+ self, file_path: str, user_id: str | None = None, auth_header: str | None = None
+ ) -> dict[str, Any]:
"""
Convert a local file via docling-serve async polling.
"""
path = Path(file_path)
file_bytes = path.read_bytes()
- task_id = await self.upload_to_docling_direct_async(path.name, file_bytes)
- return await self.get_docling_result_async(task_id)
+ task_id = await self.upload_to_docling_direct_async(
+ path.name, file_bytes, user_id=user_id, auth_header=auth_header
+ )
+ return await self.get_docling_result_async(
+ task_id, user_id=user_id, auth_header=auth_header
+ )
- async def convert_bytes(self, content: bytes, filename: str) -> Dict[str, Any]:
+ async def convert_bytes(
+ self,
+ content: bytes,
+ filename: str,
+ user_id: str | None = None,
+ auth_header: str | None = None,
+ ) -> dict[str, Any]:
"""
Convert in-memory bytes via docling-serve async polling.
"""
- task_id = await self.upload_to_docling_direct_async(filename, content)
- return await self.get_docling_result_async(task_id)
+ task_id = await self.upload_to_docling_direct_async(
+ filename, content, user_id=user_id, auth_header=auth_header
+ )
+ return await self.get_docling_result_async(
+ task_id, user_id=user_id, auth_header=auth_header
+ )
diff --git a/src/services/document_service.py b/src/services/document_service.py
index 8b9672fb3..2b920dcbf 100644
--- a/src/services/document_service.py
+++ b/src/services/document_service.py
@@ -1,13 +1,11 @@
-from typing import List
import tiktoken
+
+from config.settings import get_embedding_model, get_index_name
+from utils.document_processing import extract_relevant
from utils.logging_config import get_logger
logger = get_logger(__name__)
-from config.settings import clients, get_embedding_model, get_index_name
-from utils.document_processing import extract_relevant
-from utils.telemetry import TelemetryClient, Category, MessageId
-
def get_token_count(text: str, model: str = None) -> int:
"""Get accurate token count using tiktoken"""
@@ -22,8 +20,8 @@ def get_token_count(text: str, model: str = None) -> int:
def chunk_texts_for_embeddings(
- texts: List[str], max_tokens: int = None, model: str = None
-) -> List[List[str]]:
+ texts: list[str], max_tokens: int = None, model: str = None
+) -> list[list[str]]:
"""
Split texts into batches that won't exceed token limits.
If max_tokens is None, returns texts as single batch (no splitting).
@@ -40,7 +38,7 @@ def chunk_texts_for_embeddings(
return [texts]
batches = []
- current_batch = []
+ current_batch: list[str] = []
current_tokens = 0
for text in texts:
@@ -93,7 +91,6 @@ class DocumentService:
self.docling_service = docling_service
self._mapping_ensured = False
-
async def process_upload_file(
self,
upload_file,
@@ -103,12 +100,13 @@ class DocumentService:
owner_email: str = None,
):
"""Process an uploaded file from form data"""
- from utils.hash_utils import hash_id
- from utils.file_utils import auto_cleanup_tempfile
import os
# Default metadata for anonymous users if not provided
from session_manager import AnonymousUser
+ from utils.file_utils import auto_cleanup_tempfile
+ from utils.hash_utils import hash_id
+
anonymous_user = AnonymousUser()
owner_name = owner_name or anonymous_user.name
owner_email = owner_email or anonymous_user.email
@@ -120,7 +118,7 @@ class DocumentService:
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Stream upload file to temporary file
file_size = 0
- with open(tmp_path, 'wb') as tmp_file:
+ with open(tmp_path, "wb") as tmp_file:
while True:
chunk = await upload_file.read(1 << 20)
if not chunk:
@@ -137,16 +135,19 @@ class DocumentService:
try:
exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)
except Exception as e:
- logger.error(
- "OpenSearch exists check failed", file_hash=file_hash, error=str(e)
- )
+ logger.error("OpenSearch exists check failed", file_hash=file_hash, error=str(e))
raise
if exists:
return {"status": "unchanged", "id": file_hash}
# Use consolidated standard processing
from models.processors import TaskProcessor
- processor = TaskProcessor(document_service=self, models_service=self.models_service, docling_service=self.docling_service)
+
+ processor = TaskProcessor(
+ document_service=self,
+ models_service=self.models_service,
+ docling_service=self.docling_service,
+ )
result = await processor.process_document_standard(
file_path=tmp_path,
file_hash=file_hash,
@@ -160,7 +161,9 @@ class DocumentService:
)
return result
- async def process_upload_context(self, upload_file, filename: str = None):
+ async def process_upload_context(
+ self, upload_file, filename: str = None, user_id: str = None, jwt_token: str = None
+ ):
"""Process uploaded file and return content for context"""
import io
import os
@@ -179,11 +182,11 @@ class DocumentService:
# Check if this is a .txt or .md file - use simple processing
file_ext = os.path.splitext(filename)[1].lower()
-
- if file_ext in ('.txt', '.md'):
+
+ if file_ext in (".txt", ".md"):
# Simple text file processing for chat context
- text_content = content.read().decode('utf-8', errors='replace')
-
+ text_content = content.read().decode("utf-8", errors="replace")
+
# For context, we don't need to chunk - just return the full content
return {
"filename": filename,
@@ -192,7 +195,9 @@ class DocumentService:
"content_length": len(text_content),
}
else:
- full_doc = await self.docling_service.convert_bytes(content.read(), filename)
+ full_doc = await self.docling_service.convert_bytes(
+ content.read(), filename, user_id=user_id, auth_header=jwt_token
+ )
slim_doc = extract_relevant(full_doc)
# Extract all text content
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 397438a07..eeb69ba0a 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -1,9 +1,10 @@
-import json
import asyncio
+import json
import time
-import httpx
from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any
+
+import httpx
from config.settings import (
LANGFLOW_INGEST_FLOW_ID,
@@ -41,9 +42,9 @@ class LangflowFileService:
@staticmethod
def merge_ui_ingest_settings_into_tweaks(
- tweaks: Optional[Dict[str, Any]],
- settings: Optional[Dict[str, Any]],
- ) -> Dict[str, Any]:
+ tweaks: dict[str, Any] | None,
+ settings: dict[str, Any] | None,
+ ) -> dict[str, Any]:
"""Merge UI ingest dict (camelCase) into Langflow run ``tweaks``.
- ``chunkSize`` / ``chunkOverlap`` / ``separator`` update the flow's
@@ -58,27 +59,19 @@ class LangflowFileService:
if not settings:
return final_tweaks
- if (
- settings.get("chunkSize")
- or settings.get("chunkOverlap")
- or settings.get("separator")
- ):
+ if settings.get("chunkSize") or settings.get("chunkOverlap") or settings.get("separator"):
if "SplitText-QIKhg" not in final_tweaks:
final_tweaks["SplitText-QIKhg"] = {}
if settings.get("chunkSize"):
final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings["chunkSize"]
if settings.get("chunkOverlap"):
- final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[
- "chunkOverlap"
- ]
+ final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings["chunkOverlap"]
if settings.get("separator"):
final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]
return final_tweaks
- async def upload_user_file(
- self, file_tuple, jwt_token: Optional[str] = None
- ) -> Dict[str, Any]:
+ async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:
"""Upload a file using Langflow Files API v2: POST /api/v2/files.
Returns JSON with keys: id, name, path, size, provider.
"""
@@ -125,22 +118,22 @@ class LangflowFileService:
async def run_ingestion_flow(
self,
- file_paths: List[str],
+ file_paths: list[str],
file_tuples: list[tuple[str, str, str]],
- jwt_token: Optional[str] = None,
- session_id: Optional[str] = None,
- tweaks: Optional[Dict[str, Any]] = None,
- owner: Optional[str] = None,
- owner_name: Optional[str] = None,
- owner_email: Optional[str] = None,
- connector_type: Optional[str] = None,
- document_id: Optional[str] = None,
- source_url: Optional[str] = None,
- allowed_users: Optional[List[str]] = None,
- allowed_groups: Optional[List[str]] = None,
- selected_embedding_model: Optional[str] = None,
- docling_task_id: Optional[str] = None,
- ) -> Dict[str, Any]:
+ jwt_token: str | None = None,
+ session_id: str | None = None,
+ tweaks: dict[str, Any] | None = None,
+ owner: str | None = None,
+ owner_name: str | None = None,
+ owner_email: str | None = None,
+ connector_type: str | None = None,
+ document_id: str | None = None,
+ source_url: str | None = None,
+ allowed_users: list[str] | None = None,
+ allowed_groups: list[str] | None = None,
+ selected_embedding_model: str | None = None,
+ docling_task_id: str | None = None,
+ ) -> dict[str, Any]:
"""
Trigger the ingestion flow with provided file paths.
The flow must expose a File component path in input schema or accept files parameter.
@@ -149,7 +142,7 @@ class LangflowFileService:
logger.error("[LF] LANGFLOW_INGEST_FLOW_ID is not configured")
raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")
- payload: Dict[str, Any] = {
+ payload: dict[str, Any] = {
"input_value": "Ingest files",
"input_type": "chat",
"output_type": "text", # Changed from "json" to "text"
@@ -188,15 +181,11 @@ class LangflowFileService:
bool(jwt_token),
)
# To compute the file size in bytes, use len() on the file content (which should be bytes)
- file_size_bytes = (
- len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0
- )
+ file_size_bytes = len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0
# Avoid logging full payload to prevent leaking sensitive data (e.g., JWT)
# Extract file metadata if file_tuples is provided
- filename = (
- str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""
- )
+ filename = str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""
mimetype = (
str(file_tuples[0][2])
if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2
@@ -222,9 +211,7 @@ class LangflowFileService:
"X-Langflow-Global-Var-MIMETYPE": mimetype,
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
- "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id)
- if document_id
- else "",
+ "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id) if document_id else "",
"X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "",
"X-Langflow-Global-Var-DOCLING_TASK_ID": str(docling_task_id)
if docling_task_id
@@ -234,13 +221,9 @@ class LangflowFileService:
# Serialize ACL lists as JSON strings for Langflow global vars
# (flows will parse these back into lists before indexing)
if allowed_users is not None:
- headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(
- allowed_users or []
- )
+ headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(allowed_users or [])
if allowed_groups is not None:
- headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(
- allowed_groups or []
- )
+ headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(allowed_groups or [])
# Add provider credentials as global variables for ingestion
await add_provider_credentials_to_headers(
@@ -327,20 +310,20 @@ class LangflowFileService:
self,
docs_url: str,
crawl_depth: int,
- jwt_token: Optional[str] = None,
- owner: Optional[str] = None,
- owner_name: Optional[str] = None,
- owner_email: Optional[str] = None,
+ jwt_token: str | None = None,
+ owner: str | None = None,
+ owner_name: str | None = None,
+ owner_email: str | None = None,
connector_type: str = "url",
prevent_outside: bool = True,
- tweaks: Optional[Dict[str, Any]] = None,
- ) -> Dict[str, Any]:
+ tweaks: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
"""Run URL-based docs ingestion flow using Langflow global variable passthrough."""
if not docs_url:
raise ValueError("DEFAULT_DOCS_URL is not configured")
flow_id = await self._ensure_url_ingest_flow_id()
- payload: Dict[str, Any] = {
+ payload: dict[str, Any] = {
"input_value": docs_url,
"input_type": "chat",
"output_type": "text",
@@ -428,8 +411,7 @@ class LangflowFileService:
flow_file = Path(get_flows_path()) / "openrag_url_mcp.json"
if not flow_file.exists():
raise ValueError(
- "LANGFLOW_URL_INGEST_FLOW_ID is invalid and "
- f"flow file was not found at {flow_file}"
+ f"LANGFLOW_URL_INGEST_FLOW_ID is invalid and flow file was not found at {flow_file}"
)
with flow_file.open("r", encoding="utf-8") as f:
flow_payload = json.load(f)
@@ -502,9 +484,7 @@ class LangflowFileService:
flow_data = create_resp.json()
imported_flow_id = flow_data.get("id")
if not imported_flow_id:
- raise ValueError(
- "Langflow flow import succeeded but no flow id was returned"
- )
+ raise ValueError("Langflow flow import succeeded but no flow id was returned")
self.flow_id_url_ingest = imported_flow_id
logger.warning(
@@ -538,15 +518,15 @@ class LangflowFileService:
async def upload_and_ingest_file(
self,
file_tuple,
- session_id: Optional[str] = None,
- tweaks: Optional[Dict[str, Any]] = None,
- settings: Optional[Dict[str, Any]] = None,
- jwt_token: Optional[str] = None,
- owner: Optional[str] = None,
- owner_name: Optional[str] = None,
- owner_email: Optional[str] = None,
- connector_type: Optional[str] = None,
- ) -> Dict[str, Any]:
+ session_id: str | None = None,
+ tweaks: dict[str, Any] | None = None,
+ settings: dict[str, Any] | None = None,
+ jwt_token: str | None = None,
+ owner: str | None = None,
+ owner_name: str | None = None,
+ owner_email: str | None = None,
+ connector_type: str | None = None,
+ ) -> dict[str, Any]:
"""
Combined Docling upload and Langflow ingest operation.
First uploads the file to Docling, then runs ingestion on it via Langflow
@@ -575,7 +555,7 @@ class LangflowFileService:
# Step 1: Upload the file to Docling
try:
task_id = await self.docling_service.upload_to_docling_direct_async(
- filename, content
+ filename, content, user_id=owner, auth_header=jwt_token
)
logger.debug(
"[LF] Docling upload completed successfully",
@@ -589,12 +569,10 @@ class LangflowFileService:
"[LF] Docling upload failed during combined operation",
extra={"error": str(e)},
)
- raise Exception(f"Docling upload failed: {str(e)}")
+ raise Exception(f"Docling upload failed: {str(e)}") from e
# Step 2: Prepare for ingestion
- final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
- tweaks, settings
- )
+ final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(tweaks, settings)
if settings:
logger.debug(
"[LF] Applying ingestion settings",
diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.py
index 95bcdd503..f197b987e 100644
--- a/src/utils/opensearch_init.py
+++ b/src/utils/opensearch_init.py
@@ -8,7 +8,9 @@ that need to (re)create the documents index after onboarding completes.
from config.settings import (
API_KEYS_INDEX_BODY,
API_KEYS_INDEX_NAME,
+ IBM_AUTH_ENABLED,
INDEX_BODY,
+ PLATFORM_AUTH_DEV_MODE,
clients,
get_index_name,
get_openrag_config,
@@ -41,6 +43,10 @@ async def wait_for_opensearch(opensearch_client=None):
async def configure_alerting_security():
"""Configure OpenSearch alerting plugin security settings"""
+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
+ logger.info("Skipping alerting security configuration in IBM dev mode.")
+ return
+
try:
alerting_settings = {
"persistent": {
@@ -58,6 +64,10 @@ async def configure_alerting_security():
async def _ensure_opensearch_index():
"""Ensure OpenSearch index exists when using traditional connector service."""
+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
+ logger.info("Skipping OpenSearch index creation in IBM dev mode.")
+ return
+
try:
index_name = get_index_name()
if await clients.opensearch.indices.exists(index=index_name):
@@ -111,22 +121,29 @@ async def init_index(opensearch_client=None, admin_username: str = None):
)
else:
logger.info(
- "Index already exists, skipping creation and changing number of replicas",
+ "Index already exists, skipping creation",
index_name=index_name,
embedding_model=embedding_model,
)
- current = await os_client.indices.get_settings(index=index_name)
- current_replicas = int(
- current[index_name]["settings"]["index"].get("number_of_replicas", 1)
- )
- if current_replicas != 0:
- await os_client.indices.put_settings(
- index=index_name,
- body={"index": {"number_of_replicas": 0}},
- )
- logger.info(
- "Updated documents index settings",
- )
+ if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):
+ # Set number of replicas to 0 to not create unused nodes in OpenSearch, in case it was created with more replicas
+ try:
+ current = await os_client.indices.get_settings(index=index_name)
+ current_replicas = int(
+ current[index_name]["settings"]["index"].get("number_of_replicas", 1)
+ )
+ if current_replicas != 0:
+ await os_client.indices.put_settings(
+ index=index_name,
+ body={"index": {"number_of_replicas": 0}},
+ )
+ logger.info("Updated documents index settings")
+ except Exception as e:
+ logger.warning(
+ "Failed to check or update index replicas",
+ index_name=index_name,
+ error=str(e),
+ )
await TelemetryClient.send_event(
Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS
)
@@ -169,20 +186,28 @@ async def init_index(opensearch_client=None, admin_username: str = None):
index_name=knowledge_filter_index_name,
)
- current = await os_client.indices.get_settings(index=knowledge_filter_index_name)
- current_replicas = int(
- current[knowledge_filter_index_name]["settings"]["index"].get(
- "number_of_replicas", 1
- )
- )
- if current_replicas != 0:
- await os_client.indices.put_settings(
- index=knowledge_filter_index_name,
- body={"index": {"number_of_replicas": 0}},
- )
- logger.info(
- "Updated knowledge filters index settings",
- )
+ if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):
+ try:
+ current = await os_client.indices.get_settings(
+ index=knowledge_filter_index_name
+ )
+ current_replicas = int(
+ current[knowledge_filter_index_name]["settings"]["index"].get(
+ "number_of_replicas", 1
+ )
+ )
+ if current_replicas != 0:
+ await os_client.indices.put_settings(
+ index=knowledge_filter_index_name,
+ body={"index": {"number_of_replicas": 0}},
+ )
+ logger.info("Updated knowledge filters index settings")
+ except Exception as e:
+ logger.warning(
+ "Failed to check or update knowledge filter index replicas",
+ index_name=knowledge_filter_index_name,
+ error=str(e),
+ )
if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):
await os_client.indices.create(index=API_KEYS_INDEX_NAME, body=API_KEYS_INDEX_BODY)
@@ -199,9 +224,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):
from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error
if is_disk_space_error(e):
- logger.error(
- "OpenSearch disk space exceeded watermark. Index creation failed."
- )
+ logger.error("OpenSearch disk space exceeded watermark. Index creation failed.")
raise OpenSearchDiskSpaceError(
"OpenSearch disk space is full (watermark exceeded). "
"Please free up disk space on your Docker volume or host machine to continue."
diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.py
index b89bfdf86..46d729965 100644
--- a/src/utils/opensearch_utils.py
+++ b/src/utils/opensearch_utils.py
@@ -1,8 +1,10 @@
import asyncio
import os
import random
+
import yaml
from opensearchpy import AsyncOpenSearch
+
from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -51,6 +53,7 @@ def is_disk_space_error(error: Exception) -> bool:
error_str = str(error).lower()
return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)
+
async def wait_for_opensearch(
opensearch_client: AsyncOpenSearch,
max_retries: int = 15,
@@ -109,7 +112,7 @@ async def wait_for_opensearch(
)
if attempt < max_retries - 1:
- delay = min(base_delay * (2 ** attempt), max_delay)
+ delay = min(base_delay * (2**attempt), max_delay)
delay = random.uniform(delay / 2, delay)
logger.debug(
@@ -127,36 +130,33 @@ async def wait_for_opensearch(
async def graceful_opensearch_shutdown(opensearch_client: AsyncOpenSearch) -> None:
"""Gracefully shutdown OpenSearch client connection.
-
+
This ensures that all pending operations are completed and connections
are properly closed before the application exits.
-
+
Args:
opensearch_client: The OpenSearch client to shutdown.
"""
if opensearch_client is None:
logger.debug("OpenSearch client is None, skipping graceful shutdown")
return
-
+
try:
logger.info("Initiating graceful OpenSearch shutdown...")
-
+
# Flush any pending operations by checking cluster health one last time
try:
- await asyncio.wait_for(
- opensearch_client.cluster.health(),
- timeout=10.0
- )
+ await asyncio.wait_for(opensearch_client.cluster.health(), timeout=10.0)
logger.debug("Final cluster health check completed")
- except asyncio.TimeoutError:
+ except TimeoutError:
logger.warning("Timeout during final cluster health check")
except Exception as e:
logger.debug("[OPENSEARCH] Final cluster health check skipped", reason=str(e))
-
+
# Close the client connection
await opensearch_client.close()
logger.info("OpenSearch client connection closed gracefully")
-
+
except Exception as e:
logger.error("Error during graceful OpenSearch shutdown", error=str(e))
@@ -184,7 +184,11 @@ async def setup_opensearch_security(
This should be called during initial setup after OpenSearch is ready.
"""
- from config.settings import IBM_AUTH_ENABLED
+ from config.settings import IBM_AUTH_ENABLED, PLATFORM_AUTH_DEV_MODE
+
+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
+ logger.info("Skipping OpenSearch security configuration in IBM dev mode.")
+ return
logger.info("Initializing OpenSearch security configuration...", ibm_auth=IBM_AUTH_ENABLED)
@@ -210,13 +214,20 @@ async def setup_opensearch_security(
try:
# 1. & 2. Readiness checks
logger.info("[OPENSEARCH] Performing readiness checks...")
-
+
try:
- rolesmapping_response = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping")
- logger.info("[OPENSEARCH] Current rolesmapping retrieved", count=len(rolesmapping_response) if isinstance(rolesmapping_response, dict) else "unknown")
+ rolesmapping_response = await opensearch_client.transport.perform_request(
+ "GET", "/_plugins/_security/api/rolesmapping"
+ )
+ logger.info(
+ "[OPENSEARCH] Current rolesmapping retrieved",
+ count=len(rolesmapping_response)
+ if isinstance(rolesmapping_response, dict)
+ else "unknown",
+ )
except Exception as e:
logger.warning("[OPENSEARCH] Failed to get current rolesmapping", error=str(e))
-
+
cluster_health = await opensearch_client.cluster.health()
logger.info("[OPENSEARCH] Cluster health check passed", status=cluster_health.get("status"))
@@ -225,10 +236,13 @@ async def setup_opensearch_security(
logger.error(f"[OPENSEARCH] Roles configuration file not found: {roles_file}")
raise FileNotFoundError(f"Roles configuration file not found: {roles_file}")
- with open(roles_file, "r") as f:
+ with open(roles_file) as f:
roles_config = yaml.safe_load(f)
-
- logger.info("[OPENSEARCH] Loaded roles configuration", roles=list(roles_config.keys()) if roles_config else [])
+
+ logger.info(
+ "[OPENSEARCH] Loaded roles configuration",
+ roles=list(roles_config.keys()) if roles_config else [],
+ )
# 3. Create openrag_user_role
if "openrag_user_role" in roles_config:
@@ -236,15 +250,19 @@ async def setup_opensearch_security(
logger.info(
"[OPENSEARCH] Creating 'openrag_user_role' role",
- patterns=role_body['index_permissions'][0]['index_patterns'] if 'index_permissions' in role_body else 'default',
- allowed_actions=role_body['index_permissions'][0].get('allowed_actions', []) if 'index_permissions' in role_body else []
+ patterns=role_body["index_permissions"][0]["index_patterns"]
+ if "index_permissions" in role_body
+ else "default",
+ allowed_actions=role_body["index_permissions"][0].get("allowed_actions", [])
+ if "index_permissions" in role_body
+ else [],
)
-
+
resp = await opensearch_client.transport.perform_request(
"PUT",
"/_plugins/_security/api/roles/openrag_user_role",
body=role_body,
- headers={"Content-Type": "application/json"}
+ headers={"Content-Type": "application/json"},
)
logger.info("[OPENSEARCH] Role creation response", response=resp)
else:
@@ -255,10 +273,13 @@ async def setup_opensearch_security(
logger.error(f"[OPENSEARCH] Roles mapping file not found: {roles_mapping_file}")
raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")
- with open(roles_mapping_file, "r") as f:
+ with open(roles_mapping_file) as f:
mapping_config = yaml.safe_load(f)
-
- logger.info("[OPENSEARCH] Loaded roles mapping configuration", mappings=list(mapping_config.keys()) if mapping_config else [])
+
+ logger.info(
+ "[OPENSEARCH] Loaded roles mapping configuration",
+ mappings=list(mapping_config.keys()) if mapping_config else [],
+ )
# 4. Create openrag_user_role mapping
if "openrag_user_role" in mapping_config:
@@ -266,13 +287,13 @@ async def setup_opensearch_security(
logger.info(
"[OPENSEARCH] Creating 'openrag_user_role' mapping",
backend_roles=mapping_body.get("backend_roles", []),
- users=mapping_body.get("users", [])
+ users=mapping_body.get("users", []),
)
resp = await opensearch_client.transport.perform_request(
"PUT",
"/_plugins/_security/api/rolesmapping/openrag_user_role",
body=mapping_body,
- headers={"Content-Type": "application/json"}
+ headers={"Content-Type": "application/json"},
)
logger.info("[OPENSEARCH] Role mapping update response", response=resp)
@@ -316,9 +337,9 @@ async def setup_opensearch_security(
users=new_admin_users,
)
- merged_users = list(set(
- all_access_body.get("users", []) + existing_users + new_admin_users
- ))
+ merged_users = list(
+ set(all_access_body.get("users", []) + existing_users + new_admin_users)
+ )
all_access_body["users"] = merged_users
logger.debug("[OPENSEARCH] Merged all_access users", users=merged_users)
@@ -357,16 +378,20 @@ async def setup_opensearch_security(
"PUT",
"/_plugins/_security/api/rolesmapping/all_access",
body=all_access_body,
- headers={"Content-Type": "application/json"}
+ headers={"Content-Type": "application/json"},
)
logger.info("[OPENSEARCH] All access mapping update response", response=resp)
# 6. Final verification
logger.info("[OPENSEARCH] Verifying security configuration...")
- role_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/roles/openrag_user_role")
+ role_verify = await opensearch_client.transport.perform_request(
+ "GET", "/_plugins/_security/api/roles/openrag_user_role"
+ )
logger.info("[OPENSEARCH] Role verification", role=role_verify)
-
- mapping_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping/openrag_user_role")
+
+ mapping_verify = await opensearch_client.transport.perform_request(
+ "GET", "/_plugins/_security/api/rolesmapping/openrag_user_role"
+ )
logger.info("[OPENSEARCH] Role mapping verification", mapping=mapping_verify)
logger.info("Successfully completed OpenSearch security configuration.")
@@ -374,7 +399,9 @@ async def setup_opensearch_security(
except Exception as e:
# Check for authentication errors or if the security plugin is missing
error_str = str(e).lower()
- if any(code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]):
+ if any(
+ code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]
+ ):
logger.warning(
"Skipping OpenSearch security configuration: "
"The cluster may not have the security plugin enabled or "