OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: added docling authentication when ibm auth is enabled (#1551)
Lucas Oliveira2b275363e408
diff --git a/.env.example b/.env.exampleindex 0f933bc48..02fb7451c 100644--- a/.env.example+++ b/.env.example@@ -107,6 +107,11 @@ IBM_SESSION_COOKIE_NAME=ibm-openrag-session# Header injected by Traefik on every forwarded request containing the OpenSearch Basic credentials# (base64-encoded username:password). Override if your proxy uses a different header name.IBM_CREDENTIALS_HEADER=X-IBM-LH-Credentials+# Optional: Use these to override authentication for local development/calls when IBM_AUTH_ENABLED=true+IBM_USERNAME=+IBM_PASSWORD=+# Set to true to disable OpenSearch security roles/mapping setup during onboarding/startup+IBM_AUTH_DEV_MODE=false# make here https://console.cloud.google.com/apis/credentialsGOOGLE_OAUTH_CLIENT_ID=@@ -168,6 +173,7 @@ LANGFLOW_PUBLIC_URL=# OPTIONAL: Override the full docling-serve URL (e.g., for remote instances)# If not set, auto-detects host and uses port 5001# DOCLING_SERVE_URL=http://my-docling-server:5001+# DOCLING_SERVE_VERIFY_SSL=true# OPTIONAL: OCR engine used by docling for scanned documents. Options: "easyocr" (default), "tesseract", "tesseract_cli"# DOCLING_OCR_ENGINE=easyocrdiff --git a/.github/workflows/autofix.ci.yml b/.github/workflows/autofix.ci.ymlindex 9c09c03f5..6c3cf9a94 100644--- a/.github/workflows/autofix.ci.yml+++ b/.github/workflows/autofix.ci.yml@@ -54,7 +54,7 @@ jobs:base="${{ github.event.pull_request.base.sha }}"head="${{ github.event.pull_request.head.sha }}"merge_base=$(git merge-base "$base" "$head")- mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py')+ mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py' ':!flows/components/*')if [ "${#files[@]}" -eq 0 ]; thenecho "No Python files changed."echo "files=" >> "$GITHUB_OUTPUT"diff --git a/.github/workflows/lint-backend.yml b/.github/workflows/lint-backend.ymlindex 7e447ad10..24f6fc8b5 100644--- a/.github/workflows/lint-backend.yml+++ b/.github/workflows/lint-backend.yml@@ -45,7 +45,7 @@ jobs:head="${{ github.event.pull_request.head.sha }}"# Diff against the merge base so we only see files touched in the PR.merge_base=$(git merge-base "$base" "$head")- mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py')+ mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py' ':!flows/components/*')if [ "${#files[@]}" -eq 0 ]; thenecho "No Python files changed."echo "files=" >> "$GITHUB_OUTPUT"diff --git a/docker-compose.yml b/docker-compose.ymlindex 3b8d82bbe..004e96f0a 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -112,7 +112,12 @@ services:- INGESTION_TIMEOUT=${INGESTION_TIMEOUT:-3600}- UPLOAD_BATCH_SIZE=${UPLOAD_BATCH_SIZE:-25}- MAX_WORKERS=${MAX_WORKERS:-4}+ - IBM_AUTH_ENABLED=${IBM_AUTH_ENABLED:-false}+ - PLATFORM_AUTH_DEV_MODE=${PLATFORM_AUTH_DEV_MODE:-false}+ - PLATFORM_USERNAME=${PLATFORM_USERNAME}+ - PLATFORM_PASSWORD=${PLATFORM_PASSWORD}- DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001}+ - DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true}volumes:- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:U,z- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,zdiff --git a/flows/components/docling_remote.py b/flows/components/docling_remote.pynew file mode 100644index 000000000..177f3d455--- /dev/null+++ b/flows/components/docling_remote.py@@ -0,0 +1,347 @@+from __future__ import annotations++import base64+import json+import time+from concurrent.futures import Future, ThreadPoolExecutor+from pathlib import Path+from typing import Any++import httpx+from docling_core.types.doc import DoclingDocument+from pydantic import ValidationError++from lfx.base.data import BaseFileComponent+from lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput+from lfx.inputs.inputs import FloatInput+from lfx.schema import Data+from lfx.schema.dotdict import dotdict+from lfx.schema.message import Message+from lfx.utils.util import transform_localhost_url+++class DoclingRemoteComponent(BaseFileComponent):+ display_name = "Docling Serve"+ description = (+ "Uses Docling to process input documents connecting to your instance of Docling Serve."+ )+ documentation = "https://docling-project.github.io/docling/"+ trace_type = "tool"+ icon = "Docling"+ name = "DoclingRemote"++ MAX_500_RETRIES = 5++ # https://docling-project.github.io/docling/usage/supported_formats/+ VALID_EXTENSIONS = [+ "adoc",+ "asciidoc",+ "asc",+ "bmp",+ "csv",+ "dotx",+ "dotm",+ "docm",+ "docx",+ "htm",+ "html",+ "jpeg",+ "jpg",+ "json",+ "md",+ "pdf",+ "png",+ "potx",+ "ppsx",+ "pptm",+ "potm",+ "ppsm",+ "pptx",+ "tiff",+ "txt",+ "xls",+ "xlsx",+ "xhtml",+ "xml",+ "webp",+ ]++ inputs = [+ *BaseFileComponent.get_base_inputs(),+ StrInput(+ name="api_url",+ display_name="Server address",+ info="URL of the Docling Serve instance.",+ required=True,+ ),+ StrInput(+ name="task_id",+ display_name="Task ID",+ info=(+ "Optional task ID from a previous Docling Serve upload. "+ "If provided, file input is ignored and the component polls for this task's results."+ ),+ required=False,+ ),+ IntInput(+ name="max_concurrency",+ display_name="Concurrency",+ info="Maximum number of concurrent requests for the server.",+ advanced=True,+ value=2,+ input_types=["Message"],+ ),+ FloatInput(+ name="max_poll_timeout",+ display_name="Maximum poll time",+ info="Maximum waiting time for the document conversion to complete.",+ advanced=True,+ value=3600,+ input_types=["Message"],+ ),+ TableInput(+ name="api_headers",+ display_name="HTTP headers",+ advanced=True,+ required=False,+ info=("Optional headers required for connecting to Docling Serve."),+ table_schema=[+ {+ "name": "key",+ "display_name": "Key",+ "type": "string",+ "description": "Key name",+ },+ {+ "name": "value",+ "display_name": "Value",+ "load_from_db": True,+ "type": "string",+ "description": "Value of the header",+ },+ ],+ value=[],+ real_time_refresh=True,+ input_types=["Data", "JSON"],+ ),+ NestedDictInput(+ name="docling_serve_opts",+ display_name="Docling options",+ advanced=True,+ required=False,+ info=(+ "Optional dictionary of additional options. "+ "See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information."+ ),+ input_types=["Message"],+ ),+ ]++ outputs = [+ *BaseFileComponent.get_base_outputs(),+ ]++ def _process_headers(self) -> dict[str, str]:+ """Process the headers input into a valid dictionary."""+ if not self.api_headers:+ return {}++ component_headers_dict = {}+ # TableInput normalizes to list+ items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]++ for item in items:+ if not item:+ continue++ # Case 1: Data object+ if hasattr(item, "data") and isinstance(item.data, dict):+ data = item.data+ if "key" in data and "value" in data:+ component_headers_dict[str(data["key"])] = str(data["value"])+ else:+ # Fallback: merge all keys from Data object+ for k, v in data.items():+ if k not in ("text_key", "default_value"):+ component_headers_dict[str(k)] = str(v)++ # Case 2: Dictionary (Table row)+ elif isinstance(item, dict):+ if "key" in item and "value" in item:+ component_headers_dict[str(item["key"])] = str(item["value"])+ else:+ # Fallback: merge all keys+ for k, v in item.items():+ component_headers_dict[str(k)] = str(v)++ # Case 3: Message object+ elif hasattr(item, "text") and isinstance(item.text, str):+ try:+ parsed = json.loads(item.text)+ if isinstance(parsed, dict):+ for k, v in parsed.items():+ component_headers_dict[str(k)] = str(v)+ except json.JSONDecodeError:+ pass++ return component_headers_dict++ def update_build_config(+ self, build_config: dotdict, field_value: Any, field_name: str | None = None+ ) -> dotdict:+ if field_name == "api_headers":+ if isinstance(field_value, dict):+ # If it's a dict, convert to list of {key, value} pairs for TableInput+ # This handles migration from NestedDictInput to TableInput+ new_value = [{"key": k, "value": v} for k, v in field_value.items()]+ build_config["api_headers"]["value"] = new_value+ return build_config+ if field_value is None:+ build_config["api_headers"]["value"] = []+ return build_config++ # Default behavior+ return super().update_build_config(build_config, field_value, field_name)++ def _poll_and_fetch_result(+ self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None+ ) -> Data | None:+ """Poll for task completion and fetch the result.++ Args:+ client: The HTTP client to use for requests.+ base_url: The base URL of the Docling Serve API.+ task_id: The task ID to poll for.+ file_path: Optional file path to include in the result data.++ Returns:+ Data object with the DoclingDocument, or None if processing failed.+ """+ http_failures = 0+ retry_status_start = 500+ retry_status_end = 600+ start_wait_time = time.monotonic()++ response = client.get(f"{base_url}/status/poll/{task_id}")+ response.raise_for_status()+ task = response.json()++ while task["task_status"] not in ("success", "failure"):+ processing_time = time.monotonic() - start_wait_time+ if processing_time >= self.max_poll_timeout:+ msg = (+ f"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}."+ "Please increase the max_poll_timeout parameter or review why the processing "+ "takes long on the server."+ )+ self.log(msg)+ raise RuntimeError(msg)++ time.sleep(2)+ response = client.get(f"{base_url}/status/poll/{task_id}")++ if retry_status_start <= response.status_code < retry_status_end:+ http_failures += 1+ if http_failures > self.MAX_500_RETRIES:+ self.log(+ f"The status requests got a http response {response.status_code} too many times."+ )+ return None+ continue++ task = response.json()++ result_resp = client.get(f"{base_url}/result/{task_id}")+ result_resp.raise_for_status()+ result = result_resp.json()++ if "json_content" not in result["document"] or result["document"]["json_content"] is None:+ self.log("No JSON DoclingDocument found in the result.")+ return None++ try:+ doc = DoclingDocument.model_validate(result["document"]["json_content"])+ data_dict: dict[str, Any] = {"doc": doc}+ if file_path:+ data_dict["file_path"] = file_path+ return Data(data=data_dict)+ except ValidationError as e:+ self.log(f"Error validating the document. {e}")+ return None++ def _process_task_id(self) -> list[Data]:+ """Process an existing task by polling for status and retrieving results.++ Returns:+ List containing the result Data object, or empty list if processing failed.+ """+ transformed_url = transform_localhost_url(self.api_url)+ base_url = f"{transformed_url}/v1"++ with httpx.Client(headers=self._process_headers()) as client:+ result = self._poll_and_fetch_result(client, base_url, self.task_id)+ return [result] if result else []++ def load_files_base(self) -> list[Data]:+ """Load and process files, or poll an existing task if task_id is provided.++ Returns:+ list[Data]: Parsed data from the processed files or task.+ """+ if self.task_id:+ return self._process_task_id()+ return super().load_files_base()++ def process_files(+ self, file_list: list[BaseFileComponent.BaseFile]+ ) -> list[BaseFileComponent.BaseFile]:+ transformed_url = transform_localhost_url(self.api_url)+ base_url = f"{transformed_url}/v1"++ def _convert_document(+ client: httpx.Client, file_path: Path, options: dict[str, Any]+ ) -> Data | None:+ encoded_doc = base64.b64encode(file_path.read_bytes()).decode()+ payload = {+ "options": options,+ "sources": [+ {"kind": "file", "base64_string": encoded_doc, "filename": file_path.name}+ ],+ }++ response = client.post(f"{base_url}/convert/source/async", json=payload)+ response.raise_for_status()+ task = response.json()++ return self._poll_and_fetch_result(client, base_url, task["task_id"], str(file_path))++ docling_options = {+ "to_formats": ["json"],+ "image_export_mode": "placeholder",+ **(self.docling_serve_opts or {}),+ }++ processed_data: list[Data | None] = []+ with (+ httpx.Client(headers=self._process_headers()) as client,+ ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,+ ):+ futures: list[tuple[int, Future]] = []+ for i, file in enumerate(file_list):+ if file.path is None:+ processed_data.append(None)+ continue++ futures.append(+ (i, executor.submit(_convert_document, client, file.path, docling_options))+ )++ for _index, future in futures:+ try:+ result_data = future.result()+ processed_data.append(result_data)+ except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:+ self.log(f"Docling remote processing failed: {exc}")+ raise++ return self.rollup_data(file_list, processed_data)diff --git a/flows/components/mcp_component.py b/flows/components/mcp_component.pynew file mode 100644index 000000000..f0bffb1ad--- /dev/null+++ b/flows/components/mcp_component.py@@ -0,0 +1,835 @@+from __future__ import annotations++import asyncio+import json+import uuid++from langchain_core.tools import StructuredTool # noqa: TC002++from lfx.base.agents.utils import maybe_unflatten_dict, safe_cache_get, safe_cache_set+from lfx.base.mcp.util import (+ MCPStdioClient,+ MCPStreamableHttpClient,+ create_input_schema_from_json_schema,+ update_tools,+)+from lfx.custom.custom_component.component_with_cache import ComponentWithCache+from lfx.inputs.inputs import InputTypes # noqa: TC001+from lfx.io import (+ BoolInput,+ DictInput,+ DropdownInput,+ McpInput,+ MessageTextInput,+ Output,+ TableInput,+)+from lfx.io.schema import flatten_schema, schema_to_langflow_inputs+from lfx.log.logger import logger+from lfx.schema.dataframe import DataFrame+from lfx.schema.message import Message+from lfx.services.deps import get_storage_service, session_scope+++def resolve_mcp_config(+ server_name: str, # noqa: ARG001+ server_config_from_value: dict | None,+ server_config_from_db: dict | None,+) -> dict | None:+ """Resolve MCP server config with proper precedence.++ Resolves the configuration for an MCP server with the following precedence:+ 1. Database config (takes priority) - ensures edits are reflected+ 2. Config from value/tweaks (fallback) - allows REST API to provide config for new servers++ Args:+ server_name: Name of the MCP server+ server_config_from_value: Config provided via value/tweaks (optional)+ server_config_from_db: Config from database (optional)++ Returns:+ Final config to use (DB takes priority, falls back to value)+ Returns None if no config found in either location+ """+ if server_config_from_db:+ return server_config_from_db+ return server_config_from_value+++class MCPToolsComponent(ComponentWithCache):+ schema_inputs: list = []+ tools: list[StructuredTool] = []+ _not_load_actions: bool = False+ _tool_cache: dict = {}+ _last_selected_server: str | None = None # Cache for the last selected server++ def __init__(self, **data) -> None:+ super().__init__(**data)+ # Initialize cache keys to avoid CacheMiss when accessing them+ self._ensure_cache_structure()++ # Initialize clients with access to the component cache+ self.stdio_client: MCPStdioClient = MCPStdioClient(+ component_cache=self._shared_component_cache+ )+ self.streamable_http_client: MCPStreamableHttpClient = MCPStreamableHttpClient(+ component_cache=self._shared_component_cache+ )++ def _ensure_cache_structure(self):+ """Ensure the cache has the required structure."""+ # Check if servers key exists and is not CacheMiss+ servers_value = safe_cache_get(self._shared_component_cache, "servers")+ if servers_value is None:+ safe_cache_set(self._shared_component_cache, "servers", {})++ # Check if last_selected_server key exists and is not CacheMiss+ last_server_value = safe_cache_get(self._shared_component_cache, "last_selected_server")+ if last_server_value is None:+ safe_cache_set(self._shared_component_cache, "last_selected_server", "")++ default_keys: list[str] = [+ "code",+ "_type",+ "tool_mode",+ "tool_placeholder",+ "mcp_server",+ "tool",+ "use_cache",+ "verify_ssl",+ "headers",+ ]++ display_name = "MCP Tools"+ description = "Connect to an MCP server to use its tools."+ documentation: str = "https://docs.langflow.org/mcp-tools"+ icon = "Mcp"+ name = "MCPTools"++ inputs = [+ McpInput(+ name="mcp_server",+ display_name="MCP Server",+ info="Select the MCP Server that will be used by this component",+ real_time_refresh=True,+ ),+ BoolInput(+ name="use_cache",+ display_name="Use Cached Server",+ info=(+ "Enable caching of MCP Server and tools to improve performance. "+ "Disable to always fetch fresh tools and server updates."+ ),+ value=False,+ advanced=True,+ ),+ BoolInput(+ name="verify_ssl",+ display_name="Verify SSL Certificate",+ info=(+ "Enable SSL certificate verification for HTTPS connections. "+ "Disable only for development/testing with self-signed certificates."+ ),+ value=True,+ advanced=True,+ ),+ TableInput(+ name="headers",+ display_name="Headers",+ info=(+ "HTTP headers to include with MCP server requests. "+ "Useful for authentication (e.g., Authorization header). "+ "These headers override any headers configured in the MCP server settings."+ ),+ advanced=True,+ table_schema=[+ {+ "name": "key",+ "display_name": "Key",+ "type": "string",+ "description": "Key name",+ },+ {+ "name": "value",+ "display_name": "Value",+ "load_from_db": True,+ "type": "string",+ "description": "Value of the header",+ },+ ],+ value={},+ input_types=["Data", "JSON"],+ ),+ DropdownInput(+ name="tool",+ display_name="Tool",+ options=[],+ value="",+ info="Select the tool to execute",+ show=False,+ required=True,+ real_time_refresh=True,+ refresh_button=True,+ ),+ MessageTextInput(+ name="tool_placeholder",+ display_name="Tool Placeholder",+ info="Placeholder for the tool",+ value="",+ show=False,+ tool_mode=False,+ ),+ ]++ outputs = [+ Output(display_name="Response", name="response", method="build_output"),+ ]++ async def _validate_schema_inputs(self, tool_obj) -> list[InputTypes]:+ """Validate and process schema inputs for a tool."""+ try:+ if not tool_obj or not hasattr(tool_obj, "args_schema"):+ msg = "Invalid tool object or missing input schema"+ raise ValueError(msg)++ flat_schema = flatten_schema(tool_obj.args_schema.schema())+ input_schema = create_input_schema_from_json_schema(flat_schema)+ if not input_schema:+ msg = f"Empty input schema for tool '{tool_obj.name}'"+ raise ValueError(msg)++ schema_inputs = schema_to_langflow_inputs(input_schema)+ if not schema_inputs:+ msg = f"No input parameters defined for tool '{tool_obj.name}'"+ await logger.awarning(msg)+ return []++ except Exception as e:+ msg = f"Error validating schema inputs: {e!s}"+ await logger.aexception(msg)+ raise ValueError(msg) from e+ else:+ return schema_inputs++ async def update_tool_list(self, mcp_server_value=None):+ # Accepts mcp_server_value as dict {name, config} or uses self.mcp_server+ mcp_server = (+ mcp_server_value if mcp_server_value is not None else getattr(self, "mcp_server", None)+ )+ server_name = None+ server_config_from_value = None+ if isinstance(mcp_server, dict):+ server_name = mcp_server.get("name")+ server_config_from_value = mcp_server.get("config")+ else:+ server_name = mcp_server+ if not server_name:+ self.tools = []+ return [], {"name": server_name, "config": server_config_from_value}++ # Check if caching is enabled, default to False+ use_cache = getattr(self, "use_cache", False)++ # Use shared cache if available and caching is enabled+ cached = None+ if use_cache:+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ cached = servers_cache.get(server_name) if isinstance(servers_cache, dict) else None++ if cached is not None:+ try:+ self.tools = cached["tools"]+ self.tool_names = cached["tool_names"]+ self._tool_cache = cached["tool_cache"]+ server_config_from_value = cached["config"]+ except (TypeError, KeyError, AttributeError) as e:+ # Handle corrupted cache data by clearing it and continuing to fetch fresh tools+ msg = f"Unable to use cached data for MCP Server{server_name}: {e}"+ await logger.awarning(msg)+ # Clear the corrupted cache entry+ current_servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ if isinstance(current_servers_cache, dict) and server_name in current_servers_cache:+ current_servers_cache.pop(server_name)+ safe_cache_set(self._shared_component_cache, "servers", current_servers_cache)+ else:+ return self.tools, {"name": server_name, "config": server_config_from_value}++ try:+ # Try to fetch from database first to ensure we have the latest config+ # This ensures database updates (like editing a server) take effect+ try:+ from langflow.api.v2.mcp import get_server+ from langflow.services.database.models.user.crud import get_user_by_id++ from lfx.services.deps import get_settings_service+ except ImportError as e:+ msg = (+ "Langflow MCP server functionality is not available. "+ "This feature requires the full Langflow installation."+ )+ raise ImportError(msg) from e++ server_config_from_db = None+ async with session_scope() as db:+ if not self.user_id:+ msg = "User ID is required for fetching MCP tools."+ raise ValueError(msg)+ current_user = await get_user_by_id(db, self.user_id)++ # Try to get server config from DB/API+ server_config_from_db = await get_server(+ server_name,+ current_user,+ db,+ storage_service=get_storage_service(),+ settings_service=get_settings_service(),+ )++ # Resolve config with proper precedence: DB takes priority, falls back to value+ server_config = resolve_mcp_config(+ server_name=server_name,+ server_config_from_value=server_config_from_value,+ server_config_from_db=server_config_from_db,+ )++ if not server_config:+ self.tools = []+ return [], {"name": server_name, "config": server_config}++ # Add verify_ssl option to server config if not present+ if "verify_ssl" not in server_config:+ verify_ssl = getattr(self, "verify_ssl", True)+ server_config["verify_ssl"] = verify_ssl++ # Merge headers from component input with server config headers+ # Component headers take precedence over server config headers+ component_headers = getattr(self, "headers", None)+ if component_headers:+ # Convert list of {"key": k, "value": v}, Data objects, or dicts to dict+ component_headers_dict = {}++ # TableInput.validate_value normalizes single items to lists+ items = (+ component_headers+ if isinstance(component_headers, list)+ else [component_headers]+ )++ for item in items:+ if not item:+ continue++ # Case 1: Data object (Langflow standard)+ if hasattr(item, "data") and isinstance(item.data, dict):+ data = item.data+ if "key" in data and "value" in data:+ component_headers_dict[str(data["key"])] = str(data["value"])+ else:+ # Fallback: merge all keys from Data object (excluding internal keys)+ for k, v in data.items():+ if k not in ("text_key", "default_value"):+ component_headers_dict[str(k)] = str(v)++ # Case 2: Dictionary (Table row or direct JSON)+ elif isinstance(item, dict):+ if "key" in item and "value" in item:+ component_headers_dict[str(item["key"])] = str(item["value"])+ else:+ # Fallback: merge all keys+ for k, v in item.items():+ component_headers_dict[str(k)] = str(v)++ # Case 3: Message object+ elif hasattr(item, "text") and isinstance(item.text, str):+ try:+ parsed = json.loads(item.text)+ if isinstance(parsed, dict):+ for k, v in parsed.items():+ component_headers_dict[str(k)] = str(v)+ except json.JSONDecodeError:+ pass++ if component_headers_dict:+ existing_headers = server_config.get("headers", {}) or {}+ # Ensure existing_headers is a dict (convert from list if needed)+ if isinstance(existing_headers, list):+ existing_dict = {}+ for item in existing_headers:+ if isinstance(item, dict) and "key" in item and "value" in item:+ existing_dict[item["key"]] = item["value"]+ existing_headers = existing_dict+ merged_headers = {**existing_headers, **component_headers_dict}+ server_config["headers"] = merged_headers+ # Get request_variables from graph context for global variable resolution+ request_variables = None+ if hasattr(self, "graph") and self.graph and hasattr(self.graph, "context"):+ request_variables = self.graph.context.get("request_variables")++ # Only load global variables from database if we have headers that might use them+ # This avoids unnecessary database queries when headers are empty+ has_headers = server_config.get("headers") and len(server_config.get("headers", {})) > 0+ if not request_variables and has_headers:+ try:+ from lfx.services.deps import get_variable_service++ variable_service = get_variable_service()+ if variable_service:+ async with session_scope() as db:+ request_variables = await variable_service.get_all_decrypted_variables(+ user_id=self.user_id, session=db+ )+ except Exception as e: # noqa: BLE001+ await logger.awarning(f"Failed to load global variables for MCP component: {e}")++ _, tool_list, tool_cache = await update_tools(+ server_name=server_name,+ server_config=server_config,+ mcp_stdio_client=self.stdio_client,+ mcp_streamable_http_client=self.streamable_http_client,+ request_variables=request_variables,+ )++ self.tool_names = [tool.name for tool in tool_list if hasattr(tool, "name")]+ self._tool_cache = tool_cache+ self.tools = tool_list++ # Cache the result only if caching is enabled+ if use_cache:+ cache_data = {+ "tools": tool_list,+ "tool_names": self.tool_names,+ "tool_cache": tool_cache,+ "config": server_config,+ }++ # Safely update the servers cache+ current_servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ if isinstance(current_servers_cache, dict):+ current_servers_cache[server_name] = cache_data+ safe_cache_set(self._shared_component_cache, "servers", current_servers_cache)++ except (TimeoutError, asyncio.TimeoutError) as e:+ msg = f"Timeout updating tool list: {e!s}"+ await logger.aexception(msg)+ raise TimeoutError(msg) from e+ except Exception as e:+ msg = f"Error updating tool list: {e!s}"+ await logger.aexception(msg)+ raise ValueError(msg) from e+ else:+ return tool_list, {"name": server_name, "config": server_config}++ async def update_build_config(+ self, build_config: dict, field_value: str, field_name: str | None = None+ ) -> dict:+ """Toggle the visibility of connection-specific fields based on the selected mode."""+ try:+ if field_name == "tool":+ try:+ # Always refresh tools when cache is disabled, or when tools list is empty+ # This ensures database edits are reflected immediately when cache is disabled+ use_cache = getattr(self, "use_cache", False)+ if len(self.tools) == 0 or not use_cache:+ try:+ (+ self.tools,+ build_config["mcp_server"]["value"],+ ) = await self.update_tool_list()+ build_config["tool"]["options"] = [tool.name for tool in self.tools]+ build_config["tool"]["placeholder"] = "Select a tool"+ except (TimeoutError, asyncio.TimeoutError) as e:+ msg = f"Timeout updating tool list: {e!s}"+ await logger.aexception(msg)+ if not build_config["tools_metadata"]["show"]:+ build_config["tool"]["show"] = True+ build_config["tool"]["options"] = []+ build_config["tool"]["value"] = ""+ build_config["tool"]["placeholder"] = "Timeout on MCP server"+ else:+ build_config["tool"]["show"] = False+ except ValueError:+ if not build_config["tools_metadata"]["show"]:+ build_config["tool"]["show"] = True+ build_config["tool"]["options"] = []+ build_config["tool"]["value"] = ""+ build_config["tool"]["placeholder"] = "Error on MCP Server"+ else:+ build_config["tool"]["show"] = False++ if field_value == "":+ return build_config+ tool_obj = None+ for tool in self.tools:+ if tool.name == field_value:+ tool_obj = tool+ break+ if tool_obj is None:+ msg = f"Tool {field_value} not found in available tools: {self.tools}"+ await logger.awarning(msg)+ return build_config+ await self._update_tool_config(build_config, field_value)+ except Exception as e:+ build_config["tool"]["options"] = []+ msg = f"Failed to update tools: {e!s}"+ raise ValueError(msg) from e+ else:+ return build_config+ elif field_name == "mcp_server":+ if not field_value:+ build_config["tool"]["show"] = False+ build_config["tool"]["options"] = []+ build_config["tool"]["value"] = ""+ build_config["tool"]["placeholder"] = ""+ build_config["tool_placeholder"]["tool_mode"] = False+ self.remove_non_default_keys(build_config)+ return build_config++ build_config["tool_placeholder"]["tool_mode"] = True++ current_server_name = (+ field_value.get("name") if isinstance(field_value, dict) else field_value+ )+ _last_selected_server = safe_cache_get(+ self._shared_component_cache, "last_selected_server", ""+ )+ # Only treat as a server change if there was a previous server selection.+ # Cold cache (_last_selected_server="") on initial flow load is NOT a server change —+ # the user didn't switch anything, the backend just hasn't seen this component yet.+ server_changed = bool(+ _last_selected_server and current_server_name != _last_selected_server+ )++ # Determine if "Tool Mode" is active by checking if the tool dropdown is hidden.+ is_in_tool_mode = build_config["tools_metadata"]["show"]++ # Get use_cache setting to determine if we should use cached data+ use_cache = getattr(self, "use_cache", False)++ # Fast path: if server didn't change and we already have options, keep them as-is+ # BUT only if caching is enabled, we're in tool mode, or it's the initial load+ existing_options = build_config.get("tool", {}).get("options") or []+ if not server_changed and existing_options:+ # In non-tool mode with cache disabled, skip the fast path to force refresh+ # BUT on initial load (cold cache), always preserve saved options from the flow+ if not is_in_tool_mode and not use_cache and _last_selected_server:+ pass # Continue to refresh logic below (user-initiated with cache disabled)+ else:+ if not is_in_tool_mode:+ build_config["tool"]["show"] = True+ safe_cache_set(+ self._shared_component_cache,+ "last_selected_server",+ current_server_name,+ )+ return build_config++ # To avoid unnecessary updates, only proceed if the server has actually changed+ # OR if caching is disabled (to force refresh in non-tool mode)+ if (+ (_last_selected_server in (current_server_name, ""))+ and build_config["tool"]["show"]+ and use_cache+ ):+ if current_server_name:+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ if isinstance(servers_cache, dict):+ cached = servers_cache.get(current_server_name)+ if cached is not None and cached.get("tool_names"):+ cached_tools = cached["tool_names"]+ current_tools = build_config["tool"]["options"]+ if current_tools == cached_tools:+ return build_config+ else:+ return build_config+ safe_cache_set(+ self._shared_component_cache, "last_selected_server", current_server_name+ )++ # When cache is disabled, clear any cached data for this server+ # This ensures we always fetch fresh data from the database+ if not use_cache and current_server_name:+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ if isinstance(servers_cache, dict) and current_server_name in servers_cache:+ servers_cache.pop(current_server_name)+ safe_cache_set(self._shared_component_cache, "servers", servers_cache)++ # Check if tools are already cached for this server before clearing+ cached_tools = None+ if current_server_name and use_cache:+ servers_cache = safe_cache_get(self._shared_component_cache, "servers", {})+ if isinstance(servers_cache, dict):+ cached = servers_cache.get(current_server_name)+ if cached is not None:+ try:+ cached_tools = cached["tools"]+ self.tools = cached_tools+ self.tool_names = cached["tool_names"]+ self._tool_cache = cached["tool_cache"]+ except (TypeError, KeyError, AttributeError) as e:+ # Handle corrupted cache data by ignoring it+ msg = f"Unable to use cached data for MCP Server,{current_server_name}: {e}"+ await logger.awarning(msg)+ cached_tools = None++ # Clear tools when cache is disabled OR when we don't have cached tools+ # This ensures fresh tools are fetched after database edits+ if not cached_tools or not use_cache:+ self.tools = [] # Clear previous tools to force refresh++ # Clear previous tool inputs if:+ # 1. Server actually changed+ # 2. Cache is disabled (meaning tool list will be refreshed)+ if server_changed or not use_cache:+ self.remove_non_default_keys(build_config)++ # Only show the tool dropdown if not in tool_mode+ if not is_in_tool_mode:+ build_config["tool"]["show"] = True+ if cached_tools:+ # Use cached tools to populate options immediately+ build_config["tool"]["options"] = [tool.name for tool in cached_tools]+ build_config["tool"]["placeholder"] = "Select a tool"+ else:+ # Actually fetch tools now instead of deferring to a frontend callback.+ # The frontend has no reliable mechanism to trigger a second+ # update_build_config call for the "tool" field after this response,+ # so we must populate the options here.+ try:+ (+ self.tools,+ build_config["mcp_server"]["value"],+ ) = await self.update_tool_list(mcp_server_value=field_value)+ build_config["tool"]["options"] = [tool.name for tool in self.tools]+ build_config["tool"]["placeholder"] = "Select a tool"+ except (TimeoutError, asyncio.TimeoutError) as e:+ msg = f"Timeout loading tools for MCP server: {e!s}"+ await logger.awarning(msg)+ build_config["tool"]["options"] = []+ build_config["tool"]["placeholder"] = "Timeout on MCP server"+ except (+ ValueError,+ ImportError,+ ConnectionError,+ OSError,+ RuntimeError,+ ) as e:+ msg = f"Error loading tools for MCP server: {e!s}"+ await logger.awarning(msg)+ build_config["tool"]["options"] = []+ build_config["tool"]["placeholder"] = "Error on MCP Server"+ # Force a value refresh only when the user genuinely switched servers.+ # server_changed is only True for real user-initiated changes (not initial load).+ if server_changed:+ build_config["tool"]["value"] = uuid.uuid4()+ else:+ # Keep the tool dropdown hidden if in tool_mode+ self._not_load_actions = True+ build_config["tool"]["show"] = False++ elif field_name == "tool_mode":+ build_config["tool"]["placeholder"] = ""+ build_config["tool"]["show"] = not bool(field_value) and bool(+ build_config["mcp_server"]+ )+ self.remove_non_default_keys(build_config)+ self.tool = build_config["tool"]["value"]+ if field_value:+ self._not_load_actions = True+ else:+ build_config["tool"]["value"] = uuid.uuid4()+ build_config["tool"]["show"] = True+ # Fetch tools immediately instead of showing "Loading tools..."+ try:+ (+ self.tools,+ build_config["mcp_server"]["value"],+ ) = await self.update_tool_list()+ build_config["tool"]["options"] = [tool.name for tool in self.tools]+ build_config["tool"]["placeholder"] = "Select a tool"+ except (TimeoutError, asyncio.TimeoutError) as e:+ msg = f"Timeout loading tools when toggling tool mode: {e!s}"+ await logger.awarning(msg)+ build_config["tool"]["options"] = []+ build_config["tool"]["placeholder"] = "Timeout on MCP server"+ except (ValueError, ImportError, ConnectionError, OSError, RuntimeError) as e:+ msg = f"Error loading tools when toggling tool mode: {e!s}"+ await logger.awarning(msg)+ build_config["tool"]["options"] = []+ build_config["tool"]["placeholder"] = "Error on MCP Server"+ elif field_name == "tools_metadata":+ self._not_load_actions = False++ except Exception as e:+ msg = f"Error in update_build_config: {e!s}"+ await logger.aexception(msg)+ raise ValueError(msg) from e+ else:+ return build_config++ def get_inputs_for_all_tools(self, tools: list) -> dict:+ """Get input schemas for all tools."""+ inputs = {}+ for tool in tools:+ if not tool or not hasattr(tool, "name"):+ continue+ try:+ flat_schema = flatten_schema(tool.args_schema.schema())+ input_schema = create_input_schema_from_json_schema(flat_schema)+ langflow_inputs = schema_to_langflow_inputs(input_schema)+ inputs[tool.name] = langflow_inputs+ except (AttributeError, ValueError, TypeError, KeyError) as e:+ msg = f"Error getting inputs for tool {getattr(tool, 'name', 'unknown')}: {e!s}"+ logger.exception(msg)+ continue+ return inputs++ def remove_non_default_keys(self, build_config: dict) -> None:+ """Remove non-default keys from the build config."""+ for key in list(build_config.keys()):+ if key not in self.default_keys:+ build_config.pop(key)++ async def _update_tool_config(self, build_config: dict, tool_name: str) -> None:+ """Update tool configuration with proper error handling."""+ if not self.tools:+ self.tools, build_config["mcp_server"]["value"] = await self.update_tool_list()++ if not tool_name:+ return++ tool_obj = next((tool for tool in self.tools if tool.name == tool_name), None)+ if not tool_obj:+ msg = f"Tool {tool_name} not found in available tools: {self.tools}"+ self.remove_non_default_keys(build_config)+ build_config["tool"]["value"] = ""+ await logger.awarning(msg)+ return++ try:+ # Store current values before removing inputs (only for the current tool)+ current_values = {}+ for key, value in build_config.items():+ if key not in self.default_keys and isinstance(value, dict) and "value" in value:+ current_values[key] = value["value"]++ # Remove ALL non-default keys (all previous tool inputs)+ self.remove_non_default_keys(build_config)++ # Get and validate new inputs for the selected tool+ self.schema_inputs = await self._validate_schema_inputs(tool_obj)+ if not self.schema_inputs:+ msg = f"No input parameters to configure for tool '{tool_name}'"+ await logger.ainfo(msg)+ return++ # Add new inputs to build config for the selected tool only+ for schema_input in self.schema_inputs:+ if not schema_input or not hasattr(schema_input, "name"):+ msg = "Invalid schema input detected, skipping"+ await logger.awarning(msg)+ continue++ try:+ name = schema_input.name+ input_dict = schema_input.to_dict()+ input_dict.setdefault("value", None)+ input_dict.setdefault("required", True)++ build_config[name] = input_dict++ # Preserve existing value if the parameter name exists in current_values+ if name in current_values:+ build_config[name]["value"] = current_values[name]++ except (AttributeError, KeyError, TypeError) as e:+ msg = f"Error processing schema input {schema_input}: {e!s}"+ await logger.aexception(msg)+ continue+ except ValueError as e:+ msg = f"Schema validation error for tool {tool_name}: {e!s}"+ await logger.aexception(msg)+ self.schema_inputs = []+ return+ except (AttributeError, KeyError, TypeError) as e:+ msg = f"Error updating tool config: {e!s}"+ await logger.aexception(msg)+ raise ValueError(msg) from e++ async def build_output(self) -> DataFrame:+ """Build output with improved error handling and validation."""+ try:+ self.tools, _ = await self.update_tool_list()+ if self.tool != "":+ # Set session context for persistent MCP sessions using Langflow session ID+ session_context = self._get_session_context()+ if session_context:+ self.stdio_client.set_session_context(session_context)+ self.streamable_http_client.set_session_context(session_context)+ exec_tool = self._tool_cache[self.tool]+ tool_args = self.get_inputs_for_all_tools(self.tools)[self.tool]+ kwargs = {}+ for arg in tool_args:+ value = getattr(self, arg.name, None)+ if value is not None:+ if isinstance(value, Message):+ kwargs[arg.name] = value.text+ else:+ kwargs[arg.name] = value++ unflattened_kwargs = maybe_unflatten_dict(kwargs)++ output = await exec_tool.coroutine(**unflattened_kwargs)+ tool_content = []+ for item in output.content:+ item_dict = item.model_dump()+ item_dict = self.process_output_item(item_dict)+ tool_content.append(item_dict)++ if isinstance(tool_content, list) and all(+ isinstance(x, dict) for x in tool_content+ ):+ return DataFrame(tool_content)+ return DataFrame(data=tool_content)+ return DataFrame(data=[{"error": "You must select a tool"}])+ except Exception as e:+ msg = f"Error in build_output: {e!s}"+ await logger.aexception(msg)+ raise ValueError(msg) from e++ def process_output_item(self, item_dict):+ """Process the output of a tool."""+ if item_dict.get("type") == "text":+ text = item_dict.get("text")+ try:+ parsed = json.loads(text)+ # Ensure we always return a dictionary for DataFrame compatibility+ if isinstance(parsed, dict):+ return parsed+ # Wrap non-dict parsed values in a dictionary+ return {"text": text, "parsed_value": parsed, "type": "text"} # noqa: TRY300+ except json.JSONDecodeError:+ return item_dict+ return item_dict++ def _get_session_context(self) -> str | None:+ """Get the Langflow session ID for MCP session caching."""+ # Try to get session ID from the component's execution context+ if hasattr(self, "graph") and hasattr(self.graph, "session_id"):+ session_id = self.graph.session_id+ # Include server name to ensure different servers get different sessions+ server_name = ""+ mcp_server = getattr(self, "mcp_server", None)+ if isinstance(mcp_server, dict):+ server_name = mcp_server.get("name", "")+ elif mcp_server:+ server_name = str(mcp_server)+ return f"{session_id}_{server_name}" if session_id else None+ return None++ async def _get_tools(self):+ """Get cached tools or update if necessary."""+ mcp_server = getattr(self, "mcp_server", None)+ if not self._not_load_actions:+ tools, _ = await self.update_tool_list(mcp_server)+ return tools+ return []diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex 0ddb6b82c..79e309e64 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -828,7 +828,7 @@],"frozen": false,"icon": "braces",- "last_updated": "2026-05-05T13:33:37.431Z",+ "last_updated": "2026-05-08T19:20:38.334Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {},@@ -878,7 +878,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "48713154-dcea-4b42-b772-268029c455d7"+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"},"_type": "Component","code": {@@ -1336,9 +1336,10 @@],"frozen": false,"icon": "Docling",+ "last_updated": "2026-05-08T19:20:27.016Z","legacy": false,"metadata": {- "code_hash": "72fcef05e057",+ "code_hash": "3a18033f9e4f","dependencies": {"dependencies": [{@@ -1386,30 +1387,67 @@],"pinned": false,"template": {+ "_frontend_node_flow_id": {+ "value": "5488df7c-b93f-4f87-a446-b67028bc0813"+ },+ "_frontend_node_folder_id": {+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"+ },"_type": "Component","api_headers": {- "_input_type": "NestedDictInput",+ "_input_type": "TableInput","advanced": true,"display_name": "HTTP headers","dynamic": false,- "info": "Optional dictionary of additional headers required for connecting to Docling Serve.",+ "info": "Optional headers required for connecting to Docling Serve.","input_types": [- "Message"+ "Data",+ "JSON"],- "list": false,+ "is_list": true,"list_add_label": "Add More",+ "load_from_db": false,"name": "api_headers","override_skip": false,"placeholder": "",+ "real_time_refresh": true,"required": false,"show": true,+ "table_icon": "Table",+ "table_schema": [+ {+ "description": "Key name",+ "display_name": "Key",+ "formatter": "text",+ "name": "key",+ "type": "string"+ },+ {+ "description": "Value of the header",+ "display_name": "Value",+ "formatter": "text",+ "load_from_db": true,+ "name": "value",+ "type": "string"+ }+ ],"title_case": false,"tool_mode": false,- "trace_as_input": true,"trace_as_metadata": true,"track_in_telemetry": false,- "type": "NestedDict",- "value": {}+ "trigger_icon": "Table",+ "trigger_text": "Open table",+ "type": "table",+ "value": [+ {+ "key": "Authorization",+ "value": "JWT"+ },+ {+ "key": "X-Tenant-Id",+ "value": "OWNER"+ }+ ]},"api_url": {"_input_type": "StrInput",@@ -1448,7 +1486,7 @@"show": true,"title_case": false,"type": "code",- "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self.api_headers) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"+ "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self._process_headers()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"},"delete_server_file_after_processing": {"_input_type": "BoolInput",@@ -1569,6 +1607,7 @@"type": "bool","value": true},+ "is_refresh": false,"max_concurrency": {"_input_type": "IntInput","advanced": true,@@ -1745,14 +1784,14 @@"dragging": false,"id": "DoclingRemote-Dp3PX","measured": {- "height": 397,- "width": 320+ "height": 52,+ "width": 192},"position": {- "x": -158.35302959149863,- "y": 1766.5193585649456+ "x": 4.296782621507347,+ "y": 1764.5830512766954},- "selected": true,+ "selected": false,"type": "genericNode"},{@@ -4641,7 +4680,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "48713154-dcea-4b42-b772-268029c455d7"+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"},"_type": "Component","api_base": {@@ -6483,7 +6522,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "48713154-dcea-4b42-b772-268029c455d7"+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"},"_type": "Component","api_base": {@@ -7101,7 +7140,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "48713154-dcea-4b42-b772-268029c455d7"+ "value": "a451a847-ad7a-442f-ab14-569b43d392bd"},"_type": "Component","api_base": {@@ -7643,9 +7682,9 @@}],"viewport": {- "x": 267.6557208461858,- "y": -373.3174538705754,- "zoom": 0.475706358533402+ "x": 193.22171349647556,+ "y": -404.7149184337984,+ "zoom": 0.5164469534707714}},"description": "Load your data for chat context with Retrieval Augmented Generation.",diff --git a/pyproject.toml b/pyproject.tomlindex 1a473ad2a..e82264c17 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -91,11 +91,15 @@ override-dependencies = ["python-dotenv>=1.1.0"][tool.ruff]line-length = 100target-version = "py313"+exclude = ["flows/components"][tool.ruff.lint]select = ["E", "F", "I", "B", "UP"]ignore = ["E501"] # line-length handled by formatter+[tool.ruff.lint.flake8-bugbear]+extend-immutable-calls = ["fastapi.Depends", "fastapi.params.Depends", "Depends"]+# `bootstrap` must be the first import in any module that uses it# (it loads .env and configures structlog before anything else can# read env vars or emit logs). Define it as its own isort section@@ -119,5 +123,6 @@ python_version = "3.13"ignore_missing_imports = trueno_strict_optional = truewarn_unused_ignores = true+exclude = ["flows/components"]diff --git a/sdks/mcp/src/openrag_mcp/config.py b/sdks/mcp/src/openrag_mcp/config.pyindex 856ea763b..2ef0d88d6 100644--- a/sdks/mcp/src/openrag_mcp/config.py+++ b/sdks/mcp/src/openrag_mcp/config.py@@ -44,13 +44,13 @@ class Config:self.openrag_url = os.environ.get("OPENRAG_URL", "http://localhost:3000")self.api_key = os.environ.get("OPENRAG_API_KEY")- # IBM auth mode: user provides IBM_USERNAME + IBM_API_KEY instead of OPENRAG_API_KEY.+ # Platform auth mode: user provides PLATFORM_USERNAME + IBM_API_KEY instead of OPENRAG_API_KEY.# The MCP forwards these as headers on every SDK request to OpenRAG.- ibm_username = os.environ.get("IBM_USERNAME")+ platform_username = os.environ.get("PLATFORM_USERNAME")ibm_api_key = os.environ.get("IBM_API_KEY")self.ibm_extra_headers: dict[str, str] = {}- if ibm_username:- self.ibm_extra_headers["X-Username"] = ibm_username+ if platform_username:+ self.ibm_extra_headers["X-Username"] = platform_usernameif ibm_api_key:self.ibm_extra_headers["X-Api-Key"] = ibm_api_key@@ -63,9 +63,7 @@ class Config:# MCP httpx client configuration (OPENRAG_MCP_*)self.mcp_timeout = _parse_float("OPENRAG_MCP_TIMEOUT", 60.0)self.mcp_max_connections = _parse_int("OPENRAG_MCP_MAX_CONNECTIONS", 100)- self.mcp_max_keepalive_connections = _parse_int(- "OPENRAG_MCP_MAX_KEEPALIVE_CONNECTIONS", 20- )+ self.mcp_max_keepalive_connections = _parse_int("OPENRAG_MCP_MAX_KEEPALIVE_CONNECTIONS", 20)self.mcp_max_retries = _parse_int("OPENRAG_MCP_MAX_RETRIES", 3)self.mcp_follow_redirects = _parse_bool("OPENRAG_MCP_FOLLOW_REDIRECTS", True)diff --git a/src/api/docling.py b/src/api/docling.pyindex 25d6114a8..70ddf54a5 100644--- a/src/api/docling.py+++ b/src/api/docling.py@@ -1,15 +1,17 @@-"""Docling service proxy endpoints."""--import os-import socket-import struct-from pathlib import Path+from typing import Annotatedimport httpx-from fastapi import Request+from fastapi import Depends, Requestfrom fastapi.responses import JSONResponse-from config.settings import DOCLING_SERVE_URL, DOCLING_HOST_IP+from config.settings import (+ DOCLING_HOST_IP,+ DOCLING_SERVE_URL,+ DOCLING_SERVE_VERIFY_SSL,+ IBM_AUTH_ENABLED,+)+from dependencies import get_optional_user+from session_manager import Userfrom utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -20,43 +22,53 @@ DOCLING_SERVICE_URL = DOCLING_SERVE_URLHOST_IP = DOCLING_HOST_IP-async def health(request: Request) -> JSONResponse:+async def health(+ request: Request, user: Annotated[User | None, Depends(get_optional_user)] = None+) -> JSONResponse:"""Proxy health check to docling-serve.This allows the frontend to check docling status via same-origin request."""health_url = f"{DOCLING_SERVICE_URL}/health"+ headers = {}+ if IBM_AUTH_ENABLED and user:+ if user.jwt_token:+ headers["Authorization"] = user.jwt_token+ if user.user_id:+ headers["X-Tenant-Id"] = user.user_id+try:- async with httpx.AsyncClient() as client:- response = await client.get(- health_url,- timeout=2.0- )+ async with httpx.AsyncClient(verify=DOCLING_SERVE_VERIFY_SSL) as client:+ response = await client.get(health_url, headers=headers, timeout=2.0)if response.status_code == 200:- return JSONResponse({- "status": "healthy",- "host": HOST_IP- })+ return JSONResponse({"status": "healthy", "host": HOST_IP})else:- logger.warning("Docling health check failed", url=health_url, status_code=response.status_code)- return JSONResponse({- "status": "unhealthy",- "message": f"Health check failed with status: {response.status_code}",- "host": HOST_IP- }, status_code=503)+ logger.warning(+ "Docling health check failed", url=health_url, status_code=response.status_code+ )+ return JSONResponse(+ {+ "status": "unhealthy",+ "message": f"Health check failed with status: {response.status_code}",+ "host": HOST_IP,+ },+ status_code=503,+ )except httpx.TimeoutException:logger.warning("Docling health check timeout", url=health_url)- return JSONResponse({- "status": "unhealthy",- "message": "Connection timeout",- "host": HOST_IP- }, status_code=503)+ return JSONResponse(+ {"status": "unhealthy", "message": "Connection timeout", "host": HOST_IP},+ status_code=503,+ )except Exception as e:logger.error("Docling health check failed", url=health_url, error=str(e))- return JSONResponse({- "status": "unhealthy",- "message": str(e),- "host": HOST_IP- }, status_code=503)+ return JSONResponse(+ {+ "status": "unhealthy",+ "message": "Internal error while checking service health",+ "host": HOST_IP,+ },+ status_code=503,+ )diff --git a/src/api/upload.py b/src/api/upload.pyindex b36b9b0d4..e2712c8a0 100644--- a/src/api/upload.py+++ b/src/api/upload.py@@ -1,20 +1,20 @@-from dependencies import get_docling_service-from dependencies import get_models_serviceimport os-from typing import Optional+from typing import Annotated, Anyfrom urllib.parse import urlparseimport boto3from fastapi import Depends, File, Form, UploadFile-from pydantic import BaseModelfrom fastapi.responses import JSONResponse+from pydantic import BaseModelfrom dependencies import (- get_document_service,- get_task_service,get_chat_service,- get_session_manager,get_current_user,+ get_docling_service,+ get_document_service,+ get_models_service,+ get_session_manager,+ get_task_service,require_all_permissions,require_permission,)@@ -33,15 +33,15 @@ class UploadBucketBody(BaseModel):async def upload(- file: UploadFile = File(...),- document_service=Depends(get_document_service),- session_manager=Depends(get_session_manager),- user: User = Depends(require_permission("knowledge:upload")),+ file: Annotated[UploadFile, File(...)],+ document_service: Annotated[Any, Depends(get_document_service)],+ session_manager: Annotated[Any, Depends(get_session_manager)],+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],):"""Upload a single file"""try:-from config.settings import is_no_auth_mode+is_no_auth = is_no_auth_mode()owner_user_id = user.user_id if (user and not is_no_auth) else Noneowner_name = user.name if user else None@@ -57,10 +57,7 @@ async def upload(return JSONResponse(result, status_code=201)except Exception as e:error_msg = str(e)- if (- "AuthenticationException" in error_msg- or "access denied" in error_msg.lower()- ):+ if "AuthenticationException" in error_msg or "access denied" in error_msg.lower():logger.warning("[INGEST] Upload rejected — access denied", error=error_msg)return JSONResponse({"error": error_msg}, status_code=403)else:@@ -70,17 +67,15 @@ async def upload(async def upload_path(body: UploadPathBody,- task_service=Depends(get_task_service),- session_manager=Depends(get_session_manager),- user: User = Depends(require_permission("knowledge:upload")),+ task_service: Annotated[Any, Depends(get_task_service)],+ session_manager: Annotated[Any, Depends(get_session_manager)],+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],):"""Upload all files from a directory path"""if not body.path or not os.path.isdir(body.path):return JSONResponse({"error": "Invalid path"}, status_code=400)- file_paths = [- os.path.join(root, fn) for root, _, files in os.walk(body.path) for fn in files- ]+ file_paths = [os.path.join(root, fn) for root, _, files in os.walk(body.path) for fn in files]if not file_paths:return JSONResponse({"error": "No files found in directory"}, status_code=400)@@ -88,12 +83,14 @@ async def upload_path(jwt_token = user.jwt_tokenfrom config.settings import is_no_auth_mode+is_no_auth = is_no_auth_mode()owner_user_id = user.user_id if (user and not is_no_auth) else Noneowner_name = user.name if user else Noneowner_email = user.email if user else Nonefrom api.documents import _ensure_index_exists+await _ensure_index_exists(jwt_token)task_id = await task_service.create_upload_task(@@ -111,30 +108,32 @@ async def upload_path(async def upload_context(- file: UploadFile = File(...),- previous_response_id: Optional[str] = Form(None),- endpoint: str = Form("langflow"),- document_service=Depends(get_document_service),- chat_service=Depends(get_chat_service),- session_manager=Depends(get_session_manager),- user: User = Depends(require_all_permissions(("knowledge:upload", "chat:use"))),+ file: Annotated[UploadFile, File(...)],+ document_service: Annotated[Any, Depends(get_document_service)],+ chat_service: Annotated[Any, Depends(get_chat_service)],+ session_manager: Annotated[Any, Depends(get_session_manager)],+ user: Annotated[User, Depends(require_all_permissions(("knowledge:upload", "chat:use")))],+ previous_response_id: Annotated[str | None, Form()] = None,+ endpoint: Annotated[str, Form()] = "langflow",):"""Upload a file and add its content as context to the current conversation"""filename = file.filename or "uploaded_document"user_id = user.user_id if user else None- storage_user_id = (- (getattr(user, "db_user_id", None) or user.user_id) if user else None- )+ storage_user_id = (getattr(user, "db_user_id", None) or user.user_id) if user else Noneif previous_response_id and storage_user_id:from api.chat import _assert_owns+await _assert_owns(previous_response_id, storage_user_id)jwt_token = user.jwt_token- doc_result = await document_service.process_upload_context(file, filename)+ doc_result = await document_service.process_upload_context(+ file, filename, user_id=user_id, jwt_token=jwt_token+ )from config.settings import is_no_auth_mode+is_no_auth = is_no_auth_mode()owner_user_id = user.user_id if (user and not is_no_auth) else Noneowner_name = user.name if user else None@@ -166,29 +165,26 @@ async def upload_context(async def upload_options(- user: User = Depends(get_current_user),+ user: Annotated[User, Depends(get_current_user)],):"""Return availability of upload features"""- aws_enabled = bool(- os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY")- )+ aws_enabled = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"))from config.settings import UPLOAD_BATCH_SIZE+return JSONResponse({"aws": aws_enabled, "upload_batch_size": UPLOAD_BATCH_SIZE})async def upload_bucket(body: UploadBucketBody,- task_service=Depends(get_task_service),- models_service=Depends(get_models_service),- docling_service=Depends(get_docling_service),- session_manager=Depends(get_session_manager),- user: User = Depends(require_permission("knowledge:upload")),+ task_service: Annotated[Any, Depends(get_task_service)],+ models_service: Annotated[Any, Depends(get_models_service)],+ docling_service: Annotated[Any, Depends(get_docling_service)],+ session_manager: Annotated[Any, Depends(get_session_manager)],+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],):"""Process all files from an S3 bucket URL"""if not os.getenv("AWS_ACCESS_KEY_ID") or not os.getenv("AWS_SECRET_ACCESS_KEY"):- return JSONResponse(- {"error": "AWS credentials not configured"}, status_code=400- )+ return JSONResponse({"error": "AWS credentials not configured"}, status_code=400)if not body.s3_url or not body.s3_url.startswith("s3://"):return JSONResponse({"error": "Invalid S3 URL"}, status_code=400)@@ -211,9 +207,9 @@ async def upload_bucket(jwt_token = user.jwt_token+ from config.settings import is_no_auth_modefrom models.processors import S3FileProcessor- from config.settings import is_no_auth_modeis_no_auth = is_no_auth_mode()owner_user_id = user.user_id if (user and not is_no_auth) else Noneowner_name = user.name if user else None@@ -221,6 +217,7 @@ async def upload_bucket(task_user_id = user.user_id if (user and not is_no_auth) else Nonefrom api.documents import _ensure_index_exists+await _ensure_index_exists(jwt_token)processor = S3FileProcessor(diff --git a/src/config/settings.py b/src/config/settings.pyindex abdf51e6a..a4bd232a6 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -61,12 +61,24 @@ GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET")# IBM AMS authentication (Watsonx Data embedded mode)IBM_AUTH_ENABLED = os.getenv("IBM_AUTH_ENABLED", "false").lower() in ("true", "1", "yes")+PLATFORM_USERNAME = os.getenv("PLATFORM_USERNAME")+PLATFORM_PASSWORD = os.getenv("PLATFORM_PASSWORD")IBM_JWT_PUBLIC_KEY_URL = os.getenv("IBM_JWT_PUBLIC_KEY_URL", "")IBM_SESSION_COOKIE_NAME = os.getenv("IBM_SESSION_COOKIE_NAME", "ibm-openrag-session")IBM_CREDENTIALS_HEADER = os.getenv("IBM_CREDENTIALS_HEADER", "X-IBM-LH-Credentials")DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY", "")ENVIRONMENT = os.getenv("ENVIRONMENT", "")+PLATFORM_AUTH_DEV_MODE = os.getenv("PLATFORM_AUTH_DEV_MODE", "false").lower() in (+ "true",+ "1",+ "yes",+)+DOCLING_SERVE_VERIFY_SSL = os.getenv("DOCLING_SERVE_VERIFY_SSL", "true").lower() in (+ "true",+ "1",+ "yes",+)# Enable FastAPI's `debug` mode (verbose tracebacks in HTTP error responses# on the FastAPI app instance). Named explicitly so it isn't confused with@@ -119,19 +131,21 @@ else:logger.info("Auto-detected Docling host: %s (URL: %s)", DOCLING_HOST_IP, DOCLING_SERVE_URL)# Ingestion configuration-DISABLE_INGEST_WITH_LANGFLOW = os.getenv(- "DISABLE_INGEST_WITH_LANGFLOW", "false"-).lower() in ("true", "1", "yes")+DISABLE_INGEST_WITH_LANGFLOW = os.getenv("DISABLE_INGEST_WITH_LANGFLOW", "false").lower() in (+ "true",+ "1",+ "yes",+)# Show the "+" file upload button in the chat input-OPENRAG_INGEST_VIA_CHAT = os.getenv(- "OPENRAG_INGEST_VIA_CHAT", "false"-).lower() in ("true", "1", "yes")+OPENRAG_INGEST_VIA_CHAT = os.getenv("OPENRAG_INGEST_VIA_CHAT", "false").lower() in (+ "true",+ "1",+ "yes",+)# Ingest sample data configuration-INGEST_SAMPLE_DATA = os.getenv(- "INGEST_SAMPLE_DATA", "true"-).lower() in ("true", "1", "yes")+INGEST_SAMPLE_DATA = os.getenv("INGEST_SAMPLE_DATA", "true").lower() in ("true", "1", "yes")# Default OpenRAG docs sample ingestion source# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow@@ -139,13 +153,15 @@ INGEST_SAMPLE_DATA = os.getenv(DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag/")-#TODO: Enable this when the flow is updated to use the new variables+# TODO: Enable this when the flow is updated to use the new variablesDEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)-FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(- "FETCH_OPENRAG_DOCS_AT_STARTUP", "false"-).lower() in ("true", "1", "yes")+FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv("FETCH_OPENRAG_DOCS_AT_STARTUP", "false").lower() in (+ "true",+ "1",+ "yes",+)# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folderUPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)@@ -161,18 +177,17 @@ LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30# Default: 3600 seconds (60 minutes)INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)+def is_no_auth_mode():"""Check if we're running in no-auth mode (OAuth credentials missing)"""if IBM_AUTH_ENABLED:- return False # IBM cookie auth is a valid auth mode+ return False # IBM cookie auth is a valid auth mode (variable name kept for now as per instructions)result = not (GOOGLE_OAUTH_CLIENT_ID and GOOGLE_OAUTH_CLIENT_SECRET)return result# Webhook configuration - must be set to enable webhooks-WEBHOOK_BASE_URL = os.getenv(- "WEBHOOK_BASE_URL"-) # No default - must be explicitly configured+WEBHOOK_BASE_URL = os.getenv("WEBHOOK_BASE_URL") # No default - must be explicitly configured# OAuth callback broker URL -- when set, Google (and other providers) redirect# here instead of directly to the frontend. The broker then forwards to the@@ -393,23 +408,29 @@ class AppClients:# The property will handle lazy initialization with probe when first accessedopenai_key = os.getenv("OPENAI_API_KEY")if openai_key:- logger.info("OpenAI API key found in environment - will be initialized lazily on first use with HTTP/2 probe")+ logger.info(+ "OpenAI API key found in environment - will be initialized lazily on first use with HTTP/2 probe"+ )else:- logger.info("OpenAI API key not found in environment - will be initialized on first use if needed")+ logger.info(+ "OpenAI API key not found in environment - will be initialized on first use if needed"+ )# Initialize docling-serve HTTP client for document conversionself.docling_http_client = httpx.AsyncClient(+ verify=DOCLING_SERVE_VERIFY_SSL,timeout=httpx.Timeout(timeout=INGESTION_TIMEOUT,connect=30.0,read=INGESTION_TIMEOUT,write=30.0,pool=30.0,- )+ ),)# Eagerly initialize DoclingService to ensure thread-safetyfrom services.docling_service import DoclingService+self._docling_service = DoclingService(httpx_client=self.docling_http_client)# Initialize Langflow HTTP client with extended timeouts for large documents@@ -423,7 +444,7 @@ class AppClients:read=LANGFLOW_TIMEOUT, # Read timeout (most important for large PDFs)write=LANGFLOW_CONNECT_TIMEOUT, # Write timeoutpool=LANGFLOW_CONNECT_TIMEOUT, # Pool timeout- )+ ),)logger.info("Initialized Langflow HTTP client with extended timeouts",@@ -433,6 +454,7 @@ class AppClients:# Wait for Langflow to be healthy before generating API keyfrom utils.langflow_utils import wait_for_langflow+await wait_for_langflow(langflow_http_client=self.langflow_http_client)# Generate Langflow API key now that Langflow is confirmed ready@@ -454,9 +476,7 @@ class AppClients:logger.warning("Failed to initialize Langflow client", error=str(e))self.langflow_client = Noneif self.langflow_client is None:- logger.warning(- "No Langflow client initialized yet, will attempt later on first use"- )+ logger.warning("No Langflow client initialized yet, will attempt later on first use")return self@@ -473,9 +493,7 @@ class AppClients:)logger.info("Langflow client initialized on-demand")except Exception as e:- logger.error(- "Failed to initialize Langflow client on-demand", error=str(e)- )+ logger.error("Failed to initialize Langflow client on-demand", error=str(e))self.langflow_client = Nonereturn self.langflow_client@@ -511,7 +529,7 @@ class AppClients:os.environ["OPENAI_API_KEY"] = config.providers.openai.api_keylogger.debug("Loaded OpenAI API key from config")elif not os.environ.get("OPENAI_API_KEY"):- # Provide dummy key to satisfy AsyncOpenAI constructor;+ # Provide dummy key to satisfy AsyncOpenAI constructor;# LiteLLM/MCP will handle routing to other providers if needed.os.environ["OPENAI_API_KEY"] = "no-key-required"logger.debug("Using dummy OpenAI API key to satisfy client constructor")@@ -526,7 +544,9 @@ class AppClients:os.environ["WATSONX_API_KEY"] = config.providers.watsonx.api_keyif config.providers.watsonx.endpoint:os.environ["WATSONX_ENDPOINT"] = config.providers.watsonx.endpoint- os.environ["WATSONX_API_BASE"] = config.providers.watsonx.endpoint # LiteLLM expects this name+ os.environ["WATSONX_API_BASE"] = (+ config.providers.watsonx.endpoint+ ) # LiteLLM expects this nameif config.providers.watsonx.project_id:os.environ["WATSONX_PROJECT_ID"] = config.providers.watsonx.project_idif config.providers.watsonx.api_key:@@ -552,7 +572,6 @@ class AppClients:os.environ["OPENAI_API_KEY"] = "no-key-required"logger.debug("Using dummy OpenAI API key fallback (config load failed)")-# API key for AsyncOpenAI constructorapi_key = os.environ.get("OPENAI_API_KEY")@@ -569,16 +588,15 @@ class AppClients:logger.info(f"Probing client with HTTP/2 using model {model_name}...")try:await asyncio.wait_for(- client.embeddings.create(- model=model_name,- input=['test']- ),- timeout=5.0+ client.embeddings.create(model=model_name, input=["test"]), timeout=5.0)logger.info(f"HTTP/2 probe successful with {model_name}")return Trueexcept (TimeoutError, Exception) as probe_error:- logger.warning(f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1", error=str(probe_error))+ logger.warning(+ f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1",+ error=str(probe_error),+ )return Falsefinally:# Always close the probe client so its connections are fully@@ -614,20 +632,27 @@ class AppClients:if use_http2:self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI(api_key=api_key))- logger.info(f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})")+ logger.info(+ f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})"+ )else:http_client = httpx.AsyncClient(- http2=False,- timeout=httpx.Timeout(60.0, connect=10.0)+ http2=False, timeout=httpx.Timeout(60.0, connect=10.0))self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI(api_key=api_key, http_client=http_client))- logger.info(f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})")+ logger.info(+ f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})"+ )logger.info("Successfully initialized OpenAI client")except Exception as e:- logger.error(f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}")- raise ValueError(f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable.") from e+ logger.error(+ f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}"+ )+ raise ValueError(+ f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable."+ ) from ereturn self._patched_async_client@@ -666,6 +691,7 @@ class AppClients:return self._docling_servicefrom services.docling_service import DoclingService+self._docling_service = DoclingService(httpx_client=self.docling_http_client)return self._docling_service@@ -765,9 +791,7 @@ class AppClients:return response- async def _create_langflow_global_variable(- self, name: str, value: str, modify: bool = False- ):+ async def _create_langflow_global_variable(self, name: str, value: str, modify: bool = False):"""Create a global variable in Langflow via API"""payload = {"name": name,@@ -777,9 +801,7 @@ class AppClients:}try:- response = await self.langflow_request(- "POST", "/api/v1/variables/", json=payload- )+ response = await self.langflow_request("POST", "/api/v1/variables/", json=payload)if response.status_code in [200, 201]:logger.info(@@ -916,18 +938,15 @@ def _component_path(env_var: str, filename: str) -> str:flows_dir = get_flows_path()return os.path.join(flows_dir, "components", filename)-WATSONX_LLM_COMPONENT_PATH = _component_path(- "WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json"-)++WATSONX_LLM_COMPONENT_PATH = _component_path("WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json")WATSONX_LLM_TEXT_COMPONENT_PATH = _component_path("WATSONX_LLM_TEXT_COMPONENT_PATH", "watsonx_llm_text.json")WATSONX_EMBEDDING_COMPONENT_PATH = _component_path("WATSONX_EMBEDDING_COMPONENT_PATH", "watsonx_embedding.json")-OLLAMA_LLM_COMPONENT_PATH = _component_path(- "OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json"-)+OLLAMA_LLM_COMPONENT_PATH = _component_path("OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json")OLLAMA_LLM_TEXT_COMPONENT_PATH = _component_path("OLLAMA_LLM_TEXT_COMPONENT_PATH", "ollama_llm_text.json")@@ -940,13 +959,9 @@ OLLAMA_EMBEDDING_COMPONENT_PATH = _component_path(OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME = os.getenv("OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME", "Embedding Model")-OPENAI_LLM_COMPONENT_DISPLAY_NAME = os.getenv(- "OPENAI_LLM_COMPONENT_DISPLAY_NAME", "Language Model"-)+OPENAI_LLM_COMPONENT_DISPLAY_NAME = os.getenv("OPENAI_LLM_COMPONENT_DISPLAY_NAME", "Language Model")-AGENT_COMPONENT_DISPLAY_NAME = os.getenv(- "AGENT_COMPONENT_DISPLAY_NAME", "Agent"-)+AGENT_COMPONENT_DISPLAY_NAME = os.getenv("AGENT_COMPONENT_DISPLAY_NAME", "Agent")# Provider-specific component IDsWATSONX_EMBEDDING_COMPONENT_DISPLAY_NAME = os.getenv(@@ -994,7 +1009,9 @@ def get_agent_config():def get_embedding_model() -> str:"""Return the currently configured embedding model."""- return get_openrag_config().knowledge.embedding_model or (OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else "")+ return get_openrag_config().knowledge.embedding_model or (+ OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else ""+ )def get_index_name() -> str:diff --git a/src/dependencies.py b/src/dependencies.pyindex e7f943fd3..864a51bf6 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -17,7 +17,8 @@ Usage:import asyncioimport dataclasses-from typing import AsyncIterator, Optional, Sequence+from collections.abc import AsyncIterator, Sequence+from typing import Optionalfrom cachetools import TTLCachefrom fastapi import Depends, HTTPException, Request@@ -130,6 +131,8 @@ def get_flows_service(services: dict = Depends(get_services)):def get_docling_service(services: dict = Depends(get_services)):return services["docling_service"]++def get_rbac_service(services: dict = Depends(get_services)):return services["rbac_service"]@@ -150,12 +153,13 @@ async def get_db_session() -> AsyncIterator[AsyncSession]:if SessionLocal is None:init_engine()from db.engine import SessionLocal as _SessionLocal+assert _SessionLocal is not Noneasync with _SessionLocal() as session:yield session-async def _ensure_db_user(user: User) -> Optional[str]:+async def _ensure_db_user(user: User) -> str | None:"""Best-effort DB upsert for the authenticated user. Returns the SQL`users.id` for this user (so callers can cache the OAuth-sub → DB-idmapping). Returns None on failure.@@ -189,6 +193,7 @@ async def _ensure_db_user(user: User) -> Optional[str]:if SessionLocal is None:init_engine()from db.engine import SessionLocal as _SessionLocal+if _SessionLocal is None:return Noneasync with _SessionLocal() as session:@@ -216,7 +221,7 @@ async def _resolve_db_user_id(user: User) -> str:return resolved or user.user_id-async def _attach_db_user_id(request: Request, user: Optional[User]) -> Optional[User]:+async def _attach_db_user_id(request: Request, user: User | None) -> User | None:"""Attach the internal SQL users.id to the request user.`User.user_id` remains the external auth subject used in JWT/OpenSearch@@ -235,8 +240,8 @@ async def _attach_db_user_id(request: Request, user: Optional[User]) -> Optionaldef invalidate_user_ensured_cache(- oauth_provider: Optional[str] = None,- oauth_subject: Optional[str] = None,+ oauth_provider: str | None = None,+ oauth_subject: str | None = None,) -> None:"""Pop the ensure-cache + lock for a single identity, or clear allif neither argument is provided.@@ -353,7 +358,33 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:"""import auth.ibm_auth as ibm_authfrom auth.ibm_auth import extract_ibm_credentials- from config.settings import IBM_SESSION_COOKIE_NAME, IBM_CREDENTIALS_HEADER+ from config.settings import (+ IBM_CREDENTIALS_HEADER,+ IBM_SESSION_COOKIE_NAME,+ PLATFORM_PASSWORD,+ PLATFORM_USERNAME,+ )++ # ── Option -1: Environment variable override (local dev/calls) ───────++ if PLATFORM_USERNAME and PLATFORM_PASSWORD:+ import base64++ logger.debug("[AUTH] Using PLATFORM_USERNAME and PLATFORM_PASSWORD from environment")+ creds = f"{PLATFORM_USERNAME}:{PLATFORM_PASSWORD}"+ lh_credentials = base64.b64encode(creds.encode()).decode()+ user = User(+ user_id=PLATFORM_USERNAME,+ email=PLATFORM_USERNAME,+ name=PLATFORM_USERNAME,+ picture=None,+ provider="ibm_ams_env",+ jwt_token=f"Basic {lh_credentials}",+ opensearch_username=PLATFORM_USERNAME,+ opensearch_credentials=lh_credentials,+ )+ request.state.user = user+ return user# ── Option 0: Configurable credentials header (Traefik production) ───@@ -369,13 +400,14 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:logger.debug("[AUTH] IBM JWT claims decoded successfully")sub = claims.get("sub")if not sub:- logger.warning("IBM JWT is missing required 'sub' claim; treating as unauthenticated")+ logger.warning(+ "IBM JWT is missing required 'sub' claim; treating as unauthenticated"+ )else:user_id = claims.get("username", sub)email = claims.get("username", sub)name = claims.get("display_name", claims.get("username", sub))-if lh_credentials and lh_credentials.strip() != "":logger.debug("[AUTH] IBM LH credentials found in request headers")opensearch_username, _ = extract_ibm_credentials(lh_credentials)@@ -410,9 +442,7 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:# ibm_token = request.cookies.get(IBM_SESSION_COOKIE_NAME)if ibm_token and user_id:logger.debug("[AUTH] IBM JWT cookie present and user_id found")- logger.debug(- "[AUTH] LH credentials not available in header, reading from connections.json"- )+ logger.debug("[AUTH] LH credentials not available in header, reading from connections.json")# lh credentials not available in header, read from connections serviceconnector_service = request.app.state.services.get("connector_service")if connector_service:@@ -457,9 +487,7 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:# ── Option 2: ibm-auth-basic cookie (local dev, no Traefik) ─────────auth_header = request.cookies.get("ibm-auth-basic", "")if auth_header.startswith("Basic "):- logger.debug(- "[AUTH] Debug mode enabled, extracting IBM LH credentials from cookie"- )+ logger.debug("[AUTH] Debug mode enabled, extracting IBM LH credentials from cookie")username, _ = extract_ibm_credentials(auth_header)logger.debug("[AUTH] IBM LH credentials extracted successfully")user = User(@@ -533,7 +561,7 @@ async def get_current_user(async def get_optional_user(request: Request,session_manager=Depends(get_session_manager),-) -> Optional[User]:+) -> User | None:"""Optionally extract JWT cookie user.@@ -568,13 +596,9 @@ async def get_optional_user(user = session_manager.get_user_from_token(auth_token)# get_effective_jwt_token handles anonymous JWT creation if neededeffective_token = (- session_manager.get_effective_jwt_token(user.user_id, auth_token)- if user- else None- )- user_with_token = (- dataclasses.replace(user, jwt_token=effective_token) if user else None+ session_manager.get_effective_jwt_token(user.user_id, auth_token) if user else None)+ user_with_token = dataclasses.replace(user, jwt_token=effective_token) if user else Noneif user_with_token:return await _attach_db_user_id(request, user_with_token)@@ -664,8 +688,6 @@ async def get_api_key_user_async(# Phase 2 will populate api_key_role_ids from the SQL api_keys table.# In Phase 1 we leave it unset so require_permission falls back to the# user's live role membership (no privilege escalation possible).- request.state.api_key_role_ids = getattr(- request.state, "api_key_role_ids", None- )+ request.state.api_key_role_ids = getattr(request.state, "api_key_role_ids", None)return await _attach_db_user_id(request, user_with_token)diff --git a/src/models/processors.py b/src/models/processors.pyindex 62d3be1e7..51cb655a7 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -1,14 +1,19 @@-from typing import Any, Dict, Optional-from .tasks import UploadTask, FileTask-from utils.logging_config import get_logger+from typing import TYPE_CHECKING, Any+from utils.file_utils import (- get_file_extension,clean_connector_filename,+ get_file_extension,get_filename_aliases,)+from utils.logging_config import get_logger++from .tasks import FileTask, UploadTasklogger = get_logger(__name__)+if TYPE_CHECKING:+ from connectors.base import DocumentACL+class TaskProcessor:"""Base class for task processors with shared processing logic"""@@ -27,9 +32,10 @@ class TaskProcessor:Check if a document with the given hash already exists in OpenSearch.Consolidated hash checking for all processors."""- from config.settings import get_index_nameimport asyncio+ from config.settings import get_index_name+max_retries = 3retry_delay = 1.0@@ -37,18 +43,18 @@ class TaskProcessor:try:exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)return exists- except (asyncio.TimeoutError, Exception) as e:+ except (TimeoutError, Exception) as e:if attempt == max_retries - 1:logger.error("OpenSearch exists check failed after retries",file_hash=file_hash,error=str(e),- attempt=attempt + 1+ attempt=attempt + 1,)# On final failure, assume document doesn't exist (safer to reprocess than skip)logger.warning("Assuming document doesn't exist due to connection issues",- file_hash=file_hash+ file_hash=file_hash,)return Falseelse:@@ -57,10 +63,11 @@ class TaskProcessor:file_hash=file_hash,error=str(e),attempt=attempt + 1,- retry_in=retry_delay+ retry_in=retry_delay,)await asyncio.sleep(retry_delay)retry_delay *= 2 # Exponential backoff+ return Falseasync def check_filename_exists(self,@@ -71,9 +78,10 @@ class TaskProcessor:Check if a document with the given filename already exists in OpenSearch.Returns True if any chunks with this filename exist."""+ import asyncio+from config.settings import get_index_namefrom utils.opensearch_queries import build_filename_search_body- import asynciomax_retries = 3retry_delay = 1.0@@ -94,12 +102,9 @@ class TaskProcessor:i = 0while i < len(pending_candidates):candidate = pending_candidates[i]- search_body = build_filename_search_body(- candidate, size=1, source=False- )+ search_body = build_filename_search_body(candidate, size=1, source=False)response = await opensearch_client.search(- index=get_index_name(),- body=search_body+ index=get_index_name(), body=search_body)hits = response.get("hits", {}).get("hits", [])if hits:@@ -110,18 +115,18 @@ class TaskProcessor:continuereturn False- except (asyncio.TimeoutError, Exception) as e:+ except (TimeoutError, Exception) as e:if attempt == max_retries - 1:logger.error("OpenSearch filename check failed after retries",filename=filename,error=str(e),- attempt=attempt + 1+ attempt=attempt + 1,)# On final failure, assume document doesn't exist (safer to reprocess than skip)logger.warning("Assuming filename doesn't exist due to connection issues",- filename=filename+ filename=filename,)return Falseelse:@@ -130,10 +135,11 @@ class TaskProcessor:filename=filename,error=str(e),attempt=attempt + 1,- retry_in=retry_delay+ retry_in=retry_delay,)await asyncio.sleep(retry_delay)retry_delay *= 2 # Exponential backoff+ return Falseasync def delete_document_by_filename(self,@@ -158,22 +164,15 @@ class TaskProcessor:for candidate in candidate_filenames:delete_body = build_filename_delete_body(candidate)response = await opensearch_client.delete_by_query(- index=get_index_name(),- body=delete_body+ index=get_index_name(), body=delete_body)deleted_count += response.get("deleted", 0)logger.info(- "Deleted existing document chunks",- filename=filename,- deleted_count=deleted_count+ "Deleted existing document chunks", filename=filename, deleted_count=deleted_count)except Exception as e:- logger.error(- "Failed to delete existing document",- filename=filename,- error=str(e)- )+ logger.error("Failed to delete existing document", filename=filename, error=str(e))raiseasync def process_document_standard(@@ -206,6 +205,7 @@ class TaskProcessor:acl: DocumentACL instance with access control information"""import datetime+from config.settings import (clients,get_embedding_model,@@ -223,11 +223,7 @@ class TaskProcessor:# get_embedding_model() returns empty string when Langflow ingest is enabled,# but OpenRAG processors still need a concrete embedding model.configured_embedding_model = get_openrag_config().knowledge.embedding_model- embedding_model = (- embedding_model- or configured_embedding_model- or get_embedding_model()- )+ embedding_model = embedding_model or configured_embedding_model or get_embedding_model()# Get user's OpenSearch client with JWT for OIDC authopensearch_client = self.document_service.session_manager.get_user_opensearch_client(@@ -237,7 +233,6 @@ class TaskProcessor:# Check if already existsif await self.check_document_exists(file_hash, opensearch_client):return {"status": "unchanged", "id": file_hash}-logger.info("Processing document with embedding model",@@ -247,11 +242,13 @@ class TaskProcessor:# Check if this is a .txt or .md file - use simple processing instead of doclingimport os+file_ext = os.path.splitext(file_path)[1].lower()-- if file_ext in ('.txt', '.md'):++ if file_ext in (".txt", ".md"):# Simple text file processing without doclingfrom utils.document_processing import process_text_file+logger.info("Processing as plain text file (bypassing docling)",file_path=file_path,@@ -262,7 +259,9 @@ class TaskProcessor:if original_filename:slim_doc["filename"] = original_filenameelse:- full_doc = await self.docling_service.convert_file(file_path)+ full_doc = await self.docling_service.convert_file(+ file_path, user_id=owner_user_id, auth_header=jwt_token+ )slim_doc = extract_relevant(full_doc)if chunk_size is not None:@@ -272,11 +271,7 @@ class TaskProcessor:cs = 0if cs > 0:try:- co = (- int(chunk_overlap)- if chunk_overlap is not None- else 0- )+ co = int(chunk_overlap) if chunk_overlap is not None else 0except (TypeError, ValueError):co = 0if co < cs:@@ -286,7 +281,11 @@ class TaskProcessor:texts = [c["text"] for c in slim_doc["chunks"]]- litellm_embedding_model = await self.models_service.get_litellm_model_name(embedding_model) if self.models_service is not None else embedding_model+ litellm_embedding_model = (+ await self.models_service.get_litellm_model_name(embedding_model)+ if self.models_service is not None+ else embedding_model+ )# Split into batches to avoid token limits (8191 limit, use 8000 with buffer or 2000 if it's ollama)if "ollama" in litellm_embedding_model:@@ -299,7 +298,9 @@ class TaskProcessor:resp = await clients.patched_embedding_client.embeddings.create(model=litellm_embedding_model, input=batch)- embeddings.extend([d["embedding"] if isinstance(d, dict) else d.embedding for d in resp.data])+ embeddings.extend(+ [d["embedding"] if isinstance(d, dict) else d.embedding for d in resp.data]+ )if not embeddings or len(embeddings) == 0:logger.error(@@ -317,12 +318,10 @@ class TaskProcessor:)# Index each chunk as a separate document- for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings)):+ for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True)):chunk_doc = {"document_id": file_hash,- "filename": original_filename- if original_filename- else slim_doc["filename"],+ "filename": original_filename if original_filename else slim_doc["filename"],"mimetype": slim_doc["mimetype"],"page": chunk["page"],"text": chunk["text"],@@ -359,9 +358,7 @@ class TaskProcessor:chunk_doc["is_sample_data"] = "true"chunk_id = f"{file_hash}_{i}"try:- await opensearch_client.index(- index=get_index_name(), id=chunk_id, body=chunk_doc- )+ await opensearch_client.index(index=get_index_name(), id=chunk_id, body=chunk_doc)except Exception as e:logger.error("OpenSearch indexing failed for chunk",@@ -372,9 +369,7 @@ class TaskProcessor:raisereturn {"status": "indexed", "id": file_hash}- async def process_item(- self, upload_task: UploadTask, item: Any, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:"""Process a single item in the task.@@ -421,14 +416,13 @@ class DocumentFileProcessor(TaskProcessor):self.is_sample_data = is_sample_dataself.connector_type = connector_type- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a regular file path using consolidated methods"""+ import os+ import time+from models.tasks import TaskStatusfrom utils.hash_utils import hash_id- import time- import osfile_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -487,14 +481,12 @@ class ConnectorFileProcessor(TaskProcessor):owner_email: str = None,document_service=None,models_service=None,- ingest_settings: Optional[Dict[str, Any]] = None,+ ingest_settings: dict[str, Any] | None = None,):super().__init__(document_service=document_service,models_service=models_service,- docling_service=document_service.docling_service- if document_service- else None,+ docling_service=document_service.docling_service if document_service else None,)self.connector_service = connector_serviceself.connection_id = connection_id@@ -505,15 +497,12 @@ class ConnectorFileProcessor(TaskProcessor):self.owner_email = owner_emailself.ingest_settings = ingest_settings- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using consolidated methods"""+ import time+from models.tasks import TaskStatusfrom utils.hash_utils import hash_id- import tempfile- import time- import osfile_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -531,7 +520,7 @@ class ConnectorFileProcessor(TaskProcessor):# Get file content from connectordocument = await connector.get_file_content(file_id)-+# Update filename in task once we have it from the connectorfile_task.filename = clean_connector_filename(document.filename, document.mimetype)@@ -544,13 +533,13 @@ class ConnectorFileProcessor(TaskProcessor):suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp file- with open(tmp_path, 'wb') as f:+ with open(tmp_path, "wb") as f:f.write(document.content)# Compute hashfile_hash = hash_id(tmp_path)- standard_kwargs: Dict[str, Any] = {}+ standard_kwargs: dict[str, Any] = {}if isinstance(self.ingest_settings, dict):s = self.ingest_settingsem = s.get("embeddingModel")@@ -583,10 +572,12 @@ class ConnectorFileProcessor(TaskProcessor):)# Add connector-specific metadata- result.update({- "source_url": document.source_url,- "document_id": document.id,- })+ result.update(+ {+ "source_url": document.source_url,+ "document_id": document.id,+ }+ )file_task.status = TaskStatus.COMPLETEDfile_task.result = result@@ -613,7 +604,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):jwt_token: str = None,owner_name: str = None,owner_email: str = None,- ingest_settings: Optional[Dict[str, Any]] = None,+ ingest_settings: dict[str, Any] | None = None,):super().__init__(document_service=langflow_connector_service.task_service.document_service@@ -633,15 +624,12 @@ class LangflowConnectorFileProcessor(TaskProcessor):self.owner_email = owner_emailself.ingest_settings = ingest_settings- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using LangflowConnectorService"""+ import time+from models.tasks import TaskStatusfrom utils.hash_utils import hash_id- import tempfile- import time- import osfile_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -650,14 +638,10 @@ class LangflowConnectorFileProcessor(TaskProcessor):file_id = item # item is the connector file ID# Get the connector and connection info- connector = await self.langflow_connector_service.get_connector(+ connector = await self.langflow_connector_service.get_connector(self.connection_id)+ connection = await self.langflow_connector_service.connection_manager.get_connection(self.connection_id)- connection = (- await self.langflow_connector_service.connection_manager.get_connection(- self.connection_id- )- )if not connector or not connection:raise ValueError(f"Connection '{self.connection_id}' not found")@@ -676,15 +660,17 @@ class LangflowConnectorFileProcessor(TaskProcessor):suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp file- with open(tmp_path, 'wb') as f:+ with open(tmp_path, "wb") as f:f.write(document.content)# Compute hash and check if already existsfile_hash = hash_id(tmp_path)# Check if document already exists- opensearch_client = self.langflow_connector_service.session_manager.get_user_opensearch_client(- self.user_id, self.jwt_token+ opensearch_client = (+ self.langflow_connector_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token+ ))if await self.check_document_exists(file_hash, opensearch_client):file_task.status = TaskStatus.COMPLETED@@ -746,17 +732,12 @@ class S3FileProcessor(TaskProcessor):self.owner_name = owner_nameself.owner_email = owner_email- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Download an S3 object and process it using DocumentService"""- from models.tasks import TaskStatus- import tempfile- import osimport time- import asyncio- import datetime- from config.settings import clients, get_embedding_model, get_index_name++ from models.tasks import TaskStatus+file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -766,7 +747,7 @@ class S3FileProcessor(TaskProcessor):try:with auto_cleanup_tempfile() as tmp_path:# Download object to temporary file- with open(tmp_path, 'wb') as tmp_file:+ with open(tmp_path, "wb") as tmp_file:self.s3_client.download_fileobj(self.bucket, item, tmp_file)# Compute hash@@ -835,15 +816,14 @@ class LangflowFileProcessor(TaskProcessor):self.replace_duplicates = replace_duplicatesself.connector_type = connector_type- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a file path using LangflowFileService upload_and_ingest_file"""import mimetypesimport os- from models.tasks import TaskStatusimport time+ from models.tasks import TaskStatus+# Update task statusfile_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -873,21 +853,21 @@ class LangflowFileProcessor(TaskProcessor):await self.delete_document_by_filename(original_filename, opensearch_client)# Read file content for processing- with open(item, 'rb') as f:+ with open(item, "rb") as f:content = f.read()# Create file tuple for upload using ORIGINAL filename# This ensures the document is indexed with the original namecontent_type, _ = mimetypes.guess_type(original_filename)if not content_type:- content_type = 'application/octet-stream'+ content_type = "application/octet-stream"# Rename .txt to .md for Langflow compatibility# Langflow has issues processing text/plain fileslangflow_filename = original_filename- if original_filename.lower().endswith('.txt'):- langflow_filename = original_filename[:-4] + '.md'- content_type = 'text/markdown'+ if original_filename.lower().endswith(".txt"):+ langflow_filename = original_filename[:-4] + ".md"+ content_type = "text/markdown"logger.debug(f"Renamed {original_filename} to {langflow_filename} for Langflow")file_tuple = (langflow_filename, content, content_type)@@ -897,12 +877,10 @@ class LangflowFileProcessor(TaskProcessor):effective_jwt = self.jwt_tokenif self.session_manager and not effective_jwt:# Let session manager handle anonymous JWT creation if needed- self.session_manager.get_user_opensearch_client(- self.owner_user_id, self.jwt_token- )+ self.session_manager.get_user_opensearch_client(self.owner_user_id, self.jwt_token)# The session manager would have created anonymous JWT if needed# Get it from the session manager's internal state- if hasattr(self.session_manager, '_anonymous_jwt'):+ if hasattr(self.session_manager, "_anonymous_jwt"):effective_jwt = self.session_manager._anonymous_jwt# Prepare metadata tweaks similar to API endpoint@@ -930,10 +908,7 @@ class LangflowFileProcessor(TaskProcessor):except Exception as e:# Update task with failurefile_task.status = TaskStatus.FAILED- file_task.error_message = str(e)+ file_task.error = str(e)file_task.updated_at = time.time()upload_task.failed_files += 1raise---from .url import LangflowUrlProcessordiff --git a/src/services/auth_service.py b/src/services/auth_service.pyindex 50ffdda75..6d0676ed7 100644--- a/src/services/auth_service.py+++ b/src/services/auth_service.py@@ -1,24 +1,23 @@-import uuidimport json-import httpx-import logging+import os+import uuidfrom datetime import datetime, timedelta-from typing import Optional-import os-from config.settings import OAUTH_BROKER_URL, WEBHOOK_BASE_URL, is_no_auth_mode+import httpxfrom fastapi import HTTPException-logger = logging.getLogger(__name__)-from session_manager import SessionManager-from services.langflow_mcp_service import LangflowMCPService-from connectors.google_drive.oauth import GoogleDriveOAuth-from connectors.onedrive.oauth import OneDriveOAuth-from connectors.sharepoint.oauth import SharePointOAuth+from config.settings import OAUTH_BROKER_URL, WEBHOOK_BASE_URL, is_no_auth_modefrom connectors.google_drive import GoogleDriveConnector+from connectors.google_drive.oauth import GoogleDriveOAuthfrom connectors.onedrive import OneDriveConnector+from connectors.onedrive.oauth import OneDriveOAuthfrom connectors.sharepoint import SharePointConnector+from connectors.sharepoint.oauth import SharePointOAuth+from services.langflow_mcp_service import LangflowMCPService+from session_manager import SessionManager++logger = logging.getLogger(__name__)# Connectors that authenticate directly (no OAuth redirect required)_DIRECT_AUTH_CONNECTORS = {"ibm_cos"}@@ -34,11 +33,10 @@ class AuthService:):self.session_manager = session_managerself.connector_service = connector_service- self.used_auth_codes = set() # Track used authorization codes+ self.used_auth_codes: set[str] = set() # Track used authorization codesself.flows_service = flows_serviceself.langflow_mcp_service = langflow_mcp_service-async def init_oauth(self,connector_type: str,@@ -88,6 +86,7 @@ class AuthService:# Create connection configuration - use data directory for persistencefrom config.paths import get_data_file+token_file = get_data_file(f"{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json")effective_redirect_uri = OAUTH_BROKER_URL or redirect_uriconfig = {@@ -99,18 +98,14 @@ class AuthService:# Only add webhook URL if WEBHOOK_BASE_URL is configuredif WEBHOOK_BASE_URL:- config["webhook_url"] = (- f"{WEBHOOK_BASE_URL}/connectors/{connector_type}/webhook"- )+ config["webhook_url"] = f"{WEBHOOK_BASE_URL}/connectors/{connector_type}/webhook"# Create connection in manager- connection_id = (- await self.connector_service.connection_manager.create_connection(- connector_type=connector_type,- name=connection_name,- config=config,- user_id=user_id,- )+ connection_id = await self.connector_service.connection_manager.create_connection(+ connector_type=connector_type,+ name=connection_name,+ config=config,+ user_id=user_id,)# Direct-auth connectors (HMAC/API-key based, no OAuth redirect)@@ -126,22 +121,26 @@ class AuthService:"sharepoint": (SharePointConnector, SharePointOAuth),}- connector_class, oauth_class = connector_class_map.get(- connector_type, (None, None)- )+ connector_class, oauth_class = connector_class_map.get(connector_type, (None, None))if not connector_class or not oauth_class:raise ValueError(f"No classes found for connector type: {connector_type}")+ # Cast to Any to satisfy mypy for class attribute access+ from typing import Any++ oauth_class_any: Any = oauth_class+ connector_class_any: Any = connector_class+# Get scopes from OAuth class- scopes = oauth_class.SCOPES+ scopes = oauth_class_any.SCOPES# Get endpoints from OAuth class- auth_endpoint = oauth_class.AUTH_ENDPOINT- token_endpoint = oauth_class.TOKEN_ENDPOINT+ auth_endpoint = oauth_class_any.AUTH_ENDPOINT+ token_endpoint = oauth_class_any.TOKEN_ENDPOINT# src/services/auth_service.py- client_key = getattr(connector_class, "CLIENT_ID_ENV_VAR", None)- secret_key = getattr(connector_class, "CLIENT_SECRET_ENV_VAR", None)+ client_key = getattr(connector_class_any, "CLIENT_ID_ENV_VAR", None)+ secret_key = getattr(connector_class_any, "CLIENT_SECRET_ENV_VAR", None)def _assert_env_key(name, val):if not isinstance(val, str) or not val.strip():@@ -173,9 +172,7 @@ class AuthService:},}- async def _init_direct_connection(- self, connector_type: str, connection_id: str- ) -> dict:+ async def _init_direct_connection(self, connector_type: str, connection_id: str) -> dict:"""Authenticate a non-OAuth connector immediately using env var credentials.Creates the connection record (already done by the caller) and verifies@@ -184,10 +181,8 @@ class AuthService:is needed."""try:- connection_config = (- await self.connector_service.connection_manager.get_connection(- connection_id- )+ connection_config = await self.connector_service.connection_manager.get_connection(+ connection_id)if not connection_config:raise ValueError("Connection not found")@@ -198,25 +193,19 @@ class AuthService:authenticated = await connector.authenticate()if not authenticated:# Remove the connection so the user can retry after fixing credentials- await self.connector_service.connection_manager.delete_connection(- connection_id- )+ await self.connector_service.connection_manager.delete_connection(connection_id)raise ValueError(f"Could not authenticate with {connector_type}. ""Check that your credentials and endpoint are correct.")# Cache the authenticated connector- self.connector_service.connection_manager.active_connectors[- connection_id- ] = connector+ self.connector_service.connection_manager.active_connectors[connection_id] = connectorexcept ValueError:raiseexcept Exception as exc:- await self.connector_service.connection_manager.delete_connection(- connection_id- )+ await self.connector_service.connection_manager.delete_connection(connection_id)raise ValueError(f"Failed to connect {connector_type}: {exc}") from excreturn {@@ -236,9 +225,7 @@ class AuthService:"""Handle OAuth callback - exchange authorization code for tokens"""logger.info(f"OAuth callback state: {state}")if not all([connection_id, authorization_code]):- raise ValueError(- "Missing required parameters (connection_id, authorization_code)"- )+ raise ValueError("Missing required parameters (connection_id, authorization_code)")# Check if authorization code has already been usedif authorization_code in self.used_auth_codes:@@ -249,10 +236,8 @@ class AuthService:try:# Get connection config- connection_config = (- await self.connector_service.connection_manager.get_connection(- connection_id- )+ connection_config = await self.connector_service.connection_manager.get_connection(+ connection_id)if not connection_config:raise ValueError("Connection not found")@@ -275,15 +260,14 @@ class AuthService:"sharepoint": (SharePointConnector, SharePointOAuth),}- connector_class, oauth_class = connector_class_map.get(- connector_type, (None, None)- )+ connector_class, oauth_class = connector_class_map.get(connector_type, (None, None))if not connector_class or not oauth_class:- raise ValueError(- f"No classes found for connector type: {connector_type}"- )+ raise ValueError(f"No classes found for connector type: {connector_type}")++ from typing import Any- token_url = oauth_class.TOKEN_ENDPOINT+ oauth_class_any: Any = oauth_class+ token_url = oauth_class_any.TOKEN_ENDPOINTtoken_payload = {"code": authorization_code,@@ -311,9 +295,7 @@ class AuthService:# OAuth providers typically return scopes as a space-separated stringscopes = (- granted_scopes.split(" ")- if isinstance(granted_scopes, str)- else granted_scopes+ granted_scopes.split(" ") if isinstance(granted_scopes, str) else granted_scopes)token_file_data = {@@ -324,9 +306,7 @@ class AuthService:# Add expiry if providedif token_data.get("expires_in"):- expiry = datetime.now() + timedelta(- seconds=int(token_data["expires_in"])- )+ expiry = datetime.now() + timedelta(seconds=int(token_data["expires_in"]))token_file_data["expiry"] = expiry.isoformat()# Save tokens to file@@ -343,9 +323,7 @@ class AuthService:connection_id, connection_config, token_data, request)else:- return await self._handle_data_source_auth(- connection_id, connection_config- )+ return await self._handle_data_source_auth(connection_id, connection_config)except Exception as e:# Remove used code from set if we failed@@ -376,9 +354,6 @@ class AuthService:token_data["access_token"])---response_data = {"status": "authenticated","purpose": "app_auth",@@ -398,9 +373,7 @@ class AuthService:"purpose": "data_source","user_email": user_info.get("email"),**(- {- "webhook_url": f"{WEBHOOK_BASE_URL}/connectors/google_drive/webhook"- }+ {"webhook_url": f"{WEBHOOK_BASE_URL}/connectors/google_drive/webhook"}if WEBHOOK_BASE_URLelse {}),@@ -409,21 +382,15 @@ class AuthService:response_data["google_drive_connection_id"] = connection_idelse:# Fallback: delete connection if we can't get user info- await self.connector_service.connection_manager.delete_connection(- connection_id- )+ await self.connector_service.connection_manager.delete_connection(connection_id)return response_dataelse:# Clean up connection if session creation failed- await self.connector_service.connection_manager.delete_connection(- connection_id- )+ await self.connector_service.connection_manager.delete_connection(connection_id)raise Exception("Failed to create user session")- async def _handle_data_source_auth(- self, connection_id: str, connection_config- ) -> dict:+ async def _handle_data_source_auth(self, connection_id: str, connection_config) -> dict:"""Handle data source connection - keep the connection for syncing"""result = {"status": "authenticated",@@ -442,10 +409,8 @@ class AuthService:logger.info(f"_handle_data_source_auth: Getting connector for connection_id: {connection_id}")- connector = (- await self.connector_service.connection_manager.get_connector(- connection_id- )+ connector = await self.connector_service.connection_manager.get_connector(+ connection_id)logger.info(f"_handle_data_source_auth: Got connector: {connector is not None}, has _detect_base_url: {hasattr(connector, '_detect_base_url') if connector else False}"@@ -466,47 +431,38 @@ class AuthService:logger.info(f"_handle_data_source_auth: Updated connector.base_url to: {connector.base_url}")- await (- self.connector_service.connection_manager.save_connections()- )+ await self.connector_service.connection_manager.save_connections()result["base_url"] = detected_urllogger.info(f"_handle_data_source_auth: Auto-detected and saved base URL: {detected_url}")else:- logger.warning(- "_handle_data_source_auth: _detect_base_url returned None"- )+ logger.warning("_handle_data_source_auth: _detect_base_url returned None")else:logger.warning(- f"_handle_data_source_auth: Connector not available or doesn't have _detect_base_url"+ "_handle_data_source_auth: Connector not available or doesn't have _detect_base_url")# Clear the cached connector so next get_connector() creates a fresh instance# with the updated config (including base_url)- if (- connection_id- in self.connector_service.connection_manager.active_connectors- ):+ if connection_id in self.connector_service.connection_manager.active_connectors:logger.info(f"_handle_data_source_auth: Clearing cached connector for {connection_id}")- del self.connector_service.connection_manager.active_connectors[- connection_id- ]- except Exception as e:+ del self.connector_service.connection_manager.active_connectors[connection_id]+ except Exception:logger.exception("[AUTH] Auto-detect base URL failed")return result- async def get_user_info(self, request) -> Optional[dict]:+ async def get_user_info(self, request) -> dict | None:"""Get current user information from request"""from config.settings import IBM_AUTH_ENABLED# IBM auth mode: user is set by get_optional_user from IBM cookieif IBM_AUTH_ENABLED:user = getattr(request.state, "user", None)- if user and user.provider in ("ibm_ams", "ibm_ams_basic"):+ if user and user.provider in ("ibm_ams", "ibm_ams_basic", "ibm_ams_env"):return {"authenticated": True,"ibm_auth_mode": True,@@ -516,9 +472,7 @@ class AuthService:"name": user.name,"picture": user.picture,"provider": user.provider,- "last_login": user.last_login.isoformat()- if user.last_login- else None,+ "last_login": user.last_login.isoformat() if user.last_login else None,},}return {"authenticated": False, "ibm_auth_mode": True, "user": None}@@ -539,9 +493,7 @@ class AuthService:"name": user.name,"picture": user.picture,"provider": user.provider,- "last_login": user.last_login.isoformat()- if user.last_login- else None,+ "last_login": user.last_login.isoformat() if user.last_login else None,},}diff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex b0befd650..8c971332f 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -1,30 +1,39 @@import asyncio-import platformimport json+import platformfrom pathlib import Path-from typing import Any, Optional, Dict+from typing import Any+import httpxfrom pydantic import BaseModel-from config.settings import get_openrag_config, DOCLING_SERVE_URL+from config.settings import (+ DOCLING_SERVE_URL,+ DOCLING_SERVE_VERIFY_SSL,+ IBM_AUTH_ENABLED,+ get_openrag_config,+)from utils.logging_config import get_loggerlogger = get_logger(__name__)+class DoclingConfig(BaseModel):do_ocr: boolocr_engine: strdo_table_structure: booldo_picture_classification: booldo_picture_description: bool- picture_description_local: Optional[dict] = None+ picture_description_local: dict | None = None+class DoclingServeError(Exception):"""Raised when docling-serve conversion fails."""+def get_docling_preset_configs(table_structure=False, ocr=False, picture_descriptions=False-) -> Dict[str, Any]:+) -> dict[str, Any]:"""Get docling preset configurations based on toggle settings"""is_macos = platform.system() == "Darwin"@@ -42,13 +51,16 @@ def get_docling_preset_configs(return config+class DoclingService:- _default_client: Optional[httpx.AsyncClient] = None+ _default_client: httpx.AsyncClient | None = None- def __init__(self, docling_url: Optional[str] = None, httpx_client: Optional[httpx.AsyncClient] = None):+ def __init__(+ self, docling_url: str | None = None, httpx_client: httpx.AsyncClient | None = None+ ):"""Initialize the DoclingService.-+Args:docling_url: Base URL of the Docling Serve instance. If None, auto-detects.httpx_client: Pre-configured httpx async client.@@ -57,7 +69,7 @@ class DoclingService:self.docling_url = docling_url.rstrip("/")else:self.docling_url = DOCLING_SERVE_URL-+self.httpx_client = httpx_clientdef _get_client(self) -> httpx.AsyncClient:@@ -65,65 +77,83 @@ class DoclingService:return self.httpx_clientif DoclingService._default_client is None or DoclingService._default_client.is_closed:DoclingService._default_client = httpx.AsyncClient(- timeout=httpx.Timeout(300.0, connect=10.0)+ timeout=httpx.Timeout(300.0, connect=10.0), verify=DOCLING_SERVE_VERIFY_SSL)return DoclingService._default_client- def _build_docling_options(self) -> Dict[str, Any]:+ def _build_docling_options(self) -> dict[str, Any]:"""Build the options payload for docling from OpenRAG configs."""config = get_openrag_config()knowledge_config = config.knowledge-+preset = get_docling_preset_configs(table_structure=knowledge_config.table_structure,ocr=knowledge_config.ocr,- picture_descriptions=knowledge_config.picture_descriptions+ picture_descriptions=knowledge_config.picture_descriptions,)-- options = {- "to_formats": "json",- "image_export_mode": "placeholder",- **preset- }++ options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}return options- async def upload_to_docling_direct_async(self, filename: str, file_content: bytes) -> str:+ def _get_auth_headers(+ self, user_id: str | None = None, auth_header: str | None = None+ ) -> dict[str, str]:+ """Build authentication headers for Docling Serve if IBM auth is enabled."""+ headers = {}+ if IBM_AUTH_ENABLED:+ if auth_header:+ headers["Authorization"] = auth_header++ if user_id:+ headers["X-Tenant-Id"] = user_id+ return headers++ async def upload_to_docling_direct_async(+ self,+ filename: str,+ file_content: bytes,+ user_id: str | None = None,+ auth_header: str | None = None,+ ) -> str:"""Upload a file to Docling Serve asynchronously using direct multipart/form-data upload."""options = self._build_docling_options()-+ headers = self._get_auth_headers(user_id, auth_header)+# Docling serve async multipart endpoint /v1/convert/file/async# Options are passed as form datadata = {k: str(v).lower() if isinstance(v, bool) else vfor k, v in options.items()if not isinstance(v, dict)- } # picture_description_local needs to be JSON if it's a dict+ } # picture_description_local needs to be JSON if it's a dictif "picture_description_local" in options:data["picture_description_local"] = json.dumps(options["picture_description_local"])files = {"files": (filename, file_content)}-+client = self._get_client()should_close = client != self.httpx_client-+try:if should_close:async with client:response = await client.post(f"{self.docling_url}/v1/convert/file/async",files=files,- data=data+ data=data,+ headers=headers,)else:response = await client.post(f"{self.docling_url}/v1/convert/file/async",files=files,- data=data+ data=data,+ headers=headers,)-+response.raise_for_status()task = response.json()return task["task_id"]@@ -135,48 +165,67 @@ class DoclingService:self,task_id: str,poll_interval: float = 1.0,- timeout: float = 600.0- ) -> Dict[str, Any]:+ timeout: float = 600.0,+ user_id: str | None = None,+ auth_header: str | None = None,+ ) -> dict[str, Any]:"""Poll Docling Serve for the result of an async conversion task."""client = self._get_client()should_close = client != self.httpx_client-+try:if should_close:async with client:- return await self._poll_result(client, task_id, poll_interval, timeout)+ return await self._poll_result(+ client, task_id, poll_interval, timeout, user_id, auth_header+ )else:- return await self._poll_result(client, task_id, poll_interval, timeout)+ return await self._poll_result(+ client, task_id, poll_interval, timeout, user_id, auth_header+ )except Exception as e:logger.error("Docling result retrieval failed", task_id=task_id, error=str(e))raise- async def _poll_result(self, client: httpx.AsyncClient, task_id: str, poll_interval: float, timeout: float) -> Dict[str, Any]:+ async def _poll_result(+ self,+ client: httpx.AsyncClient,+ task_id: str,+ poll_interval: float,+ timeout: float,+ user_id: str | None = None,+ auth_header: str | None = None,+ ) -> dict[str, Any]:"""Internal polling logic."""elapsed = 0.0+ headers = self._get_auth_headers(user_id, auth_header)while elapsed < timeout:try:- response = await client.get(f"{self.docling_url}/v1/status/poll/{task_id}")+ response = await client.get(+ f"{self.docling_url}/v1/status/poll/{task_id}", headers=headers+ )response.raise_for_status()status_data = response.json()except Exception as e:logger.error("Error polling docling status", task_id=task_id, error=str(e))- raise DoclingServeError(f"Error polling docling status: {str(e)}")+ raise DoclingServeError(f"Error polling docling status: {str(e)}") from estatus = status_data.get("task_status")if status == "success":- result_response = await client.get(f"{self.docling_url}/v1/result/{task_id}")+ result_response = await client.get(+ f"{self.docling_url}/v1/result/{task_id}", headers=headers+ )result_response.raise_for_status()result_json = result_response.json()-+# Extract the json_content which matches the old convert_file/bytes returndoc_content = result_json.get("document", {}).get("json_content")if doc_content is None:raise DoclingServeError("docling-serve response missing document.json_content")-+return doc_contentelif status == "failure":raise DoclingServeError(f"Docling conversion failed: {status_data}")@@ -186,18 +235,34 @@ class DoclingService:raise TimeoutError(f"Docling task {task_id} did not complete within {timeout} seconds")- async def convert_file(self, file_path: str) -> Dict[str, Any]:+ async def convert_file(+ self, file_path: str, user_id: str | None = None, auth_header: str | None = None+ ) -> dict[str, Any]:"""Convert a local file via docling-serve async polling."""path = Path(file_path)file_bytes = path.read_bytes()- task_id = await self.upload_to_docling_direct_async(path.name, file_bytes)- return await self.get_docling_result_async(task_id)+ task_id = await self.upload_to_docling_direct_async(+ path.name, file_bytes, user_id=user_id, auth_header=auth_header+ )+ return await self.get_docling_result_async(+ task_id, user_id=user_id, auth_header=auth_header+ )- async def convert_bytes(self, content: bytes, filename: str) -> Dict[str, Any]:+ async def convert_bytes(+ self,+ content: bytes,+ filename: str,+ user_id: str | None = None,+ auth_header: str | None = None,+ ) -> dict[str, Any]:"""Convert in-memory bytes via docling-serve async polling."""- task_id = await self.upload_to_docling_direct_async(filename, content)- return await self.get_docling_result_async(task_id)+ task_id = await self.upload_to_docling_direct_async(+ filename, content, user_id=user_id, auth_header=auth_header+ )+ return await self.get_docling_result_async(+ task_id, user_id=user_id, auth_header=auth_header+ )diff --git a/src/services/document_service.py b/src/services/document_service.pyindex 8b9672fb3..2b920dcbf 100644--- a/src/services/document_service.py+++ b/src/services/document_service.py@@ -1,13 +1,11 @@-from typing import Listimport tiktoken++from config.settings import get_embedding_model, get_index_name+from utils.document_processing import extract_relevantfrom utils.logging_config import get_loggerlogger = get_logger(__name__)-from config.settings import clients, get_embedding_model, get_index_name-from utils.document_processing import extract_relevant-from utils.telemetry import TelemetryClient, Category, MessageId-def get_token_count(text: str, model: str = None) -> int:"""Get accurate token count using tiktoken"""@@ -22,8 +20,8 @@ def get_token_count(text: str, model: str = None) -> int:def chunk_texts_for_embeddings(- texts: List[str], max_tokens: int = None, model: str = None-) -> List[List[str]]:+ texts: list[str], max_tokens: int = None, model: str = None+) -> list[list[str]]:"""Split texts into batches that won't exceed token limits.If max_tokens is None, returns texts as single batch (no splitting).@@ -40,7 +38,7 @@ def chunk_texts_for_embeddings(return [texts]batches = []- current_batch = []+ current_batch: list[str] = []current_tokens = 0for text in texts:@@ -93,7 +91,6 @@ class DocumentService:self.docling_service = docling_serviceself._mapping_ensured = False-async def process_upload_file(self,upload_file,@@ -103,12 +100,13 @@ class DocumentService:owner_email: str = None,):"""Process an uploaded file from form data"""- from utils.hash_utils import hash_id- from utils.file_utils import auto_cleanup_tempfileimport os# Default metadata for anonymous users if not providedfrom session_manager import AnonymousUser+ from utils.file_utils import auto_cleanup_tempfile+ from utils.hash_utils import hash_id+anonymous_user = AnonymousUser()owner_name = owner_name or anonymous_user.nameowner_email = owner_email or anonymous_user.email@@ -120,7 +118,7 @@ class DocumentService:with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Stream upload file to temporary filefile_size = 0- with open(tmp_path, 'wb') as tmp_file:+ with open(tmp_path, "wb") as tmp_file:while True:chunk = await upload_file.read(1 << 20)if not chunk:@@ -137,16 +135,19 @@ class DocumentService:try:exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)except Exception as e:- logger.error(- "OpenSearch exists check failed", file_hash=file_hash, error=str(e)- )+ logger.error("OpenSearch exists check failed", file_hash=file_hash, error=str(e))raiseif exists:return {"status": "unchanged", "id": file_hash}# Use consolidated standard processingfrom models.processors import TaskProcessor- processor = TaskProcessor(document_service=self, models_service=self.models_service, docling_service=self.docling_service)++ processor = TaskProcessor(+ document_service=self,+ models_service=self.models_service,+ docling_service=self.docling_service,+ )result = await processor.process_document_standard(file_path=tmp_path,file_hash=file_hash,@@ -160,7 +161,9 @@ class DocumentService:)return result- async def process_upload_context(self, upload_file, filename: str = None):+ async def process_upload_context(+ self, upload_file, filename: str = None, user_id: str = None, jwt_token: str = None+ ):"""Process uploaded file and return content for context"""import ioimport os@@ -179,11 +182,11 @@ class DocumentService:# Check if this is a .txt or .md file - use simple processingfile_ext = os.path.splitext(filename)[1].lower()-- if file_ext in ('.txt', '.md'):++ if file_ext in (".txt", ".md"):# Simple text file processing for chat context- text_content = content.read().decode('utf-8', errors='replace')-+ text_content = content.read().decode("utf-8", errors="replace")+# For context, we don't need to chunk - just return the full contentreturn {"filename": filename,@@ -192,7 +195,9 @@ class DocumentService:"content_length": len(text_content),}else:- full_doc = await self.docling_service.convert_bytes(content.read(), filename)+ full_doc = await self.docling_service.convert_bytes(+ content.read(), filename, user_id=user_id, auth_header=jwt_token+ )slim_doc = extract_relevant(full_doc)# Extract all text contentdiff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 397438a07..eeb69ba0a 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -1,9 +1,10 @@-import jsonimport asyncio+import jsonimport time-import httpxfrom pathlib import Path-from typing import Any, Dict, List, Optional+from typing import Any++import httpxfrom config.settings import (LANGFLOW_INGEST_FLOW_ID,@@ -41,9 +42,9 @@ class LangflowFileService:@staticmethoddef merge_ui_ingest_settings_into_tweaks(- tweaks: Optional[Dict[str, Any]],- settings: Optional[Dict[str, Any]],- ) -> Dict[str, Any]:+ tweaks: dict[str, Any] | None,+ settings: dict[str, Any] | None,+ ) -> dict[str, Any]:"""Merge UI ingest dict (camelCase) into Langflow run ``tweaks``.- ``chunkSize`` / ``chunkOverlap`` / ``separator`` update the flow's@@ -58,27 +59,19 @@ class LangflowFileService:if not settings:return final_tweaks- if (- settings.get("chunkSize")- or settings.get("chunkOverlap")- or settings.get("separator")- ):+ if settings.get("chunkSize") or settings.get("chunkOverlap") or settings.get("separator"):if "SplitText-QIKhg" not in final_tweaks:final_tweaks["SplitText-QIKhg"] = {}if settings.get("chunkSize"):final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings["chunkSize"]if settings.get("chunkOverlap"):- final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[- "chunkOverlap"- ]+ final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings["chunkOverlap"]if settings.get("separator"):final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]return final_tweaks- async def upload_user_file(- self, file_tuple, jwt_token: Optional[str] = None- ) -> Dict[str, Any]:+ async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:"""Upload a file using Langflow Files API v2: POST /api/v2/files.Returns JSON with keys: id, name, path, size, provider."""@@ -125,22 +118,22 @@ class LangflowFileService:async def run_ingestion_flow(self,- file_paths: List[str],+ file_paths: list[str],file_tuples: list[tuple[str, str, str]],- jwt_token: Optional[str] = None,- session_id: Optional[str] = None,- tweaks: Optional[Dict[str, Any]] = None,- owner: Optional[str] = None,- owner_name: Optional[str] = None,- owner_email: Optional[str] = None,- connector_type: Optional[str] = None,- document_id: Optional[str] = None,- source_url: Optional[str] = None,- allowed_users: Optional[List[str]] = None,- allowed_groups: Optional[List[str]] = None,- selected_embedding_model: Optional[str] = None,- docling_task_id: Optional[str] = None,- ) -> Dict[str, Any]:+ jwt_token: str | None = None,+ session_id: str | None = None,+ tweaks: dict[str, Any] | None = None,+ owner: str | None = None,+ owner_name: str | None = None,+ owner_email: str | None = None,+ connector_type: str | None = None,+ document_id: str | None = None,+ source_url: str | None = None,+ allowed_users: list[str] | None = None,+ allowed_groups: list[str] | None = None,+ selected_embedding_model: str | None = None,+ docling_task_id: str | None = None,+ ) -> dict[str, Any]:"""Trigger the ingestion flow with provided file paths.The flow must expose a File component path in input schema or accept files parameter.@@ -149,7 +142,7 @@ class LangflowFileService:logger.error("[LF] LANGFLOW_INGEST_FLOW_ID is not configured")raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")- payload: Dict[str, Any] = {+ payload: dict[str, Any] = {"input_value": "Ingest files","input_type": "chat","output_type": "text", # Changed from "json" to "text"@@ -188,15 +181,11 @@ class LangflowFileService:bool(jwt_token),)# To compute the file size in bytes, use len() on the file content (which should be bytes)- file_size_bytes = (- len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0- )+ file_size_bytes = len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0# Avoid logging full payload to prevent leaking sensitive data (e.g., JWT)# Extract file metadata if file_tuples is provided- filename = (- str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""- )+ filename = str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""mimetype = (str(file_tuples[0][2])if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2@@ -222,9 +211,7 @@ class LangflowFileService:"X-Langflow-Global-Var-MIMETYPE": mimetype,"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),- "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id)- if document_id- else "",+ "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id) if document_id else "","X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "","X-Langflow-Global-Var-DOCLING_TASK_ID": str(docling_task_id)if docling_task_id@@ -234,13 +221,9 @@ class LangflowFileService:# Serialize ACL lists as JSON strings for Langflow global vars# (flows will parse these back into lists before indexing)if allowed_users is not None:- headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(- allowed_users or []- )+ headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(allowed_users or [])if allowed_groups is not None:- headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(- allowed_groups or []- )+ headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(allowed_groups or [])# Add provider credentials as global variables for ingestionawait add_provider_credentials_to_headers(@@ -327,20 +310,20 @@ class LangflowFileService:self,docs_url: str,crawl_depth: int,- jwt_token: Optional[str] = None,- owner: Optional[str] = None,- owner_name: Optional[str] = None,- owner_email: Optional[str] = None,+ jwt_token: str | None = None,+ owner: str | None = None,+ owner_name: str | None = None,+ owner_email: str | None = None,connector_type: str = "url",prevent_outside: bool = True,- tweaks: Optional[Dict[str, Any]] = None,- ) -> Dict[str, Any]:+ tweaks: dict[str, Any] | None = None,+ ) -> dict[str, Any]:"""Run URL-based docs ingestion flow using Langflow global variable passthrough."""if not docs_url:raise ValueError("DEFAULT_DOCS_URL is not configured")flow_id = await self._ensure_url_ingest_flow_id()- payload: Dict[str, Any] = {+ payload: dict[str, Any] = {"input_value": docs_url,"input_type": "chat","output_type": "text",@@ -428,8 +411,7 @@ class LangflowFileService:flow_file = Path(get_flows_path()) / "openrag_url_mcp.json"if not flow_file.exists():raise ValueError(- "LANGFLOW_URL_INGEST_FLOW_ID is invalid and "- f"flow file was not found at {flow_file}"+ f"LANGFLOW_URL_INGEST_FLOW_ID is invalid and flow file was not found at {flow_file}")with flow_file.open("r", encoding="utf-8") as f:flow_payload = json.load(f)@@ -502,9 +484,7 @@ class LangflowFileService:flow_data = create_resp.json()imported_flow_id = flow_data.get("id")if not imported_flow_id:- raise ValueError(- "Langflow flow import succeeded but no flow id was returned"- )+ raise ValueError("Langflow flow import succeeded but no flow id was returned")self.flow_id_url_ingest = imported_flow_idlogger.warning(@@ -538,15 +518,15 @@ class LangflowFileService:async def upload_and_ingest_file(self,file_tuple,- session_id: Optional[str] = None,- tweaks: Optional[Dict[str, Any]] = None,- settings: Optional[Dict[str, Any]] = None,- jwt_token: Optional[str] = None,- owner: Optional[str] = None,- owner_name: Optional[str] = None,- owner_email: Optional[str] = None,- connector_type: Optional[str] = None,- ) -> Dict[str, Any]:+ session_id: str | None = None,+ tweaks: dict[str, Any] | None = None,+ settings: dict[str, Any] | None = None,+ jwt_token: str | None = None,+ owner: str | None = None,+ owner_name: str | None = None,+ owner_email: str | None = None,+ connector_type: str | None = None,+ ) -> dict[str, Any]:"""Combined Docling upload and Langflow ingest operation.First uploads the file to Docling, then runs ingestion on it via Langflow@@ -575,7 +555,7 @@ class LangflowFileService:# Step 1: Upload the file to Doclingtry:task_id = await self.docling_service.upload_to_docling_direct_async(- filename, content+ filename, content, user_id=owner, auth_header=jwt_token)logger.debug("[LF] Docling upload completed successfully",@@ -589,12 +569,10 @@ class LangflowFileService:"[LF] Docling upload failed during combined operation",extra={"error": str(e)},)- raise Exception(f"Docling upload failed: {str(e)}")+ raise Exception(f"Docling upload failed: {str(e)}") from e# Step 2: Prepare for ingestion- final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(- tweaks, settings- )+ final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(tweaks, settings)if settings:logger.debug("[LF] Applying ingestion settings",diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.pyindex 95bcdd503..f197b987e 100644--- a/src/utils/opensearch_init.py+++ b/src/utils/opensearch_init.py@@ -8,7 +8,9 @@ that need to (re)create the documents index after onboarding completes.from config.settings import (API_KEYS_INDEX_BODY,API_KEYS_INDEX_NAME,+ IBM_AUTH_ENABLED,INDEX_BODY,+ PLATFORM_AUTH_DEV_MODE,clients,get_index_name,get_openrag_config,@@ -41,6 +43,10 @@ async def wait_for_opensearch(opensearch_client=None):async def configure_alerting_security():"""Configure OpenSearch alerting plugin security settings"""+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:+ logger.info("Skipping alerting security configuration in IBM dev mode.")+ return+try:alerting_settings = {"persistent": {@@ -58,6 +64,10 @@ async def configure_alerting_security():async def _ensure_opensearch_index():"""Ensure OpenSearch index exists when using traditional connector service."""+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:+ logger.info("Skipping OpenSearch index creation in IBM dev mode.")+ return+try:index_name = get_index_name()if await clients.opensearch.indices.exists(index=index_name):@@ -111,22 +121,29 @@ async def init_index(opensearch_client=None, admin_username: str = None):)else:logger.info(- "Index already exists, skipping creation and changing number of replicas",+ "Index already exists, skipping creation",index_name=index_name,embedding_model=embedding_model,)- current = await os_client.indices.get_settings(index=index_name)- current_replicas = int(- current[index_name]["settings"]["index"].get("number_of_replicas", 1)- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info(- "Updated documents index settings",- )+ if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):+ # Set number of replicas to 0 to not create unused nodes in OpenSearch, in case it was created with more replicas+ try:+ current = await os_client.indices.get_settings(index=index_name)+ current_replicas = int(+ current[index_name]["settings"]["index"].get("number_of_replicas", 1)+ )+ if current_replicas != 0:+ await os_client.indices.put_settings(+ index=index_name,+ body={"index": {"number_of_replicas": 0}},+ )+ logger.info("Updated documents index settings")+ except Exception as e:+ logger.warning(+ "Failed to check or update index replicas",+ index_name=index_name,+ error=str(e),+ )await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS)@@ -169,20 +186,28 @@ async def init_index(opensearch_client=None, admin_username: str = None):index_name=knowledge_filter_index_name,)- current = await os_client.indices.get_settings(index=knowledge_filter_index_name)- current_replicas = int(- current[knowledge_filter_index_name]["settings"]["index"].get(- "number_of_replicas", 1- )- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=knowledge_filter_index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info(- "Updated knowledge filters index settings",- )+ if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):+ try:+ current = await os_client.indices.get_settings(+ index=knowledge_filter_index_name+ )+ current_replicas = int(+ current[knowledge_filter_index_name]["settings"]["index"].get(+ "number_of_replicas", 1+ )+ )+ if current_replicas != 0:+ await os_client.indices.put_settings(+ index=knowledge_filter_index_name,+ body={"index": {"number_of_replicas": 0}},+ )+ logger.info("Updated knowledge filters index settings")+ except Exception as e:+ logger.warning(+ "Failed to check or update knowledge filter index replicas",+ index_name=knowledge_filter_index_name,+ error=str(e),+ )if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):await os_client.indices.create(index=API_KEYS_INDEX_NAME, body=API_KEYS_INDEX_BODY)@@ -199,9 +224,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_errorif is_disk_space_error(e):- logger.error(- "OpenSearch disk space exceeded watermark. Index creation failed."- )+ logger.error("OpenSearch disk space exceeded watermark. Index creation failed.")raise OpenSearchDiskSpaceError("OpenSearch disk space is full (watermark exceeded). ""Please free up disk space on your Docker volume or host machine to continue."diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.pyindex b89bfdf86..46d729965 100644--- a/src/utils/opensearch_utils.py+++ b/src/utils/opensearch_utils.py@@ -1,8 +1,10 @@import asyncioimport osimport random+import yamlfrom opensearchpy import AsyncOpenSearch+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -51,6 +53,7 @@ def is_disk_space_error(error: Exception) -> bool:error_str = str(error).lower()return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)+async def wait_for_opensearch(opensearch_client: AsyncOpenSearch,max_retries: int = 15,@@ -109,7 +112,7 @@ async def wait_for_opensearch()if attempt < max_retries - 1:- delay = min(base_delay * (2 ** attempt), max_delay)+ delay = min(base_delay * (2**attempt), max_delay)delay = random.uniform(delay / 2, delay)logger.debug(@@ -127,36 +130,33 @@ async def wait_for_opensearch(async def graceful_opensearch_shutdown(opensearch_client: AsyncOpenSearch) -> None:"""Gracefully shutdown OpenSearch client connection.-+This ensures that all pending operations are completed and connectionsare properly closed before the application exits.-+Args:opensearch_client: The OpenSearch client to shutdown."""if opensearch_client is None:logger.debug("OpenSearch client is None, skipping graceful shutdown")return-+try:logger.info("Initiating graceful OpenSearch shutdown...")-+# Flush any pending operations by checking cluster health one last timetry:- await asyncio.wait_for(- opensearch_client.cluster.health(),- timeout=10.0- )+ await asyncio.wait_for(opensearch_client.cluster.health(), timeout=10.0)logger.debug("Final cluster health check completed")- except asyncio.TimeoutError:+ except TimeoutError:logger.warning("Timeout during final cluster health check")except Exception as e:logger.debug("[OPENSEARCH] Final cluster health check skipped", reason=str(e))-+# Close the client connectionawait opensearch_client.close()logger.info("OpenSearch client connection closed gracefully")-+except Exception as e:logger.error("Error during graceful OpenSearch shutdown", error=str(e))@@ -184,7 +184,11 @@ async def setup_opensearch_security(This should be called during initial setup after OpenSearch is ready."""- from config.settings import IBM_AUTH_ENABLED+ from config.settings import IBM_AUTH_ENABLED, PLATFORM_AUTH_DEV_MODE++ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:+ logger.info("Skipping OpenSearch security configuration in IBM dev mode.")+ returnlogger.info("Initializing OpenSearch security configuration...", ibm_auth=IBM_AUTH_ENABLED)@@ -210,13 +214,20 @@ async def setup_opensearch_security(try:# 1. & 2. Readiness checkslogger.info("[OPENSEARCH] Performing readiness checks...")-+try:- rolesmapping_response = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping")- logger.info("[OPENSEARCH] Current rolesmapping retrieved", count=len(rolesmapping_response) if isinstance(rolesmapping_response, dict) else "unknown")+ rolesmapping_response = await opensearch_client.transport.perform_request(+ "GET", "/_plugins/_security/api/rolesmapping"+ )+ logger.info(+ "[OPENSEARCH] Current rolesmapping retrieved",+ count=len(rolesmapping_response)+ if isinstance(rolesmapping_response, dict)+ else "unknown",+ )except Exception as e:logger.warning("[OPENSEARCH] Failed to get current rolesmapping", error=str(e))-+cluster_health = await opensearch_client.cluster.health()logger.info("[OPENSEARCH] Cluster health check passed", status=cluster_health.get("status"))@@ -225,10 +236,13 @@ async def setup_opensearch_security(logger.error(f"[OPENSEARCH] Roles configuration file not found: {roles_file}")raise FileNotFoundError(f"Roles configuration file not found: {roles_file}")- with open(roles_file, "r") as f:+ with open(roles_file) as f:roles_config = yaml.safe_load(f)-- logger.info("[OPENSEARCH] Loaded roles configuration", roles=list(roles_config.keys()) if roles_config else [])++ logger.info(+ "[OPENSEARCH] Loaded roles configuration",+ roles=list(roles_config.keys()) if roles_config else [],+ )# 3. Create openrag_user_roleif "openrag_user_role" in roles_config:@@ -236,15 +250,19 @@ async def setup_opensearch_security(logger.info("[OPENSEARCH] Creating 'openrag_user_role' role",- patterns=role_body['index_permissions'][0]['index_patterns'] if 'index_permissions' in role_body else 'default',- allowed_actions=role_body['index_permissions'][0].get('allowed_actions', []) if 'index_permissions' in role_body else []+ patterns=role_body["index_permissions"][0]["index_patterns"]+ if "index_permissions" in role_body+ else "default",+ allowed_actions=role_body["index_permissions"][0].get("allowed_actions", [])+ if "index_permissions" in role_body+ else [],)-+resp = await opensearch_client.transport.perform_request("PUT","/_plugins/_security/api/roles/openrag_user_role",body=role_body,- headers={"Content-Type": "application/json"}+ headers={"Content-Type": "application/json"},)logger.info("[OPENSEARCH] Role creation response", response=resp)else:@@ -255,10 +273,13 @@ async def setup_opensearch_security(logger.error(f"[OPENSEARCH] Roles mapping file not found: {roles_mapping_file}")raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")- with open(roles_mapping_file, "r") as f:+ with open(roles_mapping_file) as f:mapping_config = yaml.safe_load(f)-- logger.info("[OPENSEARCH] Loaded roles mapping configuration", mappings=list(mapping_config.keys()) if mapping_config else [])++ logger.info(+ "[OPENSEARCH] Loaded roles mapping configuration",+ mappings=list(mapping_config.keys()) if mapping_config else [],+ )# 4. Create openrag_user_role mappingif "openrag_user_role" in mapping_config:@@ -266,13 +287,13 @@ async def setup_opensearch_security(logger.info("[OPENSEARCH] Creating 'openrag_user_role' mapping",backend_roles=mapping_body.get("backend_roles", []),- users=mapping_body.get("users", [])+ users=mapping_body.get("users", []),)resp = await opensearch_client.transport.perform_request("PUT","/_plugins/_security/api/rolesmapping/openrag_user_role",body=mapping_body,- headers={"Content-Type": "application/json"}+ headers={"Content-Type": "application/json"},)logger.info("[OPENSEARCH] Role mapping update response", response=resp)@@ -316,9 +337,9 @@ async def setup_opensearch_security(users=new_admin_users,)- merged_users = list(set(- all_access_body.get("users", []) + existing_users + new_admin_users- ))+ merged_users = list(+ set(all_access_body.get("users", []) + existing_users + new_admin_users)+ )all_access_body["users"] = merged_userslogger.debug("[OPENSEARCH] Merged all_access users", users=merged_users)@@ -357,16 +378,20 @@ async def setup_opensearch_security("PUT","/_plugins/_security/api/rolesmapping/all_access",body=all_access_body,- headers={"Content-Type": "application/json"}+ headers={"Content-Type": "application/json"},)logger.info("[OPENSEARCH] All access mapping update response", response=resp)# 6. Final verificationlogger.info("[OPENSEARCH] Verifying security configuration...")- role_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/roles/openrag_user_role")+ role_verify = await opensearch_client.transport.perform_request(+ "GET", "/_plugins/_security/api/roles/openrag_user_role"+ )logger.info("[OPENSEARCH] Role verification", role=role_verify)-- mapping_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping/openrag_user_role")++ mapping_verify = await opensearch_client.transport.perform_request(+ "GET", "/_plugins/_security/api/rolesmapping/openrag_user_role"+ )logger.info("[OPENSEARCH] Role mapping verification", mapping=mapping_verify)logger.info("Successfully completed OpenSearch security configuration.")@@ -374,7 +399,9 @@ async def setup_opensearch_security(except Exception as e:# Check for authentication errors or if the security plugin is missingerror_str = str(e).lower()- if any(code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]):+ if any(+ code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]+ ):logger.warning("Skipping OpenSearch security configuration: ""The cluster may not have the security plugin enabled or "