Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: use provider ingest settings (#1896)

Mike Fortman89e04009badd
diff --git a/src/api/connectors.py b/src/api/connectors.py
index ad3320579..9ca877b82 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -788,6 +788,7 @@ async def connector_sync(
user.user_id,
max_files=max_files,
jwt_token=jwt_token,
+ ingest_settings=body.settings,
)
else:
# No files specified - sync only files already in OpenSearch for this connector
@@ -851,6 +852,7 @@ async def connector_sync(
user.user_id,
ids_to_sync,
jwt_token=jwt_token,
+ ingest_settings=body.settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
)
else:
@@ -866,6 +868,7 @@ async def connector_sync(
max_files=None,
jwt_token=jwt_token,
filename_filter=set(existing_filenames),
+ ingest_settings=body.settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
)
task_ids = [task_id]
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 42e494d7b..3086b893b 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -180,17 +180,40 @@ class ConnectorService:
models_service=self.models_service,
docling_service=self.docling_service,
)
+ standard_kwargs: dict[str, Any] = {}
+ if isinstance(ingest_settings, dict):
+ em = ingest_settings.get("embeddingModel")
+ if isinstance(em, str) and em.strip():
+ standard_kwargs["embedding_model"] = em.strip()
+ for ui_key, param in (
+ ("chunkSize", "chunk_size"),
+ ("chunkOverlap", "chunk_overlap"),
+ ):
+ raw = ingest_settings.get(ui_key)
+ if raw is not None:
+ try:
+ standard_kwargs[param] = int(raw)
+ except (TypeError, ValueError):
+ pass
+ if "ocr" in ingest_settings:
+ standard_kwargs["ocr"] = bool(ingest_settings["ocr"])
+ if "pictureDescriptions" in ingest_settings:
+ standard_kwargs["picture_descriptions"] = bool(
+ ingest_settings["pictureDescriptions"]
+ )
+
result = await processor.process_document_standard(
file_path=tmp_path,
- file_hash=document.id, # Use connector document ID as hash
+ file_hash=document.id,
owner_user_id=owner_user_id,
- original_filename=document.filename, # Pass the original Google Doc title
+ original_filename=document.filename,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
file_size=len(document.content) if document.content else 0,
connector_type=connector_type,
acl=document.acl,
+ **standard_kwargs,
)
logger.info(
@@ -323,6 +346,7 @@ class ConnectorService:
max_files: int = None,
jwt_token: str = None,
filename_filter: set = None,
+ ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
) -> str:
"""
@@ -336,6 +360,8 @@ class ConnectorService:
filename_filter: Optional set of filenames to filter - only files with names
in this set will be synced. Used to prevent deleted files
from being re-synced.
+ ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …)
+ forwarded to ``ConnectorFileProcessor``.
"""
jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)
@@ -421,6 +447,7 @@ class ConnectorService:
else DocumentService(session_manager=self.session_manager)
),
models_service=self.models_service,
+ ingest_settings=ingest_settings,
replace_duplicates=replace_duplicates,
connector_type=connector.CONNECTOR_TYPE,
)
diff --git a/src/models/processors.py b/src/models/processors.py
index f93254848..74f11d491 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -312,6 +312,8 @@ class TaskProcessor:
is_sample_data: bool = False,
acl: "DocumentACL | None" = None,
connector_file_id: str | None = None,
+ ocr: bool | None = None,
+ picture_descriptions: bool | None = None,
):
"""
Standard processing pipeline for non-Langflow processors:
@@ -324,6 +326,8 @@ class TaskProcessor:
chunks (non-Langflow path, e.g. connector UI ``chunkSize``).
chunk_overlap: Overlap between windows; must be less than ``chunk_size``.
acl: DocumentACL instance with access control information
+ ocr: Per-request OCR override (None = use global config).
+ picture_descriptions: Per-request picture descriptions override.
"""
from services.document_service import chunk_texts_for_embeddings
@@ -361,7 +365,11 @@ class TaskProcessor:
slim_doc = process_text_file(file_path)
else:
full_doc = await self.docling_service.convert_file(
- file_path, user_id=owner_user_id, auth_header=jwt_token
+ file_path,
+ user_id=owner_user_id,
+ auth_header=jwt_token,
+ ocr=ocr,
+ picture_descriptions=picture_descriptions,
)
slim_doc = extract_relevant(full_doc)
@@ -1009,6 +1017,10 @@ class ConnectorFileProcessor(TaskProcessor):
standard_kwargs[param] = int(raw)
except (TypeError, ValueError):
pass
+ if "ocr" in s:
+ standard_kwargs["ocr"] = bool(s["ocr"])
+ if "pictureDescriptions" in s:
+ standard_kwargs["picture_descriptions"] = bool(s["pictureDescriptions"])
result = await self.process_document_standard(
file_path=tmp_path,
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index a641bd793..4a2f16ff8 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -147,15 +147,27 @@ class DoclingService:
)
return DoclingService._default_client
- def _build_docling_options(self) -> dict[str, Any]:
- """Build the options payload for docling from OpenRAG configs."""
+ def _build_docling_options(
+ self,
+ *,
+ ocr_override: bool | None = None,
+ picture_descriptions_override: bool | None = None,
+ ) -> dict[str, Any]:
+ """Build the options payload for docling from OpenRAG configs.
+
+ Per-request overrides take precedence over saved Knowledge settings.
+ """
config = get_openrag_config()
knowledge_config = config.knowledge
preset = get_docling_preset_configs(
table_structure=knowledge_config.table_structure,
- ocr=knowledge_config.ocr,
- picture_descriptions=knowledge_config.picture_descriptions,
+ ocr=ocr_override if ocr_override is not None else knowledge_config.ocr,
+ picture_descriptions=(
+ picture_descriptions_override
+ if picture_descriptions_override is not None
+ else knowledge_config.picture_descriptions
+ ),
)
options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}
@@ -180,11 +192,17 @@ class DoclingService:
file_content: bytes,
user_id: str | None = None,
auth_header: str | None = None,
+ *,
+ ocr: bool | None = None,
+ picture_descriptions: bool | None = None,
) -> str:
"""
Upload a file to Docling Serve asynchronously using direct multipart/form-data upload.
"""
- options = self._build_docling_options()
+ options = self._build_docling_options(
+ ocr_override=ocr,
+ picture_descriptions_override=picture_descriptions,
+ )
headers = self._get_auth_headers(user_id, auth_header)
# Docling serve async multipart endpoint /v1/convert/file/async
@@ -416,7 +434,13 @@ class DoclingService:
raise TimeoutError(f"Docling task {task_id} did not complete within {timeout} seconds")
async def convert_file(
- self, file_path: str, user_id: str | None = None, auth_header: str | None = None
+ self,
+ file_path: str,
+ user_id: str | None = None,
+ auth_header: str | None = None,
+ *,
+ ocr: bool | None = None,
+ picture_descriptions: bool | None = None,
) -> dict[str, Any]:
"""
Convert a local file via docling-serve async polling.
@@ -424,7 +448,12 @@ class DoclingService:
path = Path(file_path)
file_bytes = path.read_bytes()
task_id = await self.upload_to_docling_direct_async(
- path.name, file_bytes, user_id=user_id, auth_header=auth_header
+ path.name,
+ file_bytes,
+ user_id=user_id,
+ auth_header=auth_header,
+ ocr=ocr,
+ picture_descriptions=picture_descriptions,
)
return await self.get_docling_result_async(
task_id, user_id=user_id, auth_header=auth_header
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 0e4c2aea0..e61376d5f 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -817,6 +817,9 @@ class LangflowFileService:
content: bytes,
jwt_token: str | None = None,
owner: str | None = None,
+ *,
+ ocr: bool | None = None,
+ picture_descriptions: bool | None = None,
) -> str:
"""Upload a file to Docling Serve and return the task_id immediately.
@@ -831,7 +834,12 @@ class LangflowFileService:
)
try:
task_id = await self.docling_service.upload_to_docling_direct_async(
- filename, content, user_id=owner, auth_header=jwt_token
+ filename,
+ content,
+ user_id=owner,
+ auth_header=jwt_token,
+ ocr=ocr,
+ picture_descriptions=picture_descriptions,
)
logger.debug(
"[LF] Docling submission accepted",
@@ -894,12 +902,24 @@ class LangflowFileService:
filename, content, _ = file_tuple
+ ocr_override = settings.get("ocr") if isinstance(settings, dict) else None
+ pic_desc_override = (
+ settings.get("pictureDescriptions") if isinstance(settings, dict) else None
+ )
+
# ── Phase 1: submit to Docling ──────────────────────────────────
if file_task is not None:
file_task.phase = IngestionPhase.DOCLING
file_task.docling_status = DoclingPhaseStatus.PENDING
- task_id = await self.submit_to_docling(filename, content, owner=owner, jwt_token=jwt_token)
+ task_id = await self.submit_to_docling(
+ filename,
+ content,
+ owner=owner,
+ jwt_token=jwt_token,
+ ocr=ocr_override,
+ picture_descriptions=pic_desc_override,
+ )
if file_task is not None:
file_task.docling_task_id = task_id
@@ -970,6 +990,11 @@ class LangflowFileService:
if file_task is not None:
file_task.phase = IngestionPhase.LANGFLOW
+ _raw_em = settings.get("embeddingModel") if isinstance(settings, dict) else None
+ selected_embedding = (
+ _raw_em.strip() if isinstance(_raw_em, str) and _raw_em.strip() else None
+ )
+
try:
total_start_time = time.time()
ingest_result = await self.run_ingestion_flow(
@@ -985,6 +1010,7 @@ class LangflowFileService:
docling_task_id=task_id,
document_id=document_id,
source_url=source_url,
+ selected_embedding_model=selected_embedding,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,