OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: use provider ingest settings (#1896)
Mike Fortman89e04009badd
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex ad3320579..9ca877b82 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -788,6 +788,7 @@ async def connector_sync(user.user_id,max_files=max_files,jwt_token=jwt_token,+ ingest_settings=body.settings,)else:# No files specified - sync only files already in OpenSearch for this connector@@ -851,6 +852,7 @@ async def connector_sync(user.user_id,ids_to_sync,jwt_token=jwt_token,+ ingest_settings=body.settings,replace_duplicates=_connector_sync_should_replace(connector_type),)else:@@ -866,6 +868,7 @@ async def connector_sync(max_files=None,jwt_token=jwt_token,filename_filter=set(existing_filenames),+ ingest_settings=body.settings,replace_duplicates=_connector_sync_should_replace(connector_type),)task_ids = [task_id]diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 42e494d7b..3086b893b 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -180,17 +180,40 @@ class ConnectorService:models_service=self.models_service,docling_service=self.docling_service,)+ standard_kwargs: dict[str, Any] = {}+ if isinstance(ingest_settings, dict):+ em = ingest_settings.get("embeddingModel")+ if isinstance(em, str) and em.strip():+ standard_kwargs["embedding_model"] = em.strip()+ for ui_key, param in (+ ("chunkSize", "chunk_size"),+ ("chunkOverlap", "chunk_overlap"),+ ):+ raw = ingest_settings.get(ui_key)+ if raw is not None:+ try:+ standard_kwargs[param] = int(raw)+ except (TypeError, ValueError):+ pass+ if "ocr" in ingest_settings:+ standard_kwargs["ocr"] = bool(ingest_settings["ocr"])+ if "pictureDescriptions" in ingest_settings:+ standard_kwargs["picture_descriptions"] = bool(+ ingest_settings["pictureDescriptions"]+ )+result = await processor.process_document_standard(file_path=tmp_path,- file_hash=document.id, # Use connector document ID as hash+ file_hash=document.id,owner_user_id=owner_user_id,- original_filename=document.filename, # Pass the original Google Doc title+ original_filename=document.filename,jwt_token=jwt_token,owner_name=owner_name,owner_email=owner_email,file_size=len(document.content) if document.content else 0,connector_type=connector_type,acl=document.acl,+ **standard_kwargs,)logger.info(@@ -323,6 +346,7 @@ class ConnectorService:max_files: int = None,jwt_token: str = None,filename_filter: set = None,+ ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,) -> str:"""@@ -336,6 +360,8 @@ class ConnectorService:filename_filter: Optional set of filenames to filter - only files with namesin this set will be synced. Used to prevent deleted filesfrom being re-synced.+ ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …)+ forwarded to ``ConnectorFileProcessor``."""jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)@@ -421,6 +447,7 @@ class ConnectorService:else DocumentService(session_manager=self.session_manager)),models_service=self.models_service,+ ingest_settings=ingest_settings,replace_duplicates=replace_duplicates,connector_type=connector.CONNECTOR_TYPE,)diff --git a/src/models/processors.py b/src/models/processors.pyindex f93254848..74f11d491 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -312,6 +312,8 @@ class TaskProcessor:is_sample_data: bool = False,acl: "DocumentACL | None" = None,connector_file_id: str | None = None,+ ocr: bool | None = None,+ picture_descriptions: bool | None = None,):"""Standard processing pipeline for non-Langflow processors:@@ -324,6 +326,8 @@ class TaskProcessor:chunks (non-Langflow path, e.g. connector UI ``chunkSize``).chunk_overlap: Overlap between windows; must be less than ``chunk_size``.acl: DocumentACL instance with access control information+ ocr: Per-request OCR override (None = use global config).+ picture_descriptions: Per-request picture descriptions override."""from services.document_service import chunk_texts_for_embeddings@@ -361,7 +365,11 @@ class TaskProcessor:slim_doc = process_text_file(file_path)else:full_doc = await self.docling_service.convert_file(- file_path, user_id=owner_user_id, auth_header=jwt_token+ file_path,+ user_id=owner_user_id,+ auth_header=jwt_token,+ ocr=ocr,+ picture_descriptions=picture_descriptions,)slim_doc = extract_relevant(full_doc)@@ -1009,6 +1017,10 @@ class ConnectorFileProcessor(TaskProcessor):standard_kwargs[param] = int(raw)except (TypeError, ValueError):pass+ if "ocr" in s:+ standard_kwargs["ocr"] = bool(s["ocr"])+ if "pictureDescriptions" in s:+ standard_kwargs["picture_descriptions"] = bool(s["pictureDescriptions"])result = await self.process_document_standard(file_path=tmp_path,diff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex a641bd793..4a2f16ff8 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -147,15 +147,27 @@ class DoclingService:)return DoclingService._default_client- def _build_docling_options(self) -> dict[str, Any]:- """Build the options payload for docling from OpenRAG configs."""+ def _build_docling_options(+ self,+ *,+ ocr_override: bool | None = None,+ picture_descriptions_override: bool | None = None,+ ) -> dict[str, Any]:+ """Build the options payload for docling from OpenRAG configs.++ Per-request overrides take precedence over saved Knowledge settings.+ """config = get_openrag_config()knowledge_config = config.knowledgepreset = get_docling_preset_configs(table_structure=knowledge_config.table_structure,- ocr=knowledge_config.ocr,- picture_descriptions=knowledge_config.picture_descriptions,+ ocr=ocr_override if ocr_override is not None else knowledge_config.ocr,+ picture_descriptions=(+ picture_descriptions_override+ if picture_descriptions_override is not None+ else knowledge_config.picture_descriptions+ ),)options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}@@ -180,11 +192,17 @@ class DoclingService:file_content: bytes,user_id: str | None = None,auth_header: str | None = None,+ *,+ ocr: bool | None = None,+ picture_descriptions: bool | None = None,) -> str:"""Upload a file to Docling Serve asynchronously using direct multipart/form-data upload."""- options = self._build_docling_options()+ options = self._build_docling_options(+ ocr_override=ocr,+ picture_descriptions_override=picture_descriptions,+ )headers = self._get_auth_headers(user_id, auth_header)# Docling serve async multipart endpoint /v1/convert/file/async@@ -416,7 +434,13 @@ class DoclingService:raise TimeoutError(f"Docling task {task_id} did not complete within {timeout} seconds")async def convert_file(- self, file_path: str, user_id: str | None = None, auth_header: str | None = None+ self,+ file_path: str,+ user_id: str | None = None,+ auth_header: str | None = None,+ *,+ ocr: bool | None = None,+ picture_descriptions: bool | None = None,) -> dict[str, Any]:"""Convert a local file via docling-serve async polling.@@ -424,7 +448,12 @@ class DoclingService:path = Path(file_path)file_bytes = path.read_bytes()task_id = await self.upload_to_docling_direct_async(- path.name, file_bytes, user_id=user_id, auth_header=auth_header+ path.name,+ file_bytes,+ user_id=user_id,+ auth_header=auth_header,+ ocr=ocr,+ picture_descriptions=picture_descriptions,)return await self.get_docling_result_async(task_id, user_id=user_id, auth_header=auth_headerdiff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 0e4c2aea0..e61376d5f 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -817,6 +817,9 @@ class LangflowFileService:content: bytes,jwt_token: str | None = None,owner: str | None = None,+ *,+ ocr: bool | None = None,+ picture_descriptions: bool | None = None,) -> str:"""Upload a file to Docling Serve and return the task_id immediately.@@ -831,7 +834,12 @@ class LangflowFileService:)try:task_id = await self.docling_service.upload_to_docling_direct_async(- filename, content, user_id=owner, auth_header=jwt_token+ filename,+ content,+ user_id=owner,+ auth_header=jwt_token,+ ocr=ocr,+ picture_descriptions=picture_descriptions,)logger.debug("[LF] Docling submission accepted",@@ -894,12 +902,24 @@ class LangflowFileService:filename, content, _ = file_tuple+ ocr_override = settings.get("ocr") if isinstance(settings, dict) else None+ pic_desc_override = (+ settings.get("pictureDescriptions") if isinstance(settings, dict) else None+ )+# ── Phase 1: submit to Docling ──────────────────────────────────if file_task is not None:file_task.phase = IngestionPhase.DOCLINGfile_task.docling_status = DoclingPhaseStatus.PENDING- task_id = await self.submit_to_docling(filename, content, owner=owner, jwt_token=jwt_token)+ task_id = await self.submit_to_docling(+ filename,+ content,+ owner=owner,+ jwt_token=jwt_token,+ ocr=ocr_override,+ picture_descriptions=pic_desc_override,+ )if file_task is not None:file_task.docling_task_id = task_id@@ -970,6 +990,11 @@ class LangflowFileService:if file_task is not None:file_task.phase = IngestionPhase.LANGFLOW+ _raw_em = settings.get("embeddingModel") if isinstance(settings, dict) else None+ selected_embedding = (+ _raw_em.strip() if isinstance(_raw_em, str) and _raw_em.strip() else None+ )+try:total_start_time = time.time()ingest_result = await self.run_ingestion_flow(@@ -985,6 +1010,7 @@ class LangflowFileService:docling_task_id=task_id,document_id=document_id,source_url=source_url,+ selected_embedding_model=selected_embedding,allowed_users=allowed_users,allowed_groups=allowed_groups,allowed_principals=allowed_principals,