OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: sync ingest embedding from settings and persist ingest UI (#1492)
Wallgauef9f24ae1a8d
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex 91ce07899..4235c34ab 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -12,10 +12,7 @@ import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketStaimport { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery";import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";import { IngestSettings } from "@/components/cloud-picker/ingest-settings";-import {- getIngestChunkSettingsError,- type IngestSettings as IngestSettingsType,-} from "@/components/cloud-picker/types";+import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";import { Button } from "@/components/ui/button";import {Tooltip,@@ -23,6 +20,7 @@ import {TooltipTrigger,} from "@/components/ui/tooltip";import { useTask } from "@/contexts/task-context";+import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";// Connectors that sync entire buckets/repositories without a file pickerconst DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];@@ -58,13 +56,7 @@ function BucketView({const [selectedBuckets, setSelectedBuckets] = useState<Set<string>>(new Set(),);- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({- chunkSize: 1000,- chunkOverlap: 200,- ocr: false,- pictureDescriptions: false,- embeddingModel: "text-embedding-3-small",- });+ const [ingestSettings, setIngestSettings] = useSessionIngestSettings();const [isSettingsOpen, setIsSettingsOpen] = useState(false);const invalidate = () => {@@ -383,13 +375,7 @@ export default function UploadProviderPage() {const syncMutation = useSyncConnector();const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({- chunkSize: 1000,- chunkOverlap: 200,- ocr: false,- pictureDescriptions: false,- embeddingModel: "text-embedding-3-small",- });+ const [ingestSettings, setIngestSettings] = useSessionIngestSettings();const accessToken = tokenData?.access_token || null;const isLoading =@@ -405,11 +391,9 @@ export default function UploadProviderPage() {const handleFileSelected = (files: CloudFile[]) => {setSelectedFiles(files);- console.log(`Selected ${files.length} item(s) from ${provider}:`, files);- // You can add additional handling here like triggering sync, etc.};- const handleSync = async (connector: any) => {+ const handleSync = (connector: { connectionId?: string; type: string }) => {if (!connector.connectionId || selectedFiles.length === 0) return;const chunkErr = getIngestChunkSettingsError(ingestSettings);@@ -437,12 +421,13 @@ export default function UploadProviderPage() {onSuccess: (result) => {const taskIds = result.task_ids;if (taskIds && taskIds.length > 0) {- const taskId = taskIds[0]; // Use the first task ID- addTask(taskId);- // Redirect to knowledge page already to show the syncing document+ addTask(taskIds[0]);router.push("/knowledge");}},+ onError: (err) => {+ toast.error(err instanceof Error ? err.message : "Sync failed");+ },},);};@@ -612,7 +597,8 @@ export default function UploadProviderPage() {accessToken={accessToken || undefined}clientId={connector.clientId}baseUrl={connector.baseUrl}- onSettingsChange={setIngestSettings}+ ingestSettings={ingestSettings}+ onIngestSettingsChange={setIngestSettings}/></div>diff --git a/frontend/components/cloud-picker/ingest-settings.tsx b/frontend/components/cloud-picker/ingest-settings.tsxindex 8f1655175..ae56d9da8 100644--- a/frontend/components/cloud-picker/ingest-settings.tsx+++ b/frontend/components/cloud-picker/ingest-settings.tsx@@ -1,7 +1,7 @@"use client";import { ChevronRight } from "lucide-react";-import { useEffect } from "react";+import { useMemo } from "react";import {useGetIBMModelsQuery,useGetOllamaModelsQuery,@@ -34,6 +34,7 @@ import {TooltipTrigger,} from "@/components/ui/tooltip";import { useAuth } from "@/contexts/auth-context";+import { knowledgeToIngestSettings } from "@/lib/ingest-settings-knowledge";import type { IngestSettings as IngestSettingsType } from "./types";interface IngestSettingsProps {@@ -83,36 +84,54 @@ export const IngestSettings = ({? ibmModelsData: openaiModelsData;- // Get embedding model from API settings- const apiEmbeddingModel =- apiSettings.knowledge?.embedding_model ||- modelsData?.embedding_models?.find((m) => m.default)?.value ||- "text-embedding-3-small";+ const defaultEmbedding = modelsData?.embedding_models?.find(+ (m) => m.default,+ )?.value;- // Default settings - use API embedding modelconst defaultSettings: IngestSettingsType = {- chunkSize: 1000,- chunkOverlap: 200,- ocr: false,- pictureDescriptions: false,- embeddingModel: apiEmbeddingModel,+ ...knowledgeToIngestSettings(apiSettings.knowledge),+ embeddingModel:+ apiSettings.knowledge?.embedding_model?.trim() ||+ defaultEmbedding ||+ "text-embedding-3-small",};- // Use provided settings or defaults- const currentSettings = settings || defaultSettings;-- // Update settings when API embedding model changes- useEffect(() => {- if (- apiEmbeddingModel &&- (!settings || settings.embeddingModel !== apiEmbeddingModel)- ) {- onSettingsChange?.({- ...currentSettings,- embeddingModel: apiEmbeddingModel,- });+ const currentSettings = settings ?? defaultSettings;++ /** Radix Select only shows a value that exists in <SelectItem>; always include the current model. */+ const embeddingSelectOptions = useMemo(() => {+ const fallbackList = (getFallbackModels(currentProvider).embedding ??+ []) as ModelOption[];+ const fromApi = modelsData?.embedding_models;+ let base: ModelOption[] =+ fromApi && fromApi.length > 0 ? [...fromApi] : [...fallbackList];+ const v = currentSettings.embeddingModel?.trim();+ if (!v) {+ return base.length > 0+ ? base+ : [+ {+ value: "text-embedding-3-small",+ label: "text-embedding-3-small",+ },+ ];+ }+ if (!base.some((m) => m.value === v)) {+ base = [{ value: v, label: v }, ...base];}- }, [apiEmbeddingModel, settings, onSettingsChange, currentSettings]);+ return base;+ }, [+ currentProvider,+ modelsData?.embedding_models,+ currentSettings.embeddingModel,+ ]);++ const selectEmbeddingValue =+ embeddingSelectOptions.some(+ (m) => m.value === currentSettings.embeddingModel,+ ) && currentSettings.embeddingModel+ ? currentSettings.embeddingModel+ : (embeddingSelectOptions[0]?.value ?? "text-embedding-3-small");const handleSettingsChange = (newSettings: Partial<IngestSettingsType>) => {onSettingsChange?.({ ...currentSettings, ...newSettings });@@ -145,7 +164,7 @@ export const IngestSettings = ({><Selectdisabled={false}- value={currentSettings.embeddingModel}+ value={selectEmbeddingValue}onValueChange={(value) =>handleSettingsChange({ embeddingModel: value })}@@ -162,11 +181,8 @@ export const IngestSettings = ({</Tooltip><SelectContent><ModelSelectItems- models={modelsData?.embedding_models}- fallbackModels={- getFallbackModels(currentProvider)- .embedding as ModelOption[]- }+ models={embeddingSelectOptions}+ fallbackModels={[]}provider={currentProvider}/></SelectContent>diff --git a/frontend/components/cloud-picker/types.ts b/frontend/components/cloud-picker/types.tsindex a06f104aa..f3eea21f5 100644--- a/frontend/components/cloud-picker/types.ts+++ b/frontend/components/cloud-picker/types.ts@@ -23,7 +23,10 @@ export interface UnifiedCloudPickerProps {// OneDrive/SharePoint specific propsclientId?: string;baseUrl?: string;- // Ingest settings+ /** Controlled ingest state from the upload page (single source of truth). */+ ingestSettings?: IngestSettings;+ onIngestSettingsChange?: (settings: IngestSettings) => void;+ /** @deprecated Prefer ingestSettings + onIngestSettingsChange from the page. */onSettingsChange?: (settings: IngestSettings) => void;isIngesting: boolean;}diff --git a/frontend/components/cloud-picker/unified-cloud-picker.tsx b/frontend/components/cloud-picker/unified-cloud-picker.tsxindex 7f91225c0..c87d64a72 100644--- a/frontend/components/cloud-picker/unified-cloud-picker.tsx+++ b/frontend/components/cloud-picker/unified-cloud-picker.tsx@@ -21,6 +21,8 @@ export const UnifiedCloudPicker = ({onPickerStateChange,clientId,baseUrl,+ ingestSettings: ingestSettingsProp,+ onIngestSettingsChange,onSettingsChange,}: UnifiedCloudPickerProps) => {const [isPickerLoaded, setIsPickerLoaded] = useState(false);@@ -29,18 +31,28 @@ export const UnifiedCloudPicker = ({const [isLoadingBaseUrl, setIsLoadingBaseUrl] = useState(false);const [autoBaseUrl, setAutoBaseUrl] = useState<string | undefined>(undefined);- // Settings state with defaults- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({- chunkSize: 1000,- chunkOverlap: 200,- ocr: false,- pictureDescriptions: false,- embeddingModel: "text-embedding-3-small",- });-- // Handle settings changes and notify parent- const handleSettingsChange = (newSettings: IngestSettingsType) => {- setIngestSettings(newSettings);+ const isControlled =+ ingestSettingsProp !== undefined && onIngestSettingsChange !== undefined;++ const [localIngestSettings, setLocalIngestSettings] =+ useState<IngestSettingsType>({+ chunkSize: 1000,+ chunkOverlap: 200,+ ocr: false,+ pictureDescriptions: false,+ embeddingModel: "text-embedding-3-small",+ });++ const ingestSettings = isControlled+ ? ingestSettingsProp!+ : localIngestSettings;++ const handleIngestSettingsChange = (newSettings: IngestSettingsType) => {+ if (isControlled) {+ onIngestSettingsChange!(newSettings);+ } else {+ setLocalIngestSettings(newSettings);+ }onSettingsChange?.(newSettings);};@@ -98,28 +110,11 @@ export const UnifiedCloudPicker = ({]);const handleAddFiles = () => {- // === DIAGNOSTIC LOGGING ===- console.log("=== UnifiedCloudPicker handleAddFiles ===");- console.log("Provider:", provider);- console.log("Client ID:", clientId);- console.log("Base URL (from prop):", baseUrl);- console.log("Auto Base URL:", autoBaseUrl);- console.log("Effective Base URL:", effectiveBaseUrl);- console.log("Is Picker Loaded:", isPickerLoaded);- console.log("Has Access Token:", !!accessToken);-if (!isPickerLoaded || !accessToken) {- console.error(- "Cannot open picker: isPickerLoaded=",- isPickerLoaded,- "hasAccessToken=",- !!accessToken,- );return;}if ((provider === "onedrive" || provider === "sharepoint") && !clientId) {- console.error("Client ID required for OneDrive/SharePoint");return;}@@ -216,7 +211,7 @@ export const UnifiedCloudPicker = ({isOpen={isIngestSettingsOpen}onOpenChange={setIsIngestSettingsOpen}settings={ingestSettings}- onSettingsChange={handleSettingsChange}+ onSettingsChange={handleIngestSettingsChange}/></div>);diff --git a/frontend/hooks/useSessionIngestSettings.ts b/frontend/hooks/useSessionIngestSettings.tsnew file mode 100644index 000000000..2ab60cf8e--- /dev/null+++ b/frontend/hooks/useSessionIngestSettings.ts@@ -0,0 +1,38 @@+"use client";++import {+ type Dispatch,+ type SetStateAction,+ useEffect,+ useRef,+ useState,+} from "react";+import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";+import type { IngestSettings } from "@/components/cloud-picker/types";+import { useAuth } from "@/contexts/auth-context";+import { knowledgeToIngestSettings } from "@/lib/ingest-settings-knowledge";++/**+ * Ingest form state: hydrate once from GET /api/settings `knowledge`, then session-owned.+ */+export function useSessionIngestSettings(): readonly [+ IngestSettings,+ Dispatch<SetStateAction<IngestSettings>>,+] {+ const { isAuthenticated, isNoAuthMode } = useAuth();+ const { data, isSuccess } = useGetSettingsQuery({+ enabled: isAuthenticated || isNoAuthMode,+ });+ const [ingestSettings, setIngestSettings] = useState<IngestSettings>(() =>+ knowledgeToIngestSettings(undefined),+ );+ const hydrated = useRef(false);++ useEffect(() => {+ if (!isSuccess || !data || hydrated.current) return;+ setIngestSettings(knowledgeToIngestSettings(data.knowledge));+ hydrated.current = true;+ }, [isSuccess, data]);++ return [ingestSettings, setIngestSettings] as const;+}diff --git a/frontend/lib/ingest-settings-knowledge.ts b/frontend/lib/ingest-settings-knowledge.tsnew file mode 100644index 000000000..ac87ccd44--- /dev/null+++ b/frontend/lib/ingest-settings-knowledge.ts@@ -0,0 +1,20 @@+import type { KnowledgeSettings } from "@/app/api/queries/useGetSettingsQuery";+import type { IngestSettings } from "@/components/cloud-picker/types";+import { DEFAULT_KNOWLEDGE_SETTINGS } from "@/lib/constants";++/** Map saved Knowledge settings to ingest panel fields (subset of Knowledge UI). */+export function knowledgeToIngestSettings(+ knowledge: KnowledgeSettings | null | undefined,+): IngestSettings {+ return {+ chunkSize: knowledge?.chunk_size ?? DEFAULT_KNOWLEDGE_SETTINGS.chunk_size,+ chunkOverlap:+ knowledge?.chunk_overlap ?? DEFAULT_KNOWLEDGE_SETTINGS.chunk_overlap,+ ocr: knowledge?.ocr ?? DEFAULT_KNOWLEDGE_SETTINGS.ocr,+ pictureDescriptions:+ knowledge?.picture_descriptions ??+ DEFAULT_KNOWLEDGE_SETTINGS.picture_descriptions,+ embeddingModel:+ knowledge?.embedding_model?.trim() || "text-embedding-3-small",+ };+}diff --git a/src/api/connector_router.py b/src/api/connector_router.pyindex 6af6c05c9..21ebfab2a 100644--- a/src/api/connector_router.py+++ b/src/api/connector_router.py@@ -50,11 +50,22 @@ class ConnectorRouter:return await self.get_active_service().get_connector(connection_id)async def sync_specific_files(- self, connection_id: str, user_id: str, file_list: list, jwt_token: str = None, file_infos: list = None+ self,+ connection_id: str,+ user_id: str,+ file_list: list,+ jwt_token: str = None,+ file_infos: list = None,+ ingest_settings: dict = None,):"""Sync specific files using the active service."""return await self.get_active_service().sync_specific_files(- connection_id, user_id, file_list, jwt_token, file_infos=file_infos+ connection_id,+ user_id,+ file_list,+ jwt_token,+ file_infos=file_infos,+ ingest_settings=ingest_settings,)def __getattr__(self, name):diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 4b393137e..c8447097c 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1,4 +1,4 @@-from typing import Any, List, Optional+from typing import Any, Dict, List, Optionalfrom fastapi import Depends, Requestfrom pydantic import BaseModel@@ -93,6 +93,8 @@ class ConnectorSyncBody(BaseModel):sync_all: bool = False# When set, only ingest files from these buckets (IBM COS specific).bucket_filter: Optional[List[str]] = None+ # Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).+ settings: Optional[Dict[str, Any]] = Noneasync def list_connectors(@@ -204,6 +206,7 @@ async def connector_sync(selected_files,jwt_token=jwt_token,file_infos=file_infos,+ ingest_settings=body.settings,)elif body.sync_all or body.bucket_filter:# Full ingest: discover and ingest all files (or files from specific buckets).@@ -241,6 +244,7 @@ async def connector_sync(user.user_id,all_file_ids,jwt_token=jwt_token,+ ingest_settings=body.settings,)else:# sync_all: ingest everything the connector can seediff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 1c981ba5a..efe00c1d0 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -43,6 +43,7 @@ class LangflowConnectorService:jwt_token: str = None,owner_name: str = None,owner_email: str = None,+ ingest_settings: Optional[Dict[str, Any]] = None,) -> Dict[str, Any]:"""Process a document from a connector using LangflowFileService pattern"""@@ -107,8 +108,16 @@ class LangflowConnectorService:"Running Langflow ingestion flow", file_path=langflow_file_path)- # Use the same tweaks pattern as LangflowFileService- tweaks = {} # Let Langflow handle the ingestion with default settings+ connector_tweak_settings = None+ if isinstance(ingest_settings, dict):+ connector_tweak_settings = dict(ingest_settings)+ # Model selection is injected via selected_embedding_model header override.+ # Avoid hard-coding provider-specific embedding component tweak IDs here.+ connector_tweak_settings.pop("embeddingModel", None)++ tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ {}, connector_tweak_settings+ )# Extract ACL information from the connector document, if availableallowed_users: list[str] = []@@ -135,6 +144,11 @@ class LangflowConnectorService:source_url=document.source_url,allowed_users=allowed_users,allowed_groups=allowed_groups,+ selected_embedding_model=(+ ingest_settings.get("embeddingModel")+ if isinstance(ingest_settings, dict)+ else None+ ),)logger.debug("Ingestion flow completed", result=ingestion_result)@@ -279,6 +293,7 @@ class LangflowConnectorService:file_ids: List[str],jwt_token: str = None,file_infos: List[Dict[str, Any]] = None,+ ingest_settings: Optional[Dict[str, Any]] = None,) -> str:"""Sync specific files by their IDs using Langflow processing.@@ -376,6 +391,7 @@ class LangflowConnectorService:jwt_token=jwt_token,owner_name=owner_name,owner_email=owner_email,+ ingest_settings=ingest_settings,)# Create custom task using TaskServicediff --git a/src/connectors/service.py b/src/connectors/service.pyindex 1e8dc7b0f..df3a2e3f5 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -309,6 +309,7 @@ class ConnectorService:file_ids: List[str],jwt_token: str = None,file_infos: List[Dict[str, Any]] = None,+ ingest_settings: Optional[Dict[str, Any]] = None,) -> str:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).@@ -321,6 +322,8 @@ class ConnectorService:jwt_token: Optional JWT token for authenticationfile_infos: Optional list of file info dicts with {id, name, mimeType, downloadUrl, size}When provided, download URLs can be used directly without Graph API calls.+ ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …) passed to+ ``ConnectorFileProcessor`` when Langflow ingest is disabled."""if not self.task_service:raise ValueError(@@ -415,6 +418,7 @@ class ConnectorService:else DocumentService(session_manager=self.session_manager)),models_service=self.models_service,+ ingest_settings=ingest_settings,)# Create custom task using TaskServicediff --git a/src/models/processors.py b/src/models/processors.pyindex 1e5ae886b..86df12194 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -1,4 +1,4 @@-from typing import Any+from typing import Any, Dict, Optionalfrom .tasks import UploadTask, FileTaskfrom utils.logging_config import get_loggerfrom utils.file_utils import (@@ -187,6 +187,8 @@ class TaskProcessor:file_size: int = None,connector_type: str = "local",embedding_model: str = None,+ chunk_size: int = None,+ chunk_overlap: int = None,is_sample_data: bool = False,acl: "DocumentACL" = None,):@@ -197,6 +199,9 @@ class TaskProcessor:Args:embedding_model: Embedding model to use (defaults to the currentembedding model from settings)+ chunk_size: Optional character window size for re-splitting extracted+ chunks (non-Langflow path, e.g. connector UI ``chunkSize``).+ chunk_overlap: Overlap between windows; must be less than ``chunk_size``.acl: DocumentACL instance with access control information"""import datetime@@ -207,7 +212,10 @@ class TaskProcessor:get_openrag_config,)from services.document_service import chunk_texts_for_embeddings- from utils.document_processing import extract_relevant+ from utils.document_processing import (+ extract_relevant,+ resplit_chunks_character_windows,+ )from utils.embedding_fields import ensure_embedding_field_exists# Use provided embedding model or configured model.@@ -258,6 +266,25 @@ class TaskProcessor:full_doc = await convert_file(file_path, httpx_client=clients.docling_http_client)slim_doc = extract_relevant(full_doc)+ if chunk_size is not None:+ try:+ cs = int(chunk_size)+ except (TypeError, ValueError):+ cs = 0+ if cs > 0:+ try:+ co = (+ int(chunk_overlap)+ if chunk_overlap is not None+ else 0+ )+ except (TypeError, ValueError):+ co = 0+ if co < cs:+ slim_doc["chunks"] = resplit_chunks_character_windows(+ slim_doc["chunks"], cs, max(0, co)+ )+texts = [c["text"] for c in slim_doc["chunks"]]litellm_embedding_model = await self.models_service.get_litellm_model_name(embedding_model) if self.models_service is not None else embedding_model@@ -455,6 +482,7 @@ class ConnectorFileProcessor(TaskProcessor):owner_email: str = None,document_service=None,models_service=None,+ ingest_settings: Optional[Dict[str, Any]] = None,):super().__init__(document_service=document_service, models_service=models_service)self.connector_service = connector_service@@ -464,6 +492,7 @@ class ConnectorFileProcessor(TaskProcessor):self.jwt_token = jwt_tokenself.owner_name = owner_nameself.owner_email = owner_email+ self.ingest_settings = ingest_settingsasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask@@ -510,6 +539,23 @@ class ConnectorFileProcessor(TaskProcessor):# Compute hashfile_hash = hash_id(tmp_path)+ standard_kwargs: Dict[str, Any] = {}+ if isinstance(self.ingest_settings, dict):+ s = self.ingest_settings+ em = s.get("embeddingModel")+ if isinstance(em, str) and em.strip():+ standard_kwargs["embedding_model"] = em.strip()+ for ui_key, param in (+ ("chunkSize", "chunk_size"),+ ("chunkOverlap", "chunk_overlap"),+ ):+ raw = s.get(ui_key)+ if raw is not None:+ try:+ standard_kwargs[param] = int(raw)+ except (TypeError, ValueError):+ pass+# Use consolidated standard processingresult = await self.process_document_standard(file_path=tmp_path,@@ -522,6 +568,7 @@ class ConnectorFileProcessor(TaskProcessor):file_size=len(document.content),connector_type=connection.connector_type,acl=document.acl,+ **standard_kwargs,)# Add connector-specific metadata@@ -555,6 +602,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):jwt_token: str = None,owner_name: str = None,owner_email: str = None,+ ingest_settings: Optional[Dict[str, Any]] = None,):super().__init__()self.langflow_connector_service = langflow_connector_service@@ -564,6 +612,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):self.jwt_token = jwt_tokenself.owner_name = owner_nameself.owner_email = owner_email+ self.ingest_settings = ingest_settingsasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask@@ -633,6 +682,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):jwt_token=self.jwt_token,owner_name=self.owner_name,owner_email=self.owner_email,+ ingest_settings=self.ingest_settings,)file_task.status = TaskStatus.COMPLETEDdiff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex cea9efd2a..44bc95d4d 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -33,6 +33,43 @@ class LangflowFileService:),)+ @staticmethod+ def merge_ui_ingest_settings_into_tweaks(+ tweaks: Optional[Dict[str, Any]],+ settings: Optional[Dict[str, Any]],+ ) -> Dict[str, Any]:+ """Merge UI ingest dict (camelCase) into Langflow run ``tweaks``.++ - ``chunkSize`` / ``chunkOverlap`` / ``separator`` update the flow's+ ``SplitText-QIKhg`` node when any of those keys are present.+ - ``embeddingModel`` is intentionally not mapped to a component tweak.+ The embedding model should be supplied via+ ``run_ingestion_flow(..., selected_embedding_model=...)`` so Langflow+ resolves it through the global variable override, without relying on+ provider-specific component ids.+ """+ final_tweaks = dict(tweaks) if tweaks else {}+ if not settings:+ return final_tweaks++ if (+ settings.get("chunkSize")+ or settings.get("chunkOverlap")+ or settings.get("separator")+ ):+ if "SplitText-QIKhg" not in final_tweaks:+ final_tweaks["SplitText-QIKhg"] = {}+ if settings.get("chunkSize"):+ final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings["chunkSize"]+ if settings.get("chunkOverlap"):+ final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[+ "chunkOverlap"+ ]+ if settings.get("separator"):+ final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]++ return final_tweaks+async def upload_user_file(self, file_tuple, jwt_token: Optional[str] = None) -> Dict[str, Any]:@@ -95,6 +132,7 @@ class LangflowFileService:source_url: Optional[str] = None,allowed_users: Optional[List[str]] = None,allowed_groups: Optional[List[str]] = None,+ selected_embedding_model: Optional[str] = None,) -> Dict[str, Any]:"""Trigger the ingestion flow with provided file paths.@@ -156,6 +194,8 @@ class LangflowFileService:config = get_openrag_config()embedding_model = config.knowledge.embedding_model+ if selected_embedding_model:+ embedding_model = selected_embedding_modelheaders = {"X-Langflow-Global-Var-JWT": str(jwt_token),@@ -514,44 +554,13 @@ class LangflowFileService:if not file_path:raise ValueError("Upload successful but no file path returned")- # Convert UI settings to component tweaks if provided- final_tweaks = tweaks.copy() if tweaks else {}-+ final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ tweaks, settings+ )if settings:logger.debug(- "[LF] Applying ingestion settings", extra={"settings": settings}- )-- # Split Text component tweaks (SplitText-QIKhg)- if (- settings.get("chunkSize")- or settings.get("chunkOverlap")- or settings.get("separator")- ):- if "SplitText-QIKhg" not in final_tweaks:- final_tweaks["SplitText-QIKhg"] = {}- if settings.get("chunkSize"):- final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings[- "chunkSize"- ]- if settings.get("chunkOverlap"):- final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[- "chunkOverlap"- ]- if settings.get("separator"):- final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]-- # OpenAI Embeddings component tweaks (OpenAIEmbeddings-joRJ6)- if settings.get("embeddingModel"):- if "OpenAIEmbeddings-joRJ6" not in final_tweaks:- final_tweaks["OpenAIEmbeddings-joRJ6"] = {}- final_tweaks["OpenAIEmbeddings-joRJ6"]["model"] = settings[- "embeddingModel"- ]-- logger.debug(- "[LF] Final tweaks with settings applied",- extra={"tweaks": final_tweaks},+ "[LF] Applying ingestion settings",+ extra={"settings": settings, "tweaks": final_tweaks},)# Step 3: Run ingestiondiff --git a/src/utils/document_processing.py b/src/utils/document_processing.pyindex 2ecaf68ff..0ca02bfa4 100644--- a/src/utils/document_processing.py+++ b/src/utils/document_processing.py@@ -141,3 +141,42 @@ def extract_relevant(doc_dict: dict) -> dict:"mimetype": origin.get("mimetype"),"chunks": chunks,}+++def resplit_chunks_character_windows(+ chunks: list,+ chunk_size: int,+ chunk_overlap: int,+) -> list:+ """Split long chunk texts into fixed-size character windows with overlap.++ Used by the non-Langflow ingestion path when the UI supplies ``chunkSize`` /+ ``chunkOverlap``. Invalid combinations (non-positive size or overlap >= size)+ return ``chunks`` unchanged.+ """+ if chunk_size <= 0 or chunk_overlap >= chunk_size:+ return chunks+ stride = chunk_size - chunk_overlap+ if stride <= 0:+ return chunks++ out: list = []+ for ch in chunks:+ text = ch.get("text") if isinstance(ch, dict) else None+ if not isinstance(text, str):+ out.append(dict(ch) if isinstance(ch, dict) else ch)+ continue+ if len(text) <= chunk_size:+ out.append(dict(ch))+ continue+ start = 0+ while start < len(text):+ end = min(start + chunk_size, len(text))+ piece = text[start:end]+ new_ch = dict(ch)+ new_ch["text"] = piece+ out.append(new_ch)+ if end >= len(text):+ break+ start += stride+ return outdiff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.pynew file mode 100644index 000000000..bfb6b3254--- /dev/null+++ b/tests/unit/test_document_processing_resplit.py@@ -0,0 +1,37 @@+"""Tests for resplit_chunks_character_windows."""++from src.utils.document_processing import resplit_chunks_character_windows+++def test_invalid_returns_original():+ chunks = [{"page": 1, "text": "hello world" * 50}]+ assert resplit_chunks_character_windows(chunks, 0, 0) is chunks+ assert resplit_chunks_character_windows(chunks, 10, 10) is chunks+ assert resplit_chunks_character_windows(chunks, 5, 6) is chunks+++def test_non_dict_chunk_passthrough():+ chunks = ["not-a-dict", {"page": 1, "text": "x" * 30}]+ out = resplit_chunks_character_windows(chunks, 10, 2)+ assert out[0] == "not-a-dict"+ assert len(out) == 1 + 4 # one long dict chunk -> 4 windows+++def test_short_text_unchanged_count():+ chunks = [{"page": 2, "type": "text", "text": "short"}]+ out = resplit_chunks_character_windows(chunks, 100, 20)+ assert len(out) == 1+ assert out[0]["text"] == "short"+ assert out[0]["page"] == 2+++def test_long_text_windows_with_overlap():+ # 22 chars, size 10, overlap 4 -> stride 6; third window reaches end+ text = "abcdefghijklmnopqrstuv"+ assert len(text) == 22+ chunks = [{"page": 1, "text": text}]+ out = resplit_chunks_character_windows(chunks, 10, 4)+ assert len(out) == 3+ assert out[0]["text"] == "abcdefghij"+ assert out[1]["text"] == "ghijklmnop"+ assert out[2]["text"] == "mnopqrstuv"diff --git a/tests/unit/test_langflow_file_service_merge_tweaks.py b/tests/unit/test_langflow_file_service_merge_tweaks.pynew file mode 100644index 000000000..bf5c2e321--- /dev/null+++ b/tests/unit/test_langflow_file_service_merge_tweaks.py@@ -0,0 +1,68 @@+"""Unit tests for LangflowFileService.merge_ui_ingest_settings_into_tweaks."""++from src.services.langflow_file_service import LangflowFileService+++def test_merge_no_settings_returns_tweaks_copy():+ base = {"OtherNode": {"x": 1}}+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(base, None)+ assert out == base+ assert out is not base+++def test_merge_empty_settings_returns_tweaks_only():+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(None, {})+ assert out == {}+++def test_merge_chunk_fields_populate_split_text():+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ None,+ {"chunkSize": 512, "chunkOverlap": 64, "separator": "\n\n"},+ )+ assert out["SplitText-QIKhg"] == {+ "chunk_size": 512,+ "chunk_overlap": 64,+ "separator": "\n\n",+ }+++def test_merge_chunk_partial_only_sets_provided_keys():+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ None,+ {"chunkSize": 1000},+ )+ assert out["SplitText-QIKhg"] == {"chunk_size": 1000}+++def test_merge_preserves_and_extends_existing_split_text():+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ {"SplitText-QIKhg": {"chunk_size": 100, "existing": "keep"}},+ {"chunkOverlap": 20},+ )+ assert out["SplitText-QIKhg"] == {+ "chunk_size": 100,+ "existing": "keep",+ "chunk_overlap": 20,+ }+++def test_merge_embedding_model_is_ignored_in_tweaks():+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(+ None,+ {"embeddingModel": "text-embedding-3-large"},+ )+ assert out == {}+++def test_connector_style_settings_without_embedding_only_split_text():+ """Embedding model is not mapped to tweaks; split settings still apply."""+ settings = {+ "chunkSize": 800,+ "chunkOverlap": 100,+ "ocr": True,+ "embeddingModel": "ignored",+ }+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks({}, settings)+ assert "OpenAIEmbeddings-joRJ6" not in out+ assert out["SplitText-QIKhg"]["chunk_size"] == 800