Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: sync ingest embedding from settings and persist ingest UI (#1492)

Wallgauef9f24ae1a8d
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 91ce07899..4235c34ab 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -12,10 +12,7 @@ import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketSta
import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery";
import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
import { IngestSettings } from "@/components/cloud-picker/ingest-settings";
-import {
- getIngestChunkSettingsError,
- type IngestSettings as IngestSettingsType,
-} from "@/components/cloud-picker/types";
+import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";
import { Button } from "@/components/ui/button";
import {
Tooltip,
@@ -23,6 +20,7 @@ import {
TooltipTrigger,
} from "@/components/ui/tooltip";
import { useTask } from "@/contexts/task-context";
+import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
// Connectors that sync entire buckets/repositories without a file picker
const DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];
@@ -58,13 +56,7 @@ function BucketView({
const [selectedBuckets, setSelectedBuckets] = useState<Set<string>>(
new Set(),
);
- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({
- chunkSize: 1000,
- chunkOverlap: 200,
- ocr: false,
- pictureDescriptions: false,
- embeddingModel: "text-embedding-3-small",
- });
+ const [ingestSettings, setIngestSettings] = useSessionIngestSettings();
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
const invalidate = () => {
@@ -383,13 +375,7 @@ export default function UploadProviderPage() {
const syncMutation = useSyncConnector();
const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);
- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({
- chunkSize: 1000,
- chunkOverlap: 200,
- ocr: false,
- pictureDescriptions: false,
- embeddingModel: "text-embedding-3-small",
- });
+ const [ingestSettings, setIngestSettings] = useSessionIngestSettings();
const accessToken = tokenData?.access_token || null;
const isLoading =
@@ -405,11 +391,9 @@ export default function UploadProviderPage() {
const handleFileSelected = (files: CloudFile[]) => {
setSelectedFiles(files);
- console.log(`Selected ${files.length} item(s) from ${provider}:`, files);
- // You can add additional handling here like triggering sync, etc.
};
- const handleSync = async (connector: any) => {
+ const handleSync = (connector: { connectionId?: string; type: string }) => {
if (!connector.connectionId || selectedFiles.length === 0) return;
const chunkErr = getIngestChunkSettingsError(ingestSettings);
@@ -437,12 +421,13 @@ export default function UploadProviderPage() {
onSuccess: (result) => {
const taskIds = result.task_ids;
if (taskIds && taskIds.length > 0) {
- const taskId = taskIds[0]; // Use the first task ID
- addTask(taskId);
- // Redirect to knowledge page already to show the syncing document
+ addTask(taskIds[0]);
router.push("/knowledge");
}
},
+ onError: (err) => {
+ toast.error(err instanceof Error ? err.message : "Sync failed");
+ },
},
);
};
@@ -612,7 +597,8 @@ export default function UploadProviderPage() {
accessToken={accessToken || undefined}
clientId={connector.clientId}
baseUrl={connector.baseUrl}
- onSettingsChange={setIngestSettings}
+ ingestSettings={ingestSettings}
+ onIngestSettingsChange={setIngestSettings}
/>
</div>
diff --git a/frontend/components/cloud-picker/ingest-settings.tsx b/frontend/components/cloud-picker/ingest-settings.tsx
index 8f1655175..ae56d9da8 100644
--- a/frontend/components/cloud-picker/ingest-settings.tsx
+++ b/frontend/components/cloud-picker/ingest-settings.tsx
@@ -1,7 +1,7 @@
"use client";
import { ChevronRight } from "lucide-react";
-import { useEffect } from "react";
+import { useMemo } from "react";
import {
useGetIBMModelsQuery,
useGetOllamaModelsQuery,
@@ -34,6 +34,7 @@ import {
TooltipTrigger,
} from "@/components/ui/tooltip";
import { useAuth } from "@/contexts/auth-context";
+import { knowledgeToIngestSettings } from "@/lib/ingest-settings-knowledge";
import type { IngestSettings as IngestSettingsType } from "./types";
interface IngestSettingsProps {
@@ -83,36 +84,54 @@ export const IngestSettings = ({
? ibmModelsData
: openaiModelsData;
- // Get embedding model from API settings
- const apiEmbeddingModel =
- apiSettings.knowledge?.embedding_model ||
- modelsData?.embedding_models?.find((m) => m.default)?.value ||
- "text-embedding-3-small";
+ const defaultEmbedding = modelsData?.embedding_models?.find(
+ (m) => m.default,
+ )?.value;
- // Default settings - use API embedding model
const defaultSettings: IngestSettingsType = {
- chunkSize: 1000,
- chunkOverlap: 200,
- ocr: false,
- pictureDescriptions: false,
- embeddingModel: apiEmbeddingModel,
+ ...knowledgeToIngestSettings(apiSettings.knowledge),
+ embeddingModel:
+ apiSettings.knowledge?.embedding_model?.trim() ||
+ defaultEmbedding ||
+ "text-embedding-3-small",
};
- // Use provided settings or defaults
- const currentSettings = settings || defaultSettings;
-
- // Update settings when API embedding model changes
- useEffect(() => {
- if (
- apiEmbeddingModel &&
- (!settings || settings.embeddingModel !== apiEmbeddingModel)
- ) {
- onSettingsChange?.({
- ...currentSettings,
- embeddingModel: apiEmbeddingModel,
- });
+ const currentSettings = settings ?? defaultSettings;
+
+ /** Radix Select only shows a value that exists in <SelectItem>; always include the current model. */
+ const embeddingSelectOptions = useMemo(() => {
+ const fallbackList = (getFallbackModels(currentProvider).embedding ??
+ []) as ModelOption[];
+ const fromApi = modelsData?.embedding_models;
+ let base: ModelOption[] =
+ fromApi && fromApi.length > 0 ? [...fromApi] : [...fallbackList];
+ const v = currentSettings.embeddingModel?.trim();
+ if (!v) {
+ return base.length > 0
+ ? base
+ : [
+ {
+ value: "text-embedding-3-small",
+ label: "text-embedding-3-small",
+ },
+ ];
+ }
+ if (!base.some((m) => m.value === v)) {
+ base = [{ value: v, label: v }, ...base];
}
- }, [apiEmbeddingModel, settings, onSettingsChange, currentSettings]);
+ return base;
+ }, [
+ currentProvider,
+ modelsData?.embedding_models,
+ currentSettings.embeddingModel,
+ ]);
+
+ const selectEmbeddingValue =
+ embeddingSelectOptions.some(
+ (m) => m.value === currentSettings.embeddingModel,
+ ) && currentSettings.embeddingModel
+ ? currentSettings.embeddingModel
+ : (embeddingSelectOptions[0]?.value ?? "text-embedding-3-small");
const handleSettingsChange = (newSettings: Partial<IngestSettingsType>) => {
onSettingsChange?.({ ...currentSettings, ...newSettings });
@@ -145,7 +164,7 @@ export const IngestSettings = ({
>
<Select
disabled={false}
- value={currentSettings.embeddingModel}
+ value={selectEmbeddingValue}
onValueChange={(value) =>
handleSettingsChange({ embeddingModel: value })
}
@@ -162,11 +181,8 @@ export const IngestSettings = ({
</Tooltip>
<SelectContent>
<ModelSelectItems
- models={modelsData?.embedding_models}
- fallbackModels={
- getFallbackModels(currentProvider)
- .embedding as ModelOption[]
- }
+ models={embeddingSelectOptions}
+ fallbackModels={[]}
provider={currentProvider}
/>
</SelectContent>
diff --git a/frontend/components/cloud-picker/types.ts b/frontend/components/cloud-picker/types.ts
index a06f104aa..f3eea21f5 100644
--- a/frontend/components/cloud-picker/types.ts
+++ b/frontend/components/cloud-picker/types.ts
@@ -23,7 +23,10 @@ export interface UnifiedCloudPickerProps {
// OneDrive/SharePoint specific props
clientId?: string;
baseUrl?: string;
- // Ingest settings
+ /** Controlled ingest state from the upload page (single source of truth). */
+ ingestSettings?: IngestSettings;
+ onIngestSettingsChange?: (settings: IngestSettings) => void;
+ /** @deprecated Prefer ingestSettings + onIngestSettingsChange from the page. */
onSettingsChange?: (settings: IngestSettings) => void;
isIngesting: boolean;
}
diff --git a/frontend/components/cloud-picker/unified-cloud-picker.tsx b/frontend/components/cloud-picker/unified-cloud-picker.tsx
index 7f91225c0..c87d64a72 100644
--- a/frontend/components/cloud-picker/unified-cloud-picker.tsx
+++ b/frontend/components/cloud-picker/unified-cloud-picker.tsx
@@ -21,6 +21,8 @@ export const UnifiedCloudPicker = ({
onPickerStateChange,
clientId,
baseUrl,
+ ingestSettings: ingestSettingsProp,
+ onIngestSettingsChange,
onSettingsChange,
}: UnifiedCloudPickerProps) => {
const [isPickerLoaded, setIsPickerLoaded] = useState(false);
@@ -29,18 +31,28 @@ export const UnifiedCloudPicker = ({
const [isLoadingBaseUrl, setIsLoadingBaseUrl] = useState(false);
const [autoBaseUrl, setAutoBaseUrl] = useState<string | undefined>(undefined);
- // Settings state with defaults
- const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({
- chunkSize: 1000,
- chunkOverlap: 200,
- ocr: false,
- pictureDescriptions: false,
- embeddingModel: "text-embedding-3-small",
- });
-
- // Handle settings changes and notify parent
- const handleSettingsChange = (newSettings: IngestSettingsType) => {
- setIngestSettings(newSettings);
+ const isControlled =
+ ingestSettingsProp !== undefined && onIngestSettingsChange !== undefined;
+
+ const [localIngestSettings, setLocalIngestSettings] =
+ useState<IngestSettingsType>({
+ chunkSize: 1000,
+ chunkOverlap: 200,
+ ocr: false,
+ pictureDescriptions: false,
+ embeddingModel: "text-embedding-3-small",
+ });
+
+ const ingestSettings = isControlled
+ ? ingestSettingsProp!
+ : localIngestSettings;
+
+ const handleIngestSettingsChange = (newSettings: IngestSettingsType) => {
+ if (isControlled) {
+ onIngestSettingsChange!(newSettings);
+ } else {
+ setLocalIngestSettings(newSettings);
+ }
onSettingsChange?.(newSettings);
};
@@ -98,28 +110,11 @@ export const UnifiedCloudPicker = ({
]);
const handleAddFiles = () => {
- // === DIAGNOSTIC LOGGING ===
- console.log("=== UnifiedCloudPicker handleAddFiles ===");
- console.log("Provider:", provider);
- console.log("Client ID:", clientId);
- console.log("Base URL (from prop):", baseUrl);
- console.log("Auto Base URL:", autoBaseUrl);
- console.log("Effective Base URL:", effectiveBaseUrl);
- console.log("Is Picker Loaded:", isPickerLoaded);
- console.log("Has Access Token:", !!accessToken);
-
if (!isPickerLoaded || !accessToken) {
- console.error(
- "Cannot open picker: isPickerLoaded=",
- isPickerLoaded,
- "hasAccessToken=",
- !!accessToken,
- );
return;
}
if ((provider === "onedrive" || provider === "sharepoint") && !clientId) {
- console.error("Client ID required for OneDrive/SharePoint");
return;
}
@@ -216,7 +211,7 @@ export const UnifiedCloudPicker = ({
isOpen={isIngestSettingsOpen}
onOpenChange={setIsIngestSettingsOpen}
settings={ingestSettings}
- onSettingsChange={handleSettingsChange}
+ onSettingsChange={handleIngestSettingsChange}
/>
</div>
);
diff --git a/frontend/hooks/useSessionIngestSettings.ts b/frontend/hooks/useSessionIngestSettings.ts
new file mode 100644
index 000000000..2ab60cf8e
--- /dev/null
+++ b/frontend/hooks/useSessionIngestSettings.ts
@@ -0,0 +1,38 @@
+"use client";
+
+import {
+ type Dispatch,
+ type SetStateAction,
+ useEffect,
+ useRef,
+ useState,
+} from "react";
+import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";
+import type { IngestSettings } from "@/components/cloud-picker/types";
+import { useAuth } from "@/contexts/auth-context";
+import { knowledgeToIngestSettings } from "@/lib/ingest-settings-knowledge";
+
+/**
+ * Ingest form state: hydrate once from GET /api/settings `knowledge`, then session-owned.
+ */
+export function useSessionIngestSettings(): readonly [
+ IngestSettings,
+ Dispatch<SetStateAction<IngestSettings>>,
+] {
+ const { isAuthenticated, isNoAuthMode } = useAuth();
+ const { data, isSuccess } = useGetSettingsQuery({
+ enabled: isAuthenticated || isNoAuthMode,
+ });
+ const [ingestSettings, setIngestSettings] = useState<IngestSettings>(() =>
+ knowledgeToIngestSettings(undefined),
+ );
+ const hydrated = useRef(false);
+
+ useEffect(() => {
+ if (!isSuccess || !data || hydrated.current) return;
+ setIngestSettings(knowledgeToIngestSettings(data.knowledge));
+ hydrated.current = true;
+ }, [isSuccess, data]);
+
+ return [ingestSettings, setIngestSettings] as const;
+}
diff --git a/frontend/lib/ingest-settings-knowledge.ts b/frontend/lib/ingest-settings-knowledge.ts
new file mode 100644
index 000000000..ac87ccd44
--- /dev/null
+++ b/frontend/lib/ingest-settings-knowledge.ts
@@ -0,0 +1,20 @@
+import type { KnowledgeSettings } from "@/app/api/queries/useGetSettingsQuery";
+import type { IngestSettings } from "@/components/cloud-picker/types";
+import { DEFAULT_KNOWLEDGE_SETTINGS } from "@/lib/constants";
+
+/** Map saved Knowledge settings to ingest panel fields (subset of Knowledge UI). */
+export function knowledgeToIngestSettings(
+ knowledge: KnowledgeSettings | null | undefined,
+): IngestSettings {
+ return {
+ chunkSize: knowledge?.chunk_size ?? DEFAULT_KNOWLEDGE_SETTINGS.chunk_size,
+ chunkOverlap:
+ knowledge?.chunk_overlap ?? DEFAULT_KNOWLEDGE_SETTINGS.chunk_overlap,
+ ocr: knowledge?.ocr ?? DEFAULT_KNOWLEDGE_SETTINGS.ocr,
+ pictureDescriptions:
+ knowledge?.picture_descriptions ??
+ DEFAULT_KNOWLEDGE_SETTINGS.picture_descriptions,
+ embeddingModel:
+ knowledge?.embedding_model?.trim() || "text-embedding-3-small",
+ };
+}
diff --git a/src/api/connector_router.py b/src/api/connector_router.py
index 6af6c05c9..21ebfab2a 100644
--- a/src/api/connector_router.py
+++ b/src/api/connector_router.py
@@ -50,11 +50,22 @@ class ConnectorRouter:
return await self.get_active_service().get_connector(connection_id)
async def sync_specific_files(
- self, connection_id: str, user_id: str, file_list: list, jwt_token: str = None, file_infos: list = None
+ self,
+ connection_id: str,
+ user_id: str,
+ file_list: list,
+ jwt_token: str = None,
+ file_infos: list = None,
+ ingest_settings: dict = None,
):
"""Sync specific files using the active service."""
return await self.get_active_service().sync_specific_files(
- connection_id, user_id, file_list, jwt_token, file_infos=file_infos
+ connection_id,
+ user_id,
+ file_list,
+ jwt_token,
+ file_infos=file_infos,
+ ingest_settings=ingest_settings,
)
def __getattr__(self, name):
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 4b393137e..c8447097c 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
from fastapi import Depends, Request
from pydantic import BaseModel
@@ -93,6 +93,8 @@ class ConnectorSyncBody(BaseModel):
sync_all: bool = False
# When set, only ingest files from these buckets (IBM COS specific).
bucket_filter: Optional[List[str]] = None
+ # Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).
+ settings: Optional[Dict[str, Any]] = None
async def list_connectors(
@@ -204,6 +206,7 @@ async def connector_sync(
selected_files,
jwt_token=jwt_token,
file_infos=file_infos,
+ ingest_settings=body.settings,
)
elif body.sync_all or body.bucket_filter:
# Full ingest: discover and ingest all files (or files from specific buckets).
@@ -241,6 +244,7 @@ async def connector_sync(
user.user_id,
all_file_ids,
jwt_token=jwt_token,
+ ingest_settings=body.settings,
)
else:
# sync_all: ingest everything the connector can see
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 1c981ba5a..efe00c1d0 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -43,6 +43,7 @@ class LangflowConnectorService:
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
+ ingest_settings: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Process a document from a connector using LangflowFileService pattern"""
@@ -107,8 +108,16 @@ class LangflowConnectorService:
"Running Langflow ingestion flow", file_path=langflow_file_path
)
- # Use the same tweaks pattern as LangflowFileService
- tweaks = {} # Let Langflow handle the ingestion with default settings
+ connector_tweak_settings = None
+ if isinstance(ingest_settings, dict):
+ connector_tweak_settings = dict(ingest_settings)
+ # Model selection is injected via selected_embedding_model header override.
+ # Avoid hard-coding provider-specific embedding component tweak IDs here.
+ connector_tweak_settings.pop("embeddingModel", None)
+
+ tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ {}, connector_tweak_settings
+ )
# Extract ACL information from the connector document, if available
allowed_users: list[str] = []
@@ -135,6 +144,11 @@ class LangflowConnectorService:
source_url=document.source_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ selected_embedding_model=(
+ ingest_settings.get("embeddingModel")
+ if isinstance(ingest_settings, dict)
+ else None
+ ),
)
logger.debug("Ingestion flow completed", result=ingestion_result)
@@ -279,6 +293,7 @@ class LangflowConnectorService:
file_ids: List[str],
jwt_token: str = None,
file_infos: List[Dict[str, Any]] = None,
+ ingest_settings: Optional[Dict[str, Any]] = None,
) -> str:
"""
Sync specific files by their IDs using Langflow processing.
@@ -376,6 +391,7 @@ class LangflowConnectorService:
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
+ ingest_settings=ingest_settings,
)
# Create custom task using TaskService
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 1e8dc7b0f..df3a2e3f5 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -309,6 +309,7 @@ class ConnectorService:
file_ids: List[str],
jwt_token: str = None,
file_infos: List[Dict[str, Any]] = None,
+ ingest_settings: Optional[Dict[str, Any]] = None,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
@@ -321,6 +322,8 @@ class ConnectorService:
jwt_token: Optional JWT token for authentication
file_infos: Optional list of file info dicts with {id, name, mimeType, downloadUrl, size}
When provided, download URLs can be used directly without Graph API calls.
+ ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …) passed to
+ ``ConnectorFileProcessor`` when Langflow ingest is disabled.
"""
if not self.task_service:
raise ValueError(
@@ -415,6 +418,7 @@ class ConnectorService:
else DocumentService(session_manager=self.session_manager)
),
models_service=self.models_service,
+ ingest_settings=ingest_settings,
)
# Create custom task using TaskService
diff --git a/src/models/processors.py b/src/models/processors.py
index 1e5ae886b..86df12194 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Dict, Optional
from .tasks import UploadTask, FileTask
from utils.logging_config import get_logger
from utils.file_utils import (
@@ -187,6 +187,8 @@ class TaskProcessor:
file_size: int = None,
connector_type: str = "local",
embedding_model: str = None,
+ chunk_size: int = None,
+ chunk_overlap: int = None,
is_sample_data: bool = False,
acl: "DocumentACL" = None,
):
@@ -197,6 +199,9 @@ class TaskProcessor:
Args:
embedding_model: Embedding model to use (defaults to the current
embedding model from settings)
+ chunk_size: Optional character window size for re-splitting extracted
+ chunks (non-Langflow path, e.g. connector UI ``chunkSize``).
+ chunk_overlap: Overlap between windows; must be less than ``chunk_size``.
acl: DocumentACL instance with access control information
"""
import datetime
@@ -207,7 +212,10 @@ class TaskProcessor:
get_openrag_config,
)
from services.document_service import chunk_texts_for_embeddings
- from utils.document_processing import extract_relevant
+ from utils.document_processing import (
+ extract_relevant,
+ resplit_chunks_character_windows,
+ )
from utils.embedding_fields import ensure_embedding_field_exists
# Use provided embedding model or configured model.
@@ -258,6 +266,25 @@ class TaskProcessor:
full_doc = await convert_file(file_path, httpx_client=clients.docling_http_client)
slim_doc = extract_relevant(full_doc)
+ if chunk_size is not None:
+ try:
+ cs = int(chunk_size)
+ except (TypeError, ValueError):
+ cs = 0
+ if cs > 0:
+ try:
+ co = (
+ int(chunk_overlap)
+ if chunk_overlap is not None
+ else 0
+ )
+ except (TypeError, ValueError):
+ co = 0
+ if co < cs:
+ slim_doc["chunks"] = resplit_chunks_character_windows(
+ slim_doc["chunks"], cs, max(0, co)
+ )
+
texts = [c["text"] for c in slim_doc["chunks"]]
litellm_embedding_model = await self.models_service.get_litellm_model_name(embedding_model) if self.models_service is not None else embedding_model
@@ -455,6 +482,7 @@ class ConnectorFileProcessor(TaskProcessor):
owner_email: str = None,
document_service=None,
models_service=None,
+ ingest_settings: Optional[Dict[str, Any]] = None,
):
super().__init__(document_service=document_service, models_service=models_service)
self.connector_service = connector_service
@@ -464,6 +492,7 @@ class ConnectorFileProcessor(TaskProcessor):
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
+ self.ingest_settings = ingest_settings
async def process_item(
self, upload_task: UploadTask, item: str, file_task: FileTask
@@ -510,6 +539,23 @@ class ConnectorFileProcessor(TaskProcessor):
# Compute hash
file_hash = hash_id(tmp_path)
+ standard_kwargs: Dict[str, Any] = {}
+ if isinstance(self.ingest_settings, dict):
+ s = self.ingest_settings
+ em = s.get("embeddingModel")
+ if isinstance(em, str) and em.strip():
+ standard_kwargs["embedding_model"] = em.strip()
+ for ui_key, param in (
+ ("chunkSize", "chunk_size"),
+ ("chunkOverlap", "chunk_overlap"),
+ ):
+ raw = s.get(ui_key)
+ if raw is not None:
+ try:
+ standard_kwargs[param] = int(raw)
+ except (TypeError, ValueError):
+ pass
+
# Use consolidated standard processing
result = await self.process_document_standard(
file_path=tmp_path,
@@ -522,6 +568,7 @@ class ConnectorFileProcessor(TaskProcessor):
file_size=len(document.content),
connector_type=connection.connector_type,
acl=document.acl,
+ **standard_kwargs,
)
# Add connector-specific metadata
@@ -555,6 +602,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
+ ingest_settings: Optional[Dict[str, Any]] = None,
):
super().__init__()
self.langflow_connector_service = langflow_connector_service
@@ -564,6 +612,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
+ self.ingest_settings = ingest_settings
async def process_item(
self, upload_task: UploadTask, item: str, file_task: FileTask
@@ -633,6 +682,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
+ ingest_settings=self.ingest_settings,
)
file_task.status = TaskStatus.COMPLETED
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index cea9efd2a..44bc95d4d 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -33,6 +33,43 @@ class LangflowFileService:
),
)
+ @staticmethod
+ def merge_ui_ingest_settings_into_tweaks(
+ tweaks: Optional[Dict[str, Any]],
+ settings: Optional[Dict[str, Any]],
+ ) -> Dict[str, Any]:
+ """Merge UI ingest dict (camelCase) into Langflow run ``tweaks``.
+
+ - ``chunkSize`` / ``chunkOverlap`` / ``separator`` update the flow's
+ ``SplitText-QIKhg`` node when any of those keys are present.
+ - ``embeddingModel`` is intentionally not mapped to a component tweak.
+ The embedding model should be supplied via
+ ``run_ingestion_flow(..., selected_embedding_model=...)`` so Langflow
+ resolves it through the global variable override, without relying on
+ provider-specific component ids.
+ """
+ final_tweaks = dict(tweaks) if tweaks else {}
+ if not settings:
+ return final_tweaks
+
+ if (
+ settings.get("chunkSize")
+ or settings.get("chunkOverlap")
+ or settings.get("separator")
+ ):
+ if "SplitText-QIKhg" not in final_tweaks:
+ final_tweaks["SplitText-QIKhg"] = {}
+ if settings.get("chunkSize"):
+ final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings["chunkSize"]
+ if settings.get("chunkOverlap"):
+ final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[
+ "chunkOverlap"
+ ]
+ if settings.get("separator"):
+ final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]
+
+ return final_tweaks
+
async def upload_user_file(
self, file_tuple, jwt_token: Optional[str] = None
) -> Dict[str, Any]:
@@ -95,6 +132,7 @@ class LangflowFileService:
source_url: Optional[str] = None,
allowed_users: Optional[List[str]] = None,
allowed_groups: Optional[List[str]] = None,
+ selected_embedding_model: Optional[str] = None,
) -> Dict[str, Any]:
"""
Trigger the ingestion flow with provided file paths.
@@ -156,6 +194,8 @@ class LangflowFileService:
config = get_openrag_config()
embedding_model = config.knowledge.embedding_model
+ if selected_embedding_model:
+ embedding_model = selected_embedding_model
headers = {
"X-Langflow-Global-Var-JWT": str(jwt_token),
@@ -514,44 +554,13 @@ class LangflowFileService:
if not file_path:
raise ValueError("Upload successful but no file path returned")
- # Convert UI settings to component tweaks if provided
- final_tweaks = tweaks.copy() if tweaks else {}
-
+ final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ tweaks, settings
+ )
if settings:
logger.debug(
- "[LF] Applying ingestion settings", extra={"settings": settings}
- )
-
- # Split Text component tweaks (SplitText-QIKhg)
- if (
- settings.get("chunkSize")
- or settings.get("chunkOverlap")
- or settings.get("separator")
- ):
- if "SplitText-QIKhg" not in final_tweaks:
- final_tweaks["SplitText-QIKhg"] = {}
- if settings.get("chunkSize"):
- final_tweaks["SplitText-QIKhg"]["chunk_size"] = settings[
- "chunkSize"
- ]
- if settings.get("chunkOverlap"):
- final_tweaks["SplitText-QIKhg"]["chunk_overlap"] = settings[
- "chunkOverlap"
- ]
- if settings.get("separator"):
- final_tweaks["SplitText-QIKhg"]["separator"] = settings["separator"]
-
- # OpenAI Embeddings component tweaks (OpenAIEmbeddings-joRJ6)
- if settings.get("embeddingModel"):
- if "OpenAIEmbeddings-joRJ6" not in final_tweaks:
- final_tweaks["OpenAIEmbeddings-joRJ6"] = {}
- final_tweaks["OpenAIEmbeddings-joRJ6"]["model"] = settings[
- "embeddingModel"
- ]
-
- logger.debug(
- "[LF] Final tweaks with settings applied",
- extra={"tweaks": final_tweaks},
+ "[LF] Applying ingestion settings",
+ extra={"settings": settings, "tweaks": final_tweaks},
)
# Step 3: Run ingestion
diff --git a/src/utils/document_processing.py b/src/utils/document_processing.py
index 2ecaf68ff..0ca02bfa4 100644
--- a/src/utils/document_processing.py
+++ b/src/utils/document_processing.py
@@ -141,3 +141,42 @@ def extract_relevant(doc_dict: dict) -> dict:
"mimetype": origin.get("mimetype"),
"chunks": chunks,
}
+
+
+def resplit_chunks_character_windows(
+ chunks: list,
+ chunk_size: int,
+ chunk_overlap: int,
+) -> list:
+ """Split long chunk texts into fixed-size character windows with overlap.
+
+ Used by the non-Langflow ingestion path when the UI supplies ``chunkSize`` /
+ ``chunkOverlap``. Invalid combinations (non-positive size or overlap >= size)
+ return ``chunks`` unchanged.
+ """
+ if chunk_size <= 0 or chunk_overlap >= chunk_size:
+ return chunks
+ stride = chunk_size - chunk_overlap
+ if stride <= 0:
+ return chunks
+
+ out: list = []
+ for ch in chunks:
+ text = ch.get("text") if isinstance(ch, dict) else None
+ if not isinstance(text, str):
+ out.append(dict(ch) if isinstance(ch, dict) else ch)
+ continue
+ if len(text) <= chunk_size:
+ out.append(dict(ch))
+ continue
+ start = 0
+ while start < len(text):
+ end = min(start + chunk_size, len(text))
+ piece = text[start:end]
+ new_ch = dict(ch)
+ new_ch["text"] = piece
+ out.append(new_ch)
+ if end >= len(text):
+ break
+ start += stride
+ return out
diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.py
new file mode 100644
index 000000000..bfb6b3254
--- /dev/null
+++ b/tests/unit/test_document_processing_resplit.py
@@ -0,0 +1,37 @@
+"""Tests for resplit_chunks_character_windows."""
+
+from src.utils.document_processing import resplit_chunks_character_windows
+
+
+def test_invalid_returns_original():
+ chunks = [{"page": 1, "text": "hello world" * 50}]
+ assert resplit_chunks_character_windows(chunks, 0, 0) is chunks
+ assert resplit_chunks_character_windows(chunks, 10, 10) is chunks
+ assert resplit_chunks_character_windows(chunks, 5, 6) is chunks
+
+
+def test_non_dict_chunk_passthrough():
+ chunks = ["not-a-dict", {"page": 1, "text": "x" * 30}]
+ out = resplit_chunks_character_windows(chunks, 10, 2)
+ assert out[0] == "not-a-dict"
+ assert len(out) == 1 + 4 # one long dict chunk -> 4 windows
+
+
+def test_short_text_unchanged_count():
+ chunks = [{"page": 2, "type": "text", "text": "short"}]
+ out = resplit_chunks_character_windows(chunks, 100, 20)
+ assert len(out) == 1
+ assert out[0]["text"] == "short"
+ assert out[0]["page"] == 2
+
+
+def test_long_text_windows_with_overlap():
+ # 22 chars, size 10, overlap 4 -> stride 6; third window reaches end
+ text = "abcdefghijklmnopqrstuv"
+ assert len(text) == 22
+ chunks = [{"page": 1, "text": text}]
+ out = resplit_chunks_character_windows(chunks, 10, 4)
+ assert len(out) == 3
+ assert out[0]["text"] == "abcdefghij"
+ assert out[1]["text"] == "ghijklmnop"
+ assert out[2]["text"] == "mnopqrstuv"
diff --git a/tests/unit/test_langflow_file_service_merge_tweaks.py b/tests/unit/test_langflow_file_service_merge_tweaks.py
new file mode 100644
index 000000000..bf5c2e321
--- /dev/null
+++ b/tests/unit/test_langflow_file_service_merge_tweaks.py
@@ -0,0 +1,68 @@
+"""Unit tests for LangflowFileService.merge_ui_ingest_settings_into_tweaks."""
+
+from src.services.langflow_file_service import LangflowFileService
+
+
+def test_merge_no_settings_returns_tweaks_copy():
+ base = {"OtherNode": {"x": 1}}
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(base, None)
+ assert out == base
+ assert out is not base
+
+
+def test_merge_empty_settings_returns_tweaks_only():
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(None, {})
+ assert out == {}
+
+
+def test_merge_chunk_fields_populate_split_text():
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ None,
+ {"chunkSize": 512, "chunkOverlap": 64, "separator": "\n\n"},
+ )
+ assert out["SplitText-QIKhg"] == {
+ "chunk_size": 512,
+ "chunk_overlap": 64,
+ "separator": "\n\n",
+ }
+
+
+def test_merge_chunk_partial_only_sets_provided_keys():
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ None,
+ {"chunkSize": 1000},
+ )
+ assert out["SplitText-QIKhg"] == {"chunk_size": 1000}
+
+
+def test_merge_preserves_and_extends_existing_split_text():
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ {"SplitText-QIKhg": {"chunk_size": 100, "existing": "keep"}},
+ {"chunkOverlap": 20},
+ )
+ assert out["SplitText-QIKhg"] == {
+ "chunk_size": 100,
+ "existing": "keep",
+ "chunk_overlap": 20,
+ }
+
+
+def test_merge_embedding_model_is_ignored_in_tweaks():
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks(
+ None,
+ {"embeddingModel": "text-embedding-3-large"},
+ )
+ assert out == {}
+
+
+def test_connector_style_settings_without_embedding_only_split_text():
+ """Embedding model is not mapped to tweaks; split settings still apply."""
+ settings = {
+ "chunkSize": 800,
+ "chunkOverlap": 100,
+ "ocr": True,
+ "embeddingModel": "ignored",
+ }
+ out = LangflowFileService.merge_ui_ingest_settings_into_tweaks({}, settings)
+ assert "OpenAIEmbeddings-joRJ6" not in out
+ assert out["SplitText-QIKhg"]["chunk_size"] == 800