OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: connector sync and override feature (#1663)
Edwin Jose748583bdec39
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsindex e40508211..214133429 100644--- a/frontend/app/api/mutations/useSyncConnector.ts+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -72,6 +72,8 @@ const syncConnector = async ({sync_all?: boolean;/** Restrict ingest to these bucket names (IBM COS). */bucket_filter?: string[];+ /** When true, replace any indexed document with the same filename. */+ replace_duplicates?: boolean;};}): Promise<SyncResponse> => {const response = await fetch(`/api/connectors/${connectorType}/sync`, {diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex eae4389b5..a4112d86c 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -9,7 +9,7 @@ import {RefreshCw,} from "lucide-react";import { useParams, useRouter } from "next/navigation";-import { useState } from "react";+import { useRef, useState } from "react";import { toast } from "sonner";import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";@@ -19,6 +19,7 @@ import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQueryimport { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";import { IngestSettings } from "@/components/cloud-picker/ingest-settings";import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";+import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog";import { FileBrowserDialog } from "@/components/file-browser-dialog";import { Button } from "@/components/ui/button";import {@@ -28,6 +29,7 @@ import {} from "@/components/ui/tooltip";import { useTask } from "@/contexts/task-context";import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";+import { duplicateCheck } from "@/lib/upload-utils";// Connectors that sync entire buckets/repositories without a file pickerconst DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];@@ -410,6 +412,15 @@ export default function UploadProviderPage() {const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);const [ingestSettings, setIngestSettings] = useSessionIngestSettings();+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);+ const [pendingSync, setPendingSync] = useState<{+ connector: { connectionId?: string; type: string };+ allFiles: CloudFile[];+ nonDuplicateFiles: CloudFile[];+ duplicateNames: string[];+ } | null>(null);+ const isOverwriteConfirmedRef = useRef(false);const accessToken = tokenData?.access_token || null;const isLoading =@@ -427,21 +438,17 @@ export default function UploadProviderPage() {setSelectedFiles(files);};- const handleSync = (connector: { connectionId?: string; type: string }) => {- if (!connector.connectionId || selectedFiles.length === 0) return;-- const chunkErr = getIngestChunkSettingsError(ingestSettings);- if (chunkErr) {- toast.error("Could not start ingest", { description: chunkErr });- return;- }-+ const submitSync = (+ connector: { connectionId?: string; type: string },+ files: CloudFile[],+ replaceDuplicates: boolean,+ ) => {syncMutation.mutate({connectorType: connector.type,body: {connection_id: connector.connectionId,- selected_files: selectedFiles.map((file) => ({+ selected_files: files.map((file) => ({id: file.id,name: file.name,mimeType: file.mimeType,@@ -450,6 +457,7 @@ export default function UploadProviderPage() {isFolder: file.isFolder,})),settings: ingestSettings,+ replace_duplicates: replaceDuplicates,},},{@@ -467,6 +475,89 @@ export default function UploadProviderPage() {);};+ const handleSync = async (connector: {+ connectionId?: string;+ type: string;+ }) => {+ if (!connector.connectionId || selectedFiles.length === 0) return;++ const chunkErr = getIngestChunkSettingsError(ingestSettings);+ if (chunkErr) {+ toast.error("Could not start ingest", { description: chunkErr });+ return;+ }++ setIsCheckingDuplicates(true);+ try {+ const results = await Promise.all(+ selectedFiles.map(async (file) => {+ if (file.isFolder) return { file, isDuplicate: false };+ try {+ const fakeFile = new File([], file.name);+ const { exists } = await duplicateCheck(fakeFile);+ return { file, isDuplicate: exists };+ } catch (err) {+ console.error(+ `[Connector Sync] Duplicate check failed for ${file.name}:`,+ err,+ );+ return { file, isDuplicate: false };+ }+ }),+ );++ const duplicates = results.filter((r) => r.isDuplicate);+ const nonDuplicates = results+ .filter((r) => !r.isDuplicate)+ .map((r) => r.file);++ if (duplicates.length === 0) {+ submitSync(connector, selectedFiles, false);+ return;+ }++ setPendingSync({+ connector,+ allFiles: selectedFiles,+ nonDuplicateFiles: nonDuplicates,+ duplicateNames: duplicates.map((r) => r.file.name),+ });+ setDuplicateDialogOpen(true);+ } finally {+ setIsCheckingDuplicates(false);+ }+ };++ const handleOverwriteDuplicates = () => {+ if (!pendingSync) return;+ isOverwriteConfirmedRef.current = true;+ const { connector, allFiles } = pendingSync;+ submitSync(connector, allFiles, true);+ setPendingSync(null);+ };++ const handleDuplicateDialogOpenChange = (open: boolean) => {+ if (!open && pendingSync) {+ if (isOverwriteConfirmedRef.current) {+ // Overwrite already submitted in handleOverwriteDuplicates; this close+ // event fires immediately after and would otherwise re-enter the+ // "skip duplicates" branch.+ isOverwriteConfirmedRef.current = false;+ } else {+ const { connector, nonDuplicateFiles, duplicateNames } = pendingSync;+ if (nonDuplicateFiles.length > 0) {+ submitSync(connector, nonDuplicateFiles, false);+ } else {+ toast.info(+ `All ${duplicateNames.length} selected file(s) already exist. Nothing was synced.`,+ );+ }+ }+ setPendingSync(null);+ }+ setDuplicateDialogOpen(open);+ };+const getProviderDisplayName = () => {const nameMap: { [key: string]: string } = {google_drive: "Google Drive",@@ -652,8 +743,10 @@ export default function UploadProviderPage() {className="bg-foreground text-background hover:bg-foreground/90 font-semibold"variant={!hasSelectedFiles ? "secondary" : undefined}onClick={() => handleSync(connector)}- loading={isIngesting}- disabled={!hasSelectedFiles || isIngesting}+ loading={isIngesting || isCheckingDuplicates}+ disabled={+ !hasSelectedFiles || isIngesting || isCheckingDuplicates+ }>{hasSelectedFiles ? (<>@@ -673,6 +766,14 @@ export default function UploadProviderPage() {</Tooltip></div></div>++ <DuplicateHandlingDialog+ open={duplicateDialogOpen}+ onOpenChange={handleDuplicateDialogOpenChange}+ onOverwrite={handleOverwriteDuplicates}+ isLoading={isIngesting}+ duplicateNames={pendingSync?.duplicateNames}+ /></>);}diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsxindex a19faabc4..76b5a84c9 100644--- a/frontend/components/duplicate-handling-dialog.tsx+++ b/frontend/components/duplicate-handling-dialog.tsx@@ -19,8 +19,11 @@ interface DuplicateHandlingDialogProps {isLoading?: boolean;duplicateLabel?: string;duplicateCount?: number;+ duplicateNames?: string[];}+const MAX_LISTED_DUPLICATES = 5;+export const DuplicateHandlingDialog: React.FC<DuplicateHandlingDialogProps> = ({@@ -30,27 +33,40 @@ export const DuplicateHandlingDialog: React.FC<isLoading = false,duplicateLabel,duplicateCount,+ duplicateNames,}) => {const handleOverwrite = async () => {await onOverwrite();onOpenChange(false);};+ const namesProvided = duplicateNames && duplicateNames.length > 0;+ const effectiveCount = namesProvided+ ? duplicateNames!.length+ : duplicateCount;+const description =- typeof duplicateCount === "number"- ? duplicateCount === 1+ typeof effectiveCount === "number"+ ? effectiveCount === 1? "1 duplicate document already exists. Overwriting will replace the existing document version. This can't be undone."- : `${duplicateCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`+ : `${effectiveCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`: duplicateLabel? `A document named "${duplicateLabel}" already exists. Overwriting will replace the existing document version. This can't be undone.`: "Overwriting will replace the existing document with another version. This can't be undone.";const overwriteLabel =- typeof duplicateCount === "number" ? "Overwrite duplicates" : "Overwrite";+ typeof effectiveCount === "number" ? "Overwrite duplicates" : "Overwrite";const cancelLabel =- typeof duplicateCount === "number"+ typeof effectiveCount === "number"? "Skip duplicates & continue": "Cancel";+ const visibleNames = namesProvided+ ? duplicateNames!.slice(0, MAX_LISTED_DUPLICATES)+ : [];+ const remainingCount = namesProvided+ ? duplicateNames!.length - visibleNames.length+ : 0;+return (<Dialog open={open} onOpenChange={onOpenChange}><DialogContent className="sm:max-w-[520px]">@@ -61,6 +77,19 @@ export const DuplicateHandlingDialog: React.FC<</DialogDescription></DialogHeader>+ {namesProvided && (+ <ul className="text-sm text-muted-foreground list-disc pl-5 space-y-0.5">+ {visibleNames.map((name) => (+ <li key={name} className="break-all">+ {name}+ </li>+ ))}+ {remainingCount > 0 && (+ <li className="list-none italic">… and {remainingCount} more</li>+ )}+ </ul>+ )}+<DialogFooter className="flex-row gap-2 justify-end"><Buttontype="button"diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsxindex b858bcfa0..687c8eef5 100644--- a/frontend/components/knowledge-dropdown.tsx+++ b/frontend/components/knowledge-dropdown.tsx@@ -142,7 +142,7 @@ export function KnowledgeDropdown() {const [pendingFolderUpload, setPendingFolderUpload] = useState<{allFiles: File[];nonDuplicateFiles: File[];- duplicateCount: number;+ duplicateNames: string[];unsupportedCount: number;} | null>(null);const isFolderOverwriteConfirmedRef = useRef(false);@@ -382,13 +382,13 @@ export function KnowledgeDropdown() {const handleOverwriteFile = async () => {if (pendingFolderUpload) {isFolderOverwriteConfirmedRef.current = true;- const { allFiles, duplicateCount, unsupportedCount } =+ const { allFiles, duplicateNames, unsupportedCount } =pendingFolderUpload;await uploadFolderBatches(allFiles, true);const unsupportedMessage =unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";toast.success(- `Processed ${allFiles.length} file(s), including ${duplicateCount} overwrite(s)${unsupportedMessage}`,+ `Processed ${allFiles.length} file(s), including ${duplicateNames.length} overwrite(s)${unsupportedMessage}`,);resetDuplicateDialogState();return;@@ -427,13 +427,13 @@ export function KnowledgeDropdown() {if (isFolderOverwriteConfirmedRef.current) {isFolderOverwriteConfirmedRef.current = false;} else {- const { nonDuplicateFiles, duplicateCount, unsupportedCount } =+ const { nonDuplicateFiles, duplicateNames, unsupportedCount } =pendingFolderUpload;if (nonDuplicateFiles.length > 0) {await uploadFolderBatches(nonDuplicateFiles, false);const extraParts: string[] = [];- if (duplicateCount > 0) {- extraParts.push(`skipped ${duplicateCount} duplicate(s)`);+ if (duplicateNames.length > 0) {+ extraParts.push(`skipped ${duplicateNames.length} duplicate(s)`);}if (unsupportedCount > 0) {extraParts.push(`skipped ${unsupportedCount} unsupported`);@@ -515,9 +515,10 @@ export function KnowledgeDropdown() {const nonDuplicateFiles = duplicateResults.filter((r) => !r.isDuplicate).map((r) => r.file);- const duplicateCount = duplicateResults.filter(- (r) => r.isDuplicate,- ).length;+ const duplicateNames = duplicateResults+ .filter((r) => r.isDuplicate)+ .map((r) => r.file.name);+ const duplicateCount = duplicateNames.length;if (unsupportedCount > 0) {toast.error(@@ -536,7 +537,7 @@ export function KnowledgeDropdown() {setPendingFolderUpload({allFiles: cleanFiles,nonDuplicateFiles,- duplicateCount,+ duplicateNames,unsupportedCount,});setShowDuplicateDialog(true);@@ -856,7 +857,7 @@ export function KnowledgeDropdown() {onOverwrite={handleOverwriteFile}isLoading={fileUploading || folderLoading}duplicateLabel={duplicateFilename}- duplicateCount={pendingFolderUpload?.duplicateCount}+ duplicateNames={pendingFolderUpload?.duplicateNames}/></>);diff --git a/src/api/connector_router.py b/src/api/connector_router.pyindex 21ebfab2a..e8bf8228e 100644--- a/src/api/connector_router.py+++ b/src/api/connector_router.py@@ -57,6 +57,7 @@ class ConnectorRouter:jwt_token: str = None,file_infos: list = None,ingest_settings: dict = None,+ replace_duplicates: bool = False,):"""Sync specific files using the active service."""return await self.get_active_service().sync_specific_files(@@ -66,6 +67,7 @@ class ConnectorRouter:jwt_token,file_infos=file_infos,ingest_settings=ingest_settings,+ replace_duplicates=replace_duplicates,)def __getattr__(self, name):diff --git a/src/api/connectors.py b/src/api/connectors.pyindex ea31c828d..309158337 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -294,6 +294,10 @@ class ConnectorSyncBody(BaseModel):bucket_filter: list[str] | None = None# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).settings: dict[str, Any] | None = None+ # When True, files whose filename already exists in the index are replaced+ # rather than failing. Set by the provider upload UI after the user confirms+ # overwrite in the duplicate dialog.+ replace_duplicates: bool = Falseasync def list_connectors(@@ -409,6 +413,7 @@ async def connector_sync(jwt_token=jwt_token,file_infos=file_infos,ingest_settings=body.settings,+ replace_duplicates=body.replace_duplicates,)elif body.sync_all or body.bucket_filter:# Full ingest: discover and ingest all files (or files from specific buckets).diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 5cd2716de..3f2ba93a0 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -317,6 +317,7 @@ class LangflowConnectorService:jwt_token: str = None,file_infos: list[dict[str, Any]] = None,ingest_settings: dict[str, Any] | None = None,+ replace_duplicates: bool = False,) -> str:"""Sync specific files by their IDs using Langflow processing.@@ -413,6 +414,7 @@ class LangflowConnectorService:owner_name=owner_name,owner_email=owner_email,ingest_settings=ingest_settings,+ replace_duplicates=replace_duplicates,)# Create custom task using TaskServicediff --git a/src/connectors/service.py b/src/connectors/service.pyindex 146c70116..ed9b385b0 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -318,6 +318,7 @@ class ConnectorService:jwt_token: str = None,file_infos: list[dict[str, Any]] = None,ingest_settings: dict[str, Any] | None = None,+ replace_duplicates: bool = False,) -> str:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).@@ -442,6 +443,7 @@ class ConnectorService:),models_service=self.models_service,ingest_settings=ingest_settings,+ replace_duplicates=replace_duplicates,)# Create custom task using TaskServicediff --git a/src/models/processors.py b/src/models/processors.pyindex 02771678a..05ffaf4a1 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -492,6 +492,7 @@ class ConnectorFileProcessor(TaskProcessor):document_service=None,models_service=None,ingest_settings: dict[str, Any] | None = None,+ replace_duplicates: bool = False,):super().__init__(document_service=document_service,@@ -506,6 +507,7 @@ class ConnectorFileProcessor(TaskProcessor):self.owner_name = owner_nameself.owner_email = owner_emailself.ingest_settings = ingest_settings+ self.replace_duplicates = replace_duplicatesasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using consolidated methods"""@@ -548,6 +550,18 @@ class ConnectorFileProcessor(TaskProcessor):if not self.user_id:raise ValueError("user_id not provided to ConnectorFileProcessor")+ opensearch_client = self.document_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token+ )+ if await self.check_filename_exists(document.filename, opensearch_client):+ if not self.replace_duplicates:+ file_task.status = TaskStatus.FAILED+ file_task.error = f"File with name '{document.filename}' already exists"+ file_task.updated_at = time.time()+ upload_task.failed_files += 1+ return+ await self.delete_document_by_filename(document.filename, opensearch_client)+# Create temporary file from document contentsuffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:@@ -624,6 +638,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):owner_name: str = None,owner_email: str = None,ingest_settings: dict[str, Any] | None = None,+ replace_duplicates: bool = False,):super().__init__(document_service=langflow_connector_service.task_service.document_service@@ -642,6 +657,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):self.owner_name = owner_nameself.owner_email = owner_emailself.ingest_settings = ingest_settings+ self.replace_duplicates = replace_duplicatesasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using LangflowConnectorService"""@@ -668,6 +684,20 @@ class LangflowConnectorFileProcessor(TaskProcessor):if not self.user_id:raise ValueError("user_id not provided to LangflowConnectorFileProcessor")+ opensearch_client = (+ self.langflow_connector_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token+ )+ )+ if await self.check_filename_exists(document.filename, opensearch_client):+ if not self.replace_duplicates:+ file_task.status = TaskStatus.FAILED+ file_task.error = f"File with name '{document.filename}' already exists"+ file_task.updated_at = time.time()+ upload_task.failed_files += 1+ return+ await self.delete_document_by_filename(document.filename, opensearch_client)+# Create temporary file and compute hash to check for duplicatessuffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:@@ -678,12 +708,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):# Compute hash and check if already existsfile_hash = hash_id(tmp_path)- # Check if document already exists- opensearch_client = (- self.langflow_connector_service.session_manager.get_user_opensearch_client(- self.user_id, self.jwt_token- )- )if await self.check_document_exists(file_hash, opensearch_client):file_task.status = TaskStatus.COMPLETEDfile_task.result = {"status": "unchanged", "id": file_hash}diff --git a/src/utils/file_utils.py b/src/utils/file_utils.pyindex 989875b0f..f2a50dfe2 100644--- a/src/utils/file_utils.py+++ b/src/utils/file_utils.py@@ -3,11 +3,12 @@import osimport tempfilefrom contextlib import contextmanager-from typing import Optional@contextmanager-def auto_cleanup_tempfile(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None):+def auto_cleanup_tempfile(+ suffix: str | None = None, prefix: str | None = None, dir: str | None = None+):"""Context manager for temporary files that automatically cleans up.@@ -85,20 +86,20 @@ def get_file_extension(mimetype: str) -> str:def clean_connector_filename(filename: str, mimetype: str) -> str:- """Clean filename and ensure correct extension.+ """Ensure the filename ends with the extension that matches its MIME type.- If the MIME type maps to a known extension, it is enforced.- If the MIME type is unknown, the original filename (and its extension) is kept as-is- rather than appending a meaningless .bin suffix.+ The original name is preserved verbatim (spaces, slashes, casing) so that+ connector-indexed filenames match what the user sees in the source system+ and what a local upload of the same file would index as. Only the+ extension is enforced, and only when the MIME type maps to one — for an+ unknown MIME type, the original filename and extension are kept."""- clean_name = filename.replace(" ", "_").replace("/", "_")suffix = get_file_extension(mimetype)if suffix is None:- # Unknown type — keep whatever extension the file already has- return clean_name- if not clean_name.lower().endswith(suffix.lower()):- return clean_name + suffix- return clean_name+ return filename+ if not filename.lower().endswith(suffix.lower()):+ return filename + suffix+ return filenamedef get_filename_aliases(filename: str) -> list[str]:@@ -125,5 +126,9 @@ def get_filename_aliases(filename: str) -> list[str]:elif lower_name.endswith(".md"):aliases.append(normalized[:-3] + ".txt")+ # Mirror clean_connector_filename's space/slash -> underscore so lookups also+ # match files indexed through a connector ingestion path.+ aliases.extend(name.replace(" ", "_").replace("/", "_") for name in list(aliases))+# Keep order stable while removing duplicates.return list(dict.fromkeys(aliases))diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pynew file mode 100644index 000000000..4d167dc42--- /dev/null+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -0,0 +1,250 @@+"""Filename-based dedupe gate on connector ingest paths.++Covers the symmetric behavior introduced so that picking a file from the+SharePoint UI when a same-named document already exists triggers the+"Overwrite" dialog (frontend) or fails the file task (backend, when+``replace_duplicates`` is not set).+"""++from datetime import datetime+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from connectors.base import ConnectorDocument, DocumentACL+from models.processors import ConnectorFileProcessor, LangflowConnectorFileProcessor+from models.tasks import FileTask, TaskStatus, UploadTask+++def _make_document(filename: str = "My Report.pdf") -> ConnectorDocument:+ return ConnectorDocument(+ id="doc-id-1",+ filename=filename,+ mimetype="application/pdf",+ content=b"%PDF-1.4 dummy",+ source_url="https://example.sharepoint.com/file.pdf",+ acl=DocumentACL(owner="[email protected]"),+ modified_time=datetime.now(),+ created_time=datetime.now(),+ )+++def _make_search_response(has_hit: bool) -> dict:+ return {"hits": {"hits": [{"_id": "x"}] if has_hit else []}}+++def _make_upload_task() -> UploadTask:+ return UploadTask(task_id="task-1", total_files=1)+++def _make_file_task() -> FileTask:+ return FileTask(file_path="file-id-1")+++def _build_connector_processor(replace_duplicates: bool) -> ConnectorFileProcessor:+ document_service = MagicMock()+ document_service.docling_service = MagicMock()+ document_service.session_manager = MagicMock()+ document_service.session_manager.get_user_opensearch_client = MagicMock()+ connector_service = MagicMock()+ return ConnectorFileProcessor(+ connector_service=connector_service,+ connection_id="conn-1",+ files_to_process=["file-id-1"],+ user_id="user-1",+ jwt_token="jwt",+ document_service=document_service,+ models_service=MagicMock(),+ replace_duplicates=replace_duplicates,+ )+++def _wire_connector_processor(+ processor: ConnectorFileProcessor,+ document: ConnectorDocument,+ filename_exists: bool,+):+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(return_value=_make_search_response(filename_exists))+ opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 3})+ opensearch_client.exists = AsyncMock(return_value=False)+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (+ opensearch_client+ )++ connector = MagicMock()+ connector.get_file_content = AsyncMock(return_value=document)+ processor.connector_service.get_connector = AsyncMock(return_value=connector)+ connection = MagicMock()+ connection.connector_type = "sharepoint"+ processor.connector_service.connection_manager = MagicMock()+ processor.connector_service.connection_manager.get_connection = AsyncMock(+ return_value=connection+ )+ return opensearch_client+++async def test_connector_processor_fails_when_filename_exists_and_replace_false():+ processor = _build_connector_processor(replace_duplicates=False)+ document = _make_document()+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(processor, "process_document_standard", new=AsyncMock()) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.FAILED+ assert "already exists" in (file_task.error or "")+ assert upload_task.failed_files == 1+ assert upload_task.successful_files == 0+ mock_process.assert_not_called()+ opensearch_client.delete_by_query.assert_not_called()+++async def test_connector_processor_deletes_then_ingests_when_replace_true():+ processor = _build_connector_processor(replace_duplicates=True)+ document = _make_document()+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(+ processor,+ "process_document_standard",+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),+ ) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ opensearch_client.delete_by_query.assert_awaited()+ mock_process.assert_awaited_once()+ # The processed filename must be the original (with space), not a+ # sanitized variant.+ assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"+++async def test_connector_processor_proceeds_when_filename_absent():+ processor = _build_connector_processor(replace_duplicates=False)+ document = _make_document()+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=False)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(+ processor,+ "process_document_standard",+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),+ ) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ opensearch_client.delete_by_query.assert_not_called()+ mock_process.assert_awaited_once()+++def _build_langflow_processor(+ replace_duplicates: bool,+) -> LangflowConnectorFileProcessor:+ langflow_service = MagicMock()+ langflow_service.task_service = MagicMock()+ langflow_service.task_service.document_service = MagicMock()+ langflow_service.task_service.models_service = MagicMock()+ langflow_service.docling_service = MagicMock()+ langflow_service.session_manager = MagicMock()+ langflow_service.session_manager.get_user_opensearch_client = MagicMock()+ langflow_service.process_connector_document = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )+ return LangflowConnectorFileProcessor(+ langflow_connector_service=langflow_service,+ connection_id="conn-1",+ files_to_process=["file-id-1"],+ user_id="user-1",+ jwt_token="jwt",+ replace_duplicates=replace_duplicates,+ )+++def _wire_langflow_processor(+ processor: LangflowConnectorFileProcessor,+ document: ConnectorDocument,+ filename_exists: bool,+ hash_exists: bool = False,+):+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(return_value=_make_search_response(filename_exists))+ opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 2})+ opensearch_client.exists = AsyncMock(return_value=hash_exists)+ processor.langflow_connector_service.session_manager.get_user_opensearch_client.return_value = (+ opensearch_client+ )++ connector = MagicMock()+ connector.get_file_content = AsyncMock(return_value=document)+ processor.langflow_connector_service.get_connector = AsyncMock(return_value=connector)+ connection = MagicMock()+ connection.connector_type = "sharepoint"+ processor.langflow_connector_service.connection_manager = MagicMock()+ processor.langflow_connector_service.connection_manager.get_connection = AsyncMock(+ return_value=connection+ )+ return opensearch_client+++async def test_langflow_connector_processor_fails_on_filename_collision():+ processor = _build_langflow_processor(replace_duplicates=False)+ document = _make_document()+ opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.FAILED+ assert "already exists" in (file_task.error or "")+ assert upload_task.failed_files == 1+ processor.langflow_connector_service.process_connector_document.assert_not_called()+ opensearch_client.delete_by_query.assert_not_called()+++async def test_langflow_connector_processor_overwrites_when_replace_true():+ processor = _build_langflow_processor(replace_duplicates=True)+ document = _make_document()+ opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ opensearch_client.delete_by_query.assert_awaited()+ processor.langflow_connector_service.process_connector_document.assert_awaited_once()+++async def test_langflow_connector_processor_hash_unchanged_path_preserved():+ """When the filename is new but the byte hash already exists, the file+ is reported as 'unchanged' — same as before this change."""+ processor = _build_langflow_processor(replace_duplicates=False)+ document = _make_document()+ _wire_langflow_processor(processor, document, filename_exists=False, hash_exists=True)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ assert (file_task.result or {}).get("status") == "unchanged"+ processor.langflow_connector_service.process_connector_document.assert_not_called()diff --git a/tests/unit/test_file_utils_filename_aliases.py b/tests/unit/test_file_utils_filename_aliases.pynew file mode 100644index 000000000..96083b804--- /dev/null+++ b/tests/unit/test_file_utils_filename_aliases.py@@ -0,0 +1,81 @@+from utils.file_utils import clean_connector_filename, get_filename_aliases+++def test_empty_input_returns_empty_list():+ assert get_filename_aliases("") == []+ assert get_filename_aliases(None) == []+ assert get_filename_aliases(" ") == []+++def test_plain_filename_returns_single_alias():+ assert get_filename_aliases("Report.pdf") == ["Report.pdf"]+++def test_txt_md_swap_preserved():+ aliases = get_filename_aliases("notes.txt")+ assert aliases == ["notes.txt", "notes.md"]++ aliases = get_filename_aliases("notes.md")+ assert aliases == ["notes.md", "notes.txt"]+++def test_spaces_get_underscore_variant():+ """Connector ingestion replaces spaces with underscores; lookup must match+ both the original and sanitized form so a SharePoint-ingested+ 'Q1 Report.pdf' (indexed as 'Q1_Report.pdf') is detected when the user+ re-uploads it locally with the original name."""+ aliases = get_filename_aliases("Q1 Report.pdf")+ assert "Q1 Report.pdf" in aliases+ assert "Q1_Report.pdf" in aliases+++def test_slashes_get_underscore_variant():+ aliases = get_filename_aliases("folder/file.pdf")+ assert "folder/file.pdf" in aliases+ assert "folder_file.pdf" in aliases+++def test_spaces_combined_with_txt_md_swap():+ aliases = get_filename_aliases("My Notes.txt")+ assert set(aliases) == {+ "My Notes.txt",+ "My Notes.md",+ "My_Notes.txt",+ "My_Notes.md",+ }+++def test_order_stable_and_deduped():+ """Original first, then variants. Names without spaces/slashes must not+ produce duplicate entries from the sanitization pass."""+ aliases = get_filename_aliases("report.txt")+ assert aliases == ["report.txt", "report.md"]++ aliases = get_filename_aliases("My Notes.txt")+ assert aliases[0] == "My Notes.txt"+ assert len(aliases) == len(set(aliases))+++def test_clean_connector_filename_preserves_spaces_and_slashes():+ """Spaces and slashes must survive so connector-indexed filenames match+ what a local upload of the same file would index as."""+ assert clean_connector_filename("My Report.pdf", "application/pdf") == "My Report.pdf"+ assert clean_connector_filename("docs/file.txt", "text/plain") == "docs/file.txt"+++def test_clean_connector_filename_enforces_mime_extension():+ """Google Docs / Slides / Sheets export as PDF — the suffix must be added+ when the name doesn't already end with the MIME-mapped extension."""+ assert (+ clean_connector_filename("untitled", "application/vnd.google-apps.document")+ == "untitled.pdf"+ )+ # Existing matching extension is not duplicated.+ assert (+ clean_connector_filename("untitled.pdf", "application/vnd.google-apps.document")+ == "untitled.pdf"+ )+++def test_clean_connector_filename_unknown_mime_keeps_filename():+ assert clean_connector_filename("data.bin", "application/x-unknown") == "data.bin"