Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: connector sync and override feature (#1663)

Edwin Jose748583bdec39
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.ts
index e40508211..214133429 100644
--- a/frontend/app/api/mutations/useSyncConnector.ts
+++ b/frontend/app/api/mutations/useSyncConnector.ts
@@ -72,6 +72,8 @@ const syncConnector = async ({
sync_all?: boolean;
/** Restrict ingest to these bucket names (IBM COS). */
bucket_filter?: string[];
+ /** When true, replace any indexed document with the same filename. */
+ replace_duplicates?: boolean;
};
}): Promise<SyncResponse> => {
const response = await fetch(`/api/connectors/${connectorType}/sync`, {
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index eae4389b5..a4112d86c 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -9,7 +9,7 @@ import {
RefreshCw,
} from "lucide-react";
import { useParams, useRouter } from "next/navigation";
-import { useState } from "react";
+import { useRef, useState } from "react";
import { toast } from "sonner";
import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";
@@ -19,6 +19,7 @@ import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery
import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
import { IngestSettings } from "@/components/cloud-picker/ingest-settings";
import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";
+import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog";
import { FileBrowserDialog } from "@/components/file-browser-dialog";
import { Button } from "@/components/ui/button";
import {
@@ -28,6 +29,7 @@ import {
} from "@/components/ui/tooltip";
import { useTask } from "@/contexts/task-context";
import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
+import { duplicateCheck } from "@/lib/upload-utils";
// Connectors that sync entire buckets/repositories without a file picker
const DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];
@@ -410,6 +412,15 @@ export default function UploadProviderPage() {
const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);
const [ingestSettings, setIngestSettings] = useSessionIngestSettings();
+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);
+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);
+ const [pendingSync, setPendingSync] = useState<{
+ connector: { connectionId?: string; type: string };
+ allFiles: CloudFile[];
+ nonDuplicateFiles: CloudFile[];
+ duplicateNames: string[];
+ } | null>(null);
+ const isOverwriteConfirmedRef = useRef(false);
const accessToken = tokenData?.access_token || null;
const isLoading =
@@ -427,21 +438,17 @@ export default function UploadProviderPage() {
setSelectedFiles(files);
};
- const handleSync = (connector: { connectionId?: string; type: string }) => {
- if (!connector.connectionId || selectedFiles.length === 0) return;
-
- const chunkErr = getIngestChunkSettingsError(ingestSettings);
- if (chunkErr) {
- toast.error("Could not start ingest", { description: chunkErr });
- return;
- }
-
+ const submitSync = (
+ connector: { connectionId?: string; type: string },
+ files: CloudFile[],
+ replaceDuplicates: boolean,
+ ) => {
syncMutation.mutate(
{
connectorType: connector.type,
body: {
connection_id: connector.connectionId,
- selected_files: selectedFiles.map((file) => ({
+ selected_files: files.map((file) => ({
id: file.id,
name: file.name,
mimeType: file.mimeType,
@@ -450,6 +457,7 @@ export default function UploadProviderPage() {
isFolder: file.isFolder,
})),
settings: ingestSettings,
+ replace_duplicates: replaceDuplicates,
},
},
{
@@ -467,6 +475,89 @@ export default function UploadProviderPage() {
);
};
+ const handleSync = async (connector: {
+ connectionId?: string;
+ type: string;
+ }) => {
+ if (!connector.connectionId || selectedFiles.length === 0) return;
+
+ const chunkErr = getIngestChunkSettingsError(ingestSettings);
+ if (chunkErr) {
+ toast.error("Could not start ingest", { description: chunkErr });
+ return;
+ }
+
+ setIsCheckingDuplicates(true);
+ try {
+ const results = await Promise.all(
+ selectedFiles.map(async (file) => {
+ if (file.isFolder) return { file, isDuplicate: false };
+ try {
+ const fakeFile = new File([], file.name);
+ const { exists } = await duplicateCheck(fakeFile);
+ return { file, isDuplicate: exists };
+ } catch (err) {
+ console.error(
+ `[Connector Sync] Duplicate check failed for ${file.name}:`,
+ err,
+ );
+ return { file, isDuplicate: false };
+ }
+ }),
+ );
+
+ const duplicates = results.filter((r) => r.isDuplicate);
+ const nonDuplicates = results
+ .filter((r) => !r.isDuplicate)
+ .map((r) => r.file);
+
+ if (duplicates.length === 0) {
+ submitSync(connector, selectedFiles, false);
+ return;
+ }
+
+ setPendingSync({
+ connector,
+ allFiles: selectedFiles,
+ nonDuplicateFiles: nonDuplicates,
+ duplicateNames: duplicates.map((r) => r.file.name),
+ });
+ setDuplicateDialogOpen(true);
+ } finally {
+ setIsCheckingDuplicates(false);
+ }
+ };
+
+ const handleOverwriteDuplicates = () => {
+ if (!pendingSync) return;
+ isOverwriteConfirmedRef.current = true;
+ const { connector, allFiles } = pendingSync;
+ submitSync(connector, allFiles, true);
+ setPendingSync(null);
+ };
+
+ const handleDuplicateDialogOpenChange = (open: boolean) => {
+ if (!open && pendingSync) {
+ if (isOverwriteConfirmedRef.current) {
+ // Overwrite already submitted in handleOverwriteDuplicates; this close
+ // event fires immediately after and would otherwise re-enter the
+ // "skip duplicates" branch.
+ isOverwriteConfirmedRef.current = false;
+ } else {
+ const { connector, nonDuplicateFiles, duplicateNames } = pendingSync;
+ if (nonDuplicateFiles.length > 0) {
+ submitSync(connector, nonDuplicateFiles, false);
+ } else {
+ toast.info(
+ `All ${duplicateNames.length} selected file(s) already exist. Nothing was synced.`,
+ );
+ }
+ }
+ setPendingSync(null);
+ }
+ setDuplicateDialogOpen(open);
+ };
+
const getProviderDisplayName = () => {
const nameMap: { [key: string]: string } = {
google_drive: "Google Drive",
@@ -652,8 +743,10 @@ export default function UploadProviderPage() {
className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
variant={!hasSelectedFiles ? "secondary" : undefined}
onClick={() => handleSync(connector)}
- loading={isIngesting}
- disabled={!hasSelectedFiles || isIngesting}
+ loading={isIngesting || isCheckingDuplicates}
+ disabled={
+ !hasSelectedFiles || isIngesting || isCheckingDuplicates
+ }
>
{hasSelectedFiles ? (
<>
@@ -673,6 +766,14 @@ export default function UploadProviderPage() {
</Tooltip>
</div>
</div>
+
+ <DuplicateHandlingDialog
+ open={duplicateDialogOpen}
+ onOpenChange={handleDuplicateDialogOpenChange}
+ onOverwrite={handleOverwriteDuplicates}
+ isLoading={isIngesting}
+ duplicateNames={pendingSync?.duplicateNames}
+ />
</>
);
}
diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx
index a19faabc4..76b5a84c9 100644
--- a/frontend/components/duplicate-handling-dialog.tsx
+++ b/frontend/components/duplicate-handling-dialog.tsx
@@ -19,8 +19,11 @@ interface DuplicateHandlingDialogProps {
isLoading?: boolean;
duplicateLabel?: string;
duplicateCount?: number;
+ duplicateNames?: string[];
}
+const MAX_LISTED_DUPLICATES = 5;
+
export const DuplicateHandlingDialog: React.FC<
DuplicateHandlingDialogProps
> = ({
@@ -30,27 +33,40 @@ export const DuplicateHandlingDialog: React.FC<
isLoading = false,
duplicateLabel,
duplicateCount,
+ duplicateNames,
}) => {
const handleOverwrite = async () => {
await onOverwrite();
onOpenChange(false);
};
+ const namesProvided = duplicateNames && duplicateNames.length > 0;
+ const effectiveCount = namesProvided
+ ? duplicateNames!.length
+ : duplicateCount;
+
const description =
- typeof duplicateCount === "number"
- ? duplicateCount === 1
+ typeof effectiveCount === "number"
+ ? effectiveCount === 1
? "1 duplicate document already exists. Overwriting will replace the existing document version. This can't be undone."
- : `${duplicateCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`
+ : `${effectiveCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`
: duplicateLabel
? `A document named "${duplicateLabel}" already exists. Overwriting will replace the existing document version. This can't be undone.`
: "Overwriting will replace the existing document with another version. This can't be undone.";
const overwriteLabel =
- typeof duplicateCount === "number" ? "Overwrite duplicates" : "Overwrite";
+ typeof effectiveCount === "number" ? "Overwrite duplicates" : "Overwrite";
const cancelLabel =
- typeof duplicateCount === "number"
+ typeof effectiveCount === "number"
? "Skip duplicates & continue"
: "Cancel";
+ const visibleNames = namesProvided
+ ? duplicateNames!.slice(0, MAX_LISTED_DUPLICATES)
+ : [];
+ const remainingCount = namesProvided
+ ? duplicateNames!.length - visibleNames.length
+ : 0;
+
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-[520px]">
@@ -61,6 +77,19 @@ export const DuplicateHandlingDialog: React.FC<
</DialogDescription>
</DialogHeader>
+ {namesProvided && (
+ <ul className="text-sm text-muted-foreground list-disc pl-5 space-y-0.5">
+ {visibleNames.map((name) => (
+ <li key={name} className="break-all">
+ {name}
+ </li>
+ ))}
+ {remainingCount > 0 && (
+ <li className="list-none italic">… and {remainingCount} more</li>
+ )}
+ </ul>
+ )}
+
<DialogFooter className="flex-row gap-2 justify-end">
<Button
type="button"
diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx
index b858bcfa0..687c8eef5 100644
--- a/frontend/components/knowledge-dropdown.tsx
+++ b/frontend/components/knowledge-dropdown.tsx
@@ -142,7 +142,7 @@ export function KnowledgeDropdown() {
const [pendingFolderUpload, setPendingFolderUpload] = useState<{
allFiles: File[];
nonDuplicateFiles: File[];
- duplicateCount: number;
+ duplicateNames: string[];
unsupportedCount: number;
} | null>(null);
const isFolderOverwriteConfirmedRef = useRef(false);
@@ -382,13 +382,13 @@ export function KnowledgeDropdown() {
const handleOverwriteFile = async () => {
if (pendingFolderUpload) {
isFolderOverwriteConfirmedRef.current = true;
- const { allFiles, duplicateCount, unsupportedCount } =
+ const { allFiles, duplicateNames, unsupportedCount } =
pendingFolderUpload;
await uploadFolderBatches(allFiles, true);
const unsupportedMessage =
unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";
toast.success(
- `Processed ${allFiles.length} file(s), including ${duplicateCount} overwrite(s)${unsupportedMessage}`,
+ `Processed ${allFiles.length} file(s), including ${duplicateNames.length} overwrite(s)${unsupportedMessage}`,
);
resetDuplicateDialogState();
return;
@@ -427,13 +427,13 @@ export function KnowledgeDropdown() {
if (isFolderOverwriteConfirmedRef.current) {
isFolderOverwriteConfirmedRef.current = false;
} else {
- const { nonDuplicateFiles, duplicateCount, unsupportedCount } =
+ const { nonDuplicateFiles, duplicateNames, unsupportedCount } =
pendingFolderUpload;
if (nonDuplicateFiles.length > 0) {
await uploadFolderBatches(nonDuplicateFiles, false);
const extraParts: string[] = [];
- if (duplicateCount > 0) {
- extraParts.push(`skipped ${duplicateCount} duplicate(s)`);
+ if (duplicateNames.length > 0) {
+ extraParts.push(`skipped ${duplicateNames.length} duplicate(s)`);
}
if (unsupportedCount > 0) {
extraParts.push(`skipped ${unsupportedCount} unsupported`);
@@ -515,9 +515,10 @@ export function KnowledgeDropdown() {
const nonDuplicateFiles = duplicateResults
.filter((r) => !r.isDuplicate)
.map((r) => r.file);
- const duplicateCount = duplicateResults.filter(
- (r) => r.isDuplicate,
- ).length;
+ const duplicateNames = duplicateResults
+ .filter((r) => r.isDuplicate)
+ .map((r) => r.file.name);
+ const duplicateCount = duplicateNames.length;
if (unsupportedCount > 0) {
toast.error(
@@ -536,7 +537,7 @@ export function KnowledgeDropdown() {
setPendingFolderUpload({
allFiles: cleanFiles,
nonDuplicateFiles,
- duplicateCount,
+ duplicateNames,
unsupportedCount,
});
setShowDuplicateDialog(true);
@@ -856,7 +857,7 @@ export function KnowledgeDropdown() {
onOverwrite={handleOverwriteFile}
isLoading={fileUploading || folderLoading}
duplicateLabel={duplicateFilename}
- duplicateCount={pendingFolderUpload?.duplicateCount}
+ duplicateNames={pendingFolderUpload?.duplicateNames}
/>
</>
);
diff --git a/src/api/connector_router.py b/src/api/connector_router.py
index 21ebfab2a..e8bf8228e 100644
--- a/src/api/connector_router.py
+++ b/src/api/connector_router.py
@@ -57,6 +57,7 @@ class ConnectorRouter:
jwt_token: str = None,
file_infos: list = None,
ingest_settings: dict = None,
+ replace_duplicates: bool = False,
):
"""Sync specific files using the active service."""
return await self.get_active_service().sync_specific_files(
@@ -66,6 +67,7 @@ class ConnectorRouter:
jwt_token,
file_infos=file_infos,
ingest_settings=ingest_settings,
+ replace_duplicates=replace_duplicates,
)
def __getattr__(self, name):
diff --git a/src/api/connectors.py b/src/api/connectors.py
index ea31c828d..309158337 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -294,6 +294,10 @@ class ConnectorSyncBody(BaseModel):
bucket_filter: list[str] | None = None
# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).
settings: dict[str, Any] | None = None
+ # When True, files whose filename already exists in the index are replaced
+ # rather than failing. Set by the provider upload UI after the user confirms
+ # overwrite in the duplicate dialog.
+ replace_duplicates: bool = False
async def list_connectors(
@@ -409,6 +413,7 @@ async def connector_sync(
jwt_token=jwt_token,
file_infos=file_infos,
ingest_settings=body.settings,
+ replace_duplicates=body.replace_duplicates,
)
elif body.sync_all or body.bucket_filter:
# Full ingest: discover and ingest all files (or files from specific buckets).
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 5cd2716de..3f2ba93a0 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -317,6 +317,7 @@ class LangflowConnectorService:
jwt_token: str = None,
file_infos: list[dict[str, Any]] = None,
ingest_settings: dict[str, Any] | None = None,
+ replace_duplicates: bool = False,
) -> str:
"""
Sync specific files by their IDs using Langflow processing.
@@ -413,6 +414,7 @@ class LangflowConnectorService:
owner_name=owner_name,
owner_email=owner_email,
ingest_settings=ingest_settings,
+ replace_duplicates=replace_duplicates,
)
# Create custom task using TaskService
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 146c70116..ed9b385b0 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -318,6 +318,7 @@ class ConnectorService:
jwt_token: str = None,
file_infos: list[dict[str, Any]] = None,
ingest_settings: dict[str, Any] | None = None,
+ replace_duplicates: bool = False,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
@@ -442,6 +443,7 @@ class ConnectorService:
),
models_service=self.models_service,
ingest_settings=ingest_settings,
+ replace_duplicates=replace_duplicates,
)
# Create custom task using TaskService
diff --git a/src/models/processors.py b/src/models/processors.py
index 02771678a..05ffaf4a1 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -492,6 +492,7 @@ class ConnectorFileProcessor(TaskProcessor):
document_service=None,
models_service=None,
ingest_settings: dict[str, Any] | None = None,
+ replace_duplicates: bool = False,
):
super().__init__(
document_service=document_service,
@@ -506,6 +507,7 @@ class ConnectorFileProcessor(TaskProcessor):
self.owner_name = owner_name
self.owner_email = owner_email
self.ingest_settings = ingest_settings
+ self.replace_duplicates = replace_duplicates
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using consolidated methods"""
@@ -548,6 +550,18 @@ class ConnectorFileProcessor(TaskProcessor):
if not self.user_id:
raise ValueError("user_id not provided to ConnectorFileProcessor")
+ opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
+ )
+ if await self.check_filename_exists(document.filename, opensearch_client):
+ if not self.replace_duplicates:
+ file_task.status = TaskStatus.FAILED
+ file_task.error = f"File with name '{document.filename}' already exists"
+ file_task.updated_at = time.time()
+ upload_task.failed_files += 1
+ return
+ await self.delete_document_by_filename(document.filename, opensearch_client)
+
# Create temporary file from document content
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
@@ -624,6 +638,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
owner_name: str = None,
owner_email: str = None,
ingest_settings: dict[str, Any] | None = None,
+ replace_duplicates: bool = False,
):
super().__init__(
document_service=langflow_connector_service.task_service.document_service
@@ -642,6 +657,7 @@ class LangflowConnectorFileProcessor(TaskProcessor):
self.owner_name = owner_name
self.owner_email = owner_email
self.ingest_settings = ingest_settings
+ self.replace_duplicates = replace_duplicates
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using LangflowConnectorService"""
@@ -668,6 +684,20 @@ class LangflowConnectorFileProcessor(TaskProcessor):
if not self.user_id:
raise ValueError("user_id not provided to LangflowConnectorFileProcessor")
+ opensearch_client = (
+ self.langflow_connector_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
+ )
+ )
+ if await self.check_filename_exists(document.filename, opensearch_client):
+ if not self.replace_duplicates:
+ file_task.status = TaskStatus.FAILED
+ file_task.error = f"File with name '{document.filename}' already exists"
+ file_task.updated_at = time.time()
+ upload_task.failed_files += 1
+ return
+ await self.delete_document_by_filename(document.filename, opensearch_client)
+
# Create temporary file and compute hash to check for duplicates
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
@@ -678,12 +708,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):
# Compute hash and check if already exists
file_hash = hash_id(tmp_path)
- # Check if document already exists
- opensearch_client = (
- self.langflow_connector_service.session_manager.get_user_opensearch_client(
- self.user_id, self.jwt_token
- )
- )
if await self.check_document_exists(file_hash, opensearch_client):
file_task.status = TaskStatus.COMPLETED
file_task.result = {"status": "unchanged", "id": file_hash}
diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py
index 989875b0f..f2a50dfe2 100644
--- a/src/utils/file_utils.py
+++ b/src/utils/file_utils.py
@@ -3,11 +3,12 @@
import os
import tempfile
from contextlib import contextmanager
-from typing import Optional
@contextmanager
-def auto_cleanup_tempfile(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None):
+def auto_cleanup_tempfile(
+ suffix: str | None = None, prefix: str | None = None, dir: str | None = None
+):
"""
Context manager for temporary files that automatically cleans up.
@@ -85,20 +86,20 @@ def get_file_extension(mimetype: str) -> str:
def clean_connector_filename(filename: str, mimetype: str) -> str:
- """Clean filename and ensure correct extension.
+ """Ensure the filename ends with the extension that matches its MIME type.
- If the MIME type maps to a known extension, it is enforced.
- If the MIME type is unknown, the original filename (and its extension) is kept as-is
- rather than appending a meaningless .bin suffix.
+ The original name is preserved verbatim (spaces, slashes, casing) so that
+ connector-indexed filenames match what the user sees in the source system
+ and what a local upload of the same file would index as. Only the
+ extension is enforced, and only when the MIME type maps to one — for an
+ unknown MIME type, the original filename and extension are kept.
"""
- clean_name = filename.replace(" ", "_").replace("/", "_")
suffix = get_file_extension(mimetype)
if suffix is None:
- # Unknown type — keep whatever extension the file already has
- return clean_name
- if not clean_name.lower().endswith(suffix.lower()):
- return clean_name + suffix
- return clean_name
+ return filename
+ if not filename.lower().endswith(suffix.lower()):
+ return filename + suffix
+ return filename
def get_filename_aliases(filename: str) -> list[str]:
@@ -125,5 +126,9 @@ def get_filename_aliases(filename: str) -> list[str]:
elif lower_name.endswith(".md"):
aliases.append(normalized[:-3] + ".txt")
+ # Mirror clean_connector_filename's space/slash -> underscore so lookups also
+ # match files indexed through a connector ingestion path.
+ aliases.extend(name.replace(" ", "_").replace("/", "_") for name in list(aliases))
+
# Keep order stable while removing duplicates.
return list(dict.fromkeys(aliases))
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
new file mode 100644
index 000000000..4d167dc42
--- /dev/null
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -0,0 +1,250 @@
+"""Filename-based dedupe gate on connector ingest paths.
+
+Covers the symmetric behavior introduced so that picking a file from the
+SharePoint UI when a same-named document already exists triggers the
+"Overwrite" dialog (frontend) or fails the file task (backend, when
+``replace_duplicates`` is not set).
+"""
+
+from datetime import datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from connectors.base import ConnectorDocument, DocumentACL
+from models.processors import ConnectorFileProcessor, LangflowConnectorFileProcessor
+from models.tasks import FileTask, TaskStatus, UploadTask
+
+
+def _make_document(filename: str = "My Report.pdf") -> ConnectorDocument:
+ return ConnectorDocument(
+ id="doc-id-1",
+ filename=filename,
+ mimetype="application/pdf",
+ content=b"%PDF-1.4 dummy",
+ source_url="https://example.sharepoint.com/file.pdf",
+ acl=DocumentACL(owner="[email protected]"),
+ modified_time=datetime.now(),
+ created_time=datetime.now(),
+ )
+
+
+def _make_search_response(has_hit: bool) -> dict:
+ return {"hits": {"hits": [{"_id": "x"}] if has_hit else []}}
+
+
+def _make_upload_task() -> UploadTask:
+ return UploadTask(task_id="task-1", total_files=1)
+
+
+def _make_file_task() -> FileTask:
+ return FileTask(file_path="file-id-1")
+
+
+def _build_connector_processor(replace_duplicates: bool) -> ConnectorFileProcessor:
+ document_service = MagicMock()
+ document_service.docling_service = MagicMock()
+ document_service.session_manager = MagicMock()
+ document_service.session_manager.get_user_opensearch_client = MagicMock()
+ connector_service = MagicMock()
+ return ConnectorFileProcessor(
+ connector_service=connector_service,
+ connection_id="conn-1",
+ files_to_process=["file-id-1"],
+ user_id="user-1",
+ jwt_token="jwt",
+ document_service=document_service,
+ models_service=MagicMock(),
+ replace_duplicates=replace_duplicates,
+ )
+
+
+def _wire_connector_processor(
+ processor: ConnectorFileProcessor,
+ document: ConnectorDocument,
+ filename_exists: bool,
+):
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(return_value=_make_search_response(filename_exists))
+ opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 3})
+ opensearch_client.exists = AsyncMock(return_value=False)
+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(return_value=document)
+ processor.connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "sharepoint"
+ processor.connector_service.connection_manager = MagicMock()
+ processor.connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+ return opensearch_client
+
+
+async def test_connector_processor_fails_when_filename_exists_and_replace_false():
+ processor = _build_connector_processor(replace_duplicates=False)
+ document = _make_document()
+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(processor, "process_document_standard", new=AsyncMock()) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.FAILED
+ assert "already exists" in (file_task.error or "")
+ assert upload_task.failed_files == 1
+ assert upload_task.successful_files == 0
+ mock_process.assert_not_called()
+ opensearch_client.delete_by_query.assert_not_called()
+
+
+async def test_connector_processor_deletes_then_ingests_when_replace_true():
+ processor = _build_connector_processor(replace_duplicates=True)
+ document = _make_document()
+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),
+ ) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ opensearch_client.delete_by_query.assert_awaited()
+ mock_process.assert_awaited_once()
+ # The processed filename must be the original (with space), not a
+ # sanitized variant.
+ assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"
+
+
+async def test_connector_processor_proceeds_when_filename_absent():
+ processor = _build_connector_processor(replace_duplicates=False)
+ document = _make_document()
+ opensearch_client = _wire_connector_processor(processor, document, filename_exists=False)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),
+ ) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ opensearch_client.delete_by_query.assert_not_called()
+ mock_process.assert_awaited_once()
+
+
+def _build_langflow_processor(
+ replace_duplicates: bool,
+) -> LangflowConnectorFileProcessor:
+ langflow_service = MagicMock()
+ langflow_service.task_service = MagicMock()
+ langflow_service.task_service.document_service = MagicMock()
+ langflow_service.task_service.models_service = MagicMock()
+ langflow_service.docling_service = MagicMock()
+ langflow_service.session_manager = MagicMock()
+ langflow_service.session_manager.get_user_opensearch_client = MagicMock()
+ langflow_service.process_connector_document = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+ return LangflowConnectorFileProcessor(
+ langflow_connector_service=langflow_service,
+ connection_id="conn-1",
+ files_to_process=["file-id-1"],
+ user_id="user-1",
+ jwt_token="jwt",
+ replace_duplicates=replace_duplicates,
+ )
+
+
+def _wire_langflow_processor(
+ processor: LangflowConnectorFileProcessor,
+ document: ConnectorDocument,
+ filename_exists: bool,
+ hash_exists: bool = False,
+):
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(return_value=_make_search_response(filename_exists))
+ opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 2})
+ opensearch_client.exists = AsyncMock(return_value=hash_exists)
+ processor.langflow_connector_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(return_value=document)
+ processor.langflow_connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "sharepoint"
+ processor.langflow_connector_service.connection_manager = MagicMock()
+ processor.langflow_connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+ return opensearch_client
+
+
+async def test_langflow_connector_processor_fails_on_filename_collision():
+ processor = _build_langflow_processor(replace_duplicates=False)
+ document = _make_document()
+ opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.FAILED
+ assert "already exists" in (file_task.error or "")
+ assert upload_task.failed_files == 1
+ processor.langflow_connector_service.process_connector_document.assert_not_called()
+ opensearch_client.delete_by_query.assert_not_called()
+
+
+async def test_langflow_connector_processor_overwrites_when_replace_true():
+ processor = _build_langflow_processor(replace_duplicates=True)
+ document = _make_document()
+ opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ opensearch_client.delete_by_query.assert_awaited()
+ processor.langflow_connector_service.process_connector_document.assert_awaited_once()
+
+
+async def test_langflow_connector_processor_hash_unchanged_path_preserved():
+ """When the filename is new but the byte hash already exists, the file
+ is reported as 'unchanged' — same as before this change."""
+ processor = _build_langflow_processor(replace_duplicates=False)
+ document = _make_document()
+ _wire_langflow_processor(processor, document, filename_exists=False, hash_exists=True)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ assert (file_task.result or {}).get("status") == "unchanged"
+ processor.langflow_connector_service.process_connector_document.assert_not_called()
diff --git a/tests/unit/test_file_utils_filename_aliases.py b/tests/unit/test_file_utils_filename_aliases.py
new file mode 100644
index 000000000..96083b804
--- /dev/null
+++ b/tests/unit/test_file_utils_filename_aliases.py
@@ -0,0 +1,81 @@
+from utils.file_utils import clean_connector_filename, get_filename_aliases
+
+
+def test_empty_input_returns_empty_list():
+ assert get_filename_aliases("") == []
+ assert get_filename_aliases(None) == []
+ assert get_filename_aliases(" ") == []
+
+
+def test_plain_filename_returns_single_alias():
+ assert get_filename_aliases("Report.pdf") == ["Report.pdf"]
+
+
+def test_txt_md_swap_preserved():
+ aliases = get_filename_aliases("notes.txt")
+ assert aliases == ["notes.txt", "notes.md"]
+
+ aliases = get_filename_aliases("notes.md")
+ assert aliases == ["notes.md", "notes.txt"]
+
+
+def test_spaces_get_underscore_variant():
+ """Connector ingestion replaces spaces with underscores; lookup must match
+ both the original and sanitized form so a SharePoint-ingested
+ 'Q1 Report.pdf' (indexed as 'Q1_Report.pdf') is detected when the user
+ re-uploads it locally with the original name."""
+ aliases = get_filename_aliases("Q1 Report.pdf")
+ assert "Q1 Report.pdf" in aliases
+ assert "Q1_Report.pdf" in aliases
+
+
+def test_slashes_get_underscore_variant():
+ aliases = get_filename_aliases("folder/file.pdf")
+ assert "folder/file.pdf" in aliases
+ assert "folder_file.pdf" in aliases
+
+
+def test_spaces_combined_with_txt_md_swap():
+ aliases = get_filename_aliases("My Notes.txt")
+ assert set(aliases) == {
+ "My Notes.txt",
+ "My Notes.md",
+ "My_Notes.txt",
+ "My_Notes.md",
+ }
+
+
+def test_order_stable_and_deduped():
+ """Original first, then variants. Names without spaces/slashes must not
+ produce duplicate entries from the sanitization pass."""
+ aliases = get_filename_aliases("report.txt")
+ assert aliases == ["report.txt", "report.md"]
+
+ aliases = get_filename_aliases("My Notes.txt")
+ assert aliases[0] == "My Notes.txt"
+ assert len(aliases) == len(set(aliases))
+
+
+def test_clean_connector_filename_preserves_spaces_and_slashes():
+ """Spaces and slashes must survive so connector-indexed filenames match
+ what a local upload of the same file would index as."""
+ assert clean_connector_filename("My Report.pdf", "application/pdf") == "My Report.pdf"
+ assert clean_connector_filename("docs/file.txt", "text/plain") == "docs/file.txt"
+
+
+def test_clean_connector_filename_enforces_mime_extension():
+ """Google Docs / Slides / Sheets export as PDF — the suffix must be added
+ when the name doesn't already end with the MIME-mapped extension."""
+ assert (
+ clean_connector_filename("untitled", "application/vnd.google-apps.document")
+ == "untitled.pdf"
+ )
+ # Existing matching extension is not duplicated.
+ assert (
+ clean_connector_filename("untitled.pdf", "application/vnd.google-apps.document")
+ == "untitled.pdf"
+ )
+
+
+def test_clean_connector_filename_unknown_mime_keeps_filename():
+ assert clean_connector_filename("data.bin", "application/x-unknown") == "data.bin"