OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: overwite duplicate files for connectors (#1892)
Wallgau417e20655b12
diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.tsindex af7faabde..ba262ec1b 100644--- a/frontend/lib/knowledge-table-state.ts+++ b/frontend/lib/knowledge-table-state.ts@@ -28,15 +28,58 @@ export function getKnowledgeFileIdentity(file?: {return "";}+function looksLikeHttpUrl(value: string): boolean {+ return /^https?:\/\//i.test(value.trim());+}++/** Filename variants for overlay matching (mirrors backend `get_filename_aliases`). */+function getKnowledgeFilenameAliases(filename?: string): string[] {+ const normalized = filename?.trim() ?? "";+ if (!normalized) {+ return [];+ }+ if (looksLikeHttpUrl(normalized)) {+ return [normalized];+ }+ const aliases = [normalized];+ const lower = normalized.toLowerCase();+ if (lower.endsWith(".txt")) {+ aliases.push(`${normalized.slice(0, -4)}.md`);+ } else if (lower.endsWith(".md")) {+ aliases.push(`${normalized.slice(0, -3)}.txt`);+ }+ for (const name of [...aliases]) {+ aliases.push(name.replace(/ /g, "_").replace(/\//g, "_"));+ }+ return [...new Set(aliases)];+}++function addFilenameAliasKeys(keys: Set<string>, filename?: string): void {+ for (const alias of getKnowledgeFilenameAliases(filename)) {+ keys.add(alias);+ }+}++/** Lookup keys for matching task overlays to indexed rows (filename, path, basename). */+export function getKnowledgeFileAliasKeys(file?: {+ filename?: string;+ source_url?: string;+}): string[] {+ const keys = new Set<string>();+ addFilenameAliasKeys(keys, file?.filename);+ const sourceUrl = file?.source_url?.trim();+ if (sourceUrl) {+ keys.add(sourceUrl);+ addFilenameAliasKeys(keys, sourceUrl.split("/").pop());+ }+ return [...keys];+}+function isMeaningfulConnectorType(connectorType?: string): boolean {const normalized = connectorType?.trim();return Boolean(normalized && normalized !== "local");}-function looksLikeHttpUrl(value: string): boolean {- return /^https?:\/\//i.test(value.trim());-}-/** Infer connector_type for task overlays when the API does not return it. */export function inferTaskFileConnectorType(filePath: string,@@ -91,6 +134,50 @@ export function resolveKnowledgeRowConnectorType(return taskType?.trim() || backendType?.trim() || "local";}+function taskOverlayPriority(status?: string): number {+ switch (status) {+ case "processing":+ return 3;+ case "failed":+ return 2;+ case "active":+ return 1;+ default:+ return 0;+ }+}++function indexTaskFileOverlays(+ taskFilesAsFiles: SearchFile[],+): Map<string, SearchFile> {+ const map = new Map<string, SearchFile>();+ for (const file of taskFilesAsFiles) {+ for (const key of getKnowledgeFileAliasKeys(file)) {+ const existing = map.get(key);+ if (+ !existing ||+ taskOverlayPriority(file.status) >= taskOverlayPriority(existing.status)+ ) {+ map.set(key, file);+ }+ }+ }+ return map;+}++function lookupTaskFileOverlay(+ map: Map<string, SearchFile>,+ file: SearchFile,+): SearchFile | undefined {+ for (const key of getKnowledgeFileAliasKeys(file)) {+ const match = map.get(key);+ if (match) {+ return match;+ }+ }+ return undefined;+}+export function buildKnowledgeTableRows(searchData: SearchFile[],taskFiles: TaskFile[],@@ -115,27 +202,32 @@ export function buildKnowledgeTableRows(};});- const taskFileMap = new Map(- taskFilesAsFiles.map((file) => [getKnowledgeFileIdentity(file), file]),- );+ const taskFileMap = indexTaskFileOverlays(taskFilesAsFiles);const backendFiles = searchData.map((file) => {if (file.connector_type === "openrag_docs") {return file;}- const taskFile = taskFileMap.get(getKnowledgeFileIdentity(file));+ const taskFile = lookupTaskFileOverlay(taskFileMap, file);if (taskFile) {const backendStatus = file.status ?? "active";+ const status =+ taskFile.status === "processing" || taskFile.status === "failed"+ ? taskFile.status+ : backendStatus;return {...file,- filename: taskFile.filename,- source_url: taskFile.source_url,- connector_type: resolveKnowledgeRowConnectorType(- file.connector_type,- taskFile.connector_type,- backendStatus,- ),- status: backendStatus,+ // Indexed row identity: prefer backend fields so filename and source_url stay paired.+ filename: file.filename || taskFile.filename,+ source_url: file.source_url || taskFile.source_url,+ connector_type: isMeaningfulConnectorType(file.connector_type)+ ? file.connector_type!.trim()+ : resolveKnowledgeRowConnectorType(+ file.connector_type,+ taskFile.connector_type,+ status,+ ),+ status,error: taskFile.error,embedding_model: taskFile.embedding_model ?? file.embedding_model,embedding_dimensions:@@ -145,22 +237,28 @@ export function buildKnowledgeTableRows(return file;});- const backendIdentities = new Set(- backendFiles.map((f) => getKnowledgeFileIdentity(f)),- );+ const backendIdentityKeys = new Set<string>();+ for (const file of searchData) {+ for (const key of getKnowledgeFileAliasKeys(file)) {+ backendIdentityKeys.add(key);+ }+ }const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {if (taskFile.filename === "OpenRAG docs refresh" ||- taskFile.source_url.includes("openr.ag")+ (taskFile.source_url ?? "").includes("openr.ag")) {return false;}if (taskFile.connector_type === "openrag_docs") {return false;}- const identity = getKnowledgeFileIdentity(taskFile);- if (backendIdentities.has(identity)) {+ if (+ getKnowledgeFileAliasKeys(taskFile).some((key) =>+ backendIdentityKeys.has(key),+ )+ ) {return false;}// Keep "active" overlays until the index lists the file (task drops key before refetch).diff --git a/src/connectors/service.py b/src/connectors/service.pyindex ade3f9e91..42e494d7b 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -219,6 +219,7 @@ class ConnectorService:connector_type: str,jwt_token: str = None,id_field: str = "document_id",+ indexed_filename: str | None = None,):"""Update indexed chunks with connector-specific metadata"""from utils.acl_utils import update_document_acl@@ -294,7 +295,7 @@ class ConnectorService:"params": {"source_url": document.source_url,"connector_type": connector_type,- "filename": document.filename,+ "filename": indexed_filename or document.filename,"created_time": document.created_time.isoformat()if document.created_timeelse None,@@ -421,6 +422,7 @@ class ConnectorService:),models_service=self.models_service,replace_duplicates=replace_duplicates,+ connector_type=connector.CONNECTOR_TYPE,)# Use file IDs as items (no more fake file paths!)@@ -603,6 +605,7 @@ class ConnectorService:models_service=self.models_service,ingest_settings=ingest_settings,replace_duplicates=replace_duplicates,+ connector_type=connector.CONNECTOR_TYPE,)# Create custom task using TaskServicediff --git a/src/models/processors.py b/src/models/processors.pyindex c619c46a8..f93254848 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -698,6 +698,7 @@ class ConnectorFileProcessor(TaskProcessor):models_service=None,ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,+ connector_type: str | None = None,):super().__init__(document_service=document_service,@@ -713,6 +714,7 @@ class ConnectorFileProcessor(TaskProcessor):self.owner_email = owner_emailself.ingest_settings = ingest_settingsself.replace_duplicates = replace_duplicates+ self.connector_type = connector_typeasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using unified methods"""@@ -730,6 +732,8 @@ class ConnectorFileProcessor(TaskProcessor):if not connector or not connection:raise ValueError(f"Connection '{self.connection_id}' not found")+ connector_type = self.connector_type or connection.connector_type+# Validate file extension early if filename is availableVALID_EXTENSIONS = {"adoc",@@ -844,17 +848,20 @@ class ConnectorFileProcessor(TaskProcessor):# filename differs from the current one. If any were removed (a real# rename), force a re-ingest below so the file is re-indexed under# the new name instead of short-circuiting as "unchanged".+ # Match against file_task.filename — the cleaned name the file is+ # actually indexed under — so duplicate/rename detection lines up+ # with how chunks are keyed.renamed = (await self._delete_connector_chunks(document.id,opensearch_client,self.user_id,- keep_filenames=get_filename_aliases(document.filename),+ keep_filenames=get_filename_aliases(file_task.filename),)> 0)- if await self.check_filename_exists(document.filename, opensearch_client):+ if await self.check_filename_exists(file_task.filename, opensearch_client):if not self.replace_duplicates:file_task.status = TaskStatus.SKIPPEDfile_task.error = None@@ -867,13 +874,13 @@ class ConnectorFileProcessor(TaskProcessor):upload_task.successful_files += 1returnawait self.delete_document_by_filename(- document.filename,+ file_task.filename,opensearch_client,owner_user_id=self.user_id,)# Create temporary file from document content- suffix = os.path.splitext(document.filename)[1]+ suffix = os.path.splitext(file_task.filename)[1]if not suffix:suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:@@ -929,7 +936,7 @@ class ConnectorFileProcessor(TaskProcessor):# Ingest via unified Langflow pipeline (two-phase Docling + Langflow run)langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(- document.filename, document.mimetype+ file_task.filename, document.mimetype)file_tuple = (langflow_filename, document.content, processed_mimetype)@@ -962,7 +969,7 @@ class ConnectorFileProcessor(TaskProcessor):owner=self.user_id,owner_name=self.owner_name,owner_email=self.owner_email,- connector_type=connection.connector_type,+ connector_type=connector_type,docling_polling_service=self.connector_service.task_service.docling_polling_serviceif self.connector_service.task_serviceelse None,@@ -1007,12 +1014,12 @@ class ConnectorFileProcessor(TaskProcessor):file_path=tmp_path,file_hash=file_hash,owner_user_id=self.user_id,- original_filename=document.filename,+ original_filename=file_task.filename,jwt_token=self.jwt_token,owner_name=self.owner_name,owner_email=self.owner_email,file_size=len(document.content),- connector_type=connection.connector_type,+ connector_type=connector_type,acl=document.acl,connector_file_id=document.id,**standard_kwargs,@@ -1023,9 +1030,10 @@ class ConnectorFileProcessor(TaskProcessor):await self.connector_service._update_connector_metadata(document,self.user_id,- connection.connector_type,+ connector_type,self.jwt_token,id_field="connector_file_id",+ indexed_filename=file_task.filename,)# Add connector-specific metadatadiff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.pyindex cd553dbd1..20c71d687 100644--- a/tests/unit/connectors/test_update_connector_metadata_filename.py+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py@@ -100,10 +100,7 @@ async def test_update_includes_filename_in_script_and_params(monkeypatch):@pytest.mark.asyncioasync def test_filename_passed_raw_not_cleaned(monkeypatch):- """process_document_standard indexes chunks with the raw document.filename- (see src/connectors/service.py:80 + src/models/processors.py:323).- The metadata update must use the SAME raw value or chunks drift.- """+ """When indexed_filename is omitted, metadata update uses document.filename."""service, opensearch_client = _make_service()async def _noop_acl(**_kwargs):@@ -124,6 +121,28 @@ async def test_filename_passed_raw_not_cleaned(monkeypatch):assert params["filename"] == raw_name+async def test_indexed_filename_overrides_document_filename(monkeypatch):+ """Connector processor indexes under a MIME-cleaned name; metadata must match."""+ service, opensearch_client = _make_service()++ async def _noop_acl(**_kwargs):+ return {"status": "unchanged"}++ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)++ document = _make_document(filename="My Report")+ await service._update_connector_metadata(+ document,+ owner_user_id="alice",+ connector_type="google_drive",+ indexed_filename="My Report.pdf",+ )++ params = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]["params"]+ assert params["filename"] == "My Report.pdf"++@pytest.mark.asyncioasync def test_filename_overwritten_for_existing_indexed_chunks(monkeypatch):"""End-to-end intent: when SharePoint rename triggers the unchanged ->diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex 37f527187..8cfc06ae1 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -225,6 +225,64 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(assert fields["document_id"] == "file-id-1"+async def test_connector_processor_indexes_cleaned_filename(monkeypatch):+ """MIME-enforced extensions must be indexed under the same name used for dedupe."""+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)+ processor = _build_connector_processor(replace_duplicates=False)+ document = ConnectorDocument(+ id="doc-id-1",+ filename="My Report",+ mimetype="application/vnd.google-apps.document",+ content=b"%PDF-1.4 dummy",+ source_url="https://example.google.com/file",+ acl=DocumentACL(owner="[email protected]"),+ modified_time=datetime.now(),+ created_time=datetime.now(),+ )+ _wire_connector_processor(processor, document, filename_exists=False)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(+ processor,+ "process_document_standard",+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),+ ) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.filename == "My Report.pdf"+ assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"+ metadata_call = processor.connector_service._update_connector_metadata.await_args+ assert metadata_call.kwargs["indexed_filename"] == "My Report.pdf"+++async def test_langflow_connector_processor_uses_cleaned_filename(monkeypatch):+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", False)+ processor = _build_langflow_processor(replace_duplicates=False)+ document = ConnectorDocument(+ id="doc-id-1",+ filename="My Report",+ mimetype="application/vnd.google-apps.document",+ content=b"%PDF-1.4 dummy",+ source_url="https://example.google.com/file",+ acl=DocumentACL(owner="[email protected]"),+ modified_time=datetime.now(),+ created_time=datetime.now(),+ )+ _wire_langflow_processor(processor, document, filename_exists=False)++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ upload_call = processor.connector_service.langflow_service.upload_and_ingest_file.await_args+ assert upload_call.kwargs["file_tuple"][0] == "My Report.pdf"++@pytest.mark.asyncioasync def test_connector_processor_proceeds_when_filename_absent(monkeypatch):monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)