Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: overwite duplicate files for connectors (#1892)

Wallgau417e20655b12
diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.ts
index af7faabde..ba262ec1b 100644
--- a/frontend/lib/knowledge-table-state.ts
+++ b/frontend/lib/knowledge-table-state.ts
@@ -28,15 +28,58 @@ export function getKnowledgeFileIdentity(file?: {
return "";
}
+function looksLikeHttpUrl(value: string): boolean {
+ return /^https?:\/\//i.test(value.trim());
+}
+
+/** Filename variants for overlay matching (mirrors backend `get_filename_aliases`). */
+function getKnowledgeFilenameAliases(filename?: string): string[] {
+ const normalized = filename?.trim() ?? "";
+ if (!normalized) {
+ return [];
+ }
+ if (looksLikeHttpUrl(normalized)) {
+ return [normalized];
+ }
+ const aliases = [normalized];
+ const lower = normalized.toLowerCase();
+ if (lower.endsWith(".txt")) {
+ aliases.push(`${normalized.slice(0, -4)}.md`);
+ } else if (lower.endsWith(".md")) {
+ aliases.push(`${normalized.slice(0, -3)}.txt`);
+ }
+ for (const name of [...aliases]) {
+ aliases.push(name.replace(/ /g, "_").replace(/\//g, "_"));
+ }
+ return [...new Set(aliases)];
+}
+
+function addFilenameAliasKeys(keys: Set<string>, filename?: string): void {
+ for (const alias of getKnowledgeFilenameAliases(filename)) {
+ keys.add(alias);
+ }
+}
+
+/** Lookup keys for matching task overlays to indexed rows (filename, path, basename). */
+export function getKnowledgeFileAliasKeys(file?: {
+ filename?: string;
+ source_url?: string;
+}): string[] {
+ const keys = new Set<string>();
+ addFilenameAliasKeys(keys, file?.filename);
+ const sourceUrl = file?.source_url?.trim();
+ if (sourceUrl) {
+ keys.add(sourceUrl);
+ addFilenameAliasKeys(keys, sourceUrl.split("/").pop());
+ }
+ return [...keys];
+}
+
function isMeaningfulConnectorType(connectorType?: string): boolean {
const normalized = connectorType?.trim();
return Boolean(normalized && normalized !== "local");
}
-function looksLikeHttpUrl(value: string): boolean {
- return /^https?:\/\//i.test(value.trim());
-}
-
/** Infer connector_type for task overlays when the API does not return it. */
export function inferTaskFileConnectorType(
filePath: string,
@@ -91,6 +134,50 @@ export function resolveKnowledgeRowConnectorType(
return taskType?.trim() || backendType?.trim() || "local";
}
+function taskOverlayPriority(status?: string): number {
+ switch (status) {
+ case "processing":
+ return 3;
+ case "failed":
+ return 2;
+ case "active":
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+function indexTaskFileOverlays(
+ taskFilesAsFiles: SearchFile[],
+): Map<string, SearchFile> {
+ const map = new Map<string, SearchFile>();
+ for (const file of taskFilesAsFiles) {
+ for (const key of getKnowledgeFileAliasKeys(file)) {
+ const existing = map.get(key);
+ if (
+ !existing ||
+ taskOverlayPriority(file.status) >= taskOverlayPriority(existing.status)
+ ) {
+ map.set(key, file);
+ }
+ }
+ }
+ return map;
+}
+
+function lookupTaskFileOverlay(
+ map: Map<string, SearchFile>,
+ file: SearchFile,
+): SearchFile | undefined {
+ for (const key of getKnowledgeFileAliasKeys(file)) {
+ const match = map.get(key);
+ if (match) {
+ return match;
+ }
+ }
+ return undefined;
+}
+
export function buildKnowledgeTableRows(
searchData: SearchFile[],
taskFiles: TaskFile[],
@@ -115,27 +202,32 @@ export function buildKnowledgeTableRows(
};
});
- const taskFileMap = new Map(
- taskFilesAsFiles.map((file) => [getKnowledgeFileIdentity(file), file]),
- );
+ const taskFileMap = indexTaskFileOverlays(taskFilesAsFiles);
const backendFiles = searchData.map((file) => {
if (file.connector_type === "openrag_docs") {
return file;
}
- const taskFile = taskFileMap.get(getKnowledgeFileIdentity(file));
+ const taskFile = lookupTaskFileOverlay(taskFileMap, file);
if (taskFile) {
const backendStatus = file.status ?? "active";
+ const status =
+ taskFile.status === "processing" || taskFile.status === "failed"
+ ? taskFile.status
+ : backendStatus;
return {
...file,
- filename: taskFile.filename,
- source_url: taskFile.source_url,
- connector_type: resolveKnowledgeRowConnectorType(
- file.connector_type,
- taskFile.connector_type,
- backendStatus,
- ),
- status: backendStatus,
+ // Indexed row identity: prefer backend fields so filename and source_url stay paired.
+ filename: file.filename || taskFile.filename,
+ source_url: file.source_url || taskFile.source_url,
+ connector_type: isMeaningfulConnectorType(file.connector_type)
+ ? file.connector_type!.trim()
+ : resolveKnowledgeRowConnectorType(
+ file.connector_type,
+ taskFile.connector_type,
+ status,
+ ),
+ status,
error: taskFile.error,
embedding_model: taskFile.embedding_model ?? file.embedding_model,
embedding_dimensions:
@@ -145,22 +237,28 @@ export function buildKnowledgeTableRows(
return file;
});
- const backendIdentities = new Set(
- backendFiles.map((f) => getKnowledgeFileIdentity(f)),
- );
+ const backendIdentityKeys = new Set<string>();
+ for (const file of searchData) {
+ for (const key of getKnowledgeFileAliasKeys(file)) {
+ backendIdentityKeys.add(key);
+ }
+ }
const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
if (
taskFile.filename === "OpenRAG docs refresh" ||
- taskFile.source_url.includes("openr.ag")
+ (taskFile.source_url ?? "").includes("openr.ag")
) {
return false;
}
if (taskFile.connector_type === "openrag_docs") {
return false;
}
- const identity = getKnowledgeFileIdentity(taskFile);
- if (backendIdentities.has(identity)) {
+ if (
+ getKnowledgeFileAliasKeys(taskFile).some((key) =>
+ backendIdentityKeys.has(key),
+ )
+ ) {
return false;
}
// Keep "active" overlays until the index lists the file (task drops key before refetch).
diff --git a/src/connectors/service.py b/src/connectors/service.py
index ade3f9e91..42e494d7b 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -219,6 +219,7 @@ class ConnectorService:
connector_type: str,
jwt_token: str = None,
id_field: str = "document_id",
+ indexed_filename: str | None = None,
):
"""Update indexed chunks with connector-specific metadata"""
from utils.acl_utils import update_document_acl
@@ -294,7 +295,7 @@ class ConnectorService:
"params": {
"source_url": document.source_url,
"connector_type": connector_type,
- "filename": document.filename,
+ "filename": indexed_filename or document.filename,
"created_time": document.created_time.isoformat()
if document.created_time
else None,
@@ -421,6 +422,7 @@ class ConnectorService:
),
models_service=self.models_service,
replace_duplicates=replace_duplicates,
+ connector_type=connector.CONNECTOR_TYPE,
)
# Use file IDs as items (no more fake file paths!)
@@ -603,6 +605,7 @@ class ConnectorService:
models_service=self.models_service,
ingest_settings=ingest_settings,
replace_duplicates=replace_duplicates,
+ connector_type=connector.CONNECTOR_TYPE,
)
# Create custom task using TaskService
diff --git a/src/models/processors.py b/src/models/processors.py
index c619c46a8..f93254848 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -698,6 +698,7 @@ class ConnectorFileProcessor(TaskProcessor):
models_service=None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
+ connector_type: str | None = None,
):
super().__init__(
document_service=document_service,
@@ -713,6 +714,7 @@ class ConnectorFileProcessor(TaskProcessor):
self.owner_email = owner_email
self.ingest_settings = ingest_settings
self.replace_duplicates = replace_duplicates
+ self.connector_type = connector_type
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using unified methods"""
@@ -730,6 +732,8 @@ class ConnectorFileProcessor(TaskProcessor):
if not connector or not connection:
raise ValueError(f"Connection '{self.connection_id}' not found")
+ connector_type = self.connector_type or connection.connector_type
+
# Validate file extension early if filename is available
VALID_EXTENSIONS = {
"adoc",
@@ -844,17 +848,20 @@ class ConnectorFileProcessor(TaskProcessor):
# filename differs from the current one. If any were removed (a real
# rename), force a re-ingest below so the file is re-indexed under
# the new name instead of short-circuiting as "unchanged".
+ # Match against file_task.filename — the cleaned name the file is
+ # actually indexed under — so duplicate/rename detection lines up
+ # with how chunks are keyed.
renamed = (
await self._delete_connector_chunks(
document.id,
opensearch_client,
self.user_id,
- keep_filenames=get_filename_aliases(document.filename),
+ keep_filenames=get_filename_aliases(file_task.filename),
)
> 0
)
- if await self.check_filename_exists(document.filename, opensearch_client):
+ if await self.check_filename_exists(file_task.filename, opensearch_client):
if not self.replace_duplicates:
file_task.status = TaskStatus.SKIPPED
file_task.error = None
@@ -867,13 +874,13 @@ class ConnectorFileProcessor(TaskProcessor):
upload_task.successful_files += 1
return
await self.delete_document_by_filename(
- document.filename,
+ file_task.filename,
opensearch_client,
owner_user_id=self.user_id,
)
# Create temporary file from document content
- suffix = os.path.splitext(document.filename)[1]
+ suffix = os.path.splitext(file_task.filename)[1]
if not suffix:
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
@@ -929,7 +936,7 @@ class ConnectorFileProcessor(TaskProcessor):
# Ingest via unified Langflow pipeline (two-phase Docling + Langflow run)
langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(
- document.filename, document.mimetype
+ file_task.filename, document.mimetype
)
file_tuple = (langflow_filename, document.content, processed_mimetype)
@@ -962,7 +969,7 @@ class ConnectorFileProcessor(TaskProcessor):
owner=self.user_id,
owner_name=self.owner_name,
owner_email=self.owner_email,
- connector_type=connection.connector_type,
+ connector_type=connector_type,
docling_polling_service=self.connector_service.task_service.docling_polling_service
if self.connector_service.task_service
else None,
@@ -1007,12 +1014,12 @@ class ConnectorFileProcessor(TaskProcessor):
file_path=tmp_path,
file_hash=file_hash,
owner_user_id=self.user_id,
- original_filename=document.filename,
+ original_filename=file_task.filename,
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=len(document.content),
- connector_type=connection.connector_type,
+ connector_type=connector_type,
acl=document.acl,
connector_file_id=document.id,
**standard_kwargs,
@@ -1023,9 +1030,10 @@ class ConnectorFileProcessor(TaskProcessor):
await self.connector_service._update_connector_metadata(
document,
self.user_id,
- connection.connector_type,
+ connector_type,
self.jwt_token,
id_field="connector_file_id",
+ indexed_filename=file_task.filename,
)
# Add connector-specific metadata
diff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.py
index cd553dbd1..20c71d687 100644
--- a/tests/unit/connectors/test_update_connector_metadata_filename.py
+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py
@@ -100,10 +100,7 @@ async def test_update_includes_filename_in_script_and_params(monkeypatch):
@pytest.mark.asyncio
async def test_filename_passed_raw_not_cleaned(monkeypatch):
- """process_document_standard indexes chunks with the raw document.filename
- (see src/connectors/service.py:80 + src/models/processors.py:323).
- The metadata update must use the SAME raw value or chunks drift.
- """
+ """When indexed_filename is omitted, metadata update uses document.filename."""
service, opensearch_client = _make_service()
async def _noop_acl(**_kwargs):
@@ -124,6 +121,28 @@ async def test_filename_passed_raw_not_cleaned(monkeypatch):
assert params["filename"] == raw_name
+async def test_indexed_filename_overrides_document_filename(monkeypatch):
+ """Connector processor indexes under a MIME-cleaned name; metadata must match."""
+ service, opensearch_client = _make_service()
+
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ document = _make_document(filename="My Report")
+ await service._update_connector_metadata(
+ document,
+ owner_user_id="alice",
+ connector_type="google_drive",
+ indexed_filename="My Report.pdf",
+ )
+
+ params = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]["params"]
+ assert params["filename"] == "My Report.pdf"
+
+
@pytest.mark.asyncio
async def test_filename_overwritten_for_existing_indexed_chunks(monkeypatch):
"""End-to-end intent: when SharePoint rename triggers the unchanged ->
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
index 37f527187..8cfc06ae1 100644
--- a/tests/unit/test_connector_processor_filename_dedupe.py
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -225,6 +225,64 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(
assert fields["document_id"] == "file-id-1"
+async def test_connector_processor_indexes_cleaned_filename(monkeypatch):
+ """MIME-enforced extensions must be indexed under the same name used for dedupe."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_connector_processor(replace_duplicates=False)
+ document = ConnectorDocument(
+ id="doc-id-1",
+ filename="My Report",
+ mimetype="application/vnd.google-apps.document",
+ content=b"%PDF-1.4 dummy",
+ source_url="https://example.google.com/file",
+ acl=DocumentACL(owner="[email protected]"),
+ modified_time=datetime.now(),
+ created_time=datetime.now(),
+ )
+ _wire_connector_processor(processor, document, filename_exists=False)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),
+ ) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.filename == "My Report.pdf"
+ assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"
+ metadata_call = processor.connector_service._update_connector_metadata.await_args
+ assert metadata_call.kwargs["indexed_filename"] == "My Report.pdf"
+
+
+async def test_langflow_connector_processor_uses_cleaned_filename(monkeypatch):
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", False)
+ processor = _build_langflow_processor(replace_duplicates=False)
+ document = ConnectorDocument(
+ id="doc-id-1",
+ filename="My Report",
+ mimetype="application/vnd.google-apps.document",
+ content=b"%PDF-1.4 dummy",
+ source_url="https://example.google.com/file",
+ acl=DocumentACL(owner="[email protected]"),
+ modified_time=datetime.now(),
+ created_time=datetime.now(),
+ )
+ _wire_langflow_processor(processor, document, filename_exists=False)
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ upload_call = processor.connector_service.langflow_service.upload_and_ingest_file.await_args
+ assert upload_call.kwargs["file_tuple"][0] == "My Report.pdf"
+
+
@pytest.mark.asyncio
async def test_connector_processor_proceeds_when_filename_absent(monkeypatch):
monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)