OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: tighten filename alias handling in duplicate checks
Olfa Maslahb3eeaf998ff8
diff --git a/src/api/documents.py b/src/api/documents.pyindex 963ed568a..a211dc8ad 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -123,12 +123,14 @@ async def check_filename_exists(from utils.opensearch_queries import build_filename_search_bodyfrom utils.file_utils import get_filename_aliases- candidate_filenames = get_filename_aliases(filename) or [filename]+ candidate_filenames = get_filename_aliases(filename)+ if not candidate_filenames:+ return JSONResponse({"exists": False, "filename": filename}, status_code=200)logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())+ exists = Falsetry:- exists = Falsefor candidate in candidate_filenames:search_body = build_filename_search_body(candidate, size=1, source=["filename"])response = await opensearch_client.search(diff --git a/src/models/processors.py b/src/models/processors.pyindex 0a8af28ef..0931295f3 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -76,11 +76,22 @@ class TaskProcessor:max_retries = 3retry_delay = 1.0- candidate_filenames = get_filename_aliases(filename) or [filename]+ candidate_filenames = get_filename_aliases(filename)+ if not candidate_filenames:+ return False+ # Keep track of aliases that still need checking across retries.+ # If one alias was already checked successfully with no hits, we avoid+ # re-querying it when another alias fails transiently.+ pending_candidates = list(candidate_filenames)+ # Retry strategy: only retry aliases that have not completed successfully.+ # This avoids re-querying aliases already checked with no hits when a later+ # alias fails transiently (e.g., timeout).for attempt in range(max_retries):try:- for candidate in candidate_filenames:+ i = 0+ while i < len(pending_candidates):+ candidate = pending_candidates[i]search_body = build_filename_search_body(candidate, size=1, source=False)@@ -91,6 +102,10 @@ class TaskProcessor:hits = response.get("hits", {}).get("hits", [])if hits:return True+ # Successfully checked this alias with no hits; don't+ # re-query it on future retries.+ pending_candidates.pop(i)+ continuereturn Falseexcept (asyncio.TimeoutError, Exception) as e:@@ -131,7 +146,13 @@ class TaskProcessor:try:deleted_count = 0- candidate_filenames = get_filename_aliases(filename) or [filename]+ candidate_filenames = get_filename_aliases(filename)+ if not candidate_filenames:+ logger.info(+ "Skipped delete_by_filename due to empty filename input",+ filename=filename,+ )+ returnfor candidate in candidate_filenames:delete_body = build_filename_delete_body(candidate)response = await opensearch_client.delete_by_query(diff --git a/src/utils/file_utils.py b/src/utils/file_utils.pyindex ff1789fee..989875b0f 100644--- a/src/utils/file_utils.py+++ b/src/utils/file_utils.py@@ -104,8 +104,14 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:def get_filename_aliases(filename: str) -> list[str]:"""Return equivalent filename variants used by ingestion/indexing.- Legacy Langflow ingest may index `.txt` uploads as `.md`. This helper keeps- duplicate detection/deletion consistent by checking both forms.+ Legacy Langflow ingest indexes `.txt` uploads as `.md` (see+ `LangflowFileProcessor`). The alias always uses a lowercase extension+ to match the rename behavior:+ `original_filename[:-4] + ".md"`+ So `"FOO.TXT"` aliases to `"FOO.md"`, not `"FOO.MD"`.++ This helper keeps duplicate detection/deletion consistent by checking+ both `.txt` and `.md` forms."""normalized = (filename or "").strip()if not normalized:diff --git a/tests/unit/test_processors_filename_alias_retry.py b/tests/unit/test_processors_filename_alias_retry.pynew file mode 100644index 000000000..7a690a4ef--- /dev/null+++ b/tests/unit/test_processors_filename_alias_retry.py@@ -0,0 +1,37 @@+import asyncio+from unittest.mock import AsyncMock++import pytest++from models.processors import TaskProcessor+++async def test_check_filename_exists_does_not_requery_successfully_checked_aliases_on_retry():+ """Once an alias is checked successfully, retries should continue from pending aliases."""+ processor = TaskProcessor()+ opensearch_client = AsyncMock()++ async def _search_side_effect(*, index, body):+ candidate = body["query"]["term"]["filename"]+ if candidate == "report.md" and _search_side_effect.md_calls == 0:+ _search_side_effect.md_calls += 1+ raise asyncio.TimeoutError("transient timeout")+ return {"hits": {"hits": []}}++ _search_side_effect.md_calls = 0+ opensearch_client.search.side_effect = _search_side_effect++ exists = await processor.check_filename_exists("report.txt", opensearch_client)++ assert exists is False++ queried_candidates = [+ call.kwargs["body"]["query"]["term"]["filename"]+ for call in opensearch_client.search.await_args_list+ ]+ # Expected sequence:+ # 1) report.txt succeeds (no hits)+ # 2) report.md times out+ # 3) retry continues from pending alias only -> report.md+ assert queried_candidates == ["report.txt", "report.md", "report.md"]