Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: tighten filename alias handling in duplicate checks

Olfa Maslahb3eeaf998ff8
diff --git a/src/api/documents.py b/src/api/documents.py
index 963ed568a..a211dc8ad 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -123,12 +123,14 @@ async def check_filename_exists(
from utils.opensearch_queries import build_filename_search_body
from utils.file_utils import get_filename_aliases
- candidate_filenames = get_filename_aliases(filename) or [filename]
+ candidate_filenames = get_filename_aliases(filename)
+ if not candidate_filenames:
+ return JSONResponse({"exists": False, "filename": filename}, status_code=200)
logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())
+ exists = False
try:
- exists = False
for candidate in candidate_filenames:
search_body = build_filename_search_body(candidate, size=1, source=["filename"])
response = await opensearch_client.search(
diff --git a/src/models/processors.py b/src/models/processors.py
index 0a8af28ef..0931295f3 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -76,11 +76,22 @@ class TaskProcessor:
max_retries = 3
retry_delay = 1.0
- candidate_filenames = get_filename_aliases(filename) or [filename]
+ candidate_filenames = get_filename_aliases(filename)
+ if not candidate_filenames:
+ return False
+ # Keep track of aliases that still need checking across retries.
+ # If one alias was already checked successfully with no hits, we avoid
+ # re-querying it when another alias fails transiently.
+ pending_candidates = list(candidate_filenames)
+ # Retry strategy: only retry aliases that have not completed successfully.
+ # This avoids re-querying aliases already checked with no hits when a later
+ # alias fails transiently (e.g., timeout).
for attempt in range(max_retries):
try:
- for candidate in candidate_filenames:
+ i = 0
+ while i < len(pending_candidates):
+ candidate = pending_candidates[i]
search_body = build_filename_search_body(
candidate, size=1, source=False
)
@@ -91,6 +102,10 @@ class TaskProcessor:
hits = response.get("hits", {}).get("hits", [])
if hits:
return True
+ # Successfully checked this alias with no hits; don't
+ # re-query it on future retries.
+ pending_candidates.pop(i)
+ continue
return False
except (asyncio.TimeoutError, Exception) as e:
@@ -131,7 +146,13 @@ class TaskProcessor:
try:
deleted_count = 0
- candidate_filenames = get_filename_aliases(filename) or [filename]
+ candidate_filenames = get_filename_aliases(filename)
+ if not candidate_filenames:
+ logger.info(
+ "Skipped delete_by_filename due to empty filename input",
+ filename=filename,
+ )
+ return
for candidate in candidate_filenames:
delete_body = build_filename_delete_body(candidate)
response = await opensearch_client.delete_by_query(
diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py
index ff1789fee..989875b0f 100644
--- a/src/utils/file_utils.py
+++ b/src/utils/file_utils.py
@@ -104,8 +104,14 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:
def get_filename_aliases(filename: str) -> list[str]:
"""Return equivalent filename variants used by ingestion/indexing.
- Legacy Langflow ingest may index `.txt` uploads as `.md`. This helper keeps
- duplicate detection/deletion consistent by checking both forms.
+ Legacy Langflow ingest indexes `.txt` uploads as `.md` (see
+ `LangflowFileProcessor`). The alias always uses a lowercase extension
+ to match the rename behavior:
+ `original_filename[:-4] + ".md"`
+ So `"FOO.TXT"` aliases to `"FOO.md"`, not `"FOO.MD"`.
+
+ This helper keeps duplicate detection/deletion consistent by checking
+ both `.txt` and `.md` forms.
"""
normalized = (filename or "").strip()
if not normalized:
diff --git a/tests/unit/test_processors_filename_alias_retry.py b/tests/unit/test_processors_filename_alias_retry.py
new file mode 100644
index 000000000..7a690a4ef
--- /dev/null
+++ b/tests/unit/test_processors_filename_alias_retry.py
@@ -0,0 +1,37 @@
+import asyncio
+from unittest.mock import AsyncMock
+
+import pytest
+
+from models.processors import TaskProcessor
+
+
+async def test_check_filename_exists_does_not_requery_successfully_checked_aliases_on_retry():
+ """Once an alias is checked successfully, retries should continue from pending aliases."""
+ processor = TaskProcessor()
+ opensearch_client = AsyncMock()
+
+ async def _search_side_effect(*, index, body):
+ candidate = body["query"]["term"]["filename"]
+ if candidate == "report.md" and _search_side_effect.md_calls == 0:
+ _search_side_effect.md_calls += 1
+ raise asyncio.TimeoutError("transient timeout")
+ return {"hits": {"hits": []}}
+
+ _search_side_effect.md_calls = 0
+ opensearch_client.search.side_effect = _search_side_effect
+
+ exists = await processor.check_filename_exists("report.txt", opensearch_client)
+
+ assert exists is False
+
+ queried_candidates = [
+ call.kwargs["body"]["query"]["term"]["filename"]
+ for call in opensearch_client.search.await_args_list
+ ]
+ # Expected sequence:
+ # 1) report.txt succeeds (no hits)
+ # 2) report.md times out
+ # 3) retry continues from pending alias only -> report.md
+ assert queried_candidates == ["report.txt", "report.md", "report.md"]