Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Treat .txt and .md filenames as duplicates.

Olfa Maslah6fce94b5c947
diff --git a/src/api/documents.py b/src/api/documents.py
index 8169a3d4b..963ed568a 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -121,16 +121,24 @@ async def check_filename_exists(
)
from utils.opensearch_queries import build_filename_search_body
+ from utils.file_utils import get_filename_aliases
- search_body = build_filename_search_body(filename, size=1, source=["filename"])
+ candidate_filenames = get_filename_aliases(filename) or [filename]
logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())
try:
- response = await opensearch_client.search(
- index=get_index_name(),
- body=search_body
- )
+ exists = False
+ for candidate in candidate_filenames:
+ search_body = build_filename_search_body(candidate, size=1, source=["filename"])
+ response = await opensearch_client.search(
+ index=get_index_name(),
+ body=search_body
+ )
+ hits = response.get("hits", {}).get("hits", [])
+ if hits:
+ exists = True
+ break
except Exception as search_err:
if "index_not_found_exception" in str(search_err):
logger.info("Index does not exist, creating it now before upload")
@@ -138,9 +146,6 @@ async def check_filename_exists(
return JSONResponse({"exists": False, "filename": filename}, status_code=200)
raise
- hits = response.get("hits", {}).get("hits", [])
- exists = len(hits) > 0
-
return JSONResponse({"exists": exists, "filename": filename}, status_code=200)
except Exception as e:
diff --git a/src/models/processors.py b/src/models/processors.py
index f4fb72694..0a8af28ef 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1,7 +1,11 @@
from typing import Any
from .tasks import UploadTask, FileTask
from utils.logging_config import get_logger
-from utils.file_utils import get_file_extension, clean_connector_filename
+from utils.file_utils import (
+ get_file_extension,
+ clean_connector_filename,
+ get_filename_aliases,
+)
logger = get_logger(__name__)
@@ -72,19 +76,22 @@ class TaskProcessor:
max_retries = 3
retry_delay = 1.0
+ candidate_filenames = get_filename_aliases(filename) or [filename]
+
for attempt in range(max_retries):
try:
- # Search for any document with this exact filename
- search_body = build_filename_search_body(filename, size=1, source=False)
-
- response = await opensearch_client.search(
- index=get_index_name(),
- body=search_body
- )
-
- # Check if any hits were found
- hits = response.get("hits", {}).get("hits", [])
- return len(hits) > 0
+ for candidate in candidate_filenames:
+ search_body = build_filename_search_body(
+ candidate, size=1, source=False
+ )
+ response = await opensearch_client.search(
+ index=get_index_name(),
+ body=search_body
+ )
+ hits = response.get("hits", {}).get("hits", [])
+ if hits:
+ return True
+ return False
except (asyncio.TimeoutError, Exception) as e:
if attempt == max_retries - 1:
@@ -123,15 +130,15 @@ class TaskProcessor:
from utils.opensearch_queries import build_filename_delete_body
try:
- # Delete all documents with this filename
- delete_body = build_filename_delete_body(filename)
-
- response = await opensearch_client.delete_by_query(
- index=get_index_name(),
- body=delete_body
- )
-
- deleted_count = response.get("deleted", 0)
+ deleted_count = 0
+ candidate_filenames = get_filename_aliases(filename) or [filename]
+ for candidate in candidate_filenames:
+ delete_body = build_filename_delete_body(candidate)
+ response = await opensearch_client.delete_by_query(
+ index=get_index_name(),
+ body=delete_body
+ )
+ deleted_count += response.get("deleted", 0)
logger.info(
"Deleted existing document chunks",
filename=filename,
diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py
index 44d0bb98e..ff1789fee 100644
--- a/src/utils/file_utils.py
+++ b/src/utils/file_utils.py
@@ -98,4 +98,26 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:
return clean_name
if not clean_name.lower().endswith(suffix.lower()):
return clean_name + suffix
- return clean_name
\ No newline at end of file
+ return clean_name
+
+
+def get_filename_aliases(filename: str) -> list[str]:
+ """Return equivalent filename variants used by ingestion/indexing.
+
+ Legacy Langflow ingest may index `.txt` uploads as `.md`. This helper keeps
+ duplicate detection/deletion consistent by checking both forms.
+ """
+ normalized = (filename or "").strip()
+ if not normalized:
+ return []
+
+ aliases = [normalized]
+ lower_name = normalized.lower()
+
+ if lower_name.endswith(".txt"):
+ aliases.append(normalized[:-4] + ".md")
+ elif lower_name.endswith(".md"):
+ aliases.append(normalized[:-3] + ".txt")
+
+ # Keep order stable while removing duplicates.
+ return list(dict.fromkeys(aliases))