OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Treat .txt and .md filenames as duplicates.
Olfa Maslah6fce94b5c947
diff --git a/src/api/documents.py b/src/api/documents.pyindex 8169a3d4b..963ed568a 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -121,16 +121,24 @@ async def check_filename_exists()from utils.opensearch_queries import build_filename_search_body+ from utils.file_utils import get_filename_aliases- search_body = build_filename_search_body(filename, size=1, source=["filename"])+ candidate_filenames = get_filename_aliases(filename) or [filename]logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())try:- response = await opensearch_client.search(- index=get_index_name(),- body=search_body- )+ exists = False+ for candidate in candidate_filenames:+ search_body = build_filename_search_body(candidate, size=1, source=["filename"])+ response = await opensearch_client.search(+ index=get_index_name(),+ body=search_body+ )+ hits = response.get("hits", {}).get("hits", [])+ if hits:+ exists = True+ breakexcept Exception as search_err:if "index_not_found_exception" in str(search_err):logger.info("Index does not exist, creating it now before upload")@@ -138,9 +146,6 @@ async def check_filename_exists(return JSONResponse({"exists": False, "filename": filename}, status_code=200)raise- hits = response.get("hits", {}).get("hits", [])- exists = len(hits) > 0-return JSONResponse({"exists": exists, "filename": filename}, status_code=200)except Exception as e:diff --git a/src/models/processors.py b/src/models/processors.pyindex f4fb72694..0a8af28ef 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -1,7 +1,11 @@from typing import Anyfrom .tasks import UploadTask, FileTaskfrom utils.logging_config import get_logger-from utils.file_utils import get_file_extension, clean_connector_filename+from utils.file_utils import (+ get_file_extension,+ clean_connector_filename,+ get_filename_aliases,+)logger = get_logger(__name__)@@ -72,19 +76,22 @@ class TaskProcessor:max_retries = 3retry_delay = 1.0+ candidate_filenames = get_filename_aliases(filename) or [filename]+for attempt in range(max_retries):try:- # Search for any document with this exact filename- search_body = build_filename_search_body(filename, size=1, source=False)-- response = await opensearch_client.search(- index=get_index_name(),- body=search_body- )-- # Check if any hits were found- hits = response.get("hits", {}).get("hits", [])- return len(hits) > 0+ for candidate in candidate_filenames:+ search_body = build_filename_search_body(+ candidate, size=1, source=False+ )+ response = await opensearch_client.search(+ index=get_index_name(),+ body=search_body+ )+ hits = response.get("hits", {}).get("hits", [])+ if hits:+ return True+ return Falseexcept (asyncio.TimeoutError, Exception) as e:if attempt == max_retries - 1:@@ -123,15 +130,15 @@ class TaskProcessor:from utils.opensearch_queries import build_filename_delete_bodytry:- # Delete all documents with this filename- delete_body = build_filename_delete_body(filename)-- response = await opensearch_client.delete_by_query(- index=get_index_name(),- body=delete_body- )-- deleted_count = response.get("deleted", 0)+ deleted_count = 0+ candidate_filenames = get_filename_aliases(filename) or [filename]+ for candidate in candidate_filenames:+ delete_body = build_filename_delete_body(candidate)+ response = await opensearch_client.delete_by_query(+ index=get_index_name(),+ body=delete_body+ )+ deleted_count += response.get("deleted", 0)logger.info("Deleted existing document chunks",filename=filename,diff --git a/src/utils/file_utils.py b/src/utils/file_utils.pyindex 44d0bb98e..ff1789fee 100644--- a/src/utils/file_utils.py+++ b/src/utils/file_utils.py@@ -98,4 +98,26 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:return clean_nameif not clean_name.lower().endswith(suffix.lower()):return clean_name + suffix- return clean_name\ No newline at end of file+ return clean_name+++def get_filename_aliases(filename: str) -> list[str]:+ """Return equivalent filename variants used by ingestion/indexing.++ Legacy Langflow ingest may index `.txt` uploads as `.md`. This helper keeps+ duplicate detection/deletion consistent by checking both forms.+ """+ normalized = (filename or "").strip()+ if not normalized:+ return []++ aliases = [normalized]+ lower_name = normalized.lower()++ if lower_name.endswith(".txt"):+ aliases.append(normalized[:-4] + ".md")+ elif lower_name.endswith(".md"):+ aliases.append(normalized[:-3] + ".txt")++ # Keep order stable while removing duplicates.+ return list(dict.fromkeys(aliases))