OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Centralize .txt->.md workaround for Langflow (#1651)
Edwin Jose7e33b4ece7e8
diff --git a/src/api/langflow_files.py b/src/api/langflow_files.pyindex b3b80c4bb..dc848067e 100644--- a/src/api/langflow_files.py+++ b/src/api/langflow_files.py@@ -1,43 +1,43 @@import jsonimport osimport tempfile-from typing import List, Optionalfrom fastapi import Depends, File, Form, UploadFile-from pydantic import BaseModelfrom fastapi.responses import JSONResponse+from pydantic import BaseModelfrom dependencies import (+ get_current_user,get_langflow_file_service,+ get_optional_user,get_session_manager,get_task_service,- get_current_user,- get_optional_user,)from session_manager import User+from utils.file_utils import langflow_safe_filename_and_mimetypefrom utils.logging_config import get_loggerlogger = get_logger(__name__)class RunIngestionBody(BaseModel):- file_paths: List[str] = []- file_ids: Optional[List[str]] = None- file_metadata: Optional[List[dict]] = None- session_id: Optional[str] = None- tweaks: Optional[dict] = None- settings: Optional[dict] = None+ file_paths: list[str] = []+ file_ids: list[str] | None = None+ file_metadata: list[dict] | None = None+ session_id: str | None = None+ tweaks: dict | None = None+ settings: dict | None = Noneclass DeleteFilesBody(BaseModel):- file_ids: List[str]+ file_ids: list[str]async def upload_user_file(file: UploadFile = File(...),langflow_file_service=Depends(get_langflow_file_service),session_manager=Depends(get_session_manager),- user: Optional[User] = Depends(get_optional_user),+ user: User | None = Depends(get_optional_user),):"""Upload a file to Langflow's Files API"""try:@@ -45,11 +45,11 @@ async def upload_user_file(logger.debug("Processing file", filename=file.filename, size=file.size)content = await file.read()- file_tuple = (- file.filename,- content,- file.content_type or "application/octet-stream",+ # Langflow's docling chokes on text/plain — rename .txt -> .md.+ upload_filename, upload_mimetype = langflow_safe_filename_and_mimetype(+ file.filename, file.content_type)+ file_tuple = (upload_filename, content, upload_mimetype)jwt_token = user.jwt_token if user else session_manager.get_effective_jwt_token(None, None)@@ -60,6 +60,7 @@ async def upload_user_file(except Exception as e:logger.error("upload_user_file endpoint failed", error_type=type(e).__name__, error=str(e))import traceback+logger.error("Full traceback", traceback=traceback.format_exc())return JSONResponse({"error": str(e)}, status_code=500)@@ -111,6 +112,7 @@ async def run_ingestion(if jwt_token:from auth_context import set_auth_context+set_auth_context(user.user_id, jwt_token)try:@@ -132,9 +134,9 @@ async def run_ingestion(async def upload_and_ingest_user_file(file: UploadFile = File(...),- session_id: Optional[str] = Form(None),- settings_json: Optional[str] = Form(None, alias="settings"),- tweaks_json: Optional[str] = Form(None, alias="tweaks"),+ session_id: str | None = Form(None),+ settings_json: str | None = Form(None, alias="settings"),+ tweaks_json: str | None = Form(None, alias="tweaks"),langflow_file_service=Depends(get_langflow_file_service),session_manager=Depends(get_session_manager),task_service=Depends(get_task_service),@@ -191,6 +193,7 @@ async def upload_and_ingest_user_file()except Exception:from utils.file_utils import safe_unlink+safe_unlink(temp_path)raise@@ -219,4 +222,4 @@ async def delete_user_files("errors": errors,},status_code=status,- )\ No newline at end of file+ )diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex e7ee81cd2..8214f2a63 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -3,7 +3,11 @@ from typing import Any# Create custom processor for connector files using Langflowfrom models.processors import LangflowConnectorFileProcessorfrom services.langflow_file_service import LangflowFileService-from utils.file_utils import clean_connector_filename, get_file_extension+from utils.file_utils import (+ clean_connector_filename,+ get_file_extension,+ langflow_safe_filename_and_mimetype,+)from utils.logging_config import get_loggerfrom .base import BaseConnector, ConnectorDocument@@ -76,11 +80,15 @@ class LangflowConnectorService:# Clean filename and ensure we don't add a double extensionprocessed_filename = clean_connector_filename(document.filename, document.mimetype)+ # Langflow's docling chokes on text/plain — rename .txt -> .md.+ processed_filename, processed_mimetype = langflow_safe_filename_and_mimetype(+ processed_filename, document.mimetype+ )file_tuple = (processed_filename,content,- document.mimetype or "application/octet-stream",+ processed_mimetype,)# Step 0: Delete existing chunks for this file before re-ingesting.diff --git a/src/models/processors.py b/src/models/processors.pyindex ba7ff205b..3c84ffe7e 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -17,6 +17,7 @@ from utils.file_utils import (clean_connector_filename,get_file_extension,get_filename_aliases,+ langflow_safe_filename_and_mimetype,)from utils.hash_utils import hash_idfrom utils.logging_config import get_logger@@ -899,14 +900,10 @@ class LangflowFileProcessor(TaskProcessor):if not content_type:content_type = "application/octet-stream"- # Rename .txt to .md for Langflow compatibility- # Langflow has issues processing text/plain files- langflow_filename = original_filename- if original_filename.lower().endswith(".txt"):- langflow_filename = original_filename[:-4] + ".md"- content_type = "text/markdown"- logger.debug(f"Renamed {original_filename} to {langflow_filename} for Langflow")-+ # Langflow's docling chokes on text/plain — rename .txt -> .md.+ langflow_filename, content_type = langflow_safe_filename_and_mimetype(+ original_filename, content_type+ )file_tuple = (langflow_filename, content, content_type)# Get JWT token using same logic as DocumentFileProcessordiff --git a/src/utils/file_utils.py b/src/utils/file_utils.pyindex f2a50dfe2..845da8647 100644--- a/src/utils/file_utils.py+++ b/src/utils/file_utils.py@@ -102,6 +102,25 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:return filename+def langflow_safe_filename_and_mimetype(filename: str, mimetype: str | None) -> tuple[str, str]:+ """Apply the .txt -> .md workaround for Langflow ingestion.++ Langflow's docling component fails on text/plain (see commit f6b9fe0).+ The workaround is to rename .txt files to .md with text/markdown before+ handing the file to Langflow. Centralised here so every Langflow-upload+ site applies it identically — local user uploads, connector-driven+ ingestion, and the direct upload route.++ Returns the (possibly rewritten) (filename, mimetype) pair. Pass-through+ for all other extensions; falls back to application/octet-stream when+ mimetype is missing.+ """+ safe_mimetype = mimetype or "application/octet-stream"+ if filename and filename.lower().endswith(".txt"):+ return filename[:-4] + ".md", "text/markdown"+ return filename, safe_mimetype++def get_filename_aliases(filename: str) -> list[str]:"""Return equivalent filename variants used by ingestion/indexing.diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.pynew file mode 100644index 000000000..cb267aff5--- /dev/null+++ b/tests/unit/connectors/test_langflow_connector_txt_to_md.py@@ -0,0 +1,131 @@+"""Regression guard for SharePoint .txt ingestion via the Langflow connector+path.++Background: Langflow's docling component fails on text/plain. The+user-upload path already applies a .txt -> .md rename (commit f6b9fe0).+This test pins that the connector path+(`LangflowConnectorService.process_connector_document`) applies the same+rule, so a SharePoint .txt reaches Langflow as .md / text/markdown — not+the unhandled text/plain shape that previously broke ingestion.++This test drives the production code path; if the rename is ever removed+or shorted around, the assertions will catch it.+"""++import sys+from datetime import datetime+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_service():+ """Build a LangflowConnectorService with the surface+ process_connector_document touches stubbed:+ - session_manager (only needed if pre-delete fires; we let it run as+ a no-op against a mocked OpenSearch client).+ - langflow_service.upload_user_file / run_ingestion_flow / delete_user_file.+ """+ from connectors.langflow_connector_service import LangflowConnectorService++ service = LangflowConnectorService.__new__(LangflowConnectorService)++ # OpenSearch mock for the pre-delete step (collect+delete). Empty result+ # so the pre-delete is a no-op; the test isn't about that path.+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})++ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ service.session_manager = session_manager+ service.docling_service = MagicMock()+ service.connection_manager = MagicMock()++ # Mock Langflow service — the surface we're asserting against is its+ # upload_user_file (the file_tuple it receives).+ langflow_service = MagicMock()+ langflow_service.upload_user_file = AsyncMock(+ return_value={"id": "lf-file-id", "path": "/lf/path/file"}+ )+ langflow_service.run_ingestion_flow = AsyncMock(return_value={"status": "ok"})+ langflow_service.delete_user_file = AsyncMock()+ service.langflow_service = langflow_service++ return service, langflow_service+++def _make_document(filename: str, mimetype: str, content: bytes = b"hello world"):+ from connectors.base import ConnectorDocument, DocumentACL++ return ConnectorDocument(+ id="graph-item-id-stable",+ filename=filename,+ mimetype=mimetype,+ content=content,+ source_url="https://contoso.sharepoint.com/.../notes.txt",+ acl=DocumentACL(owner="alice"),+ modified_time=datetime(2026, 5, 21),+ created_time=datetime(2026, 5, 1),+ metadata={"site": "marketing"},+ )+++async def test_sharepoint_txt_is_uploaded_to_langflow_as_md():+ """A connector document with mimetype text/plain must reach Langflow's+ upload_user_file with a .md filename and text/markdown mimetype."""+ service, langflow_service = _make_service()+ document = _make_document(filename="notes.txt", mimetype="text/plain")++ # auto_cleanup_tempfile is a context manager that yields a real path;+ # let it run normally — it just creates a temp file we won't use.+ await service.process_connector_document(+ document=document,+ owner_user_id="alice",+ connector_type="sharepoint",+ jwt_token="jwt",+ )++ langflow_service.upload_user_file.assert_awaited_once()+ (file_tuple, _jwt) = langflow_service.upload_user_file.await_args.args+ filename, content, mimetype = file_tuple++ assert filename.endswith(".md"), (+ f"SharePoint .txt must reach Langflow as .md; got {filename!r}. "+ "Langflow's docling component fails on text/plain — see "+ "langflow_safe_filename_and_mimetype."+ )+ assert mimetype == "text/markdown", (+ f"SharePoint .txt must reach Langflow as text/markdown; got {mimetype!r}."+ )+ # Content bytes are unchanged by the rename.+ assert content == document.content+++async def test_sharepoint_pdf_passes_through_untouched():+ """The rename rule must NOT touch non-.txt files."""+ service, langflow_service = _make_service()+ document = _make_document(+ filename="report.pdf", mimetype="application/pdf", content=b"%PDF-1.4..."+ )++ await service.process_connector_document(+ document=document,+ owner_user_id="alice",+ connector_type="sharepoint",+ jwt_token="jwt",+ )++ (file_tuple, _jwt) = langflow_service.upload_user_file.await_args.args+ filename, _content, mimetype = file_tuple+ assert filename == "report.pdf"+ assert mimetype == "application/pdf"diff --git a/tests/unit/test_langflow_safe_filename.py b/tests/unit/test_langflow_safe_filename.pynew file mode 100644index 000000000..cdf164157--- /dev/null+++ b/tests/unit/test_langflow_safe_filename.py@@ -0,0 +1,66 @@+"""Pin the .txt -> .md workaround applied at every Langflow upload site.++Langflow's docling component fails on text/plain (commit f6b9fe0). The+helper at `src/utils/file_utils.py::langflow_safe_filename_and_mimetype` is+the single source of truth for that rename — every place that builds a+file_tuple for `LangflowFileService.upload_user_file` must call it.+"""++import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++from utils.file_utils import langflow_safe_filename_and_mimetype # noqa: E402+++ "filename,mimetype,expected_filename,expected_mimetype",+ [+ # Core rule: .txt becomes .md, mimetype switches to text/markdown.+ ("foo.txt", "text/plain", "foo.md", "text/markdown"),+ # Case-insensitive: extension match ignores case but preserves the+ # original stem case; only the extension is rewritten to ".md".+ ("FOO.TXT", "text/plain", "FOO.md", "text/markdown"),+ ("Report.Txt", "text/plain", "Report.md", "text/markdown"),+ # Pass-through: .md is already fine.+ ("notes.md", "text/markdown", "notes.md", "text/markdown"),+ # Pass-through: unrelated extensions.+ ("doc.pdf", "application/pdf", "doc.pdf", "application/pdf"),+ ("sheet.csv", "text/csv", "sheet.csv", "text/csv"),+ # Missing mimetype: still rewrite .txt to .md (the bug we're fixing is+ # about Langflow tripping on text/plain — once we rewrite, we own the+ # mimetype). Non-.txt without mimetype gets the octet-stream default.+ ("foo.txt", None, "foo.md", "text/markdown"),+ ("foo.pdf", None, "foo.pdf", "application/octet-stream"),+ # Defensive: empty filename returns the octet-stream default, never crashes.+ ("", "text/plain", "", "text/plain"),+ ("", None, "", "application/octet-stream"),+ ],+)+def test_helper_applies_txt_to_md_rule(filename, mimetype, expected_filename, expected_mimetype):+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype(filename, mimetype)+ assert out_filename == expected_filename+ assert out_mimetype == expected_mimetype+++def test_helper_does_not_match_txt_substring_in_middle_of_name():+ """`.txt` must only match as a SUFFIX, otherwise filenames like+ `notes.txt.bak` or `mytxt.pdf` would be mangled."""+ # A real edge case: file genuinely named `something.txt.bak` is NOT a .txt.+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype(+ "notes.txt.bak", "application/octet-stream"+ )+ assert out_filename == "notes.txt.bak"+ assert out_mimetype == "application/octet-stream"++ # Different real case: `mytxt.pdf` — txt is just part of the stem.+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype("mytxt.pdf", "application/pdf")+ assert out_filename == "mytxt.pdf"+ assert out_mimetype == "application/pdf"