Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Centralize .txt->.md workaround for Langflow (#1651)

Edwin Jose7e33b4ece7e8
diff --git a/src/api/langflow_files.py b/src/api/langflow_files.py
index b3b80c4bb..dc848067e 100644
--- a/src/api/langflow_files.py
+++ b/src/api/langflow_files.py
@@ -1,43 +1,43 @@
import json
import os
import tempfile
-from typing import List, Optional
from fastapi import Depends, File, Form, UploadFile
-from pydantic import BaseModel
from fastapi.responses import JSONResponse
+from pydantic import BaseModel
from dependencies import (
+ get_current_user,
get_langflow_file_service,
+ get_optional_user,
get_session_manager,
get_task_service,
- get_current_user,
- get_optional_user,
)
from session_manager import User
+from utils.file_utils import langflow_safe_filename_and_mimetype
from utils.logging_config import get_logger
logger = get_logger(__name__)
class RunIngestionBody(BaseModel):
- file_paths: List[str] = []
- file_ids: Optional[List[str]] = None
- file_metadata: Optional[List[dict]] = None
- session_id: Optional[str] = None
- tweaks: Optional[dict] = None
- settings: Optional[dict] = None
+ file_paths: list[str] = []
+ file_ids: list[str] | None = None
+ file_metadata: list[dict] | None = None
+ session_id: str | None = None
+ tweaks: dict | None = None
+ settings: dict | None = None
class DeleteFilesBody(BaseModel):
- file_ids: List[str]
+ file_ids: list[str]
async def upload_user_file(
file: UploadFile = File(...),
langflow_file_service=Depends(get_langflow_file_service),
session_manager=Depends(get_session_manager),
- user: Optional[User] = Depends(get_optional_user),
+ user: User | None = Depends(get_optional_user),
):
"""Upload a file to Langflow's Files API"""
try:
@@ -45,11 +45,11 @@ async def upload_user_file(
logger.debug("Processing file", filename=file.filename, size=file.size)
content = await file.read()
- file_tuple = (
- file.filename,
- content,
- file.content_type or "application/octet-stream",
+ # Langflow's docling chokes on text/plain — rename .txt -> .md.
+ upload_filename, upload_mimetype = langflow_safe_filename_and_mimetype(
+ file.filename, file.content_type
)
+ file_tuple = (upload_filename, content, upload_mimetype)
jwt_token = user.jwt_token if user else session_manager.get_effective_jwt_token(None, None)
@@ -60,6 +60,7 @@ async def upload_user_file(
except Exception as e:
logger.error("upload_user_file endpoint failed", error_type=type(e).__name__, error=str(e))
import traceback
+
logger.error("Full traceback", traceback=traceback.format_exc())
return JSONResponse({"error": str(e)}, status_code=500)
@@ -111,6 +112,7 @@ async def run_ingestion(
if jwt_token:
from auth_context import set_auth_context
+
set_auth_context(user.user_id, jwt_token)
try:
@@ -132,9 +134,9 @@ async def run_ingestion(
async def upload_and_ingest_user_file(
file: UploadFile = File(...),
- session_id: Optional[str] = Form(None),
- settings_json: Optional[str] = Form(None, alias="settings"),
- tweaks_json: Optional[str] = Form(None, alias="tweaks"),
+ session_id: str | None = Form(None),
+ settings_json: str | None = Form(None, alias="settings"),
+ tweaks_json: str | None = Form(None, alias="tweaks"),
langflow_file_service=Depends(get_langflow_file_service),
session_manager=Depends(get_session_manager),
task_service=Depends(get_task_service),
@@ -191,6 +193,7 @@ async def upload_and_ingest_user_file(
)
except Exception:
from utils.file_utils import safe_unlink
+
safe_unlink(temp_path)
raise
@@ -219,4 +222,4 @@ async def delete_user_files(
"errors": errors,
},
status_code=status,
- )
\ No newline at end of file
+ )
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index e7ee81cd2..8214f2a63 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -3,7 +3,11 @@ from typing import Any
# Create custom processor for connector files using Langflow
from models.processors import LangflowConnectorFileProcessor
from services.langflow_file_service import LangflowFileService
-from utils.file_utils import clean_connector_filename, get_file_extension
+from utils.file_utils import (
+ clean_connector_filename,
+ get_file_extension,
+ langflow_safe_filename_and_mimetype,
+)
from utils.logging_config import get_logger
from .base import BaseConnector, ConnectorDocument
@@ -76,11 +80,15 @@ class LangflowConnectorService:
# Clean filename and ensure we don't add a double extension
processed_filename = clean_connector_filename(document.filename, document.mimetype)
+ # Langflow's docling chokes on text/plain — rename .txt -> .md.
+ processed_filename, processed_mimetype = langflow_safe_filename_and_mimetype(
+ processed_filename, document.mimetype
+ )
file_tuple = (
processed_filename,
content,
- document.mimetype or "application/octet-stream",
+ processed_mimetype,
)
# Step 0: Delete existing chunks for this file before re-ingesting.
diff --git a/src/models/processors.py b/src/models/processors.py
index ba7ff205b..3c84ffe7e 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -17,6 +17,7 @@ from utils.file_utils import (
clean_connector_filename,
get_file_extension,
get_filename_aliases,
+ langflow_safe_filename_and_mimetype,
)
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
@@ -899,14 +900,10 @@ class LangflowFileProcessor(TaskProcessor):
if not content_type:
content_type = "application/octet-stream"
- # Rename .txt to .md for Langflow compatibility
- # Langflow has issues processing text/plain files
- langflow_filename = original_filename
- if original_filename.lower().endswith(".txt"):
- langflow_filename = original_filename[:-4] + ".md"
- content_type = "text/markdown"
- logger.debug(f"Renamed {original_filename} to {langflow_filename} for Langflow")
-
+ # Langflow's docling chokes on text/plain — rename .txt -> .md.
+ langflow_filename, content_type = langflow_safe_filename_and_mimetype(
+ original_filename, content_type
+ )
file_tuple = (langflow_filename, content, content_type)
# Get JWT token using same logic as DocumentFileProcessor
diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py
index f2a50dfe2..845da8647 100644
--- a/src/utils/file_utils.py
+++ b/src/utils/file_utils.py
@@ -102,6 +102,25 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:
return filename
+def langflow_safe_filename_and_mimetype(filename: str, mimetype: str | None) -> tuple[str, str]:
+ """Apply the .txt -> .md workaround for Langflow ingestion.
+
+ Langflow's docling component fails on text/plain (see commit f6b9fe0).
+ The workaround is to rename .txt files to .md with text/markdown before
+ handing the file to Langflow. Centralised here so every Langflow-upload
+ site applies it identically — local user uploads, connector-driven
+ ingestion, and the direct upload route.
+
+ Returns the (possibly rewritten) (filename, mimetype) pair. Pass-through
+ for all other extensions; falls back to application/octet-stream when
+ mimetype is missing.
+ """
+ safe_mimetype = mimetype or "application/octet-stream"
+ if filename and filename.lower().endswith(".txt"):
+ return filename[:-4] + ".md", "text/markdown"
+ return filename, safe_mimetype
+
+
def get_filename_aliases(filename: str) -> list[str]:
"""Return equivalent filename variants used by ingestion/indexing.
diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.py
new file mode 100644
index 000000000..cb267aff5
--- /dev/null
+++ b/tests/unit/connectors/test_langflow_connector_txt_to_md.py
@@ -0,0 +1,131 @@
+"""Regression guard for SharePoint .txt ingestion via the Langflow connector
+path.
+
+Background: Langflow's docling component fails on text/plain. The
+user-upload path already applies a .txt -> .md rename (commit f6b9fe0).
+This test pins that the connector path
+(`LangflowConnectorService.process_connector_document`) applies the same
+rule, so a SharePoint .txt reaches Langflow as .md / text/markdown — not
+the unhandled text/plain shape that previously broke ingestion.
+
+This test drives the production code path; if the rename is ever removed
+or shorted around, the assertions will catch it.
+"""
+
+import sys
+from datetime import datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_service():
+ """Build a LangflowConnectorService with the surface
+ process_connector_document touches stubbed:
+ - session_manager (only needed if pre-delete fires; we let it run as
+ a no-op against a mocked OpenSearch client).
+ - langflow_service.upload_user_file / run_ingestion_flow / delete_user_file.
+ """
+ from connectors.langflow_connector_service import LangflowConnectorService
+
+ service = LangflowConnectorService.__new__(LangflowConnectorService)
+
+ # OpenSearch mock for the pre-delete step (collect+delete). Empty result
+ # so the pre-delete is a no-op; the test isn't about that path.
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ service.session_manager = session_manager
+ service.docling_service = MagicMock()
+ service.connection_manager = MagicMock()
+
+ # Mock Langflow service — the surface we're asserting against is its
+ # upload_user_file (the file_tuple it receives).
+ langflow_service = MagicMock()
+ langflow_service.upload_user_file = AsyncMock(
+ return_value={"id": "lf-file-id", "path": "/lf/path/file"}
+ )
+ langflow_service.run_ingestion_flow = AsyncMock(return_value={"status": "ok"})
+ langflow_service.delete_user_file = AsyncMock()
+ service.langflow_service = langflow_service
+
+ return service, langflow_service
+
+
+def _make_document(filename: str, mimetype: str, content: bytes = b"hello world"):
+ from connectors.base import ConnectorDocument, DocumentACL
+
+ return ConnectorDocument(
+ id="graph-item-id-stable",
+ filename=filename,
+ mimetype=mimetype,
+ content=content,
+ source_url="https://contoso.sharepoint.com/.../notes.txt",
+ acl=DocumentACL(owner="alice"),
+ modified_time=datetime(2026, 5, 21),
+ created_time=datetime(2026, 5, 1),
+ metadata={"site": "marketing"},
+ )
+
+
+async def test_sharepoint_txt_is_uploaded_to_langflow_as_md():
+ """A connector document with mimetype text/plain must reach Langflow's
+ upload_user_file with a .md filename and text/markdown mimetype."""
+ service, langflow_service = _make_service()
+ document = _make_document(filename="notes.txt", mimetype="text/plain")
+
+ # auto_cleanup_tempfile is a context manager that yields a real path;
+ # let it run normally — it just creates a temp file we won't use.
+ await service.process_connector_document(
+ document=document,
+ owner_user_id="alice",
+ connector_type="sharepoint",
+ jwt_token="jwt",
+ )
+
+ langflow_service.upload_user_file.assert_awaited_once()
+ (file_tuple, _jwt) = langflow_service.upload_user_file.await_args.args
+ filename, content, mimetype = file_tuple
+
+ assert filename.endswith(".md"), (
+ f"SharePoint .txt must reach Langflow as .md; got {filename!r}. "
+ "Langflow's docling component fails on text/plain — see "
+ "langflow_safe_filename_and_mimetype."
+ )
+ assert mimetype == "text/markdown", (
+ f"SharePoint .txt must reach Langflow as text/markdown; got {mimetype!r}."
+ )
+ # Content bytes are unchanged by the rename.
+ assert content == document.content
+
+
+async def test_sharepoint_pdf_passes_through_untouched():
+ """The rename rule must NOT touch non-.txt files."""
+ service, langflow_service = _make_service()
+ document = _make_document(
+ filename="report.pdf", mimetype="application/pdf", content=b"%PDF-1.4..."
+ )
+
+ await service.process_connector_document(
+ document=document,
+ owner_user_id="alice",
+ connector_type="sharepoint",
+ jwt_token="jwt",
+ )
+
+ (file_tuple, _jwt) = langflow_service.upload_user_file.await_args.args
+ filename, _content, mimetype = file_tuple
+ assert filename == "report.pdf"
+ assert mimetype == "application/pdf"
diff --git a/tests/unit/test_langflow_safe_filename.py b/tests/unit/test_langflow_safe_filename.py
new file mode 100644
index 000000000..cdf164157
--- /dev/null
+++ b/tests/unit/test_langflow_safe_filename.py
@@ -0,0 +1,66 @@
+"""Pin the .txt -> .md workaround applied at every Langflow upload site.
+
+Langflow's docling component fails on text/plain (commit f6b9fe0). The
+helper at `src/utils/file_utils.py::langflow_safe_filename_and_mimetype` is
+the single source of truth for that rename — every place that builds a
+file_tuple for `LangflowFileService.upload_user_file` must call it.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+from utils.file_utils import langflow_safe_filename_and_mimetype # noqa: E402
+
+
+ "filename,mimetype,expected_filename,expected_mimetype",
+ [
+ # Core rule: .txt becomes .md, mimetype switches to text/markdown.
+ ("foo.txt", "text/plain", "foo.md", "text/markdown"),
+ # Case-insensitive: extension match ignores case but preserves the
+ # original stem case; only the extension is rewritten to ".md".
+ ("FOO.TXT", "text/plain", "FOO.md", "text/markdown"),
+ ("Report.Txt", "text/plain", "Report.md", "text/markdown"),
+ # Pass-through: .md is already fine.
+ ("notes.md", "text/markdown", "notes.md", "text/markdown"),
+ # Pass-through: unrelated extensions.
+ ("doc.pdf", "application/pdf", "doc.pdf", "application/pdf"),
+ ("sheet.csv", "text/csv", "sheet.csv", "text/csv"),
+ # Missing mimetype: still rewrite .txt to .md (the bug we're fixing is
+ # about Langflow tripping on text/plain — once we rewrite, we own the
+ # mimetype). Non-.txt without mimetype gets the octet-stream default.
+ ("foo.txt", None, "foo.md", "text/markdown"),
+ ("foo.pdf", None, "foo.pdf", "application/octet-stream"),
+ # Defensive: empty filename returns the octet-stream default, never crashes.
+ ("", "text/plain", "", "text/plain"),
+ ("", None, "", "application/octet-stream"),
+ ],
+)
+def test_helper_applies_txt_to_md_rule(filename, mimetype, expected_filename, expected_mimetype):
+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype(filename, mimetype)
+ assert out_filename == expected_filename
+ assert out_mimetype == expected_mimetype
+
+
+def test_helper_does_not_match_txt_substring_in_middle_of_name():
+ """`.txt` must only match as a SUFFIX, otherwise filenames like
+ `notes.txt.bak` or `mytxt.pdf` would be mangled."""
+ # A real edge case: file genuinely named `something.txt.bak` is NOT a .txt.
+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype(
+ "notes.txt.bak", "application/octet-stream"
+ )
+ assert out_filename == "notes.txt.bak"
+ assert out_mimetype == "application/octet-stream"
+
+ # Different real case: `mytxt.pdf` — txt is just part of the stem.
+ out_filename, out_mimetype = langflow_safe_filename_and_mimetype("mytxt.pdf", "application/pdf")
+ assert out_filename == "mytxt.pdf"
+ assert out_mimetype == "application/pdf"