Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): derive url file suffix from content-type media type in load_from_url (#42364)

coderdailyonefea6b6e99939
diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py
index 1d2f98075ef..dd73b436089 100644
--- a/api/core/rag/extractor/extract_processor.py
+++ b/api/core/rag/extractor/extract_processor.py
@@ -83,10 +83,12 @@ class ExtractProcessor:
suffix = Path(url).suffix
if not suffix and suffix != ".":
# get content-type
- if response.headers.get("Content-Type"):
- suffix = "." + response.headers.get("Content-Type").split("/")[-1]
+ content_type = response.headers.get("Content-Type")
+ if content_type:
+ # strip parameters such as "; charset=utf-8" before using the subtype as suffix
+ suffix = "." + content_type.split(";")[0].strip().split("/")[-1]
else:
- content_disposition = response.headers.get("Content-Disposition")
+ content_disposition = response.headers.get("Content-Disposition", "")
filename_match = re.search(r'filename="([^"]+)"', content_disposition)
if filename_match:
filename = unquote(filename_match.group(1))
diff --git a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
index 6b81f24fcb8..2f00fda06db 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
@@ -105,6 +105,16 @@ class TestExtractProcessorLoaders:
[
("https://example.com/file.txt", {"Content-Type": "text/plain"}, ".txt"),
("https://example.com/no_suffix", {"Content-Type": "application/pdf"}, ".pdf"),
+ (
+ "https://example.com/no_suffix",
+ {"Content-Type": "application/pdf; charset=binary"},
+ ".pdf",
+ ),
+ (
+ "https://example.com/no_suffix",
+ {"Content-Type": "text/html; charset=utf-8"},
+ ".html",
+ ),
(
"https://example.com/no_suffix",
{"Content-Disposition": 'attachment; filename="report.md"'},
@@ -115,6 +125,7 @@ class TestExtractProcessorLoaders:
{"Content-Disposition": 'attachment; filename="report"'},
"",
),
+ ("https://example.com/no_suffix", {}, ""),
],
)
def test_load_from_url_builds_temp_file_with_correct_suffix(
@@ -167,6 +178,21 @@ class TestExtractProcessorLoaders:
assert args[1] is None
assert args[2] is None
+ def test_load_from_url_routes_parameterized_content_type(self, monkeypatch: pytest.MonkeyPatch):
+ # Servers commonly append parameters such as "; charset=binary" to
+ # Content-Type; the media type must be parsed before deriving the suffix.
+ response = SimpleNamespace(
+ headers={"Content-Type": "application/pdf; charset=binary"}, content=b"%PDF-1.1 body"
+ )
+ monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)
+ factory = _patch_all_extractors(monkeypatch)
+ apply_config_overrides(monkeypatch, ETL_TYPE="dify")
+
+ text = ExtractProcessor.load_from_url("https://example.com/report", return_text=True)
+
+ assert text == "extracted-by-PdfExtractor"
+ assert factory.calls[-1][0] == "PdfExtractor"
+
class TestExtractProcessorFileRouting:
@pytest.fixture(autouse=True)