Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: handle long text URLs in webscraper (#40757)

林玮 (Jade Lin)4e44ade07f1f
diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py
index 7eed135bbf0..0a008b5f31e 100644
--- a/api/core/rag/extractor/extract_processor.py
+++ b/api/core/rag/extractor/extract_processor.py
@@ -132,7 +132,6 @@ class ExtractProcessor:
storage.download(upload_file.key, file_path)
input_file = Path(file_path)
file_extension = input_file.suffix.lower()
- assert upload_file is not None, "upload_file is required"
etl_type = dify_config.ETL_TYPE
extractor: BaseExtractor | None = None
if etl_type == "Unstructured":
@@ -140,6 +139,7 @@ class ExtractProcessor:
unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or ""
if file_extension in {".xlsx", ".xls"}:
+ assert upload_file is not None, "upload_file is required"
extractor = ExcelExtractor(
file_path,
upload_file.tenant_id,
@@ -187,6 +187,7 @@ class ExtractProcessor:
extractor = TextExtractor(file_path, autodetect_encoding=True)
else:
if file_extension in {".xlsx", ".xls"}:
+ assert upload_file is not None, "upload_file is required"
extractor = ExcelExtractor(
file_path,
upload_file.tenant_id,
diff --git a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
index 19d16e1d552..0b73d488fa0 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
@@ -118,6 +118,16 @@ class TestExtractProcessorLoaders:
assert len(docs) == 2
assert text == "u1\nu2"
+ def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch):
+ content = "a" * 100_000
+ response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode())
+ monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)
+ monkeypatch.setattr(processor_module.dify_config, "ETL_TYPE", "SelfHosted")
+
+ text = ExtractProcessor.load_from_url("https://example.com/response.txt", return_text=True)
+
+ assert text == content
+
class TestExtractProcessorFileRouting:
@pytest.fixture(autouse=True)