Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix: handle long text URLs in webscraper (#40757)
林玮 (Jade Lin)4e44ade07f1f
diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.pyindex 7eed135bbf0..0a008b5f31e 100644--- a/api/core/rag/extractor/extract_processor.py+++ b/api/core/rag/extractor/extract_processor.py@@ -132,7 +132,6 @@ class ExtractProcessor:storage.download(upload_file.key, file_path)input_file = Path(file_path)file_extension = input_file.suffix.lower()- assert upload_file is not None, "upload_file is required"etl_type = dify_config.ETL_TYPEextractor: BaseExtractor | None = Noneif etl_type == "Unstructured":@@ -140,6 +139,7 @@ class ExtractProcessor:unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or ""if file_extension in {".xlsx", ".xls"}:+ assert upload_file is not None, "upload_file is required"extractor = ExcelExtractor(file_path,upload_file.tenant_id,@@ -187,6 +187,7 @@ class ExtractProcessor:extractor = TextExtractor(file_path, autodetect_encoding=True)else:if file_extension in {".xlsx", ".xls"}:+ assert upload_file is not None, "upload_file is required"extractor = ExcelExtractor(file_path,upload_file.tenant_id,diff --git a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.pyindex 19d16e1d552..0b73d488fa0 100644--- a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py@@ -118,6 +118,16 @@ class TestExtractProcessorLoaders:assert len(docs) == 2assert text == "u1\nu2"+ def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch):+ content = "a" * 100_000+ response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode())+ monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)+ monkeypatch.setattr(processor_module.dify_config, "ETL_TYPE", "SelfHosted")++ text = ExtractProcessor.load_from_url("https://example.com/response.txt", return_text=True)++ assert text == content+class TestExtractProcessorFileRouting:@pytest.fixture(autouse=True)