Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use SQLite sessions in datasource files (#39048)
Asuka Minato78129cf6c3bc
diff --git a/api/core/datasource/datasource_file_manager.py b/api/core/datasource/datasource_file_manager.pyindex 521791206db..317f362d662 100644--- a/api/core/datasource/datasource_file_manager.py+++ b/api/core/datasource/datasource_file_manager.py@@ -12,8 +12,8 @@ from uuid import uuid4import httpxfrom configs import dify_config+from core.db.session_factory import session_factoryfrom core.file import remote_fetcher-from extensions.ext_database import dbfrom extensions.ext_storage import storagefrom extensions.storage.storage_type import StorageTypefrom models.enums import CreatorUserRole@@ -54,6 +54,7 @@ class DatasourceFileManager:mimetype: str,filename: str | None = None,) -> UploadFile:+ """Persist an uploaded datasource file and its storage payload."""extension = guess_extension(mimetype) or ".bin"unique_name = uuid4().hexunique_filename = f"{unique_name}{extension}"@@ -82,9 +83,10 @@ class DatasourceFileManager:created_at=datetime.now(),)- db.session.add(upload_file)- db.session.commit()- db.session.refresh(upload_file)+ with session_factory.create_session() as session:+ session.add(upload_file)+ session.commit()+ session.refresh(upload_file)return upload_file@@ -95,6 +97,7 @@ class DatasourceFileManager:file_url: str,conversation_id: str | None = None,) -> ToolFile:+ """Download a remote file and persist its tool-file metadata."""# try to download imagetry:response = remote_fetcher.make_request("GET", file_url)@@ -125,8 +128,9 @@ class DatasourceFileManager:size=len(blob),)- db.session.add(tool_file)- db.session.commit()+ with session_factory.create_session() as session:+ session.add(tool_file)+ session.commit()return tool_file@@ -139,7 +143,8 @@ class DatasourceFileManager::return: the binary of the file, mime type"""- upload_file: UploadFile | None = db.session.get(UploadFile, id)+ with session_factory.create_session() as session:+ upload_file: UploadFile | None = session.get(UploadFile, id)if not upload_file:return None@@ -157,21 +162,24 @@ class DatasourceFileManager::return: the binary of the file, mime type"""- message_file: MessageFile | None = db.session.get(MessageFile, id)-- # Check if message_file is not None- if message_file is not None:- # get tool file id- if message_file.url is not None:- tool_file_id = message_file.url.split("/")[-1]- # trim extension- tool_file_id = tool_file_id.split(".")[0]+ with session_factory.create_session() as session:+ message_file: MessageFile | None = session.get(MessageFile, id)++ # Check if message_file is not None+ if message_file is not None:+ # get tool file id+ if message_file.url is not None:+ tool_file_id = message_file.url.split("/")[-1]+ # trim extension+ tool_file_id = tool_file_id.split(".")[0]+ else:+ tool_file_id = Noneelse:tool_file_id = None- else:- tool_file_id = None- tool_file: ToolFile | None = db.session.get(ToolFile, tool_file_id)+ if not tool_file_id:+ return None+ tool_file: ToolFile | None = session.get(ToolFile, tool_file_id)if not tool_file:return None@@ -185,11 +193,12 @@ class DatasourceFileManager:"""get file binary- :param tool_file_id: the id of the tool file+ :param upload_file_id: the id of the upload file:return: the binary of the file, mime type"""- upload_file: UploadFile | None = db.session.get(UploadFile, upload_file_id)+ with session_factory.create_session() as session:+ upload_file: UploadFile | None = session.get(UploadFile, upload_file_id)if not upload_file:return None, Nonediff --git a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.pyindex 827950fae2f..3bbc8db1b66 100644--- a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py+++ b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py@@ -1,14 +1,62 @@-from datetime import datetime+from datetime import UTC, datetimefrom unittest.mock import MagicMock, patchimport httpximport pytest+from sqlalchemy.orm import Sessionfrom core.datasource.datasource_file_manager import DatasourceFileManager+from extensions.storage.storage_type import StorageType+from models.enums import CreatorUserRolefrom models.model import MessageFile, UploadFilefrom models.tools import ToolFile+def _upload_file(id: str, *, key: str, mime_type: str) -> UploadFile:+ upload_file = UploadFile(+ tenant_id="tenant-1",+ storage_type=StorageType.LOCAL,+ key=key,+ name="file.png",+ size=4,+ extension=".png",+ mime_type=mime_type,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-1",+ created_at=datetime.now(UTC).replace(tzinfo=None),+ used=False,+ )+ upload_file.id = id+ return upload_file+++def _tool_file(id: str, *, key: str = "tool_key", mimetype: str = "image/png") -> ToolFile:+ tool_file = ToolFile(+ tenant_id="tenant-1",+ user_id="user-1",+ conversation_id=None,+ file_key=key,+ mimetype=mimetype,+ name="tool.png",+ size=4,+ )+ tool_file.id = id+ return tool_file+++def _message_file(id: str, *, url: str | None) -> MessageFile:+ message_file = MessageFile(+ message_id="message-1",+ type="image",+ transfer_method="remote_url",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-1",+ url=url,+ )+ message_file.id = id+ return message_file++class TestDatasourceFileManager:@patch("core.datasource.datasource_file_manager.time.time")@patch("core.datasource.datasource_file_manager.os.urandom")@@ -32,11 +80,10 @@ class TestDatasourceFileManager:assert f"nonce={mock_urandom.return_value.hex()}" in signed_urlassert "sign=" in signed_url- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")@patch("core.datasource.datasource_file_manager.dify_config")- def test_create_file_by_raw(self, mock_config, mock_uuid, mock_storage, mock_db):+ def test_create_file_by_raw(self, mock_config, mock_uuid, mock_storage, sqlite_session: Session):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_config.STORAGE_TYPE = "local"@@ -64,14 +111,14 @@ class TestDatasourceFileManager:assert upload_file.key == f"datasources/{tenant_id}/unique_hex.png"mock_storage.save.assert_called_once_with(upload_file.key, file_binary)- mock_db.session.add.assert_called_once()- mock_db.session.commit.assert_called_once()+ persisted_file = sqlite_session.get(UploadFile, upload_file.id)+ assert persisted_file is not None+ assert persisted_file.key == upload_file.key- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")@patch("core.datasource.datasource_file_manager.dify_config")- def test_create_file_by_raw_filename_no_extension(self, mock_config, mock_uuid, mock_storage, mock_db):+ def test_create_file_by_raw_filename_no_extension(self, mock_config, mock_uuid, mock_storage):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_config.STORAGE_TYPE = "local"@@ -94,12 +141,11 @@ class TestDatasourceFileManager:# Verifyassert upload_file.name == "test.png" # Should append extension- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")@patch("core.datasource.datasource_file_manager.dify_config")@patch("core.datasource.datasource_file_manager.guess_extension")- def test_create_file_by_raw_unknown_extension(self, mock_guess_ext, mock_config, mock_uuid, mock_storage, mock_db):+ def test_create_file_by_raw_unknown_extension(self, mock_guess_ext, mock_config, mock_uuid, mock_storage):# Setupmock_guess_ext.return_value = None # Cannot guessmock_uuid.return_value = MagicMock(hex="unique_hex")@@ -118,11 +164,10 @@ class TestDatasourceFileManager:assert upload_file.extension == ".bin"assert upload_file.name == "unique_hex.bin"- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")@patch("core.datasource.datasource_file_manager.dify_config")- def test_create_file_by_raw_no_filename(self, mock_config, mock_uuid, mock_storage, mock_db):+ def test_create_file_by_raw_no_filename(self, mock_config, mock_uuid, mock_storage):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_config.STORAGE_TYPE = "local"@@ -141,10 +186,9 @@ class TestDatasourceFileManager:assert upload_file.extension == ".pdf"@patch("core.datasource.datasource_file_manager.remote_fetcher")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")- def test_create_file_by_url_mimetype_from_guess(self, mock_uuid, mock_storage, mock_db, mock_ssrf):+ def test_create_file_by_url_mimetype_from_guess(self, mock_uuid, mock_storage, mock_ssrf):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_response = MagicMock()@@ -154,17 +198,18 @@ class TestDatasourceFileManager:# Executetool_file = DatasourceFileManager.create_file_by_url(- user_id="user_123", tenant_id="tenant_456", file_url="https://example.com/photo.png"+ user_id="user_123",+ tenant_id="tenant_456",+ file_url="https://example.com/photo.png",)# Verifyassert tool_file.mimetype == "image/png" # Guessed from .png in URL@patch("core.datasource.datasource_file_manager.remote_fetcher")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")- def test_create_file_by_url_mimetype_default(self, mock_uuid, mock_storage, mock_db, mock_ssrf):+ def test_create_file_by_url_mimetype_default(self, mock_uuid, mock_storage, mock_ssrf):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_response = MagicMock()@@ -183,10 +228,9 @@ class TestDatasourceFileManager:assert tool_file.mimetype == "application/octet-stream"@patch("core.datasource.datasource_file_manager.remote_fetcher")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")@patch("core.datasource.datasource_file_manager.uuid4")- def test_create_file_by_url_success(self, mock_uuid, mock_storage, mock_db, mock_ssrf):+ def test_create_file_by_url_success(self, mock_uuid, mock_storage, mock_ssrf):# Setupmock_uuid.return_value = MagicMock(hex="unique_hex")mock_response = MagicMock()@@ -196,7 +240,9 @@ class TestDatasourceFileManager:# Executetool_file = DatasourceFileManager.create_file_by_url(- user_id="user_123", tenant_id="tenant_456", file_url="https://example.com/photo.jpg"+ user_id="user_123",+ tenant_id="tenant_456",+ file_url="https://example.com/photo.jpg",)# Verify@@ -216,152 +262,59 @@ class TestDatasourceFileManager:user_id="user_123", tenant_id="tenant_456", file_url="https://example.com/large.file")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")- def test_get_file_binary(self, mock_storage, mock_db):- # Setup- mock_upload_file = UploadFile(- tenant_id="tenant-id",- storage_type="opendal",- key="some_key",- name="test.txt",- size=0,- extension="txt",- mime_type="image/png",- created_by_role="account",- created_by="account-id",- created_at=datetime.now(),- used=False,- )-- mock_db.session.get.return_value = mock_upload_file+ def test_get_file_binary(self, mock_storage, sqlite_session: Session):+ sqlite_session.add(_upload_file("file_id", key="some_key", mime_type="image/png"))+ sqlite_session.commit()mock_storage.load_once.return_value = b"file content"- # Executeresult = DatasourceFileManager.get_file_binary("file_id")# Verifyassert result == (b"file content", "image/png")- # Case: Not found- mock_db.session.get.return_value = Noneassert DatasourceFileManager.get_file_binary("unknown") is None- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")- def test_get_file_binary_by_message_file_id(self, mock_storage, mock_db):- # Setup- mock_message_file = MessageFile(- message_id="message-id",- type="image",- transfer_method="local_file",- created_by_role="user",- created_by="user-id",- url="http://localhost/files/tools/tool_id.png",- )-- mock_tool_file = ToolFile(- user_id="user-id",- tenant_id="tenant-id",- conversation_id="conversation-id",- file_key="tool_key",- mimetype="image/png",+ def test_get_file_binary_by_message_file_id(self, mock_storage, sqlite_session: Session):+ sqlite_session.add_all(+ [+ _message_file("msg_file_id", url="http://localhost/files/tools/tool_id.png"),+ _tool_file("tool_id"),+ ])-- def mock_get(model, id):- if model == MessageFile:- return mock_message_file- elif model == ToolFile:- return mock_tool_file- return None-- mock_db.session.get.side_effect = mock_get+ sqlite_session.commit()mock_storage.load_once.return_value = b"tool content"- # Executeresult = DatasourceFileManager.get_file_binary_by_message_file_id("msg_file_id")# Verifyassert result == (b"tool content", "image/png")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")- def test_get_file_binary_by_message_file_id_with_extension(self, mock_storage, mock_db):- # Test that it correctly parses tool_id even with extension in URL- mock_message_file = MessageFile(- message_id="message-id",- type="image",- transfer_method="local_file",- created_by_role="user",- created_by="user-id",- url="http://localhost/files/tools/abcdef.png",+ def test_get_file_binary_by_message_file_id_with_extension(self, mock_storage, sqlite_session: Session):+ sqlite_session.add_all(+ [_message_file("m", url="http://localhost/files/tools/abcdef.png"), _tool_file("abcdef", key="tk")])-- mock_tool_file = ToolFile(- user_id="user-id",- tenant_id="tenant-id",- conversation_id="conversation-id",- file_key="tk",- mimetype="image/png",- )- mock_tool_file.id = "abcdef"-- def mock_get(model, id):- if model == MessageFile:- return mock_message_file- return mock_tool_file-- mock_db.session.get.side_effect = mock_get+ sqlite_session.commit()mock_storage.load_once.return_value = b"bits"result = DatasourceFileManager.get_file_binary_by_message_file_id("m")assert result == (b"bits", "image/png")- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")- def test_get_file_binary_by_message_file_id_failures(self, mock_storage, mock_db):- # Case 1: Message file not found- mock_db.session.get.return_value = None+ def test_get_file_binary_by_message_file_id_failures(self, mock_storage, sqlite_session: Session):assert DatasourceFileManager.get_file_binary_by_message_file_id("none") is None- # Case 2: Message file found but tool file not found- mock_message_file = MessageFile(- message_id="message-id",- type="image",- transfer_method="local_file",- created_by_role="user",- created_by="user-id",- url=None,- )-- def mock_get_v2(model, id):- if model == MessageFile:- return mock_message_file- return None-- mock_db.session.get.side_effect = mock_get_v2+ sqlite_session.add(_message_file("msg_id", url=None))+ sqlite_session.commit()assert DatasourceFileManager.get_file_binary_by_message_file_id("msg_id") is None- @patch("core.datasource.datasource_file_manager.db")@patch("core.datasource.datasource_file_manager.storage")- def test_get_file_generator_by_upload_file_id(self, mock_storage, mock_db):- # Setup- mock_upload_file = UploadFile(- tenant_id="tenant-id",- storage_type="opendal",- key="upload_key",- name="test.txt",- size=0,- extension="txt",- mime_type="text/plain",- created_by_role="account",- created_by="account-id",- created_at=datetime.now(),- used=False,- )-- mock_db.session.get.return_value = mock_upload_file+ def test_get_file_generator_by_upload_file_id(self, mock_storage, sqlite_session: Session):+ sqlite_session.add(_upload_file("upload_id", key="upload_key", mime_type="text/plain"))+ sqlite_session.commit()mock_storage.load_stream.return_value = iter([b"chunk1", b"chunk2"])@@ -372,8 +325,6 @@ class TestDatasourceFileManager:assert mimetype == "text/plain"assert list(stream) == [b"chunk1", b"chunk2"]- # Case: Not found- mock_db.session.get.return_value = Nonestream, mimetype = DatasourceFileManager.get_file_generator_by_upload_file_id("none")assert stream is Noneassert mimetype is None