Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: exercise tenant indexing queue serialization (#43188)
Asuka Minatoe23c02340a87
diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.pyindex 4312e24fe9a..48d5e0ea671 100644--- a/api/tests/unit_tests/conftest.py+++ b/api/tests/unit_tests/conftest.py@@ -80,6 +80,18 @@ def redis_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[ext_redisyield wrapper, commands+def tenant_queue_commands(+ redis_transport: tuple[ext_redis.RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch+) -> MagicMock:+ """Run tenant queue serialization and command building without Redis I/O."""+ from core.rag.pipeline import queue++ redis, commands = redis_transport+ monkeypatch.setattr(queue, "redis_client", redis)+ return commands++@pytest.fixturedef app() -> Flask:return CACHED_APPdiff --git a/api/tests/unit_tests/services/test_document_indexing_task_proxy.py b/api/tests/unit_tests/services/test_document_indexing_task_proxy.pyindex ace9abcb49e..f92d59181b1 100644--- a/api/tests/unit_tests/services/test_document_indexing_task_proxy.py+++ b/api/tests/unit_tests/services/test_document_indexing_task_proxy.py@@ -1,4 +1,5 @@-from unittest.mock import Mock, patch+import json+from unittest.mock import MagicMock, Mock, call, patchfrom core.entities.document_task import DocumentTaskfrom core.rag.pipeline.queue import TenantIsolatedTaskQueue@@ -19,15 +20,6 @@ class DocumentIndexingTaskProxyTestDataFactory:features.billing.subscription.plan = planreturn features- @staticmethod- def create_mock_tenant_queue(has_task_key: bool = False) -> Mock:- """Create mock TenantIsolatedTaskQueue."""- queue = Mock(spec=TenantIsolatedTaskQueue)- queue.get_task_key.return_value = "task_key" if has_task_key else None- queue.push_tasks = Mock()- queue.set_task_waiting_time = Mock()- return queue-@staticmethoddef create_document_task_proxy(tenant_id: str = "tenant-123", dataset_id: str = "dataset-456", document_ids: list[str] | None = None@@ -93,21 +85,21 @@ class TestDocumentIndexingTaskProxy:)@patch("services.document_indexing_proxy.document_indexing_task_proxy.normal_document_indexing_task")- def test_send_to_tenant_queue_with_existing_task_key(self, mock_task):+ def test_send_to_tenant_queue_with_existing_task_key(self, mock_task, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when task key exists."""# Arrangeproxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()- proxy._tenant_isolated_task_queue = DocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=True- )+ tenant_queue_commands.return_value = "task_key"mock_task.delay = Mock()# Actproxy._send_to_tenant_queue(mock_task)# Assert- proxy._tenant_isolated_task_queue.push_tasks.assert_called_once()- pushed_tasks = proxy._tenant_isolated_task_queue.push_tasks.call_args[0][0]+ assert [entry.args[0] for entry in tenant_queue_commands.call_args_list] == ["GET", "LPUSH"]+ push = tenant_queue_commands.call_args_list[1]+ assert push.args[1] == "tenant_self_document_indexing_task_queue:tenant-123"+ pushed_tasks = [json.loads(push.args[2])["data"]]assert len(pushed_tasks) == 1assert isinstance(DocumentTask(**pushed_tasks[0]), DocumentTask)assert pushed_tasks[0]["tenant_id"] == "tenant-123"@@ -116,24 +108,29 @@ class TestDocumentIndexingTaskProxy:mock_task.delay.assert_not_called()@patch("services.document_indexing_proxy.document_indexing_task_proxy.normal_document_indexing_task")- def test_send_to_tenant_queue_without_task_key(self, mock_task):+ def test_send_to_tenant_queue_without_task_key(self, mock_task, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when no task key exists."""# Arrangeproxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()- proxy._tenant_isolated_task_queue = DocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=False- )+ tenant_queue_commands.return_value = Nonemock_task.delay = Mock()# Actproxy._send_to_tenant_queue(mock_task)# Assert- proxy._tenant_isolated_task_queue.set_task_waiting_time.assert_called_once()+ tenant_queue_commands.assert_has_calls(+ [+ call(+ "GET", "tenant_document_indexing_task:tenant-123", keys=["tenant_document_indexing_task:tenant-123"]+ ),+ call("SETEX", "tenant_document_indexing_task:tenant-123", 3600, 1),+ ]+ )mock_task.delay.assert_called_once_with(tenant_id="tenant-123", dataset_id="dataset-456", document_ids=["doc-1", "doc-2", "doc-3"])- proxy._tenant_isolated_task_queue.push_tasks.assert_not_called()+ assert all(entry.args[0] != "LPUSH" for entry in tenant_queue_commands.call_args_list)def test_send_to_default_tenant_queue(self):"""Test _send_to_default_tenant_queue method."""diff --git a/api/tests/unit_tests/services/test_duplicate_document_indexing_task_proxy.py b/api/tests/unit_tests/services/test_duplicate_document_indexing_task_proxy.pyindex 7a5039e94cf..64077459cf6 100644--- a/api/tests/unit_tests/services/test_duplicate_document_indexing_task_proxy.py+++ b/api/tests/unit_tests/services/test_duplicate_document_indexing_task_proxy.py@@ -1,4 +1,5 @@-from unittest.mock import Mock, patch+import json+from unittest.mock import MagicMock, Mock, call, patchfrom core.entities.document_task import DocumentTaskfrom core.rag.pipeline.queue import TenantIsolatedTaskQueue@@ -21,15 +22,6 @@ class DuplicateDocumentIndexingTaskProxyTestDataFactory:features.billing.subscription.plan = planreturn features- @staticmethod- def create_mock_tenant_queue(has_task_key: bool = False) -> Mock:- """Create mock TenantIsolatedTaskQueue."""- queue = Mock(spec=TenantIsolatedTaskQueue)- queue.get_task_key.return_value = "task_key" if has_task_key else None- queue.push_tasks = Mock()- queue.set_task_waiting_time = Mock()- return queue-@staticmethoddef create_duplicate_document_task_proxy(tenant_id: str = "tenant-123", dataset_id: str = "dataset-456", document_ids: list[str] | None = None@@ -116,21 +108,21 @@ class TestDuplicateDocumentIndexingTaskProxy:@patch("services.document_indexing_proxy.duplicate_document_indexing_task_proxy.normal_duplicate_document_indexing_task")- def test_send_to_tenant_queue_with_existing_task_key(self, mock_task):+ def test_send_to_tenant_queue_with_existing_task_key(self, mock_task, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when task key exists."""# Arrangeproxy = DuplicateDocumentIndexingTaskProxyTestDataFactory.create_duplicate_document_task_proxy()- proxy._tenant_isolated_task_queue = DuplicateDocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=True- )+ tenant_queue_commands.return_value = "task_key"mock_task.delay = Mock()# Actproxy._send_to_tenant_queue(mock_task)# Assert- proxy._tenant_isolated_task_queue.push_tasks.assert_called_once()- pushed_tasks = proxy._tenant_isolated_task_queue.push_tasks.call_args[0][0]+ assert [entry.args[0] for entry in tenant_queue_commands.call_args_list] == ["GET", "LPUSH"]+ push = tenant_queue_commands.call_args_list[1]+ assert push.args[1] == "tenant_self_duplicate_document_indexing_task_queue:tenant-123"+ pushed_tasks = [json.loads(push.args[2])["data"]]assert len(pushed_tasks) == 1assert isinstance(DocumentTask(**pushed_tasks[0]), DocumentTask)assert pushed_tasks[0]["tenant_id"] == "tenant-123"@@ -141,24 +133,31 @@ class TestDuplicateDocumentIndexingTaskProxy:@patch("services.document_indexing_proxy.duplicate_document_indexing_task_proxy.normal_duplicate_document_indexing_task")- def test_send_to_tenant_queue_without_task_key(self, mock_task):+ def test_send_to_tenant_queue_without_task_key(self, mock_task, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when no task key exists."""# Arrangeproxy = DuplicateDocumentIndexingTaskProxyTestDataFactory.create_duplicate_document_task_proxy()- proxy._tenant_isolated_task_queue = DuplicateDocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=False- )+ tenant_queue_commands.return_value = Nonemock_task.delay = Mock()# Actproxy._send_to_tenant_queue(mock_task)# Assert- proxy._tenant_isolated_task_queue.set_task_waiting_time.assert_called_once()+ tenant_queue_commands.assert_has_calls(+ [+ call(+ "GET",+ "tenant_duplicate_document_indexing_task:tenant-123",+ keys=["tenant_duplicate_document_indexing_task:tenant-123"],+ ),+ call("SETEX", "tenant_duplicate_document_indexing_task:tenant-123", 3600, 1),+ ]+ )mock_task.delay.assert_called_once_with(tenant_id="tenant-123", dataset_id="dataset-456", document_ids=["doc-1", "doc-2", "doc-3"])- proxy._tenant_isolated_task_queue.push_tasks.assert_not_called()+ assert all(entry.args[0] != "LPUSH" for entry in tenant_queue_commands.call_args_list)def test_send_to_default_tenant_queue(self):"""Test _send_to_default_tenant_queue method."""diff --git a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.pyindex 0905c78deba..eff006c8b09 100644--- a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py+++ b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py@@ -1,7 +1,7 @@import jsonimport loggingfrom datetime import datetime-from unittest.mock import MagicMock, Mock, patch+from unittest.mock import MagicMock, Mock, call, patchimport pytest@@ -27,15 +27,6 @@ class RagPipelineTaskProxyTestDataFactory:features.billing.subscription.plan = planreturn features- @staticmethod- def create_mock_tenant_queue(has_task_key: bool = False) -> Mock:- """Create mock TenantIsolatedTaskQueue."""- queue = Mock(spec=TenantIsolatedTaskQueue)- queue.get_task_key.return_value = "task_key" if has_task_key else None- queue.push_tasks = Mock()- queue.set_task_waiting_time = Mock()- return queue-@staticmethoddef create_rag_pipeline_invoke_entity(pipeline_id: str = "pipeline-123",@@ -226,11 +217,10 @@ class TestRagPipelineTaskProxy:assert parsed_json[0]["pipeline_id"] == "pipeline-1"assert parsed_json[1]["pipeline_id"] == "pipeline-2"- def test_send_to_direct_queue(self):+ def test_send_to_direct_queue(self, tenant_queue_commands: MagicMock):"""Test _send_to_direct_queue method."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()- proxy._tenant_isolated_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue()upload_file_id = "file-123"mock_task = Mock()@@ -238,20 +228,18 @@ class TestRagPipelineTaskProxy:proxy._send_to_direct_queue(upload_file_id, mock_task)# If sent to direct queue, tenant_isolated_task_queue should not be called- proxy._tenant_isolated_task_queue.push_tasks.assert_not_called()+ assert all(entry.args[0] != "LPUSH" for entry in tenant_queue_commands.call_args_list)# Celery should be called directlymock_task.delay.assert_called_once_with(rag_pipeline_invoke_entities_file_id=upload_file_id, tenant_id="tenant-123")- def test_send_to_tenant_queue_with_existing_task_key(self):+ def test_send_to_tenant_queue_with_existing_task_key(self, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when task key exists."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()- proxy._tenant_isolated_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=True- )+ tenant_queue_commands.return_value = "task_key"upload_file_id = "file-123"mock_task = Mock()@@ -259,17 +247,15 @@ class TestRagPipelineTaskProxy:proxy._send_to_tenant_queue(upload_file_id, mock_task)# If task key exists, should push tasks to the queue- proxy._tenant_isolated_task_queue.push_tasks.assert_called_once_with([upload_file_id])+ tenant_queue_commands.assert_any_call("LPUSH", "tenant_self_pipeline_task_queue:tenant-123", upload_file_id)# Celery should not be called directlymock_task.delay.assert_not_called()- def test_send_to_tenant_queue_without_task_key(self):+ def test_send_to_tenant_queue_without_task_key(self, tenant_queue_commands: MagicMock):"""Test _send_to_tenant_queue when no task key exists."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()- proxy._tenant_isolated_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue(- has_task_key=False- )+ tenant_queue_commands.return_value = Noneupload_file_id = "file-123"mock_task = Mock()@@ -277,13 +263,18 @@ class TestRagPipelineTaskProxy:proxy._send_to_tenant_queue(upload_file_id, mock_task)# If no task key, should set task waiting time key first- proxy._tenant_isolated_task_queue.set_task_waiting_time.assert_called_once()+ tenant_queue_commands.assert_has_calls(+ [+ call("GET", "tenant_pipeline_task:tenant-123", keys=["tenant_pipeline_task:tenant-123"]),+ call("SETEX", "tenant_pipeline_task:tenant-123", 3600, 1),+ ]+ )mock_task.delay.assert_called_once_with(rag_pipeline_invoke_entities_file_id=upload_file_id, tenant_id="tenant-123")# The first task should be sent to celery directly, so push tasks should not be called- proxy._tenant_isolated_task_queue.push_tasks.assert_not_called()+ assert all(entry.args[0] != "LPUSH" for entry in tenant_queue_commands.call_args_list)def test_send_to_default_tenant_queue(self):"""Test _send_to_default_tenant_queue method."""diff --git a/api/tests/unit_tests/tasks/test_duplicate_document_indexing_task.py b/api/tests/unit_tests/tasks/test_duplicate_document_indexing_task.pyindex f6dbc4275b4..c4f80731bca 100644--- a/api/tests/unit_tests/tasks/test_duplicate_document_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_duplicate_document_indexing_task.py@@ -1,11 +1,11 @@"""Unit tests for queue/wrapper behaviors in duplicate document indexing tasks (non-database logic)."""+import jsonimport uuidfrom unittest.mock import Mock, patchimport pytest-from core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom tasks.duplicate_document_indexing_task import (_duplicate_document_indexing_task_with_tenant_queue,duplicate_document_indexing_task,@@ -29,17 +29,6 @@ def document_ids():return [str(uuid.uuid4()) for _ in range(3)]-def mock_tenant_isolated_queue():- with patch("tasks.duplicate_document_indexing_task.TenantIsolatedTaskQueue", autospec=True) as mock_queue_class:- mock_queue = Mock(spec=TenantIsolatedTaskQueue)- mock_queue.pull_tasks.return_value = []- mock_queue.delete_task_key = Mock()- mock_queue.set_task_waiting_time = Mock()- mock_queue_class.return_value = mock_queue- yield mock_queue--class TestDuplicateDocumentIndexingTask:"""Tests for the deprecated duplicate_document_indexing_task function."""@@ -72,7 +61,7 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:def test_tenant_queue_wrapper_calls_core_function(self,mock_core_func,- mock_tenant_isolated_queue,+ tenant_queue_commands,tenant_id,dataset_id,document_ids,@@ -91,7 +80,7 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:def test_tenant_queue_wrapper_deletes_key_when_no_tasks(self,mock_core_func,- mock_tenant_isolated_queue,+ tenant_queue_commands,tenant_id,dataset_id,document_ids,@@ -99,19 +88,19 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:"""Test that tenant queue wrapper deletes task key when no more tasks."""# Arrangemock_task_func = Mock()- mock_tenant_isolated_queue.pull_tasks.return_value = []+ tenant_queue_commands.return_value = None# Act_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)# Assert- mock_tenant_isolated_queue.delete_task_key.assert_called_once()+ tenant_queue_commands.assert_any_call("DEL", f"tenant_duplicate_document_indexing_task:{tenant_id}")@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)def test_tenant_queue_wrapper_processes_next_tasks(self,mock_core_func,- mock_tenant_isolated_queue,+ tenant_queue_commands,tenant_id,dataset_id,document_ids,@@ -124,13 +113,13 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:"dataset_id": dataset_id,"document_ids": document_ids,}- mock_tenant_isolated_queue.pull_tasks.return_value = [next_task]+ tenant_queue_commands.side_effect = [json.dumps({"data": next_task}), None, None]# Act_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)# Assert- mock_tenant_isolated_queue.set_task_waiting_time.assert_called_once()+ tenant_queue_commands.assert_any_call("SETEX", f"tenant_duplicate_document_indexing_task:{tenant_id}", 3600, 1)mock_task_func.delay.assert_called_once_with(tenant_id=tenant_id,dataset_id=dataset_id,@@ -141,7 +130,7 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:def test_tenant_queue_wrapper_handles_core_function_error(self,mock_core_func,- mock_tenant_isolated_queue,+ tenant_queue_commands,tenant_id,dataset_id,document_ids,@@ -156,7 +145,7 @@ class TestDuplicateDocumentIndexingTaskWithTenantQueue:# Assert# Should still check for next tasks even after error- mock_tenant_isolated_queue.pull_tasks.assert_called_once()+ tenant_queue_commands.assert_any_call("RPOP", f"tenant_self_duplicate_document_indexing_task_queue:{tenant_id}")class TestNormalDuplicateDocumentIndexingTask: