Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: migrate dataset indexing task sessions to SQLite (#40091)
Asuka Minato3f62f3cd05a0
diff --git a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py b/api/tests/unit_tests/tasks/test_dataset_indexing_task.pyindex bc692dc0023..3466543762b 100644--- a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_dataset_indexing_task.py@@ -1,29 +1,24 @@-"""-Unit tests for dataset indexing tasks.--This module tests the document indexing task functionality including:-- Task enqueuing to different queues (normal, priority, tenant-isolated)-- Batch processing of multiple documents-- Progress tracking through task lifecycle-- Error handling and retry mechanisms-- Task cancellation and cleanup+"""SQLite-backed tests for document indexing tasks.++The indexing task deliberately uses separate transactions for validation,+status persistence, indexing, and summary dispatch. These tests persist real+ORM rows so each phase observes only committed database state."""-import loggingimport uuidfrom contextlib import nullcontextfrom types import SimpleNamespacefrom unittest.mock import MagicMock, Mock, patchimport pytest+from sqlalchemy.orm import Sessionfrom core.indexing_runner import DocumentIsPausedErrorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom enums.cloud_plan import CloudPlanfrom extensions.ext_redis import redis_clientfrom models.dataset import Dataset, Document-from models.enums import IndexingStatus+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatusfrom services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxyfrom tasks.document_indexing_task import (_document_indexing,@@ -33,40 +28,26 @@ from tasks.document_indexing_task import (priority_document_indexing_task,)-# ============================================================================-# Fixtures-# ============================================================================-@pytest.fixture-def tenant_id():- """Generate a unique tenant ID for testing."""+def tenant_id() -> str:return str(uuid.uuid4())@pytest.fixture-def dataset_id():- """Generate a unique dataset ID for testing."""+def dataset_id() -> str:return str(uuid.uuid4())@pytest.fixture-def document_ids():- """Generate a list of document IDs for testing."""+def document_ids() -> list[str]:return [str(uuid.uuid4()) for _ in range(3)]@pytest.fixture-def mock_redis():- """Mock Redis client operations."""- # Redis is already mocked globally in conftest.py- # Reset it for each test+def mock_redis() -> MagicMock:+ """Reset the external Redis boundary used by tenant-isolated queues."""redis_client.reset_mock()- redis_client.get.reset_mock()- redis_client.setex.reset_mock()- redis_client.delete.reset_mock()- redis_client.lpush.reset_mock()- redis_client.rpop.reset_mock()redis_client.get.return_value = Noneredis_client.setex.return_value = Trueredis_client.delete.return_value = True@@ -75,1923 +56,495 @@ def mock_redis():return redis_client-# Additional fixtures required by tests in this module--@pytest.fixture-def mock_db_session():- """Mock session_factory.create_session() to return a session whose queries use shared test data.-- Tests set session._shared_data = {"dataset": <Dataset>, "documents": [<Document>, ...]}- This fixture makes session.scalar(select(Dataset)...) return the shared dataset,- and session.scalars(select(Document)...).all() return the shared documents.- """- with patch("tasks.document_indexing_task.session_factory") as mock_sf:- session = MagicMock()- session._shared_data = {"dataset": None, "documents": []}-- def _get_entity(stmt) -> type | None:- """Extract the mapped entity class from a SQLAlchemy select statement."""- try:- descs = stmt.column_descriptions- if descs:- return descs[0].get("entity")- except (AttributeError, TypeError):- pass- return None-- def _extract_id_from_where(stmt) -> str | None:- """Return the value bound to the 'id' column in the WHERE clause, if present."""- try:- where = stmt.whereclause- if where is None:- clauses = []- else:- try:- clauses = list(where.clauses)- except AttributeError:- clauses = [where]- except Exception:- return None-- for clause in clauses:- try:- left = clause.left- right = clause.right- except AttributeError:- continue- try:- key = left.key- except AttributeError:- continue- if key == "id":- try:- return right.value- except AttributeError:- return None- return None-- def _scalar_side_effect(stmt):- entity = _get_entity(stmt)- if entity is not None:- if entity.__name__ == "Dataset":- return session._shared_data.get("dataset")- elif entity.__name__ == "Document":- docs = session._shared_data.get("documents", [])- if not docs:- return None- queried_id = _extract_id_from_where(stmt)- if queried_id:- doc_map = {d.id: d for d in docs}- return doc_map.get(queried_id, docs[0])- return docs[0]- return None-- def _scalars_side_effect(stmt):- entity = _get_entity(stmt)- result = MagicMock()- if entity is not None:- if entity.__name__ == "Document":- result.all.return_value = list(session._shared_data.get("documents", []))- elif entity.__name__ == "Dataset":- ds = session._shared_data.get("dataset")- result.all.return_value = [ds] if ds else []- else:- result.all.return_value = []- else:- result.all.return_value = []- return result-- session.scalar.side_effect = _scalar_side_effect- session.scalars.side_effect = _scalars_side_effect-- # Implement session.begin() context manager that commits on exit- session.commit = MagicMock()- bm = MagicMock()- bm.__enter__.return_value = session-- def _bm_exit_side_effect(*args, **kwargs):- session.commit()-- bm.__exit__.side_effect = _bm_exit_side_effect- session.begin.return_value = bm-- # Context manager behavior for create_session(): ensure close() is called on exit- session.close = MagicMock()- cm = MagicMock()- cm.__enter__.return_value = session-- def _exit_side_effect(*args, **kwargs):- session.close()-- cm.__exit__.side_effect = _exit_side_effect- mock_sf.create_session.return_value = cm-- yield session+def indexing_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:+ runner = MagicMock()+ runner_class = MagicMock(return_value=runner)+ monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", runner_class)+ runner._constructor_mock = runner_class+ return runner+++def _features(+ *,+ billing_enabled: bool = False,+ plan: CloudPlan = CloudPlan.PROFESSIONAL,+ vector_limit: int = 1000,+ vector_size: int = 0,+) -> SimpleNamespace:+ return SimpleNamespace(+ billing=SimpleNamespace(enabled=billing_enabled, subscription=SimpleNamespace(plan=plan)),+ vector_space=SimpleNamespace(limit=vector_limit, size=vector_size),+ )-def mock_dataset(dataset_id, tenant_id):- """Create a Dataset model object."""+def _patch_features(monkeypatch: pytest.MonkeyPatch, features: SimpleNamespace) -> MagicMock:+ get_features = MagicMock(return_value=features)+ monkeypatch.setattr("tasks.document_indexing_task.FeatureService.get_features", get_features)+ return get_features+++def _persist_indexing_rows(+ session: Session,+ *,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ indexing_technique: IndexTechniqueType = IndexTechniqueType.HIGH_QUALITY,+ summary_index_setting: dict[str, bool] | None = None,+ document_forms: list[IndexStructureType] | None = None,+ need_summary: list[bool] | None = None,+) -> tuple[Dataset, list[Document]]:+ """Persist one tenant-owned dataset and the requested document rows."""+ created_by = str(uuid.uuid4())dataset = Dataset(id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ name="Indexing dataset",+ data_source_type=DataSourceType.UPLOAD_FILE,+ indexing_technique=indexing_technique,embedding_model_provider="openai",- embedding_model="text-embedding-ada-002",+ embedding_model="text-embedding-3-small",+ summary_index_setting=summary_index_setting,+ created_by=created_by,)- return dataset---def mock_documents(document_ids, dataset_id):- """Create Document model objects."""- documents = []- for doc_id in document_ids:- documents.append(- Document(- id=doc_id,- dataset_id=dataset_id,- indexing_status="waiting",- error=None,- stopped_at=None,- processing_started_at=None,- doc_form=IndexStructureType.PARAGRAPH_INDEX,- )- )- return documents---def mock_indexing_runner():- """Mock IndexingRunner for document_indexing_task module."""- with patch("tasks.document_indexing_task.IndexingRunner") as mock_runner_class:- mock_runner = MagicMock()- mock_runner_class.return_value = mock_runner- mock_runner._constructor_mock = mock_runner_class- yield mock_runner---def mock_feature_service():- """Mock FeatureService for document_indexing_task module."""- with patch("tasks.document_indexing_task.FeatureService") as mock_service:- mock_features = Mock()- mock_features.billing = Mock()- mock_features.billing.enabled = False- mock_features.vector_space = Mock()- mock_features.vector_space.size = 0- mock_features.vector_space.limit = 1000- mock_service.get_features.return_value = mock_features- yield mock_service---# ============================================================================-# Test Task Enqueuing-# ============================================================================+ documents = [+ Document(+ id=document_id,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=position,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name=f"document-{position}.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by=created_by,+ indexing_status=IndexingStatus.WAITING,+ doc_form=(document_forms or [IndexStructureType.PARAGRAPH_INDEX] * len(document_ids))[position - 1],+ need_summary=(need_summary or [False] * len(document_ids))[position - 1],+ )+ for position, document_id in enumerate(document_ids, start=1)+ ]+ session.add_all([dataset, *documents])+ session.commit()+ return dataset, documents+++def _persisted_documents(session: Session, document_ids: list[str]) -> list[Document]:+ session.expire_all()+ return [document for document_id in document_ids if (document := session.get(Document, document_id)) is not None]class TestTaskEnqueuing:- """Test cases for task enqueuing to different queues."""-- def test_enqueue_to_priority_direct_queue_for_self_hosted(self, tenant_id, dataset_id, document_ids, mock_redis):- """- Test enqueuing to priority direct queue for self-hosted deployments.-- When billing is disabled (self-hosted), tasks should go directly to- the priority queue without tenant isolation.- """- # Arrange- with patch.object(DocumentIndexingTaskProxy, "features") as mock_features:- mock_features.billing.enabled = False-- # Mock the class variable directly- mock_task = Mock()- with patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", mock_task):- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)-- # Act- proxy.delay()-- # Assert- mock_task.delay.assert_called_once_with(- tenant_id=tenant_id, dataset_id=dataset_id, document_ids=document_ids- )-- def test_enqueue_to_normal_tenant_queue_for_sandbox_plan(self, tenant_id, dataset_id, document_ids, mock_redis):- """- Test enqueuing to normal tenant queue for sandbox plan.-- Sandbox plan users should have their tasks queued with tenant isolation- in the normal priority queue.- """- # Arrange- mock_redis.get.return_value = None # No existing task-- with patch.object(DocumentIndexingTaskProxy, "features") as mock_features:- mock_features.billing.enabled = True- mock_features.billing.subscription.plan = CloudPlan.SANDBOX-- # Mock the class variable directly- mock_task = Mock()- with patch.object(DocumentIndexingTaskProxy, "NORMAL_TASK_FUNC", mock_task):- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)-- # Act- proxy.delay()-- # Assert - Should set task key and call delay- assert mock_redis.setex.called- mock_task.delay.assert_called_once()-- def test_enqueue_to_priority_tenant_queue_for_paid_plan(self, tenant_id, dataset_id, document_ids, mock_redis):- """- Test enqueuing to priority tenant queue for paid plans.-- Paid plan users should have their tasks queued with tenant isolation- in the priority queue.- """- # Arrange- mock_redis.get.return_value = None # No existing task-- with patch.object(DocumentIndexingTaskProxy, "features") as mock_features:- mock_features.billing.enabled = True- mock_features.billing.subscription.plan = CloudPlan.PROFESSIONAL-- # Mock the class variable directly- mock_task = Mock()- with patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", mock_task):- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)-- # Act- proxy.delay()-- # Assert- assert mock_redis.setex.called- mock_task.delay.assert_called_once()-- def test_enqueue_adds_to_waiting_queue_when_task_running(self, tenant_id, dataset_id, document_ids, mock_redis):- """- Test that new tasks are added to waiting queue when a task is already running.-- If a task is already running for the tenant (task key exists),- new tasks should be pushed to the waiting queue.- """- # Arrange- mock_redis.get.return_value = b"1" # Task already running-- with patch.object(DocumentIndexingTaskProxy, "features") as mock_features:- mock_features.billing.enabled = True- mock_features.billing.subscription.plan = CloudPlan.PROFESSIONAL-- # Mock the class variable directly- mock_task = Mock()- with patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", mock_task):- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)-- # Act- proxy.delay()-- # Assert - Should push to queue, not call delay- assert mock_redis.lpush.called- mock_task.delay.assert_not_called()-- def test_legacy_document_indexing_task_still_works(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_documents, mock_indexing_runner- ):- """- Test that the legacy document_indexing_task function still works.-- This ensures backward compatibility for existing code that may still- use the deprecated function.- """- # Arrange- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- document_indexing_task(dataset_id, document_ids)-- # Assert- mock_indexing_runner.run.assert_called_once()---# ============================================================================-# Test Batch Processing-# ============================================================================---class TestBatchProcessing:- """Test cases for batch processing of multiple documents."""-- def test_batch_processing_multiple_documents(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test batch processing of multiple documents.-- All documents in the batch should be processed together and their- status should be updated to 'parsing'.- """- # Arrange - Create actual document objects that can be modified- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - All documents should be set to 'parsing' status- for doc in mock_documents:- assert doc.indexing_status == IndexingStatus.PARSING- assert doc.processing_started_at is not None-- # IndexingRunner should be called with all documents- mock_indexing_runner._constructor_mock.assert_called_once_with(enforce_vector_space_admission=True)- mock_indexing_runner.run.assert_called_once()- call_args = mock_indexing_runner.run.call_args[0][0]- assert len(call_args) == len(document_ids)-- def test_batch_processing_with_limit_check(self, dataset_id, mock_db_session, mock_dataset, mock_feature_service):- """- Test batch processing respects upload limits.-- When the number of documents exceeds the batch upload limit,- an error should be raised and all documents should be marked as error.- """- # Arrange- batch_limit = 10- document_ids = [str(uuid.uuid4()) for _ in range(batch_limit + 1)]-- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- mock_feature_service.get_features.return_value.billing.enabled = True- mock_feature_service.get_features.return_value.billing.subscription.plan = CloudPlan.PROFESSIONAL- mock_feature_service.get_features.return_value.vector_space.limit = 1000- mock_feature_service.get_features.return_value.vector_space.size = 0-- with patch("tasks.document_indexing_task.dify_config.BATCH_UPLOAD_LIMIT", str(batch_limit)):- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - All documents should have error status- for doc in mock_documents:- assert doc.indexing_status == "error"- assert doc.error is not None- assert "batch upload limit" in doc.error-- def test_batch_processing_sandbox_plan_single_document_only(- self, dataset_id, mock_db_session, mock_dataset, mock_feature_service- ):- """- Test that sandbox plan only allows single document upload.-- Sandbox plan should reject batch uploads (more than 1 document).- """- # Arrange- document_ids = [str(uuid.uuid4()) for _ in range(2)]-- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- mock_feature_service.get_features.return_value.billing.enabled = True- mock_feature_service.get_features.return_value.billing.subscription.plan = CloudPlan.SANDBOX- mock_feature_service.get_features.return_value.vector_space.limit = 1000- mock_feature_service.get_features.return_value.vector_space.size = 0-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - All documents should have error status- for doc in mock_documents:- assert doc.indexing_status == "error"- assert "does not support batch upload" in doc.error-- def test_batch_processing_empty_document_list(- self, dataset_id, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test batch processing with empty document list.-- Should handle empty list gracefully without errors.- """- # Arrange- document_ids = []-- # Set shared mock data with empty documents list- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = []-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - IndexingRunner should still be called with empty list- mock_indexing_runner.run.assert_called_once_with([], mock_db_session)---# ============================================================================-# Test Progress Tracking-# ============================================================================---class TestProgressTracking:- """Test cases for progress tracking through task lifecycle."""-- def test_document_status_progression(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test document status progresses correctly through lifecycle.-- Documents should transition from 'waiting' -> 'parsing' -> processed.- """- # Arrange - Create actual document objects- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - Status should be 'parsing'- for doc in mock_documents:- assert doc.indexing_status == IndexingStatus.PARSING- assert doc.processing_started_at is not None-- # Verify commit was called to persist status- assert mock_db_session.commit.called-- def test_processing_started_timestamp_set(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test that processing_started_at timestamp is set correctly.-- When documents start processing, the timestamp should be recorded.- """- # Arrange - Create actual document objects- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert- for doc in mock_documents:- assert doc.processing_started_at is not None-- def test_tenant_queue_processes_next_task_after_completion(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test that tenant queue processes next waiting task after completion.-- After a task completes, the system should check for waiting tasks- and process the next one.- """- # Arrange- next_task_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": ["next_doc_id"]}-- # Simulate next task in queue- from core.rag.pipeline.queue import TaskWrapper+ def test_self_hosted_dispatches_directly_to_priority_task(+ self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock+ ) -> None:+ with (+ patch.object(DocumentIndexingTaskProxy, "features") as features,+ patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,+ ):+ features.billing.enabled = False+ DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()++ task.delay.assert_called_once_with(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=document_ids,+ )++ @pytest.mark.parametrize(+ ("plan", "task_attribute"),+ [+ (CloudPlan.SANDBOX, "NORMAL_TASK_FUNC"),+ (CloudPlan.PROFESSIONAL, "PRIORITY_TASK_FUNC"),+ ],+ )+ def test_cloud_dispatches_first_task_through_tenant_queue(+ self,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ mock_redis: MagicMock,+ plan: CloudPlan,+ task_attribute: str,+ ) -> None:+ with (+ patch.object(DocumentIndexingTaskProxy, "features") as features,+ patch.object(DocumentIndexingTaskProxy, task_attribute, Mock()) as task,+ ):+ features.billing.enabled = True+ features.billing.subscription.plan = plan+ DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()++ mock_redis.setex.assert_called()+ task.delay.assert_called_once()++ def test_running_tenant_task_queues_followup_work(+ self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock+ ) -> None:+ mock_redis.get.return_value = b"1"+ with (+ patch.object(DocumentIndexingTaskProxy, "features") as features,+ patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,+ ):+ features.billing.enabled = True+ features.billing.subscription.plan = CloudPlan.PROFESSIONAL+ DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()- wrapper = TaskWrapper(data=next_task_data)- mock_redis.rpop.return_value = wrapper.serialize()-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert - Next task should be enqueued- mock_task.apply_async.assert_called()- # Task key should be set for next task- assert mock_redis.setex.called-- def test_tenant_queue_clears_flag_when_no_more_tasks(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test that tenant queue clears flag when no more tasks are waiting.-- When there are no more tasks in the queue, the task key should be deleted.- """- # Arrange- mock_redis.rpop.return_value = None # No more tasks-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)+ mock_redis.lpush.assert_called_once()+ task.delay.assert_not_called()- # Assert - Task key should be deleted- assert mock_redis.delete.called+class TestDocumentIndexing:+ def test_legacy_task_persists_parsing_before_running(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=document_ids,+ )+ _patch_features(monkeypatch, _features())-# ============================================================================-# Test Error Handling and Retries-# ============================================================================+ def assert_committed_parsing(documents: list[Document], session: Session) -> None:+ assert all(document.indexing_status == IndexingStatus.PARSING for document in documents)+ assert all(document.processing_started_at is not None for document in documents)+ assert all(session.get(Document, document.id) is document for document in documents)+ indexing_runner.run.side_effect = assert_committed_parsing+ document_indexing_task.run(dataset_id, document_ids)-class TestErrorHandling:- """Test cases for error handling and retry mechanisms."""+ persisted = _persisted_documents(sqlite_session, document_ids)+ assert [document.indexing_status for document in persisted] == [IndexingStatus.PARSING] * 3+ indexing_runner._constructor_mock.assert_called_once_with(enforce_vector_space_admission=True)+ indexing_runner.run.assert_called_once()+ assert isinstance(indexing_runner.run.call_args.args[1], Session)- def test_error_handling_sets_document_error_status(+ def test_only_existing_documents_are_processed(self,- dataset_id,- document_ids,- mock_db_session,- mock_dataset,- mock_feature_service,- ):- """- Test that errors during validation set document error status.-- When validation fails (e.g., limit exceeded), documents should be- marked with error status and error message.- """- # Arrange - Create actual document objects- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Set up to trigger vector space limit error- mock_feature_service.get_features.return_value.billing.enabled = True- mock_feature_service.get_features.return_value.billing.subscription.plan = CloudPlan.PROFESSIONAL- mock_feature_service.get_features.return_value.vector_space.size = 100- mock_feature_service.get_features.return_value.vector_space.limit = 100-- # Act- _document_indexing(dataset_id, document_ids)+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ existing_ids = [document_ids[0], document_ids[2]]+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=existing_ids,+ )+ _patch_features(monkeypatch, _features())- # Assert- for doc in mock_documents:- assert doc.indexing_status == "error"- assert doc.error is not None- assert "over the limit" in doc.error- assert doc.stopped_at is not None-- def test_error_handling_during_indexing_runner(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_documents, mock_indexing_runner- ):- """- Test error handling when IndexingRunner raises an exception.-- Errors during indexing should be caught and logged, but not crash the task.- """- # Arrange- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Make IndexingRunner raise an exception- mock_indexing_runner.run.side_effect = Exception("Indexing failed")-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act - Should not raise exception- _document_indexing(dataset_id, document_ids)-- # Assert - Session should be closed even after error- assert mock_db_session.close.called-- def test_document_paused_error_handling(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_documents, mock_indexing_runner- ):- """- Test handling of DocumentIsPausedError.-- When a document is paused, the error should be caught and logged- but not treated as a failure.- """- # Arrange- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Make IndexingRunner raise DocumentIsPausedError- mock_indexing_runner.run.side_effect = DocumentIsPausedError("Document is paused")-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act - Should not raise exception- _document_indexing(dataset_id, document_ids)-- # Assert - Session should be closed- assert mock_db_session.close.called-- def test_dataset_not_found_error_handling(self, dataset_id, document_ids, mock_db_session):- """- Test handling when dataset is not found.-- If the dataset doesn't exist, the task should exit gracefully.- """- # Arrange - dataset is not in _shared_data (None by default), so scalar() returns None-- # Act_document_indexing(dataset_id, document_ids)- # Assert - Session should be closed- assert mock_db_session.close.called+ processed = indexing_runner.run.call_args.args[0]+ assert {document.id for document in processed} == set(existing_ids)+ assert sqlite_session.get(Document, document_ids[1]) is None- def test_tenant_queue_error_handling_still_processes_next_task(- self,- tenant_id,- dataset_id,- document_ids,- mock_redis,- mock_db_session,- mock_dataset,- mock_indexing_runner,- caplog: pytest.LogCaptureFixture,- ):- """- Test that errors don't prevent processing next task in tenant queue.-- Even if the current task fails, the next task should still be processed.- """- # Arrange- next_task_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": ["next_doc_id"]}-- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=next_task_data)- # Set up rpop to return task once for concurrency check- mock_redis.rpop.side_effect = [wrapper.serialize(), None]-- # Make _document_indexing raise an error- with patch("tasks.document_indexing_task._document_indexing") as mock_indexing:- mock_indexing.side_effect = Exception("Processing failed")-- with caplog.at_level(logging.ERROR, logger="tasks.document_indexing_task"):- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert - Next task should still be enqueued despite error- mock_task.apply_async.assert_called()- assert (- f"Error processing document indexing {dataset_id} for tenant {tenant_id}: {document_ids}"- in caplog.messages- )-- def test_concurrent_task_limit_respected(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset- ):- """- Test that tenant isolated task concurrency limit is respected.-- Should pull only TENANT_ISOLATED_TASK_CONCURRENCY tasks at a time.- """- # Arrange- concurrency_limit = 2-- # Create multiple tasks in queue- tasks = []- for i in range(5):- task_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": [f"doc_{i}"]}- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=task_data)- tasks.append(wrapper.serialize())-- # Mock rpop to return tasks one by one- mock_redis.rpop.side_effect = tasks[:concurrency_limit] + [None]-- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.dify_config.TENANT_ISOLATED_TASK_CONCURRENCY", concurrency_limit):- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert - Should enqueue exactly concurrency_limit tasks- assert mock_task.apply_async.call_count == concurrency_limit---# ============================================================================-# Test Task Cancellation-# ============================================================================---class TestTaskCancellation:- """Test cases for task cancellation and cleanup."""-- def test_task_isolation_between_tenants(self, mock_redis):- """- Test that tasks are properly isolated between different tenants.-- Each tenant should have their own queue and task key.- """- # Arrange- tenant_1 = str(uuid.uuid4())- tenant_2 = str(uuid.uuid4())- dataset_id = str(uuid.uuid4())- document_ids = [str(uuid.uuid4())]-- # Act- queue_1 = TenantIsolatedTaskQueue(tenant_1, "document_indexing")- queue_2 = TenantIsolatedTaskQueue(tenant_2, "document_indexing")-- # Assert - Different tenants should have different queue keys- assert queue_1._queue != queue_2._queue- assert queue_1._task_key != queue_2._task_key- assert tenant_1 in queue_1._queue- assert tenant_2 in queue_2._queue---# ============================================================================-# Integration Tests-# ============================================================================---class TestAdvancedScenarios:- """Advanced test scenarios for edge cases and complex workflows."""-- def test_multiple_documents_with_mixed_success_and_failure(- self, dataset_id, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test handling of mixed success and failure scenarios in batch processing.-- When processing multiple documents, some may succeed while others fail.- This tests that the system handles partial failures gracefully.-- Scenario:- - Process 3 documents in a batch- - First document succeeds- - Second document is not found (skipped)- - Third document succeeds-- Expected behavior:- - Only found documents are processed- - Missing documents are skipped without crashing- - IndexingRunner receives only valid documents- """- # Arrange - Create document IDs with one missing- document_ids = [str(uuid.uuid4()) for _ in range(3)]-- # Create only 2 documents (simulate one missing)- # The new code uses .all() which will only return existing documents- mock_documents = []- for i, doc_id in enumerate([document_ids[0], document_ids[2]]): # Skip middle one- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data - .all() will only return existing documents- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - Only 2 documents should be processed (missing one skipped)- mock_indexing_runner.run.assert_called_once()- call_args = mock_indexing_runner.run.call_args[0][0]- assert len(call_args) == 2 # Only found documents-- def test_tenant_queue_with_multiple_concurrent_tasks(- self, tenant_id, dataset_id, mock_redis, mock_db_session, mock_dataset- ):- """- Test concurrent task processing with tenant isolation.-- This tests the scenario where multiple tasks are queued for the same tenant- and need to be processed respecting the concurrency limit.-- Scenario:- - 5 tasks are waiting in the queue- - Concurrency limit is 2- - After current task completes, pull and enqueue next 2 tasks-- Expected behavior:- - Exactly 2 tasks are pulled from queue (respecting concurrency)- - Each task is enqueued with correct parameters- - Task waiting time is set for each new task- """- # Arrange- concurrency_limit = 2- document_ids = [str(uuid.uuid4())]-- # Create multiple waiting tasks- waiting_tasks = []- for i in range(5):- task_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": [f"doc_{i}"]}- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=task_data)- waiting_tasks.append(wrapper.serialize())-- # Mock rpop to return tasks up to concurrency limit- mock_redis.rpop.side_effect = waiting_tasks[:concurrency_limit] + [None]- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.dify_config.TENANT_ISOLATED_TASK_CONCURRENCY", concurrency_limit):- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert- # Should enqueue exactly concurrency_limit tasks- assert mock_task.apply_async.call_count == concurrency_limit-- # Verify task waiting time was set for each task- assert mock_redis.setex.call_count >= concurrency_limit-- def test_vector_space_limit_edge_case_at_exact_limit(+ def test_empty_batch_still_reaches_runner(self,- dataset_id,- document_ids,- mock_db_session,- mock_dataset,- mock_feature_service,- ):- """- Test vector space limit validation at exact boundary.-- Edge case: When vector space is exactly at the limit (not over),- the upload should still be rejected.-- Scenario:- - Vector space limit: 100- - Current size: 100 (exactly at limit)- - Try to upload 3 documents-- Expected behavior:- - Upload is rejected with appropriate error message- - All documents are marked with error status- """- # Arrange- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Set vector space exactly at limit- mock_feature_service.get_features.return_value.billing.enabled = True- mock_feature_service.get_features.return_value.billing.subscription.plan = CloudPlan.PROFESSIONAL- mock_feature_service.get_features.return_value.vector_space.size = 100- mock_feature_service.get_features.return_value.vector_space.limit = 100-- # Act- _document_indexing(dataset_id, document_ids)-- # Assert - All documents should have error status- for doc in mock_documents:- assert doc.indexing_status == "error"- assert "over the limit" in doc.error-- def test_task_queue_fifo_ordering(self, tenant_id, dataset_id, mock_redis, mock_db_session, mock_dataset):- """- Test that tasks are processed in FIFO (First-In-First-Out) order.-- The tenant isolated queue should maintain task order, ensuring- that tasks are processed in the sequence they were added.-- Scenario:- - Task A added first- - Task B added second- - Task C added third- - When pulling tasks, should get A, then B, then C-- Expected behavior:- - Tasks are retrieved in the order they were added- - FIFO ordering is maintained throughout processing- """- # Arrange- document_ids = [str(uuid.uuid4())]-- # Create tasks with identifiable document IDs to track order- task_order = ["task_A", "task_B", "task_C"]- tasks = []- for task_name in task_order:- task_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": [task_name]}- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=task_data)- tasks.append(wrapper.serialize())-- # Mock rpop to return tasks in FIFO order- mock_redis.rpop.side_effect = tasks + [None]- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.dify_config.TENANT_ISOLATED_TASK_CONCURRENCY", 3):- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert - Verify tasks were enqueued in correct order- assert mock_task.apply_async.call_count == 3-- # Check that document_ids in calls match expected order- for i, call_obj in enumerate(mock_task.apply_async.call_args_list):- called_doc_ids = call_obj[1]["kwargs"]["document_ids"]- assert called_doc_ids == [task_order[i]]-- def test_empty_queue_after_task_completion_cleans_up(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset- ):- """- Test cleanup behavior when queue becomes empty after task completion.-- After processing the last task in the queue, the system should:- 1. Detect that no more tasks are waiting- 2. Delete the task key to indicate tenant is idle- 3. Allow new tasks to start fresh processing-- Scenario:- - Process a task- - Check queue for next tasks- - Queue is empty- - Task key should be deleted-- Expected behavior:- - Task key is deleted when queue is empty- - Tenant is marked as idle (no active tasks)- """- # Arrange- mock_redis.rpop.return_value = None # Empty queue- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert- expected_task_key = f"tenant_document_indexing_task:{tenant_id}"-- # Verify the task key for this tenant was deleted (do not assert call count; fixtures may be shared).- mock_redis.delete.assert_any_call(expected_task_key)-- deleted_keys = [delete_call.args[0] for delete_call in mock_redis.delete.call_args_list if delete_call.args]- assert expected_task_key in deleted_keys-- deleted_task_key = next(key for key in deleted_keys if key == expected_task_key)- assert tenant_id in deleted_task_key- assert "document_indexing" in deleted_task_key-- def test_billing_disabled_skips_limit_checks(- self, dataset_id, document_ids, mock_db_session, mock_dataset, mock_indexing_runner, mock_feature_service- ):- """- Test that billing limit checks are skipped when billing is disabled.-- For self-hosted or enterprise deployments where billing is disabled,- the system should not enforce vector space or batch upload limits.-- Scenario:- - Billing is disabled- - Upload 100 documents (would normally exceed limits)- - No limit checks should be performed-- Expected behavior:- - Documents are processed without limit validation- - No errors related to limits- - All documents proceed to indexing- """- # Arrange - Create many documents- large_batch_ids = [str(uuid.uuid4()) for _ in range(100)]-- mock_documents = []- for doc_id in large_batch_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Billing disabled - limits should not be checked- mock_feature_service.get_features.return_value.billing.enabled = False-- # Act- _document_indexing(dataset_id, large_batch_ids)-- # Assert- # All documents should be set to parsing (no limit errors)- for doc in mock_documents:- assert doc.indexing_status == IndexingStatus.PARSING-- # IndexingRunner should be called with all documents- mock_indexing_runner.run.assert_called_once()- call_args = mock_indexing_runner.run.call_args[0][0]- assert len(call_args) == 100---class TestIntegration:- """Integration tests for complete task workflows."""-- def test_complete_workflow_normal_task(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test complete workflow for normal document indexing task.-- This tests the full flow from task receipt to completion.- """- # Arrange - Create actual document objects- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set up rpop to return None for concurrency check (no more tasks)- mock_redis.rpop.side_effect = [None]- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- normal_document_indexing_task(tenant_id, dataset_id, document_ids)-- # Assert- # Documents should be processed- mock_indexing_runner.run.assert_called_once()- # Session should be closed- assert mock_db_session.close.called- # Task key should be deleted (no more tasks)- assert mock_redis.delete.called-- def test_complete_workflow_priority_task(- self, tenant_id, dataset_id, document_ids, mock_redis, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test complete workflow for priority document indexing task.-- Priority tasks should follow the same flow as normal tasks.- """- # Arrange - Create actual document objects- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set up rpop to return None for concurrency check (no more tasks)- mock_redis.rpop.side_effect = [None]- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- # Act- priority_document_indexing_task(tenant_id, dataset_id, document_ids)-- # Assert- mock_indexing_runner.run.assert_called_once()- assert mock_db_session.close.called- assert mock_redis.delete.called-- def test_queue_chain_processing(- self, tenant_id, dataset_id, mock_redis, mock_db_session, mock_dataset, mock_indexing_runner- ):- """- Test that multiple tasks in queue are processed in sequence.-- When tasks are queued, they should be processed one after another.- """- # Arrange- task_1_docs = [str(uuid.uuid4())]- task_2_docs = [str(uuid.uuid4())]-- task_2_data = {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": task_2_docs}-- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=task_2_data)-- # First call returns task 2, second call returns None- mock_redis.rpop.side_effect = [wrapper.serialize(), None]-- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.FeatureService.get_features") as mock_features:- mock_features.return_value.billing.enabled = False-- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act - Process first task- _document_indexing_with_tenant_queue(tenant_id, dataset_id, task_1_docs, mock_task)-- # Assert - Second task should be enqueued- assert mock_task.apply_async.called- call_args = mock_task.apply_async.call_args- assert call_args[1]["kwargs"]["document_ids"] == task_2_docs---# ============================================================================-# Additional Edge Case Tests-# ============================================================================---class TestEdgeCases:- """Test edge cases and boundary conditions."""-- def test_rapid_successive_task_enqueuing(self, tenant_id, dataset_id, mock_redis):- """- Test rapid successive task enqueuing to the same tenant queue.-- When multiple tasks are enqueued rapidly for the same tenant,- the system should queue them properly without race conditions.-- Scenario:- - First task starts processing (task key exists)- - Multiple tasks enqueued rapidly while first is running- - All should be added to waiting queue-- Expected behavior:- - All tasks are queued (not executed immediately)- - No tasks are lost- - Queue maintains all tasks- """- # Arrange- document_ids_list = [[str(uuid.uuid4())] for _ in range(5)]-- # Simulate task already running- mock_redis.get.return_value = b"1"+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=[],+ )+ _patch_features(monkeypatch, _features())- with patch.object(DocumentIndexingTaskProxy, "features") as mock_features:- mock_features.billing.enabled = True- mock_features.billing.subscription.plan = CloudPlan.PROFESSIONAL+ _document_indexing(dataset_id, [])- # Mock the class variable directly- mock_task = Mock()- with patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", mock_task):- # Act - Enqueue multiple tasks rapidly- for doc_ids in document_ids_list:- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, doc_ids)- proxy.delay()+ assert indexing_runner.run.call_args.args[0] == []+ assert isinstance(indexing_runner.run.call_args.args[1], Session)- # Assert - All tasks should be pushed to queue, none executed- assert mock_redis.lpush.call_count == 5- mock_task.delay.assert_not_called()+ def test_missing_dataset_returns_before_feature_lookup(+ self, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch+ ) -> None:+ get_features = _patch_features(monkeypatch, _features())+ runner_class = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", runner_class)+ _document_indexing(dataset_id, document_ids)-class TestPerformanceScenarios:- """Test performance-related scenarios and optimizations."""+ get_features.assert_not_called()+ runner_class.assert_not_called()- def test_large_document_batch_processing(+ @pytest.mark.parametrize(+ ("features", "batch_limit", "message"),+ [+ (_features(billing_enabled=True), 1, "batch upload limit"),+ (_features(billing_enabled=True, plan=CloudPlan.SANDBOX), 100, "does not support batch upload"),+ (_features(billing_enabled=True, vector_limit=100, vector_size=100), 100, "over the limit"),+ ],+ )+ def test_validation_failure_marks_every_scoped_document_error(self,- dataset_id,- mock_db_session,- mock_dataset,- mock_indexing_runner,- mock_feature_service,- ):- """- Test processing a large batch of documents at batch limit.-- When processing the maximum allowed batch size, the system- should handle it efficiently without errors.-- Scenario:- - Process exactly batch_upload_limit documents (e.g., 50)- - All documents are valid- - Billing is enabled-- Expected behavior:- - All documents are processed successfully- - No timeout or memory issues- - Batch limit is not exceeded- """- # Arrange- batch_limit = 50- document_ids = [str(uuid.uuid4()) for _ in range(batch_limit)]-- mock_documents = []- for doc_id in document_ids:- doc = Document()- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.processing_started_at = None- mock_documents.append(doc)-- # Set shared mock data so all sessions can access it- mock_db_session._shared_data["dataset"] = mock_dataset- mock_db_session._shared_data["documents"] = mock_documents-- # Configure billing with sufficient limits- mock_feature_service.get_features.return_value.billing.enabled = True- mock_feature_service.get_features.return_value.billing.subscription.plan = CloudPlan.PROFESSIONAL- mock_feature_service.get_features.return_value.vector_space.size = 40.75- mock_feature_service.get_features.return_value.vector_space.limit = 10000-- with patch("tasks.document_indexing_task.dify_config.BATCH_UPLOAD_LIMIT", str(batch_limit)):- # Act- _document_indexing(dataset_id, document_ids)-- # Assert- for doc in mock_documents:- assert doc.indexing_status == IndexingStatus.PARSING-- mock_indexing_runner.run.assert_called_once()- call_args = mock_indexing_runner.run.call_args[0][0]- assert len(call_args) == batch_limit- mock_feature_service.get_features.assert_called_once_with(mock_dataset.tenant_id)-- def test_tenant_queue_handles_burst_traffic(self, tenant_id, dataset_id, mock_redis, mock_db_session, mock_dataset):- """- Test tenant queue handling burst traffic scenarios.-- When many tasks arrive in a burst for the same tenant,- the queue should handle them efficiently without dropping tasks.-- Scenario:- - 20 tasks arrive rapidly- - Concurrency limit is 3- - Tasks should be queued and processed in batches-- Expected behavior:- - First 3 tasks are processed immediately- - Remaining tasks wait in queue- - No tasks are lost- """- # Arrange- num_tasks = 20- concurrency_limit = 3- document_ids = [str(uuid.uuid4())]-- # Create waiting tasks- waiting_tasks = []- for i in range(num_tasks):- task_data = {- "tenant_id": tenant_id,- "dataset_id": dataset_id,- "document_ids": [f"doc_{i}"],- }- from core.rag.pipeline.queue import TaskWrapper-- wrapper = TaskWrapper(data=task_data)- waiting_tasks.append(wrapper.serialize())-- # Mock rpop to return tasks up to concurrency limit- mock_redis.rpop.side_effect = waiting_tasks[:concurrency_limit] + [None]- mock_db_session._shared_data["dataset"] = mock_dataset-- with patch("tasks.document_indexing_task.dify_config.TENANT_ISOLATED_TASK_CONCURRENCY", concurrency_limit):- with patch("tasks.document_indexing_task.normal_document_indexing_task") as mock_task:- # Act- _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task)-- # Assert - Should process exactly concurrency_limit tasks- assert mock_task.apply_async.call_count == concurrency_limit-- def test_multiple_tenants_isolated_processing(self, mock_redis):- """- Test that multiple tenants process tasks in isolation.-- When multiple tenants have tasks running simultaneously,- they should not interfere with each other.-- Scenario:- - Tenant A has tasks in queue- - Tenant B has tasks in queue- - Both process independently-- Expected behavior:- - Each tenant has separate queue- - Each tenant has separate task key- - No cross-tenant interference- """- # Arrange- tenant_a = str(uuid.uuid4())- tenant_b = str(uuid.uuid4())- dataset_id = str(uuid.uuid4())- document_ids = [str(uuid.uuid4())]-- # Create queues for both tenants- queue_a = TenantIsolatedTaskQueue(tenant_a, "document_indexing")- queue_b = TenantIsolatedTaskQueue(tenant_b, "document_indexing")-- # Act - Set task keys for both tenants- queue_a.set_task_waiting_time()- queue_b.set_task_waiting_time()-- # Assert - Each tenant has independent queue and key- assert queue_a._queue != queue_b._queue- assert queue_a._task_key != queue_b._task_key- assert tenant_a in queue_a._queue- assert tenant_b in queue_b._queue- assert tenant_a in queue_a._task_key- assert tenant_b in queue_b._task_key---class TestRobustness:- """Test system robustness and resilience."""-- def test_task_proxy_handles_feature_service_failure(self, tenant_id, dataset_id, document_ids, mock_redis):- """- Test that task proxy handles FeatureService failures gracefully.-- If FeatureService fails to retrieve features, the system should- have a fallback or handle the error appropriately.-- Scenario:- - FeatureService.get_features() raises an exception during dispatch- - Task enqueuing should handle the error-- Expected behavior:- - Exception is raised when trying to dispatch- - System doesn't crash unexpectedly- - Error is propagated appropriately- """- # Arrange- with patch("services.document_indexing_proxy.base.FeatureService.get_features") as mock_get_features:- # Simulate FeatureService failure- mock_get_features.side_effect = Exception("Feature service unavailable")-- # Create proxy instance- proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)-- # Act & Assert - Should raise exception when trying to delay (which accesses features)- with pytest.raises(Exception) as exc_info:- proxy.delay()-- # Verify the exception message- assert "Feature service" in str(exc_info.value) or isinstance(exc_info.value, Exception)---class _SessionContext:- def __init__(self, session: MagicMock) -> None:- self._session = session-- def __enter__(self) -> MagicMock:- return self._session-- def __exit__(self, exc_type, exc, tb) -> None: # type: ignore[override]- return None---class TestDocumentIndexingTaskSummaryFlow:- """Additional coverage for summary and tenant queue branches."""-- def test_should_return_when_dataset_missing(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test early return when dataset does not exist."""- # Arrange- session = MagicMock()- session = MagicMock()- session.scalar.return_value = None # dataset not found-- create_session_mock = MagicMock(return_value=_SessionContext(session))- monkeypatch.setattr("tasks.document_indexing_task.session_factory.create_session", create_session_mock)- features_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.FeatureService.get_features", features_mock)-- # Act- _document_indexing("dataset-1", ["doc-1"])-- # Assert- features_mock.assert_not_called()-- def test_should_mark_documents_error_when_batch_upload_limit_exceeded(- self, monkeypatch: pytest.MonkeyPatch+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ monkeypatch: pytest.MonkeyPatch,+ features: SimpleNamespace,+ batch_limit: int,+ message: str,) -> None:- """Test batch upload limit triggers error handling."""- # Arrange- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = SimpleNamespace(id="doc-1", indexing_status=None, error=None, stopped_at=None)-- session = MagicMock()+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=document_ids,+ )+ control_dataset_id = str(uuid.uuid4())+ control_document_id = str(uuid.uuid4())+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=str(uuid.uuid4()),+ dataset_id=control_dataset_id,+ document_ids=[control_document_id],+ )+ _patch_features(monkeypatch, features)+ monkeypatch.setattr("tasks.document_indexing_task.dify_config.BATCH_UPLOAD_LIMIT", str(batch_limit))- def _scalar_se(stmt):- entity = stmt.column_descriptions[0].get("entity")- if entity is Dataset:- return dataset- return document-- session.scalar.side_effect = _scalar_se+ _document_indexing(dataset_id, document_ids)- monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(return_value=_SessionContext(session)),- )+ persisted = _persisted_documents(sqlite_session, document_ids)+ assert all(document.indexing_status == IndexingStatus.ERROR for document in persisted)+ assert all(document.error and message in document.error for document in persisted)+ assert all(document.stopped_at is not None for document in persisted)+ control = sqlite_session.get(Document, control_document_id)+ assert control is not None+ assert control.indexing_status == IndexingStatus.WAITING- features = SimpleNamespace(- billing=SimpleNamespace(- enabled=True,- subscription=SimpleNamespace(plan=CloudPlan.PROFESSIONAL),- ),- vector_space=SimpleNamespace(limit=0, size=0),- )- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)- )- monkeypatch.setattr("tasks.document_indexing_task.dify_config.BATCH_UPLOAD_LIMIT", "1")-- # Act- _document_indexing("dataset-1", ["doc-1", "doc-2"])-- # Assert- assert document.indexing_status == "error"- assert "batch upload limit" in document.error- session.commit.assert_called_once()-- def test_should_queue_summary_generation_for_completed_documents(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test summary generation is queued for eligible documents."""- # Arrange- dataset = SimpleNamespace(- id="dataset-1",- tenant_id="tenant-1",- indexing_technique="high_quality",+ @pytest.mark.parametrize("error", [DocumentIsPausedError("paused"), RuntimeError("boom")])+ def test_runner_failure_stops_before_summary_dispatch(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ error: Exception,+ ) -> None:+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=document_ids,summary_index_setting={"enable": True},+ need_summary=[True] * len(document_ids),)+ _patch_features(monkeypatch, _features())+ indexing_runner.run.side_effect = error+ summary_delay = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)- doc_eligible = SimpleNamespace(- id="doc-1",- indexing_status="completed",- doc_form="text",- need_summary=True,- )- doc_skip_form = SimpleNamespace(- id="doc-2",- indexing_status="completed",- doc_form="qa_model",- need_summary=True,- )- doc_skip_status = SimpleNamespace(- id="doc-3",- indexing_status="processing",- doc_form="text",- need_summary=True,- )-- phase1_docs = [SimpleNamespace(id="doc-1"), SimpleNamespace(id="doc-2"), SimpleNamespace(id="doc-3")]-- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session3 = MagicMock()- session4 = MagicMock()-- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=phase1_docs))- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=phase1_docs))- session4.scalar.return_value = dataset- session4.scalars.return_value = MagicMock(- all=MagicMock(return_value=[doc_eligible, doc_skip_form, doc_skip_status])- )-- create_session_mock = MagicMock(- side_effect=[- _SessionContext(session1),- _SessionContext(session2),- _SessionContext(session3),- _SessionContext(session4),- ]- )- monkeypatch.setattr("tasks.document_indexing_task.session_factory.create_session", create_session_mock)-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),- )- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)- )-- indexing_runner = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=indexing_runner))- delay_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)+ _document_indexing(dataset_id, document_ids)- # Act- _document_indexing("dataset-1", ["doc-1", "doc-2", "doc-3"])+ summary_delay.assert_not_called()+ persisted = _persisted_documents(sqlite_session, document_ids)+ assert all(document.indexing_status == IndexingStatus.PARSING for document in persisted)- # Assert- delay_mock.assert_called_once_with("dataset-1", "doc-1", None)- def test_should_continue_when_summary_queue_fails(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test summary queueing errors are swallowed."""- # Arrange- dataset = SimpleNamespace(- id="dataset-1",- tenant_id="tenant-1",- indexing_technique="high_quality",+class TestSummaryDispatch:+ def test_only_eligible_completed_documents_queue_summaries(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=document_ids,summary_index_setting={"enable": True},+ document_forms=[+ IndexStructureType.PARAGRAPH_INDEX,+ IndexStructureType.QA_INDEX,+ IndexStructureType.PARAGRAPH_INDEX,+ ],+ need_summary=[True, True, True],)+ _patch_features(monkeypatch, _features())- doc_eligible = SimpleNamespace(- id="doc-1",- indexing_status="completed",- doc_form="text",- need_summary=True,- )+ def finish_documents(documents: list[Document], _session: Session) -> None:+ documents[0].indexing_status = IndexingStatus.COMPLETED+ documents[1].indexing_status = IndexingStatus.COMPLETED+ documents[2].indexing_status = IndexingStatus.INDEXING- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session3 = MagicMock()- session4 = MagicMock()-- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session4.scalar.return_value = dataset- session4.scalars.return_value = MagicMock(all=MagicMock(return_value=[doc_eligible]))-- monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(- side_effect=[- _SessionContext(session1),- _SessionContext(session2),- _SessionContext(session3),- _SessionContext(session4),- ]- ),- )-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),- )- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)- )+ indexing_runner.run.side_effect = finish_documents+ summary_delay = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)- indexing_runner = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=indexing_runner))- delay_mock = MagicMock(side_effect=Exception("boom"))- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)-- # Act- _document_indexing("dataset-1", ["doc-1"])-- # Assert- delay_mock.assert_called_once_with("dataset-1", "doc-1", None)-- def test_should_return_when_dataset_missing_after_indexing(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test early return when dataset is missing after indexing."""- # Arrange- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")-- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session3 = MagicMock()- session4 = MagicMock()- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session4.scalar.return_value = None # dataset not found after indexing+ _document_indexing(dataset_id, document_ids)- monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(- side_effect=[- _SessionContext(session1),- _SessionContext(session2),- _SessionContext(session3),- _SessionContext(session4),- ]- ),- )+ summary_delay.assert_called_once_with(dataset_id, document_ids[0], None)- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),+ def test_summary_queue_failure_does_not_fail_indexing(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ document_id = str(uuid.uuid4())+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=[document_id],+ summary_index_setting={"enable": True},+ need_summary=[True],)- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)+ _patch_features(monkeypatch, _features())+ indexing_runner.run.side_effect = lambda documents, _session: setattr(+ documents[0], "indexing_status", IndexingStatus.COMPLETED)- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=MagicMock()))+ summary_delay = MagicMock(side_effect=RuntimeError("queue unavailable"))+ monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)- # Act- _document_indexing("dataset-1", ["doc-1"])+ _document_indexing(dataset_id, [document_id])- # Assert- session4.scalar.assert_called()+ summary_delay.assert_called_once_with(dataset_id, document_id, None)+ persisted = _persisted_documents(sqlite_session, [document_id])[0]+ assert persisted.indexing_status == IndexingStatus.COMPLETED- def test_should_skip_summary_when_not_high_quality(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test summary generation skipped when indexing_technique is not high_quality."""- # Arrange- dataset = SimpleNamespace(- id="dataset-1",- tenant_id="tenant-1",- indexing_technique="economy",+ def test_economy_indexing_skips_summary_generation(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ document_id = str(uuid.uuid4())+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=[document_id],+ indexing_technique=IndexTechniqueType.ECONOMY,summary_index_setting={"enable": True},+ need_summary=[True],)- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session3 = MagicMock()- session4 = MagicMock()-- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session4.scalar.return_value = dataset-- monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(- side_effect=[- _SessionContext(session1),- _SessionContext(session2),- _SessionContext(session3),- _SessionContext(session4),- ]- ),- )-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),+ _patch_features(monkeypatch, _features())+ indexing_runner.run.side_effect = lambda documents, _session: setattr(+ documents[0], "indexing_status", IndexingStatus.COMPLETED)- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)- )- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=MagicMock()))-- delay_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)-- # Act- _document_indexing("dataset-1", ["doc-1"])+ summary_delay = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)- # Assert- delay_mock.assert_not_called()+ _document_indexing(dataset_id, [document_id])- def test_should_skip_summary_generation_when_indexing_paused(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test summary generation is skipped when indexing is paused."""- # Arrange- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ summary_delay.assert_not_called()- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3 = MagicMock()- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))-- create_session_mock = MagicMock(- side_effect=[_SessionContext(session1), _SessionContext(session2), _SessionContext(session3)]- )- monkeypatch.setattr("tasks.document_indexing_task.session_factory.create_session", create_session_mock)-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),- )- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)+ def test_dataset_removed_by_runner_is_absent_from_summary_phase(+ self,+ sqlite_session: Session,+ tenant_id: str,+ dataset_id: str,+ indexing_runner: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ document_id = str(uuid.uuid4())+ _persist_indexing_rows(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_ids=[document_id],+ summary_index_setting={"enable": True},+ need_summary=[True],)+ _patch_features(monkeypatch, _features())- runner = MagicMock()- runner.run.side_effect = DocumentIsPausedError("paused")- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=runner))- delay_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)+ def remove_dataset(_documents: list[Document], session: Session) -> None:+ dataset = session.get(Dataset, dataset_id)+ assert dataset is not None+ session.delete(dataset)- # Act- _document_indexing("dataset-1", ["doc-1"])+ indexing_runner.run.side_effect = remove_dataset+ summary_delay = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)- # Assert- delay_mock.assert_not_called()+ _document_indexing(dataset_id, [document_id])- def test_should_handle_indexing_runner_exception(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test generic indexing runner exception is handled."""- # Arrange- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(Dataset, dataset_id) is None+ summary_delay.assert_not_called()- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3 = MagicMock()- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))+class TestTenantQueue:+ def test_followup_tasks_are_dispatched_with_one_shared_producer(+ self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch+ ) -> None:+ next_documents = [str(uuid.uuid4())]+ queue = MagicMock()+ queue.pull_tasks.return_value = [+ {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents}+ ]+ monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))+ monkeypatch.setattr("tasks.document_indexing_task._document_indexing", MagicMock())+ producer = object()monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(side_effect=[_SessionContext(session1), _SessionContext(session2), _SessionContext(session3)]),- )-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),- )- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)+ "tasks.document_indexing_task.current_app.producer_or_acquire",+ MagicMock(return_value=nullcontext(producer)),)+ task = MagicMock()- runner = MagicMock()- runner.run.side_effect = RuntimeError("boom")- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=runner))-- delay_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)-- # Act- _document_indexing("dataset-1", ["doc-1"])-- # Assert- delay_mock.assert_not_called()-- def test_should_log_missing_document_entry_in_summary_list(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test falsey document entries are handled in summary iteration."""+ _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, task)- # Arrange- class _FalseyDocument:- def __init__(self, doc_id: str) -> None:- self.id = doc_id-- def __bool__(self) -> bool:- return False-- dataset = SimpleNamespace(- id="dataset-1",- tenant_id="tenant-1",- indexing_technique="high_quality",- summary_index_setting={"enable": True},- )- session1 = MagicMock()- session2 = MagicMock()- session2.begin.return_value = nullcontext()- session3 = MagicMock()- session4 = MagicMock()-- session1.scalar.return_value = dataset- session2.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session3.scalar.return_value = dataset- session3.scalars.return_value = MagicMock(all=MagicMock(return_value=[SimpleNamespace(id="doc-1")]))- session4.scalar.return_value = dataset- session4.scalars.return_value = MagicMock(all=MagicMock(return_value=[_FalseyDocument("missing-doc")]))-- monkeypatch.setattr(- "tasks.document_indexing_task.session_factory.create_session",- MagicMock(- side_effect=[- _SessionContext(session1),- _SessionContext(session2),- _SessionContext(session3),- _SessionContext(session4),- ]- ),- )-- features = SimpleNamespace(- billing=SimpleNamespace(enabled=False),- vector_space=SimpleNamespace(limit=0, size=0),+ task.apply_async.assert_called_once_with(+ kwargs={"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents},+ producer=producer,)- monkeypatch.setattr(- "tasks.document_indexing_task.FeatureService.get_features", MagicMock(return_value=features)- )- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", MagicMock(return_value=MagicMock()))-- delay_mock = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", delay_mock)-- # Act- _document_indexing("dataset-1", ["doc-1"])+ queue.set_task_waiting_time.assert_called_once()+ queue.delete_task_key.assert_not_called()- # Assert- delay_mock.assert_not_called()-- def test_normal_document_indexing_task_should_delegate(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test normal indexing task delegates to tenant queue handler."""- # Arrange- handler = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task._document_indexing_with_tenant_queue", handler)+ def test_queue_cleanup_runs_when_indexing_fails(+ self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch+ ) -> None:+ queue = MagicMock()+ queue.pull_tasks.return_value = []+ monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))+ indexing = MagicMock(side_effect=RuntimeError("indexing failed"))+ monkeypatch.setattr("tasks.document_indexing_task._document_indexing", indexing)- # Act- normal_document_indexing_task("tenant-1", "dataset-1", ["doc-1"])+ _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, MagicMock())- # Assert- handler.assert_called_once_with("tenant-1", "dataset-1", ["doc-1"], normal_document_indexing_task)+ queue.delete_task_key.assert_called_once()- def test_priority_document_indexing_task_should_delegate(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Test priority indexing task delegates to tenant queue handler."""- # Arrange- handler = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task._document_indexing_with_tenant_queue", handler)+ @pytest.mark.parametrize("task", [normal_document_indexing_task, priority_document_indexing_task])+ def test_celery_entrypoints_delegate_to_tenant_queue(+ self,+ task: object,+ tenant_id: str,+ dataset_id: str,+ document_ids: list[str],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ delegate = MagicMock()+ monkeypatch.setattr("tasks.document_indexing_task._document_indexing_with_tenant_queue", delegate)- # Act- priority_document_indexing_task("tenant-1", "dataset-1", ["doc-1"])+ task.run(tenant_id, dataset_id, document_ids) # type: ignore[attr-defined]- # Assert- handler.assert_called_once_with("tenant-1", "dataset-1", ["doc-1"], priority_document_indexing_task)+ delegate.assert_called_once()