Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
perf(api): batch indexing status segment counts (#42191)
CoralGarden525b58f5146033
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.pyindex b85ea3bc9eb..4e07b814713 100644--- a/api/controllers/console/datasets/datasets.py+++ b/api/controllers/console/datasets/datasets.py@@ -47,9 +47,9 @@ from graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import build_icon_url, dump_response, to_timestampfrom libs.login import login_requiredfrom libs.url_utils import normalize_api_base_url-from models import Account, ApiToken, App, Dataset, Document, DocumentSegment, UploadFile+from models import Account, ApiToken, App, Dataset, Document, UploadFilefrom models.dataset import DatasetPermission, DatasetPermissionEnum, DatasetQuery-from models.enums import ApiTokenType, SegmentStatus+from models.enums import ApiTokenTypefrom models.provider_ids import ModelProviderIDfrom services import dataset_api_key_servicefrom services.api_token_service import ApiTokenCache@@ -1054,31 +1054,13 @@ class DatasetIndexingStatusApi(Resource):documents = session.scalars(select(Document).where(Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id)).all()+ segment_counts = DocumentService.get_document_segment_counts(+ documents,+ session=session,+ )documents_status = []for document in documents:- completed_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.completed_at.isnot(None),- DocumentSegment.tenant_id == dataset.tenant_id,- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- total_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.tenant_id == dataset.tenant_id,- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))# Create a dictionary with document attributes and additional fieldsdocument_dict = {"id": document.id,diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.pyindex dd02a4011d8..2f34de91f2e 100644--- a/api/controllers/console/datasets/datasets_document.py+++ b/api/controllers/console/datasets/datasets_document.py@@ -939,27 +939,10 @@ class DocumentBatchIndexingStatusApi(DocumentResource):def get(self, session: Session, current_user: Account, dataset_id: UUID, batch: str):dataset_id_str = str(dataset_id)documents = self.get_batch_documents(session, dataset_id_str, batch, current_user)+ segment_counts = DocumentService.get_document_segment_counts(documents, session=session)documents_status = []for document in documents:- completed_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.completed_at.isnot(None),- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- total_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))# Create a dictionary with document attributes and additional fieldsdocument_dict = {"id": document.id,diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.pyindex 37170840c93..b18bb485971 100644--- a/api/controllers/service_api/dataset/document.py+++ b/api/controllers/service_api/dataset/document.py@@ -23,7 +23,7 @@ from pydantic import (model_validator,)from pydantic.json_schema import SkipJsonSchema-from sqlalchemy import desc, func, select+from sqlalchemy import desc, selectfrom sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound@@ -76,8 +76,7 @@ from fields.document_fields import (from libs.helper import dump_responsefrom libs.login import current_userfrom libs.pagination import paginate_query-from models.dataset import Dataset, Document, DocumentSegment-from models.enums import SegmentStatus+from models.dataset import Dataset, Documentfrom services.dataset_service import DatasetService, DocumentServicefrom services.entities.knowledge_entities.knowledge_entities import (DocForm,@@ -1158,27 +1157,10 @@ class DocumentIndexingStatusApi(DatasetApiResource):documents = DocumentService.get_batch_documents(dataset_id_str, batch, session)if not documents:raise NotFound("Documents not found.")+ segment_counts = DocumentService.get_document_segment_counts(documents, session=session)documents_status = []for document in documents:- completed_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.completed_at.isnot(None),- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- total_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))# Create a dictionary with document attributes and additional fieldsdocument_dict = {"id": document.id,diff --git a/api/services/dataset_service.py b/api/services/dataset_service.pyindex 8b3464bf5ae..e5c29958574 100644--- a/api/services/dataset_service.py+++ b/api/services/dataset_service.py@@ -12,7 +12,7 @@ from typing import Annotated, Any, Literal, TypedDict, castimport sqlalchemy as safrom pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validatorfrom redis.exceptions import LockNotOwnedError-from sqlalchemy import ColumnElement, delete, exists, func, select, update+from sqlalchemy import ColumnElement, case, delete, exists, func, select, tuple_, updatefrom sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound@@ -1972,6 +1972,41 @@ class DocumentService:return documents+ @staticmethod+ def get_document_segment_counts(+ documents: Sequence[Document],+ session: Session,+ ) -> dict[str, tuple[int, int]]:+ """Get completed and total segment counts for multiple documents in one query."""+ if not documents:+ return {}++ document_owner_keys = {+ (str(document.tenant_id), str(document.dataset_id), str(document.id)) for document in documents+ }++ rows = session.execute(+ select(+ DocumentSegment.document_id,+ func.count(DocumentSegment.id).label("total_segments"),+ func.coalesce(func.sum(case((DocumentSegment.completed_at.isnot(None), 1), else_=0)), 0).label(+ "completed_segments"+ ),+ )+ .where(+ tuple_(DocumentSegment.tenant_id, DocumentSegment.dataset_id, DocumentSegment.document_id).in_(+ document_owner_keys+ ),+ DocumentSegment.status != SegmentStatus.RE_SEGMENT,+ )+ .group_by(DocumentSegment.document_id)+ )++ return {+ str(document_id): (int(completed_segments or 0), int(total_segments or 0))+ for document_id, total_segments, completed_segments in rows+ }+@staticmethoddef get_document_file_detail(file_id: str, session: Session):file_detail = session.get(UploadFile, file_id)diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.pyindex 4e7918d0850..1f31d150a5b 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py@@ -8,6 +8,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock, call, patchimport pytestfrom flask import Flask+from sqlalchemy import eventfrom sqlalchemy.orm import Sessionfrom werkzeug.exceptions import BadRequest, Forbidden, NotFound@@ -46,7 +47,7 @@ from core.rag.retrieval.retrieval_methods import RetrievalMethodfrom extensions.storage.storage_type import StorageTypefrom models.account import Account, TenantAccountRolefrom models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegment-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatusfrom models.model import ApiToken, App, AppMode, IconType, UploadFilefrom services.dataset_ref_service import DatasetReffrom services.dataset_service import DatasetPermissionService, DatasetService@@ -171,16 +172,25 @@ def make_document_status(**overrides) -> Document:return Document(**base)-def make_document_segment(*, position: int, completed: bool) -> DocumentSegment:+def make_document_segment(+ *,+ position: int,+ completed: bool,+ document_id: str = "doc-1",+ tenant_id: str = "tenant-1",+ dataset_id: str = "dataset-1",+ status: SegmentStatus = SegmentStatus.WAITING,+) -> DocumentSegment:return DocumentSegment(- tenant_id="tenant-1",- dataset_id="dataset-1",- document_id="doc-1",+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document_id,position=position,content=f"segment {position}",word_count=2,tokens=2,created_by="account-1",+ status=status,completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) if completed else None,)@@ -1396,6 +1406,65 @@ class TestDatasetIndexingStatusApi(_UsesSQLiteSession):assert item["completed_segments"] == 2assert item["total_segments"] == 5+ def test_get_batch_segment_counts_uses_one_aggregate_query(self, app: Flask, sqlite_engine):+ api = DatasetIndexingStatusApi()+ method = unwrap(api.get)+ dataset = make_dataset(id="dataset-1")+ documents = [make_document_status(id=f"doc-{index}", position=index) for index in range(1, 6)]+ session = self.session+ session.add_all(documents)+ session.add_all(+ [+ make_document_segment(+ position=position,+ completed=position <= 2,+ document_id=document.id,+ )+ for document in documents+ for position in range(1, 4)+ ]+ )+ session.add_all(+ [+ make_document_segment(+ position=4,+ completed=True,+ document_id=documents[0].id,+ status=SegmentStatus.RE_SEGMENT,+ ),+ make_document_segment(+ position=5,+ completed=True,+ document_id=documents[0].id,+ tenant_id="other-tenant",+ ),+ ]+ )+ session.flush()++ select_statements: list[str] = []++ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):+ if statement.lstrip().upper().startswith("SELECT"):+ select_statements.append(statement)++ event.listen(sqlite_engine, "before_cursor_execute", record_select)+ try:+ with (+ app.test_request_context("/"),+ patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),+ patch.object(DatasetService, "check_dataset_permission"),+ ):+ response, status = method(api, session, "tenant-1", make_account(), "dataset-1")+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", record_select)++ assert status == 200+ assert len(response["data"]) == 5+ assert all(item["completed_segments"] == 2 for item in response["data"])+ assert all(item["total_segments"] == 3 for item in response["data"])+ assert len(select_statements) == 2+class TestDatasetApiKeyApi(_UsesSQLiteSession):def test_get_api_keys_success(self, app: Flask):diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.pyindex 010ebbd67fd..44b91144143 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py@@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patchimport pytestfrom flask import Flask+from sqlalchemy import eventfrom sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound@@ -128,11 +129,11 @@ def make_account(role: TenantAccountRole = TenantAccountRole.EDITOR) -> Account:return account-def make_segment(*, position: int, completed: bool = True) -> DocumentSegment:+def make_segment(*, position: int, completed: bool = True, document_id: str = "doc-1") -> DocumentSegment:return DocumentSegment(tenant_id="tenant-1",dataset_id="ds-1",- document_id="doc-1",+ document_id=document_id,position=position,content=f"segment {position}",word_count=2,@@ -1336,6 +1337,43 @@ class TestDocumentBatchIndexingStatusApi(_UsesSQLiteSession):with pytest.raises(NotFound):method(api, self.session, user, "ds-1", "invalid-batch")+ def test_get_batch_status_uses_one_aggregate_query(self, app: Flask, patch_tenant, sqlite_engine):+ api = DocumentBatchIndexingStatusApi()+ method = unwrap(api.get)+ user, _ = patch_tenant+ documents = [make_document(id=f"doc-{index}", position=index) for index in range(1, 6)]+ session = self.session+ session.add_all(+ [+ make_segment(+ position=position,+ completed=position <= 2,+ document_id=document.id,+ )+ for document in documents+ for position in range(1, 4)+ ]+ )+ session.flush()++ select_statements: list[str] = []++ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):+ if statement.lstrip().upper().startswith("SELECT"):+ select_statements.append(statement)++ event.listen(sqlite_engine, "before_cursor_execute", record_select)+ try:+ with app.test_request_context("/"), patch.object(api, "get_batch_documents", return_value=documents):+ response = method(api, session, user, "ds-1", "batch-1")+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", record_select)++ assert len(response["data"]) == 5+ assert all(item["completed_segments"] == 2 for item in response["data"])+ assert all(item["total_segments"] == 3 for item in response["data"])+ assert len(select_statements) == 1+class TestDocumentIndexingStatusApi(_UsesSQLiteSession):def test_get_status_success_serializes_status_shape(self, app: Flask, patch_tenant):diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.pyindex 5fc0837cc9f..efb38d0e340 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py@@ -25,7 +25,7 @@ from unittest.mock import Mock, PropertyMock, patchimport pytestfrom flask import Flask-from sqlalchemy import Engine+from sqlalchemy import Engine, eventfrom sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound@@ -1291,6 +1291,7 @@ class TestDocumentIndexingStatusApi(SQLiteControllerTest):)mock_doc_svc.get_batch_documents.return_value = [document]+ mock_doc_svc.get_document_segment_counts.return_value = {document.id: (5, 5)}self._persist_dataset(mock_dataset)self.session.add_all(@@ -1392,6 +1393,68 @@ class TestDocumentIndexingStatusApi(SQLiteControllerTest):batch=batch_id,)+ def test_get_indexing_status_uses_one_aggregate_query(self, app: Flask, mock_tenant, mock_dataset, sqlite_engine):+ batch_id = "batch_123"+ documents = [+ make_serializable_document(+ id=str(uuid.uuid4()),+ tenant_id=mock_tenant,+ dataset_id=mock_dataset.id,+ )+ for _ in range(5)+ ]+ self._persist_dataset(mock_dataset)+ self.session.add_all(+ [+ DocumentSegment(+ tenant_id=mock_tenant,+ dataset_id=mock_dataset.id,+ document_id=document.id,+ position=position,+ content=f"Segment {position}",+ word_count=2,+ tokens=2,+ created_by="user-1",+ status=SegmentStatus.COMPLETED,+ completed_at=datetime(2021, 1, 1, tzinfo=UTC) if position <= 2 else None,+ )+ for document in documents+ for position in range(1, 4)+ ]+ )+ self.session.commit()++ select_statements: list[str] = []++ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):+ if statement.lstrip().upper().startswith("SELECT"):+ select_statements.append(statement)++ event.listen(sqlite_engine, "before_cursor_execute", record_select)+ try:+ with (+ app.test_request_context(+ f"/datasets/{mock_dataset.id}/documents/{batch_id}/indexing-status",+ method="GET",+ ),+ patch.object(DocumentService, "get_batch_documents", return_value=documents),+ ):+ api = DocumentIndexingStatusApi()+ response = inspect.unwrap(type(api).get)(+ api,+ self.session,+ tenant_id=mock_tenant,+ dataset_id=mock_dataset.id,+ batch=batch_id,+ )+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", record_select)++ assert len(response["data"]) == 5+ assert all(item["completed_segments"] == 2 for item in response["data"])+ assert all(item["total_segments"] == 3 for item in response["data"])+ assert len(select_statements) == 2+class TestDocumentAddByTextApi(SQLiteControllerTest):"""Test suite for DocumentAddByTextApi.post() endpoint.