Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

perf(api): batch indexing status segment counts (#42191)

CoralGarden525b58f5146033
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index b85ea3bc9eb..4e07b814713 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -47,9 +47,9 @@ from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import build_icon_url, dump_response, to_timestamp
from libs.login import login_required
from libs.url_utils import normalize_api_base_url
-from models import Account, ApiToken, App, Dataset, Document, DocumentSegment, UploadFile
+from models import Account, ApiToken, App, Dataset, Document, UploadFile
from models.dataset import DatasetPermission, DatasetPermissionEnum, DatasetQuery
-from models.enums import ApiTokenType, SegmentStatus
+from models.enums import ApiTokenType
from models.provider_ids import ModelProviderID
from services import dataset_api_key_service
from services.api_token_service import ApiTokenCache
@@ -1054,31 +1054,13 @@ class DatasetIndexingStatusApi(Resource):
documents = session.scalars(
select(Document).where(Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id)
).all()
+ segment_counts = DocumentService.get_document_segment_counts(
+ documents,
+ session=session,
+ )
documents_status = []
for document in documents:
- completed_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.completed_at.isnot(None),
- DocumentSegment.tenant_id == dataset.tenant_id,
- DocumentSegment.dataset_id == dataset.id,
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
- total_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.tenant_id == dataset.tenant_id,
- DocumentSegment.dataset_id == dataset.id,
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))
# Create a dictionary with document attributes and additional fields
document_dict = {
"id": document.id,
diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py
index dd02a4011d8..2f34de91f2e 100644
--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@@ -939,27 +939,10 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
def get(self, session: Session, current_user: Account, dataset_id: UUID, batch: str):
dataset_id_str = str(dataset_id)
documents = self.get_batch_documents(session, dataset_id_str, batch, current_user)
+ segment_counts = DocumentService.get_document_segment_counts(documents, session=session)
documents_status = []
for document in documents:
- completed_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.completed_at.isnot(None),
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
- total_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))
# Create a dictionary with document attributes and additional fields
document_dict = {
"id": document.id,
diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py
index 37170840c93..b18bb485971 100644
--- a/api/controllers/service_api/dataset/document.py
+++ b/api/controllers/service_api/dataset/document.py
@@ -23,7 +23,7 @@ from pydantic import (
model_validator,
)
from pydantic.json_schema import SkipJsonSchema
-from sqlalchemy import desc, func, select
+from sqlalchemy import desc, select
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
@@ -76,8 +76,7 @@ from fields.document_fields import (
from libs.helper import dump_response
from libs.login import current_user
from libs.pagination import paginate_query
-from models.dataset import Dataset, Document, DocumentSegment
-from models.enums import SegmentStatus
+from models.dataset import Dataset, Document
from services.dataset_service import DatasetService, DocumentService
from services.entities.knowledge_entities.knowledge_entities import (
DocForm,
@@ -1158,27 +1157,10 @@ class DocumentIndexingStatusApi(DatasetApiResource):
documents = DocumentService.get_batch_documents(dataset_id_str, batch, session)
if not documents:
raise NotFound("Documents not found.")
+ segment_counts = DocumentService.get_document_segment_counts(documents, session=session)
documents_status = []
for document in documents:
- completed_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.completed_at.isnot(None),
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
- total_segments = (
- session.scalar(
- select(func.count(DocumentSegment.id)).where(
- DocumentSegment.document_id == str(document.id),
- DocumentSegment.status != SegmentStatus.RE_SEGMENT,
- )
- )
- or 0
- )
+ completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))
# Create a dictionary with document attributes and additional fields
document_dict = {
"id": document.id,
diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py
index 8b3464bf5ae..e5c29958574 100644
--- a/api/services/dataset_service.py
+++ b/api/services/dataset_service.py
@@ -12,7 +12,7 @@ from typing import Annotated, Any, Literal, TypedDict, cast
import sqlalchemy as sa
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from redis.exceptions import LockNotOwnedError
-from sqlalchemy import ColumnElement, delete, exists, func, select, update
+from sqlalchemy import ColumnElement, case, delete, exists, func, select, tuple_, update
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
@@ -1972,6 +1972,41 @@ class DocumentService:
return documents
+ @staticmethod
+ def get_document_segment_counts(
+ documents: Sequence[Document],
+ session: Session,
+ ) -> dict[str, tuple[int, int]]:
+ """Get completed and total segment counts for multiple documents in one query."""
+ if not documents:
+ return {}
+
+ document_owner_keys = {
+ (str(document.tenant_id), str(document.dataset_id), str(document.id)) for document in documents
+ }
+
+ rows = session.execute(
+ select(
+ DocumentSegment.document_id,
+ func.count(DocumentSegment.id).label("total_segments"),
+ func.coalesce(func.sum(case((DocumentSegment.completed_at.isnot(None), 1), else_=0)), 0).label(
+ "completed_segments"
+ ),
+ )
+ .where(
+ tuple_(DocumentSegment.tenant_id, DocumentSegment.dataset_id, DocumentSegment.document_id).in_(
+ document_owner_keys
+ ),
+ DocumentSegment.status != SegmentStatus.RE_SEGMENT,
+ )
+ .group_by(DocumentSegment.document_id)
+ )
+
+ return {
+ str(document_id): (int(completed_segments or 0), int(total_segments or 0))
+ for document_id, total_segments, completed_segments in rows
+ }
+
@staticmethod
def get_document_file_detail(file_id: str, session: Session):
file_detail = session.get(UploadFile, file_id)
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
index 4e7918d0850..1f31d150a5b 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
@@ -8,6 +8,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock, call, patch
import pytest
from flask import Flask
+from sqlalchemy import event
from sqlalchemy.orm import Session
from werkzeug.exceptions import BadRequest, Forbidden, NotFound
@@ -46,7 +47,7 @@ from core.rag.retrieval.retrieval_methods import RetrievalMethod
from extensions.storage.storage_type import StorageType
from models.account import Account, TenantAccountRole
from models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegment
-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
from models.model import ApiToken, App, AppMode, IconType, UploadFile
from services.dataset_ref_service import DatasetRef
from services.dataset_service import DatasetPermissionService, DatasetService
@@ -171,16 +172,25 @@ def make_document_status(**overrides) -> Document:
return Document(**base)
-def make_document_segment(*, position: int, completed: bool) -> DocumentSegment:
+def make_document_segment(
+ *,
+ position: int,
+ completed: bool,
+ document_id: str = "doc-1",
+ tenant_id: str = "tenant-1",
+ dataset_id: str = "dataset-1",
+ status: SegmentStatus = SegmentStatus.WAITING,
+) -> DocumentSegment:
return DocumentSegment(
- tenant_id="tenant-1",
- dataset_id="dataset-1",
- document_id="doc-1",
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ document_id=document_id,
position=position,
content=f"segment {position}",
word_count=2,
tokens=2,
created_by="account-1",
+ status=status,
completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) if completed else None,
)
@@ -1396,6 +1406,65 @@ class TestDatasetIndexingStatusApi(_UsesSQLiteSession):
assert item["completed_segments"] == 2
assert item["total_segments"] == 5
+ def test_get_batch_segment_counts_uses_one_aggregate_query(self, app: Flask, sqlite_engine):
+ api = DatasetIndexingStatusApi()
+ method = unwrap(api.get)
+ dataset = make_dataset(id="dataset-1")
+ documents = [make_document_status(id=f"doc-{index}", position=index) for index in range(1, 6)]
+ session = self.session
+ session.add_all(documents)
+ session.add_all(
+ [
+ make_document_segment(
+ position=position,
+ completed=position <= 2,
+ document_id=document.id,
+ )
+ for document in documents
+ for position in range(1, 4)
+ ]
+ )
+ session.add_all(
+ [
+ make_document_segment(
+ position=4,
+ completed=True,
+ document_id=documents[0].id,
+ status=SegmentStatus.RE_SEGMENT,
+ ),
+ make_document_segment(
+ position=5,
+ completed=True,
+ document_id=documents[0].id,
+ tenant_id="other-tenant",
+ ),
+ ]
+ )
+ session.flush()
+
+ select_statements: list[str] = []
+
+ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):
+ if statement.lstrip().upper().startswith("SELECT"):
+ select_statements.append(statement)
+
+ event.listen(sqlite_engine, "before_cursor_execute", record_select)
+ try:
+ with (
+ app.test_request_context("/"),
+ patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),
+ patch.object(DatasetService, "check_dataset_permission"),
+ ):
+ response, status = method(api, session, "tenant-1", make_account(), "dataset-1")
+ finally:
+ event.remove(sqlite_engine, "before_cursor_execute", record_select)
+
+ assert status == 200
+ assert len(response["data"]) == 5
+ assert all(item["completed_segments"] == 2 for item in response["data"])
+ assert all(item["total_segments"] == 3 for item in response["data"])
+ assert len(select_statements) == 2
+
class TestDatasetApiKeyApi(_UsesSQLiteSession):
def test_get_api_keys_success(self, app: Flask):
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py
index 010ebbd67fd..44b91144143 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py
@@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
+from sqlalchemy import event
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
@@ -128,11 +129,11 @@ def make_account(role: TenantAccountRole = TenantAccountRole.EDITOR) -> Account:
return account
-def make_segment(*, position: int, completed: bool = True) -> DocumentSegment:
+def make_segment(*, position: int, completed: bool = True, document_id: str = "doc-1") -> DocumentSegment:
return DocumentSegment(
tenant_id="tenant-1",
dataset_id="ds-1",
- document_id="doc-1",
+ document_id=document_id,
position=position,
content=f"segment {position}",
word_count=2,
@@ -1336,6 +1337,43 @@ class TestDocumentBatchIndexingStatusApi(_UsesSQLiteSession):
with pytest.raises(NotFound):
method(api, self.session, user, "ds-1", "invalid-batch")
+ def test_get_batch_status_uses_one_aggregate_query(self, app: Flask, patch_tenant, sqlite_engine):
+ api = DocumentBatchIndexingStatusApi()
+ method = unwrap(api.get)
+ user, _ = patch_tenant
+ documents = [make_document(id=f"doc-{index}", position=index) for index in range(1, 6)]
+ session = self.session
+ session.add_all(
+ [
+ make_segment(
+ position=position,
+ completed=position <= 2,
+ document_id=document.id,
+ )
+ for document in documents
+ for position in range(1, 4)
+ ]
+ )
+ session.flush()
+
+ select_statements: list[str] = []
+
+ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):
+ if statement.lstrip().upper().startswith("SELECT"):
+ select_statements.append(statement)
+
+ event.listen(sqlite_engine, "before_cursor_execute", record_select)
+ try:
+ with app.test_request_context("/"), patch.object(api, "get_batch_documents", return_value=documents):
+ response = method(api, session, user, "ds-1", "batch-1")
+ finally:
+ event.remove(sqlite_engine, "before_cursor_execute", record_select)
+
+ assert len(response["data"]) == 5
+ assert all(item["completed_segments"] == 2 for item in response["data"])
+ assert all(item["total_segments"] == 3 for item in response["data"])
+ assert len(select_statements) == 1
+
class TestDocumentIndexingStatusApi(_UsesSQLiteSession):
def test_get_status_success_serializes_status_shape(self, app: Flask, patch_tenant):
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
index 5fc0837cc9f..efb38d0e340 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
@@ -25,7 +25,7 @@ from unittest.mock import Mock, PropertyMock, patch
import pytest
from flask import Flask
-from sqlalchemy import Engine
+from sqlalchemy import Engine, event
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
@@ -1291,6 +1291,7 @@ class TestDocumentIndexingStatusApi(SQLiteControllerTest):
)
mock_doc_svc.get_batch_documents.return_value = [document]
+ mock_doc_svc.get_document_segment_counts.return_value = {document.id: (5, 5)}
self._persist_dataset(mock_dataset)
self.session.add_all(
@@ -1392,6 +1393,68 @@ class TestDocumentIndexingStatusApi(SQLiteControllerTest):
batch=batch_id,
)
+ def test_get_indexing_status_uses_one_aggregate_query(self, app: Flask, mock_tenant, mock_dataset, sqlite_engine):
+ batch_id = "batch_123"
+ documents = [
+ make_serializable_document(
+ id=str(uuid.uuid4()),
+ tenant_id=mock_tenant,
+ dataset_id=mock_dataset.id,
+ )
+ for _ in range(5)
+ ]
+ self._persist_dataset(mock_dataset)
+ self.session.add_all(
+ [
+ DocumentSegment(
+ tenant_id=mock_tenant,
+ dataset_id=mock_dataset.id,
+ document_id=document.id,
+ position=position,
+ content=f"Segment {position}",
+ word_count=2,
+ tokens=2,
+ created_by="user-1",
+ status=SegmentStatus.COMPLETED,
+ completed_at=datetime(2021, 1, 1, tzinfo=UTC) if position <= 2 else None,
+ )
+ for document in documents
+ for position in range(1, 4)
+ ]
+ )
+ self.session.commit()
+
+ select_statements: list[str] = []
+
+ def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):
+ if statement.lstrip().upper().startswith("SELECT"):
+ select_statements.append(statement)
+
+ event.listen(sqlite_engine, "before_cursor_execute", record_select)
+ try:
+ with (
+ app.test_request_context(
+ f"/datasets/{mock_dataset.id}/documents/{batch_id}/indexing-status",
+ method="GET",
+ ),
+ patch.object(DocumentService, "get_batch_documents", return_value=documents),
+ ):
+ api = DocumentIndexingStatusApi()
+ response = inspect.unwrap(type(api).get)(
+ api,
+ self.session,
+ tenant_id=mock_tenant,
+ dataset_id=mock_dataset.id,
+ batch=batch_id,
+ )
+ finally:
+ event.remove(sqlite_engine, "before_cursor_execute", record_select)
+
+ assert len(response["data"]) == 5
+ assert all(item["completed_segments"] == 2 for item in response["data"])
+ assert all(item["total_segments"] == 3 for item in response["data"])
+ assert len(select_statements) == 2
+
class TestDocumentAddByTextApi(SQLiteControllerTest):
"""Test suite for DocumentAddByTextApi.post() endpoint.