Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

perf(api): batch segment list child chunks and attachments (#45050)

Tianyu Caia709c29669e4
diff --git a/api/repositories/knowledge/dataset_read_repository.py b/api/repositories/knowledge/dataset_read_repository.py
index 81a072790fa..06c8f90f2d1 100644
--- a/api/repositories/knowledge/dataset_read_repository.py
+++ b/api/repositories/knowledge/dataset_read_repository.py
@@ -132,6 +132,85 @@ def load_document_read_batch(documents: Sequence[Document], *, session: Session)
)
+@dataclass(frozen=True)
+class SegmentReadBatch:
+ child_chunks: dict[str, list[ChildChunk]] = field(default_factory=dict)
+ attachments: dict[str, list[UploadFile]] = field(default_factory=dict)
+
+
+def load_segment_read_batch(segments: Sequence[DocumentSegment], *, session: Session) -> SegmentReadBatch:
+ if not segments:
+ return SegmentReadBatch()
+ document_owners = list({(s.tenant_id, s.dataset_id, s.document_id) for s in segments})
+ documents = {
+ document.id: document
+ for document in session.scalars(
+ select(Document).where(tuple_(Document.tenant_id, Document.dataset_id, Document.id).in_(document_owners))
+ )
+ }
+ rule_owners = {
+ (doc.dataset_process_rule_id, doc.dataset_id) for doc in documents.values() if doc.dataset_process_rule_id
+ }
+ process_rules = (
+ {
+ rule.id: rule
+ for rule in session.scalars(
+ select(DatasetProcessRule).where(
+ tuple_(DatasetProcessRule.id, DatasetProcessRule.dataset_id).in_(rule_owners)
+ )
+ )
+ }
+ if rule_owners
+ else {}
+ )
+ child_document_ids: set[str] = set()
+ for document in documents.values():
+ if not document.dataset_process_rule_id:
+ continue
+ process_rule = process_rules.get(document.dataset_process_rule_id)
+ if process_rule and process_rule.mode == "hierarchical":
+ rules_dict = process_rule.rules_dict
+ if rules_dict:
+ rules = Rule.model_validate(rules_dict)
+ if rules.parent_mode and rules.parent_mode != ParentMode.FULL_DOC:
+ child_document_ids.add(document.id)
+
+ child_owners = [
+ (s.tenant_id, s.dataset_id, s.document_id, s.id) for s in segments if s.document_id in child_document_ids
+ ]
+ child_chunks: dict[str, list[ChildChunk]] = {}
+ if child_owners:
+ for chunk in session.scalars(
+ select(ChildChunk)
+ .where(
+ tuple_(ChildChunk.tenant_id, ChildChunk.dataset_id, ChildChunk.document_id, ChildChunk.segment_id).in_(
+ child_owners
+ )
+ )
+ .order_by(ChildChunk.position.asc())
+ ):
+ child_chunks.setdefault(chunk.segment_id, []).append(chunk)
+
+ attachment_owners = [(s.tenant_id, s.dataset_id, s.document_id, s.id) for s in segments]
+ attachments: dict[str, list[UploadFile]] = {}
+ for segment_id, file in session.execute(
+ select(SegmentAttachmentBinding.segment_id, UploadFile)
+ .join(SegmentAttachmentBinding, UploadFile.id == SegmentAttachmentBinding.attachment_id)
+ .where(
+ tuple_(
+ SegmentAttachmentBinding.tenant_id,
+ SegmentAttachmentBinding.dataset_id,
+ SegmentAttachmentBinding.document_id,
+ SegmentAttachmentBinding.segment_id,
+ ).in_(attachment_owners),
+ UploadFile.tenant_id == SegmentAttachmentBinding.tenant_id,
+ )
+ ):
+ attachments.setdefault(segment_id, []).append(file)
+
+ return SegmentReadBatch(child_chunks=child_chunks, attachments=attachments)
+
+
def get_segment_content_file_ids(
segment: DocumentSegment, *, upload_ids: set[str], tool_ids: set[str], session: Session
) -> tuple[set[str], set[str]]:
diff --git a/api/repositories/knowledge/segment_repository.py b/api/repositories/knowledge/segment_repository.py
index 47766f46644..9badeb30c04 100644
--- a/api/repositories/knowledge/segment_repository.py
+++ b/api/repositories/knowledge/segment_repository.py
@@ -32,11 +32,19 @@ from models.dataset import (
SegmentAttachmentBinding,
)
from models.enums import IndexingStatus, SegmentStatus, SegmentType, SummaryStatus
-from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table, get_segment_child_chunks
+from repositories.knowledge.dataset_read_repository import (
+ SegmentReadBatch,
+ get_dataset_keyword_table,
+ get_segment_child_chunks,
+ load_segment_read_batch,
+)
from repositories.knowledge.dataset_repository import _get_dataset
from repositories.knowledge.document_repository import _get_document, require_indexing_document
from repositories.knowledge.keyword_table_repository import persist_keyword_table
-from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_content
+from repositories.knowledge.segment_read_adapter import (
+ format_segment_attachments,
+ sign_segment_content,
+)
from services.knowledge.entities.segments import ChildChunkRecord, SegmentRecord
from services.knowledge.indexing.execution import IndexingDocument
from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef
@@ -220,11 +228,13 @@ class SQLAlchemySegmentRepository:
if items
else {}
)
+ batch = load_segment_read_batch(items, session=session)
return SegmentPage(
items=tuple(
_segment_data(
segment,
summaries[segment.id].summary_content if segment.id in summaries else None,
+ batch,
session,
)
for segment in items
@@ -622,8 +632,9 @@ def _child_state(child: ChildChunk) -> ChildChunkState:
def _detail(session: Session, segment: DocumentSegment, document: Document) -> SegmentDetail:
summary = _summary(session, segment)
+ batch = load_segment_read_batch([segment], session=session)
return SegmentDetail(
- data=_segment_data(segment, summary.summary_content if summary else None, session),
+ data=_segment_data(segment, summary.summary_content if summary else None, batch, session),
doc_form=str(document.doc_form),
)
@@ -663,7 +674,9 @@ def _get_child_chunk(session: Session, segment_ref: SegmentRef, child_chunk_id:
)
-def _segment_data(segment: DocumentSegment, summary: str | None, session: Session) -> SegmentRecord:
+def _segment_data(
+ segment: DocumentSegment, summary: str | None, batch: SegmentReadBatch, session: Session
+) -> SegmentRecord:
return SegmentRecord.model_validate(
{
"id": segment.id,
@@ -691,8 +704,8 @@ def _segment_data(segment: DocumentSegment, summary: str | None, session: Sessio
"stopped_at": segment.stopped_at,
"sign_content": sign_segment_content(segment, session=session),
"summary": summary,
- "child_chunks": get_segment_child_chunks(segment, session=session, include_full_doc=False),
- "attachments": get_segment_attachments(segment, session=session),
+ "child_chunks": batch.child_chunks.get(segment.id, []),
+ "attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),
}
)
diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.py
index 97aba42cb61..46bc1df4d6d 100644
--- a/api/services/knowledge/dataset_read_service.py
+++ b/api/services/knowledge/dataset_read_service.py
@@ -16,16 +16,20 @@ from models.model import UploadFile
from repositories.knowledge.dataset_read_repository import (
DatasetDetailPrefetch,
DocumentReadBatch,
+ SegmentReadBatch,
build_dataset_detail_prefetch,
get_document_dataset,
get_document_metadata_rows,
get_document_upload_file,
get_document_uploader,
get_query_upload_files,
- get_segment_child_chunks,
load_document_read_batch,
+ load_segment_read_batch,
+)
+from repositories.knowledge.segment_read_adapter import (
+ format_segment_attachments,
+ sign_segment_content,
)
-from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_content
from services.knowledge.entities.datasets import DatasetDetailRecord
@@ -127,9 +131,13 @@ def load_document_details(documents: Iterable[Document], *, session: Session) ->
]
-def load_segment_detail(segment: DocumentSegment, summary: str | None, *, session: Session) -> dict[str, Any]:
+def load_segment_detail(
+ segment: DocumentSegment, summary: str | None, *, session: Session, batch: SegmentReadBatch | None = None
+) -> dict[str, Any]:
"""Load relations and authorize file links before handing plain values to the response model."""
- child_chunks = get_segment_child_chunks(segment, session=session, include_full_doc=False)
+ if batch is None:
+ batch = load_segment_read_batch([segment], session=session)
+ child_chunks = batch.child_chunks.get(segment.id, [])
return {
"id": segment.id,
"position": segment.position,
@@ -168,7 +176,7 @@ def load_segment_detail(segment: DocumentSegment, summary: str | None, *, sessio
}
for chunk in child_chunks
],
- "attachments": get_segment_attachments(segment, session=session),
+ "attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),
"summary": summary,
}
@@ -176,7 +184,9 @@ def load_segment_detail(segment: DocumentSegment, summary: str | None, *, sessio
def load_segment_details(
segments: Iterable[DocumentSegment], summaries: Mapping[str, str | None], *, session: Session
) -> list[dict[str, Any]]:
- return [load_segment_detail(segment, summaries.get(segment.id), session=session) for segment in segments]
+ items = list(segments)
+ batch = load_segment_read_batch(items, session=session)
+ return [load_segment_detail(segment, summaries.get(segment.id), session=session, batch=batch) for segment in items]
def format_document_source_detail(document: Document, file: UploadFile | None) -> dict[str, Any]:
diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py
index 1b6e235d223..06aec9a789b 100644
--- a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py
+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py
@@ -1,6 +1,7 @@
"""Behavioral and architecture regressions for explicit knowledge reads."""
import ast
+import json
from collections.abc import Generator
from contextlib import contextmanager
from datetime import datetime
@@ -19,24 +20,37 @@ from fields.document_fields import DocumentResponse
from graphon.file import helpers as file_helpers
from models import dataset as dataset_module
from models.dataset import (
+ ChildChunk,
Dataset,
DatasetMetadata,
DatasetMetadataBinding,
+ DatasetProcessRule,
DatasetQuery,
Document,
DocumentSegment,
ExternalKnowledgeApis,
ExternalKnowledgeBindings,
+ SegmentAttachmentBinding,
)
-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode
from models.model import UploadFile
from models.tools import ToolFile
-from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch
+from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch, load_segment_read_batch
from repositories.knowledge.segment_read_adapter import sign_segment_content
from services.knowledge.dataset_read_service import (
get_dataset_queries,
get_document_metadata_details,
load_document_details,
+ load_segment_details,
+)
+
+_HIERARCHICAL_RULES = json.dumps(
+ {
+ "pre_processing_rules": [],
+ "segmentation": {"separator": "\n", "max_tokens": 500},
+ "parent_mode": "paragraph",
+ "subchunk_segmentation": {"separator": "\n", "max_tokens": 200},
+ }
)
@@ -250,6 +264,135 @@ def test_document_page_load_is_bounded_and_serialization_performs_no_queries(sql
assert serialization_queries == []
+def test_segment_page_load_is_bounded_for_hierarchical_documents(sqlite_session: Session) -> None:
+ rule = DatasetProcessRule(
+ dataset_id="dataset-1", mode=ProcessRuleMode.HIERARCHICAL, rules=_HIERARCHICAL_RULES, created_by="account-1"
+ )
+ document = _document()
+ sqlite_session.add_all([_dataset(), rule, document])
+ sqlite_session.flush()
+ document.dataset_process_rule_id = rule.id
+ segments = [
+ DocumentSegment(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ position=index,
+ content=f"segment {index}",
+ word_count=1,
+ tokens=1,
+ created_by="account-1",
+ )
+ for index in range(1, 21)
+ ]
+ sqlite_session.add_all(segments)
+ sqlite_session.flush()
+ for segment in segments:
+ upload = _upload()
+ sqlite_session.add(upload)
+ sqlite_session.flush()
+ sqlite_session.add_all(
+ [
+ ChildChunk(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=segment.id,
+ position=1,
+ content="child",
+ word_count=1,
+ created_by="account-1",
+ ),
+ SegmentAttachmentBinding(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=segment.id,
+ attachment_id=upload.id,
+ ),
+ ]
+ )
+ sqlite_session.flush()
+ with _queries(sqlite_session) as single_queries:
+ load_segment_details(segments[:1], {}, session=sqlite_session)
+ with _queries(sqlite_session) as page_queries:
+ load_segment_details(segments, {}, session=sqlite_session)
+
+ assert len(page_queries) == len(single_queries)
+
+
+def test_segment_read_batch_assigns_relations_to_their_segments(sqlite_session: Session) -> None:
+ rule = DatasetProcessRule(
+ dataset_id="dataset-1", mode=ProcessRuleMode.HIERARCHICAL, rules=_HIERARCHICAL_RULES, created_by="account-1"
+ )
+ document = _document()
+ sqlite_session.add_all([_dataset(), rule, document])
+ sqlite_session.flush()
+ document.dataset_process_rule_id = rule.id
+
+ segment_a, segment_b = [
+ DocumentSegment(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ position=index,
+ content=f"segment {index}",
+ word_count=1,
+ tokens=1,
+ created_by="account-1",
+ )
+ for index in (1, 2)
+ ]
+ sqlite_session.add_all([segment_a, segment_b])
+ sqlite_session.flush()
+
+ for position in (2, 1):
+ sqlite_session.add(
+ ChildChunk(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=segment_a.id,
+ position=position,
+ content=f"child {position}",
+ word_count=1,
+ created_by="account-1",
+ )
+ )
+ sqlite_session.flush()
+
+ upload_a = _upload()
+ upload_b = _upload()
+ upload_other_tenant = _upload("tenant-2")
+ sqlite_session.add_all([upload_a, upload_b, upload_other_tenant])
+ sqlite_session.flush()
+
+ sqlite_session.add_all(
+ [
+ SegmentAttachmentBinding(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=segment.id,
+ attachment_id=upload.id,
+ )
+ for segment, upload in [
+ (segment_a, upload_a),
+ (segment_b, upload_b),
+ (segment_b, upload_other_tenant),
+ ]
+ ]
+ )
+ sqlite_session.flush()
+
+ batch = load_segment_read_batch([segment_a, segment_b], session=sqlite_session)
+
+ assert [chunk.position for chunk in batch.child_chunks[segment_a.id]] == [1, 2]
+ assert segment_b.id not in batch.child_chunks
+ assert [file.id for file in batch.attachments[segment_a.id]] == [upload_a.id]
+ assert [file.id for file in batch.attachments[segment_b.id]] == [upload_b.id]
+
+
def test_external_api_bindings_are_batched_and_tenant_scoped(sqlite_session: Session) -> None:
dataset = _dataset()
apis = [
diff --git a/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py b/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py
index 4ed1bda2840..3f53abbcd7d 100644
--- a/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py
+++ b/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py
@@ -7,13 +7,15 @@ from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.models.document import ChildDocument
from core.rag.models.document import Document as IndexDocument
-from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment
+from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment, SegmentAttachmentBinding
from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
from services.knowledge.indexing.errors import DocumentIsDeletedPausedError
from services.knowledge.indexing.execution import IndexingDocument
from services.knowledge.resource_scope import DatasetRef
+from services.knowledge.segments.application import SegmentListFilter
+from tests.unit_tests.repositories.knowledge.test_dataset_read_repository import _queries, _upload
from tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _document
IndexingRows = tuple[SQLAlchemySegmentRepository, IndexingDocument, list[IndexDocument]]
@@ -117,3 +119,71 @@ def test_segment_mutations_reject_foreign_tenant(indexing_rows: IndexingRows) ->
]:
with pytest.raises(DocumentIsDeletedPausedError):
operation()
+
+
+def test_list_segments_query_count_is_bounded_for_hierarchical_documents(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ with sqlite_session_factory.begin() as session:
+ document = _document("document-1")
+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX
+ rule = DatasetProcessRule(
+ dataset_id="dataset-1",
+ mode=ProcessRuleMode.HIERARCHICAL,
+ rules='{"parent_mode": "paragraph"}',
+ created_by="account-1",
+ )
+ rule.id = "rule-1"
+ document.dataset_process_rule_id = rule.id
+ session.add_all([_dataset("dataset-1", "workspace-1"), document, rule])
+
+ rows = [
+ DocumentSegment(
+ tenant_id="workspace-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ position=index,
+ content=f"segment {index}",
+ word_count=1,
+ tokens=1,
+ created_by="account-1",
+ )
+ for index in range(1, 21)
+ ]
+ uploads = [_upload("workspace-1") for _ in rows]
+ session.add_all([*rows, *uploads])
+ session.flush()
+
+ for row, upload in zip(rows, uploads, strict=True):
+ session.add_all(
+ [
+ ChildChunk(
+ tenant_id="workspace-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=row.id,
+ position=1,
+ content="child",
+ word_count=1,
+ created_by="account-1",
+ ),
+ SegmentAttachmentBinding(
+ tenant_id="workspace-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ segment_id=row.id,
+ attachment_id=upload.id,
+ ),
+ ]
+ )
+
+ segments = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)
+ document_ref = DatasetRef("workspace-1", "dataset-1").document("document-1")
+ with sqlite_session_factory() as session, _queries(session) as single_queries:
+ segments.list_segments(document_ref, SegmentListFilter(limit=1))
+ with sqlite_session_factory() as session, _queries(session) as page_queries:
+ full_page = segments.list_segments(document_ref, SegmentListFilter(limit=20))
+
+ assert len(page_queries) == len(single_queries)
+ assert len(full_page.items) == 20
+ assert all(len(item.child_chunks) == 1 and len(item.attachments) == 1 for item in full_page.items)