Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
perf(api): batch segment list child chunks and attachments (#45050)
Tianyu Caia709c29669e4
diff --git a/api/repositories/knowledge/dataset_read_repository.py b/api/repositories/knowledge/dataset_read_repository.pyindex 81a072790fa..06c8f90f2d1 100644--- a/api/repositories/knowledge/dataset_read_repository.py+++ b/api/repositories/knowledge/dataset_read_repository.py@@ -132,6 +132,85 @@ def load_document_read_batch(documents: Sequence[Document], *, session: Session))+@dataclass(frozen=True)+class SegmentReadBatch:+ child_chunks: dict[str, list[ChildChunk]] = field(default_factory=dict)+ attachments: dict[str, list[UploadFile]] = field(default_factory=dict)+++def load_segment_read_batch(segments: Sequence[DocumentSegment], *, session: Session) -> SegmentReadBatch:+ if not segments:+ return SegmentReadBatch()+ document_owners = list({(s.tenant_id, s.dataset_id, s.document_id) for s in segments})+ documents = {+ document.id: document+ for document in session.scalars(+ select(Document).where(tuple_(Document.tenant_id, Document.dataset_id, Document.id).in_(document_owners))+ )+ }+ rule_owners = {+ (doc.dataset_process_rule_id, doc.dataset_id) for doc in documents.values() if doc.dataset_process_rule_id+ }+ process_rules = (+ {+ rule.id: rule+ for rule in session.scalars(+ select(DatasetProcessRule).where(+ tuple_(DatasetProcessRule.id, DatasetProcessRule.dataset_id).in_(rule_owners)+ )+ )+ }+ if rule_owners+ else {}+ )+ child_document_ids: set[str] = set()+ for document in documents.values():+ if not document.dataset_process_rule_id:+ continue+ process_rule = process_rules.get(document.dataset_process_rule_id)+ if process_rule and process_rule.mode == "hierarchical":+ rules_dict = process_rule.rules_dict+ if rules_dict:+ rules = Rule.model_validate(rules_dict)+ if rules.parent_mode and rules.parent_mode != ParentMode.FULL_DOC:+ child_document_ids.add(document.id)++ child_owners = [+ (s.tenant_id, s.dataset_id, s.document_id, s.id) for s in segments if s.document_id in child_document_ids+ ]+ child_chunks: dict[str, list[ChildChunk]] = {}+ if child_owners:+ for chunk in session.scalars(+ select(ChildChunk)+ .where(+ tuple_(ChildChunk.tenant_id, ChildChunk.dataset_id, ChildChunk.document_id, ChildChunk.segment_id).in_(+ child_owners+ )+ )+ .order_by(ChildChunk.position.asc())+ ):+ child_chunks.setdefault(chunk.segment_id, []).append(chunk)++ attachment_owners = [(s.tenant_id, s.dataset_id, s.document_id, s.id) for s in segments]+ attachments: dict[str, list[UploadFile]] = {}+ for segment_id, file in session.execute(+ select(SegmentAttachmentBinding.segment_id, UploadFile)+ .join(SegmentAttachmentBinding, UploadFile.id == SegmentAttachmentBinding.attachment_id)+ .where(+ tuple_(+ SegmentAttachmentBinding.tenant_id,+ SegmentAttachmentBinding.dataset_id,+ SegmentAttachmentBinding.document_id,+ SegmentAttachmentBinding.segment_id,+ ).in_(attachment_owners),+ UploadFile.tenant_id == SegmentAttachmentBinding.tenant_id,+ )+ ):+ attachments.setdefault(segment_id, []).append(file)++ return SegmentReadBatch(child_chunks=child_chunks, attachments=attachments)++def get_segment_content_file_ids(segment: DocumentSegment, *, upload_ids: set[str], tool_ids: set[str], session: Session) -> tuple[set[str], set[str]]:diff --git a/api/repositories/knowledge/segment_repository.py b/api/repositories/knowledge/segment_repository.pyindex 47766f46644..9badeb30c04 100644--- a/api/repositories/knowledge/segment_repository.py+++ b/api/repositories/knowledge/segment_repository.py@@ -32,11 +32,19 @@ from models.dataset import (SegmentAttachmentBinding,)from models.enums import IndexingStatus, SegmentStatus, SegmentType, SummaryStatus-from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table, get_segment_child_chunks+from repositories.knowledge.dataset_read_repository import (+ SegmentReadBatch,+ get_dataset_keyword_table,+ get_segment_child_chunks,+ load_segment_read_batch,+)from repositories.knowledge.dataset_repository import _get_datasetfrom repositories.knowledge.document_repository import _get_document, require_indexing_documentfrom repositories.knowledge.keyword_table_repository import persist_keyword_table-from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_content+from repositories.knowledge.segment_read_adapter import (+ format_segment_attachments,+ sign_segment_content,+)from services.knowledge.entities.segments import ChildChunkRecord, SegmentRecordfrom services.knowledge.indexing.execution import IndexingDocumentfrom services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef@@ -220,11 +228,13 @@ class SQLAlchemySegmentRepository:if itemselse {})+ batch = load_segment_read_batch(items, session=session)return SegmentPage(items=tuple(_segment_data(segment,summaries[segment.id].summary_content if segment.id in summaries else None,+ batch,session,)for segment in items@@ -622,8 +632,9 @@ def _child_state(child: ChildChunk) -> ChildChunkState:def _detail(session: Session, segment: DocumentSegment, document: Document) -> SegmentDetail:summary = _summary(session, segment)+ batch = load_segment_read_batch([segment], session=session)return SegmentDetail(- data=_segment_data(segment, summary.summary_content if summary else None, session),+ data=_segment_data(segment, summary.summary_content if summary else None, batch, session),doc_form=str(document.doc_form),)@@ -663,7 +674,9 @@ def _get_child_chunk(session: Session, segment_ref: SegmentRef, child_chunk_id:)-def _segment_data(segment: DocumentSegment, summary: str | None, session: Session) -> SegmentRecord:+def _segment_data(+ segment: DocumentSegment, summary: str | None, batch: SegmentReadBatch, session: Session+) -> SegmentRecord:return SegmentRecord.model_validate({"id": segment.id,@@ -691,8 +704,8 @@ def _segment_data(segment: DocumentSegment, summary: str | None, session: Sessio"stopped_at": segment.stopped_at,"sign_content": sign_segment_content(segment, session=session),"summary": summary,- "child_chunks": get_segment_child_chunks(segment, session=session, include_full_doc=False),- "attachments": get_segment_attachments(segment, session=session),+ "child_chunks": batch.child_chunks.get(segment.id, []),+ "attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),})diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.pyindex 97aba42cb61..46bc1df4d6d 100644--- a/api/services/knowledge/dataset_read_service.py+++ b/api/services/knowledge/dataset_read_service.py@@ -16,16 +16,20 @@ from models.model import UploadFilefrom repositories.knowledge.dataset_read_repository import (DatasetDetailPrefetch,DocumentReadBatch,+ SegmentReadBatch,build_dataset_detail_prefetch,get_document_dataset,get_document_metadata_rows,get_document_upload_file,get_document_uploader,get_query_upload_files,- get_segment_child_chunks,load_document_read_batch,+ load_segment_read_batch,+)+from repositories.knowledge.segment_read_adapter import (+ format_segment_attachments,+ sign_segment_content,)-from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_contentfrom services.knowledge.entities.datasets import DatasetDetailRecord@@ -127,9 +131,13 @@ def load_document_details(documents: Iterable[Document], *, session: Session) ->]-def load_segment_detail(segment: DocumentSegment, summary: str | None, *, session: Session) -> dict[str, Any]:+def load_segment_detail(+ segment: DocumentSegment, summary: str | None, *, session: Session, batch: SegmentReadBatch | None = None+) -> dict[str, Any]:"""Load relations and authorize file links before handing plain values to the response model."""- child_chunks = get_segment_child_chunks(segment, session=session, include_full_doc=False)+ if batch is None:+ batch = load_segment_read_batch([segment], session=session)+ child_chunks = batch.child_chunks.get(segment.id, [])return {"id": segment.id,"position": segment.position,@@ -168,7 +176,7 @@ def load_segment_detail(segment: DocumentSegment, summary: str | None, *, sessio}for chunk in child_chunks],- "attachments": get_segment_attachments(segment, session=session),+ "attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),"summary": summary,}@@ -176,7 +184,9 @@ def load_segment_detail(segment: DocumentSegment, summary: str | None, *, sessiodef load_segment_details(segments: Iterable[DocumentSegment], summaries: Mapping[str, str | None], *, session: Session) -> list[dict[str, Any]]:- return [load_segment_detail(segment, summaries.get(segment.id), session=session) for segment in segments]+ items = list(segments)+ batch = load_segment_read_batch(items, session=session)+ return [load_segment_detail(segment, summaries.get(segment.id), session=session, batch=batch) for segment in items]def format_document_source_detail(document: Document, file: UploadFile | None) -> dict[str, Any]:diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.pyindex 1b6e235d223..06aec9a789b 100644--- a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_repository.py@@ -1,6 +1,7 @@"""Behavioral and architecture regressions for explicit knowledge reads."""import ast+import jsonfrom collections.abc import Generatorfrom contextlib import contextmanagerfrom datetime import datetime@@ -19,24 +20,37 @@ from fields.document_fields import DocumentResponsefrom graphon.file import helpers as file_helpersfrom models import dataset as dataset_modulefrom models.dataset import (+ ChildChunk,Dataset,DatasetMetadata,DatasetMetadataBinding,+ DatasetProcessRule,DatasetQuery,Document,DocumentSegment,ExternalKnowledgeApis,ExternalKnowledgeBindings,+ SegmentAttachmentBinding,)-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleModefrom models.model import UploadFilefrom models.tools import ToolFile-from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch+from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch, load_segment_read_batchfrom repositories.knowledge.segment_read_adapter import sign_segment_contentfrom services.knowledge.dataset_read_service import (get_dataset_queries,get_document_metadata_details,load_document_details,+ load_segment_details,+)++_HIERARCHICAL_RULES = json.dumps(+ {+ "pre_processing_rules": [],+ "segmentation": {"separator": "\n", "max_tokens": 500},+ "parent_mode": "paragraph",+ "subchunk_segmentation": {"separator": "\n", "max_tokens": 200},+ })@@ -250,6 +264,135 @@ def test_document_page_load_is_bounded_and_serialization_performs_no_queries(sqlassert serialization_queries == []+def test_segment_page_load_is_bounded_for_hierarchical_documents(sqlite_session: Session) -> None:+ rule = DatasetProcessRule(+ dataset_id="dataset-1", mode=ProcessRuleMode.HIERARCHICAL, rules=_HIERARCHICAL_RULES, created_by="account-1"+ )+ document = _document()+ sqlite_session.add_all([_dataset(), rule, document])+ sqlite_session.flush()+ document.dataset_process_rule_id = rule.id+ segments = [+ DocumentSegment(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ position=index,+ content=f"segment {index}",+ word_count=1,+ tokens=1,+ created_by="account-1",+ )+ for index in range(1, 21)+ ]+ sqlite_session.add_all(segments)+ sqlite_session.flush()+ for segment in segments:+ upload = _upload()+ sqlite_session.add(upload)+ sqlite_session.flush()+ sqlite_session.add_all(+ [+ ChildChunk(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=segment.id,+ position=1,+ content="child",+ word_count=1,+ created_by="account-1",+ ),+ SegmentAttachmentBinding(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=segment.id,+ attachment_id=upload.id,+ ),+ ]+ )+ sqlite_session.flush()+ with _queries(sqlite_session) as single_queries:+ load_segment_details(segments[:1], {}, session=sqlite_session)+ with _queries(sqlite_session) as page_queries:+ load_segment_details(segments, {}, session=sqlite_session)++ assert len(page_queries) == len(single_queries)+++def test_segment_read_batch_assigns_relations_to_their_segments(sqlite_session: Session) -> None:+ rule = DatasetProcessRule(+ dataset_id="dataset-1", mode=ProcessRuleMode.HIERARCHICAL, rules=_HIERARCHICAL_RULES, created_by="account-1"+ )+ document = _document()+ sqlite_session.add_all([_dataset(), rule, document])+ sqlite_session.flush()+ document.dataset_process_rule_id = rule.id++ segment_a, segment_b = [+ DocumentSegment(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ position=index,+ content=f"segment {index}",+ word_count=1,+ tokens=1,+ created_by="account-1",+ )+ for index in (1, 2)+ ]+ sqlite_session.add_all([segment_a, segment_b])+ sqlite_session.flush()++ for position in (2, 1):+ sqlite_session.add(+ ChildChunk(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=segment_a.id,+ position=position,+ content=f"child {position}",+ word_count=1,+ created_by="account-1",+ )+ )+ sqlite_session.flush()++ upload_a = _upload()+ upload_b = _upload()+ upload_other_tenant = _upload("tenant-2")+ sqlite_session.add_all([upload_a, upload_b, upload_other_tenant])+ sqlite_session.flush()++ sqlite_session.add_all(+ [+ SegmentAttachmentBinding(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=segment.id,+ attachment_id=upload.id,+ )+ for segment, upload in [+ (segment_a, upload_a),+ (segment_b, upload_b),+ (segment_b, upload_other_tenant),+ ]+ ]+ )+ sqlite_session.flush()++ batch = load_segment_read_batch([segment_a, segment_b], session=sqlite_session)++ assert [chunk.position for chunk in batch.child_chunks[segment_a.id]] == [1, 2]+ assert segment_b.id not in batch.child_chunks+ assert [file.id for file in batch.attachments[segment_a.id]] == [upload_a.id]+ assert [file.id for file in batch.attachments[segment_b.id]] == [upload_b.id]++def test_external_api_bindings_are_batched_and_tenant_scoped(sqlite_session: Session) -> None:dataset = _dataset()apis = [diff --git a/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py b/api/tests/unit_tests/repositories/knowledge/test_segment_repository.pyindex 4ed1bda2840..3f53abbcd7d 100644--- a/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py+++ b/api/tests/unit_tests/repositories/knowledge/test_segment_repository.py@@ -7,13 +7,15 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.rag.models.document import ChildDocumentfrom core.rag.models.document import Document as IndexDocument-from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment+from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment, SegmentAttachmentBindingfrom models.enums import IndexingStatus, ProcessRuleMode, SegmentStatusfrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom repositories.knowledge.segment_repository import SQLAlchemySegmentRepositoryfrom services.knowledge.indexing.errors import DocumentIsDeletedPausedErrorfrom services.knowledge.indexing.execution import IndexingDocumentfrom services.knowledge.resource_scope import DatasetRef+from services.knowledge.segments.application import SegmentListFilter+from tests.unit_tests.repositories.knowledge.test_dataset_read_repository import _queries, _uploadfrom tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _documentIndexingRows = tuple[SQLAlchemySegmentRepository, IndexingDocument, list[IndexDocument]]@@ -117,3 +119,71 @@ def test_segment_mutations_reject_foreign_tenant(indexing_rows: IndexingRows) ->]:with pytest.raises(DocumentIsDeletedPausedError):operation()+++def test_list_segments_query_count_is_bounded_for_hierarchical_documents(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ document = _document("document-1")+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.HIERARCHICAL,+ rules='{"parent_mode": "paragraph"}',+ created_by="account-1",+ )+ rule.id = "rule-1"+ document.dataset_process_rule_id = rule.id+ session.add_all([_dataset("dataset-1", "workspace-1"), document, rule])++ rows = [+ DocumentSegment(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ position=index,+ content=f"segment {index}",+ word_count=1,+ tokens=1,+ created_by="account-1",+ )+ for index in range(1, 21)+ ]+ uploads = [_upload("workspace-1") for _ in rows]+ session.add_all([*rows, *uploads])+ session.flush()++ for row, upload in zip(rows, uploads, strict=True):+ session.add_all(+ [+ ChildChunk(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=row.id,+ position=1,+ content="child",+ word_count=1,+ created_by="account-1",+ ),+ SegmentAttachmentBinding(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=row.id,+ attachment_id=upload.id,+ ),+ ]+ )++ segments = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ document_ref = DatasetRef("workspace-1", "dataset-1").document("document-1")+ with sqlite_session_factory() as session, _queries(session) as single_queries:+ segments.list_segments(document_ref, SegmentListFilter(limit=1))+ with sqlite_session_factory() as session, _queries(session) as page_queries:+ full_page = segments.list_segments(document_ref, SegmentListFilter(limit=20))++ assert len(page_queries) == len(single_queries)+ assert len(full_page.items) == 20+ assert all(len(item.child_chunks) == 1 and len(item.attachments) == 1 for item in full_page.items)