Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): avoid per-row MAX(position) queries in batch segment import (#42233)
CoralGarden525389f70719ac
diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.pyindex 250788d0aab..a4a5feed649 100644--- a/api/tasks/batch_create_segment_to_index_task.py+++ b/api/tasks/batch_create_segment_to_index_task.py@@ -141,20 +141,24 @@ def batch_create_segment_to_index_task(tokens_list = [0] * len(content)with session_factory.create_session() as session, session.begin():+ max_position = (+ session.scalar(+ select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == document_config["id"])+ )+ or 0+ )for segment, tokens in zip(content, tokens_list):content = segment["content"]doc_id = str(uuid.uuid4())segment_hash = helper.generate_text_hash(content)- max_position = session.scalar(- select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == document_config["id"])- )+ max_position += 1segment_document = DocumentSegment(tenant_id=tenant_id,dataset_id=dataset_id,document_id=document_id,index_node_id=doc_id,index_node_hash=segment_hash,- position=max_position + 1 if max_position else 1,+ position=max_position,content=content,word_count=len(content),tokens=tokens,diff --git a/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.py b/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.pynew file mode 100644index 00000000000..c3ea579e365--- /dev/null+++ b/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.py@@ -0,0 +1,117 @@+from datetime import UTC, datetime+from pathlib import Path+from unittest.mock import patch+from uuid import uuid4++from sqlalchemy import event, select+from sqlalchemy.engine import Engine+from sqlalchemy.orm import Session++from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from extensions.storage.storage_type import StorageType+from models.dataset import Dataset, Document, DocumentSegment+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from models.model import UploadFile+from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task+++def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch(+ sqlite_session: Session,+ sqlite_engine: Engine,+) -> None:+ tenant_id = str(uuid4())+ user_id = str(uuid4())+ dataset = Dataset(+ id=str(uuid4()),+ tenant_id=tenant_id,+ name="Batch import dataset",+ data_source_type=DataSourceType.UPLOAD_FILE,+ created_by=user_id,+ indexing_technique=IndexTechniqueType.ECONOMY,+ )+ document = Document(+ id=str(uuid4()),+ tenant_id=tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name="document.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by=user_id,+ indexing_status=IndexingStatus.COMPLETED,+ enabled=True,+ archived=False,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ word_count=0,+ )+ upload_file = UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key="batch-import.csv",+ name="batch-import.csv",+ size=32,+ extension=".csv",+ mime_type="text/csv",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=user_id,+ created_at=datetime.now(UTC),+ used=False,+ )+ existing_segment = DocumentSegment(+ tenant_id=tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ position=7,+ content="existing",+ word_count=1,+ tokens=1,+ created_by=user_id,+ index_node_id="existing-node",+ index_node_hash="existing-hash",+ status=SegmentStatus.COMPLETED,+ )+ sqlite_session.add_all([dataset, document, upload_file, existing_segment])+ sqlite_session.commit()++ def mock_download(_key: str, file_path: str) -> None:+ Path(file_path).write_text("content\nfirst\nsecond\nthird\n", encoding="utf-8")++ max_position_queries: list[str] = []++ def count_max_position_query(+ _connection: object,+ _cursor: object,+ statement: str,+ _parameters: object,+ _context: object,+ _executemany: bool,+ ) -> None:+ normalized_statement = statement.lower()+ if "max(" in normalized_statement and "document_segments.position" in normalized_statement:+ max_position_queries.append(statement)++ event.listen(sqlite_engine, "before_cursor_execute", count_max_position_query)+ try:+ with (+ patch("tasks.batch_create_segment_to_index_task.storage.download", side_effect=mock_download),+ patch("tasks.batch_create_segment_to_index_task.VectorService.create_segments_vector"),+ ):+ batch_create_segment_to_index_task(+ job_id=str(uuid4()),+ upload_file_id=upload_file.id,+ dataset_id=dataset.id,+ document_id=document.id,+ tenant_id=tenant_id,+ user_id=user_id,+ )+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", count_max_position_query)++ positions = sqlite_session.scalars(+ select(DocumentSegment.position)+ .where(DocumentSegment.document_id == document.id)+ .order_by(DocumentSegment.position)+ ).all()+ assert positions == [7, 8, 9, 10]+ assert len(max_position_queries) == 1