Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): establish dataset application boundaries (#41729)
Byron.wanga67cd87eb02b
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERSindex 249c2932f65..006e3fd55ba 100644--- a/.github/CODEOWNERS+++ b/.github/CODEOWNERS@@ -57,13 +57,10 @@# Backend - RAG (Retrieval Augmented Generation)/api/core/rag/ @JohnJyong+/api/services/knowledge/ @JohnJyong/api/services/rag_pipeline/ @JohnJyong-/api/services/dataset_service.py @JohnJyong/api/services/knowledge_service.py @JohnJyong-/api/services/external_knowledge_service.py @JohnJyong/api/services/hit_testing_service.py @JohnJyong-/api/services/metadata_service.py @JohnJyong-/api/services/vector_service.py @JohnJyong/api/services/entities/knowledge_entities/ @JohnJyong/api/services/entities/external_knowledge_entities/ @JohnJyong/api/controllers/console/datasets/ @JohnJyongdiff --git a/api/.importlinter b/api/.importlinterindex badb1a6e95a..ff8d7d6fe5d 100644--- a/api/.importlinter+++ b/api/.importlinter@@ -1,12 +1,14 @@[importlinter]root_packages =core+ core.rag.extractor.entityconstantscontextconfigscontrollersextensionsfactories+ fieldslibsmachinerymodels@@ -16,7 +18,7 @@ root_packages =include_external_packages = True[importlinter:contract:backend-layers]-name = Backend modules follow the transport, application, domain, and shared-library layers+name = Backend modules follow the transport, application, execution-kernel, and shared-library layerstype = layerslayers =controllers@@ -27,7 +29,7 @@ layers =# error makes this list shrink whenever a legacy dependency is removed.unmatched_ignore_imports_alerting = errorignore_imports =- core.app.app_config.easy_ui_based_app.dataset.manager -> services.dataset_service+ core.app.app_config.easy_ui_based_app.dataset.manager -> services.knowledge.dataset_servicecore.app.apps.advanced_chat.app_generator -> controllers.console.app.workflowcore.app.apps.advanced_chat.app_generator -> services.conversation_servicecore.app.apps.advanced_chat.app_generator -> services.errors.conversation@@ -48,45 +50,44 @@ ignore_imports =core.app.apps.message_based_app_generator -> services.errors.conversationcore.app.apps.message_based_app_generator -> services.errors.messagecore.app.apps.pipeline.pipeline_generator -> services- core.app.apps.pipeline.pipeline_generator -> services.dataset_service- core.app.apps.pipeline.pipeline_generator -> services.datasource_provider_service+ core.app.apps.pipeline.pipeline_generator -> services.knowledge.dataset_service+ core.app.apps.pipeline.pipeline_generator -> services.data_source.provider_servicecore.app.apps.pipeline.pipeline_generator -> services.feature_servicecore.app.apps.pipeline.pipeline_generator -> services.workflow_draft_variable_service- core.app.apps.pipeline.pipeline_runner -> services.dataset_ref_servicecore.app.apps.workflow.app_generator -> controllers.console.app.workflowcore.app.apps.workflow.app_generator -> services.snippet_generate_servicecore.app.apps.workflow.app_generator -> services.workflow_draft_variable_servicecore.app.apps.workflow_app_runner -> tasks.mail_human_input_delivery_taskcore.app.features.annotation_reply.annotation_reply -> services.annotation_service- core.app.features.annotation_reply.annotation_reply -> services.dataset_service+ core.app.features.annotation_reply.annotation_reply -> services.knowledge.dataset_servicecore.app.llm.quota -> services.credit_pool_servicecore.app.task_pipeline.message_cycle_manager -> services.account_servicecore.app.task_pipeline.message_cycle_manager -> services.annotation_servicecore.app.workflow.layers.persistence -> services.workflow.inspector_events- core.datasource.datasource_manager -> services.datasource_provider_servicecore.helper.credential_utils -> services.enterprise.plugin_manager_servicecore.helper.credential_utils -> services.system_feature_service- core.indexing_runner -> services.vector_space_admission_servicecore.mcp.auth_client -> services- core.provider_manager -> services.credential_permission_servicecore.provider_manager -> services.credit_pool_servicecore.provider_manager -> services.feature_service- core.rag.datasource.retrieval_service -> services.external_knowledge_service+ core.rag.datasource.retrieval_service -> services.knowledge.external.servicecore.rag.index_processor.index_processor -> services.vector_space_admission_servicecore.rag.index_processor.index_processor_base -> services.file_servicecore.rag.index_processor.processor.paragraph_index_processor -> services.account_service- core.rag.index_processor.processor.paragraph_index_processor -> services.summary_index_service+ core.rag.index_processor.processor.paragraph_index_processor -> services.knowledge.summaries.adapterscore.rag.index_processor.processor.parent_child_index_processor -> services.account_service- core.rag.index_processor.processor.parent_child_index_processor -> services.summary_index_service- core.rag.index_processor.processor.qa_index_processor -> services.summary_index_service- core.rag.retrieval.dataset_retrieval -> services.external_knowledge_service+ core.rag.index_processor.processor.parent_child_index_processor -> services.knowledge.summaries.adapters+ core.rag.index_processor.processor.qa_index_processor -> services.knowledge.summaries.adapters+ core.rag.retrieval.dataset_retrieval -> services.knowledge.external.service+ core.rag.retrieval.dataset_retrieval -> services.knowledge.retrieval.attachmentscore.rag.retrieval.dataset_retrieval -> services.feature_service- core.rag.summary_index.summary_index -> services.summary_index_service+ core.rag.summary_index.summary_index -> services.knowledge.summaries.adapterscore.rag.summary_index.summary_index -> tasks.generate_summary_index_taskcore.repositories.sqlalchemy_workflow_node_execution_repository -> services.file_servicecore.repositories.sqlalchemy_workflow_node_execution_repository -> services.variable_truncatorcore.tools.tool_manager -> servicescore.workflow.node_runtime -> services+ # Node construction delegates datasource dependency assembly to the composition root.+ core.workflow.node_factory -> extensions.application_services.data_sourcescore.workflow.nodes.agent.message_transformer -> servicescore.workflow.nodes.agent_v2.agent_node -> services.agent.prompt_mentionscore.workflow.nodes.agent_v2.agent_node -> services.agent.workspace_service@@ -99,7 +100,7 @@ ignore_imports =core.workflow.nodes.agent_v2.workspace_retirement_layer -> tasks.collect_agent_resources_tasklibs.email_i18n -> services.entities.feature_entitieslibs.email_i18n -> services.system_feature_service- libs.external_api -> core+ libs.external_api -> core.pluginlibs.external_api -> core.errors.errorlibs.external_api -> extensions.ext_logginglibs.flask_utils -> models@@ -168,6 +169,141 @@ forbidden_modules =sqlalchemywerkzeug+[importlinter:contract:dataset-application-boundary]+name = Dataset application services and contracts are framework and persistence neutral+type = forbidden+source_modules =+ core.rag.extractor.entity.extract_setting+ services.knowledge.entities.segments+ services.knowledge.entities.datasets+ services.knowledge.entities.document_creation+ services.data_source.binding_application_service+ services.data_source.credential_gateway+ services.data_source.notion_import_application_service+ services.data_source.entities.credential+ services.data_source.entities.notion_import+ services.knowledge.entities.indexing_estimate+ services.knowledge.dataset_access+ services.knowledge.datasets.application+ services.knowledge.datasets.retrieval+ services.knowledge.document_sync+ services.knowledge.documents.application+ services.knowledge.external.application+ services.knowledge.indexing.errors+ services.knowledge.indexing.estimate+ services.knowledge.indexing.execution+ services.knowledge.resource_scope+ services.knowledge.segments.application+ services.knowledge.metadata.application+ services.knowledge.summaries.application+forbidden_modules =+ fields+ configs+ controllers+ extensions+ flask+ models+ repositories+ sqlalchemy+ tasks+ werkzeug+ services.knowledge.dataset_service+ services.data_source.provider_service+ services.knowledge.indexing.adapters+ services.knowledge.segments.adapters+ services.knowledge.segments.indexing+ core.datasource.datasource_manager+ core.rag.extractor.notion_extractor++[importlinter:contract:knowledge-repository-boundary]+name = Knowledge persistence adapters only depend on their consumer-owned application contracts+type = forbidden+source_modules =+ repositories.knowledge+forbidden_modules =+ controllers+ extensions+ flask+ services+ tasks+ werkzeug+allow_indirect_imports = True+ignore_imports =+ repositories.knowledge.dataset_api_key_repository -> services.auth.api_key_contracts+ repositories.knowledge.dataset_api_key_repository -> services.knowledge.api_key_service+ repositories.knowledge.dataset_repository -> services.knowledge.segments.application+ repositories.knowledge.dataset_repository -> services.knowledge.dataset_access+ repositories.knowledge.dataset_repository -> services.knowledge.indexing.estimate+ repositories.knowledge.dataset_repository -> services.knowledge.resource_scope+ repositories.knowledge.document_repository -> services.knowledge.document_sync+ repositories.knowledge.document_repository -> services.knowledge.indexing.estimate+ repositories.knowledge.document_repository -> services.knowledge.indexing.errors+ repositories.knowledge.document_repository -> services.knowledge.indexing.execution+ repositories.knowledge.document_repository -> services.knowledge.resource_scope+ repositories.knowledge.segment_repository -> services.knowledge.resource_scope+ repositories.knowledge.segment_repository -> services.knowledge.indexing.execution+ repositories.knowledge.segment_repository -> services.knowledge.segments.application+ repositories.knowledge.segment_repository -> services.knowledge.entities.segments+ repositories.knowledge.metadata_repository -> services.errors.metadata+ repositories.knowledge.metadata_repository -> services.knowledge.metadata.application+ repositories.knowledge.metadata_repository -> services.knowledge.resource_scope+ repositories.knowledge.summary_repository -> services.knowledge.summaries.application++[importlinter:contract:datasource-credential-repository-boundary]+name = Data source credential persistence does not depend on SQLAlchemy service helpers+type = forbidden+source_modules =+ repositories.credentials.query_repository+ repositories.data_source.credential_repository+forbidden_modules =+ services.data_source.provider_service+allow_indirect_imports = True++[importlinter:contract:datasource-provider-persistence-boundary]+name = Datasource provider service consumes injected persistence contracts+type = forbidden+source_modules =+ services.data_source.provider_service+forbidden_modules =+ repositories+ extensions.application_services+ extensions.ext_application_services+allow_indirect_imports = True++[importlinter:contract:credential-permission-query-policy-boundary]+name = Credential query contracts do not expose persistence implementations+type = forbidden+source_modules =+ services.credentials.query+forbidden_modules =+ repositories+ sqlalchemy+ models+ extensions+ flask+allow_indirect_imports = True++[importlinter:contract:credential-query-implementation-boundary]+name = Credential query implementations are wired only by composition or other repositories+type = forbidden+source_modules =+ controllers+ services+ core+forbidden_modules =+ repositories.credentials.query_repository+allow_indirect_imports = True++[importlinter:contract:pipeline-document-lookup-boundary]+name = Pipeline core depends on its document store port+type = forbidden+source_modules =+ core.app.apps.pipeline.pipeline_generator+ core.app.apps.pipeline.pipeline_runner+forbidden_modules =+ repositories+allow_indirect_imports = True+[importlinter:contract:init-validation-service-boundary]name = Initialization validation application service is framework and persistence neutraltype = forbidden@@ -479,7 +615,7 @@ forbidden_modules =name = Data source OAuth application service is framework and persistence neutraltype = forbiddensource_modules =- services.data_source_oauth_service+ services.data_source.oauth_serviceforbidden_modules =configscontrollers@@ -800,6 +936,46 @@ forbidden_modules =taskswerkzeug+[importlinter:contract:knowledge-segment-presentation-boundary]+name = Knowledge operations do not serialize HTTP responses+type = forbidden+allow_indirect_imports = True+source_modules =+ services.knowledge.segments+ services.knowledge.documents+ services.knowledge.datasets+ services.knowledge.external+forbidden_modules =+ controllers+ fields++[importlinter:contract:knowledge-response-persistence-boundary]+name = Knowledge response schemas do not load database state+type = forbidden+allow_indirect_imports = True+source_modules =+ fields.dataset_fields+ fields.document_fields+ fields.segment_fields+forbidden_modules =+ sqlalchemy+ models+ repositories+ services.knowledge++[importlinter:contract:dataset-model-infrastructure-boundary]+name = Dataset models do not execute infrastructure or application operations+type = forbidden+allow_indirect_imports = True+source_modules =+ models.dataset+forbidden_modules =+ repositories+ services+ extensions+ core.tools+ models.engine+[importlinter:contract:app-mcp-server-service-boundary]name = App MCP server application service is framework and persistence neutraltype = forbiddendiff --git a/api/AGENTS.md b/api/AGENTS.mdindex 160ae644a28..86f2ef34664 100644--- a/api/AGENTS.md+++ b/api/AGENTS.md@@ -15,7 +15,8 @@ Run direct Python commands through `uv run --project api`. Docker-backed integra## Architecture And Boundaries-- Keep transport parsing and serialization in controllers, orchestration in services, and domain policy in `core/` or its domain owner. Keep `libs/` business-agnostic and reuse existing owners before adding abstractions.+- Treat `api/core/` as a migration-only area: do not add files or move implementations into it. Move responsibilities out; limit changes to existing callers to the adaptations needed for that migration.+- Keep transport parsing and serialization in controllers, orchestration in services, and domain policy in its domain owner. New or extracted implementations must stay outside `core/`. Keep `libs/` business-agnostic and reuse existing owners before adding abstractions.- Before changing controller schemas, generated API contracts, or `SystemFeatureModel`, read `controllers/API_SCHEMA_GUIDE.md`. Treat `/system-features` as a minimal unauthenticated bootstrap allowlist, not a general configuration registry.- Scope tenant-owned reads and writes by the complete owner chain, and propagate `tenant_id` across every affected layer. Reconstruct trusted internal references from validated database state after payload or async boundaries.- Keep write transactions explicit and bounded. Do not perform external I/O inside an open transaction unless a documented consistency contract requires it.diff --git a/api/commands/vector.py b/api/commands/vector.pyindex 437ea40f529..d0555332e5e 100644--- a/api/commands/vector.py+++ b/api/commands/vector.py@@ -19,6 +19,7 @@ from models.dataset import Dataset, DatasetCollectionBinding, DatasetMetadata, Dfrom models.dataset import Document as DatasetDocumentfrom models.enums import DatasetMetadataType, IndexingStatus, SegmentStatusfrom models.model import App, AppAnnotationSetting, MessageAnnotation+from repositories.knowledge.dataset_read_repository import get_segment_child_chunks@click.command("vdb-migrate", help="Migrate vector db.")@@ -278,7 +279,7 @@ def migrate_knowledge_vector_database():},)if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=db_session)+ child_chunks = get_segment_child_chunks(segment, session=db_session)if child_chunks:child_documents = []for child_chunk in child_chunks:diff --git a/api/controllers/common/rbac/__init__.py b/api/controllers/common/rbac/__init__.pyindex 6f227d5034d..7ed48814fc5 100644--- a/api/controllers/common/rbac/__init__.py+++ b/api/controllers/common/rbac/__init__.py@@ -4,6 +4,7 @@ from .checks import RBAC_CHECKS_ATTR, RBACCheck, enforce_rbac_checksfrom .locators import (AgentBehindApp,AgentId,+ DatasetByDocument,DatasetByPipeline,DatasetId,PlainApp,@@ -16,6 +17,7 @@ __all__ = ["RBAC_CHECKS_ATTR","AgentBehindApp","AgentId",+ "DatasetByDocument","DatasetByPipeline","DatasetId","PlainApp",diff --git a/api/controllers/common/rbac/locators.py b/api/controllers/common/rbac/locators.pyindex 16790059045..1fde75f517a 100644--- a/api/controllers/common/rbac/locators.py+++ b/api/controllers/common/rbac/locators.py@@ -15,6 +15,7 @@ if TYPE_CHECKING:__all__ = ["AgentBehindApp","AgentId",+ "DatasetByDocument","DatasetByPipeline","DatasetId","PlainApp",@@ -133,6 +134,18 @@ class DatasetId(_ParamLocator):return RBACResourceService.get_dataset_maintainer(db.session, tenant_id, identity.id)+class DatasetByDocument(DatasetId):+ default_param = "document_id"++ @override+ def locate(self, tenant_id: str, path_args: Mapping[str, object]) -> ResourceIdentity | None:+ document_id = _required(path_args, self.param)+ dataset_id = RBACResourceService.get_dataset_id_by_document(db.session, tenant_id, document_id)+ if dataset_id is None:+ raise NotFound("Document not found")+ return ResourceIdentity(self.scope, dataset_id)++class DatasetByPipeline(DatasetId):default_param = "pipeline_id"diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.pyindex 58019cbdda1..e1db9164dca 100644--- a/api/controllers/console/app/app.py+++ b/api/controllers/console/app/app.py@@ -79,7 +79,8 @@ from services.entities.app_entities import (UpdateAppParams,)from services.entities.dsl_entities import ImportStatus-from services.entities.knowledge_entities.knowledge_entities import (+from services.errors.account import NoPermissionError+from services.knowledge.entities.knowledge_entities import (DataSource,InfoList,NotionIcon,@@ -91,7 +92,6 @@ from services.entities.knowledge_entities.knowledge_entities import (WeightModel,WeightVectorSetting,)-from services.errors.account import NoPermissionErrorregister_enum_models(console_ns, IconType)diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.pyindex f42aca2997e..279748d7720 100644--- a/api/controllers/console/auth/data_source_bearer_auth.py+++ b/api/controllers/console/auth/data_source_bearer_auth.py@@ -27,7 +27,7 @@ from services.auth.errors import (InvalidDataSourceApiKeyAuthCredentialsError,UnsupportedDataSourceApiKeyAuthProviderError,)-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthCredentials,)@@ -80,7 +80,7 @@ class ApiKeyAuthDataSource(Resource):@console_ns.response(HTTPStatus.OK, "Success", console_ns.models[ApiKeyAuthDataSourceListResponse.__name__])@console_account_admission()def get(self, request_context: RequestContext):- bindings = application_services().data_source_api_key_auth.list_bindings(request_context)+ bindings = application_services().data_sources.api_key_auth.list_bindings(request_context)return dump_response(ApiKeyAuthDataSourceListResponse,{@@ -120,7 +120,7 @@ class ApiKeyAuthDataSourceBinding(Resource):),)try:- application_services().data_source_api_key_auth.create_binding(request_context, command)+ application_services().data_sources.api_key_auth.create_binding(request_context, command)except UnsupportedDataSourceApiKeyAuthProviderError as exc:raise DataSourceApiKeyAuthProviderNotSupportedError() from excexcept InvalidDataSourceApiKeyAuthCredentialsError as exc:@@ -140,5 +140,5 @@ class ApiKeyAuthDataSourceBindingDelete(Resource):rbac_checks=[RBACCheck(RBACPermission.CREDENTIAL_MANAGE, Workspace())],)def delete(self, request_context: RequestContext, binding_id: UUID):- application_services().data_source_api_key_auth.delete_binding(request_context, str(binding_id))+ application_services().data_sources.api_key_auth.delete_binding(request_context, str(binding_id))return "", HTTPStatus.NO_CONTENTdiff --git a/api/controllers/console/auth/data_source_oauth.py b/api/controllers/console/auth/data_source_oauth.pyindex 8c4147ea7fa..0fce33ff038 100644--- a/api/controllers/console/auth/data_source_oauth.py+++ b/api/controllers/console/auth/data_source_oauth.py@@ -24,13 +24,13 @@ from fields.base import ResponseModelfrom libs.helper import dump_responsefrom machinery.context import RequestContextfrom models.account import TenantAccountRole-from services.data_source_oauth_service import (+from services.data_source.entities.oauth import DataSourceOAuthCallback+from services.data_source.oauth_service import (DataSourceOAuthConfigurationError,DataSourceOAuthError,InvalidDataSourceOAuthCodeError,InvalidDataSourceOAuthProviderError,)-from services.entities.data_source_oauth_entities import DataSourceOAuthCallbackfrom .. import console_ns@@ -116,7 +116,7 @@ class OAuthDataSource(Resource):)def get(self, request_context: RequestContext, provider: str):try:- service = application_services().resolve_data_source_oauth(provider)+ service = application_services().data_sources.resolve_oauth(provider)authorization = service.start_authorization(request_context)except InvalidDataSourceOAuthProviderError:return _invalid_provider_response()@@ -141,7 +141,7 @@ class OAuthDataSourceCallback(Resource):def get(self, provider: str):query = query_params_from_request(OAuthDataSourceCallbackQuery)try:- service = application_services().resolve_data_source_oauth(provider)+ service = application_services().data_sources.resolve_oauth(provider)callback = service.complete_callback(code=query.code,error=query.error,@@ -172,7 +172,7 @@ class OAuthDataSourceBinding(Resource):if not query.code:return {"error": "Invalid code"}, HTTPStatus.BAD_REQUESTtry:- service = application_services().resolve_data_source_oauth(provider)+ service = application_services().data_sources.resolve_oauth(provider)service.bind(request_context, code=query.code)except InvalidDataSourceOAuthProviderError:return _invalid_provider_response()@@ -198,7 +198,7 @@ class OAuthDataSourceSync(Resource):@console_account_admission()def get(self, request_context: RequestContext, provider: str, binding_id: UUID):try:- service = application_services().resolve_data_source_oauth(provider)+ service = application_services().data_sources.resolve_oauth(provider)service.sync(request_context,binding_id=str(binding_id),diff --git a/api/controllers/console/datasets/data_source.py b/api/controllers/console/datasets/data_source.pyindex e4ad4b3b819..2272ed89894 100644--- a/api/controllers/console/datasets/data_source.py+++ b/api/controllers/console/datasets/data_source.py@@ -1,53 +1,59 @@-import json-from collections.abc import Generatorfrom datetime import datetime-from typing import Any, Literal, cast+from typing import Neverfrom uuid import UUID-from flask import requestfrom flask_restx import Resourcefrom pydantic import BaseModel, Field, field_serializer-from sqlalchemy import select-from sqlalchemy.orm import Session-from werkzeug.exceptions import NotFound+from configs import dify_config+from controllers.common.errors import InvalidArgumentError, NotFoundErrorfrom controllers.common.fields import SimpleResultResponse, TextContentResponsefrom controllers.common.rbac import DatasetId, RBACCheck, Workspacefrom controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models-from controllers.common.session import with_session-from core.datasource.entities.datasource_entities import DatasourceProviderType, OnlineDocumentPagesMessage-from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin+from controllers.console import console_ns+from controllers.console.app.error import ProviderNotInitializeError+from controllers.console.datasets.error import (+ DatasetAccessDeniedRequestError,+ IndexingEstimateError,+ InvalidActionError,+)+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import model_validatefrom core.entities.knowledge_entities import IndexingEstimate-from core.indexing_runner import IndexingRunner-from core.rag.extractor.entity.datasource_type import DatasourceType-from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo-from core.rag.extractor.notion_extractor import NotionExtractor-from extensions.ext_database import db+from core.rbac import RBACPermission+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModel-from libs.datetime_utils import naive_utc_now+from fields.dataset_fields import NotionEstimateWorkspacePayloadfrom libs.helper import dump_response, to_timestamp-from libs.login import login_required-from models import Account, DataSourceOauthBinding, Document-from services.dataset_service import DatasetService, DocumentService-from services.datasource_provider_service import DatasourceProviderService-from tasks.document_indexing_sync_task import document_indexing_sync_task--from .. import console_ns-from ..wraps import (- RBACPermission,- account_initialization_required,- is_admin_or_owner_required,- model_validate,- rbac_permission_required,- setup_required,- with_current_tenant_id,- with_current_user,+from machinery.context import RequestContext+from models.account import TenantAccountRole+from services.data_source.binding_application_service import (+ BindingAction,+ DataSourceBindingNotFoundError,+ DataSourceBindingStateError,+)+from services.data_source.entities.notion_import import NotionPageType+from services.data_source.notion_import_application_service import (+ DatasetIsNotNotionSourceError,+ NotionImportCredentialUnavailableError,+)+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.document_sync import SyncDocumentNotFoundError, SyncDocumentSourceError+from services.knowledge.entities.indexing_estimate import NewSourcesEstimateCommand, NotionEstimateSource+from services.knowledge.indexing.estimate import (+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,)+_ADMIN_OR_OWNER_ROLES = frozenset({TenantAccountRole.ADMIN, TenantAccountRole.OWNER})+_DATA_SOURCE_OAUTH_BASE_PATH = "/console/api/oauth/data-source"+class NotionEstimatePayload(BaseModel):- notion_info_list: list[dict[str, Any]]- process_rule: dict[str, Any]+ notion_info_list: list[NotionEstimateWorkspacePayload] = Field(min_length=1)+ process_rule: dict[str, object]doc_form: str = Field(default="text_model")doc_language: str = Field(default="English")@@ -72,7 +78,7 @@ class DataSourceIntegratePageResponse(ResponseModel):page_id: strpage_icon: DataSourceIntegrateIconResponse | Noneparent_id: str- type: str+ type: NotionPageTypeclass DataSourceIntegrateWorkspaceResponse(ResponseModel):@@ -106,7 +112,7 @@ class NotionIntegratePageResponse(ResponseModel):page_id: strpage_icon: DataSourceIntegrateIconResponse | Noneparent_id: str | None- type: str+ page_type: NotionPageType = Field(alias="type")is_bound: bool@@ -132,325 +138,204 @@ register_response_schema_models()-@console_ns.route(- "/data-source/integrates",- "/data-source/integrates/<uuid:binding_id>/<string:action>",-)-class DataSourceApi(Resource):- @setup_required- @login_required- @is_admin_or_owner_required- @rbac_permission_required(RBACCheck(RBACPermission.CREDENTIAL_MANAGE, Workspace()))- @account_initialization_required- @console_ns.response(200, "Success", console_ns.models[DataSourceIntegrateListResponse.__name__])- @with_current_tenant_id- def get(self, current_tenant_id: str) -> tuple[dict[str, Any], int]:- # get workspace data source integrates- data_source_integrates = db.session.scalars(- select(DataSourceOauthBinding).where(- DataSourceOauthBinding.tenant_id == current_tenant_id,- DataSourceOauthBinding.disabled == False,- )- ).all()-- base_url = request.url_root.rstrip("/")- data_source_oauth_base_path = "/console/api/oauth/data-source"- providers = ["notion"]-- integrate_data: list[dict[str, bool | datetime | dict[str, Any] | str | None]] = []- for provider in providers:- # existing_integrate = next((ai for ai in data_source_integrates if ai.provider == provider), None)- existing_integrates = filter(lambda item: item.provider == provider, data_source_integrates)- if existing_integrates:- for existing_integrate in list(existing_integrates):- integrate_data.append(- {- "id": existing_integrate.id,- "provider": provider,- "created_at": existing_integrate.created_at,- "is_bound": True,- "disabled": existing_integrate.disabled,- "source_info": existing_integrate.source_info,- "link": f"{base_url}{data_source_oauth_base_path}/{provider}",- }- )- else:- integrate_data.append(- {- "id": None,- "provider": provider,- "created_at": None,- "source_info": None,- "is_bound": False,- "disabled": None,- "link": f"{base_url}{data_source_oauth_base_path}/{provider}",- }- )- return dump_response(DataSourceIntegrateListResponse, {"data": integrate_data}), 200+def _raise_dataset_access_error(error: DatasetNotFoundError | DatasetAccessDeniedError) -> Never:+ if isinstance(error, DatasetNotFoundError):+ raise NotFoundError(description="Dataset not found.") from None+ raise DatasetAccessDeniedRequestError(description=str(error)) from None+- @setup_required- @login_required- @is_admin_or_owner_required- @rbac_permission_required(RBACCheck(RBACPermission.CREDENTIAL_MANAGE, Workspace()))- @account_initialization_required+@console_ns.route("/data-source/integrates")+class DataSourceIntegrationListApi(Resource):+ @console_ns.response(200, "Success", console_ns.models[DataSourceIntegrateListResponse.__name__])+ @console_account_admission(+ allowed_roles=_ADMIN_OR_OWNER_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.CREDENTIAL_MANAGE, Workspace()),),+ )+ def get(self, request_context: RequestContext) -> tuple[dict[str, object], int]:+ bindings = application_services().data_sources.bindings.list_integrations(request_context)+ base_url = dify_config.CONSOLE_API_URL.rstrip("/")+ data = [+ {+ "id": binding.id,+ "provider": binding.provider,+ "created_at": binding.created_at,+ "is_bound": True,+ "disabled": binding.disabled,+ "source_info": binding.source_info,+ "link": f"{base_url}{_DATA_SOURCE_OAUTH_BASE_PATH}/{binding.provider}",+ }+ for binding in bindings+ ]+ return dump_response(DataSourceIntegrateListResponse, {"data": data}), 200+++@console_ns.route("/data-source/integrates/<uuid:binding_id>/<string:action>")+class DataSourceIntegrationApi(Resource):@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @with_current_tenant_id- @with_session+ @console_account_admission(+ allowed_roles=_ADMIN_OR_OWNER_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.CREDENTIAL_MANAGE, Workspace()),),+ )def patch(- self, session: Session, current_tenant_id: str, binding_id: UUID, action: Literal["enable", "disable"]- ) -> tuple[dict[str, str], int]:- binding_id_str = str(binding_id)- data_source_binding = session.scalar(- select(DataSourceOauthBinding).where(- DataSourceOauthBinding.id == binding_id_str, DataSourceOauthBinding.tenant_id == current_tenant_id+ self,+ request_context: RequestContext,+ binding_id: UUID,+ action: str,+ ) -> tuple[dict[str, object], int]:+ try:+ validated_action = BindingAction(action)+ except ValueError as error:+ raise InvalidActionError() from error+ try:+ application_services().data_sources.bindings.change_state(+ request_context,+ str(binding_id),+ validated_action,)- )- if data_source_binding is None:- raise NotFound("Data source binding not found.")- # enable binding- match action:- case "enable":- if data_source_binding.disabled:- data_source_binding.disabled = False- data_source_binding.updated_at = naive_utc_now()- else:- raise ValueError("Data source is not disabled.")- # disable binding- case "disable":- if not data_source_binding.disabled:- data_source_binding.disabled = True- data_source_binding.updated_at = naive_utc_now()- else:- raise ValueError("Data source is disabled.")- return {"result": "success"}, 200+ except DataSourceBindingNotFoundError as error:+ raise NotFoundError(description="Data source binding not found.") from error+ except DataSourceBindingStateError as error:+ raise InvalidArgumentError(description=str(error)) from error+ return dump_response(SimpleResultResponse, {"result": "success"}), 200@console_ns.route("/notion/pre-import/pages")class DataSourceNotionListApi(Resource):- @setup_required- @login_required- @account_initialization_required@console_ns.doc(params=query_params_from_model(DataSourceNotionListQuery))@console_ns.response(200, "Success", console_ns.models[NotionIntegrateInfoListResponse.__name__])- @with_current_user- @with_current_tenant_id- @with_session(write=False)+ @console_account_admission()@model_validate(DataSourceNotionListQuery)def get(self,req_data: DataSourceNotionListQuery,- session: Session,- current_tenant_id: str,- current_user: Account,- ) -> tuple[dict[str, Any], int]:- datasource_provider_service = DatasourceProviderService()- credential = datasource_provider_service.get_datasource_credentials(- tenant_id=current_tenant_id,- credential_id=req_data.credential_id,- provider="notion_datasource",- plugin_id="langgenius/notion_datasource",- )- if not credential:- raise NotFound("Credential not found.")- exist_page_ids = []- # import notion in the exist dataset- if req_data.dataset_id:- dataset = DatasetService.get_dataset(req_data.dataset_id, session)- if not dataset:- raise NotFound("Dataset not found.")- if dataset.data_source_type != "notion_import":- raise ValueError("Dataset is not notion type.")-- documents = session.scalars(- select(Document).where(- Document.dataset_id == req_data.dataset_id,- Document.tenant_id == current_tenant_id,- Document.data_source_type == "notion_import",- Document.enabled.is_(True),- )- ).all()- if documents:- for document in documents:- data_source_info = json.loads(document.data_source_info)- exist_page_ids.append(data_source_info["notion_page_id"])- # get all authorized pages- from core.datasource.datasource_manager import DatasourceManager-- datasource_runtime = DatasourceManager.get_datasource_runtime(- provider_id="langgenius/notion_datasource/notion_datasource",- datasource_name="notion_datasource",- tenant_id=current_tenant_id,- datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,- )- datasource_provider_service = DatasourceProviderService()- if credential:- datasource_runtime.runtime.credentials = credential- datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)- online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (- datasource_runtime.get_online_document_pages(- user_id=current_user.id,- datasource_parameters={},- provider_type=datasource_runtime.datasource_provider_type(),- )- )+ request_context: RequestContext,+ ) -> tuple[dict[str, object], int]:try:- pages = []- workspace_info = {}- for message in online_document_result:- result = message.result- for info in result:- workspace_info = {- "workspace_id": info.workspace_id,- "workspace_name": info.workspace_name,- "workspace_icon": info.workspace_icon,- }- for page in info.pages:- page_info = {- "page_id": page.page_id,- "page_name": page.page_name,- "type": page.type,- "parent_id": page.parent_id,- "is_bound": page.page_id in exist_page_ids,- "page_icon": page.page_icon,- }- pages.append(page_info)- except Exception as e:- raise e- notion_info = [{**workspace_info, "pages": pages}] if workspace_info else []- return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200+ result = application_services().data_sources.notion_imports.list_pages(+ request_context,+ credential_id=req_data.credential_id,+ dataset_id=req_data.dataset_id,+ )+ except NotionImportCredentialUnavailableError as error:+ raise NotFoundError(description="Credential not found.") from error+ except (DatasetNotFoundError, DatasetAccessDeniedError) as error:+ _raise_dataset_access_error(error)+ except DatasetIsNotNotionSourceError as error:+ raise InvalidArgumentError(description="Dataset is not notion type.") from error+ return dump_response(NotionIntegrateInfoListResponse, {"notion_info": result.workspaces}), 200@console_ns.route("/notion/pages/<uuid:page_id>/<string:page_type>/preview")class DataSourceNotionPreviewApi(Resource):"""Preview one authorized Notion page through the datasource credential."""- @setup_required- @login_required- @account_initialization_required@console_ns.doc(params=query_params_from_model(DataSourceNotionPreviewQuery))@console_ns.response(200, "Success", console_ns.models[TextContentResponse.__name__])- @with_current_tenant_id+ @console_account_admission()@model_validate(DataSourceNotionPreviewQuery)def get(self,req_data: DataSourceNotionPreviewQuery,- current_tenant_id: str,+ request_context: RequestContext,page_id: UUID,page_type: str,- ) -> tuple[dict[str, str], int]:-- datasource_provider_service = DatasourceProviderService()- credential = datasource_provider_service.get_datasource_credentials(- tenant_id=current_tenant_id,- credential_id=req_data.credential_id,- provider="notion_datasource",- plugin_id="langgenius/notion_datasource",- )-- page_id_str = str(page_id)-- extractor = NotionExtractor(- notion_workspace_id="",- notion_obj_id=page_id_str,- notion_page_type=page_type,- notion_access_token=credential.get("integration_secret"),- tenant_id=current_tenant_id,- )-- text_docs = extractor.extract()- return {"content": "\n".join([doc.page_content for doc in text_docs])}, 200+ ) -> tuple[dict[str, object], int]:+ try:+ validated_page_type = NotionPageType(page_type)+ except ValueError as error:+ raise InvalidArgumentError(description="Invalid Notion page type.") from error+ try:+ content = application_services().data_sources.notion_imports.preview_page(+ request_context,+ credential_id=req_data.credential_id,+ page_id=str(page_id),+ page_type=validated_page_type,+ )+ except NotionImportCredentialUnavailableError as error:+ raise NotFoundError(description="Credential not found.") from error+ return dump_response(TextContentResponse, {"content": content}), 200@console_ns.route("/datasets/notion-indexing-estimate")class DataSourceNotionIndexingEstimateApi(Resource):"""Estimate indexing work for selected Notion pages."""- @setup_required- @login_required- @account_initialization_required@console_ns.expect(console_ns.models[NotionEstimatePayload.__name__])@console_ns.response(200, "Success", console_ns.models[IndexingEstimate.__name__])- @with_current_tenant_id- @with_session+ @console_account_admission()@model_validate(NotionEstimatePayload)def post(self,req_data: NotionEstimatePayload,- session: Session,- current_tenant_id: str,- ) -> tuple[dict[str, Any], int]:- args = req_data.model_dump()- # validate args- DocumentService.estimate_args_validate(args)- notion_info_list = req_data.notion_info_list- extract_settings = []- for notion_info in notion_info_list:- workspace_id = notion_info["workspace_id"]- credential_id = notion_info.get("credential_id")- for page in notion_info["pages"]:- extract_setting = ExtractSetting(- datasource_type=DatasourceType.NOTION,- notion_info=NotionInfo.model_validate(- {- "credential_id": credential_id,- "notion_workspace_id": workspace_id,- "notion_obj_id": page["page_id"],- "notion_page_type": page["type"],- "tenant_id": current_tenant_id,- }- ),- document_model=args["doc_form"],+ request_context: RequestContext,+ ) -> tuple[dict[str, object], int]:+ command = NewSourcesEstimateCommand(+ sources=tuple(+ NotionEstimateSource(+ workspace_id=workspace.workspace_id,+ page_id=page.page_id,+ page_type=page.page_type,+ credential_id=workspace.credential_id,)- extract_settings.append(extract_setting)- indexing_runner = IndexingRunner()- response = indexing_runner.indexing_estimate(- tenant_id=current_tenant_id,- extract_settings=extract_settings,- tmp_processing_rule=args["process_rule"],- doc_form=args["doc_form"],- doc_language=args["doc_language"],- session=session,+ for workspace in req_data.notion_info_list+ for page in workspace.pages+ ),+ process_rule=req_data.process_rule,+ doc_form=req_data.doc_form,+ doc_language=req_data.doc_language,)+ try:+ response = application_services().knowledge.indexing_estimates.estimate_new_sources(+ request_context,+ command,+ )+ except IndexingEstimateCredentialUnavailableError as error:+ raise NotFoundError(description="Credential not found.") from error+ except EstimateSourceNotFoundError as error:+ raise NotFoundError(description=str(error)) from error+ except IndexingEstimateProviderUnavailableError as error:+ raise ProviderNotInitializeError(str(error)) from error+ except IndexingEstimateExecutionError as error:+ raise IndexingEstimateError(str(error)) from errorreturn dump_response(IndexingEstimate, response), 200@console_ns.route("/datasets/<uuid:dataset_id>/notion/sync")class DataSourceNotionDatasetSyncApi(Resource):- @setup_required- @login_required- @account_initialization_required@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session(write=False)- def get(self, session: Session, dataset_id: UUID) -> tuple[dict[str, str], int]:- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-- documents = DocumentService.get_document_by_dataset_id(dataset_id_str, session)- for document in documents:- document_indexing_sync_task.delay(dataset_id_str, document.id)- return {"result": "success"}, 200+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ def get(self, request_context: RequestContext, dataset_id: UUID) -> tuple[dict[str, object], int]:+ try:+ application_services().knowledge.document_sync.sync_dataset(request_context, str(dataset_id))+ except (DatasetNotFoundError, DatasetAccessDeniedError) as error:+ _raise_dataset_access_error(error)+ return dump_response(SimpleResultResponse, {"result": "success"}), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/notion/sync")class DataSourceNotionDocumentSyncApi(Resource):- @setup_required- @login_required- @account_initialization_required@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session(write=False)- def get(self, session: Session, dataset_id: UUID, document_id: UUID) -> tuple[dict[str, str], int]:- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if document is None:- raise NotFound("Document not found.")- document_indexing_sync_task.delay(dataset_id_str, document_id_str)- return {"result": "success"}, 200+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ def get(+ self,+ request_context: RequestContext,+ dataset_id: UUID,+ document_id: UUID,+ ) -> tuple[dict[str, object], int]:+ try:+ application_services().knowledge.document_sync.sync_document(+ request_context,+ str(dataset_id),+ str(document_id),+ )+ except (DatasetNotFoundError, DatasetAccessDeniedError) as error:+ _raise_dataset_access_error(error)+ except SyncDocumentNotFoundError as error:+ raise NotFoundError(description="Document not found.") from error+ except SyncDocumentSourceError as error:+ raise InvalidArgumentError(description=str(error)) from error+ return dump_response(SimpleResultResponse, {"result": "success"}), 200diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.pyindex 24a3dd81d50..fa4f19fd10e 100644--- a/api/controllers/console/datasets/datasets.py+++ b/api/controllers/console/datasets/datasets.py@@ -1,92 +1,89 @@-from dataclasses import dataclassfrom datetime import datetimefrom http import HTTPStatus-from typing import Any+from typing import Any, Neverfrom uuid import UUIDfrom flask import requestfrom flask_restx import Resource-from pydantic import BaseModel, Field, field_validator, model_validator-from sqlalchemy import select-from sqlalchemy.orm import Session+from pydantic import BaseModel, Field, TypeAdapter, field_validator, model_validatorfrom werkzeug.exceptions import Forbidden, NotFound-import services-from configs import dify_config+from controllers.common.errors import InvalidArgumentError, NotFoundErrorfrom controllers.common.fields import ApiBaseUrlResponse, SimpleResultResponse, UsageCheckResponsefrom controllers.common.rbac import DatasetId, RBACCheck, Workspace, enforce_rbac_checksfrom controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models-from controllers.common.session import with_sessionfrom controllers.console import console_nsfrom controllers.console.apikey import API_KEY_DELETE_ROLES, api_key_errorsfrom controllers.console.app.error import ProviderNotInitializeError-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError+from controllers.console.datasets.error import (+ DatasetAccessDeniedRequestError,+ DatasetInUseError,+ DatasetNameDuplicateError,+ IndexingEstimateError,+)from controllers.console.flask_admission import console_account_admissionfrom controllers.console.wraps import (RBACPermission,- account_initialization_required,cloud_edition_billing_rate_limit_check,- enterprise_license_required,model_validate,- rbac_permission_required,- setup_required,- with_current_tenant_id,- with_current_user,)from core.entities.knowledge_entities import IndexingEstimate-from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError-from core.indexing_runner import IndexingRunner-from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager-from core.rag.datasource.vdb.vector_type import VectorType-from core.rag.extractor.entity.datasource_type import DatasourceType-from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo-from core.rag.index_processor.constant.index_type import IndexTechniqueType-from core.rag.retrieval.retrieval_methods import RetrievalMethod+from core.rag.index_processor.constant.index_type import IndexStructureTypefrom extensions.ext_application_services import application_servicesfrom fields.api_key_fields import ApiKeyItem, ApiKeyList, build_masked_api_key_listfrom fields.base import ResponseModelfrom fields.dataset_fields import (DatasetDetailResponse,- build_dataset_detail_prefetch,- dataset_detail_response_source,+ NotionEstimateWorkspacePayload,)-from graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import build_icon_url, dump_response, to_timestamp-from libs.login import login_required-from libs.pagination import clamp_pagination-from libs.url_utils import normalize_api_base_urlfrom machinery.context import RequestContext-from models import Account, Dataset, Document, UploadFile-from models.dataset import DatasetPermission, DatasetPermissionEnum, DatasetQuery-from models.provider_ids import ModelProviderID-from services.dataset_ref_service import DatasetRefService-from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService-from services.enterprise import rbac_service as enterprise_rbac_service--register_response_schema_models(console_ns, ApiBaseUrlResponse, SimpleResultResponse, UsageCheckResponse)+from models.account import TenantAccountRole+from models.enums import PermissionEnum as DatasetPermissionEnum+from services.errors.dataset import DatasetInUseError as DatasetInUseFailure+from services.errors.dataset import DatasetNameDuplicateError as DatasetNameDuplicateFailure+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.datasets.application import DatasetListFilter+from services.knowledge.entities.indexing_estimate import (+ NewEstimateSource,+ NewSourcesEstimateCommand,+ NotionEstimateSource,+ UploadFileEstimateSource,+ WebsiteEstimateSource,+)+from services.knowledge.entities.knowledge_entities import FileInfo, WebsiteInfo+from services.knowledge.indexing.estimate import (+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+ UnsupportedEstimateSourceError,+)-DATASET_LIST_PERMISSION_KEYS = frozenset({"dataset.preview", "dataset.acl.preview", "dataset.full_access"})+_DATASET_EDIT_ROLES = frozenset(+ {TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR, TenantAccountRole.DATASET_OPERATOR}+)-def _has_dataset_list_permission(permission_keys: list[str]) -> bool:- return any(permission_key in DATASET_LIST_PERMISSION_KEYS for permission_key in permission_keys)+def _raise_dataset_error(error: Exception) -> Never:+ if isinstance(error, DatasetNotFoundError):+ raise NotFound("Dataset not found.") from error+ if isinstance(error, DatasetAccessDeniedError):+ raise Forbidden(str(error)) from error+ if isinstance(error, DatasetNameDuplicateFailure):+ raise DatasetNameDuplicateError() from error+ if isinstance(error, DatasetInUseFailure):+ raise DatasetInUseError() from error+ raise error-def _get_accessible_dataset(dataset_id: UUID, tenant_id: str, current_user: Account, session: Session) -> Dataset:- dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- return dataset+register_response_schema_models(console_ns, ApiBaseUrlResponse, SimpleResultResponse, UsageCheckResponse)def _validate_indexing_technique(value: str | None) -> str | None:if value is None:return value- if value not in Dataset.INDEXING_TECHNIQUE_LIST:+ if value not in {"high_quality", "economy"}:raise ValueError("Invalid indexing technique.")return value@@ -94,7 +91,7 @@ def _validate_indexing_technique(value: str | None) -> str | None:def _validate_doc_form(value: str | None) -> str | None:if value is None:return value- if value not in Dataset.DOC_FORM_LIST:+ if value not in {member.value for member in IndexStructureType}:raise ValueError("Invalid doc_form.")return value@@ -116,7 +113,7 @@ class DatasetCreatePayload(BaseModel):@field_validator("provider")@classmethoddef validate_provider(cls, value: str) -> str:- if value not in Dataset.PROVIDER_LIST:+ if value not in {"vendor", "external"}:raise ValueError("Invalid provider.")return value@@ -168,6 +165,44 @@ class IndexingEstimatePayload(BaseModel):return result+_NOTION_SELECTIONS = TypeAdapter(list[NotionEstimateWorkspacePayload])+++def _new_estimate_sources(info_list: dict[str, Any]) -> tuple[NewEstimateSource, ...]:+ match info_list.get("data_source_type"):+ case "upload_file":+ files = FileInfo.model_validate(info_list.get("file_info_list"))+ return tuple(UploadFileEstimateSource(file_id=file_id) for file_id in dict.fromkeys(files.file_ids))+ case "notion_import":+ workspaces = _NOTION_SELECTIONS.validate_python(info_list.get("notion_info_list"))+ return tuple(+ NotionEstimateSource(+ workspace_id=workspace.workspace_id,+ credential_id=workspace.credential_id,+ page_id=page.page_id,+ page_type=page.page_type,+ )+ for workspace in workspaces+ for page in workspace.pages+ )+ case "website_crawl":+ values = info_list.get("website_info_list")+ if not isinstance(values, dict):+ raise ValueError("Website info list is required")+ website = WebsiteInfo.model_validate({"only_main_content": False, **values})+ return tuple(+ WebsiteEstimateSource(+ provider=website.provider,+ job_id=website.job_id,+ url=url,+ only_main_content=website.only_main_content,+ )+ for url in website.urls+ )+ case _:+ raise ValueError("Data source type not support")++class DatasetApiKeyCreatePayload(BaseModel):# Knowledge bases to scope the key to. Absent/empty => the key can access every# dataset in the tenant (default). Declared so the generated client can send it.@@ -229,21 +264,6 @@ class DatasetQueryDetailResponse(ResponseModel):return to_timestamp(value)-@dataclass(frozen=True)-class _DatasetQueryResponseSource:- """Expose query content through the request's database session."""-- query: DatasetQuery- session: Session-- @property- def queries(self) -> list[dict[str, Any]]:- return self.query.get_queries(session=self.session)-- def __getattr__(self, name: str) -> Any:- return getattr(self.query, name) # guard-ignore: no-new-getattr -- delegates model fields--class DatasetQueryListResponse(ResponseModel):data: list[DatasetQueryDetailResponse]has_more: bool@@ -353,84 +373,6 @@ register_response_schema_models()-def _get_retrieval_methods_by_vector_type(vector_type: str | None, is_mock: bool = False) -> dict[str, list[str]]:- """- Get supported retrieval methods based on vector database type.-- Args:- vector_type: Vector database type, can be None- is_mock: Whether this is a Mock API, affects MILVUS handling-- Returns:- Dictionary containing supported retrieval methods-- Raises:- ValueError: If vector_type is None or unsupported- """- if vector_type is None:- raise ValueError("Vector store type is not configured.")-- # Define vector database types that only support semantic search- semantic_only_types = {- VectorType.RELYT,- VectorType.CHROMA,- VectorType.PGVECTO_RS,- VectorType.VIKINGDB,- VectorType.UPSTASH,- }-- # Define vector database types that support all retrieval methods- full_search_types = {- VectorType.QDRANT,- VectorType.WEAVIATE,- VectorType.OPENSEARCH,- VectorType.ANALYTICDB,- VectorType.MYSCALE,- VectorType.ORACLE,- VectorType.ELASTICSEARCH,- VectorType.ELASTICSEARCH_JA,- VectorType.PGVECTOR,- VectorType.VASTBASE,- VectorType.TIDB_ON_QDRANT,- VectorType.LINDORM,- VectorType.COUCHBASE,- VectorType.OPENGAUSS,- VectorType.OCEANBASE,- VectorType.SEEKDB,- VectorType.TABLESTORE,- VectorType.HUAWEI_CLOUD,- VectorType.TENCENT,- VectorType.MATRIXONE,- VectorType.CLICKZETTA,- VectorType.BAIDU,- VectorType.ALIBABACLOUD_MYSQL,- VectorType.IRIS,- VectorType.HOLOGRES,- }-- semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}- full_methods = {- "retrieval_method": [- RetrievalMethod.SEMANTIC_SEARCH.value,- RetrievalMethod.FULL_TEXT_SEARCH.value,- RetrievalMethod.HYBRID_SEARCH.value,- ]- }-- if vector_type == VectorType.MILVUS:- return semantic_methods if is_mock else full_methods-- if vector_type == VectorType.TIDB_VECTOR:- return full_methods if dify_config.TIDB_VECTOR_ENABLE_FULLTEXT_SEARCH else semantic_methods-- if vector_type in semantic_only_types:- return semantic_methods- elif vector_type in full_search_types:- return full_methods- else:- raise ValueError(f"Unsupported vector db type {vector_type}.")--@console_ns.route("/datasets")class DatasetListApi(Resource):@console_ns.doc("get_datasets")@@ -439,208 +381,39 @@ class DatasetListApi(Resource):@console_ns.response(HTTPStatus.OK, "Datasets retrieved successfully", console_ns.models[DatasetListResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required- @with_current_user- @with_current_tenant_id- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account):- # Convert query parameters to dict, handling list parameters correctly+ @console_account_admission(require_valid_enterprise_license=True)+ def get(self, request_context: RequestContext):query_params: dict[str, str | list[str]] = dict(request.args.to_dict())- # Handle ids and tag_ids as lists (Flask request.args.getlist returns list even for single value)- if "ids" in request.args:- query_params["ids"] = request.args.getlist("ids")- if "tag_ids" in request.args:- query_params["tag_ids"] = request.args.getlist("tag_ids")+ for key in ("ids", "tag_ids"):+ if key in request.args:+ query_params[key] = request.args.getlist(key)query = ConsoleDatasetListQuery.model_validate(query_params)-- permissions = enterprise_rbac_service.RBACService.MyPermissions.get(- str(current_tenant_id),- current_user.id,- session=session,+ result = application_services().knowledge.datasets.list_datasets(+ request_context, DatasetListFilter(**query.model_dump()))-- accessible_dataset_ids: list[str] | None = None- include_own_datasets = False- if dify_config.RBAC_ENABLED:- whitelist_scope = enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources(- str(current_tenant_id),- current_user.id,- )- has_default_readonly = _has_dataset_list_permission(- permissions.dataset.default_permission_keys- ) or _has_dataset_list_permission(permissions.workspace.permission_keys)- permission_dataset_ids: set[str] | None = None- if not has_default_readonly:- permission_dataset_ids = {- override.resource_id- for override in permissions.dataset.overrides- if _has_dataset_list_permission(override.permission_keys)- }- if getattr(whitelist_scope, "unrestricted", False):- filtered_dataset_ids = permission_dataset_ids- include_own_datasets = "dataset.create_and_management" in permissions.workspace.permission_keys- else:- # A restricted dataset whitelist is the highest-priority visibility gate:- # default readonly, per-dataset permission overrides, and own-dataset- # management must not expose datasets outside this set.- filtered_dataset_ids = set(whitelist_scope.resource_ids)- if filtered_dataset_ids is not None:- accessible_dataset_ids = sorted(filtered_dataset_ids)-- effective_page, effective_limit = clamp_pagination(query.page, query.limit, 100)- if query.ids:- datasets, total = DatasetService.get_datasets_by_ids(- query.ids,- current_tenant_id,- user=current_user,- accessible_dataset_ids=accessible_dataset_ids,- include_own_datasets=include_own_datasets,- session=session,- )- # This branch resolves the ids it was handed in a single page- # (`per_page=len(ids)`), so there is never a next one to ask for.- has_more = False- else:- datasets, total = DatasetService.get_datasets(- effective_page,- effective_limit,- session,- current_tenant_id,- current_user,- query.keyword,- query.tag_ids,- query.include_all,- tags=application_services().tags,- accessible_dataset_ids=accessible_dataset_ids,- include_own_datasets=include_own_datasets,- )- has_more = effective_page * effective_limit < total-- permission_keys_map = {}- if datasets:- dataset_ids = [str(dataset.id) for dataset in datasets]- permission_keys_map = permissions.dataset.permission_keys_by_resource_ids(dataset_ids)-- # check embedding setting- provider_manager = create_plugin_provider_manager(tenant_id=current_tenant_id)- configurations = provider_manager.get_configurations(tenant_id=current_tenant_id)-- embedding_models = configurations.get_models(model_type=ModelType.TEXT_EMBEDDING, only_active=True)-- model_names = []- for embedding_model in embedding_models:- model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")-- prefetch = build_dataset_detail_prefetch(datasets, session=session)- data = [- dump_response(- DatasetDetailResponse, dataset_detail_response_source(dataset, session=session, prefetch=prefetch)- )- for dataset in datasets- ]- dataset_ids = [item["id"] for item in data if item.get("permission") == "partial_members"]- partial_members_map: dict[str, list[str]] = {}- if dataset_ids:- partial_member_rows = session.execute(- select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(- DatasetPermission.dataset_id.in_(dataset_ids)- )- ).all()-- for dataset_id, account_id in partial_member_rows:- partial_members_map.setdefault(dataset_id, []).append(account_id)-- for item in data:- # convert embedding_model_provider to plugin standard format- if item["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY and item["embedding_model_provider"]:- item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))- item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"- if item_model in model_names:- item["embedding_available"] = True- else:- item["embedding_available"] = False- else:- item["embedding_available"] = True-- if item.get("permission") == "partial_members":- item.update({"partial_member_list": partial_members_map.get(item["id"], [])})- else:- item.update({"partial_member_list": []})- item["permission_keys"] = permission_keys_map.get(str(item["id"]), [])-- response = {- "data": data,- "has_more": has_more,- "limit": effective_limit,- "total": total,- "page": effective_page,- }- return dump_response(DatasetListResponse, response), HTTPStatus.OK+ return dump_response(DatasetListResponse, result), HTTPStatus.OK@console_ns.doc("create_dataset")@console_ns.doc(description="Create a new dataset")@console_ns.expect(console_ns.models[DatasetCreatePayload.__name__])- @console_ns.response(201, "Dataset created successfully", console_ns.models[DatasetDetailResponse.__name__])+ @console_ns.response(+ 201, "Dataset created successfully", console_ns.models[DatasetDetailWithPartialMembersResponse.__name__]+ )@console_ns.response(400, "Invalid request parameters")- @setup_required- @login_required- @account_initialization_required- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace()))+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace()),),+ )@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @with_current_tenant_id- @with_session@model_validate(DatasetCreatePayload)- def post(self, req_data: DatasetCreatePayload, session: Session, current_tenant_id: str, current_user: Account):- # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator- if not current_user.is_dataset_editor:- raise Forbidden()-- if dify_config.RBAC_ENABLED:- permission = DatasetPermissionEnum.ALL_TEAM- else:- permission = req_data.permission or DatasetPermissionEnum.ONLY_ME-+ def post(self, req_data: DatasetCreatePayload, request_context: RequestContext):try:- dataset = DatasetService.create_empty_dataset(- session=session,- tenant_id=current_tenant_id,- name=req_data.name,- description=req_data.description,- indexing_technique=req_data.indexing_technique,- account=current_user,- permission=permission,- provider=req_data.provider,- external_knowledge_api_id=req_data.external_knowledge_api_id,- external_knowledge_id=req_data.external_knowledge_id,+ result = application_services().knowledge.datasets.create_dataset(+ request_context, values=req_data.model_dump())- except services.errors.dataset.DatasetNameDuplicateError:- raise DatasetNameDuplicateError()-- permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(- current_tenant_id,- current_user.id,- [dataset.id],- session=session,- )-- item = DatasetDetailWithPartialMembersResponse.model_validate(- dataset_detail_response_source(dataset, session=session), from_attributes=True- ).model_dump(mode="json")- item["permission_keys"] = permission_keys_map.get(dataset.id, [])-- if dify_config.RBAC_ENABLED:- enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist(- current_tenant_id,- current_user.id,- dataset.id,- enterprise_rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),- )-- return item, 201+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(DatasetDetailWithPartialMembersResponse, result), 201@console_ns.route("/datasets/<uuid:dataset_id>")@@ -655,59 +428,13 @@ class DatasetApi(Resource):)@console_ns.response(404, "Dataset not found")@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-+ @console_account_admission()+ def get(self, request_context: RequestContext, dataset_id: UUID):try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- permissions = enterprise_rbac_service.RBACService.MyPermissions.get(- current_tenant_id,- current_user.id,- dataset_id=dataset_id_str,- session=session,- )- permission_keys_map = permissions.dataset.permission_keys_by_resource_ids([dataset_id_str])- data = dump_response(DatasetDetailResponse, dataset_detail_response_source(dataset, session=session))- data["permission_keys"] = permission_keys_map.get(dataset_id_str, [])- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- if dataset.embedding_model_provider:- provider_id = ModelProviderID(dataset.embedding_model_provider)- data["embedding_model_provider"] = str(provider_id)- if data.get("permission") == "partial_members":- part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, session)- data.update({"partial_member_list": part_users_list})-- # check embedding setting- provider_manager = create_plugin_provider_manager(tenant_id=current_tenant_id)- configurations = provider_manager.get_configurations(tenant_id=current_tenant_id)-- embedding_models = configurations.get_models(model_type=ModelType.TEXT_EMBEDDING, only_active=True)-- model_names = []- for embedding_model in embedding_models:- model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")-- if data["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY:- item_model = f"{data['embedding_model']}:{data['embedding_model_provider']}"- if item_model in model_names:- data["embedding_available"] = True- else:- data["embedding_available"] = False- else:- data["embedding_available"] = True-- return dump_response(DatasetDetailWithPartialMembersResponse, data), 200+ result = application_services().knowledge.datasets.get_dataset(request_context, dataset_id=str(dataset_id))+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(DatasetDetailWithPartialMembersResponse, result), 200@console_ns.doc("update_dataset")@console_ns.doc(description="Update dataset details")@@ -719,95 +446,31 @@ class DatasetApi(Resource):)@console_ns.response(404, "Dataset not found")@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(DatasetUpdatePayload)- def patch(- self,- req_data: DatasetUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- ):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-- # check embedding model setting- if (- req_data.indexing_technique == IndexTechniqueType.HIGH_QUALITY- and req_data.embedding_model_provider is not None- and req_data.embedding_model is not None- ):- is_multimodal = DatasetService.check_is_multimodal_model(- dataset.tenant_id, req_data.embedding_model_provider, req_data.embedding_model- )- req_data.is_multimodal = is_multimodal- payload_data = req_data.model_dump(exclude_unset=True)- # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator- if not dify_config.RBAC_ENABLED:- DatasetPermissionService.check_permission(- current_user, dataset, req_data.permission, req_data.partial_member_list, session=session- )-- dataset = DatasetService.update_dataset(dataset_id_str, payload_data, current_user, session=session)-- if dataset is None:- raise NotFound("Dataset not found.")-- permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(- current_tenant_id,- current_user.id,- [dataset_id_str],- session=session,- )- result_data = dump_response(DatasetDetailResponse, dataset_detail_response_source(dataset, session=session))- result_data["permission_keys"] = permission_keys_map.get(dataset_id_str, [])- tenant_id = current_tenant_id-- if req_data.partial_member_list is not None and req_data.permission == DatasetPermissionEnum.PARTIAL_TEAM:- DatasetPermissionService.update_partial_member_list(- tenant_id, dataset_id_str, req_data.partial_member_list, session+ def patch(self, req_data: DatasetUpdatePayload, request_context: RequestContext, dataset_id: UUID):+ try:+ result = application_services().knowledge.datasets.update_dataset(+ request_context, dataset_id=str(dataset_id), values=req_data.model_dump(exclude_unset=True))- # clear partial member list when permission is only_me or all_team_members- elif req_data.permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:- DatasetPermissionService.clear_partial_member_list(dataset_id_str, session)+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(DatasetDetailWithPartialMembersResponse, result), 200- partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, session)- result_data.update({"partial_member_list": partial_member_list})-- return dump_response(DatasetDetailWithPartialMembersResponse, result_data), 200-- @setup_required- @login_required- @account_initialization_required- @cloud_edition_billing_rate_limit_check("knowledge")@console_ns.response(204, "Dataset deleted successfully")- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_DELETE, DatasetId()))- @with_session- def delete(self, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)-- if not (current_user.has_edit_permission or current_user.is_dataset_operator):- raise Forbidden()-+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_DELETE, DatasetId()),)+ )+ @cloud_edition_billing_rate_limit_check("knowledge")+ def delete(self, request_context: RequestContext, dataset_id: UUID):try:- if DatasetService.delete_dataset(dataset_id_str, current_user, session):- DatasetPermissionService.clear_partial_member_list(dataset_id_str, session)- return "", 204- else:- raise NotFound("Dataset not found.")- except services.errors.dataset.DatasetInUseError:- raise DatasetInUseError()+ application_services().knowledge.datasets.delete_dataset(request_context, dataset_id=str(dataset_id))+ except Exception as error:+ _raise_dataset_error(error)+ return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/use-check")@@ -820,17 +483,13 @@ class DatasetUseCheckApi(Resource):"Dataset use status retrieved successfully",console_ns.models[UsageCheckResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset = _get_accessible_dataset(dataset_id, current_tenant_id, current_user, session)- dataset_is_using = DatasetService.dataset_use_check(DatasetRefService.create_dataset_ref(dataset), session)- return UsageCheckResponse(is_using=dataset_is_using).model_dump(mode="json"), 200+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):+ try:+ result = application_services().knowledge.datasets.is_in_use(request_context, dataset_id=str(dataset_id))+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(UsageCheckResponse, {"is_using": result}), 200@console_ns.route("/datasets/<uuid:dataset_id>/queries")@@ -843,39 +502,18 @@ class DatasetQueryApi(Resource):"Query history retrieved successfully",console_ns.models[DatasetQueryListResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- page = request.args.get("page", default=1, type=int)- limit = request.args.get("limit", default=20, type=int)-- effective_page, effective_limit = clamp_pagination(page, limit, 100)- dataset_queries, total = DatasetService.get_dataset_queries(- dataset_id=dataset.id, page=effective_page, per_page=effective_limit, session=session- )-- response = {- "data": [_DatasetQueryResponseSource(query=query, session=session) for query in dataset_queries],- "has_more": effective_page * effective_limit < total,- "limit": effective_limit,- "total": total,- "page": effective_page,- }- return dump_response(DatasetQueryListResponse, response), 200+ result = application_services().knowledge.datasets.queries(+ request_context,+ dataset_id=str(dataset_id),+ page=request.args.get("page", default=1, type=int),+ limit=request.args.get("limit", default=20, type=int),+ )+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(DatasetQueryListResponse, result), 200@console_ns.route("/datasets/indexing-estimate")@@ -887,115 +525,49 @@ class DatasetIndexingEstimateApi(Resource):"Indexing estimate calculated successfully",console_ns.models[IndexingEstimateResponse.__name__],)- @setup_required- @login_required- @account_initialization_required@console_ns.expect(console_ns.models[IndexingEstimatePayload.__name__])- @with_current_user- @with_current_tenant_id- @with_session+ @console_account_admission()@model_validate(IndexingEstimatePayload)- def post(- self,- req_data: IndexingEstimatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,- ):- args = req_data.model_dump()- dataset_id = req_data.dataset_id- if dataset_id:+ def post(self, req_data: IndexingEstimatePayload, request_context: RequestContext):+ if req_data.dataset_id:checks = [RBACCheck(RBACPermission.DATASET_USE, DatasetId())]- path_args = {"dataset_id": dataset_id}+ path_args = {"dataset_id": req_data.dataset_id}else:checks = [RBACCheck(RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace())]path_args = Noneenforce_rbac_checks(- tenant_id=current_tenant_id,- account_id=current_user.id,+ tenant_id=request_context.active_workspace_id,+ account_id=request_context.account_id,checks=checks,path_args=path_args,)- # validate args- DocumentService.estimate_args_validate(args)- extract_settings = []- match args["info_list"]["data_source_type"]:- case "upload_file":- file_ids = args["info_list"]["file_info_list"]["file_ids"]- file_details = session.scalars(- select(UploadFile).where(UploadFile.tenant_id == current_tenant_id, UploadFile.id.in_(file_ids))- ).all()- if not file_details:- raise NotFound("File not found.")-- if file_details:- for file_detail in file_details:- extract_setting = ExtractSetting(- datasource_type=DatasourceType.FILE,- upload_file=file_detail,- document_model=args["doc_form"],- )- extract_settings.append(extract_setting)- case "notion_import":- notion_info_list = args["info_list"]["notion_info_list"]- for notion_info in notion_info_list:- workspace_id = notion_info["workspace_id"]- credential_id = notion_info.get("credential_id")- for page in notion_info["pages"]:- extract_setting = ExtractSetting(- datasource_type=DatasourceType.NOTION,- notion_info=NotionInfo.model_validate(- {- "credential_id": credential_id,- "notion_workspace_id": workspace_id,- "notion_obj_id": page["page_id"],- "notion_page_type": page["type"],- "tenant_id": current_tenant_id,- }- ),- document_model=args["doc_form"],- )- extract_settings.append(extract_setting)- case "website_crawl":- website_info_list = args["info_list"]["website_info_list"]- for url in website_info_list["urls"]:- extract_setting = ExtractSetting(- datasource_type=DatasourceType.WEBSITE,- website_info=WebsiteInfo.model_validate(- {- "provider": website_info_list["provider"],- "job_id": website_info_list["job_id"],- "url": url,- "tenant_id": current_tenant_id,- "mode": "crawl",- "only_main_content": website_info_list["only_main_content"],- }- ),- document_model=args["doc_form"],- )- extract_settings.append(extract_setting)- case _:- raise ValueError("Data source type not support")- indexing_runner = IndexingRunner()+ command = NewSourcesEstimateCommand(+ sources=_new_estimate_sources(req_data.info_list),+ process_rule=req_data.process_rule,+ doc_form=req_data.doc_form,+ doc_language=req_data.doc_language,+ dataset_id=req_data.dataset_id,+ indexing_technique=req_data.indexing_technique,+ )try:- response = indexing_runner.indexing_estimate(- tenant_id=current_tenant_id,- extract_settings=extract_settings,- tmp_processing_rule=args["process_rule"],- doc_form=args["doc_form"],- doc_language=args["doc_language"],- dataset_id=args["dataset_id"],- indexing_technique=args["indexing_technique"],- session=session,+ response = application_services().knowledge.indexing_estimates.estimate_new_sources(+ request_context,+ command,)- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- except Exception as e:- raise IndexingEstimateError(str(e))+ except (+ IndexingEstimateCredentialUnavailableError,+ EstimateSourceNotFoundError,+ DatasetNotFoundError,+ ) as error:+ raise NotFoundError(description=str(error)) from error+ except DatasetAccessDeniedError as error:+ raise DatasetAccessDeniedRequestError(description=str(error)) from error+ except UnsupportedEstimateSourceError as error:+ raise InvalidArgumentError(description=str(error)) from error+ except IndexingEstimateProviderUnavailableError as error:+ raise ProviderNotInitializeError(str(error)) from error+ except IndexingEstimateExecutionError as error:+ raise IndexingEstimateError(str(error)) from errorreturn (IndexingEstimateResponse(@@ -1020,30 +592,13 @@ class DatasetRelatedAppListApi(Resource):"Related apps retrieved successfully",console_ns.models[RelatedAppListResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- app_dataset_joins = DatasetService.get_related_apps(dataset.id, session)-- related_apps = application_services().apps.queries.related_apps(- dataset.tenant_id, [join.app_id for join in app_dataset_joins]- )-- return dump_response(RelatedAppListResponse, {"data": related_apps, "total": len(related_apps)}), HTTPStatus.OK+ result = application_services().knowledge.datasets.related_apps(request_context, dataset_id=str(dataset_id))+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(RelatedAppListResponse, result), HTTPStatus.OK@console_ns.route("/datasets/<uuid:dataset_id>/indexing-status")@@ -1056,43 +611,15 @@ class DatasetIndexingStatusApi(Resource):"Indexing status retrieved successfully",console_ns.models[DocumentStatusListResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset = _get_accessible_dataset(dataset_id, current_tenant_id, current_user, session)-- documents = session.scalars(- select(Document).where(Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id)- ).all()- segment_counts = DocumentService.get_document_segment_counts(- documents,- session=session,- )- documents_status = []- for document in documents:- completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))- # Create a dictionary with document attributes and additional fields- document_dict = {- "id": document.id,- "indexing_status": document.indexing_status,- "processing_started_at": document.processing_started_at,- "parsing_completed_at": document.parsing_completed_at,- "cleaning_completed_at": document.cleaning_completed_at,- "splitting_completed_at": document.splitting_completed_at,- "completed_at": document.completed_at,- "paused_at": document.paused_at,- "error": document.error,- "stopped_at": document.stopped_at,- "completed_segments": completed_segments,- "total_segments": total_segments,- }- documents_status.append(document_dict)- return dump_response(DocumentStatusListResponse, {"data": documents_status}), 200+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):+ try:+ result = application_services().knowledge.datasets.indexing_status(+ request_context, dataset_id=str(dataset_id)+ )+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(DocumentStatusListResponse, result), 200@console_ns.route("/datasets/api-keys")@@ -1102,7 +629,7 @@ class DatasetApiKeyApi(Resource):@console_ns.response(200, "API keys retrieved successfully", console_ns.models[ApiKeyList.__name__])@console_account_admission(allowed_roles=API_KEY_DELETE_ROLES,- rbac_checks=[RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace())],+ rbac_checks=(RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace()),),)def get(self, request_context: RequestContext):with api_key_errors():@@ -1114,16 +641,16 @@ class DatasetApiKeyApi(Resource):@console_ns.response(400, "Maximum keys exceeded")@console_account_admission(allowed_roles=API_KEY_DELETE_ROLES,- rbac_checks=[RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace())],+ rbac_checks=(RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace()),),)def post(self, request_context: RequestContext):# Preserve the optional scope and validation response of the workspace route.payload = request.get_json(silent=True) or {}- raw_dataset_ids = payload.get("dataset_ids") or []- if not isinstance(raw_dataset_ids, list) or any(not isinstance(item, str) for item in raw_dataset_ids):+ dataset_ids = payload.get("dataset_ids") or []+ if not isinstance(dataset_ids, list) or any(not isinstance(item, str) for item in dataset_ids):console_ns.abort(400, message="dataset_ids must be a list of strings.")with api_key_errors():- key = application_services().dataset_api_keys.create_workspace_key(request_context, tuple(raw_dataset_ids))+ key = application_services().dataset_api_keys.create_workspace_key(request_context, tuple(dataset_ids))return dump_response(ApiKeyItem, key), 200@@ -1135,7 +662,7 @@ class DatasetApiDeleteApi(Resource):@console_ns.response(204, "API key deleted successfully")@console_account_admission(allowed_roles=API_KEY_DELETE_ROLES,- rbac_checks=[RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace())],+ rbac_checks=(RBACCheck(RBACPermission.DATASET_API_KEY_MANAGE, Workspace()),),)def delete(self, request_context: RequestContext, api_key_id: UUID):with api_key_errors():@@ -1145,21 +672,17 @@ class DatasetApiDeleteApi(Resource):@console_ns.route("/datasets/<uuid:dataset_id>/api-keys/<string:status>")class DatasetEnableApiApi(Resource):- @setup_required- @login_required- @account_initialization_required@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def post(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, status: str):- dataset = _get_accessible_dataset(dataset_id, current_tenant_id, current_user, session)- if not current_user.is_dataset_editor:- raise Forbidden()-- DatasetService.update_dataset_api_status(dataset, status == "enable", current_user, session)-+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )+ def post(self, request_context: RequestContext, dataset_id: UUID, status: str):+ try:+ application_services().knowledge.datasets.set_api_enabled(+ request_context, dataset_id=str(dataset_id), status=status+ )+ except Exception as error:+ _raise_dataset_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@@ -1168,12 +691,12 @@ class DatasetApiBaseUrlApi(Resource):@console_ns.doc("get_dataset_api_base_info")@console_ns.doc(description="Get dataset API base information")@console_ns.response(200, "API base info retrieved successfully", console_ns.models[ApiBaseUrlResponse.__name__])- @setup_required- @login_required- @account_initialization_required- def get(self):- base = dify_config.SERVICE_API_URL or request.host_url.rstrip("/")- return ApiBaseUrlResponse(api_base_url=normalize_api_base_url(base)).model_dump(mode="json")+ @console_account_admission()+ def get(self, request_context: RequestContext):+ base = application_services().knowledge.datasets.api_base_url(+ request_context, request_base_url=request.host_url.rstrip("/")+ )+ return ApiBaseUrlResponse(api_base_url=base).model_dump(mode="json")@console_ns.route("/datasets/retrieval-setting")@@ -1183,15 +706,10 @@ class DatasetRetrievalSettingApi(Resource):@console_ns.response(200, "Retrieval settings retrieved successfully", console_ns.models[RetrievalSettingResponse.__name__])- @setup_required- @login_required- @account_initialization_required- def get(self):- vector_type = dify_config.VECTOR_STORE- return dump_response(- RetrievalSettingResponse,- _get_retrieval_methods_by_vector_type(vector_type, is_mock=False),- )+ @console_account_admission()+ def get(self, request_context: RequestContext):+ result = application_services().knowledge.datasets.retrieval_settings(request_context)+ return dump_response(RetrievalSettingResponse, result)@console_ns.route("/datasets/retrieval-setting/<string:vector_type>")@@ -1204,14 +722,12 @@ class DatasetRetrievalSettingMockApi(Resource):"Mock retrieval settings retrieved successfully",console_ns.models[RetrievalSettingResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- def get(self, vector_type: str):- return dump_response(- RetrievalSettingResponse,- _get_retrieval_methods_by_vector_type(vector_type, is_mock=True),+ @console_account_admission()+ def get(self, request_context: RequestContext, vector_type: str):+ result = application_services().knowledge.datasets.retrieval_settings(+ request_context, vector_type=vector_type, is_mock=True)+ return dump_response(RetrievalSettingResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/error-docs")@@ -1221,20 +737,15 @@ class DatasetErrorDocs(Resource):@console_ns.doc(params={"dataset_id": "Dataset ID"})@console_ns.response(200, "Error documents retrieved successfully", console_ns.models[ErrorDocsResponse.__name__])@console_ns.response(404, "Dataset not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset = _get_accessible_dataset(dataset_id, current_tenant_id, current_user, session)- results = DocumentService.get_error_documents_by_dataset_ref(- DatasetRefService.create_dataset_ref(dataset), session- )-- return dump_response(ErrorDocsResponse, {"data": results, "total": len(results)}), 200+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):+ try:+ result = application_services().knowledge.datasets.error_documents(+ request_context, dataset_id=str(dataset_id)+ )+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(ErrorDocsResponse, result), 200@console_ns.route("/datasets/<uuid:dataset_id>/permission-part-users")@@ -1249,25 +760,15 @@ class DatasetPermissionUserListApi(Resource):)@console_ns.response(404, "Dataset not found")@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, session)-- return dump_response(PartialMemberListResponse, {"data": partial_members_list}), 200+ result = application_services().knowledge.datasets.partial_members(+ request_context, dataset_id=str(dataset_id)+ )+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(PartialMemberListResponse, {"data": result}), 200@console_ns.route("/datasets/<uuid:dataset_id>/auto-disable-logs")@@ -1281,16 +782,12 @@ class DatasetAutoDisableLogApi(Resource):console_ns.models[AutoDisableLogsResponse.__name__],)@console_ns.response(404, "Dataset not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset = _get_accessible_dataset(dataset_id, current_tenant_id, current_user, session)- auto_disable_logs = DatasetService.get_dataset_auto_disable_logs(- DatasetRefService.create_dataset_ref(dataset), session- )- return dump_response(AutoDisableLogsResponse, auto_disable_logs), 200+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):+ try:+ result = application_services().knowledge.datasets.auto_disable_logs(+ request_context, dataset_id=str(dataset_id)+ )+ except Exception as error:+ _raise_dataset_error(error)+ return dump_response(AutoDisableLogsResponse, result), 200diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.pyindex 72abbbbd264..76e6c21a727 100644--- a/api/controllers/console/datasets/datasets_document.py+++ b/api/controllers/console/datasets/datasets_document.py@@ -1,95 +1,106 @@-import json-import logging-from argparse import ArgumentTypeError-from collections.abc import Mapping, Sequencefrom contextlib import ExitStackfrom datetime import datetime-from typing import Any, Literal, cast+from typing import Any, Literal, Neverfrom uuid import UUID-import sqlalchemy as safrom flask import request, send_filefrom flask_restx import Resourcefrom pydantic import BaseModel, Field, JsonValue, field_validator-from sqlalchemy import asc, desc, func, select-from sqlalchemy.orm import Session-from werkzeug.exceptions import Forbidden, NotFound+from werkzeug.exceptions import BadRequest, Forbidden, NotFound-import services-from configs import dify_configfrom controllers.common.controller_schemas import DocumentBatchDownloadZipPayload+from controllers.common.errors import InvalidArgumentError, NotFoundErrorfrom controllers.common.fields import SimpleResultMessageResponse, SimpleResultResponse, UrlResponse-from controllers.common.rbac import DatasetId, RBACCheck, enforce_rbac_checks+from controllers.common.rbac import DatasetByDocument, DatasetId, RBACCheck, enforce_rbac_checksfrom controllers.common.schema import register_response_schema_models, register_schema_models-from controllers.common.session import with_sessionfrom controllers.console import console_ns-from controllers.console.wraps import RBACPermission, model_validate, rbac_permission_required-from core.entities.knowledge_entities import IndexingEstimate-from core.errors.error import (- LLMBadRequestError,- ModelCurrentlyNotSupportError,- ProviderTokenNotInitError,- QuotaExceededError,-)-from core.indexing_runner import IndexingRunner-from core.model_manager import ModelManager-from core.plugin.impl.exc import PluginDaemonClientSideError-from core.rag.entities import Rule-from core.rag.extractor.entity.datasource_type import DatasourceType-from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo-from core.rag.index_processor.constant.index_type import IndexTechniqueType-from fields.base import ResponseModel-from fields.document_fields import (- DocumentMetadataResponse,- DocumentResponse,- DocumentStatusListResponse,- DocumentStatusResponse,- DocumentWithSession,- document_response,- document_responses,- normalize_enum,-)-from graphon.model_runtime.entities.model_entities import ModelType-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError-from libs.datetime_utils import naive_utc_now-from libs.helper import dump_response, to_timestamp-from libs.login import login_required-from libs.pagination import paginate_query-from models import Account, Document, DocumentSegment, UploadFile-from models.dataset import DatasetPermissionEnum, DocumentPipelineExecutionLog-from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus-from services.dataset_ref_service import DatasetRefService-from services.dataset_service import DatasetService, DocumentService-from services.enterprise import rbac_service as enterprise_rbac_service-from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel-from services.file_service import FileService-from services.vector_space_admission_service import get_vector_space_admission_error_fields-from tasks.generate_summary_index_task import generate_summary_index_task--from ..app.error import (+from controllers.console.app.error import (ProviderModelCurrentlyNotSupportError,ProviderNotInitializeError,ProviderQuotaExceededError,)-from ..datasets.error import (+from controllers.console.datasets.error import (ArchivedDocumentImmutableError,+ DatasetAccessDeniedRequestError,DocumentAlreadyFinishedError,DocumentIndexingError,IndexingEstimateError,InvalidActionError,InvalidMetadataError,)-from ..wraps import (- account_initialization_required,+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import (+ RBACPermission,check_knowledge_rate_limit,cloud_edition_billing_rate_limit_check,cloud_edition_billing_resource_check,- setup_required,- with_current_tenant_id,- with_current_user,+ model_validate,+)+from core.entities.knowledge_entities import IndexingEstimate+from core.rag.entities import Rule+from extensions.ext_application_services import application_services+from fields.base import ResponseModel+from fields.document_fields import (+ DocumentMetadataResponse,+ DocumentResponse,+ DocumentStatusListResponse,+ DocumentStatusResponse,+ normalize_enum,+)+from libs.helper import dump_response, to_timestamp+from machinery.context import RequestContext+from models.account import TenantAccountRole+from models.enums import ProcessRuleMode+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.documents.application import (+ DocumentArchivedError,+ DocumentIndexingStateError,+ DocumentInvalidActionError,+ DocumentListFilter,+ DocumentNotFoundError,+ DocumentProviderError,+)+from services.knowledge.entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel+from services.knowledge.indexing.estimate import (+ EstimateDocumentAlreadyFinishedError,+ EstimateDocumentNotFoundError,+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+ UnsupportedEstimateSourceError,+)++_DATASET_EDIT_ROLES = frozenset(+ {+ TenantAccountRole.OWNER,+ TenantAccountRole.ADMIN,+ TenantAccountRole.EDITOR,+ TenantAccountRole.DATASET_OPERATOR,+ })-logger = logging.getLogger(__name__)++def _raise_document_error(error: Exception) -> Never:+ if isinstance(error, DatasetNotFoundError):+ raise NotFound("Dataset not found.") from error+ if isinstance(error, DocumentNotFoundError):+ raise NotFound(str(error)) from error+ if isinstance(error, DatasetAccessDeniedError):+ raise Forbidden(str(error)) from error+ if isinstance(error, DocumentArchivedError):+ raise ArchivedDocumentImmutableError() from error+ if isinstance(error, DocumentIndexingStateError):+ raise DocumentIndexingError(str(error)) from error+ if isinstance(error, DocumentInvalidActionError):+ raise InvalidActionError(str(error)) from error+ if isinstance(error, DocumentProviderError):+ if error.kind == "quota":+ raise ProviderQuotaExceededError() from error+ if error.kind == "unsupported":+ raise ProviderModelCurrentlyNotSupportError() from error+ raise ProviderNotInitializeError(str(error)) from error+ raise errorclass DatasetResponse(ResponseModel):@@ -119,24 +130,6 @@ class DocumentWithSegmentsResponse(DocumentResponse):total_segments: int | None = Field(default=None, exclude_if=lambda value: value is None)-class DocumentWithSegmentsSession(DocumentWithSession):- @property- def process_rule_dict(self) -> Any:- process_rule = self.document.get_dataset_process_rule(session=self.session)- return process_rule.to_dict() if process_rule else None---def document_with_segments_responses(- documents: Sequence[Document], *, session: Session-) -> list[DocumentWithSegmentsResponse]:- return [- DocumentWithSegmentsResponse.model_validate(- DocumentWithSegmentsSession(document=document, session=session), from_attributes=True- )- for document in documents- ]--class DatasetAndDocumentResponse(ResponseModel):dataset: DatasetResponsedocuments: list[DocumentResponse]@@ -183,6 +176,20 @@ class IndexingEstimateResponse(IndexingEstimate):currency: str+def _serialize_indexing_estimate(estimate: IndexingEstimate) -> tuple[dict[str, object], int]:+ return (+ IndexingEstimateResponse(+ tokens=0,+ total_price=0,+ currency="USD",+ total_segments=estimate.total_segments,+ preview=estimate.preview,+ qa_preview=estimate.qa_preview,+ ).model_dump(mode="json", exclude_none=True),+ 200,+ )++class DocumentDetailResponse(ResponseModel):id: strposition: int | None = None@@ -294,96 +301,29 @@ register_response_schema_models()-class DocumentResource(Resource):- def get_document(- self, session: Session, dataset_id: str, document_id: str, current_user: Account, current_tenant_id: str- ) -> Document:- dataset = DatasetService.get_dataset_for_tenant(dataset_id, current_tenant_id, session=session)- if not dataset:- raise NotFound("Dataset not found.")-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, document_id)- document = DatasetRefService.get_document_by_ref(document_ref, session=session)-- if not document:- raise NotFound("Document not found.")-- return document-- def get_batch_documents(- self, session: Session, dataset_id: str, batch: str, current_user: Account- ) -> Sequence[Document]:- dataset = DatasetService.get_dataset(dataset_id, session)- if not dataset:- raise NotFound("Dataset not found.")-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- documents = DocumentService.get_batch_documents(dataset_id, batch, session)-- if not documents:- raise NotFound("Documents not found.")-- return documents--@console_ns.route("/datasets/process-rule")class GetProcessRuleApi(Resource):@console_ns.doc("get_process_rule")@console_ns.doc(description="Get dataset document processing rules")@console_ns.doc(params={"document_id": "Document ID (optional)"})@console_ns.response(200, "Process rules retrieved successfully", console_ns.models[ProcessRuleResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_session(write=False)- def get(self, session: Session, current_user: Account):- req_data = request.args-- document_id = req_data.get("document_id")-- # get default rules- mode = DocumentService.DEFAULT_RULES["mode"]- rules = DocumentService.DEFAULT_RULES["rules"]- limits = DocumentService.DEFAULT_RULES["limits"]+ @console_account_admission()+ def get(self, request_context: RequestContext):+ document_id = request.args.get("document_id")if document_id:- document = DocumentService.get_document_by_id(document_id, session)- if document is None:- raise NotFound("Document not found.")-- dataset = DatasetService.get_dataset(document.dataset_id, session)-- if not dataset:- raise NotFound("Dataset not found.")-enforce_rbac_checks(- tenant_id=dataset.tenant_id,- account_id=current_user.id,- checks=[RBACCheck(RBACPermission.DATASET_READONLY, DatasetId())],- path_args={"dataset_id": dataset.id},+ tenant_id=request_context.active_workspace_id,+ account_id=request_context.account_id,+ checks=[RBACCheck(RBACPermission.DATASET_READONLY, DatasetByDocument())],+ path_args={"document_id": document_id},)-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- dataset_process_rule = dataset.get_latest_process_rule(session=session)- if dataset_process_rule:- mode = dataset_process_rule.mode- rules = dataset_process_rule.rules_dict-- return dump_response(ProcessRuleResponse, {"mode": mode, "rules": rules, "limits": limits})+ try:+ result = application_services().knowledge.documents.get_process_rule(+ request_context, document_id=document_id+ )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(ProcessRuleResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/documents")@@ -406,224 +346,56 @@ class DatasetDocumentListApi(Resource):"Documents retrieved successfully",console_ns.models[DocumentWithSegmentsListResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- raw_args = request.args.to_dict()- param = DocumentDatasetListParam.model_validate(raw_args)- page = param.page- limit = param.limit- search = param.search- sort = param.sort_by- status = param.status- # "yes", "true", "t", "y", "1" convert to True, while others convert to False.+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID):+ args = DocumentDatasetListParam.model_validate(request.args.to_dict())try:- fetch_val = param.fetch_val- if isinstance(fetch_val, bool):- fetch = fetch_val- else:- if fetch_val.lower() in ("yes", "true", "t", "y", "1"):- fetch = True- elif fetch_val.lower() in ("no", "false", "f", "n", "0"):- fetch = False- else:- raise ArgumentTypeError(- f"Truthy value expected: got {fetch_val} but expected one of yes/no, true/false, t/f, y/n, 1/0 "- f"(case insensitive)."- )- except (ArgumentTypeError, ValueError, Exception):- fetch = False- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- query = select(Document).where(Document.dataset_id == dataset_id_str, Document.tenant_id == current_tenant_id)-- if status:- query = DocumentService.apply_display_status_filter(query, status)-- if search:- search = f"%{search}%"- query = query.where(Document.name.like(search))-- if sort.startswith("-"):- sort_logic = desc- sort = sort[1:]- else:- sort_logic = asc-- match sort:- case "hit_count":- sub_query = (- sa.select(- DocumentSegment.document_id, sa.func.sum(DocumentSegment.hit_count).label("total_hit_count")- )- .where(DocumentSegment.dataset_id == dataset_id_str)- .group_by(DocumentSegment.document_id)- .subquery()- )-- query = query.outerjoin(sub_query, sub_query.c.document_id == Document.id).order_by(- sort_logic(sa.func.coalesce(sub_query.c.total_hit_count, 0)),- sort_logic(Document.position),- )- case "created_at":- query = query.order_by(- sort_logic(Document.created_at),- sort_logic(Document.position),- )- case _:- query = query.order_by(- desc(Document.created_at),- desc(Document.position),- )-- paginated_documents = paginate_query(query, session=session, page=page, per_page=limit, max_per_page=100)- documents = paginated_documents.items-- DocumentService.enrich_documents_with_summary_index_status(- documents=documents,- dataset=dataset,- tenant_id=current_tenant_id,- session=session,- )-- if fetch:- for document in documents:- completed_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.completed_at.isnot(None),- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- total_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.document_id == str(document.id),- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- document.completed_segments = completed_segments- document.total_segments = total_segments- response = {- "data": document_with_segments_responses(documents, session=session),- # The result object already knows: it was built from the page the query- # ran with, where `len(documents) == limit` is only ever a guess that a- # full page means another one follows.- "has_more": paginated_documents.has_next,- "limit": paginated_documents.per_page,- "total": paginated_documents.total,- "page": paginated_documents.page,- }-- return dump_response(DocumentWithSegmentsListResponse, response)+ result = application_services().knowledge.documents.list_documents(+ request_context,+ dataset_id=str(dataset_id),+ query=DocumentListFilter(+ page=args.page,+ limit=args.limit,+ search=args.search or "",+ sort=args.sort_by,+ status=args.status,+ fetch=args.fetch_val.lower() in {"yes", "true", "t", "y", "1"},+ ),+ )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DocumentWithSegmentsListResponse, result)- @setup_required- @login_required- @account_initialization_required- @cloud_edition_billing_resource_check("vector_space")- @cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[KnowledgeConfig.__name__])@console_ns.response(200, "Documents created successfully", console_ns.models[DatasetAndDocumentResponse.__name__])- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_USE, DatasetId()))- @with_session- def post(self, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)-- dataset = DatasetService.get_dataset(dataset_id_str, session)-- if not dataset:- raise NotFound("Dataset not found.")-- # The role of the current user in the ta table must be admin, owner, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- knowledge_config = KnowledgeConfig.model_validate(console_ns.payload or {})-- if not dataset.indexing_technique and not knowledge_config.indexing_technique:- raise ValueError("indexing_technique is required.")-- # validate args- DocumentService.document_create_args_validate(knowledge_config)-+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_USE, DatasetId()),)+ )+ @cloud_edition_billing_resource_check("vector_space")+ @cloud_edition_billing_rate_limit_check("knowledge")+ def post(self, request_context: RequestContext, dataset_id: UUID):+ config = KnowledgeConfig.model_validate(console_ns.payload or {})try:- documents, batch = DocumentService.save_document_with_dataset_id(- dataset, knowledge_config, current_user, session=session+ result = application_services().knowledge.documents.create_documents(+ request_context, dataset_id=str(dataset_id), settings=config.model_dump())- dataset = DatasetService.get_dataset(dataset_id_str, session)-- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- except QuotaExceededError:- raise ProviderQuotaExceededError()- except ModelCurrentlyNotSupportError:- raise ProviderModelCurrentlyNotSupportError()+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DatasetAndDocumentResponse, result)- return dump_response(- DatasetAndDocumentResponse,- {"dataset": dataset, "documents": document_responses(documents, session=session), "batch": batch},- )-- @setup_required- @login_required- @account_initialization_required@console_ns.response(204, "Documents deleted successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_DELETE_FILE, DatasetId()))- @with_session- def delete(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- ):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")-- if not current_user.is_dataset_editor:- raise Forbidden()-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_DELETE_FILE, DatasetId()),),+ )+ def delete(self, request_context: RequestContext, dataset_id: UUID):check_knowledge_rate_limit()try:- document_ids = request.args.getlist("document_id")- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- DocumentService.delete_documents(dataset_ref, document_ids, dataset.get_doc_form(session=session), session)- except services.errors.document.DocumentIndexingError:- raise DocumentIndexingError("Cannot delete document during indexing.")-+ application_services().knowledge.documents.delete_documents(+ request_context, dataset_id=str(dataset_id), document_ids=request.args.getlist("document_id")+ )+ except Exception as error:+ _raise_document_error(error)return "", 204@@ -636,87 +408,22 @@ class DatasetInitApi(Resource):200, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__])@console_ns.response(400, "Invalid request parameters")- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(allowed_roles=_DATASET_EDIT_ROLES)@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @with_current_tenant_id- @with_session- def post(self, session: Session, current_tenant_id: str, current_user: Account):- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- knowledge_config = KnowledgeConfig.model_validate(console_ns.payload or {})- if knowledge_config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- if knowledge_config.embedding_model is None or knowledge_config.embedding_model_provider is None:- raise ValueError("embedding model and embedding model provider are required for high quality indexing.")- try:- model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)- model_manager.get_model_instance(- tenant_id=current_tenant_id,- provider=knowledge_config.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=knowledge_config.embedding_model,- )- is_multimodal = DatasetService.check_is_multimodal_model(- current_tenant_id, knowledge_config.embedding_model_provider, knowledge_config.embedding_model- )- knowledge_config.is_multimodal = is_multimodal # pyrefly: ignore[bad-assignment]- except InvokeAuthorizationError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)-- # validate args- DocumentService.document_create_args_validate(knowledge_config)-+ def post(self, request_context: RequestContext):+ config = KnowledgeConfig.model_validate(console_ns.payload or {})try:- dataset, documents, batch = DocumentService.save_document_without_dataset_id(- tenant_id=current_tenant_id,- knowledge_config=knowledge_config,- account=current_user,- session=session,- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- except QuotaExceededError:- raise ProviderQuotaExceededError()- except ModelCurrentlyNotSupportError:- raise ProviderModelCurrentlyNotSupportError()-- if dify_config.RBAC_ENABLED:- dataset.permission = DatasetPermissionEnum.ALL_TEAM- else:- dataset.permission = DatasetPermissionEnum.ONLY_ME- session.flush()-- if dify_config.RBAC_ENABLED:- enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist(- current_tenant_id,- current_user.id,- dataset.id,- enterprise_rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),- )- enterprise_rbac_service.try_sync_creator_access_policy_member_bindings(- current_tenant_id,- current_user.id,- enterprise_rbac_service.RBACResourceType.DATASET,- dataset.id,+ result = application_services().knowledge.documents.initialize_dataset(+ request_context, settings=config.model_dump())-- return dump_response(- DatasetAndDocumentResponse,- {"dataset": dataset, "documents": document_responses(documents, session=session), "batch": batch},- )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DatasetAndDocumentResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-estimate")-class DocumentIndexingEstimateApi(DocumentResource):+class DocumentIndexingEstimateApi(Resource):@console_ns.doc("estimate_document_indexing")@console_ns.doc(description="Estimate document indexing cost")@console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})@@ -727,256 +434,91 @@ class DocumentIndexingEstimateApi(DocumentResource):)@console_ns.response(404, "Document not found")@console_ns.response(400, "Document already finished")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_USE, DatasetId()))- @with_session- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-- if document.indexing_status in {IndexingStatus.COMPLETED, IndexingStatus.ERROR}:- raise DocumentAlreadyFinishedError()-- data_process_rule = document.get_dataset_process_rule(session=session)- data_process_rule_dict: Mapping[str, Any] = data_process_rule.to_dict() if data_process_rule else {}-- if document.data_source_type == "upload_file":- data_source_info = document.data_source_info_dict- if data_source_info and "upload_file_id" in data_source_info:- file_id = data_source_info["upload_file_id"]-- file = session.scalar(- select(UploadFile)- .where(UploadFile.tenant_id == document.tenant_id, UploadFile.id == file_id)- .limit(1)- )-- # raise error if file not found- if not file:- raise NotFound("File not found.")-- extract_setting = ExtractSetting(- datasource_type=DatasourceType.FILE, upload_file=file, document_model=document.doc_form- )-- indexing_runner = IndexingRunner()-- try:- estimate_response = indexing_runner.indexing_estimate(- tenant_id=current_tenant_id,- extract_settings=[extract_setting],- tmp_processing_rule=data_process_rule_dict,- doc_form=document.doc_form,- doc_language="English",- dataset_id=dataset_id_str,- session=session,- )- return (- # TODO: why using zero here? the same for the below endpoint- IndexingEstimateResponse(- tokens=0,- total_price=0,- currency="USD",- total_segments=estimate_response.total_segments,- preview=estimate_response.preview,- qa_preview=estimate_response.qa_preview,- ).model_dump(mode="json", exclude_none=True),- 200,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider "- "in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- except PluginDaemonClientSideError as ex:- raise ProviderNotInitializeError(ex.description)- except Exception as e:- raise IndexingEstimateError(str(e))-- return (- IndexingEstimateResponse(- tokens=0,- total_price=0,- currency="USD",- total_segments=0,- preview=[],- ).model_dump(mode="json", exclude_none=True),- 200,- )+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_USE, DatasetId()),),+ )+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):+ try:+ estimate = application_services().knowledge.indexing_estimates.estimate_document(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ )+ except (+ DatasetNotFoundError,+ EstimateDocumentNotFoundError,+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ ) as error:+ raise NotFoundError(description=str(error)) from error+ except DatasetAccessDeniedError as error:+ raise DatasetAccessDeniedRequestError(description=str(error)) from error+ except EstimateDocumentAlreadyFinishedError as error:+ raise DocumentAlreadyFinishedError() from error+ except UnsupportedEstimateSourceError as error:+ raise InvalidArgumentError(description=str(error)) from error+ except IndexingEstimateProviderUnavailableError as error:+ raise ProviderNotInitializeError(str(error)) from error+ except IndexingEstimateExecutionError as error:+ raise IndexingEstimateError(str(error)) from error+ return _serialize_indexing_estimate(estimate)@console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-estimate")-class DocumentBatchIndexingEstimateApi(DocumentResource):+class DocumentBatchIndexingEstimateApi(Resource):@console_ns.response(200,"Indexing estimate calculated successfully",console_ns.models[IndexingEstimateResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_USE, DatasetId()))- @with_session- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, batch: str):- dataset_id_str = str(dataset_id)- documents = self.get_batch_documents(session, dataset_id_str, batch, current_user)- if not documents:- return (- IndexingEstimateResponse(- tokens=0,- total_price=0,- currency="USD",- total_segments=0,- preview=[],- ).model_dump(mode="json", exclude_none=True),- 200,+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_USE, DatasetId()),),+ )+ def get(self, request_context: RequestContext, dataset_id: UUID, batch: str):+ try:+ estimate = application_services().knowledge.indexing_estimates.estimate_batch(+ request_context,+ dataset_id=str(dataset_id),+ batch=batch,)- data_process_rule = documents[0].get_dataset_process_rule(session=session)- data_process_rule_dict: Mapping[str, Any] = data_process_rule.to_dict() if data_process_rule else {}- extract_settings = []- for document in documents:- if document.indexing_status in {IndexingStatus.COMPLETED, IndexingStatus.ERROR}:- raise DocumentAlreadyFinishedError()- data_source_info = document.data_source_info_dict- match document.data_source_type:- case "upload_file":- if not data_source_info:- continue- file_id = data_source_info["upload_file_id"]- file_detail = session.scalar(- select(UploadFile)- .where(UploadFile.tenant_id == current_tenant_id, UploadFile.id == file_id)- .limit(1)- )-- if file_detail is None:- raise NotFound("File not found.")-- extract_setting = ExtractSetting(- datasource_type=DatasourceType.FILE, upload_file=file_detail, document_model=document.doc_form- )- extract_settings.append(extract_setting)- case "notion_import":- if not data_source_info:- continue- extract_setting = ExtractSetting(- datasource_type=DatasourceType.NOTION,- notion_info=NotionInfo.model_validate(- {- "credential_id": data_source_info.get("credential_id"),- "notion_workspace_id": data_source_info["notion_workspace_id"],- "notion_obj_id": data_source_info["notion_page_id"],- "notion_page_type": data_source_info["type"],- "tenant_id": current_tenant_id,- }- ),- document_model=document.doc_form,- )- extract_settings.append(extract_setting)- case "website_crawl":- if not data_source_info:- continue- extract_setting = ExtractSetting(- datasource_type=DatasourceType.WEBSITE,- website_info=WebsiteInfo.model_validate(- {- "provider": data_source_info["provider"],- "job_id": data_source_info["job_id"],- "url": data_source_info["url"],- "tenant_id": current_tenant_id,- "mode": data_source_info["mode"],- "only_main_content": data_source_info["only_main_content"],- }- ),- document_model=document.doc_form,- )- extract_settings.append(extract_setting)-- case _:- raise ValueError("Data source type not support")- indexing_runner = IndexingRunner()- try:- response = indexing_runner.indexing_estimate(- tenant_id=current_tenant_id,- extract_settings=extract_settings,- tmp_processing_rule=data_process_rule_dict,- doc_form=document.doc_form,- doc_language="English",- dataset_id=dataset_id_str,- session=session,- )- return (- IndexingEstimateResponse(- tokens=0,- total_price=0,- currency="USD",- total_segments=response.total_segments,- preview=response.preview,- qa_preview=response.qa_preview,- ).model_dump(mode="json", exclude_none=True),- 200,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- except PluginDaemonClientSideError as ex:- raise ProviderNotInitializeError(ex.description)- except Exception as e:- raise IndexingEstimateError(str(e))+ except (+ DatasetNotFoundError,+ EstimateDocumentNotFoundError,+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ ) as error:+ raise NotFoundError(description=str(error)) from error+ except DatasetAccessDeniedError as error:+ raise DatasetAccessDeniedRequestError(description=str(error)) from error+ except EstimateDocumentAlreadyFinishedError as error:+ raise DocumentAlreadyFinishedError() from error+ except UnsupportedEstimateSourceError as error:+ raise InvalidArgumentError(description=str(error)) from error+ except IndexingEstimateProviderUnavailableError as error:+ raise ProviderNotInitializeError(str(error)) from error+ except IndexingEstimateExecutionError as error:+ raise IndexingEstimateError(str(error)) from error+ return _serialize_indexing_estimate(estimate)@console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-status")-class DocumentBatchIndexingStatusApi(DocumentResource):+class DocumentBatchIndexingStatusApi(Resource):@console_ns.response(200, "Indexing status retrieved successfully", console_ns.models[DocumentStatusListResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_user: Account, dataset_id: UUID, batch: str):- dataset_id_str = str(dataset_id)- documents = self.get_batch_documents(session, dataset_id_str, batch, current_user)- segment_counts = DocumentService.get_document_segment_counts(documents, session=session)- documents_status = []- for document in documents:- completed_segments, total_segments = segment_counts.get(str(document.id), (0, 0))- # Create a dictionary with document attributes and additional fields- document_dict = {- "id": document.id,- "indexing_status": IndexingStatus.PAUSED if document.is_paused else document.indexing_status,- "processing_started_at": document.processing_started_at,- "parsing_completed_at": document.parsing_completed_at,- "cleaning_completed_at": document.cleaning_completed_at,- "splitting_completed_at": document.splitting_completed_at,- "completed_at": document.completed_at,- "paused_at": document.paused_at,- "error": document.error,- **get_vector_space_admission_error_fields(document.error),- "stopped_at": document.stopped_at,- "completed_segments": completed_segments,- "total_segments": total_segments,- }- documents_status.append(document_dict)- return dump_response(DocumentStatusListResponse, {"data": documents_status})+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID, batch: str):+ try:+ result = application_services().knowledge.documents.get_batch_indexing_status(+ request_context, dataset_id=str(dataset_id), batch=batch+ )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DocumentStatusListResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-status")-class DocumentIndexingStatusApi(DocumentResource):+class DocumentIndexingStatusApi(Resource):@console_ns.doc("get_document_indexing_status")@console_ns.doc(description="Get document indexing status")@console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})@@ -984,59 +526,19 @@ class DocumentIndexingStatusApi(DocumentResource):200, "Indexing status retrieved successfully", console_ns.models[DocumentStatusResponse.__name__])@console_ns.response(404, "Document not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-- completed_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.completed_at.isnot(None),- DocumentSegment.document_id == document_id_str,- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )- )- or 0- )- total_segments = (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.document_id == document_id_str,- DocumentSegment.status != SegmentStatus.RE_SEGMENT,- )+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):+ try:+ result = application_services().knowledge.documents.get_indexing_status(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id))- or 0- )-- # Create a dictionary with document attributes and additional fields- document_dict = {- "id": document.id,- "indexing_status": IndexingStatus.PAUSED if document.is_paused else document.indexing_status,- "processing_started_at": document.processing_started_at,- "parsing_completed_at": document.parsing_completed_at,- "cleaning_completed_at": document.cleaning_completed_at,- "splitting_completed_at": document.splitting_completed_at,- "completed_at": document.completed_at,- "paused_at": document.paused_at,- "error": document.error,- **get_vector_space_admission_error_fields(document.error),- "stopped_at": document.stopped_at,- "completed_segments": completed_segments,- "total_segments": total_segments,- }- return dump_response(DocumentStatusResponse, document_dict)+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DocumentStatusResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")-class DocumentApi(DocumentResource):+class DocumentApi(Resource):METADATA_CHOICES = {"all", "only", "without"}@console_ns.doc("get_document")@@ -1050,179 +552,103 @@ class DocumentApi(DocumentResource):)@console_ns.response(200, "Document retrieved successfully", console_ns.models[DocumentDetailResponse.__name__])@console_ns.response(404, "Document not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):metadata = request.args.get("metadata", "all")if metadata not in self.METADATA_CHOICES:raise InvalidMetadataError(f"Invalid metadata value: {metadata}")-- metadata_fields = {"doc_type", "doc_metadata"}- if metadata == "only":- return (- dump_response(- DocumentDetailResponse,- {- "id": document.id,- "doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),- },- include={"id", *metadata_fields},- exclude_unset=True,- ),- 200,+ try:+ result = application_services().knowledge.documents.get_document(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ metadata_only=metadata == "only",)+ except Exception as error:+ _raise_document_error(error)- dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, session)- document_process_rule = document.get_dataset_process_rule(session=session)- document_process_rules: Mapping[str, Any] = document_process_rule.to_dict() if document_process_rule else {}- segment_count = document.get_segment_count(session=session)- response_payload = {- "id": document.id,- "position": document.position,- "data_source_type": document.data_source_type,- "data_source_info": document.data_source_info_dict,- "data_source_detail_dict": document.get_data_source_detail_dict(session=session),- "dataset_process_rule_id": document.dataset_process_rule_id,- "dataset_process_rule": dataset_process_rules,- "document_process_rule": document_process_rules,- "name": document.name,- "created_from": document.created_from,- "created_by": document.created_by,- "created_at": int(document.created_at.timestamp()),- "tokens": document.tokens,- "indexing_status": document.indexing_status,- "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,- "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,- "indexing_latency": document.indexing_latency,- "error": document.error,- "enabled": document.enabled,- "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,- "disabled_by": document.disabled_by,- "archived": document.archived,- "doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),- "segment_count": segment_count,- "average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,- "hit_count": document.get_hit_count(session=session),- "display_status": document.display_status,- "doc_form": document.doc_form,- "doc_language": document.doc_language,- "need_summary": document.need_summary if document.need_summary is not None else False,- }- exclude = metadata_fields if metadata == "without" else None- return dump_response(DocumentDetailResponse, response_payload, exclude=exclude, exclude_unset=True), 200+ metadata_fields = {"doc_type", "doc_metadata"}+ return dump_response(+ DocumentDetailResponse,+ result,+ include={"id", *metadata_fields} if metadata == "only" else None,+ exclude=metadata_fields if metadata == "without" else None,+ exclude_unset=True,+ ), 200- @setup_required- @login_required- @account_initialization_required- @cloud_edition_billing_rate_limit_check("knowledge")@console_ns.response(204, "Document deleted successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_DELETE_FILE, DatasetId()))- @with_session- def delete(- self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID- ):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)-- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_DELETE_FILE, DatasetId()),),+ )+ @cloud_edition_billing_rate_limit_check("knowledge")+ def delete(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):try:- DocumentService.delete_document(document, session)- except services.errors.document.DocumentIndexingError:- raise DocumentIndexingError("Cannot delete document during indexing.")-+ application_services().knowledge.documents.delete_document(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/download")-class DocumentDownloadApi(DocumentResource):+class DocumentDownloadApi(Resource):"""Return a signed download URL for a dataset document's original uploaded file."""@console_ns.doc("get_dataset_document_download_url")@console_ns.doc(description="Get a signed download URL for a dataset document's original uploaded file")@console_ns.response(200, "Download URL generated successfully", console_ns.models[UrlResponse.__name__])- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_DOCUMENT_DOWNLOAD, DatasetId()),))@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_DOCUMENT_DOWNLOAD, DatasetId()))- @with_session(write=False)- def get(- self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID- ) -> dict[str, Any]:- # Reuse the shared permission/tenant checks implemented in DocumentResource.- document = self.get_document(session, str(dataset_id), str(document_id), current_user, current_tenant_id)- return UrlResponse(url=DocumentService.get_document_download_url(document, session)).model_dump(mode="json")+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):+ try:+ result = application_services().knowledge.documents.get_download_url(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)+ return UrlResponse(url=result).model_dump(mode="json")@console_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")-class DocumentBatchDownloadZipApi(DocumentResource):+class DocumentBatchDownloadZipApi(Resource):"""Download multiple uploaded-file documents as a single ZIP (avoids browser multi-download limits)."""@console_ns.doc("download_dataset_documents_as_zip")@console_ns.doc(description="Download selected dataset documents as a single ZIP archive (upload-file only)")@console_ns.response(200, "ZIP archive downloaded successfully")- @setup_required- @login_required- @account_initialization_required- @cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[DocumentBatchDownloadZipPayload.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_DOCUMENT_DOWNLOAD, DatasetId()))- @with_session(write=False)- def post(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_DOCUMENT_DOWNLOAD, DatasetId()),),+ )+ @cloud_edition_billing_rate_limit_check("knowledge")+ def post(self, request_context: RequestContext, dataset_id: UUID):"""Stream a ZIP archive containing the requested uploaded documents."""- # Parse and validate request payload.payload = DocumentBatchDownloadZipPayload.model_validate(console_ns.payload or {})-- dataset_id_str = str(dataset_id)- document_ids: list[str] = [str(document_id) for document_id in payload.document_ids]- upload_files, download_name = DocumentService.prepare_document_batch_download_zip(- dataset_id=dataset_id_str,- document_ids=document_ids,- tenant_id=current_tenant_id,- current_user=current_user,- session=session,- )-- # Delegate ZIP packing to FileService, but keep Flask response+cleanup in the route.- with ExitStack() as stack:- zip_path = stack.enter_context(FileService.build_upload_files_zip_tempfile(upload_files=upload_files))- response = send_file(- zip_path,- mimetype="application/zip",- as_attachment=True,- download_name=download_name,- )- cleanup = stack.pop_all()- response.call_on_close(cleanup.close)+ try:+ with ExitStack() as stack:+ archive = stack.enter_context(+ application_services().knowledge.documents.build_download_zip(+ request_context,+ dataset_id=str(dataset_id),+ document_ids=[str(value) for value in payload.document_ids],+ )+ )+ response = send_file(+ archive.path, mimetype="application/zip", as_attachment=True, download_name=archive.filename+ )+ cleanup = stack.pop_all()+ response.call_on_close(cleanup.close)+ except Exception as error:+ _raise_document_error(error)# response-contract:ignore binary ZIP download responsereturn response@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/<string:action>")-class DocumentProcessingApi(DocumentResource):+class DocumentProcessingApi(Resource):@console_ns.doc("update_document_processing")@console_ns.doc(description="Update document processing status (pause/resume)")@console_ns.doc(@@ -1235,53 +661,24 @@ class DocumentProcessingApi(DocumentResource):)@console_ns.response(404, "Document not found")@console_ns.response(400, "Invalid action")- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_sessiondef patch(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- document_id: UUID,- action: Literal["pause", "resume"],+ self, request_context: RequestContext, dataset_id: UUID, document_id: UUID, action: Literal["pause", "resume"]):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- match action:- case "pause":- if document.indexing_status != IndexingStatus.INDEXING:- raise InvalidActionError("Document not in indexing state.")-- document.paused_by = current_user.id- document.paused_at = naive_utc_now()- document.is_paused = True-- case "resume":- if document.indexing_status not in {IndexingStatus.PAUSED, IndexingStatus.ERROR}:- raise InvalidActionError("Document not in paused or error state.")-- document.paused_by = None- document.paused_at = None- document.is_paused = False-+ try:+ application_services().knowledge.documents.update_processing(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id), action=action+ )+ except Exception as error:+ _raise_document_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")-class DocumentMetadataApi(DocumentResource):+class DocumentMetadataApi(Resource):@console_ns.doc("update_document_metadata")@console_ns.doc(description="Update document metadata")@console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})@@ -1293,368 +690,167 @@ class DocumentMetadataApi(DocumentResource):)@console_ns.response(404, "Document not found")@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@model_validate(DocumentMetadataUpdatePayload)def put(self,req_data: DocumentMetadataUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)-- doc_type = req_data.doc_type- doc_metadata = req_data.doc_metadata-- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- if doc_type is None or doc_metadata is None:- raise ValueError("Both doc_type and doc_metadata must be provided.")-- if doc_type not in DocumentService.DOCUMENT_METADATA_SCHEMA:- raise ValueError("Invalid doc_type.")-- if not isinstance(doc_metadata, dict):- raise ValueError("doc_metadata must be a dictionary.")- metadata_schema: dict[str, Any] = cast(dict[str, Any], DocumentService.DOCUMENT_METADATA_SCHEMA[doc_type])-- document.doc_metadata = {}- if doc_type == "others":- document.doc_metadata = doc_metadata- else:- for key, value_type in metadata_schema.items():- value = doc_metadata.get(key)- if value is not None and isinstance(value, value_type):- document.doc_metadata[key] = value-- document.doc_type = doc_type- document.updated_at = naive_utc_now()-+ try:+ application_services().knowledge.documents.update_metadata(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ doc_type=req_data.doc_type,+ doc_metadata=req_data.doc_metadata,+ )+ except Exception as error:+ _raise_document_error(error)return SimpleResultMessageResponse(result="success", message="Document metadata updated.").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")-class DocumentStatusApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class DocumentStatusApi(Resource):+ @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")- @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_sessiondef patch(self,- session: Session,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,action: Literal["enable", "disable", "archive", "un_archive"],):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-- # The role of the current user in the ta table must be admin, owner, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)-- # check user's permission- DatasetService.check_dataset_permission(dataset, current_user, session)-- document_ids = request.args.getlist("document_id")-try:- DocumentService.batch_update_document_status(dataset, document_ids, action, current_user, session)- except services.errors.document.DocumentIndexingError as e:- raise InvalidActionError(str(e))- except ValueError as e:- raise InvalidActionError(str(e))- except NotFound as e:- raise NotFound(str(e))-+ application_services().knowledge.documents.change_status(+ request_context,+ dataset_id=str(dataset_id),+ document_ids=request.args.getlist("document_id"),+ action=action,+ )+ except Exception as error:+ _raise_document_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")-class DocumentPauseApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class DocumentPauseApi(Resource):@console_ns.response(204, "Document paused successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def patch(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- document_id: UUID,- ):+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )+ def patch(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):"""pause document."""- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)-- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)- if not current_user.is_dataset_editor:- raise Forbidden()-- # 403 if document is archived- if DocumentService.check_archived(document):- raise ArchivedDocumentImmutableError()-check_knowledge_rate_limit()try:- # pause document- DocumentService.pause_document(document, session)- except services.errors.document.DocumentIndexingError:- raise DocumentIndexingError("Cannot pause completed document.")-+ application_services().knowledge.documents.pause_document(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/resume")-class DocumentRecoverApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class DocumentRecoverApi(Resource):@console_ns.response(204, "Document resumed successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def patch(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- document_id: UUID,- ):+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )+ def patch(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):"""recover document."""- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)-- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)- if not current_user.is_dataset_editor:- raise Forbidden()-- # 403 if document is archived- if DocumentService.check_archived(document):- raise ArchivedDocumentImmutableError()check_knowledge_rate_limit()try:- # pause document- DocumentService.recover_document(document, session)- except services.errors.document.DocumentIndexingError:- raise DocumentIndexingError("Document is not in paused status.")-+ application_services().knowledge.documents.recover_document(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/retry")-class DocumentRetryApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class DocumentRetryApi(Resource):@console_ns.expect(console_ns.models[DocumentRetryPayload.__name__])@console_ns.response(204, "Documents retry started successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@model_validate(DocumentRetryPayload)- def post(- self,- req_data: DocumentRetryPayload,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- ):+ def post(self, req_data: DocumentRetryPayload, request_context: RequestContext, dataset_id: UUID):"""retry document."""- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if not dataset:- raise NotFound("Dataset not found.")-- if not current_user.is_dataset_editor:- raise Forbidden()-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- documents = DocumentService.get_documents_by_ids(- DatasetRefService.create_dataset_ref(dataset), req_data.document_ids, session- )- documents_by_id = {document.id: document for document in documents}- retry_documents = []- for document_id in req_data.document_ids:- try:- document = documents_by_id.get(document_id)-- # 404 if document not found- if document is None:- raise NotFound("Document Not Exists.")-- # 403 if document is archived- if DocumentService.check_archived(document):- raise ArchivedDocumentImmutableError()-- # 400 if document is completed- if document.indexing_status == IndexingStatus.COMPLETED:- raise DocumentAlreadyFinishedError()- retry_documents.append(document)- except Exception:- logger.exception("Failed to retry document, document id: %s", document_id)- continue- # retry documentcheck_knowledge_rate_limit()- DocumentService.retry_document(dataset_id_str, retry_documents, session)-+ try:+ application_services().knowledge.documents.retry_documents(+ request_context, dataset_id=str(dataset_id), document_ids=req_data.document_ids+ )+ except Exception as error:+ _raise_document_error(error)return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/rename")-class DocumentRenameApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class DocumentRenameApi(Resource):@console_ns.response(200, "Document renamed successfully", console_ns.models[DocumentResponse.__name__])@console_ns.expect(console_ns.models[DocumentRenamePayload.__name__])- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@model_validate(DocumentRenamePayload)def post(- self,- req_data: DocumentRenamePayload,- session: Session,- current_user: Account,- dataset_id: UUID,- document_id: UUID,+ self, req_data: DocumentRenamePayload, request_context: RequestContext, dataset_id: UUID, document_id: UUID):- # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator- if not current_user.is_dataset_editor:- raise Forbidden()- dataset = DatasetService.get_dataset(dataset_id, session)- if not dataset:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_operator_permission(current_user, dataset, session=session)-try:- document = DocumentService.rename_document(str(dataset_id), str(document_id), req_data.name, session)- except services.errors.document.DocumentIndexingError:- raise DocumentIndexingError("Cannot delete document during indexing.")-- return dump_response(DocumentResponse, document_response(document, session=session))+ result = application_services().knowledge.documents.rename_document(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id), name=req_data.name+ )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DocumentResponse, result)@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/website-sync")-class WebsiteDocumentSyncApi(DocumentResource):- @setup_required- @login_required- @account_initialization_required+class WebsiteDocumentSyncApi(Resource):@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def get(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- document_id: UUID,- ):+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):"""sync website document."""- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if not dataset:- raise NotFound("Dataset not found.")- if not current_user.is_dataset_editor:- raise Forbidden()- document_id_str = str(document_id)- document = self.get_document(session, dataset.id, document_id_str, current_user, current_tenant_id)- if document.data_source_type != "website_crawl":- raise ValueError("Document is not a website document.")- # 403 if document is archived- if DocumentService.check_archived(document):- raise ArchivedDocumentImmutableError()- # sync document- DocumentService.sync_website_document(dataset, document, session)-+ try:+ application_services().knowledge.documents.sync_website(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/pipeline-execution-log")-class DocumentPipelineExecutionLogApi(DocumentResource):+class DocumentPipelineExecutionLogApi(Resource):@console_ns.response(200,"Pipeline execution log retrieved successfully",console_ns.models[DocumentPipelineExecutionLogResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- document_id: UUID,- ):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)-- document = self.get_document(session, dataset_id_str, document_id_str, current_user, current_tenant_id)- log = session.scalar(- select(DocumentPipelineExecutionLog)- .where(DocumentPipelineExecutionLog.document_id == document.id)- .order_by(DocumentPipelineExecutionLog.created_at.desc())- .limit(1)- )- if not log:- return DocumentPipelineExecutionLogResponse().model_dump(mode="json"), 200- return dump_response(- DocumentPipelineExecutionLogResponse,- {- "datasource_info": json.loads(log.datasource_info),- "datasource_type": log.datasource_type,- "input_data": log.input_data,- "datasource_node_id": log.datasource_node_id,- },- ), 200+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):+ try:+ result = application_services().knowledge.documents.get_execution_log(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)+ return dump_response(DocumentPipelineExecutionLogResponse, result), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/generate-summary")@@ -1671,15 +867,12 @@ class DocumentGenerateSummaryApi(Resource):@console_ns.response(400, "Invalid request or dataset configuration")@console_ns.response(403, "Permission denied")@console_ns.response(404, "Dataset not found")- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES, rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),)+ )@cloud_edition_billing_rate_limit_check("knowledge")- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(GenerateSummaryPayload)- def post(self, req_data: GenerateSummaryPayload, session: Session, current_user: Account, dataset_id: UUID):+ def post(self, req_data: GenerateSummaryPayload, request_context: RequestContext, dataset_id: UUID):"""Generate summary index for specified documents.@@ -1687,83 +880,19 @@ class DocumentGenerateSummaryApi(Resource):(indexing_technique must be 'high_quality' and summary_index_setting.enable must be true),then asynchronously generates summary indexes for the provided documents."""- dataset_id_str = str(dataset_id)-- # Get dataset- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")-- # Check permissions- if not current_user.is_dataset_editor:- raise Forbidden()-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- document_list = req_data.document_list-- if not document_list:- from werkzeug.exceptions import BadRequest-+ if not req_data.document_list:raise BadRequest("document_list cannot be empty.")-- # Check if dataset configuration supports summary generation- if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:- raise ValueError(- f"Summary generation is only available for 'high_quality' indexing technique. "- f"Current indexing technique: {dataset.indexing_technique}"- )-- summary_index_setting = dataset.summary_index_setting- if not summary_index_setting or not summary_index_setting.get("enable"):- raise ValueError("Summary index is not enabled for this dataset. Please enable it in the dataset settings.")-- # Verify all documents exist and belong to the dataset- documents = DocumentService.get_documents_by_ids(- DatasetRefService.create_dataset_ref(dataset), document_list, session- )-- if len(documents) != len(document_list):- found_ids = {doc.id for doc in documents}- missing_ids = set(document_list) - found_ids- raise NotFound(f"Some documents not found: {list(missing_ids)}")-- # Update need_summary to True for documents that don't have it set- # This handles the case where documents were created when summary_index_setting was disabled- documents_to_update = [doc for doc in documents if not doc.need_summary and doc.doc_form != "qa_model"]-- if documents_to_update:- document_ids_to_update = [str(doc.id) for doc in documents_to_update]- DocumentService.update_documents_need_summary(- dataset_id=dataset_id_str,- document_ids=document_ids_to_update,- session=session,- need_summary=True,- )-- # Dispatch async tasks for each document- for document in documents:- # Skip qa_model documents as they don't generate summaries- if document.doc_form == "qa_model":- logger.info("Skipping summary generation for qa_model document %s", document.id)- continue-- # Dispatch async task- generate_summary_index_task.delay(dataset_id_str, document.id)- logger.info(- "Dispatched summary generation task for document %s in dataset %s",- document.id,- dataset_id_str,+ try:+ application_services().knowledge.documents.generate_summary(+ request_context, dataset_id=str(dataset_id), document_ids=req_data.document_list)-+ except Exception as error:+ _raise_document_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/summary-status")-class DocumentSummaryStatusApi(DocumentResource):+class DocumentSummaryStatusApi(Resource):@console_ns.doc("get_document_summary_status")@console_ns.doc(description="Get summary index generation status for a document")@console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})@@ -1773,13 +902,8 @@ class DocumentSummaryStatusApi(DocumentResource):console_ns.models[DocumentSummaryStatusResponse.__name__],)@console_ns.response(404, "Document not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_user: Account, dataset_id: UUID, document_id: UUID):+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),))+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):"""Get summary index generation status for a document.@@ -1793,27 +917,10 @@ class DocumentSummaryStatusApi(DocumentResource):- timeout: Number of summaries that timed out- summaries: List of summary records with status and content preview"""- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)-- # Get dataset- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")-- # Check permissionstry:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- # Get summary status detail from service- from services.summary_index_service import SummaryIndexService-- result = SummaryIndexService.get_document_summary_status_detail(- document_id=document_id_str,- dataset_id=dataset_id_str,- session=session,- )-+ result = application_services().knowledge.documents.get_summary_status(+ request_context, dataset_id=str(dataset_id), document_id=str(document_id)+ )+ except Exception as error:+ _raise_document_error(error)return dump_response(DocumentSummaryStatusResponse, result), 200diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.pyindex fdaa47c13f5..44e35cf0fbe 100644--- a/api/controllers/console/datasets/datasets_segments.py+++ b/api/controllers/console/datasets/datasets_segments.py@@ -1,18 +1,10 @@-import uuid-from typing import Literal-from typing import cast as type_cast+from typing import Literal, Neverfrom uuid import UUID-from flask import requestfrom flask_restx import Resourcefrom pydantic import BaseModel, Field-from sqlalchemy import String, case, cast, func, literal, or_, select-from sqlalchemy.dialects.postgresql import JSONB-from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound-import services-from configs import dify_configfrom controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayloadfrom controllers.common.fields import SimpleResultResponsefrom controllers.common.rbac import DatasetId, RBACCheck@@ -22,7 +14,6 @@ from controllers.common.schema import (register_response_schema_models,register_schema_models,)-from controllers.common.session import with_sessionfrom controllers.console import console_nsfrom controllers.console.app.error import ProviderNotInitializeErrorfrom controllers.console.datasets.error import (@@ -30,22 +21,15 @@ from controllers.console.datasets.error import (ChildChunkIndexingError,InvalidActionError,)+from controllers.console.flask_admission import console_account_admissionfrom controllers.console.wraps import (RBACPermission,- account_initialization_required,cloud_edition_billing_knowledge_limit_check,cloud_edition_billing_rate_limit_check,cloud_edition_billing_resource_check,model_validate,- rbac_permission_required,- setup_required,- with_current_tenant_id,- with_current_user,)-from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError-from core.model_manager import ModelManager-from core.rag.index_processor.constant.index_type import IndexTechniqueType-from extensions.ext_redis import redis_client+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModelfrom fields.segment_fields import (ChildChunkDetailResponse,@@ -53,23 +37,39 @@ from fields.segment_fields import (ChildChunkResponse,SegmentDetailResponse,SegmentResponse,- segment_response_with_summary,- segment_responses_with_summaries,)-from graphon.model_runtime.entities.model_entities import ModelType-from libs.helper import dump_response, escape_like_pattern-from libs.login import login_required-from libs.pagination import paginate_query-from models import Account-from models.dataset import Dataset, Document, DocumentSegment-from models.model import UploadFile-from services.dataset_ref_service import DatasetRefService, SegmentRef-from services.dataset_service import DatasetService, DocumentService, SegmentService-from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs-from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError-from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError-from services.summary_index_service import SummaryIndexService-from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task+from libs.helper import dump_response+from machinery.context import RequestContext+from models.account import TenantAccountRole+from services.knowledge.entities.segments import ChildChunkUpdateArgs+from services.knowledge.segments.application import (+ ChildChunkDeleteIndexApplicationError,+ ChildChunkIndexingApplicationError,+ ChildChunkListFilter,+ ChildChunkNotFoundError,+ SegmentBatchImportDispatchError,+ SegmentBatchImportNotFoundError,+ SegmentDatasetModelUnavailableError,+ SegmentDatasetNotFoundError,+ SegmentDocumentIndexingError,+ SegmentDocumentNotFoundError,+ SegmentEmbeddingModelUnavailableError,+ SegmentInvalidFileTypeError,+ SegmentListFilter,+ SegmentNotFoundError,+ SegmentPermissionDeniedError,+ SegmentStatusUpdateError,+ SegmentUploadFileNotFoundError,+)++_DATASET_EDIT_ROLES = frozenset(+ {+ TenantAccountRole.OWNER,+ TenantAccountRole.ADMIN,+ TenantAccountRole.EDITOR,+ TenantAccountRole.DATASET_OPERATOR,+ }+)class SegmentListQuery(BaseModel):@@ -166,19 +166,32 @@ register_response_schema_models()-def _get_segment_for_document(- session: Session, dataset: Dataset, document: Document, segment_id: str-) -> tuple[SegmentRef, DocumentSegment]:- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref(dataset_ref, document)- if document_ref is None:- raise NotFound("Document not found.")-- segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)- segment = SegmentService.get_segment_by_ref(segment_ref, session=session)- if not segment:- raise NotFound("Segment not found.")- return segment_ref, segment+def _raise_segment_error(error: Exception) -> Never:+ if isinstance(error, SegmentDatasetNotFoundError):+ raise NotFound("Dataset not found.") from error+ if isinstance(error, SegmentDocumentNotFoundError):+ raise NotFound("Document not found.") from error+ if isinstance(error, SegmentNotFoundError):+ raise NotFound("Segment not found.") from error+ if isinstance(error, ChildChunkNotFoundError):+ raise NotFound("Child chunk not found.") from error+ if isinstance(error, SegmentUploadFileNotFoundError):+ raise NotFound("UploadFile not found.") from error+ if isinstance(error, SegmentPermissionDeniedError):+ raise Forbidden(str(error)) from error+ if isinstance(error, SegmentEmbeddingModelUnavailableError):+ raise ProviderNotInitializeError(str(error)) from error+ if isinstance(error, SegmentDatasetModelUnavailableError):+ raise ValueError(str(error)) from error+ if isinstance(error, (SegmentDocumentIndexingError, SegmentStatusUpdateError)):+ raise InvalidActionError(str(error)) from error+ if isinstance(error, SegmentInvalidFileTypeError):+ raise ValueError(str(error)) from error+ if isinstance(error, ChildChunkIndexingApplicationError):+ raise ChildChunkIndexingError(str(error)) from error+ if isinstance(error, ChildChunkDeleteIndexApplicationError):+ raise ChildChunkDeleteIndexError(str(error)) from error+ raise error@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")@@ -186,147 +199,60 @@ class DatasetDocumentSegmentListApi(Resource):@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)@console_ns.doc(params=query_params_from_model(SegmentListQuery))@console_ns.response(200, "Segments retrieved successfully", console_ns.models[ConsoleSegmentListResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):- dataset_id_str = str(dataset_id)- document_id_str = str(document_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)-- if not document:- raise NotFound("Document not found.")-+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),),+ )+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID):args = query_params_from_request(SegmentListQuery, list_fields=("status",))-- page = args.page- limit = min(args.limit, 100)- status_list = args.status- hit_count_gte = args.hit_count_gte- keyword = args.keyword-- query = (- select(DocumentSegment)- .where(- DocumentSegment.document_id == document_id_str,- DocumentSegment.tenant_id == current_tenant_id,- )- .order_by(DocumentSegment.position.asc())- )-- if status_list:- query = query.where(DocumentSegment.status.in_(status_list))-- if hit_count_gte is not None:- query = query.where(DocumentSegment.hit_count >= hit_count_gte)-- if keyword:- # Escape special characters in keyword to prevent SQL injection via LIKE wildcards- escaped_keyword = escape_like_pattern(keyword)- # Search in both content and keywords fields- # Use database-specific methods for JSON array search- if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql":- # PostgreSQL: Use jsonb_array_elements_text to properly handle Unicode/Chinese text- # Feed the set-returning function a JSON array in every row. Filtering in- # the subquery is not enough because PostgreSQL can still evaluate the- # SRF on scalar JSON before applying the predicate.- keywords_jsonb = cast(DocumentSegment.keywords, JSONB)- keywords_array = case(- (func.jsonb_typeof(keywords_jsonb) == "array", keywords_jsonb),- else_=cast(literal("[]"), JSONB),- )- keywords_condition = func.array_to_string(- func.array(- select(func.jsonb_array_elements_text(keywords_array))- .correlate(DocumentSegment)- .scalar_subquery()- ),- ",",- ).ilike(f"%{escaped_keyword}%", escape="\\")- else:- # MySQL: Cast JSON to string for pattern matching- # MySQL stores Chinese text directly in JSON without Unicode escaping- keywords_condition = cast(DocumentSegment.keywords, String).ilike(f"%{escaped_keyword}%", escape="\\")-- query = query.where(- or_(- DocumentSegment.content.ilike(f"%{escaped_keyword}%", escape="\\"),- keywords_condition,- )- )-- if args.enabled.lower() != "all":- if args.enabled.lower() == "true":- query = query.where(DocumentSegment.enabled == True)- elif args.enabled.lower() == "false":- query = query.where(DocumentSegment.enabled == False)-- segments = paginate_query(query, session=session, page=page, per_page=limit, max_per_page=100)-- segment_list = list(segments.items)- segment_ids = [segment.id for segment in segment_list]- summaries: dict[str, str | None] = {}- if segment_ids:- summary_records = SummaryIndexService.get_segments_summaries(- segment_ids=segment_ids, dataset_id=dataset_id_str, session=session+ try:+ result = application_services().knowledge.segments.list_segments(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ query=SegmentListFilter(+ page=args.page,+ limit=args.limit,+ statuses=tuple(args.status),+ hit_count_gte=args.hit_count_gte,+ enabled=args.enabled,+ keyword=args.keyword,+ ),)- summaries = {chunk_id: summary.summary_content for chunk_id, summary in summary_records.items()}-+ except Exception as error:+ _raise_segment_error(error)response = {- "data": segment_responses_with_summaries(segment_list, summaries, session=session),- "limit": limit,- "total": segments.total,- "total_pages": segments.pages,- "page": page,+ "data": result.items,+ "limit": result.limit,+ "total": result.total,+ "total_pages": result.total_pages,+ "page": result.page,}return dump_response(ConsoleSegmentListResponse, response), 200- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)@console_ns.doc(params=query_params_from_model(SegmentIdListQuery))@console_ns.response(204, "Segments deleted successfully")- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def delete(self, session: Session, current_user: Account, dataset_id: UUID, document_id: UUID):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- segment_ids = request.args.getlist("segment_id")-- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()+ def delete(+ self,+ request_context: RequestContext,+ dataset_id: UUID,+ document_id: UUID,+ ):+ args = query_params_from_request(SegmentIdListQuery, list_fields=("segment_id",))try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- SegmentService.delete_segments(segment_ids, document, dataset, session)+ application_services().knowledge.segments.delete_segments(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_ids=args.segment_id,+ )+ except Exception as error:+ _raise_segment_error(error)return "", 204@@ -334,141 +260,68 @@ class DatasetDocumentSegmentListApi(Resource):class DatasetDocumentSegmentApi(Resource):@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_ACTION)@console_ns.doc(params=query_params_from_model(SegmentIdListQuery))- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_sessiondef patch(self,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,action: Literal["enable", "disable"],):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()-- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # check embedding model setting- try:- model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)- model_manager.get_model_instance(- tenant_id=current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- segment_ids = request.args.getlist("segment_id")-- document_indexing_cache_key = f"document_{document.id}_indexing"- cache_result = redis_client.get(document_indexing_cache_key)- if cache_result is not None:- raise InvalidActionError("Document is being indexed, please try again later")+ if action not in ("enable", "disable"):+ raise InvalidActionError()+ args = query_params_from_request(SegmentIdListQuery, list_fields=("segment_id",))try:- SegmentService.update_segments_status(segment_ids, action, dataset, document, session)- except Exception as e:- raise InvalidActionError(str(e))+ application_services().knowledge.segments.change_segment_status(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_ids=args.segment_id,+ action=action,+ )+ except Exception as error:+ _raise_segment_error(error)return SimpleResultResponse(result="success").model_dump(mode="json"), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment")class DatasetDocumentSegmentAddApi(Resource):@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_knowledge_limit_check("add_segment")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[SegmentCreatePayload.__name__])@console_ns.response(200, "Segment created successfully", console_ns.models[SegmentDetailResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(SegmentCreatePayload)def post(self,req_data: SegmentCreatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- if not current_user.is_dataset_editor:- raise Forbidden()- # check embedding model setting- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- try:- model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)- model_manager.get_model_instance(- tenant_id=current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- # validate args- payload_dict = req_data.model_dump(exclude_none=True)- SegmentService.segment_create_args_validate(payload_dict, document)- segment = type_cast(DocumentSegment, SegmentService.create_segment(payload_dict, document, dataset, session))- summary = SummaryIndexService.get_segment_summary(- segment_id=segment.id, dataset_id=dataset_id_str, session=session- )+ result = application_services().knowledge.segments.create_segment(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ values=req_data.model_dump(exclude_none=True),+ )+ except Exception as error:+ _raise_segment_error(error)response = {- "data": segment_response_with_summary(- segment, summary.summary_content if summary else None, session=session- ),- "doc_form": document.doc_form,+ "data": result.data,+ "doc_form": result.doc_form,}return dump_response(SegmentDetailResponse, response), 200@@ -476,338 +329,173 @@ class DatasetDocumentSegmentAddApi(Resource):@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>")class DatasetDocumentSegmentUpdateApi(Resource):@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[SegmentUpdatePayload.__name__])@console_ns.response(200, "Segment updated successfully", console_ns.models[SegmentDetailResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(SegmentUpdatePayload)def patch(self,req_data: SegmentUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # check embedding model setting- try:- model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)- model_manager.get_model_instance(- tenant_id=current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- segment_id_str = str(segment_id)- _, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- # validate args- payload_dict = req_data.model_dump(exclude_none=True)- SegmentService.segment_create_args_validate(payload_dict, document)-- # Update segment (summary update with change detection is handled in SegmentService.update_segment)- segment = SegmentService.update_segment(- SegmentUpdateArgs.model_validate(req_data.model_dump(exclude_none=True)),- segment,- document,- dataset,- session,- )- summary = SummaryIndexService.get_segment_summary(- segment_id=segment.id, dataset_id=dataset_id_str, session=session- )+ result = application_services().knowledge.segments.update_segment(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ values=req_data.model_dump(exclude_none=True),+ )+ except Exception as error:+ _raise_segment_error(error)response = {- "data": segment_response_with_summary(- segment, summary.summary_content if summary else None, session=session- ),- "doc_form": document.doc_form,+ "data": result.data,+ "doc_form": result.doc_form,}return dump_response(SegmentDetailResponse, response), 200- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)@console_ns.response(204, "Segment deleted successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_sessiondef delete(self,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- segment_id_str = str(segment_id)- _, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- SegmentService.delete_segment(segment, document, dataset, session)+ application_services().knowledge.segments.delete_segment(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ )+ except Exception as error:+ _raise_segment_error(error)return "", 204-@console_ns.route(- "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/batch_import",- "/datasets/batch_import_status/<uuid:job_id>",-)+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/batch_import")class DatasetDocumentSegmentBatchImportApi(Resource):@console_ns.response(200, "Batch import started", console_ns.models[SegmentBatchImportStatusResponse.__name__])- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_knowledge_limit_check("add_segment")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[BatchImportPayload.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(BatchImportPayload)def post(self,req_data: BatchImportPayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if not dataset:- raise NotFound("Dataset not found.")-- # check document- document_id_str = str(document_id)- document_ref = DatasetRefService.create_document_ref_from_id(- DatasetRefService.create_dataset_ref(dataset), document_id_str- )- document = DatasetRefService.get_document_by_ref(document_ref, session=session)- if not document:- raise NotFound("Document not found.")-- upload_file_id = req_data.upload_file_id-- upload_file = session.scalar(select(UploadFile).where(UploadFile.id == upload_file_id).limit(1))- if not upload_file:- raise NotFound("UploadFile not found.")-- # check file type- if not upload_file.name or not upload_file.name.lower().endswith(".csv"):- raise ValueError("Invalid file type. Only CSV files are allowed")-try:- # async job- job_id = str(uuid.uuid4())- indexing_cache_key = f"segment_batch_import_{job_id}"- # send batch add segments task- redis_client.setnx(indexing_cache_key, "waiting")- batch_create_segment_to_index_task.delay(- job_id,- upload_file_id,- dataset_id_str,- document_id_str,- current_tenant_id,- current_user.id,+ result = application_services().knowledge.segments.start_batch_import(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ upload_file_id=req_data.upload_file_id,)- except Exception as e:- return {"error": str(e)}, 500- return dump_response(SegmentBatchImportStatusResponse, {"job_id": job_id, "job_status": "waiting"}), 200+ except SegmentBatchImportDispatchError as error:+ return {"error": str(error)}, 500+ except Exception as error:+ _raise_segment_error(error)+ return dump_response(SegmentBatchImportStatusResponse, result), 200++@console_ns.route("/datasets/batch_import_status/<uuid:job_id>")+class DatasetDocumentSegmentBatchImportStatusApi(Resource):@console_ns.response(200, "Batch import status", console_ns.models[SegmentBatchImportStatusResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- def get(self, job_id=None, dataset_id: UUID | None = None, document_id: UUID | None = None):- if job_id is None:- raise NotFound("The job does not exist.")- job_id = str(job_id)- indexing_cache_key = f"segment_batch_import_{job_id}"- cache_result = redis_client.get(indexing_cache_key)- if cache_result is None:- raise ValueError("The job does not exist.")-- response = {"job_id": job_id, "job_status": cache_result.decode()}- return dump_response(SegmentBatchImportStatusResponse, response), 200+ @console_account_admission()+ def get(self, _request_context: RequestContext, job_id: UUID):+ try:+ result = application_services().knowledge.segments.get_batch_import_status(str(job_id))+ except SegmentBatchImportNotFoundError as error:+ raise ValueError(str(error)) from error+ return dump_response(SegmentBatchImportStatusResponse, result), 200@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks")class ChildChunkAddApi(Resource):@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_knowledge_limit_check("add_segment")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.expect(console_ns.models[ChildChunkCreatePayload.__name__])@console_ns.response(200, "Child chunk created successfully", console_ns.models[ChildChunkDetailResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(ChildChunkCreatePayload)def post(self,req_data: ChildChunkCreatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- if not current_user.is_dataset_editor:- raise Forbidden()try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- # check embedding model setting- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- try:- model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)- model_manager.get_model_instance(- tenant_id=current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- except LLMBadRequestError:- raise ProviderNotInitializeError(- "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."- )- except ProviderTokenNotInitError as ex:- raise ProviderNotInitializeError(ex.description)- segment_id_str = str(segment_id)- _, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- # validate args- try:- child_chunk = SegmentService.create_child_chunk(req_data.content, segment, document, dataset, session)- except ChildChunkIndexingServiceError as e:- raise ChildChunkIndexingError(str(e))+ child_chunk = application_services().knowledge.segments.create_child_chunk(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ content=req_data.content,+ )+ except Exception as error:+ _raise_segment_error(error)return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)@console_ns.doc(params=query_params_from_model(ChildChunkListQuery))@console_ns.response(200, "Child chunks retrieved successfully", console_ns.models[ChildChunkListResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- segment_id_str = str(segment_id)- _get_segment_for_document(session, dataset, document, segment_id_str)+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),),+ )+ def get(self, request_context: RequestContext, dataset_id: UUID, document_id: UUID, segment_id: UUID):args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)-- page = args.page- limit = min(args.limit, 100)- keyword = args.keyword-- child_chunks = SegmentService.get_child_chunks(- segment_id_str,- document_id_str,- dataset_id_str,- page,- limit,- keyword,- session=session,- )+ try:+ result = application_services().knowledge.segments.list_child_chunks(+ workspace_id=request_context.active_workspace_id,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ query=ChildChunkListFilter(page=args.page, limit=args.limit, keyword=args.keyword),+ )+ except Exception as error:+ _raise_segment_error(error)response = {- "data": child_chunks.items,- "total": child_chunks.total,- "total_pages": child_chunks.pages,- "page": page,- "limit": limit,+ "data": result.items,+ "total": result.total,+ "total_pages": result.total_pages,+ "page": result.page,+ "limit": result.limit,}return dump_response(ChildChunkListResponse, response), 200- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)@@ -817,47 +505,25 @@ class ChildChunkAddApi(Resource):console_ns.models[ChildChunkBatchUpdateResponse.__name__],)@console_ns.expect(console_ns.models[ChildChunkBatchUpdatePayload.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(ChildChunkBatchUpdatePayload)def patch(self,req_data: ChildChunkBatchUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- segment_id_str = str(segment_id)- _, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- # validate args- try:- child_chunks = SegmentService.update_child_chunks(req_data.chunks, segment, document, dataset, session)- except ChildChunkIndexingServiceError as e:- raise ChildChunkIndexingError(str(e))+ child_chunks = application_services().knowledge.segments.update_child_chunks(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ chunks=req_data.chunks,+ )+ except Exception as error:+ _raise_segment_error(error)return dump_response(ChildChunkBatchUpdateResponse, {"data": child_chunks}), 200@@ -865,111 +531,61 @@ class ChildChunkAddApi(Resource):"/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>")class ChildChunkUpdateApi(Resource):- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)@console_ns.response(204, "Child chunk deleted successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_sessiondef delete(self,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,child_chunk_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- segment_id_str = str(segment_id)- segment_ref, _ = _get_segment_for_document(session, dataset, document, segment_id_str)- child_chunk_id_str = str(child_chunk_id)- child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)- if not child_chunk:- raise NotFound("Child chunk not found.")try:- SegmentService.delete_child_chunk(child_chunk, dataset, session)- except ChildChunkDeleteIndexServiceError as e:- raise ChildChunkDeleteIndexError(str(e))+ application_services().knowledge.segments.delete_child_chunk(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ child_chunk_id=str(child_chunk_id),+ )+ except Exception as error:+ _raise_segment_error(error)return "", 204- @setup_required- @login_required- @account_initialization_required+ @console_account_admission(+ allowed_roles=_DATASET_EDIT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )@cloud_edition_billing_resource_check("vector_space")@cloud_edition_billing_rate_limit_check("knowledge")@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)@console_ns.expect(console_ns.models[ChildChunkUpdatePayload.__name__])@console_ns.response(200, "Child chunk updated successfully", console_ns.models[ChildChunkDetailResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session@model_validate(ChildChunkUpdatePayload)def patch(self,req_data: ChildChunkUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,+ request_context: RequestContext,dataset_id: UUID,document_id: UUID,segment_id: UUID,child_chunk_id: UUID,):- # check dataset- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if not dataset:- raise NotFound("Dataset not found.")- # check user's model setting- DatasetService.check_dataset_model_setting(dataset)- # check document- document_id_str = str(document_id)- document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)- if not document:- raise NotFound("Document not found.")- # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor- if not current_user.is_dataset_editor:- raise Forbidden()- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- segment_id_str = str(segment_id)- segment_ref, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- child_chunk_id_str = str(child_chunk_id)- child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)- if not child_chunk:- raise NotFound("Child chunk not found.")- # validate argstry:- child_chunk = SegmentService.update_child_chunk(- req_data.content, child_chunk, segment, document, dataset, session+ child_chunk = application_services().knowledge.segments.update_child_chunk(+ request_context,+ dataset_id=str(dataset_id),+ document_id=str(document_id),+ segment_id=str(segment_id),+ child_chunk_id=str(child_chunk_id),+ content=req_data.content,)- except ChildChunkIndexingServiceError as e:- raise ChildChunkIndexingError(str(e))+ except Exception as error:+ _raise_segment_error(error)return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200diff --git a/api/controllers/console/datasets/error.py b/api/controllers/console/datasets/error.pyindex ac09ec16b27..924bd391784 100644--- a/api/controllers/console/datasets/error.py+++ b/api/controllers/console/datasets/error.py@@ -25,6 +25,12 @@ class InvalidActionError(BaseHTTPException):code = 400+class DatasetAccessDeniedRequestError(BaseHTTPException):+ error_code = "dataset_access_denied"+ description = "Dataset access denied."+ code = 403++class DocumentAlreadyFinishedError(BaseHTTPException):error_code = "document_already_finished"description = "The document has been processed. Please refresh the page or go to the document details."diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.pyindex fd93100d9fc..1ceb31469d9 100644--- a/api/controllers/console/datasets/external.py+++ b/api/controllers/console/datasets/external.py@@ -1,16 +1,13 @@from __future__ import annotations-from dataclasses import dataclassfrom datetime import datetime-from typing import Any+from typing import Any, Neverfrom uuid import UUIDfrom flask_restx import Resourcefrom pydantic import AliasChoices, BaseModel, Field, field_validator-from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, InternalServerError, NotFound-import servicesfrom controllers.common.fields import UsageCountResponsefrom controllers.common.rbac import DatasetId, RBACCheck, Workspacefrom controllers.common.schema import (@@ -18,31 +15,42 @@ from controllers.common.schema import (register_response_schema_models,register_schema_models,)-from controllers.common.session import with_sessionfrom controllers.console import console_nsfrom controllers.console.datasets.error import DatasetNameDuplicateError+from controllers.console.flask_admission import console_account_admissionfrom controllers.console.wraps import (RBACPermission,- account_initialization_required,- edit_permission_required,model_validate,- rbac_permission_required,- setup_required,- with_current_tenant_id,- with_current_user,)+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModel-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source+from fields.dataset_fields import (+ DatasetDetailResponse,+)from libs.helper import dump_response-from libs.login import login_required-from libs.pagination import clamp_pagination-from models import Account-from models.dataset import ExternalKnowledgeApis-from services.dataset_service import DatasetService-from services.enterprise import rbac_service as enterprise_rbac_service+from machinery.context import RequestContext+from models.account import TenantAccountRolefrom services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload-from services.external_knowledge_service import ExternalDatasetService-from services.hit_testing_service import HitTestingService+from services.errors.dataset import DatasetNameDuplicateError as DatasetNameDuplicateFailure+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.external.application import ExternalHitTestingError, ExternalTemplateNotFoundError++_DATASET_EDIT_ROLES = frozenset(+ {TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR, TenantAccountRole.DATASET_OPERATOR}+)+_EXTERNAL_CONNECT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR})+++def _raise_external_error(error: Exception) -> Never:+ if isinstance(error, (DatasetNotFoundError, ExternalTemplateNotFoundError)):+ raise NotFound(str(error)) from error+ if isinstance(error, DatasetAccessDeniedError):+ raise Forbidden(str(error)) from error+ if isinstance(error, DatasetNameDuplicateFailure):+ raise DatasetNameDuplicateError() from error+ if isinstance(error, ExternalHitTestingError):+ raise InternalServerError(str(error)) from error+ raise errorclass ExternalKnowledgeApiPayload(BaseModel):@@ -85,28 +93,6 @@ class ExternalKnowledgeApiResponse(ResponseModel):return value-@dataclass(frozen=True)-class ExternalKnowledgeApiResponseSource:- external_knowledge_api: ExternalKnowledgeApis- session: Session-- @property- def dataset_bindings(self) -> Any:- return self.external_knowledge_api.get_dataset_bindings(session=self.session)-- def __getattr__(self, name: str) -> Any:- return getattr(self.external_knowledge_api, name) # guard-ignore: no-new-getattr -- delegates model fields---def external_knowledge_api_response(- external_knowledge_api: ExternalKnowledgeApis, *, session: Session-) -> ExternalKnowledgeApiResponse:- return ExternalKnowledgeApiResponse.model_validate(- ExternalKnowledgeApiResponseSource(external_knowledge_api=external_knowledge_api, session=session),- from_attributes=True,- )--class ExternalKnowledgeApiListResponse(ResponseModel):data: list[ExternalKnowledgeApiResponse]has_more: bool@@ -161,25 +147,13 @@ class ExternalApiTemplateListApi(Resource):"External API templates retrieved successfully",console_ns.models[ExternalKnowledgeApiListResponse.__name__],)- @setup_required- @login_required- @with_current_tenant_id- @account_initialization_required- @with_session(write=False)+ @console_account_admission()@model_validate(ExternalApiTemplateListQuery)- def get(self, req_data: ExternalApiTemplateListQuery, session: Session, current_tenant_id: str):-- effective_page, effective_limit = clamp_pagination(req_data.page, req_data.limit, 100)- external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(- effective_page, effective_limit, current_tenant_id, req_data.keyword, session=session+ def get(self, req_data: ExternalApiTemplateListQuery, request_context: RequestContext):+ result = application_services().knowledge.external.list_templates(+ request_context, page=req_data.page, limit=req_data.limit, keyword=req_data.keyword)- return ExternalKnowledgeApiListResponse(- data=[external_knowledge_api_response(item, session=session) for item in external_knowledge_apis],- has_more=effective_page * effective_limit < total,- limit=effective_limit,- total=total,- page=effective_page,- ).model_dump(mode="json"), 200+ return dump_response(ExternalKnowledgeApiListResponse, result), 200@console_ns.doc("create_external_api_template")@console_ns.doc(description="Create external knowledge API template")@@ -190,48 +164,16 @@ class ExternalApiTemplateListApi(Resource):console_ns.models[ExternalKnowledgeApiResponse.__name__],)@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @with_session+ @console_account_admission(allowed_roles=_DATASET_EDIT_ROLES)@model_validate(ExternalKnowledgeApiPayload)- def post(- self,- req_data: ExternalKnowledgeApiPayload,- session: Session,- current_tenant_id: str,- current_user: Account,- ):-- ExternalDatasetService.validate_api_list(req_data.settings)-- # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator- if not current_user.is_dataset_editor:- raise Forbidden()-+ def post(self, req_data: ExternalKnowledgeApiPayload, request_context: RequestContext):try:- external_knowledge_api = ExternalDatasetService.create_external_knowledge_api(- tenant_id=current_tenant_id,- user_id=current_user.id,- args=req_data.model_dump(),- session=session,+ result = application_services().knowledge.external.create_template(+ request_context, name=req_data.name, settings=req_data.settings)- except services.errors.dataset.DatasetNameDuplicateError:- raise DatasetNameDuplicateError()-- # if dify_config.RBAC_ENABLED:- # enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist(- # current_tenant_id,- # current_user.id,- # external_knowledge_api.id,- # enterprise_rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=True),- # )- # initialize_created_app_rbac_access_task.delay(- # current_tenant_id, current_user.id, dataset_id=external_knowledge_api.id)-- return external_knowledge_api_response(external_knowledge_api, session=session).model_dump(mode="json"), 201+ except Exception as error:+ _raise_external_error(error)+ return dump_response(ExternalKnowledgeApiResponse, result), 201@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")@@ -245,20 +187,15 @@ class ExternalApiTemplateApi(Resource):console_ns.models[ExternalKnowledgeApiResponse.__name__],)@console_ns.response(404, "Template not found")- @setup_required- @login_required- @account_initialization_required- @with_current_tenant_id- @with_session- def get(self, session: Session, current_tenant_id: str, external_knowledge_api_id: UUID):- external_knowledge_api_id_str = str(external_knowledge_api_id)- external_knowledge_api = ExternalDatasetService.get_external_knowledge_api(- external_knowledge_api_id=external_knowledge_api_id_str, tenant_id=current_tenant_id, session=session- )- if external_knowledge_api is None:- raise NotFound("API template not found.")-- return external_knowledge_api_response(external_knowledge_api, session=session).model_dump(mode="json"), 200+ @console_account_admission()+ def get(self, request_context: RequestContext, external_knowledge_api_id: UUID):+ try:+ result = application_services().knowledge.external.get_template(+ request_context, template_id=str(external_knowledge_api_id)+ )+ except Exception as error:+ _raise_external_error(error)+ return dump_response(ExternalKnowledgeApiResponse, result), 200@console_ns.doc("update_external_api_template")@console_ns.doc(description="Update external knowledge API template")@@ -270,51 +207,31 @@ class ExternalApiTemplateApi(Resource):console_ns.models[ExternalKnowledgeApiResponse.__name__],)@console_ns.response(404, "Template not found")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @with_current_tenant_id- @with_session+ @console_account_admission()@model_validate(ExternalKnowledgeApiPayload)def patch(- self,- req_data: ExternalKnowledgeApiPayload,- session: Session,- current_tenant_id: str,- current_user: Account,- external_knowledge_api_id: UUID,+ self, req_data: ExternalKnowledgeApiPayload, request_context: RequestContext, external_knowledge_api_id: UUID):- external_knowledge_api_id_str = str(external_knowledge_api_id)-- ExternalDatasetService.validate_api_list(req_data.settings)-- external_knowledge_api = ExternalDatasetService.update_external_knowledge_api(- tenant_id=current_tenant_id,- user_id=current_user.id,- external_knowledge_api_id=external_knowledge_api_id_str,- args=req_data.model_dump(),- session=session,- )-- return external_knowledge_api_response(external_knowledge_api, session=session).model_dump(mode="json"), 200+ try:+ result = application_services().knowledge.external.update_template(+ request_context,+ template_id=str(external_knowledge_api_id),+ name=req_data.name,+ settings=req_data.settings,+ )+ except Exception as error:+ _raise_external_error(error)+ return dump_response(ExternalKnowledgeApiResponse, result), 200- @setup_required- @login_required- @account_initialization_required@console_ns.response(204, "External knowledge API deleted successfully")- @with_current_user- @with_current_tenant_id- @with_session- def delete(self, session: Session, current_tenant_id: str, current_user: Account, external_knowledge_api_id: UUID):- external_knowledge_api_id_str = str(external_knowledge_api_id)-- if not (current_user.has_edit_permission or current_user.is_dataset_operator):- raise Forbidden()-- ExternalDatasetService.delete_external_knowledge_api(- current_tenant_id, external_knowledge_api_id_str, session=session- )+ @console_account_admission(allowed_roles=_DATASET_EDIT_ROLES)+ def delete(self, request_context: RequestContext, external_knowledge_api_id: UUID):+ try:+ application_services().knowledge.external.delete_template(+ request_context, template_id=str(external_knowledge_api_id)+ )+ except Exception as error:+ _raise_external_error(error)return "", 204@@ -324,18 +241,12 @@ class ExternalApiUseCheckApi(Resource):@console_ns.doc(description="Check if external knowledge API is being used")@console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})@console_ns.response(200, "Usage check completed successfully", console_ns.models[UsageCountResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @with_current_tenant_id- @with_session- def get(self, session: Session, current_tenant_id: str, external_knowledge_api_id: UUID):- external_knowledge_api_id_str = str(external_knowledge_api_id)-- external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(- external_knowledge_api_id_str, current_tenant_id, session=session+ @console_account_admission()+ def get(self, request_context: RequestContext, external_knowledge_api_id: UUID):+ is_using, count = application_services().knowledge.external.template_usage(+ request_context, template_id=str(external_knowledge_api_id))- return UsageCountResponse(is_using=external_knowledge_api_is_using, count=count).model_dump(mode="json"), 200+ return dump_response(UsageCountResponse, {"is_using": is_using, "count": count}), 200@console_ns.route("/datasets/external")@@ -348,50 +259,17 @@ class ExternalDatasetCreateApi(Resource):)@console_ns.response(400, "Invalid parameters")@console_ns.response(403, "Permission denied")- @setup_required- @login_required- @account_initialization_required- @edit_permission_required- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()))- @with_current_user- @with_current_tenant_id- @with_session+ @console_account_admission(+ allowed_roles=_EXTERNAL_CONNECT_ROLES,+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()),),+ )@model_validate(ExternalDatasetCreatePayload)- def post(- self,- req_data: ExternalDatasetCreatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,- ):- # The role of the current user in the ta table must be admin, owner, or editor-- # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator- if not current_user.is_dataset_editor:- raise Forbidden()-+ def post(self, req_data: ExternalDatasetCreatePayload, request_context: RequestContext):try:- dataset = ExternalDatasetService.create_external_dataset(- tenant_id=current_tenant_id,- user_id=current_user.id,- args=req_data,- session=session,- )- except services.errors.dataset.DatasetNameDuplicateError:- raise DatasetNameDuplicateError()-- dataset_id_str = str(dataset.id)- permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(- str(current_tenant_id),- current_user.id,- [dataset_id_str],- session=session,- )- data = DatasetDetailResponse.model_validate(- dataset_detail_response_source(dataset, session=session)- ).model_dump(mode="json")- data["permission_keys"] = permission_keys_map.get(dataset_id_str, [])- return data, 201+ result = application_services().knowledge.external.create_dataset(request_context, payload=req_data)+ except Exception as error:+ _raise_external_error(error)+ return dump_response(DatasetDetailResponse, result), 201@console_ns.route("/datasets/<uuid:dataset_id>/external-hit-testing")@@ -407,36 +285,17 @@ class ExternalKnowledgeHitTestingApi(Resource):)@console_ns.response(404, "Dataset not found")@console_ns.response(400, "Invalid parameters")- @setup_required- @login_required- @account_initialization_required- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))- @with_session+ @console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()),))@model_validate(ExternalHitTestingPayload)- def post(self, req_data: ExternalHitTestingPayload, session: Session, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")-+ def post(self, req_data: ExternalHitTestingPayload, request_context: RequestContext, dataset_id: UUID):try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))-- HitTestingService.hit_testing_args_check(req_data.model_dump())-- try:- response = HitTestingService.external_retrieve(- session=session,- dataset=dataset,+ result = application_services().knowledge.external.hit_testing(+ request_context,+ dataset_id=str(dataset_id),query=req_data.query,- account=current_user,- external_retrieval_model=req_data.external_retrieval_model,- metadata_filtering_conditions=req_data.metadata_filtering_conditions,+ retrieval_model=req_data.external_retrieval_model,+ metadata_filters=req_data.metadata_filtering_conditions,)-- return dump_response(ExternalHitTestingResponse, response)- except Exception as e:- raise InternalServerError(str(e))+ except Exception as error:+ _raise_external_error(error)+ return dump_response(ExternalHitTestingResponse, result)diff --git a/api/controllers/console/datasets/hit_testing_base.py b/api/controllers/console/datasets/hit_testing_base.pyindex e211507d5ab..0e0a3114d98 100644--- a/api/controllers/console/datasets/hit_testing_base.py+++ b/api/controllers/console/datasets/hit_testing_base.py@@ -23,9 +23,9 @@ from graphon.model_runtime.errors.invoke import InvokeErrorfrom libs.login import resolve_account_fallbackfrom models.account import Accountfrom models.dataset import Dataset-from services.dataset_service import DatasetService-from services.entities.knowledge_entities.knowledge_entities import ExternalRetrievalModel, RetrievalModelfrom services.hit_testing_service import HitTestingService+from services.knowledge.dataset_service import DatasetService+from services.knowledge.entities.knowledge_entities import ExternalRetrievalModel, RetrievalModellogger = logging.getLogger(__name__)diff --git a/api/controllers/console/datasets/metadata.py b/api/controllers/console/datasets/metadata.pyindex 4aa6a9073e6..1b9255e87b8 100644--- a/api/controllers/console/datasets/metadata.py+++ b/api/controllers/console/datasets/metadata.py@@ -1,43 +1,47 @@-from typing import Literal+from collections.abc import Callable+from functools import wrapsfrom uuid import UUIDfrom flask_restx import Resource-from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound-import servicesfrom controllers.common.controller_schemas import MetadataUpdatePayloadfrom controllers.common.rbac import DatasetId, RBACCheckfrom controllers.common.schema import register_response_schema_models, register_schema_models-from controllers.common.session import with_sessionfrom controllers.console import console_ns-from controllers.console.wraps import (- RBACPermission,- account_initialization_required,- enterprise_license_required,- model_validate,- rbac_permission_required,- setup_required,- with_current_tenant_id,- with_current_user,-)+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import model_validate+from core.rbac import RBACPermission+from extensions.ext_application_services import application_servicesfrom fields.dataset_fields import (DatasetMetadataBuiltInFieldsResponse,DatasetMetadataListResponse,DatasetMetadataResponse,)from libs.helper import dump_response-from libs.login import login_required-from models.account import Account-from services.dataset_service import DatasetService-from services.entities.knowledge_entities.knowledge_entities import (+from machinery.context import RequestContext+from services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.entities.knowledge_entities import (DocumentMetadataOperation,MetadataArgs,MetadataDetail,MetadataOperationData,)-from services.errors.metadata import MetadataResourceNotFoundError-from services.metadata_service import MetadataService+++def metadata_errors[**P, T](method: Callable[P, T]) -> Callable[P, T]:+ @wraps(method)+ def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:+ try:+ return method(*args, **kwargs)+ except (DatasetNotFoundError, MetadataResourceNotFoundError) as error:+ raise NotFound(str(error)) from error+ except DatasetAccessDeniedError as error:+ raise Forbidden(str(error)) from error++ return wrapped+register_schema_models(console_ns, MetadataArgs, MetadataOperationData, MetadataUpdatePayload, DocumentMetadataOperation, MetadataDetail@@ -52,199 +56,111 @@ register_response_schema_models(@console_ns.route("/datasets/<uuid:dataset_id>/metadata")class DatasetMetadataCreateApi(Resource):- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(201, "Metadata created successfully", console_ns.models[DatasetMetadataResponse.__name__])@console_ns.expect(console_ns.models[MetadataArgs.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ @metadata_errors@model_validate(MetadataArgs)- def post(- self,- req_data: MetadataArgs,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- ):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_permission(dataset, current_user, session)-- metadata = MetadataService.create_metadata(- dataset_id_str, req_data, current_user, current_tenant_id, session=session- )+ def post(self, req_data: MetadataArgs, request_context: RequestContext, dataset_id: UUID):+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ metadata = service.create_metadata(ref, req_data, actor_id=request_context.account_id)return dump_response(DatasetMetadataResponse, metadata), 201- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(200, "Metadata retrieved successfully", console_ns.models[DatasetMetadataListResponse.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()))- @with_session(write=False)- def get(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")- try:- DatasetService.check_dataset_permission(dataset, current_user, session)- except services.errors.account.NoPermissionError as e:- raise Forbidden(str(e))- metadata = MetadataService.get_dataset_metadatas(dataset, session)- return dump_response(DatasetMetadataListResponse, metadata), 200+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_READONLY, DatasetId()),),+ )+ @metadata_errors+ def get(self, request_context: RequestContext, dataset_id: UUID):+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ return dump_response(DatasetMetadataListResponse, service.get_dataset_metadatas(ref)), 200@console_ns.route("/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")class DatasetMetadataApi(Resource):- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(200, "Metadata updated successfully", console_ns.models[DatasetMetadataResponse.__name__])@console_ns.expect(console_ns.models[MetadataUpdatePayload.__name__])- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ @metadata_errors@model_validate(MetadataUpdatePayload)def patch(- self,- req_data: MetadataUpdatePayload,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- metadata_id: UUID,+ self, req_data: MetadataUpdatePayload, request_context: RequestContext, dataset_id: UUID, metadata_id: UUID):- name = req_data.name-- dataset_id_str = str(dataset_id)- metadata_id_str = str(metadata_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_permission(dataset, current_user, session)-- metadata = MetadataService.update_metadata_name(dataset, metadata_id_str, name, current_user, session=session)+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ metadata = service.update_metadata_name(+ ref, str(metadata_id), req_data.name, actor_id=request_context.account_id+ )return dump_response(DatasetMetadataResponse, metadata), 200- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(204, "Metadata deleted successfully")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def delete(- self,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- metadata_id: UUID,- ):- dataset_id_str = str(dataset_id)- metadata_id_str = str(metadata_id)- dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, current_tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_permission(dataset, current_user, session)-- MetadataService.delete_metadata(dataset, metadata_id_str, session)- # Frontend callers only await success and invalidate metadata caches; no response body is consumed.+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ @metadata_errors+ def delete(self, request_context: RequestContext, dataset_id: UUID, metadata_id: UUID):+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ service.delete_metadata(ref, str(metadata_id))return "", 204@console_ns.route("/datasets/metadata/built-in")class DatasetMetadataBuiltInFieldApi(Resource):- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(200,"Built-in fields retrieved successfully",console_ns.models[DatasetMetadataBuiltInFieldsResponse.__name__],)- def get(self):- built_in_fields = MetadataService.get_built_in_fields()- return dump_response(DatasetMetadataBuiltInFieldsResponse, {"fields": built_in_fields}), 200+ @console_account_admission()+ @metadata_errors+ def get(self, request_context: RequestContext):+ fields = application_services().knowledge.metadata.get_built_in_fields()+ return dump_response(DatasetMetadataBuiltInFieldsResponse, {"fields": fields}), 200@console_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in/<string:action>")class DatasetMetadataBuiltInFieldActionApi(Resource):- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.response(204, "Action completed successfully")- @with_current_user- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session- def post(self, session: Session, current_user: Account, dataset_id: UUID, action: Literal["enable", "disable"]):- dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)- if dataset is None:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_permission(dataset, current_user, session)-- match action:- case "enable":- MetadataService.enable_built_in_field(dataset, session)- case "disable":- MetadataService.disable_built_in_field(dataset, session)- # Frontend callers only await success and invalidate metadata caches; no response body is consumed.+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ @metadata_errors+ def post(self, request_context: RequestContext, dataset_id: UUID, action: str):+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ if action == "enable":+ service.enable_built_in_field(ref)+ elif action == "disable":+ service.disable_built_in_field(ref)+ else:+ raise ValueError("Invalid action.")return "", 204@console_ns.route("/datasets/<uuid:dataset_id>/documents/metadata")class DocumentMetadataEditApi(Resource):- @setup_required- @login_required- @account_initialization_required- @enterprise_license_required@console_ns.expect(console_ns.models[MetadataOperationData.__name__])@console_ns.response(204,"Documents metadata updated successfully",)@console_ns.response(404, "Dataset, document, or metadata not found")- @with_current_user- @with_current_tenant_id- @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))- @with_session+ @console_account_admission(+ rbac_checks=(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()),),+ )+ @metadata_errors@model_validate(MetadataOperationData)- def post(- self,- req_data: MetadataOperationData,- session: Session,- current_tenant_id: str,- current_user: Account,- dataset_id: UUID,- ):- dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), current_tenant_id, session=session)- if dataset is None:- raise NotFound("Dataset not found.")- DatasetService.check_dataset_permission(dataset, current_user, session)-- try:- MetadataService.update_documents_metadata(dataset, req_data, current_user, session=session)- except MetadataResourceNotFoundError as exc:- raise NotFound(str(exc)) from exc-- # Frontend callers only await success and invalidate caches; no response body is consumed.+ def post(self, req_data: MetadataOperationData, request_context: RequestContext, dataset_id: UUID):+ service = application_services().knowledge.metadata+ ref = service.require_dataset(request_context, str(dataset_id))+ service.update_documents_metadata(ref, req_data, actor_id=request_context.account_id)return "", 204diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_auth.py b/api/controllers/console/datasets/rag_pipeline/datasource_auth.pyindex 672a8a3d83a..ab20f673d32 100644--- a/api/controllers/console/datasets/rag_pipeline/datasource_auth.py+++ b/api/controllers/console/datasets/rag_pipeline/datasource_auth.py@@ -24,6 +24,7 @@ from core.entities.provider_entities import ProviderConfigfrom core.plugin.entities.plugin_daemon import PluginOAuthAuthorizationUrlResponsefrom core.plugin.impl.oauth import OAuthHandlerfrom core.tools.entities.common_entities import I18nObject+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom fields.base import ResponseModelfrom graphon.model_runtime.errors.validate import CredentialsValidateFailedError@@ -32,7 +33,6 @@ from libs.login import login_requiredfrom models import Accountfrom models.enums import PermissionEnumfrom models.provider_ids import DatasourceProviderID-from services.datasource_provider_service import DatasourceProviderServicefrom services.plugin.oauth_service import OAuthProxyService@@ -176,7 +176,7 @@ class DatasourcePluginOAuthAuthorizationUrl(Resource):datasource_provider_id = DatasourceProviderID(provider_id)provider_name = datasource_provider_id.provider_nameplugin_id = datasource_provider_id.plugin_id- oauth_config = DatasourceProviderService().get_oauth_client(+ oauth_config = application_services().data_sources.providers.get_oauth_client(tenant_id=tenant_id,datasource_provider_id=datasource_provider_id,)@@ -246,7 +246,7 @@ class DatasourceOAuthCallback(Resource):tenant_id: str = context["tenant_id"]datasource_provider_id = DatasourceProviderID(provider_id)plugin_id = datasource_provider_id.plugin_id- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersoauth_client_params = datasource_provider_service.get_oauth_client(tenant_id=tenant_id,datasource_provider_id=datasource_provider_id,@@ -315,7 +315,7 @@ class DatasourceAuth(Resource):@model_validate(DatasourceCredentialPayload)def post(self, req_data: DatasourceCredentialPayload, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providerstry:datasource_provider_service.add_datasource_api_key_provider(@@ -342,14 +342,14 @@ class DatasourceAuth(Resource):@with_current_tenant_iddef get(self, current_tenant_id: str, user: Account, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasources = datasource_provider_service.list_datasource_credentials(tenant_id=current_tenant_id,provider=datasource_provider_id.provider_name,plugin_id=datasource_provider_id.plugin_id,user=user,- session=db.session(),+ credential_query=application_services().credential_queries,)return dump_response(DatasourceCredentialListResponse, {"result": datasources}), 200@@ -370,7 +370,7 @@ class DatasourceAuthDeleteApi(Resource):plugin_id = datasource_provider_id.plugin_idprovider_name = datasource_provider_id.provider_name- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.remove_datasource_credentials(tenant_id=current_tenant_id,auth_id=req_data.credential_id,@@ -397,7 +397,7 @@ class DatasourceAuthUpdateApi(Resource):def post(self, req_data: DatasourceCredentialUpdatePayload, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.update_datasource_credentials(tenant_id=current_tenant_id,auth_id=req_data.credential_id,@@ -422,9 +422,12 @@ class DatasourceAuthListApi(Resource):@with_current_user@with_current_tenant_iddef get(self, current_tenant_id: str, user: Account):- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasources = datasource_provider_service.get_all_datasource_credentials(- tenant_id=current_tenant_id, session=db.session(), user=user+ tenant_id=current_tenant_id,+ session=db.session(),+ user=user,+ credential_query=application_services().credential_queries,)return dump_response(DatasourceProviderAuthListResponse, {"result": datasources}), 200@@ -442,9 +445,12 @@ class DatasourceHardCodeAuthListApi(Resource):@with_current_user@with_current_tenant_iddef get(self, current_tenant_id: str, user: Account):- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasources = datasource_provider_service.get_hard_code_datasource_credentials(- tenant_id=current_tenant_id, session=db.session(), user=user+ tenant_id=current_tenant_id,+ session=db.session(),+ user=user,+ credential_query=application_services().credential_queries,)return dump_response(DatasourceProviderAuthListResponse, {"result": datasources}), 200@@ -464,7 +470,7 @@ class DatasourceAuthOauthCustomClient(Resource):@model_validate(DatasourceCustomClientPayload)def post(self, req_data: DatasourceCustomClientPayload, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.setup_oauth_custom_client_params(tenant_id=current_tenant_id,datasource_provider_id=datasource_provider_id,@@ -482,7 +488,7 @@ class DatasourceAuthOauthCustomClient(Resource):@with_current_tenant_iddef delete(self, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.remove_oauth_custom_client_params(tenant_id=current_tenant_id,datasource_provider_id=datasource_provider_id,@@ -503,7 +509,7 @@ class DatasourceAuthDefaultApi(Resource):@model_validate(DatasourceDefaultPayload)def post(self, req_data: DatasourceDefaultPayload, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.set_default_datasource_provider(tenant_id=current_tenant_id,datasource_provider_id=datasource_provider_id,@@ -525,7 +531,7 @@ class DatasourceUpdateProviderNameApi(Resource):@model_validate(DatasourceUpdateNamePayload)def post(self, req_data: DatasourceUpdateNamePayload, current_tenant_id: str, provider_id: str):datasource_provider_id = DatasourceProviderID(provider_id)- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = application_services().data_sources.providersdatasource_provider_service.update_datasource_provider_name(tenant_id=current_tenant_id,datasource_provider_id=datasource_provider_id,diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.pyindex 07ae94d3924..af4993d9d51 100644--- a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py+++ b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py@@ -9,6 +9,7 @@ from controllers.common.schema import register_schema_modelsfrom controllers.console import console_nsfrom controllers.console.datasets.wraps import get_rag_pipelinefrom controllers.console.wraps import account_initialization_required, model_validate, setup_required, with_current_user+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom libs.login import login_requiredfrom models import Account@@ -51,5 +52,6 @@ class DataSourceContentPreviewApi(Resource):datasource_type=datasource_type,is_published=True,credential_id=req_data.credential_id,+ datasource_providers=application_services().data_sources.providers,)return preview_content, 200diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline.pyindex a3d49f149b2..a69f2fa37fb 100644--- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline.py+++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline.py@@ -35,10 +35,11 @@ from libs.helper import dump_responsefrom libs.login import login_requiredfrom models.account import Accountfrom models.dataset import Pipeline-from services.dataset_service import DatasetService+from repositories.knowledge.dataset_read_repository import get_pipeline_datasetfrom services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntityfrom services.errors.account import NoPermissionErrorfrom services.errors.rag_pipeline import RagPipelineResourceNotFoundError+from services.knowledge.dataset_service import DatasetServicefrom services.rag_pipeline.rag_pipeline import RagPipelineServicelogger: logging.Logger = logging.getLogger(__name__)@@ -232,7 +233,7 @@ class PublishCustomizedPipelineTemplateApi(Resource):pipeline: Pipeline,) -> tuple[str, int]:session = db.session()- dataset = pipeline.retrieve_dataset(session=session)+ dataset = get_pipeline_dataset(pipeline, session=session)if dataset is None:raise NotFound("Dataset not found")diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.pyindex 73ca05aba37..b6d1f6ff35a 100644--- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py+++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py@@ -17,14 +17,17 @@ from controllers.console.wraps import (with_current_user,)from extensions.ext_database import db-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source+from fields.dataset_fields import (+ DatasetDetailResponse,+)from libs.helper import dump_responsefrom libs.login import login_requiredfrom models import Accountfrom models.dataset import DatasetPermissionEnum-from services.dataset_service import DatasetPermissionService, DatasetServicefrom services.enterprise import rbac_service as enterprise_rbac_servicefrom services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, RagPipelineDatasetCreateEntity+from services.knowledge.dataset_read_service import load_dataset_detail+from services.knowledge.dataset_service import DatasetPermissionService, DatasetServicefrom services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService@@ -136,4 +139,4 @@ class CreateEmptyRagPipelineDatasetApi(Resource):),session=session,)- return dump_response(DatasetDetailResponse, dataset_detail_response_source(dataset, session=session)), 201+ return dump_response(DatasetDetailResponse, load_dataset_detail(dataset, session=session)), 201diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.pyindex 86f9ea3534e..52898b07c34 100644--- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py+++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py@@ -46,6 +46,7 @@ from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom factories import variable_factoryfrom fields.base import ResponseModel@@ -66,10 +67,10 @@ from models.dataset import Pipelinefrom models.model import EndUserfrom models.workflow import Workflowfrom services.agent.retirement_service import WorkflowAgentRetirementService-from services.dataset_service import DatasetServicefrom services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundErrorfrom services.errors.llm import InvokeRateLimitErrorfrom services.errors.rag_pipeline import RagPipelineResourceNotFoundError+from services.knowledge.dataset_service import DatasetServicefrom services.rag_pipeline.pipeline_generate_service import PipelineGenerateServicefrom services.rag_pipeline.rag_pipeline import RagPipelineServicefrom services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService@@ -303,7 +304,13 @@ class RagPipelineDraftRunIterationNodeApi(Resource):try:response = PipelineGenerateService.generate_single_iteration(- pipeline=pipeline, user=current_user, node_id=node_id, args=args, session=db.session(), streaming=True+ generator=application_services().knowledge.pipeline_generator,+ pipeline=pipeline,+ user=current_user,+ node_id=node_id,+ args=args,+ session=db.session(),+ streaming=True,)return helper.compact_generate_response(response)@@ -338,7 +345,13 @@ class RagPipelineDraftRunLoopNodeApi(Resource):try:response = PipelineGenerateService.generate_single_loop(- pipeline=pipeline, user=current_user, node_id=node_id, args=args, session=db.session(), streaming=True+ generator=application_services().knowledge.pipeline_generator,+ pipeline=pipeline,+ user=current_user,+ node_id=node_id,+ args=args,+ session=db.session(),+ streaming=True,)return helper.compact_generate_response(response)@@ -374,6 +387,7 @@ class DraftRagPipelineRunApi(Resource):try:response = PipelineGenerateService.generate(+ generator=application_services().knowledge.pipeline_generator,session=session,pipeline=pipeline,user=current_user,@@ -409,6 +423,7 @@ class PublishedRagPipelineRunApi(Resource):try:response = PipelineGenerateService.generate(+ generator=application_services().knowledge.pipeline_generator,session=session,pipeline=pipeline,user=current_user,@@ -450,6 +465,7 @@ class RagPipelinePublishedDatasourceNodeRunApi(Resource):datasource_type=req_data.datasource_type,is_published=False,credential_id=req_data.credential_id,+ datasource_providers=application_services().data_sources.providers,)))@@ -483,6 +499,7 @@ class RagPipelineDraftDatasourceNodeRunApi(Resource):datasource_type=req_data.datasource_type,is_published=False,credential_id=req_data.credential_id,+ datasource_providers=application_services().data_sources.providers,)))@@ -1016,7 +1033,9 @@ class DatasourceListApi(Resource):@account_initialization_required@with_current_tenant_iddef get(self, current_tenant_id: str):- providers = RagPipelineManageService.list_rag_pipeline_datasources(current_tenant_id)+ providers = RagPipelineManageService.list_rag_pipeline_datasources(+ current_tenant_id, datasource_providers=application_services().data_sources.providers+ )return RagPipelineDatasourceListResponse.model_validate(providers, from_attributes=True).model_dump(mode="json")diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.pyindex 9411d128195..4a9e9af0912 100644--- a/api/controllers/console/datasets/website.py+++ b/api/controllers/console/datasets/website.py@@ -7,8 +7,9 @@ from controllers.common.schema import query_params_from_model, register_responsefrom controllers.console import console_nsfrom controllers.console.datasets.error import WebsiteCrawlErrorfrom controllers.console.wraps import account_initialization_required, model_validate, setup_required+from extensions.ext_application_services import application_servicesfrom libs.login import login_required-from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService+from services.data_source.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteServiceclass WebsiteCrawlPayload(BaseModel):@@ -49,7 +50,7 @@ class WebsiteCrawlApi(Resource):# Crawl URL using typed requesttry:- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(providers=application_services().data_sources.providers).crawl_url(api_request)except Exception as e:raise WebsiteCrawlError(str(e))return result, 200@@ -77,7 +78,9 @@ class WebsiteCrawlStatusApi(Resource):# Get crawl status using typed requesttry:- result = WebsiteService.get_crawl_status_typed(api_request)+ result = WebsiteService(providers=application_services().data_sources.providers).get_crawl_status_typed(+ api_request+ )except Exception as e:raise WebsiteCrawlError(str(e))return result, 200diff --git a/api/controllers/console/flask_admission.py b/api/controllers/console/flask_admission.pyindex cb923fb4606..7d492c04ee2 100644--- a/api/controllers/console/flask_admission.py+++ b/api/controllers/console/flask_admission.py@@ -71,7 +71,7 @@ def console_account_admission[T, **P, R]() -> Callable[Concatenate[T, P], R | Response]:admitted_view = cloud_edition_billing_resource_check(billing_resource)(view) if billing_resource else view- @wraps(view, updated=())+ @wraps(view)def inject_request_context(self: T, /, *args: P.args, **kwargs: P.kwargs) -> R:account_with_tenant = current_account_with_tenant()account = account_with_tenant.accountdiff --git a/api/controllers/console/workspace/model_providers.py b/api/controllers/console/workspace/model_providers.pyindex dd67b9ec131..23f467ef8be 100644--- a/api/controllers/console/workspace/model_providers.py+++ b/api/controllers/console/workspace/model_providers.py@@ -33,7 +33,7 @@ from services.entities.model_provider_entities import (ModelProviderSummaryResponse,ProviderResponse,)-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServicefrom services.workspace_service import WorkspaceServicediff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.pyindex 85488610667..759887ba5f9 100644--- a/api/controllers/console/workspace/models.py+++ b/api/controllers/console/workspace/models.py@@ -25,6 +25,7 @@ from controllers.console.wraps import ()from core.entities.model_entities import DefaultModelSettingfrom core.entities.provider_entities import CredentialConfiguration+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom fields.base import ResponseModelfrom graphon.model_runtime.entities.model_entities import ModelType, ParameterRule@@ -38,7 +39,7 @@ from services.entities.model_provider_entities import (ProviderWithModelsResponse,)from services.model_load_balancing_service import ModelLoadBalancingService-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServicelogger = logging.getLogger(__name__)@@ -376,6 +377,7 @@ class ModelProviderModelCredentialApi(Resource):tenant_id=tenant_id,provider=provider,user=user,+ credential_query=application_services().credential_queries,)else:available_credentials = model_provider_service.get_provider_model_available_credentials(diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.pyindex 34b89d5e767..01b1e661289 100644--- a/api/controllers/console/workspace/tool_providers.py+++ b/api/controllers/console/workspace/tool_providers.py@@ -62,6 +62,7 @@ from core.tools.entities.tool_entities import (WorkflowToolParameterConfiguration,)from enums import DeploymentEdition+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom fields.base import ResponseModelfrom libs.helper import alphanumeric, dump_response, uuid_value@@ -664,9 +665,9 @@ class ToolBuiltinProviderGetCredentialsApi(Resource):BuiltinToolManageService.get_builtin_tool_provider_credentials(tenant_id=tenant_id,provider_name=provider,- session=db.session(),user=user,include_credential_ids=query.include_credential_ids or None,+ credential_query=application_services().credential_queries,),)@@ -1363,9 +1364,9 @@ class ToolBuiltinProviderGetCredentialInfoApi(Resource):BuiltinToolManageService.get_builtin_tool_provider_credential_info(tenant_id=tenant_id,provider=provider,- session=db.session(),user=user,include_credential_ids=query.include_credential_ids or None,+ credential_query=application_services().credential_queries,),)diff --git a/api/controllers/console/workspace/trigger_providers.py b/api/controllers/console/workspace/trigger_providers.pyindex 8d15c3a99b4..c9d6074aa44 100644--- a/api/controllers/console/workspace/trigger_providers.py+++ b/api/controllers/console/workspace/trigger_providers.py@@ -22,6 +22,7 @@ from core.trigger.entities.api_entities import ()from core.trigger.entities.entities import RequestLog, SubscriptionBuilderUpdaterfrom core.trigger.trigger_manager import TriggerManager+from extensions.ext_application_services import application_servicesfrom extensions.ext_database import dbfrom fields.base import ResponseModelfrom libs.helper import dump_response@@ -213,6 +214,7 @@ class TriggerSubscriptionListApi(Resource):tenant_id=tenant_id,provider_id=TriggerProviderID(provider),user=user,+ credential_query=application_services().credential_queries,),)except ValueError as e:diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.pyindex 00c8d18894d..9c6a2d6750d 100644--- a/api/controllers/service_api/dataset/dataset.py+++ b/api/controllers/service_api/dataset/dataset.py@@ -37,11 +37,6 @@ from core.plugin.impl.model_runtime_factory import create_plugin_provider_managefrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom extensions.ext_application_services import application_servicesfrom fields.base import ResponseModel-from fields.dataset_fields import (- DatasetDetailPrefetch,- build_dataset_detail_prefetch,- dataset_detail_response_source,-)from fields.dataset_fields import DatasetDetailResponse as BaseDatasetDetailResponsefrom graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import dump_response@@ -51,9 +46,11 @@ from models.account import Accountfrom models.dataset import DatasetPermissionEnumfrom models.enums import TagTypefrom models.provider_ids import ModelProviderID-from services.dataset_service import DatasetPermissionService, DatasetService, DocumentServicefrom services.enterprise import rbac_service as enterprise_rbac_service-from services.entities.knowledge_entities.knowledge_entities import (+from services.knowledge.dataset_read_service import load_dataset_detail, load_dataset_details+from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService+from services.knowledge.entities.datasets import DatasetDetailRecord+from services.knowledge.entities.knowledge_entities import (ExternalRetrievalModel,KnowledgeProvider,RetrievalModel,@@ -104,12 +101,8 @@ _SERVICE_DATASET_DETAIL_EXCLUDE = {"permission_keys"}_SERVICE_DATASET_LIST_EXCLUDE = {"data": {"__all__": _SERVICE_DATASET_DETAIL_EXCLUDE}}-def _dump_service_dataset_detail(- dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None-) -> dict[str, Any]:- return DatasetDetailResponse.model_validate(- dataset_detail_response_source(dataset, session=session, prefetch=prefetch), from_attributes=True- ).model_dump(+def _dump_service_dataset_detail(detail: DatasetDetailRecord) -> dict[str, Any]:+ return DatasetDetailResponse.model_validate(detail).model_dump(mode="json",exclude=_SERVICE_DATASET_DETAIL_EXCLUDE,)@@ -470,8 +463,7 @@ class DatasetListApi(DatasetApiResource):for embedding_model in embedding_models:model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")- prefetch = build_dataset_detail_prefetch(datasets, session=session)- data = [_dump_service_dataset_detail(dataset, session=session, prefetch=prefetch) for dataset in datasets]+ data = [_dump_service_dataset_detail(detail) for detail in load_dataset_details(datasets, session=session)]for item in data:if item["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY and item["embedding_model_provider"]:item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))@@ -571,7 +563,7 @@ class DatasetListApi(DatasetApiResource):enterprise_rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),)- return _dump_service_dataset_detail(dataset, session=session), 200+ return _dump_service_dataset_detail(load_dataset_detail(dataset, session=session)), 200@service_api_ns.route("/datasets/<uuid:dataset_id>")@@ -617,7 +609,7 @@ class DatasetApi(DatasetApiResource):DatasetService.check_dataset_permission(dataset, current_user, session)except services.errors.account.NoPermissionError as e:raise Forbidden(str(e))- data = _dump_service_dataset_detail(dataset, session=session)+ data = _dump_service_dataset_detail(load_dataset_detail(dataset, session=session))# check embedding settingassert isinstance(current_user, Account)cid = current_user.current_tenant_id@@ -734,7 +726,7 @@ class DatasetApi(DatasetApiResource):if dataset is None:raise NotFound("Dataset not found.")- result_data = _dump_service_dataset_detail(dataset, session=session)+ result_data = _dump_service_dataset_detail(load_dataset_detail(dataset, session=session))assert isinstance(current_user, Account)tenant_id = current_user.current_tenant_iddiff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.pyindex 50e5ebb3308..72870918ced 100644--- a/api/controllers/service_api/dataset/document.py+++ b/api/controllers/service_api/dataset/document.py@@ -69,25 +69,36 @@ from fields.document_fields import (DocumentMetadataResponse,DocumentResponse,DocumentStatusListResponse,- document_response,- document_responses,normalize_enum,)from libs.helper import dump_responsefrom libs.login import current_userfrom libs.pagination import clamp_pagination, paginate_queryfrom models.dataset import Dataset, Document-from services.dataset_service import DatasetService, DocumentService-from services.entities.knowledge_entities.knowledge_entities import (+from repositories.knowledge.dataset_read_repository import (+ get_dataset_creator,+ get_document_hit_count,+ get_document_process_rule,+ get_document_segment_count,+ get_latest_dataset_process_rule,+)+from services.feature_service import FeatureService+from services.file_service import FileService+from services.knowledge.dataset_read_service import (+ get_document_metadata_details,+ get_document_source_detail,+ load_document_detail,+ load_document_details,+)+from services.knowledge.dataset_service import DatasetService, DocumentService+from services.knowledge.entities.knowledge_entities import (DocForm,IndexingTechnique,KnowledgeConfig,ProcessRule,RetrievalModel,)-from services.feature_service import FeatureService-from services.file_service import FileService-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterclass DocumentTextCreatePayload(BaseModel):@@ -304,7 +315,7 @@ class DocumentAndBatchResponse(ResponseModel):def _document_and_batch_response(document: Document, batch: str, *, session: Session) -> dict[str, Any]:return dump_response(DocumentAndBatchResponse,- {"document": document_response(document, session=session), "batch": batch},+ {"document": load_document_detail(document, session=session), "batch": batch},)@@ -445,7 +456,7 @@ def _create_document_by_text(session: Session, tenant_id: str, dataset_id: UUID)dataset=dataset,knowledge_config=knowledge_config,account=current_user,- dataset_process_rule=dataset.get_latest_process_rule(session=session)+ dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,created_from="api",@@ -509,7 +520,7 @@ def _update_document_by_text(dataset=dataset,knowledge_config=knowledge_config,account=current_user,- dataset_process_rule=dataset.get_latest_process_rule(session=session)+ dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,created_from="api",@@ -822,7 +833,9 @@ class DocumentAddByFileApi(DatasetApiResource):knowledge_config = KnowledgeConfig.model_validate(args)DocumentService.document_create_args_validate(knowledge_config)- dataset_process_rule = dataset.get_latest_process_rule(session=session) if "process_rule" not in args else None+ dataset_process_rule = (+ get_latest_dataset_process_rule(dataset, session=session) if "process_rule" not in args else None+ )if not knowledge_config.original_document_id and not dataset_process_rule and not knowledge_config.process_rule:raise ValueError("process_rule is required.")@@ -830,7 +843,7 @@ class DocumentAddByFileApi(DatasetApiResource):documents, batch = DocumentService.save_document_with_dataset_id(dataset=dataset,knowledge_config=knowledge_config,- account=dataset.get_created_by_account(session=session),+ account=get_dataset_creator(dataset, session=session),dataset_process_rule=dataset_process_rule,created_from="api",session=session,@@ -909,8 +922,8 @@ def _update_document_by_file(documents, _ = DocumentService.save_document_with_dataset_id(dataset=dataset,knowledge_config=knowledge_config,- account=dataset.get_created_by_account(session=session),- dataset_process_rule=dataset.get_latest_process_rule(session=session)+ account=get_dataset_creator(dataset, session=session),+ dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,created_from="api",@@ -1046,9 +1059,7 @@ class DocumentListApi(DatasetApiResource):)response = {- "data": document_responses(documents, session=session),- # The result object already knows: it was built from the page the query- # ran with, while the requested values are only ever a request.+ "data": load_document_details(documents, session=session),"has_more": paginated_documents.has_next,"limit": paginated_documents.per_page,"total": paginated_documents.total,@@ -1303,7 +1314,7 @@ class DocumentApi(DatasetApiResource):summary_index_status = Nonehas_summary_index = dataset.summary_index_setting and dataset.summary_index_setting.get("enable") is Trueif has_summary_index and document.need_summary is True:- summary_index_status = SummaryIndexService.get_document_summary_index_status(+ summary_index_status = SummaryIndexAdapter.get_document_summary_index_status(document_id=document_id_str,dataset_id=dataset_id_str,tenant_id=tenant_id,@@ -1315,15 +1326,15 @@ class DocumentApi(DatasetApiResource):response = {"id": document.id,"doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),+ "doc_metadata": get_document_metadata_details(document, session=session),}elif metadata == "without":dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, session)response_exclude = {"doc_type", "doc_metadata"}- document_process_rule = document.get_dataset_process_rule(session=session)+ document_process_rule = get_document_process_rule(document, session=session)document_process_rules: Mapping[str, Any] = document_process_rule.to_dict() if document_process_rule else {}- data_source_info = document.get_data_source_detail_dict(session=session)- segment_count = document.get_segment_count(session=session)+ data_source_info = get_document_source_detail(document, session=session)+ segment_count = get_document_segment_count(document, session=session)response = {"id": document.id,"position": document.position,@@ -1348,7 +1359,7 @@ class DocumentApi(DatasetApiResource):"archived": document.archived,"segment_count": segment_count,"average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,- "hit_count": document.get_hit_count(session=session),+ "hit_count": get_document_hit_count(document, session=session),"display_status": document.display_status,"doc_form": document.doc_form,"doc_language": document.doc_language,@@ -1357,10 +1368,10 @@ class DocumentApi(DatasetApiResource):}else:dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, session)- document_process_rule = document.get_dataset_process_rule(session=session)+ document_process_rule = get_document_process_rule(document, session=session)document_process_rules = document_process_rule.to_dict() if document_process_rule else {}- data_source_info = document.get_data_source_detail_dict(session=session)- segment_count = document.get_segment_count(session=session)+ data_source_info = get_document_source_detail(document, session=session)+ segment_count = get_document_segment_count(document, session=session)response = {"id": document.id,"position": document.position,@@ -1384,10 +1395,10 @@ class DocumentApi(DatasetApiResource):"disabled_by": document.disabled_by,"archived": document.archived,"doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),+ "doc_metadata": get_document_metadata_details(document, session=session),"segment_count": segment_count,"average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,- "hit_count": document.get_hit_count(session=session),+ "hit_count": get_document_hit_count(document, session=session),"display_status": document.display_status,"doc_form": document.doc_form,"doc_language": document.doc_language,diff --git a/api/controllers/service_api/dataset/metadata.py b/api/controllers/service_api/dataset/metadata.pyindex 693ba8377b9..99c7c2458ec 100644--- a/api/controllers/service_api/dataset/metadata.py+++ b/api/controllers/service_api/dataset/metadata.py@@ -1,4 +1,4 @@-from typing import Literal, cast+from typing import Literalfrom uuid import UUIDfrom flask_login import current_user@@ -11,6 +11,7 @@ from controllers.common.session import with_sessionfrom controllers.console.wraps import model_validatefrom controllers.service_api import service_api_nsfrom controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check+from extensions.ext_application_services import application_servicesfrom fields.dataset_fields import (DatasetMetadataActionResponse,DatasetMetadataBuiltInFieldsResponse,@@ -18,16 +19,15 @@ from fields.dataset_fields import (DatasetMetadataResponse,)from libs.helper import dump_response-from models import Account-from services.dataset_service import DatasetService-from services.entities.knowledge_entities.knowledge_entities import (+from services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.dataset_service import DatasetService+from services.knowledge.entities.knowledge_entities import (DocumentMetadataOperation,MetadataArgs,MetadataDetail,MetadataOperationData,)-from services.errors.metadata import MetadataResourceNotFoundError-from services.metadata_service import MetadataService+from services.knowledge.resource_scope import DatasetRefBUILT_IN_METADATA_ACTION_PARAM = {"description": "`enable` to activate built-in metadata fields, `disable` to deactivate them.",@@ -87,12 +87,18 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):"""Create metadata for a dataset."""dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)+ dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)if dataset is None:raise NotFound("Dataset not found.")DatasetService.check_dataset_permission(dataset, current_user, session)- metadata = MetadataService.create_metadata(dataset_id_str, metadata_args, session=session)+ session.commit()+ try:+ metadata = application_services().knowledge.metadata.create_metadata(+ DatasetRef(str(tenant_id), dataset_id_str), metadata_args, actor_id=current_user.id+ )+ except MetadataResourceNotFoundError as exc:+ raise NotFound(str(exc)) from excreturn dump_response(DatasetMetadataResponse, metadata), 201@service_api_ns.doc(@@ -123,10 +129,16 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):def get(self, session: Session, tenant_id, dataset_id: UUID):"""Get all metadata for a dataset."""dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)+ dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)if dataset is None:raise NotFound("Dataset not found.")- metadata = MetadataService.get_dataset_metadatas(dataset, session)+ session.commit()+ try:+ metadata = application_services().knowledge.metadata.get_dataset_metadatas(+ DatasetRef(str(tenant_id), dataset_id_str)+ )+ except MetadataResourceNotFoundError as exc:+ raise NotFound(str(exc)) from excreturn dump_response(DatasetMetadataListResponse, metadata), 200@@ -168,9 +180,13 @@ class DatasetMetadataServiceApi(DatasetApiResource):raise NotFound("Dataset not found.")DatasetService.check_dataset_permission(dataset, current_user, session)- metadata = MetadataService.update_metadata_name(- dataset, metadata_id_str, payload.name, cast(Account, current_user), session=session- )+ session.commit()+ try:+ metadata = application_services().knowledge.metadata.update_metadata_name(+ DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str, payload.name, actor_id=current_user.id+ )+ except MetadataResourceNotFoundError as exc:+ raise NotFound(str(exc)) from excreturn dump_response(DatasetMetadataResponse, metadata), 200@service_api_ns.doc(@@ -206,7 +222,13 @@ class DatasetMetadataServiceApi(DatasetApiResource):raise NotFound("Dataset not found.")DatasetService.check_dataset_permission(dataset, current_user, session)- MetadataService.delete_metadata(dataset, metadata_id_str, session)+ session.commit()+ try:+ application_services().knowledge.metadata.delete_metadata(+ DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str+ )+ except MetadataResourceNotFoundError as exc:+ raise NotFound(str(exc)) from excreturn "", 204@@ -239,7 +261,7 @@ class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource):)def get(self, tenant_id, dataset_id: UUID):"""Get all built-in metadata fields."""- built_in_fields = MetadataService.get_built_in_fields()+ built_in_fields = application_services().knowledge.metadata.get_built_in_fields()return dump_response(DatasetMetadataBuiltInFieldsResponse, {"fields": built_in_fields}), 200@@ -271,16 +293,24 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):def post(self, session: Session, tenant_id, dataset_id: UUID, action: Literal["enable", "disable"]):"""Enable or disable built-in metadata field."""dataset_id_str = str(dataset_id)- dataset = DatasetService.get_dataset(dataset_id_str, session)+ dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)if dataset is None:raise NotFound("Dataset not found.")DatasetService.check_dataset_permission(dataset, current_user, session)- match action:- case "enable":- MetadataService.enable_built_in_field(dataset, session)- case "disable":- MetadataService.disable_built_in_field(dataset, session)+ session.commit()+ try:+ match action:+ case "enable":+ application_services().knowledge.metadata.enable_built_in_field(+ DatasetRef(str(tenant_id), dataset_id_str)+ )+ case "disable":+ application_services().knowledge.metadata.disable_built_in_field(+ DatasetRef(str(tenant_id), dataset_id_str)+ )+ except MetadataResourceNotFoundError as exc:+ raise NotFound(str(exc)) from excreturn dump_response(DatasetMetadataActionResponse, {"result": "success"}), 200@@ -324,9 +354,10 @@ class DocumentMetadataEditServiceApi(DatasetApiResource):raise NotFound("Dataset not found.")DatasetService.check_dataset_permission(dataset, current_user, session)+ session.commit()try:- MetadataService.update_documents_metadata(- dataset, metadata_args, cast(Account, current_user), session=session+ application_services().knowledge.metadata.update_documents_metadata(+ DatasetRef(str(tenant_id), str(dataset_id)), metadata_args, actor_id=current_user.id)except MetadataResourceNotFoundError as exc:raise NotFound(str(exc)) from excdiff --git a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.pyindex 245466367b7..64c5ab76006 100644--- a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py+++ b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py@@ -32,6 +32,7 @@ from controllers.service_api.wraps import DatasetApiResourcefrom core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.entities.knowledge_entities import PipelineDataset, PipelineDocument+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModelfrom libs import helperfrom libs.helper import dump_response@@ -183,7 +184,11 @@ class DatasourcePluginsApi(DatasetApiResource):rag_pipeline_service = RagPipelineService(db.session())datasource_plugins: list[dict[Any, Any]] = rag_pipeline_service.get_datasource_plugins(- tenant_id=tenant_id, dataset_id=dataset_id_str, is_published=query.is_published+ tenant_id=tenant_id,+ dataset_id=dataset_id_str,+ is_published=query.is_published,+ credential_query=application_services().credential_queries,+ datasource_providers=application_services().data_sources.providers,)return dump_response(DatasourcePluginListResponse, datasource_plugins), 200@@ -247,6 +252,7 @@ class DatasourceNodeRunApi(DatasetApiResource):datasource_type=datasource_node_run_api_entity.datasource_type,is_published=datasource_node_run_api_entity.is_published,credential_id=datasource_node_run_api_entity.credential_id,+ datasource_providers=application_services().data_sources.providers,)))@@ -310,6 +316,7 @@ class PipelineRunApi(DatasetApiResource):pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)try:response: dict[Any, Any] | Generator[str, Any, None] = PipelineGenerateService.generate(+ generator=application_services().knowledge.pipeline_generator,session=session,pipeline=pipeline,user=current_user,diff --git a/api/controllers/service_api/dataset/segment.py b/api/controllers/service_api/dataset/segment.pyindex 921c59a0942..7d93859970c 100644--- a/api/controllers/service_api/dataset/segment.py+++ b/api/controllers/service_api/dataset/segment.py@@ -27,26 +27,26 @@ from controllers.service_api.wraps import (from core.errors.error import LLMBadRequestError, ProviderTokenNotInitErrorfrom core.model_manager import ModelManagerfrom core.rag.index_processor.constant.index_type import IndexTechniqueType+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModelfrom fields.segment_fields import (ChildChunkDetailResponse,ChildChunkListResponse,SegmentDetailResponse,SegmentResponse,- segment_response_with_summary,- segment_responses_with_summaries,)from graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import dump_responsefrom libs.login import current_account_with_tenantfrom models.dataset import Dataset, Document, DocumentSegment-from services.dataset_ref_service import DatasetRefService, SegmentRef-from services.dataset_service import DatasetService, DocumentService, SegmentService-from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgsfrom services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingErrorfrom services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceErrorfrom services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError-from services.summary_index_service import SummaryIndexService+from services.knowledge.dataset_read_service import load_segment_detail, load_segment_details+from services.knowledge.dataset_service import DatasetService, DocumentService, SegmentService+from services.knowledge.entities.segments import SegmentUpdateArgs+from services.knowledge.resource_scope import DatasetRef, SegmentRef+from services.knowledge.summaries.adapters import SummaryIndexAdapterclass SegmentCreateItemPayload(BaseModel):@@ -133,12 +133,11 @@ register_response_schema_models(def _get_segment_for_document(session: Session, dataset: Dataset, document: Document, segment_id: str) -> tuple[SegmentRef, DocumentSegment]:- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref(dataset_ref, document)- if document_ref is None:+ if document.tenant_id != dataset.tenant_id or document.dataset_id != dataset.id:raise NotFound("Document not found.")- segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ segment_ref = dataset_ref.document(document.id).segment(segment_id)segment = SegmentService.get_segment_by_ref(segment_ref, session=session)if not segment:raise NotFound("Segment not found.")@@ -230,17 +229,24 @@ class SegmentApi(DatasetApiResource):for args_item in segment_items:SegmentService.segment_create_args_validate(args_item, document)segments = cast(- list[DocumentSegment], SegmentService.multi_create_segment(segment_items, document, dataset, session)+ list[DocumentSegment],+ SegmentService.multi_create_segment(+ segment_items,+ document,+ dataset,+ session,+ mutations=application_services().knowledge.segments.mutations,+ ),)segment_ids = [segment.id for segment in segments]summaries: dict[str, str | None] = {}if segment_ids:- summary_records = SummaryIndexService.get_segments_summaries(+ summary_records = SummaryIndexAdapter.get_segments_summaries(segment_ids=segment_ids, dataset_id=dataset_id_str, session=session)summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}response = {- "data": segment_responses_with_summaries(segments, summaries, session=session),+ "data": load_segment_details(segments, summaries, session=session),"doc_form": document.doc_form,}return dump_response(SegmentCreateListResponse, response), 200@@ -322,13 +328,13 @@ class SegmentApi(DatasetApiResource):segment_ids = [segment.id for segment in segments]summaries: dict[str, str | None] = {}if segment_ids:- summary_records = SummaryIndexService.get_segments_summaries(+ summary_records = SummaryIndexAdapter.get_segments_summaries(segment_ids=segment_ids, dataset_id=dataset_id_str, session=session)summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}response = {- "data": segment_responses_with_summaries(segments, summaries, session=session),+ "data": load_segment_details(segments, summaries, session=session),"doc_form": document.doc_form,"total": total,"has_more": page * limit < total,@@ -380,7 +386,9 @@ class DatasetSegmentApi(DatasetApiResource):raise NotFound("Document not found.")segment_id_str = str(segment_id)_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- SegmentService.delete_segment(segment, document, dataset, session)+ SegmentService.delete_segment(+ segment, document, dataset, session, mutations=application_services().knowledge.segments.mutations+ )return "", 204@service_api_ns.doc(@@ -417,7 +425,7 @@ class DatasetSegmentApi(DatasetApiResource):document_id: UUID,segment_id: UUID,):- _, current_tenant_id = current_account_with_tenant()+ current_account, current_tenant_id = current_account_with_tenant()dataset_id_str = str(dataset_id)# check datasetdataset = session.scalar(@@ -451,14 +459,20 @@ class DatasetSegmentApi(DatasetApiResource):segment_id_str = str(segment_id)_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset, session)- summary = SummaryIndexService.get_segment_summary(+ updated_segment = SegmentService.update_segment(+ payload.segment,+ segment,+ document,+ dataset,+ session,+ actor_id=current_account.id,+ mutations=application_services().knowledge.segments.mutations,+ )+ summary = SummaryIndexAdapter.get_segment_summary(segment_id=updated_segment.id, dataset_id=dataset_id_str, session=session)response = {- "data": segment_response_with_summary(- updated_segment, summary.summary_content if summary else None, session=session- ),+ "data": load_segment_detail(updated_segment, summary.summary_content if summary else None, session=session),"doc_form": document.doc_form,}return dump_response(SegmentDetailResponse, response), 200@@ -510,13 +524,11 @@ class DatasetSegmentApi(DatasetApiResource):segment_id_str = str(segment_id)_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)- summary = SummaryIndexService.get_segment_summary(+ summary = SummaryIndexAdapter.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str, session=session)response = {- "data": segment_response_with_summary(- segment, summary.summary_content if summary else None, session=session- ),+ "data": load_segment_detail(segment, summary.summary_content if summary else None, session=session),"doc_form": document.doc_form,}return dump_response(SegmentDetailResponse, response), 200@@ -567,7 +579,7 @@ class ChildChunkApi(DatasetApiResource):document_id: UUID,segment_id: UUID,):- _, current_tenant_id = current_account_with_tenant()+ current_account, current_tenant_id = current_account_with_tenant()"""Create child chunk."""dataset_id_str = str(dataset_id)# check dataset@@ -604,7 +616,15 @@ class ChildChunkApi(DatasetApiResource):raise ProviderNotInitializeError(ex.description)try:- child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset, session)+ child_chunk = SegmentService.create_child_chunk(+ payload.content,+ segment,+ document,+ dataset,+ session,+ actor_id=current_account.id,+ mutations=application_services().knowledge.segments.mutations,+ )except ChildChunkIndexingServiceError as e:raise ChildChunkIndexingError(str(e))@@ -744,7 +764,9 @@ class DatasetChildChunkApi(DatasetApiResource):raise NotFound("Child chunk not found.")try:- SegmentService.delete_child_chunk(child_chunk, dataset, session)+ SegmentService.delete_child_chunk(+ child_chunk, dataset, session, mutations=application_services().knowledge.segments.mutations+ )except ChildChunkDeleteIndexServiceError as e:raise ChildChunkDeleteIndexError(str(e))@@ -790,7 +812,7 @@ class DatasetChildChunkApi(DatasetApiResource):segment_id: UUID,child_chunk_id: UUID,):- current_account_with_tenant()+ current_account, _ = current_account_with_tenant()"""Update child chunk."""dataset_id_str = str(dataset_id)# check dataset@@ -817,7 +839,14 @@ class DatasetChildChunkApi(DatasetApiResource):try:child_chunk = SegmentService.update_child_chunk(- payload.content, child_chunk, segment, document, dataset, session+ payload.content,+ child_chunk,+ segment,+ document,+ dataset,+ session,+ actor_id=current_account.id,+ mutations=application_services().knowledge.segments.mutations,)except ChildChunkIndexingServiceError as e:raise ChildChunkIndexingError(str(e))diff --git a/api/controllers/service_api/workspace/models.py b/api/controllers/service_api/workspace/models.pyindex c2ea1a72cf5..0606906fe40 100644--- a/api/controllers/service_api/workspace/models.py+++ b/api/controllers/service_api/workspace/models.py@@ -7,7 +7,7 @@ from controllers.service_api.wraps import validate_dataset_tokenfrom fields.base import ResponseModelfrom graphon.model_runtime.utils.encoders import jsonable_encoderfrom services.entities.model_provider_entities import ProviderWithModelsResponse-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceMODEL_TYPE_PARAM = {"description": "Type of model to retrieve.",diff --git a/api/core/app/app_config/easy_ui_based_app/dataset/manager.py b/api/core/app/app_config/easy_ui_based_app/dataset/manager.pyindex 0cba996ad6f..802952b3d16 100644--- a/api/core/app/app_config/easy_ui_based_app/dataset/manager.py+++ b/api/core/app/app_config/easy_ui_based_app/dataset/manager.py@@ -12,7 +12,7 @@ from core.app.app_config.entities import (from core.entities.agent_entities import PlanningStrategyfrom core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDictfrom models.model import AppMode, AppModelConfigDict-from services.dataset_service import DatasetService+from services.knowledge.dataset_service import DatasetServiceclass DatasetConfigManager:diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.pyindex c3e1be5a95b..17f84ff313d 100644--- a/api/core/app/apps/pipeline/pipeline_generator.py+++ b/api/core/app/apps/pipeline/pipeline_generator.py@@ -22,7 +22,7 @@ from core.app.apps.draft_variable_saver import DraftVariableSaverFactoryfrom core.app.apps.exc import GenerateTaskStoppedErrorfrom core.app.apps.pipeline.pipeline_config_manager import PipelineConfigManagerfrom core.app.apps.pipeline.pipeline_queue_manager import PipelineQueueManager-from core.app.apps.pipeline.pipeline_runner import PipelineRunner+from core.app.apps.pipeline.pipeline_runner import PipelineDocumentStore, PipelineRunnerfrom core.app.apps.workflow.generate_response_converter import WorkflowAppGenerateResponseConverterfrom core.app.apps.workflow.generate_task_pipeline import WorkflowAppGenerateTaskPipelinefrom core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity@@ -52,7 +52,7 @@ from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrofrom models.dataset import Document, DocumentPipelineExecutionLog, Pipelinefrom models.enums import WorkflowRunTriggeredFromfrom models.model import AppMode-from services.datasource_provider_service import DatasourceProviderService+from services.data_source.provider_service import DatasourceProviderServicefrom services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxyfrom services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService@@ -60,6 +60,10 @@ logger = logging.getLogger(__name__)class PipelineGenerator(BaseAppGenerator):+ def __init__(self, *, documents: PipelineDocumentStore, datasource_providers: DatasourceProviderService) -> None:+ self._documents = documents+ self._datasource_providers = datasource_providers+@overloaddef generate(self,@@ -124,7 +128,7 @@ class PipelineGenerator(BaseAppGenerator):) -> Mapping[str, Any] | Generator[Mapping | str, None, None] | None:# Add null check for dataset- dataset = pipeline.retrieve_dataset(session)+ dataset = self._documents.get_pipeline_dataset(pipeline, session=session)if not dataset:raise ValueError("Pipeline dataset is required")inputs: Mapping[str, Any] = args["inputs"]@@ -140,8 +144,8 @@ class PipelineGenerator(BaseAppGenerator):)documents: list[Document] = []if invoke_from == InvokeFrom.PUBLISHED_PIPELINE and not is_retry and not args.get("original_document_id"):- from services.dataset_service import DocumentServicefrom services.feature_service import FeatureService+ from services.knowledge.dataset_service import DocumentServicefeatures = FeatureService.get_features(pipeline.tenant_id)DocumentService.check_document_creation_limits(len(datasource_info_list), features)@@ -407,7 +411,7 @@ class PipelineGenerator(BaseAppGenerator):pipeline=pipeline, workflow=workflow, start_node_id=args.get("start_node_id", "shared"))- dataset = pipeline.retrieve_dataset(session)+ dataset = self._documents.get_pipeline_dataset(pipeline, session=session)if not dataset:raise ValueError("Pipeline dataset is required")@@ -504,7 +508,7 @@ class PipelineGenerator(BaseAppGenerator):if args.get("inputs") is None:raise ValueError("inputs is required")- dataset = pipeline.retrieve_dataset(session)+ dataset = self._documents.get_pipeline_dataset(pipeline, session=session)if not dataset:raise ValueError("Pipeline dataset is required")@@ -635,6 +639,7 @@ class PipelineGenerator(BaseAppGenerator):system_user_id=system_user_id,workflow_execution_repository=workflow_execution_repository,workflow_node_execution_repository=workflow_node_execution_repository,+ documents=self._documents,)runner.run()@@ -779,8 +784,7 @@ class PipelineGenerator(BaseAppGenerator):tenant_id=pipeline.tenant_id,datasource_type=DatasourceProviderType(datasource_type),)- datasource_provider_service = DatasourceProviderService()- credentials = datasource_provider_service.get_datasource_credentials(+ credentials = self._datasource_providers.get_datasource_credentials(tenant_id=pipeline.tenant_id,provider=datasource_node_data.get("provider_name"),plugin_id=datasource_node_data.get("plugin_id"),diff --git a/api/core/app/apps/pipeline/pipeline_runner.py b/api/core/app/apps/pipeline/pipeline_runner.pyindex 651b670fcce..5a3f6142201 100644--- a/api/core/app/apps/pipeline/pipeline_runner.py+++ b/api/core/app/apps/pipeline/pipeline_runner.py@@ -1,6 +1,6 @@import loggingimport time-from typing import cast+from typing import Protocol, castfrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -28,14 +28,23 @@ from graphon.graph_events import GraphEngineEvent, GraphRunFailedEventfrom graphon.runtime import GraphRuntimeState, VariablePoolfrom graphon.variable_loader import VariableLoaderfrom graphon.variables.variables import RAGPipelineVariable, RAGPipelineVariableInput-from models.dataset import Pipeline+from models.dataset import Dataset, Pipelinefrom models.model import EndUserfrom models.workflow import Workflow-from services.dataset_ref_service import DatasetRefService, DocumentReflogger = logging.getLogger(__name__)+class PipelineDocumentStore(Protocol):+ """Knowledge state required by a pipeline run, with explicit session ownership."""++ def get_pipeline_dataset(self, pipeline: Pipeline, *, session: Session) -> Dataset | None: ...++ def exists(self, *, workspace_id: str, dataset_id: str, document_id: str) -> bool: ...++ def mark_failed(self, *, workspace_id: str, dataset_id: str, document_id: str, error: str) -> None: ...++class PipelineRunner(WorkflowBasedAppRunner):"""Pipeline Application Runner@@ -50,6 +59,7 @@ class PipelineRunner(WorkflowBasedAppRunner):system_user_id: str,workflow_execution_repository: WorkflowExecutionRepository,workflow_node_execution_repository: WorkflowNodeExecutionRepository,+ documents: PipelineDocumentStore,workflow_thread_pool_id: str | None = None,) -> None:"""@@ -68,6 +78,7 @@ class PipelineRunner(WorkflowBasedAppRunner):self._sys_user_id = system_user_idself._workflow_execution_repository = workflow_execution_repositoryself._workflow_node_execution_repository = workflow_node_execution_repository+ self._documents = documentsdef _get_app_id(self) -> str:return self.application_generate_entity.app_config.app_id@@ -98,7 +109,7 @@ class PipelineRunner(WorkflowBasedAppRunner):if not pipeline or pipeline.tenant_id != app_config.tenant_id:raise ValueError("Pipeline not found")- dataset = pipeline.retrieve_dataset(session)+ dataset = self._documents.get_pipeline_dataset(pipeline, session=session)if (not datasetor dataset.tenant_id != pipeline.tenant_id@@ -108,23 +119,20 @@ class PipelineRunner(WorkflowBasedAppRunner):document_id = self.application_generate_entity.document_idoriginal_document_id = self.application_generate_entity.original_document_id- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = (- DatasetRefService.create_document_ref_from_id(- dataset_ref,- document_id,- )- if document_id- else None- )- if document_ref and DatasetRefService.get_document_by_ref(document_ref, session=session) is None:+ dataset_workspace_id = dataset.tenant_id+ dataset_id = dataset.id+ if document_id and not self._documents.exists(+ workspace_id=dataset_workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ ):raise ValueError("Pipeline document not found")if original_document_id and original_document_id != document_id:- original_document_ref = DatasetRefService.create_document_ref_from_id(- dataset_ref,- original_document_id,- )- if DatasetRefService.get_document_by_ref(original_document_ref, session=session) is None:+ if not self._documents.exists(+ workspace_id=dataset_workspace_id,+ dataset_id=dataset_id,+ document_id=original_document_id,+ ):raise ValueError("Pipeline original document not found")workflow = self.get_workflow(session=session, pipeline=pipeline, workflow_id=app_config.workflow_id)@@ -237,7 +245,12 @@ class PipelineRunner(WorkflowBasedAppRunner):generator = workflow_entry.run()for event in generator:- self._update_document_status(event, document_ref)+ self._update_document_status(+ event,+ workspace_id=dataset_workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ )self._handle_event(workflow_entry, event)def get_workflow(self, session: Session, pipeline: Pipeline, workflow_id: str) -> Workflow | None:@@ -325,14 +338,21 @@ class PipelineRunner(WorkflowBasedAppRunner):return graph- def _update_document_status(self, event: GraphEngineEvent, document_ref: DocumentRef | None) -> None:+ def _update_document_status(+ self,+ event: GraphEngineEvent,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str | None,+ ) -> None:"""Set an owner-bound document to error after a failed graph run, if it exists."""- if not isinstance(event, GraphRunFailedEvent) or document_ref is None:+ if not isinstance(event, GraphRunFailedEvent) or document_id is None:return- with create_session() as session, session.begin():- document = DatasetRefService.get_document_by_ref(document_ref, session=session)- if document:- document.indexing_status = "error"- document.error = event.error or "Unknown error"- session.add(document)+ self._documents.mark_failed(+ workspace_id=workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ error=event.error or "Unknown error",+ )diff --git a/api/core/app/features/annotation_reply/annotation_reply.py b/api/core/app/features/annotation_reply/annotation_reply.pyindex 049f7ac2d5a..454a2394942 100644--- a/api/core/app/features/annotation_reply/annotation_reply.py+++ b/api/core/app/features/annotation_reply/annotation_reply.py@@ -10,7 +10,7 @@ from models.dataset import Datasetfrom models.enums import CollectionBindingType, ConversationFromSourcefrom models.model import AnnotationReplyEnabledConfig, App, Message, MessageAnnotation, load_annotation_reply_configfrom services.annotation_service import AppAnnotationService-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/core/datasource/datasource_manager.py b/api/core/datasource/datasource_manager.pyindex b96d98d3f9c..cdf563b7632 100644--- a/api/core/datasource/datasource_manager.py+++ b/api/core/datasource/datasource_manager.py@@ -1,5 +1,5 @@import logging-from collections.abc import Generator+from collections.abc import Generator, Mappingfrom threading import Lockfrom typing import Any, cast@@ -33,7 +33,6 @@ from graphon.file import File, FileTransferMethod, FileType, get_file_type_by_mifrom graphon.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEventfrom models.model import UploadFilefrom models.tools import ToolFile-from services.datasource_provider_service import DatasourceProviderServicelogger = logging.getLogger(__name__)_file_access_controller = DatabaseFileAccessController()@@ -147,9 +146,7 @@ class DatasourceManager:datasource_type: str,provider_id: str,tenant_id: str,- provider: str,- plugin_id: str,- credential_id: str,+ credentials: Mapping[str, Any],datasource_param: DatasourceParameter | None = None,online_drive_request: OnlineDriveDownloadFileParam | None = None,) -> Generator[DatasourceMessage, None, Any]:@@ -166,18 +163,10 @@ class DatasourceManager:datasource_type=ds_type,)- dsp_service = DatasourceProviderService()- credentials = dsp_service.get_datasource_credentials(- tenant_id=tenant_id,- provider=provider,- plugin_id=plugin_id,- credential_id=credential_id,- )-if ds_type == DatasourceProviderType.ONLINE_DOCUMENT:doc_runtime = cast(OnlineDocumentDatasourcePlugin, runtime)if credentials:- doc_runtime.runtime.credentials = credentials+ doc_runtime.runtime.credentials = dict(credentials)if datasource_param is None:raise ValueError("datasource_param is required for ONLINE_DOCUMENT streaming")inner_gen: Generator[DatasourceMessage, None, None] = doc_runtime.get_online_document_page_content(@@ -192,7 +181,7 @@ class DatasourceManager:elif ds_type == DatasourceProviderType.ONLINE_DRIVE:drive_runtime = cast(OnlineDriveDatasourcePlugin, runtime)if credentials:- drive_runtime.runtime.credentials = credentials+ drive_runtime.runtime.credentials = dict(credentials)if online_drive_request is None:raise ValueError("online_drive_request is required for ONLINE_DRIVE streaming")inner_gen = drive_runtime.online_drive_download_file(@@ -221,9 +210,7 @@ class DatasourceManager:datasource_type: str,provider_id: str,tenant_id: str,- provider: str,- plugin_id: str,- credential_id: str,+ credentials: Mapping[str, Any],parameters_for_log: dict[str, Any],datasource_info: dict[str, Any],variable_pool: Any,@@ -238,9 +225,7 @@ class DatasourceManager:datasource_type=datasource_type,provider_id=provider_id,tenant_id=tenant_id,- provider=provider,- plugin_id=plugin_id,- credential_id=credential_id,+ credentials=credentials,datasource_param=datasource_param,online_drive_request=online_drive_request,)diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.pydeleted file mode 100644index 2163a5c7fbc..00000000000--- a/api/core/indexing_runner.py+++ /dev/null@@ -1,938 +0,0 @@-import concurrent.futures-import json-import logging-import re-import threading-import time-import uuid-from collections.abc import Mapping-from typing import Any--from flask import Flask, current_app-from sqlalchemy import delete, func, select, update-from sqlalchemy.orm import Session-from sqlalchemy.orm.exc import ObjectDeletedError--from configs import dify_config-from core.credit_usage import CreditUsageCreatedBy-from core.db.session_factory import session_factory-from core.entities.knowledge_entities import IndexingEstimate, PreviewDetail, QAPreviewDetail-from core.errors.error import ProviderTokenNotInitError-from core.model_context import with_credit_usage_created_by-from core.model_manager import ModelInstance, ModelManager-from core.rag.cleaner.clean_processor import CleanProcessor-from core.rag.datasource.keyword.keyword_factory import Keyword-from core.rag.docstore.dataset_docstore import DatasetDocumentStore-from core.rag.embedding.token_counter import calculate_segment_token_counts-from core.rag.extractor.entity.datasource_type import DatasourceType-from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo-from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from core.rag.index_processor.index_processor_base import BaseIndexProcessor-from core.rag.index_processor.index_processor_factory import IndexProcessorFactory-from core.rag.models.document import ChildDocument, Document-from core.rag.splitter.fixed_text_splitter import (- EnhanceRecursiveCharacterTextSplitter,- FixedRecursiveCharacterTextSplitter,-)-from core.rag.splitter.text_splitter import TextSplitter-from core.tools.utils.web_reader_tool import get_image_upload_file_ids-from enums import DeploymentEdition-from extensions.ext_redis import redis_client-from extensions.ext_storage import storage-from extensions.otel import propagate_context-from graphon.model_runtime.entities.model_entities import ModelType-from libs import helper-from libs.datetime_utils import naive_utc_now-from models import Account-from models.dataset import AutomaticRulesConfig, ChildChunk, Dataset, DatasetProcessRule, DocumentSegment-from models.dataset import Document as DatasetDocument-from models.enums import DataSourceType, IndexingStatus, ProcessRuleMode, SegmentStatus-from models.model import UploadFile-from services.vector_space_admission_service import VectorSpaceAdmissionService--logger = logging.getLogger(__name__)---class IndexingRunner:- def __init__(- self,- *,- enforce_vector_space_admission: bool = False,- ):- self.storage = storage- self.enforce_vector_space_admission = enforce_vector_space_admission-- @staticmethod- def _get_model_manager(tenant_id: str) -> ModelManager:- return ModelManager.for_tenant(tenant_id=tenant_id)-- def _handle_indexing_error(self, document_id: str, error: Exception, session: Session) -> None:- """Handle indexing errors by updating document status."""- logger.exception("consume document failed")- document = session.get(DatasetDocument, document_id)- if document:- document.indexing_status = IndexingStatus.ERROR- error_message = getattr(error, "description", str(error))- document.error = str(error_message)- document.stopped_at = naive_utc_now()- session.flush()-- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def run(self, dataset_documents: list[DatasetDocument], session: Session):- """Run indexing with commits before slow transforms and parallel index workers.-- The phase commits keep document locks short and make newly created segments- visible to the worker sessions used for keyword and vector indexing.- """- vector_space_admission = VectorSpaceAdmissionService()- for dataset_document in dataset_documents:- document_id = dataset_document.id- try:- # Re-query the document to ensure it's bound to the current session- requeried_document = session.get(DatasetDocument, document_id)- if not requeried_document:- logger.warning("Document not found, skipping document id: %s", document_id)- continue-- # get dataset- dataset = session.get(Dataset, requeried_document.dataset_id)-- if not dataset:- raise ValueError("no dataset found")- # get the process rule- stmt = select(DatasetProcessRule).where(- DatasetProcessRule.id == requeried_document.dataset_process_rule_id- )- processing_rule = session.scalar(stmt)- if not processing_rule:- raise ValueError("no process rule found")- index_type = requeried_document.doc_form- index_processor = IndexProcessorFactory(index_type).init_index_processor()- # extract- text_docs = self._extract(index_processor, requeried_document, processing_rule.to_dict(), session)- session.commit()-- # transform- current_user = session.get(Account, requeried_document.created_by)- if not current_user:- raise ValueError("no current user found")- current_user.set_tenant_id_with_session(dataset.tenant_id, session=session)- documents = self._transform(- index_processor,- dataset,- text_docs,- requeried_document.doc_language,- processing_rule.to_dict(),- current_user=current_user,- session=session,- )- if self.enforce_vector_space_admission:- vector_space_admission.ensure_document_can_be_indexed(- dataset=dataset,- document_id=requeried_document.id,- doc_form=requeried_document.doc_form,- documents=documents,- include_summaries=bool(requeried_document.need_summary),- session=session,- )- token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)- total_tokens = sum(token_counts)- # save segment- self._load_segments(- session=session,- dataset=dataset,- dataset_document=requeried_document,- documents=documents,- token_counts=token_counts,- )- session.commit()-- # load- self._load(- session=session,- dataset=dataset,- dataset_document=requeried_document,- documents=documents,- total_tokens=total_tokens,- )- except DocumentIsPausedError:- raise DocumentIsPausedError(f"Document paused, document id: {document_id}")- except ProviderTokenNotInitError as e:- self._handle_indexing_error(document_id, e, session)- except ObjectDeletedError:- logger.warning("Document deleted, document id: %s", document_id)- except Exception as e:- self._handle_indexing_error(document_id, e, session)-- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def run_in_splitting_status(self, dataset_document: DatasetDocument, session: Session):- """Run the indexing process when the index_status is splitting."""- document_id = dataset_document.id- try:- # Re-query the document to ensure it's bound to the current session- requeried_document = session.get(DatasetDocument, document_id)- if not requeried_document:- logger.warning("Document not found: %s", document_id)- return-- # get dataset- dataset = session.get(Dataset, requeried_document.dataset_id)-- if not dataset:- raise ValueError("no dataset found")-- # get exist document_segment list and delete- document_segments = session.scalars(- select(DocumentSegment).where(- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == requeried_document.id,- )- ).all()-- for document_segment in document_segments:- session.delete(document_segment)- if requeried_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- # delete child chunks- session.execute(delete(ChildChunk).where(ChildChunk.segment_id == document_segment.id))- session.commit()- # get the process rule- stmt = select(DatasetProcessRule).where(DatasetProcessRule.id == requeried_document.dataset_process_rule_id)- processing_rule = session.scalar(stmt)- if not processing_rule:- raise ValueError("no process rule found")-- index_type = requeried_document.doc_form- index_processor = IndexProcessorFactory(index_type).init_index_processor()- # extract- text_docs = self._extract(index_processor, requeried_document, processing_rule.to_dict(), session)- session.commit()-- # transform- current_user = session.get(Account, requeried_document.created_by)- if not current_user:- raise ValueError("no current user found")- current_user.set_tenant_id_with_session(dataset.tenant_id, session=session)- documents = self._transform(- index_processor,- dataset,- text_docs,- requeried_document.doc_language,- processing_rule.to_dict(),- current_user=current_user,- session=session,- )- token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)- total_tokens = sum(token_counts)- # save segment- self._load_segments(- session=session,- dataset=dataset,- dataset_document=requeried_document,- documents=documents,- token_counts=token_counts,- )- session.commit()-- # load- self._load(- session=session,- dataset=dataset,- dataset_document=requeried_document,- documents=documents,- total_tokens=total_tokens,- )- except DocumentIsPausedError:- raise DocumentIsPausedError(f"Document paused, document id: {document_id}")- except ProviderTokenNotInitError as e:- self._handle_indexing_error(document_id, e, session)- except Exception as e:- self._handle_indexing_error(document_id, e, session)-- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def run_in_indexing_status(self, dataset_document: DatasetDocument, session: Session):- """Run the indexing process when the index_status is indexing."""- document_id = dataset_document.id- try:- # Re-query the document to ensure it's bound to the current session- requeried_document = session.get(DatasetDocument, document_id)- if not requeried_document:- logger.warning("Document not found: %s", document_id)- return-- # get dataset- dataset = session.get(Dataset, requeried_document.dataset_id)-- if not dataset:- raise ValueError("no dataset found")-- # get existing document segments- document_segments = session.scalars(- select(DocumentSegment).where(- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == requeried_document.id,- )- ).all()-- documents = []- if document_segments:- for document_segment in document_segments:- # transform segment to node- if document_segment.status != SegmentStatus.COMPLETED:- document = Document(- page_content=document_segment.content,- metadata={- "doc_id": document_segment.index_node_id,- "doc_hash": document_segment.index_node_hash,- "document_id": document_segment.document_id,- "dataset_id": document_segment.dataset_id,- },- )- if requeried_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = document_segment.get_child_chunks(session=session)- if child_chunks:- child_documents = []- for child_chunk in child_chunks:- child_document = ChildDocument(- page_content=child_chunk.content,- metadata={- "doc_id": child_chunk.index_node_id,- "doc_hash": child_chunk.index_node_hash,- "document_id": document_segment.document_id,- "dataset_id": document_segment.dataset_id,- },- )- child_documents.append(child_document)- document.children = child_documents- documents.append(document)- # Preserve the full document total even when only incomplete segments are re-indexed.- total_tokens = sum(document_segment.tokens for document_segment in document_segments)- # build index- self._load(- session=session,- dataset=dataset,- dataset_document=requeried_document,- documents=documents,- total_tokens=total_tokens,- )- except DocumentIsPausedError:- raise DocumentIsPausedError(f"Document paused, document id: {document_id}")- except ProviderTokenNotInitError as e:- self._handle_indexing_error(document_id, e, session)- except Exception as e:- self._handle_indexing_error(document_id, e, session)-- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def indexing_estimate(- self,- tenant_id: str,- extract_settings: list[ExtractSetting],- tmp_processing_rule: Mapping[str, Any],- doc_form: str | None = None,- doc_language: str = "English",- dataset_id: str | None = None,- indexing_technique: str = IndexTechniqueType.ECONOMY,- *,- session: Session,- ) -> IndexingEstimate:- """- Estimate the indexing for the document.- """- # check document limit- if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:- count = len(extract_settings)- batch_upload_limit = dify_config.BATCH_UPLOAD_LIMIT- if count > batch_upload_limit:- raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.")-- embedding_model_instance = None- if dataset_id:- dataset = session.get(Dataset, dataset_id)- if not dataset:- raise ValueError("Dataset not found.")- if IndexTechniqueType.HIGH_QUALITY in {dataset.indexing_technique, indexing_technique}:- if dataset.embedding_model_provider:- embedding_model_instance = self._get_model_manager(tenant_id).get_model_instance(- tenant_id=tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- else:- embedding_model_instance = self._get_model_manager(tenant_id).get_default_model_instance(- tenant_id=tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )- else:- if indexing_technique == IndexTechniqueType.HIGH_QUALITY:- embedding_model_instance = self._get_model_manager(tenant_id).get_default_model_instance(- tenant_id=tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )- # keep separate, avoid union-list ambiguity- preview_texts: list[PreviewDetail] = []- qa_preview_texts: list[QAPreviewDetail] = []-- total_segments = 0- # doc_form represents the segmentation method (general, parent-child, QA)- index_type = doc_form- index_processor = IndexProcessorFactory(index_type).init_index_processor()- # one extract_setting is one source document- for extract_setting in extract_settings:- # extract- processing_rule = {- "mode": tmp_processing_rule["mode"],- "rules": tmp_processing_rule.get("rules"),- }- # Extract document content- text_docs = index_processor.extract(- extract_setting, process_rule_mode=tmp_processing_rule["mode"], session=session- )- # Cleaning and segmentation- documents = index_processor.transform(- text_docs,- current_user=None,- embedding_model_instance=embedding_model_instance,- process_rule=processing_rule,- tenant_id=tenant_id,- doc_language=doc_language,- preview=True,- session=session,- )- total_segments += len(documents)- for document in documents:- preview_count = len(qa_preview_texts) if doc_form == "qa_model" else len(preview_texts)- if preview_count < 10:- if doc_form and doc_form == "qa_model":- qa_detail = QAPreviewDetail(- question=document.page_content, answer=document.metadata.get("answer") or ""- )- qa_preview_texts.append(qa_detail)- else:- preview_detail = PreviewDetail(content=document.page_content)- if document.children:- preview_detail.child_chunks = [child.page_content for child in document.children]- preview_texts.append(preview_detail)-- # delete image files and related db records- image_upload_file_ids = get_image_upload_file_ids(document.page_content)- for upload_file_id in image_upload_file_ids:- stmt = select(UploadFile).where(UploadFile.id == upload_file_id)- image_file = session.scalar(stmt)- if image_file is None:- continue- try:- storage.delete(image_file.key)- except Exception:- logger.exception(- "Delete image_files failed while indexing_estimate, \- image_upload_file_is: %s",- upload_file_id,- )- session.delete(image_file)-- # Persist preview cleanup and release the caller transaction before- # summary workers query through their own sessions.- session.commit()-- if doc_form and doc_form == "qa_model":- return IndexingEstimate(total_segments=total_segments * 20, qa_preview=qa_preview_texts, preview=[])-- # Generate summary preview- summary_index_setting = tmp_processing_rule.get("summary_index_setting")- if summary_index_setting and summary_index_setting.get("enable") and preview_texts:- preview_texts = index_processor.generate_summary_preview(- tenant_id, preview_texts, summary_index_setting, doc_language, session=session- )-- return IndexingEstimate(total_segments=total_segments, preview=preview_texts)-- def _extract(- self,- index_processor: BaseIndexProcessor,- dataset_document: DatasetDocument,- process_rule: Mapping[str, Any],- session: Session,- ) -> list[Document]:- data_source_info = dataset_document.data_source_info_dict- text_docs = []- match dataset_document.data_source_type:- case DataSourceType.UPLOAD_FILE:- if not data_source_info or "upload_file_id" not in data_source_info:- raise ValueError("no upload file found")- stmt = select(UploadFile).where(UploadFile.id == data_source_info["upload_file_id"])- file_detail = session.scalars(stmt).one_or_none()-- if file_detail:- extract_setting = ExtractSetting(- datasource_type=DatasourceType.FILE,- upload_file=file_detail,- document_model=dataset_document.doc_form,- )- text_docs = index_processor.extract(- extract_setting, process_rule_mode=process_rule["mode"], session=session- )- case DataSourceType.NOTION_IMPORT:- if (- not data_source_info- or "notion_workspace_id" not in data_source_info- or "notion_page_id" not in data_source_info- ):- raise ValueError("no notion import info found")- extract_setting = ExtractSetting(- datasource_type=DatasourceType.NOTION,- notion_info=NotionInfo.model_validate(- {- "credential_id": data_source_info.get("credential_id"),- "notion_workspace_id": data_source_info["notion_workspace_id"],- "notion_obj_id": data_source_info["notion_page_id"],- "notion_page_type": data_source_info["type"],- "document": dataset_document,- "tenant_id": dataset_document.tenant_id,- }- ),- document_model=dataset_document.doc_form,- )- text_docs = index_processor.extract(- extract_setting, process_rule_mode=process_rule["mode"], session=session- )- case DataSourceType.WEBSITE_CRAWL:- if (- not data_source_info- or "provider" not in data_source_info- or "url" not in data_source_info- or "job_id" not in data_source_info- ):- raise ValueError("no website import info found")- extract_setting = ExtractSetting(- datasource_type=DatasourceType.WEBSITE,- website_info=WebsiteInfo.model_validate(- {- "provider": data_source_info["provider"],- "job_id": data_source_info["job_id"],- "tenant_id": dataset_document.tenant_id,- "url": data_source_info["url"],- "mode": data_source_info["mode"],- "only_main_content": data_source_info["only_main_content"],- }- ),- document_model=dataset_document.doc_form,- )- text_docs = index_processor.extract(- extract_setting, process_rule_mode=process_rule["mode"], session=session- )- case _:- return []- # update document status to splitting- self._update_document_index_status(- session=session,- document_id=dataset_document.id,- after_indexing_status=IndexingStatus.SPLITTING,- extra_update_params={- DatasetDocument.parsing_completed_at: naive_utc_now(),- },- )-- # replace doc id to document model id- for text_doc in text_docs:- if text_doc.metadata is not None:- text_doc.metadata["document_id"] = dataset_document.id- text_doc.metadata["dataset_id"] = dataset_document.dataset_id-- return text_docs-- @staticmethod- def filter_string(text: str):- text = re.sub(r"<\|", "<", text)- text = re.sub(r"\|>", ">", text)- text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)- # Unicode U+FFFE- text = re.sub("\ufffe", "", text)- return text-- @staticmethod- def _get_splitter(- processing_rule_mode: str,- max_tokens: int,- chunk_overlap: int,- separator: str,- embedding_model_instance: ModelInstance | None,- ) -> TextSplitter:- """- Get the NodeParser object according to the processing rule.- """- character_splitter: TextSplitter- if processing_rule_mode in ["custom", "hierarchical"]:- # The user-defined segmentation rule- max_segmentation_tokens_length = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH- if max_tokens < 50 or max_tokens > max_segmentation_tokens_length:- raise ValueError(f"Custom segment length should be between 50 and {max_segmentation_tokens_length}.")-- if separator:- separator = separator.replace("\\n", "\n")-- character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(- chunk_size=max_tokens,- chunk_overlap=chunk_overlap,- fixed_separator=separator,- separators=["\n\n", "。", ". ", " ", ""],- embedding_model_instance=embedding_model_instance,- )- else:- # Automatic segmentation- automatic_rules: dict[str, Any] = dict(DatasetProcessRule.AUTOMATIC_RULES["segmentation"])- character_splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(- chunk_size=automatic_rules["max_tokens"],- chunk_overlap=automatic_rules["chunk_overlap"],- separators=["\n\n", "。", ". ", " ", ""],- embedding_model_instance=embedding_model_instance,- )-- return character_splitter-- def _split_to_documents_for_estimate(- self, text_docs: list[Document], splitter: TextSplitter, processing_rule: DatasetProcessRule- ) -> list[Document]:- """- Split the text documents into nodes.- """- all_documents: list[Document] = []- for text_doc in text_docs:- # document clean- document_text = self._document_clean(text_doc.page_content, processing_rule)- text_doc.page_content = document_text-- # parse document to nodes- documents = splitter.split_documents([text_doc])-- split_documents = []- for document in documents:- if document.page_content is None or not document.page_content.strip():- continue- if document.metadata is not None:- doc_id = str(uuid.uuid4())- hash = helper.generate_text_hash(document.page_content)- document.metadata["doc_id"] = doc_id- document.metadata["doc_hash"] = hash-- split_documents.append(document)-- all_documents.extend(split_documents)-- return all_documents-- @staticmethod- def _document_clean(text: str, processing_rule: DatasetProcessRule) -> str:- """- Clean the document text according to the processing rules.- """- rules: AutomaticRulesConfig | dict[str, Any]- if processing_rule.mode == ProcessRuleMode.AUTOMATIC:- rules = DatasetProcessRule.AUTOMATIC_RULES- else:- rules = json.loads(processing_rule.rules) if processing_rule.rules else {}- document_text = CleanProcessor.clean(text, {"rules": rules})-- return document_text-- @staticmethod- def format_split_text(text: str) -> list[QAPreviewDetail]:- regex = r"Q\d+:\s*(.*?)\s*A\d+:\s*([\s\S]*?)(?=Q\d+:|$)"- matches = re.findall(regex, text, re.UNICODE)-- return [QAPreviewDetail(question=q, answer=re.sub(r"\n\s*", "\n", a.strip())) for q, a in matches if q and a]-- def _load(- self,- session: Session,- dataset: Dataset,- dataset_document: DatasetDocument,- documents: list[Document],- total_tokens: int,- ) -> None:- """Build indexes and mark the document complete using the token total computed before hash sharding."""-- # Build indexes using the existing hash-based worker groups.- indexing_start_at = time.perf_counter()- create_keyword_thread = None- if (- dataset_document.doc_form != IndexStructureType.PARENT_CHILD_INDEX- and dataset.indexing_technique == IndexTechniqueType.ECONOMY- ):- # create keyword index- create_keyword_thread = threading.Thread(- target=propagate_context(self._process_keyword_index),- args=(current_app._get_current_object(), dataset.id, dataset_document.id, documents), # type: ignore- )- create_keyword_thread.start()-- max_workers = 10- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:- futures = []-- # Distribute documents into multiple groups based on the hash values of page_content- # This is done to prevent multiple threads from processing the same document,- # Thereby avoiding potential database insertion deadlocks- document_groups: list[list[Document]] = [[] for _ in range(max_workers)]- for document in documents:- hash = helper.generate_text_hash(document.page_content)- group_index = int(hash, 16) % max_workers- document_groups[group_index].append(document)- for chunk_documents in document_groups:- if len(chunk_documents) == 0:- continue- futures.append(- executor.submit(- propagate_context(self._process_chunk),- current_app._get_current_object(), # type: ignore- dataset_document.doc_form,- chunk_documents,- dataset.id,- dataset_document.id,- )- )-- for future in futures:- future.result()- if (- dataset_document.doc_form != IndexStructureType.PARENT_CHILD_INDEX- and dataset.indexing_technique == IndexTechniqueType.ECONOMY- and create_keyword_thread is not None- ):- create_keyword_thread.join()- indexing_end_at = time.perf_counter()-- # update document status to completed- self._update_document_index_status(- session=session,- document_id=dataset_document.id,- after_indexing_status=IndexingStatus.COMPLETED,- extra_update_params={- DatasetDocument.tokens: total_tokens,- DatasetDocument.completed_at: naive_utc_now(),- DatasetDocument.indexing_latency: indexing_end_at - indexing_start_at,- DatasetDocument.error: None,- },- )-- @staticmethod- def _process_keyword_index(flask_app: Flask, dataset_id: str, document_id: str, documents: list[Document]):- with flask_app.app_context():- with session_factory.create_session() as session:- dataset = session.get(Dataset, dataset_id)- if not dataset:- raise ValueError("no dataset found")- keyword = Keyword(dataset)- keyword.create(documents, session)- if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:- document_ids = [document.metadata["doc_id"] for document in documents]- session.execute(- update(DocumentSegment)- .where(- DocumentSegment.document_id == document_id,- DocumentSegment.dataset_id == dataset_id,- DocumentSegment.index_node_id.in_(document_ids),- DocumentSegment.status == SegmentStatus.INDEXING,- )- .values(- status=SegmentStatus.COMPLETED,- enabled=True,- completed_at=naive_utc_now(),- )- )- session.commit()-- def _process_chunk(- self,- flask_app: Flask,- index_type: str,- chunk_documents: list[Document],- dataset_id: str,- dataset_document_id: str,- ) -> None:- with flask_app.app_context():- with session_factory.create_session() as session:- dataset = session.get(Dataset, dataset_id)- if not dataset:- raise ValueError("no dataset found")-- dataset_document = session.get(DatasetDocument, dataset_document_id)- if not dataset_document:- raise ValueError("no document found")-- # check document is paused- self._check_document_paused_status(dataset_document.id)-- multimodal_documents = []- for document in chunk_documents:- if document.attachments and dataset.is_multimodal:- multimodal_documents.extend(document.attachments)-- # load index- index_processor = IndexProcessorFactory(index_type).init_index_processor()- index_processor.load(- dataset,- chunk_documents,- multimodal_documents=multimodal_documents,- with_keywords=False,- session=session,- )-- document_ids = [document.metadata["doc_id"] for document in chunk_documents]- session.execute(- update(DocumentSegment)- .where(- DocumentSegment.document_id == dataset_document.id,- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.index_node_id.in_(document_ids),- DocumentSegment.status == SegmentStatus.INDEXING,- )- .values(- status=SegmentStatus.COMPLETED,- enabled=True,- completed_at=naive_utc_now(),- )- )-- session.commit()-- @staticmethod- def _check_document_paused_status(document_id: str):- indexing_cache_key = f"document_{document_id}_is_paused"- result = redis_client.get(indexing_cache_key)- if result:- raise DocumentIsPausedError()-- @staticmethod- def _update_document_index_status(- document_id: str,- after_indexing_status: IndexingStatus,- extra_update_params: Mapping[Any, Any] | None = None,- *,- session: Session,- ):- """- Update the document indexing status.- """- count = (- session.scalar(- select(func.count())- .select_from(DatasetDocument)- .where(DatasetDocument.id == document_id, DatasetDocument.is_paused == True)- )- or 0- )- if count > 0:- raise DocumentIsPausedError()- document = session.get(DatasetDocument, document_id)- if not document:- raise DocumentIsDeletedPausedError()-- update_params = {DatasetDocument.indexing_status: after_indexing_status}-- if extra_update_params:- update_params.update(extra_update_params)- session.execute(update(DatasetDocument).where(DatasetDocument.id == document_id).values(update_params)) # type: ignore- session.flush()-- @staticmethod- def _update_segments_by_document(dataset_document_id: str, update_params: Mapping[Any, Any], session: Session):- """- Update the document segment by document id.- """- session.execute(- update(DocumentSegment).where(DocumentSegment.document_id == dataset_document_id).values(update_params)- )- session.flush()-- def _transform(- self,- index_processor: BaseIndexProcessor,- dataset: Dataset,- text_docs: list[Document],- doc_language: str,- process_rule: Mapping[str, Any],- current_user: Account | None = None,- *,- session: Session,- ) -> list[Document]:- # get embedding model instance- embedding_model_instance = None- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- if dataset.embedding_model_provider:- embedding_model_instance = self._get_model_manager(dataset.tenant_id).get_model_instance(- tenant_id=dataset.tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- else:- embedding_model_instance = self._get_model_manager(dataset.tenant_id).get_default_model_instance(- tenant_id=dataset.tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )-- documents = index_processor.transform(- text_docs,- current_user,- embedding_model_instance=embedding_model_instance,- process_rule=process_rule,- tenant_id=dataset.tenant_id,- doc_language=doc_language,- session=session,- )-- return documents-- def _load_segments(- self,- session: Session,- dataset: Dataset,- dataset_document: DatasetDocument,- documents: list[Document],- token_counts: list[int],- ) -> None:- """Persist transformed documents and their precomputed token counts before indexing starts."""- # save node to document segment- doc_store = DatasetDocumentStore(- dataset=dataset, user_id=dataset_document.created_by, document_id=dataset_document.id- )-- # add document segments- doc_store.add_documents(- session=session,- docs=documents,- save_child=dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX,- token_counts=token_counts,- )-- # update document status to indexing- cur_time = naive_utc_now()- self._update_document_index_status(- session=session,- document_id=dataset_document.id,- after_indexing_status=IndexingStatus.INDEXING,- extra_update_params={- DatasetDocument.cleaning_completed_at: cur_time,- DatasetDocument.splitting_completed_at: cur_time,- DatasetDocument.word_count: sum(len(doc.page_content) for doc in documents),- },- )-- # update segment status to indexing- self._update_segments_by_document(- session=session,- dataset_document_id=dataset_document.id,- update_params={- DocumentSegment.status: SegmentStatus.INDEXING,- DocumentSegment.indexing_at: naive_utc_now(),- },- )---class DocumentIsPausedError(Exception):- pass---class DocumentIsDeletedPausedError(Exception):- passdiff --git a/api/core/provider_manager.py b/api/core/provider_manager.pyindex 837beffc37c..b62c7747bb4 100644--- a/api/core/provider_manager.py+++ b/api/core/provider_manager.py@@ -65,7 +65,6 @@ from services.feature_service import FeatureServiceif TYPE_CHECKING:from graphon.model_runtime.protocols.runtime import ModelRuntime- from models.account import Accountlogger = logging.getLogger(__name__)@@ -1162,50 +1161,6 @@ class ProviderManager:provider_names.append(str(model_provider_id))return provider_names- @staticmethod- def get_provider_available_credentials(- tenant_id: str,- provider_name: str,- user: Account | None = None,- ) -> list[CredentialConfiguration]:- """- Get provider all credentials, filtered by visibility.-- :param tenant_id: workspace id- :param provider_name: provider name- :param user: current user (id + admin flag drive the visibility filter)- :return:- """- from models.credential_permission import CredentialType as CredPermType- from services.credential_permission_service import CredentialPermissionService-- with session_factory.create_session() as session:- stmt = (- select(ProviderCredential)- .where(- ProviderCredential.tenant_id == tenant_id,- ProviderCredential.provider_name.in_(ProviderManager._get_provider_names(provider_name)),- )- .order_by(ProviderCredential.created_at.desc())- )-- if user is not None:- stmt = CredentialPermissionService.apply_visibility_filter(- stmt,- model_id_column=ProviderCredential.id,- model_user_id_column=ProviderCredential.user_id,- model_visibility_column=ProviderCredential.visibility,- credential_type=CredPermType.PROVIDER_CREDENTIAL,- user=user,- )-- available_credentials = session.scalars(stmt).all()-- return [- CredentialConfiguration(credential_id=credential.id, credential_name=credential.credential_name)- for credential in available_credentials- ]-@staticmethoddef get_provider_model_available_credentials(tenant_id: str, provider_name: str, model_name: str, model_type: strdiff --git a/api/core/rag/datasource/keyword/jieba/jieba.py b/api/core/rag/datasource/keyword/jieba/jieba.pyindex 62e79d8c84d..eea88349388 100644--- a/api/core/rag/datasource/keyword/jieba/jieba.py+++ b/api/core/rag/datasource/keyword/jieba/jieba.py@@ -1,7 +1,8 @@from collections import defaultdict-from typing import Any, TypedDict, override+from collections.abc import Callable, Mapping, Sequence+from contextlib import AbstractContextManager+from typing import Any, override-import orjsonfrom pydantic import BaseModelfrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -12,12 +13,15 @@ from core.rag.datasource.keyword.keyword_base import BaseKeywordfrom core.rag.models.document import Documentfrom extensions.ext_redis import redis_clientfrom extensions.ext_storage import storage-from models.dataset import Dataset, DatasetKeywordTable, DocumentSegment---class PreSegmentData(TypedDict):- segment: DocumentSegment- keywords: list[str]+from models.dataset import Dataset, DocumentSegment+from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table+from repositories.knowledge.keyword_table_repository import (+ keyword_file_key,+ load_dataset_keyword_table,+ load_keyword_table,+ persist_keyword_table,+ save_keyword_table,+)class KeywordTableConfig(BaseModel):@@ -31,80 +35,114 @@ class Jieba(BaseKeyword):@overridedef create(self, texts: list[Document], session: Session, **kwargs: Any) -> BaseKeyword:- lock_name = f"keyword_indexing_lock_{self.dataset.id}"- with redis_client.lock(lock_name, timeout=600):- keyword_table_handler = JiebaKeywordTableHandler()- keyword_table = self._get_dataset_keyword_table(session=session)- keyword_number = self.dataset.keyword_number or self._config.max_keywords_per_chunk-- for text in texts:- keywords = keyword_table_handler.extract_keywords(text.page_content, keyword_number)- if text.metadata is not None:- self._update_segment_keywords(self.dataset.id, text.metadata["doc_id"], list(keywords), session)- keyword_table = self._add_text_to_keyword_table(- keyword_table or {}, text.metadata["doc_id"], list(keywords)- )-- self._save_dataset_keyword_table(keyword_table, session)-- return self+ self.add_texts(texts, session)+ return self@overridedef add_texts(self, texts: list[Document], session: Session, **kwargs: Any):- lock_name = f"keyword_indexing_lock_{self.dataset.id}"- with redis_client.lock(lock_name, timeout=600):- keyword_table_handler = JiebaKeywordTableHandler()+ self.update_texts(+ texts,+ read=lambda: self._read_keyword_record(session),+ write=lambda storage_type, data, keywords: persist_keyword_table(+ session,+ tenant_id=self.dataset.tenant_id,+ dataset_id=self.dataset.id,+ storage_type=storage_type,+ data=data,+ keywords=keywords,+ ),+ lock=redis_client.lock(f"keyword_indexing_lock_{self.dataset.id}", timeout=600),+ keywords_list=kwargs.get("keywords_list"),+ )- keyword_table = self._get_dataset_keyword_table(session=session)- keywords_list = kwargs.get("keywords_list")+ def update_texts(+ self,+ texts: Sequence[Document],+ *,+ read: Callable[[], tuple[str, str | None]],+ write: Callable[[str, str, Mapping[str, Sequence[str]]], None],+ lock: AbstractContextManager[object],+ keywords_list: Sequence[Sequence[str] | None] | None = None,+ replace_existing: bool = False,+ deleted_ids: Sequence[str] = (),+ ) -> None:+ """Mutate the shared index with explicit persistence and one dataset lock.++ Repository callbacks may close their sessions before returning, keeping+ extraction and object-storage I/O outside database transactions.+ """+ with lock:+ storage_type, data = read()+ payload = load_keyword_table(+ storage=storage,+ tenant_id=self.dataset.tenant_id,+ dataset_id=self.dataset.id,+ storage_type=storage_type,+ data=data,+ )+ table = dict(payload["__data__"]["table"] or {}) if payload else {}+ removed = list(deleted_ids)+ if replace_existing:+ removed.extend(text.metadata["doc_id"] for text in texts)+ if removed:+ table = self._delete_ids_from_keyword_table(table, removed)+ handler: JiebaKeywordTableHandler | None = Nonekeyword_number = self.dataset.keyword_number or self._config.max_keywords_per_chunk- for i in range(len(texts)):- text = texts[i]- if keywords_list:- keywords = keywords_list[i]- if not keywords:- keywords = keyword_table_handler.extract_keywords(text.page_content, keyword_number)- else:- keywords = keyword_table_handler.extract_keywords(text.page_content, keyword_number)- if text.metadata is not None:- self._update_segment_keywords(self.dataset.id, text.metadata["doc_id"], list(keywords), session)- keyword_table = self._add_text_to_keyword_table(- keyword_table or {}, text.metadata["doc_id"], list(keywords)- )+ selected: dict[str, Sequence[str]] = {}+ for position, text in enumerate(texts):+ keywords = keywords_list[position] if keywords_list else None+ if not keywords:+ if handler is None:+ handler = JiebaKeywordTableHandler()+ keywords = list(handler.extract_keywords(text.page_content, keyword_number))+ node_id = text.metadata["doc_id"]+ selected[node_id] = list(keywords)+ table = self._add_text_to_keyword_table(table, node_id, list(keywords))+ encoded = save_keyword_table(+ storage=storage,+ tenant_id=self.dataset.tenant_id,+ dataset_id=self.dataset.id,+ storage_type=storage_type,+ table=table,+ )+ write(storage_type, encoded, selected)- self._save_dataset_keyword_table(keyword_table, session)+ def _read_keyword_record(self, session: Session) -> tuple[str, str | None]:+ row = get_dataset_keyword_table(self.dataset, session=session)+ return (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)@overridedef text_exists(self, id: str, *, session: Session) -> bool:- dataset_keyword_table = self.dataset.get_dataset_keyword_table(session=session)keyword_table = None- keyword_table_dict = (- dataset_keyword_table.get_keyword_table_dict(session=session) if dataset_keyword_table else None- )+ keyword_table_dict = load_dataset_keyword_table(self.dataset, session=session, storage=storage)if keyword_table_dict:data: Any = keyword_table_dict["__data__"]keyword_table = dict(data["table"])if keyword_table is None:return False- return id in set.union(*keyword_table.values())+ return any(id in node_ids for node_ids in keyword_table.values())@overridedef delete_by_ids(self, ids: list[str], session: Session, **kwargs: Any):- lock_name = f"keyword_indexing_lock_{self.dataset.id}"- with redis_client.lock(lock_name, timeout=600):- keyword_table = self._get_dataset_keyword_table(session)- if keyword_table is not None:- keyword_table = self._delete_ids_from_keyword_table(keyword_table, ids)-- self._save_dataset_keyword_table(keyword_table, session)+ self.update_texts(+ [],+ read=lambda: self._read_keyword_record(session),+ write=lambda storage_type, data, keywords: persist_keyword_table(+ session,+ tenant_id=self.dataset.tenant_id,+ dataset_id=self.dataset.id,+ storage_type=storage_type,+ data=data,+ keywords=keywords,+ ),+ lock=redis_client.lock(f"keyword_indexing_lock_{self.dataset.id}", timeout=600),+ deleted_ids=ids,+ )@overridedef search(self, query: str, *, session: Session, **kwargs: Any) -> list[Document]:- dataset_keyword_table = self.dataset.get_dataset_keyword_table(session=session)keyword_table = None- keyword_table_dict = (- dataset_keyword_table.get_keyword_table_dict(session=session) if dataset_keyword_table else None- )+ keyword_table_dict = load_dataset_keyword_table(self.dataset, session=session, storage=storage)if keyword_table_dict:data: Any = keyword_table_dict["__data__"]keyword_table = dict(data["table"])@@ -145,62 +183,14 @@ class Jieba(BaseKeyword):def delete(self, *, session: Session):lock_name = f"keyword_indexing_lock_{self.dataset.id}"with redis_client.lock(lock_name, timeout=600):- dataset_keyword_table = self.dataset.get_dataset_keyword_table(session=session)+ dataset_keyword_table = get_dataset_keyword_table(self.dataset, session=session)if dataset_keyword_table:session.delete(dataset_keyword_table)session.commit()if dataset_keyword_table.data_source_type != "database":- file_key = "keyword_files/" + self.dataset.tenant_id + "/" + self.dataset.id + ".txt"+ file_key = keyword_file_key(self.dataset.tenant_id, self.dataset.id)storage.delete(file_key)- def _save_dataset_keyword_table(self, keyword_table: dict[str, set[str]] | None, session: Session):- keyword_table_dict = {- "__type__": "keyword_table",- "__data__": {"index_id": self.dataset.id, "summary": None, "table": keyword_table},- }- dataset_keyword_table = session.scalar(- select(DatasetKeywordTable).where(DatasetKeywordTable.dataset_id == self.dataset.id)- )- keyword_data_source_type = dataset_keyword_table.data_source_type if dataset_keyword_table else "file"- if keyword_data_source_type == "database":- if dataset_keyword_table is None:- return- dataset_keyword_table.keyword_table = dumps_with_sets(keyword_table_dict)- session.flush()- else:- file_key = "keyword_files/" + self.dataset.tenant_id + "/" + self.dataset.id + ".txt"- if storage.exists(file_key):- storage.delete(file_key)- storage.save(file_key, dumps_with_sets(keyword_table_dict).encode("utf-8"))-- def _get_dataset_keyword_table(self, session: Session) -> dict[str, set[str]] | None:- dataset_keyword_table = session.scalar(- select(DatasetKeywordTable).where(DatasetKeywordTable.dataset_id == self.dataset.id)- )- if dataset_keyword_table:- keyword_table_dict = dataset_keyword_table.get_keyword_table_dict(session=session)- if keyword_table_dict:- data: Any = keyword_table_dict["__data__"]- return dict(data["table"])- else:- keyword_data_source_type = dify_config.KEYWORD_DATA_SOURCE_TYPE- dataset_keyword_table = DatasetKeywordTable(- dataset_id=self.dataset.id,- keyword_table="",- data_source_type=keyword_data_source_type,- )- if keyword_data_source_type == "database":- dataset_keyword_table.keyword_table = dumps_with_sets(- {- "__type__": "keyword_table",- "__data__": {"index_id": self.dataset.id, "summary": None, "table": {}},- }- )- session.add(dataset_keyword_table)- session.flush()-- return {}-def _add_text_to_keyword_table(self, keyword_table: dict[str, set[str]], id: str, keywords: list[str]) -> dict[str, set[str]]:@@ -245,58 +235,3 @@ class Jieba(BaseKeyword):)return sorted_chunk_indices[:k]-- def _update_segment_keywords(self, dataset_id: str, node_id: str, keywords: list[str], session: Session):- stmt = select(DocumentSegment).where(- DocumentSegment.dataset_id == dataset_id, DocumentSegment.index_node_id == node_id- )- document_segment = session.scalar(stmt)- if document_segment:- document_segment.keywords = keywords- session.add(document_segment)- session.flush()-- def create_segment_keywords(self, node_id: str, keywords: list[str], session: Session):- keyword_table = self._get_dataset_keyword_table(session)- self._update_segment_keywords(self.dataset.id, node_id, keywords, session)- keyword_table = self._add_text_to_keyword_table(keyword_table or {}, node_id, keywords)- self._save_dataset_keyword_table(keyword_table, session)-- def multi_create_segment_keywords(self, pre_segment_data_list: list[PreSegmentData], session: Session):- keyword_table_handler = JiebaKeywordTableHandler()- keyword_table = self._get_dataset_keyword_table(session)- for pre_segment_data in pre_segment_data_list:- segment = pre_segment_data["segment"]- if pre_segment_data["keywords"]:- segment.keywords = pre_segment_data["keywords"]- assert segment.index_node_id- keyword_table = self._add_text_to_keyword_table(- keyword_table or {}, segment.index_node_id, pre_segment_data["keywords"]- )- else:- keyword_number = self.dataset.keyword_number or self._config.max_keywords_per_chunk-- keywords = keyword_table_handler.extract_keywords(segment.content, keyword_number)- segment.keywords = list(keywords)- assert segment.index_node_id- keyword_table = self._add_text_to_keyword_table(- keyword_table or {}, segment.index_node_id, list(keywords)- )- self._save_dataset_keyword_table(keyword_table, session)-- def update_segment_keywords_index(self, node_id: str, keywords: list[str], session: Session):- keyword_table = self._get_dataset_keyword_table(session)- keyword_table = self._add_text_to_keyword_table(keyword_table or {}, node_id, keywords)- self._save_dataset_keyword_table(keyword_table, session)---def set_orjson_default(obj: Any):- """Default function for orjson serialization of set types"""- if isinstance(obj, set):- return list(obj)- raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")---def dumps_with_sets(obj: Any) -> str:- """JSON dumps with set support using orjson"""- return orjson.dumps(obj, default=set_orjson_default).decode("utf-8")diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.pyindex 35a70103623..e97984d4291 100644--- a/api/core/rag/datasource/retrieval_service.py+++ b/api/core/rag/datasource/retrieval_service.py@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutorfrom typing import Any, NotRequired, TypedDictfrom flask import Flask, current_app-from sqlalchemy import select+from sqlalchemy import select, tuple_from sqlalchemy.orm import Session, load_onlyfrom configs import dify_config@@ -35,7 +35,7 @@ from models.dataset import ()from models.dataset import Document as DatasetDocumentfrom models.model import UploadFile-from services.external_knowledge_service import ExternalDatasetService+from services.knowledge.external.service import ExternalDatasetServiceclass SegmentAttachmentResult(TypedDict):@@ -486,7 +486,14 @@ class RetrievalService:for doc in session.scalars(select(DatasetDocument).where(DatasetDocument.id.in_(document_ids))- .options(load_only(DatasetDocument.id, DatasetDocument.doc_form, DatasetDocument.dataset_id))+ .options(+ load_only(+ DatasetDocument.id,+ DatasetDocument.tenant_id,+ DatasetDocument.doc_form,+ DatasetDocument.dataset_id,+ )+ )).all()}@@ -559,7 +566,14 @@ class RetrievalService:doc_segment_map: dict[str, list[str]] = {}segment_summary_map: dict[str, str] = {} # Map segment_id to summary content- attachments = cls.get_segment_attachment_infos(image_doc_ids, session)+ dataset_refs = tuple(+ {(document.tenant_id, document.dataset_id) for document in valid_dataset_documents.values()}+ )+ attachments = cls.get_segment_attachment_infos(+ image_doc_ids,+ dataset_refs=dataset_refs,+ session=session,+ )for attachment in attachments:segment_ids.append(attachment["segment_id"])@@ -947,15 +961,30 @@ class RetrievalService:@classmethoddef get_segment_attachment_infos(- cls, attachment_ids: list[str], session: Session+ cls,+ attachment_ids: list[str],+ *,+ dataset_refs: Sequence[tuple[str, str]],+ session: Session,) -> list[SegmentAttachmentInfoResult]:+ if not attachment_ids or not dataset_refs:+ return []attachment_infos: list[SegmentAttachmentInfoResult] = []granted_upload_file_ids: list[str] = []- upload_files = session.scalars(select(UploadFile).where(UploadFile.id.in_(attachment_ids))).all()+ tenant_ids = tuple({tenant_id for tenant_id, _ in dataset_refs})+ upload_files = session.scalars(+ select(UploadFile).where(+ UploadFile.id.in_(attachment_ids),+ UploadFile.tenant_id.in_(tenant_ids),+ )+ ).all()if upload_files:upload_file_ids = [upload_file.id for upload_file in upload_files]attachment_bindings = session.scalars(- select(SegmentAttachmentBinding).where(SegmentAttachmentBinding.attachment_id.in_(upload_file_ids))+ select(SegmentAttachmentBinding).where(+ SegmentAttachmentBinding.attachment_id.in_(upload_file_ids),+ tuple_(SegmentAttachmentBinding.tenant_id, SegmentAttachmentBinding.dataset_id).in_(dataset_refs),+ )).all()attachment_binding_map = {binding.attachment_id: binding for binding in attachment_bindings}diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.pyindex 0806dfb2745..9bd9eed6245 100644--- a/api/core/rag/datasource/vdb/vector_factory.py+++ b/api/core/rag/datasource/vdb/vector_factory.py@@ -2,6 +2,7 @@ import base64import loggingimport timefrom abc import ABC, abstractmethod+from collections.abc import Mappingfrom typing import Any, overridefrom sqlalchemy import select@@ -105,7 +106,14 @@ class _LazyEmbeddings(Embeddings):class Vector:- def __init__(self, dataset: Dataset, attributes: list | None = None, *, session: Session):+ def __init__(+ self,+ dataset: Dataset,+ attributes: list | None = None,+ *,+ session: Session | None,+ vector_type: str | None = None,+ ):if attributes is None:# `is_summary` and `original_chunk_id` are stored on summary vectors# by `SummaryIndexService` and read back by `RetrievalService` to@@ -132,7 +140,14 @@ class Vector:self._embeddings: Embeddings = _LazyEmbeddings(dataset)self._attributes = attributesself._session = session- self._vector_processor = self._init_vector(session=session)+ if vector_type is not None:+ self._vector_processor = self.get_vector_factory(vector_type)().init_vector(+ dataset, self._attributes, self._embeddings+ )+ else:+ if session is None:+ raise ValueError("A resolved vector type is required without a database session")+ self._vector_processor = self._init_vector(session=session)@staticmethoddef resolve_vector_type(dataset: Dataset, *, session: Session) -> str:@@ -192,7 +207,9 @@ class Vector:self._vector_processor.create(texts=batch, embeddings=batch_embeddings, **kwargs)logger.info("Embedding %s texts took %s s", len(texts), time.time() - start)- def create_multimodal(self, file_documents: list | None = None, **kwargs):+ def create_multimodal(+ self, file_documents: list | None = None, *, upload_files: Mapping[str, UploadFile] | None = None, **kwargs+ ):if file_documents:start = time.time()logger.info("start embedding %s files %s", len(file_documents), start)@@ -205,9 +222,16 @@ class Vector:# Batch query all upload files to avoid N+1 queriesattachment_ids = [doc.metadata["doc_id"] for doc in batch]- stmt = select(UploadFile).where(UploadFile.id.in_(attachment_ids))- upload_files = self._session.scalars(stmt).all()- upload_file_map = {str(f.id): f for f in upload_files}+ upload_file_map: Mapping[str, UploadFile]+ if upload_files is None:+ if self._session is None:+ raise ValueError("Upload files must be loaded before indexing without a database session")+ stmt = select(UploadFile).where(+ UploadFile.id.in_(attachment_ids), UploadFile.tenant_id == self._dataset.tenant_id+ )+ upload_file_map = {str(f.id): f for f in self._session.scalars(stmt).all()}+ else:+ upload_file_map = upload_filesfile_base64_list = []real_batch = []@@ -264,6 +288,8 @@ class Vector:return self._vector_processor.search_by_vector(query_vector, **kwargs)def search_by_file(self, file_id: str, **kwargs: Any) -> list[Document]:+ if self._session is None:+ raise ValueError("File search requires a database session")upload_file: UploadFile | None = self._session.get(UploadFile, file_id)if not upload_file:diff --git a/api/core/rag/docstore/dataset_docstore.py b/api/core/rag/docstore/dataset_docstore.pyindex 456df05daa4..e5f563fb170 100644--- a/api/core/rag/docstore/dataset_docstore.py+++ b/api/core/rag/docstore/dataset_docstore.py@@ -235,8 +235,12 @@ class DatasetDocumentStore:def get_document_segment(self, doc_id: str, session: Session) -> DocumentSegment | None:stmt = select(DocumentSegment).where(- DocumentSegment.dataset_id == self._dataset.id, DocumentSegment.index_node_id == doc_id+ DocumentSegment.tenant_id == self._dataset.tenant_id,+ DocumentSegment.dataset_id == self._dataset.id,+ DocumentSegment.index_node_id == doc_id,)+ if self._document_id is not None:+ stmt = stmt.where(DocumentSegment.document_id == self._document_id)document_segment = session.scalar(stmt)return document_segmentdiff --git a/api/core/rag/entities/__init__.py b/api/core/rag/entities/__init__.pyindex 373b68894bf..1e87472b2c4 100644--- a/api/core/rag/entities/__init__.py+++ b/api/core/rag/entities/__init__.py@@ -3,7 +3,13 @@ from core.rag.entities.context_entities import DocumentContextfrom core.rag.entities.event import DatasourceCompletedEvent, DatasourceErrorEvent, DatasourceProcessingEventfrom core.rag.entities.index_entities import EconomySetting, EmbeddingSetting, IndexMethodfrom core.rag.entities.metadata_entities import Condition, MetadataFilteringCondition, SupportedComparisonOperator-from core.rag.entities.processing_entities import ParentMode, PreProcessingRule, Rule, Segmentation+from core.rag.entities.processing_entities import (+ ParentMode,+ PreProcessingRule,+ PreProcessingRuleKey,+ Rule,+ Segmentation,+)from core.rag.entities.retrieval_settings import (KeywordSetting,RerankingModelConfig,@@ -24,6 +30,7 @@ __all__ = ["MetadataFilteringCondition","ParentMode","PreProcessingRule",+ "PreProcessingRuleKey","RerankingModelConfig","RetrievalSourceMetadata","Rule",diff --git a/api/core/rag/entities/processing_entities.py b/api/core/rag/entities/processing_entities.pyindex 6b08bb858c8..3a568062f40 100644--- a/api/core/rag/entities/processing_entities.py+++ b/api/core/rag/entities/processing_entities.py@@ -1,7 +1,7 @@from enum import StrEnum-from typing import Annotated, Literal+from typing import Literal-from pydantic import AliasChoices, BaseModel, Field, WithJsonSchema+from pydantic import AliasChoices, BaseModel, Fieldclass ParentMode(StrEnum):@@ -9,19 +9,11 @@ class ParentMode(StrEnum):PARAGRAPH = "paragraph"-PreProcessingRuleID = Annotated[- str,- WithJsonSchema(- {- "enum": ["remove_stopwords", "remove_extra_spaces", "remove_urls_emails"],- "type": "string",- }- ),-]+type PreProcessingRuleKey = Literal["remove_stopwords", "remove_extra_spaces", "remove_urls_emails"]class PreProcessingRule(BaseModel):- id: PreProcessingRuleID = Field(description="Rule identifier.")+ id: PreProcessingRuleKey = Field(description="Rule identifier.")enabled: bool = Field(description="Whether this preprocessing rule is enabled.")diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.pyindex 0c420340732..f31acb4ebed 100644--- a/api/core/rag/extractor/entity/extract_setting.py+++ b/api/core/rag/extractor/entity/extract_setting.py@@ -1,7 +1,24 @@-from pydantic import BaseModel, ConfigDict+from pydantic import BaseModel, ConfigDict, Field-from models.dataset import Document-from models.model import UploadFile++class UploadFileExtractionInput(BaseModel):+ """Detached values required to extract one tenant-owned upload."""++ id: str+ tenant_id: str+ key: str+ created_by: str+ model_config = ConfigDict(from_attributes=True)+++class StoredDocumentExtractionInput(BaseModel):+ """Detached document values needed to record a source's last edited time."""++ id: str+ tenant_id: str+ dataset_id: str+ data_source_info_dict: dict[str, object] | None+ model_config = ConfigDict(from_attributes=True)class NotionInfo(BaseModel):@@ -13,9 +30,9 @@ class NotionInfo(BaseModel):notion_workspace_id: str | None = ""notion_obj_id: strnotion_page_type: str- document: Document | None = None+ document: StoredDocumentExtractionInput | None = Nonetenant_id: str- model_config = ConfigDict(arbitrary_types_allowed=True)+ notion_access_token: str | None = Field(default=None, exclude=True, repr=False)class WebsiteInfo(BaseModel):@@ -23,8 +40,6 @@ class WebsiteInfo(BaseModel):website import info."""- model_config = ConfigDict(arbitrary_types_allowed=True)-provider: strjob_id: strurl: str@@ -39,8 +54,7 @@ class ExtractSetting(BaseModel):"""datasource_type: str- upload_file: UploadFile | None = None+ upload_file: UploadFileExtractionInput | None = Nonenotion_info: NotionInfo | None = Nonewebsite_info: WebsiteInfo | None = Nonedocument_model: str | None = None- model_config = ConfigDict(arbitrary_types_allowed=True)diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.pyindex dd73b436089..a8d83d57d83 100644--- a/api/core/rag/extractor/extract_processor.py+++ b/api/core/rag/extractor/extract_processor.py@@ -7,10 +7,11 @@ from urllib.parse import unquotefrom sqlalchemy.orm import Sessionfrom configs import dify_config+from core.db.session_factory import get_session_makerfrom core.file import remote_fetcherfrom core.rag.extractor.csv_extractor import CSVExtractorfrom core.rag.extractor.entity.datasource_type import DatasourceType-from core.rag.extractor.entity.extract_setting import ExtractSetting+from core.rag.extractor.entity.extract_setting import ExtractSetting, UploadFileExtractionInputfrom core.rag.extractor.excel_extractor import ExcelExtractorfrom core.rag.extractor.extractor_base import BaseExtractorfrom core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor@@ -59,7 +60,9 @@ class ExtractProcessor:cls, upload_file: UploadFile, return_text: bool = False, is_automatic: bool = False) -> list[Document] | str:extract_setting = ExtractSetting(- datasource_type=DatasourceType.FILE, upload_file=upload_file, document_model="text_model"+ datasource_type=DatasourceType.FILE,+ upload_file=UploadFileExtractionInput.model_validate(upload_file),+ document_model="text_model",)if return_text:delimiter = "\n"@@ -223,17 +226,29 @@ class ExtractProcessor:extractor = TextExtractor(file_path, autodetect_encoding=True)return extractor.extract()elif extract_setting.datasource_type == DatasourceType.NOTION:+ from extensions.application_services.data_sources import build_data_source_credentials+assert extract_setting.notion_info is not None, "notion_info is required"+ notion_info = extract_setting.notion_infoextractor = NotionExtractor(notion_workspace_id=extract_setting.notion_info.notion_workspace_id or "",notion_obj_id=extract_setting.notion_info.notion_obj_id,notion_page_type=extract_setting.notion_info.notion_page_type,document_model=extract_setting.notion_info.document,tenant_id=extract_setting.notion_info.tenant_id,+ notion_access_token=extract_setting.notion_info.notion_access_token,credential_id=extract_setting.notion_info.credential_id,+ notion_token_loader=lambda: build_data_source_credentials(+ database_client=get_session_maker()+ ).providers.get_stored_notion_access_token(+ tenant_id=notion_info.tenant_id, credential_id=notion_info.credential_id+ ),)return extractor.extract()elif extract_setting.datasource_type == DatasourceType.WEBSITE:+ from extensions.application_services.data_sources import build_website_service++ website_service = build_website_service(database_client=get_session_maker())assert extract_setting.website_info is not None, "website_info is required"match extract_setting.website_info.provider:case "firecrawl":@@ -243,6 +258,7 @@ class ExtractProcessor:tenant_id=extract_setting.website_info.tenant_id,mode=extract_setting.website_info.mode,only_main_content=extract_setting.website_info.only_main_content,+ website_service=website_service,)return extractor.extract()case "watercrawl":@@ -252,6 +268,7 @@ class ExtractProcessor:tenant_id=extract_setting.website_info.tenant_id,mode=extract_setting.website_info.mode,only_main_content=extract_setting.website_info.only_main_content,+ website_service=website_service,)return extractor.extract()case "jinareader":@@ -261,6 +278,7 @@ class ExtractProcessor:tenant_id=extract_setting.website_info.tenant_id,mode=extract_setting.website_info.mode,only_main_content=extract_setting.website_info.only_main_content,+ website_service=website_service,)return extractor.extract()case _:diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.pyindex 740527a32d7..01ac232898d 100644--- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py+++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py@@ -2,7 +2,7 @@ from typing import overridefrom core.rag.extractor.extractor_base import BaseExtractorfrom core.rag.models.document import Document-from services.website_service import WebsiteService+from services.data_source.website_service import WebsiteServiceclass FirecrawlWebExtractor(BaseExtractor):@@ -24,8 +24,11 @@ class FirecrawlWebExtractor(BaseExtractor):tenant_id: str,mode: str = "crawl",only_main_content: bool = True,+ *,+ website_service: WebsiteService,):"""Initialize with url, api_key, base_url and mode."""+ self._website_service = website_serviceself._url = urlself.job_id = job_idself.tenant_id = tenant_id@@ -37,7 +40,7 @@ class FirecrawlWebExtractor(BaseExtractor):"""Extract content from the URL."""documents = []if self.mode == "crawl":- crawl_data = WebsiteService.get_crawl_url_data(self.job_id, "firecrawl", self._url, self.tenant_id)+ crawl_data = self._website_service.get_crawl_url_data(self.job_id, "firecrawl", self._url, self.tenant_id)if crawl_data is None:return []document = Document(@@ -50,7 +53,7 @@ class FirecrawlWebExtractor(BaseExtractor):)documents.append(document)elif self.mode == "scrape":- scrape_data = WebsiteService.get_scrape_url_data(+ scrape_data = self._website_service.get_scrape_url_data("firecrawl", self._url, self.tenant_id, self.only_main_content)diff --git a/api/core/rag/extractor/jina_reader_extractor.py b/api/core/rag/extractor/jina_reader_extractor.pyindex b184e8fac65..489ec1082f8 100644--- a/api/core/rag/extractor/jina_reader_extractor.py+++ b/api/core/rag/extractor/jina_reader_extractor.py@@ -2,7 +2,7 @@ from typing import overridefrom core.rag.extractor.extractor_base import BaseExtractorfrom core.rag.models.document import Document-from services.website_service import WebsiteService+from services.data_source.website_service import WebsiteServiceclass JinaReaderWebExtractor(BaseExtractor):@@ -17,8 +17,11 @@ class JinaReaderWebExtractor(BaseExtractor):tenant_id: str,mode: str = "crawl",only_main_content: bool = False,+ *,+ website_service: WebsiteService,):"""Initialize with url, api_key, base_url and mode."""+ self._website_service = website_serviceself._url = urlself.job_id = job_idself.tenant_id = tenant_id@@ -30,7 +33,7 @@ class JinaReaderWebExtractor(BaseExtractor):"""Extract content from the URL."""documents = []if self.mode == "crawl":- crawl_data = WebsiteService.get_crawl_url_data(self.job_id, "jinareader", self._url, self.tenant_id)+ crawl_data = self._website_service.get_crawl_url_data(self.job_id, "jinareader", self._url, self.tenant_id)if crawl_data is None:return []document = Document(diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.pyindex 10f1bd984a6..680b049a9ea 100644--- a/api/core/rag/extractor/notion_extractor.py+++ b/api/core/rag/extractor/notion_extractor.py@@ -1,17 +1,17 @@import jsonimport loggingimport operator+from collections.abc import Callablefrom typing import Any, cast, overrideimport httpxfrom sqlalchemy import update-from configs import dify_config+from core.rag.extractor.entity.extract_setting import StoredDocumentExtractionInputfrom core.rag.extractor.extractor_base import BaseExtractorfrom core.rag.models.document import Documentfrom extensions.ext_database import dbfrom models.dataset import Document as DocumentModel-from services.datasource_provider_service import DatasourceProviderServicelogger = logging.getLogger(__name__)@@ -41,36 +41,21 @@ class NotionExtractor(BaseExtractor):notion_obj_id: str,notion_page_type: str,tenant_id: str,- document_model: DocumentModel | None = None,+ document_model: StoredDocumentExtractionInput | None = None,notion_access_token: str | None = None,credential_id: str | None = None,+ *,+ notion_token_loader: Callable[[], str],):self._notion_access_token = Noneself._document_model = document_modelself._notion_workspace_id = notion_workspace_idself._notion_obj_id = notion_obj_idself._notion_page_type = notion_page_type- self._credential_id = credential_idif notion_access_token:self._notion_access_token = notion_access_tokenelse:- try:- self._notion_access_token = self._get_access_token(tenant_id, self._credential_id)- except Exception as e:- logger.warning(- (- "Failed to get Notion access token from datasource credentials: %s, "- "falling back to environment variable NOTION_INTEGRATION_TOKEN"- ),- e,- )- integration_token = dify_config.NOTION_INTEGRATION_TOKEN- if integration_token is None:- raise ValueError(- "Must specify `integration_token` or set environment variable `NOTION_INTEGRATION_TOKEN`."- ) from e-- self._notion_access_token = integration_token+ self._notion_access_token = notion_token_loader()@overridedef extract(self) -> list[Document]:@@ -344,7 +329,7 @@ class NotionExtractor(BaseExtractor):result_lines = "\n".join(result_lines_arr)return result_lines- def update_last_edited_time(self, document_model: DocumentModel | None):+ def update_last_edited_time(self, document_model: StoredDocumentExtractionInput | None):if not document_model:return@@ -355,7 +340,11 @@ class NotionExtractor(BaseExtractor):db.session.execute(update(DocumentModel)- .where(DocumentModel.id == document_model.id)+ .where(+ DocumentModel.id == document_model.id,+ DocumentModel.tenant_id == document_model.tenant_id,+ DocumentModel.dataset_id == document_model.dataset_id,+ ).values({DocumentModel.data_source_info: json.dumps(data_source_info)}))db.session.commit()@@ -385,20 +374,3 @@ class NotionExtractor(BaseExtractor):data = res.json()return cast(str, data["last_edited_time"])-- @classmethod- def _get_access_token(cls, tenant_id: str, credential_id: str | None) -> str:- # get credential from tenant_id and credential_id- if not credential_id:- raise Exception(f"No credential id found for tenant {tenant_id}")- datasource_provider_service = DatasourceProviderService()- credential = datasource_provider_service.get_datasource_credentials(- tenant_id=tenant_id,- credential_id=credential_id,- provider="notion_datasource",- plugin_id="langgenius/notion_datasource",- )- if not credential:- raise Exception(f"No notion credential found for tenant {tenant_id} and credential {credential_id}")-- return cast(str, credential["integration_secret"])diff --git a/api/core/rag/extractor/watercrawl/extractor.py b/api/core/rag/extractor/watercrawl/extractor.pyindex fdfe75b6770..0359a70491b 100644--- a/api/core/rag/extractor/watercrawl/extractor.py+++ b/api/core/rag/extractor/watercrawl/extractor.py@@ -2,7 +2,7 @@ from typing import overridefrom core.rag.extractor.extractor_base import BaseExtractorfrom core.rag.models.document import Document-from services.website_service import WebsiteService+from services.data_source.website_service import WebsiteServiceclass WaterCrawlWebExtractor(BaseExtractor):@@ -25,8 +25,11 @@ class WaterCrawlWebExtractor(BaseExtractor):tenant_id: str,mode: str = "crawl",only_main_content: bool = True,+ *,+ website_service: WebsiteService,):"""Initialize with url, api_key, base_url and mode."""+ self._website_service = website_serviceself._url = urlself.job_id = job_idself.tenant_id = tenant_id@@ -38,7 +41,7 @@ class WaterCrawlWebExtractor(BaseExtractor):"""Extract content from the URL."""documents = []if self.mode == "crawl":- crawl_data = WebsiteService.get_crawl_url_data(self.job_id, "watercrawl", self._url, self.tenant_id)+ crawl_data = self._website_service.get_crawl_url_data(self.job_id, "watercrawl", self._url, self.tenant_id)if crawl_data is None:return []document = Document(@@ -51,7 +54,7 @@ class WaterCrawlWebExtractor(BaseExtractor):)documents.append(document)elif self.mode == "scrape":- scrape_data = WebsiteService.get_scrape_url_data(+ scrape_data = self._website_service.get_scrape_url_data("watercrawl", self._url, self.tenant_id, self.only_main_content)diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.pyindex 06bacd68669..a2fea9bde9e 100644--- a/api/core/rag/index_processor/index_processor_base.py+++ b/api/core/rag/index_processor/index_processor_base.py@@ -147,7 +147,12 @@ class BaseIndexProcessor(ABC):return character_splitterdef _get_content_files(- self, document: Document, current_user: Account | None = None, *, session: Session+ self,+ document: Document,+ current_user: Account | None = None,+ *,+ tenant_id: str,+ session: Session,) -> list[AttachmentDocument]:"""Get the content files from the document.@@ -185,7 +190,12 @@ class BaseIndexProcessor(ABC):if match:if current_user:tool_file_id = match.group(1)- upload_file_id = self._download_tool_file(tool_file_id, current_user, session=session)+ upload_file_id = self._download_tool_file(+ tool_file_id,+ current_user,+ tenant_id=tenant_id,+ session=session,+ )if upload_file_id:upload_file_id_list.append(upload_file_id)continue@@ -208,7 +218,12 @@ class BaseIndexProcessor(ABC):# Get unique IDs for database queryunique_upload_file_ids = list(set(upload_file_id_list))- upload_files = session.scalars(select(UploadFile).where(UploadFile.id.in_(unique_upload_file_ids))).all()+ upload_files = session.scalars(+ select(UploadFile).where(+ UploadFile.tenant_id == tenant_id,+ UploadFile.id.in_(unique_upload_file_ids),+ )+ ).all()# Create a mapping from ID to UploadFile for quick lookupupload_file_map = {upload_file.id: upload_file for upload_file in upload_files}@@ -314,13 +329,22 @@ class BaseIndexProcessor(ABC):logging.warning("Unexpected error downloading image from %s", image_url, exc_info=True)return None- def _download_tool_file(self, tool_file_id: str, current_user: Account, *, session: Session) -> str | None:+ def _download_tool_file(+ self,+ tool_file_id: str,+ current_user: Account,+ *,+ tenant_id: str,+ session: Session,+ ) -> str | None:"""Download the tool file from the ID."""from services.file_service import FileService- tool_file = session.get(ToolFile, tool_file_id)+ tool_file = session.scalar(+ select(ToolFile).where(ToolFile.id == tool_file_id, ToolFile.tenant_id == tenant_id).limit(1)+ )if not tool_file:return Noneblob = storage.load_once(tool_file.file_key)diff --git a/api/core/rag/index_processor/processor/paragraph_index_processor.py b/api/core/rag/index_processor/processor/paragraph_index_processor.pyindex d0b322db70c..2102fe966c6 100644--- a/api/core/rag/index_processor/processor/paragraph_index_processor.py+++ b/api/core/rag/index_processor/processor/paragraph_index_processor.py@@ -3,6 +3,7 @@import loggingimport reimport uuid+from collections.abc import Callablefrom typing import Any, TypedDict, cast, overridefrom sqlalchemy import select@@ -46,7 +47,7 @@ from models.account import Accountfrom models.dataset import Dataset, DatasetProcessRule, DocumentSegment, SegmentAttachmentBindingfrom models.dataset import Document as DatasetDocumentfrom services.account_service import AccountService-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterlogger = logging.getLogger(__name__)@@ -113,7 +114,12 @@ class ParagraphIndexProcessor(BaseIndexProcessor):document_node.metadata["doc_id"] = doc_iddocument_node.metadata["doc_hash"] = hashmultimodal_documents = (- self._get_content_files(document_node, current_user, session=session)+ self._get_content_files(+ document_node,+ current_user,+ tenant_id=kwargs["tenant_id"],+ session=session,+ )if document_node.metadataelse None)@@ -172,10 +178,10 @@ class ParagraphIndexProcessor(BaseIndexProcessor):).all()segment_ids = [segment.id for segment in segments]if segment_ids:- SummaryIndexService.delete_summaries_for_segments(dataset, segment_ids, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, segment_ids, session=session)else:# Delete all summaries for the dataset- SummaryIndexService.delete_summaries_for_segments(dataset, None, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, None, session=session)if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:vector = Vector(dataset, session=session)@@ -204,7 +210,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):"doc_hash": helper.generate_text_hash(content),}doc = Document(page_content=content, metadata=metadata)- attachments = self._get_content_files(doc, session=session)+ attachments = self._get_content_files(doc, tenant_id=dataset.tenant_id, session=session)if attachments:doc.attachments = attachmentsall_multimodal_documents.extend(attachments)@@ -240,7 +246,12 @@ class ParagraphIndexProcessor(BaseIndexProcessor):account = AccountService.load_user(document.created_by, account_session)if not account:raise ValueError("Invalid account")- doc.attachments = self._get_content_files(doc, current_user=account, session=session)+ doc.attachments = self._get_content_files(+ doc,+ current_user=account,+ tenant_id=dataset.tenant_id,+ session=session,+ )if doc.attachments:all_multimodal_documents.extend(doc.attachments)documents.append(doc)@@ -378,7 +389,6 @@ class ParagraphIndexProcessor(BaseIndexProcessor):return preview_texts@staticmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)def generate_summary(tenant_id: str,text: str,@@ -404,6 +414,30 @@ class ParagraphIndexProcessor(BaseIndexProcessor):Returns:Tuple of (summary_content, llm_usage) where llm_usage is LLMUsage object"""++ def load_images() -> list[File]:+ images = (+ ParagraphIndexProcessor._extract_images_from_segment_attachments(tenant_id, segment_id, session)+ if segment_id+ else []+ )+ return images or ParagraphIndexProcessor._extract_images_from_text(tenant_id, text, session)++ return ParagraphIndexProcessor.generate_summary_from_inputs(+ tenant_id, text, summary_index_setting, document_language=document_language, image_loader=load_images+ )++ @staticmethod+ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def generate_summary_from_inputs(+ tenant_id: str,+ text: str,+ summary_index_setting: SummaryIndexSettingDict | None,+ *,+ document_language: str | None,+ image_loader: Callable[[], list[File]],+ ) -> tuple[str, LLMUsage]:+ """Generate a summary after an explicit loader materializes any required images."""if not summary_index_setting or not summary_index_setting.get("enable"):raise ValueError("summary_index_setting is required and must be enabled to generate summary.")@@ -447,15 +481,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):# Extract images if model supports visionimage_files = []if supports_vision:- # First, try to get images from SegmentAttachmentBinding (preferred method)- if segment_id:- image_files = ParagraphIndexProcessor._extract_images_from_segment_attachments(- tenant_id, segment_id, session- )-- # If no images from attachments, fall back to extracting from text- if not image_files:- image_files = ParagraphIndexProcessor._extract_images_from_text(tenant_id, text, session)+ image_files = image_loader()# Build prompt messagesprompt_messages = []@@ -603,6 +629,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):.where(SegmentAttachmentBinding.segment_id == segment_id,SegmentAttachmentBinding.tenant_id == tenant_id,+ UploadFile.tenant_id == tenant_id,)).all()diff --git a/api/core/rag/index_processor/processor/parent_child_index_processor.py b/api/core/rag/index_processor/processor/parent_child_index_processor.pyindex b31875c2f20..0c98b401800 100644--- a/api/core/rag/index_processor/processor/parent_child_index_processor.py+++ b/api/core/rag/index_processor/processor/parent_child_index_processor.py@@ -29,7 +29,7 @@ from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegmfrom models.dataset import Document as DatasetDocumentfrom models.enums import ProcessRuleModefrom services.account_service import AccountService-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterlogger = logging.getLogger(__name__)@@ -99,11 +99,16 @@ class ParentChildIndexProcessor(BaseIndexProcessor):page_content = page_contentif len(page_content) > 0:document_node.page_content = page_content- multimodel_documents = self._get_content_files(document_node, current_user, session=session)+ multimodel_documents = self._get_content_files(+ document_node,+ current_user,+ tenant_id=kwargs["tenant_id"],+ session=session,+ )if multimodel_documents:document_node.attachments = multimodel_documents# parse document to child nodes- child_nodes = self._split_child_nodes(+ child_nodes = self.split_child_nodes(document_node, rules, process_rule.get("mode"), kwargs.get("embedding_model_instance"))document_node.children = child_nodes@@ -112,11 +117,15 @@ class ParentChildIndexProcessor(BaseIndexProcessor):elif rules.parent_mode == ParentMode.FULL_DOC:page_content = "\n".join([document.page_content for document in documents])document = Document(page_content=page_content, metadata=documents[0].metadata)- multimodel_documents = self._get_content_files(document, session=session)+ multimodel_documents = self._get_content_files(+ document,+ tenant_id=kwargs["tenant_id"],+ session=session,+ )if multimodel_documents:document.attachments = multimodel_documents# parse document to child nodes- child_nodes = self._split_child_nodes(+ child_nodes = self.split_child_nodes(document, rules, process_rule.get("mode"), kwargs.get("embedding_model_instance"))if kwargs.get("preview"):@@ -176,10 +185,10 @@ class ParentChildIndexProcessor(BaseIndexProcessor):).all()segment_ids = [segment.id for segment in segments]if segment_ids:- SummaryIndexService.delete_summaries_for_segments(dataset, segment_ids, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, segment_ids, session=session)else:# Delete all summaries for the dataset- SummaryIndexService.delete_summaries_for_segments(dataset, None, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, None, session=session)if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:delete_child_chunks = kwargs.get("delete_child_chunks") or False@@ -227,7 +236,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor):)session.flush()- def _split_child_nodes(+ def split_child_nodes(self,document_node: Document,rules: Rule,@@ -302,7 +311,12 @@ class ParentChildIndexProcessor(BaseIndexProcessor):account = AccountService.load_user(document.created_by, account_session)if not account:raise ValueError("Invalid account")- doc.attachments = self._get_content_files(doc, current_user=account, session=session)+ doc.attachments = self._get_content_files(+ doc,+ current_user=account,+ tenant_id=dataset.tenant_id,+ session=session,+ )documents.append(doc)if documents:token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)diff --git a/api/core/rag/index_processor/processor/qa_index_processor.py b/api/core/rag/index_processor/processor/qa_index_processor.pyindex e1971da27fa..c39a83d4489 100644--- a/api/core/rag/index_processor/processor/qa_index_processor.py+++ b/api/core/rag/index_processor/processor/qa_index_processor.py@@ -29,7 +29,7 @@ from libs import helperfrom models.account import Accountfrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocument-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterlogger = logging.getLogger(__name__)@@ -180,10 +180,10 @@ class QAIndexProcessor(BaseIndexProcessor):).all()segment_ids = [segment.id for segment in segments]if segment_ids:- SummaryIndexService.delete_summaries_for_segments(dataset, segment_ids, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, segment_ids, session=session)else:# Delete all summaries for the dataset- SummaryIndexService.delete_summaries_for_segments(dataset, None, session=session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, None, session=session)vector = Vector(dataset, session=session)if node_ids:diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.pyindex 2caa266c37c..04002bc87d1 100644--- a/api/core/rag/retrieval/dataset_retrieval.py+++ b/api/core/rag/retrieval/dataset_retrieval.py@@ -28,7 +28,6 @@ from core.app.entities.app_invoke_entities import (ModelConfigWithCredentialsEntity,get_credit_usage_app_type,)-from core.app.file_access import grant_retriever_segment_access, grant_upload_file_accessfrom core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandlerfrom core.db.session_factory import session_factoryfrom core.entities.agent_entities import PlanningStrategy@@ -83,7 +82,6 @@ from graphon.model_runtime.entities.model_entities import ModelFeature, ModelTypfrom graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModelfrom libs.helper import parse_uuid_str_or_nonefrom libs.json_in_md_parser import parse_and_check_json_markdown-from models import UploadFilefrom models.dataset import (ChildChunk,Dataset,@@ -96,8 +94,11 @@ from models.dataset import (from models.dataset import Document as DatasetDocumentfrom models.dataset import Document as DocumentModelfrom models.enums import CreatorUserRole, DatasetQuerySource-from services.external_knowledge_service import ExternalDatasetService+from repositories.knowledge.dataset_read_repository import get_dataset_available_document_count+from repositories.knowledge.segment_read_adapter import sign_segment_contentfrom services.feature_service import FeatureService+from services.knowledge.external.service import ExternalDatasetService+from services.knowledge.retrieval.attachments import authorize_retrieved_segmentdefault_retrieval_model: DefaultRetrievalModelDict = {"search_method": RetrievalMethod.SEMANTIC_SEARCH,@@ -310,25 +311,36 @@ class DatasetRetrieval:retrieval_resource_list.append(source)# deal with dify documentsif dify_documents:- with Session(bind=session.get_bind()) as format_session:- records = RetrievalService.format_retrieval_documents(format_session, dify_documents)- dataset_ids = [i.segment.dataset_id for i in records]- document_ids = [i.segment.document_id for i in records]-- with session_factory.create_session() as session:- datasets = session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()- documents = session.scalars(select(DatasetDocument).where(DatasetDocument.id.in_(document_ids))).all()+ # Materialize authorized, signed response values before closing the owned session.+ with Session(bind=session.get_bind()) as retrieval_session:+ records = RetrievalService.format_retrieval_documents(retrieval_session, dify_documents)+ dataset_ids = [i.segment.dataset_id for i in records]+ document_ids = [i.segment.document_id for i in records]+ datasets = retrieval_session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()+ documents = retrieval_session.scalars(+ select(DatasetDocument).where(DatasetDocument.id.in_(document_ids))+ ).all()++ dataset_map = {i.id: i for i in datasets}+ document_map = {i.id: i for i in documents}- dataset_map = {i.id: i for i in datasets}- document_map = {i.id: i for i in documents}-- if records:for record in records:segment = record.segment+ if (+ authorize_retrieved_segment(+ segment,+ tenant_id=request.tenant_id,+ dataset_ids=available_datasets_ids,+ session=retrieval_session,+ )+ is None+ ):+ continuedataset = dataset_map.get(segment.dataset_id)document = document_map.get(segment.document_id)if dataset and document:+ content = sign_segment_content(segment, session=retrieval_session)source = Source(metadata=SourceMetadata(source="knowledge",@@ -358,15 +370,14 @@ class DatasetRetrieval:),title=document.name,files=list(record.files) if record.files else None,- content=segment.get_sign_content(),+ content=content,)if segment.answer:- source.content = f"question:{segment.get_sign_content()} \nanswer:{segment.answer}"+ source.content = f"question:{content} \nanswer:{segment.answer}"if record.summary:source.summary = record.summary- grant_retriever_segment_access([str(segment.id)])retrieval_resource_list.append(source)if retrieval_resource_list:@@ -533,13 +544,22 @@ class DatasetRetrieval:with Session(bind=session.get_bind()) as format_session:records = RetrievalService.format_retrieval_documents(format_session, dify_documents)if records:+ authorized_records = []for record in records:segment = record.segment+ attachments = authorize_retrieved_segment(+ segment, tenant_id=tenant_id, dataset_ids=available_datasets_ids, session=session+ )+ if attachments is None:+ continue+ authorized_records.append(record)# Build content: if summary exists, add it before the segment contentif segment.answer:- segment_content = f"question:{segment.get_sign_content()} answer:{segment.answer}"+ segment_content = (+ f"question:{sign_segment_content(segment, session=session)} answer:{segment.answer}"+ )else:- segment_content = segment.get_sign_content()+ segment_content = sign_segment_content(segment, session=session)# If summary exists, prepend it to the contentif record.summary:@@ -554,18 +574,8 @@ class DatasetRetrieval:))if vision_enabled:- attachments_with_bindings = session.execute(- select(SegmentAttachmentBinding, UploadFile)- .join(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)- .where(- SegmentAttachmentBinding.segment_id == segment.id,- )- ).all()- if attachments_with_bindings:- grant_upload_file_access(- str(upload_file.id) for _, upload_file in attachments_with_bindings- )- for _, upload_file in attachments_with_bindings:+ if attachments:+ for upload_file in attachments:attachment_info = File(file_id=upload_file.id,filename=upload_file.name,@@ -583,8 +593,8 @@ class DatasetRetrieval:)context_files.append(attachment_info)if show_retrieve_source:- dataset_ids = [record.segment.dataset_id for record in records]- document_ids = [record.segment.document_id for record in records]+ dataset_ids = [record.segment.dataset_id for record in authorized_records]+ document_ids = [record.segment.document_id for record in authorized_records]dataset_document_stmt = select(DatasetDocument).where(DatasetDocument.id.in_(document_ids),DatasetDocument.enabled == True,@@ -597,7 +607,7 @@ class DatasetRetrieval:datasets = session.execute(dataset_stmt).scalars().all() # type: ignoredataset_map = {i.id: i for i in datasets}document_map = {i.id: i for i in documents}- for record in records:+ for record in authorized_records:segment = record.segmentdataset_item = dataset_map.get(segment.dataset_id)document_item = document_map.get(segment.document_id)@@ -1333,7 +1343,7 @@ class DatasetRetrieval:if (datasetand dataset.provider != "external"- and dataset.get_total_available_documents(session=session) == 0+ and get_dataset_available_document_count(dataset, session=session) == 0):continuediff --git a/api/core/rag/summary_index/summary_index.py b/api/core/rag/summary_index/summary_index.pyindex 89788c6ab53..81fe9a1be73 100644--- a/api/core/rag/summary_index/summary_index.py+++ b/api/core/rag/summary_index/summary_index.py@@ -8,7 +8,7 @@ from core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.index_processor.index_processor_base import SummaryIndexSettingDictfrom extensions.otel import propagate_contextfrom models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummary-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterfrom tasks.generate_summary_index_task import generate_summary_index_tasklogger = logging.getLogger(__name__)@@ -82,7 +82,7 @@ class SummaryIndex:if segment is None:returntry:- SummaryIndexService.generate_and_vectorize_summary(+ SummaryIndexAdapter.generate_and_vectorize_summary(segment, dataset, summary_index_setting, session=session)except Exception:diff --git a/api/core/tools/builtin_tool/providers/audio/tools/asr.py b/api/core/tools/builtin_tool/providers/audio/tools/asr.pyindex c315764c3b3..44543667815 100644--- a/api/core/tools/builtin_tool/providers/audio/tools/asr.py+++ b/api/core/tools/builtin_tool/providers/audio/tools/asr.py@@ -14,7 +14,7 @@ from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameterfrom graphon.file import FileTypefrom graphon.file.file_manager import downloadfrom graphon.model_runtime.entities.model_entities import ModelType-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceclass ASRTool(BuiltinTool):diff --git a/api/core/tools/builtin_tool/providers/audio/tools/tts.py b/api/core/tools/builtin_tool/providers/audio/tools/tts.pyindex a8083e74e25..8350af1e040 100644--- a/api/core/tools/builtin_tool/providers/audio/tools/tts.py+++ b/api/core/tools/builtin_tool/providers/audio/tools/tts.py@@ -12,7 +12,7 @@ from core.tools.builtin_tool.tool import BuiltinToolfrom core.tools.entities.common_entities import I18nObjectfrom core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameterfrom graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceclass TTSTool(BuiltinTool):diff --git a/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py b/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.pyindex da7c10d5ac4..3f2d64e3434 100644--- a/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py+++ b/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py@@ -18,6 +18,7 @@ from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import Datasfrom extensions.ext_database import dbfrom graphon.model_runtime.entities.model_entities import ModelTypefrom models.dataset import Dataset, Document, DocumentSegment+from repositories.knowledge.segment_read_adapter import sign_segment_contentdefault_retrieval_model: DefaultRetrievalModelDict = {"search_method": RetrievalMethod.SEMANTIC_SEARCH,@@ -105,9 +106,11 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):)for segment in sorted_segments:if segment.answer:- document_context_list.append(f"question:{segment.get_sign_content()} answer:{segment.answer}")+ document_context_list.append(+ f"question:{sign_segment_content(segment, session=session)} answer:{segment.answer}"+ )else:- document_context_list.append(segment.get_sign_content())+ document_context_list.append(sign_segment_content(segment, session=session))if self.return_resource:context_list: list[RetrievalSourceMetadata] = []resource_number = 1diff --git a/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py b/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.pyindex 489170053a8..ff0a1c16930 100644--- a/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py+++ b/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py@@ -14,7 +14,8 @@ from core.rag.retrieval.retrieval_methods import RetrievalMethodfrom core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseToolfrom models.dataset import Datasetfrom models.dataset import Document as DatasetDocument-from services.external_knowledge_service import ExternalDatasetService+from repositories.knowledge.segment_read_adapter import sign_segment_content+from services.knowledge.external.service import ExternalDatasetServicedefault_retrieval_model: DefaultRetrievalModelDict = {"search_method": RetrievalMethod.SEMANTIC_SEARCH,@@ -175,9 +176,11 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):segment = record.segment# Build content: if summary exists, add it before the segment contentif segment.answer:- segment_content = f"question:{segment.get_sign_content()} answer:{segment.answer}"+ segment_content = (+ f"question:{sign_segment_content(segment, session=session)} answer:{segment.answer}"+ )else:- segment_content = segment.get_sign_content()+ segment_content = sign_segment_content(segment, session=session)# If summary exists, prepend it to the contentif record.summary:diff --git a/api/core/workflow/node_factory.py b/api/core/workflow/node_factory.pyindex 1d868c58f9c..7f67415fb05 100644--- a/api/core/workflow/node_factory.py+++ b/api/core/workflow/node_factory.py@@ -11,7 +11,7 @@ from configs import dify_configfrom core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContextfrom core.app.llm.model_access import build_dify_model_access, fetch_model_configfrom core.credit_usage import created_by_from_app_type-from core.db.session_factory import session_factory+from core.db.session_factory import get_session_maker, session_factoryfrom core.file import remote_fetcherfrom core.helper.code_executor.code_executor import (CodeExecutionError,@@ -55,6 +55,7 @@ from core.workflow.nodes.human_input.callback import DifyHITLCallbackfrom core.workflow.nodes.human_input.entities import HumanInputNodeData as DifyHumanInputNodeDatafrom core.workflow.system_variables import SystemVariableKey, get_system_text, system_variable_selectorfrom core.workflow.template_rendering import CodeExecutorJinja2TemplateRenderer+from extensions.application_services.data_sources import build_data_source_credentialsfrom graphon.entities.base_node_data import BaseNodeDatafrom graphon.entities.graph_config import NodeConfigDict, NodeConfigDictAdapterfrom graphon.enums import BuiltinNodeTypes, NodeType@@ -431,6 +432,11 @@ class DifyNodeFactory(NodeFactory):resolved_node_data = self._resolve_llm_model_reference(cast(LLMNodeData, resolved_node_data))node: Node | None = Nonenode_init_kwargs_factories: Mapping[NodeType, Callable[[], dict[str, object]]] = {+ BuiltinNodeTypes.DATASOURCE: lambda: {+ "datasource_credentials": build_data_source_credentials(+ database_client=get_session_maker()+ ).providers.get_datasource_credentials,+ },BuiltinNodeTypes.CODE: lambda: {"code_executor": self._code_executor,"code_limits": self._code_limits,diff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.pyindex f2ded99db3e..f90c06537aa 100644--- a/api/core/workflow/nodes/datasource/datasource_node.py+++ b/api/core/workflow/nodes/datasource/datasource_node.py@@ -1,4 +1,4 @@-from collections.abc import Generator, Mapping, Sequence+from collections.abc import Callable, Generator, Mapping, Sequencefrom typing import TYPE_CHECKING, Any, overridefrom core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContext@@ -40,6 +40,7 @@ class DatasourceNode(Node[DatasourceNodeData]):*,graph_init_params: "GraphInitParams",graph_runtime_state: "GraphRuntimeState",+ datasource_credentials: Callable[[str, str, str, str | None], dict[str, Any]],) -> None:super().__init__(node_id=node_id,@@ -48,6 +49,7 @@ class DatasourceNode(Node[DatasourceNodeData]):graph_runtime_state=graph_runtime_state,)self.datasource_manager = DatasourceManager+ self._datasource_credentials = datasource_credentials@overridedef populate_start_event(self, event) -> None:@@ -117,9 +119,9 @@ class DatasourceNode(Node[DatasourceNodeData]):datasource_type=datasource_type.value,provider_id=provider_id,tenant_id=dify_ctx.tenant_id,- provider=node_data.provider_name,- plugin_id=node_data.plugin_id,- credential_id=credential_id,+ credentials=self._datasource_credentials(+ dify_ctx.tenant_id, node_data.provider_name, node_data.plugin_id, credential_id+ ),parameters_for_log=parameters_for_log,datasource_info=datasource_info,variable_pool=variable_pool,diff --git a/api/core/workflow/nodes/datasource/protocols.py b/api/core/workflow/nodes/datasource/protocols.pydeleted file mode 100644index 776e2673172..00000000000--- a/api/core/workflow/nodes/datasource/protocols.py+++ /dev/null@@ -1,35 +0,0 @@-from collections.abc import Generator-from typing import Any, Protocol--from graphon.file import File-from graphon.node_events import StreamChunkEvent, StreamCompletedEvent--from .entities import DatasourceParameter, OnlineDriveDownloadFileParam---class DatasourceManagerProtocol(Protocol):- @classmethod- def get_icon_url(cls, provider_id: str, tenant_id: str, datasource_name: str, datasource_type: str) -> str: ...-- @classmethod- def stream_node_events(- cls,- *,- node_id: str,- user_id: str,- datasource_name: str,- datasource_type: str,- provider_id: str,- tenant_id: str,- provider: str,- plugin_id: str,- credential_id: str,- parameters_for_log: dict[str, Any],- datasource_info: dict[str, Any],- variable_pool: Any,- datasource_param: DatasourceParameter | None = None,- online_drive_request: OnlineDriveDownloadFileParam | None = None,- ) -> Generator[StreamChunkEvent | StreamCompletedEvent, None, None]: ...-- @classmethod- def get_upload_file_by_id(cls, file_id: str, tenant_id: str) -> File: ...diff --git a/api/events/event_handlers/clean_when_dataset_deleted.py b/api/events/event_handlers/clean_when_dataset_deleted.pyindex ce0cf66f8f1..209317b7f78 100644--- a/api/events/event_handlers/clean_when_dataset_deleted.py+++ b/api/events/event_handlers/clean_when_dataset_deleted.py@@ -1,13 +1,14 @@from events.dataset_event import dataset_was_deletedfrom extensions.ext_database import dbfrom models import Dataset+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom tasks.clean_dataset_task import clean_dataset_task@dataset_was_deleted.connectdef handle(sender: Dataset, **kwargs):dataset = sender- doc_form = dataset.get_doc_form(session=db.session())+ doc_form = get_dataset_doc_form(dataset, session=db.session())if not doc_form or not dataset.indexing_technique:returnclean_dataset_task.delay(diff --git a/api/events/event_handlers/create_document_index.py b/api/events/event_handlers/create_document_index.pyindex aba387dd187..427a340baa2 100644--- a/api/events/event_handlers/create_document_index.py+++ b/api/events/event_handlers/create_document_index.py@@ -6,11 +6,13 @@ from sqlalchemy import selectfrom werkzeug.exceptions import NotFoundfrom core.db.session_factory import session_factory-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom events.document_index_event import document_index_createdfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Documentfrom models.enums import IndexingStatus+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -21,7 +23,9 @@ def handle(sender, **kwargs):document_ids = kwargs.get("document_ids", [])start_at = time.perf_counter()try:- indexing_runner = IndexingRunner(enforce_vector_space_admission=True)+ indexing_service = build_document_indexing_service(+ session_factory=session_factory.get_session_maker(), enforce_vector_space_admission=True+ )with session_factory.create_session() as session:documents = []for document_id in document_ids:@@ -41,11 +45,10 @@ def handle(sender, **kwargs):document.processing_started_at = naive_utc_now()documents.append(document)session.add(document)+ document_refs = [DatasetRef(doc.tenant_id, doc.dataset_id).document(doc.id) for doc in documents]# Persist the status transition before extraction and indexing begin.session.commit()-- indexing_runner.run(documents, session)- session.commit()+ indexing_service.run(document_refs)end_at = time.perf_counter()logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))except DocumentIsPausedError as ex:diff --git a/api/extensions/application_services/data_sources.py b/api/extensions/application_services/data_sources.pynew file mode 100644index 00000000000..0818c635e47--- /dev/null+++ b/api/extensions/application_services/data_sources.py@@ -0,0 +1,124 @@+"""Composition root for data-source services and shared credential gateways."""++from collections.abc import Mapping+from dataclasses import dataclass++from sqlalchemy.orm import Session, sessionmaker++from configs import dify_config+from core.datasource.datasource_manager import DatasourceManager+from core.helper.ssrf_proxy import ssrf_proxy+from core.plugin.impl.datasource import PluginDatasourceManager+from core.plugin.impl.oauth import OAuthHandler+from repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from repositories.data_source.oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from services.data_source.auth.api_key_gateways import (+ ProviderApiKeyAuthCredentialValidator,+ TenantApiKeyAuthCredentialEncryptor,+)+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService+from services.data_source.binding_application_service import DataSourceBindingApplicationService+from services.data_source.credential_adapters import (+ OAuthDatasourceCredentialRefresher,+ PluginDatasourceCredentialCodec,+ PluginDatasourceOAuthClientResolver,+)+from services.data_source.credential_gateway import (+ ActorAwareDatasourceCredentialGateway,+ TrustedStoredDatasourceCredentialGateway,+)+from services.data_source.notion_import_adapters import PluginNotionSourceGateway+from services.data_source.notion_import_application_service import NotionImportApplicationService+from services.data_source.notion_oauth_gateway import NotionDataSourceGateway+from services.data_source.oauth_service import DataSourceOAuthService, InvalidDataSourceOAuthProviderError+from services.data_source.provider_service import DatasourceProviderService+from services.data_source.website_service import WebsiteService+from services.knowledge.dataset_access import DatasetAccessService+++@dataclass(frozen=True, slots=True)+class DataSourceServices:+ providers: DatasourceProviderService+ api_key_auth: DataSourceApiKeyAuthService+ oauth: Mapping[str, DataSourceOAuthService]+ bindings: DataSourceBindingApplicationService+ notion_imports: NotionImportApplicationService++ def resolve_oauth(self, provider: str) -> DataSourceOAuthService:+ service = self.oauth.get(provider)+ if service is None:+ raise InvalidDataSourceOAuthProviderError("Invalid provider")+ return service+++@dataclass(frozen=True, slots=True)+class DataSourceCredentials:+ providers: DatasourceProviderService+ actor: ActorAwareDatasourceCredentialGateway+ stored: TrustedStoredDatasourceCredentialGateway+++def build_data_source_credentials(*, database_client: sessionmaker[Session]) -> DataSourceCredentials:+ credentials = SQLAlchemyDatasourceCredentialRepository(session_factory=database_client)+ provider_manager = PluginDatasourceManager()+ codec = PluginDatasourceCredentialCodec(provider_manager=provider_manager)+ refresher = OAuthDatasourceCredentialRefresher(+ oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=provider_manager),+ oauth_handler=OAuthHandler(),+ )+ return DataSourceCredentials(+ providers=DatasourceProviderService(credentials=credentials),+ actor=ActorAwareDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),+ stored=TrustedStoredDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),+ )+++def build_data_source_services(+ *,+ database_client: sessionmaker[Session],+ dataset_access: DatasetAccessService,+ datasets: SQLAlchemyDatasetRepository,+ documents: SQLAlchemyDocumentRepository,+ actor_credentials: ActorAwareDatasourceCredentialGateway,+ providers: DatasourceProviderService,+) -> DataSourceServices:+ oauth_bindings = SQLAlchemyDataSourceOAuthBindingRepository(session_factory=database_client)+ notion_gateway = NotionDataSourceGateway(+ client_id=dify_config.NOTION_CLIENT_ID or "",+ client_secret=dify_config.NOTION_CLIENT_SECRET or "",+ redirect_uri=dify_config.CONSOLE_API_URL + "/console/api/oauth/data-source/callback/notion",+ http_client=ssrf_proxy,+ )+ return DataSourceServices(+ providers=providers,+ api_key_auth=DataSourceApiKeyAuthService(+ bindings=SQLAlchemyDataSourceApiKeyAuthBindingRepository(session_factory=database_client),+ validator=ProviderApiKeyAuthCredentialValidator(),+ encryptor=TenantApiKeyAuthCredentialEncryptor(),+ ),+ oauth={+ "notion": DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=notion_gateway,+ bindings=oauth_bindings,+ is_internal_provider=dify_config.NOTION_INTEGRATION_TYPE == "internal",+ internal_access_token=dify_config.NOTION_INTERNAL_SECRET,+ )+ },+ bindings=DataSourceBindingApplicationService(bindings=oauth_bindings),+ notion_imports=NotionImportApplicationService(+ dataset_access=dataset_access,+ datasets=datasets,+ documents=documents,+ source=PluginNotionSourceGateway(+ credentials=actor_credentials, runtime_loader=DatasourceManager.get_datasource_runtime+ ),+ ),+ )+++def build_website_service(*, database_client: sessionmaker[Session]) -> WebsiteService:+ return WebsiteService(providers=build_data_source_credentials(database_client=database_client).providers)diff --git a/api/extensions/application_services/datasets.py b/api/extensions/application_services/datasets.pynew file mode 100644index 00000000000..053282d6beb--- /dev/null+++ b/api/extensions/application_services/datasets.py@@ -0,0 +1,32 @@+"""Shared dataset dependencies for knowledge and data-source use cases."""++from dataclasses import dataclass++from sqlalchemy.orm import Session, sessionmaker++from configs import dify_config+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from services.knowledge.dataset_access import DatasetAccessService, WorkspaceRoleReader+++@dataclass(frozen=True, slots=True)+class DatasetDependencies:+ datasets: SQLAlchemyDatasetRepository+ documents: SQLAlchemyDocumentRepository+ access: DatasetAccessService+++def build_dataset_dependencies(+ *, database_client: sessionmaker[Session], workspace_roles: WorkspaceRoleReader+) -> DatasetDependencies:+ datasets = SQLAlchemyDatasetRepository(session_factory=database_client)+ return DatasetDependencies(+ datasets=datasets,+ documents=SQLAlchemyDocumentRepository(session_factory=database_client),+ access=DatasetAccessService(+ datasets=datasets,+ workspace_roles=workspace_roles,+ legacy_permissions_enabled=not dify_config.RBAC_ENABLED,+ ),+ )diff --git a/api/extensions/application_services/knowledge.py b/api/extensions/application_services/knowledge.pyindex e3300814ae3..91733434af4 100644--- a/api/extensions/application_services/knowledge.py+++ b/api/extensions/application_services/knowledge.py@@ -1,16 +1,174 @@-"""Composition of knowledge base application services."""+"""Composition root for dataset-controller application services."""++from dataclasses import dataclass+from uuid import uuid4from sqlalchemy.orm import Session, sessionmakerfrom configs import dify_config+from core.app.apps.pipeline.pipeline_generator import PipelineGenerator+from core.rag.extractor.entity.datasource_type import DatasourceType+from libs.helper import generate_text_hashfrom repositories.knowledge.dataset_api_key_repository import DatasetApiKeyRepository+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepositoryfrom services.api_token_service import ApiTokenCache+from services.app.query_service import AppQueryService+from services.data_source.credential_gateway import (+ ActorAwareDatasourceCredentialGateway,+ TrustedStoredDatasourceCredentialGateway,+)+from services.data_source.provider_service import DatasourceProviderServicefrom services.knowledge.api_key_service import DatasetApiKeyService+from services.knowledge.dataset_access import DatasetAccessService+from services.knowledge.dataset_service import DocumentService+from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations+from services.knowledge.datasets.application import DatasetApplicationService+from services.knowledge.document_sync import DocumentSyncApplicationService+from services.knowledge.document_sync_adapters import CeleryDocumentSyncDispatcher+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations+from services.knowledge.documents.application import DatasetDocumentApplicationService+from services.knowledge.external.adapters import SQLAlchemyExternalKnowledgeOperations+from services.knowledge.external.application import ExternalKnowledgeApplicationService+from services.knowledge.indexing.adapters.estimate import IndexingEstimateAdapter, SQLAlchemyProcessRuleReader+from services.knowledge.indexing.adapters.sources import (+ CompositeStoredSourceResolver,+ FileSourceAdapter,+ NotionSourceResolver,+ WebsiteSourceAdapter,+)+from services.knowledge.indexing.estimate import IndexingEstimateApplicationService+from services.knowledge.metadata.application import MetadataService+from services.knowledge.segments.adapters import (+ CelerySegmentBatchImportDispatcher,+ ModelManagerSegmentGuard,+ RedisSegmentClient,+ RedisSegmentIndexingState,+)+from services.knowledge.segments.application import DatasetSegmentApplicationService+from services.knowledge.segments.indexing import SegmentIndexingGateway+from services.tag_application_service import TagTargetQuery+from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task+from tasks.delete_segment_from_index_task import delete_segment_from_index_task+from tasks.disable_segments_from_index_task import disable_segments_from_index_task+from tasks.document_indexing_sync_task import document_indexing_sync_task+from tasks.enable_segments_to_index_task import enable_segments_to_index_task+++@dataclass(frozen=True, slots=True)+class KnowledgeServices:+ metadata: MetadataService+ datasets: DatasetApplicationService+ external: ExternalKnowledgeApplicationService+ documents: DatasetDocumentApplicationService+ document_sync: DocumentSyncApplicationService+ indexing_estimates: IndexingEstimateApplicationService+ segments: DatasetSegmentApplicationService+ pipeline_generator: PipelineGenerator-def build_dataset_api_key_service(*, database_client: sessionmaker[Session]) -> DatasetApiKeyService:+def build_dataset_api_key_service(+ *, database_client: sessionmaker[Session], dataset_access: DatasetAccessService+) -> DatasetApiKeyService:return DatasetApiKeyService(keys=DatasetApiKeyRepository(session_factory=database_client),cache=ApiTokenCache,+ access=dataset_access,rbac_enabled=lambda: dify_config.RBAC_ENABLED,)+++def build_knowledge_services(+ *,+ database_client: sessionmaker[Session],+ dataset_access: DatasetAccessService,+ datasets: SQLAlchemyDatasetRepository,+ documents: SQLAlchemyDocumentRepository,+ actor_credentials: ActorAwareDatasourceCredentialGateway,+ stored_credentials: TrustedStoredDatasourceCredentialGateway,+ providers: DatasourceProviderService,+ redis: RedisSegmentClient,+ tags: TagTargetQuery,+ app_queries: AppQueryService,+) -> KnowledgeServices:+ """Build the dataset-controller knowledge use cases."""++ uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=database_client)+ notion_sources = NotionSourceResolver(+ actor_credentials=actor_credentials,+ stored_credentials=stored_credentials,+ )+ file_sources = FileSourceAdapter(uploads=uploads)+ website_sources = WebsiteSourceAdapter()+ stored_sources = CompositeStoredSourceResolver(+ adapters={+ DatasourceType.FILE.value: file_sources,+ DatasourceType.NOTION.value: notion_sources,+ DatasourceType.WEBSITE.value: website_sources,+ }+ )+ segments = SQLAlchemySegmentRepository(session_factory=database_client)+ segment_index = SegmentIndexingGateway(+ segments=segments,+ uploads=uploads,+ redis=redis,+ new_session=database_client,+ delete_task=delete_segment_from_index_task.delay,+ enable_task=enable_segments_to_index_task.delay,+ disable_task=disable_segments_from_index_task.delay,+ )+ return KnowledgeServices(+ metadata=MetadataService(+ store=SQLAlchemyMetadataRepository(session_factory=database_client), dataset_access=dataset_access+ ),+ datasets=DatasetApplicationService(+ dataset_access=dataset_access,+ operations=SQLAlchemyDatasetOperations(session_factory=database_client, tags=tags, app_queries=app_queries),+ rbac_enabled=dify_config.RBAC_ENABLED,+ service_api_url=dify_config.SERVICE_API_URL,+ vector_store=dify_config.VECTOR_STORE,+ tidb_fulltext=dify_config.TIDB_VECTOR_ENABLE_FULLTEXT_SEARCH,+ ),+ external=ExternalKnowledgeApplicationService(+ dataset_access=dataset_access,+ operations=SQLAlchemyExternalKnowledgeOperations(session_factory=database_client),+ ),+ documents=DatasetDocumentApplicationService(+ dataset_access=dataset_access,+ operations=SQLAlchemyDocumentOperations(session_factory=database_client),+ metadata_schema=DocumentService.DOCUMENT_METADATA_SCHEMA,+ ),+ pipeline_generator=PipelineGenerator(documents=documents, datasource_providers=providers),+ document_sync=DocumentSyncApplicationService(+ dataset_access=dataset_access,+ documents=documents,+ dispatcher=CeleryDocumentSyncDispatcher(delay=document_indexing_sync_task.delay),+ ),+ indexing_estimates=IndexingEstimateApplicationService(+ dataset_access=dataset_access,+ datasets=datasets,+ documents=documents,+ files=file_sources,+ websites=website_sources,+ stored_sources=stored_sources,+ notion=notion_sources,+ process_rules=SQLAlchemyProcessRuleReader(session_factory=database_client),+ runner=IndexingEstimateAdapter(session_factory=database_client),+ ),+ segments=DatasetSegmentApplicationService(+ dataset_access=dataset_access,+ scopes=datasets,+ store=segments,+ index=segment_index,+ limits=dify_config,+ text_hash=generate_text_hash,+ uploads=uploads,+ model_guard=ModelManagerSegmentGuard(),+ indexing_state=RedisSegmentIndexingState(redis),+ batch_dispatcher=CelerySegmentBatchImportDispatcher(delay=batch_create_segment_to_index_task.delay),+ job_id_factory=lambda: str(uuid4()),+ ),+ )diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex 6c5c343cd0c..0aa35bf44b1 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -19,14 +19,23 @@ from configs import dify_configfrom constants.dsl_version import CURRENT_APP_DSL_VERSIONfrom constants.languages import languagesfrom core.db.session_factory import get_session_maker-from core.helper.ssrf_proxy import ssrf_proxyfrom core.schemas.schema_manager import SchemaManagerfrom core.tools.tool_file_manager import ToolFileManagerfrom enums import DeploymentEdition, WebAppAccessModefrom extensions.application_services.agent import AgentAppServices, build_agent_app_servicesfrom extensions.application_services.app import AppServices, build_app_api_key_service, build_app_services+from extensions.application_services.data_sources import (+ DataSourceServices,+ build_data_source_credentials,+ build_data_source_services,+)+from extensions.application_services.datasets import build_dataset_dependenciesfrom extensions.application_services.installed_app import InstalledAppServices, build_installed_app_services-from extensions.application_services.knowledge import build_dataset_api_key_service+from extensions.application_services.knowledge import (+ KnowledgeServices,+ build_dataset_api_key_service,+ build_knowledge_services,+)from extensions.application_services.trial_app import TrialAppServices, build_trial_app_servicesfrom extensions.ext_redis import RedisClientWrapper, redis_clientfrom extensions.ext_storage import storage@@ -52,8 +61,7 @@ from repositories.app_definition_query_repository import AppDefinitionQueryReposfrom repositories.app_preview_query_repository import AppPreviewQueryRepositoryfrom repositories.app_scoped_end_user_repository import AppScopedEndUserRepofrom repositories.app_statistic_query_repository import AppStatisticQueryRepository-from repositories.data_source_api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository-from repositories.data_source_oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository+from repositories.credentials.query_repository import CredentialQueryRepositoryfrom repositories.explore_banner_query_repository import ExploreBannerQueryRepositoryfrom repositories.factory import DifyAPIRepositoryFactoryfrom repositories.file_grant_repository import FileGrantRepository@@ -164,15 +172,10 @@ from services.app_statistic_query import AppStatisticQueryfrom services.app_task_service import AppTaskControlServicefrom services.app_tracing_config_gateway import OpsTraceManagerGatewayfrom services.app_tracing_config_service import AppTracingConfigService-from services.auth.data_source_api_key_auth_gateways import (- ProviderApiKeyAuthCredentialValidator,- TenantApiKeyAuthCredentialEncryptor,-)-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthServicefrom services.billing_portal_service import BillingPortalServicefrom services.billing_service import BillingServicefrom services.compliance_download_service import ComplianceDownloadService-from services.data_source_oauth_service import DataSourceOAuthService, InvalidDataSourceOAuthProviderError+from services.credentials.query import CredentialQueryfrom services.enterprise.enterprise_service import EnterpriseServicefrom services.entities.file_grant_entities import FileGrantLimitsfrom services.errors.enterprise import EnterpriseServiceError@@ -191,7 +194,6 @@ from services.message_suggested_questions_adapters import MessageSuggestedQuestifrom services.message_suggested_questions_service import MessageSuggestedQuestionsfrom services.notification_gateway import BillingNotificationGatewayfrom services.notification_service import NotificationService-from services.notion_data_source_gateway import NotionDataSourceGatewayfrom services.oauth_device_adapters import (DifyConfigOAuthDeviceSettings,EnterpriseOAuthDeviceSSOGateway,@@ -337,6 +339,7 @@ class AppScopedEndUserServices:class ApplicationServices:agent_apps: AgentAppServicesadvanced_prompt_templates: AdvancedPromptTemplateService+ credential_queries: CredentialQueryaccounts: AccountServicesapp_api_keys: AppApiKeyServicedataset_api_keys: DatasetApiKeyService@@ -351,8 +354,8 @@ class ApplicationServices:app_tracing_configs: AppTracingConfigServicebilling_portal: BillingPortalServicecompliance_downloads: ComplianceDownloadService- data_source_api_key_auth: DataSourceApiKeyAuthService- data_source_oauth: Mapping[str, DataSourceOAuthService]+ data_sources: DataSourceServices+ knowledge: KnowledgeServicesapp_scoped_end_users: AppScopedEndUserServiceswebapp_access: WebAppAccessQueryServiceweb_app_runtime: WebAppRuntimeQueryService@@ -392,34 +395,6 @@ class ApplicationServices:tags: TagApplicationServiceworkflow_statistics: WorkflowStatisticQueryService- def resolve_data_source_oauth(self, provider: str) -> DataSourceOAuthService:- service = self.data_source_oauth.get(provider)- if service is None:- raise InvalidDataSourceOAuthProviderError("Invalid provider")- return service---def _build_data_source_oauth_services(- *,- database_client: sessionmaker[Session],-) -> Mapping[str, DataSourceOAuthService]:- notion_data_source = NotionDataSourceGateway(- client_id=dify_config.NOTION_CLIENT_ID or "",- client_secret=dify_config.NOTION_CLIENT_SECRET or "",- redirect_uri=dify_config.CONSOLE_API_URL + "/console/api/oauth/data-source/callback/notion",- http_client=ssrf_proxy,- )- bindings = SQLAlchemyDataSourceOAuthBindingRepository(session_factory=database_client)- return {- "notion": DataSourceOAuthService(- provider_name="notion",- provider_gateway=notion_data_source,- bindings=bindings,- is_internal_provider=dify_config.NOTION_INTEGRATION_TYPE == "internal",- internal_access_token=dify_config.NOTION_INTERNAL_SECRET,- )- }-def _build_oauth_server_service(*,@@ -547,7 +522,6 @@ def build_application_services(redis: RedisClientWrapper,) -> ApplicationServices:installation_state = InstallationStateRepository(session_factory=database_client)- data_source_api_key_auth_bindings = SQLAlchemyDataSourceApiKeyAuthBindingRepository(session_factory=database_client)app_definition_repository = AppDefinitionQueryRepository(session_factory=database_client)app_definitions = AppDefinitionQueryService(definitions=app_definition_repository,@@ -587,6 +561,39 @@ def build_application_services(sources=database_catalog, exporter=RosterAgentPackageExporter())workspace_query_repository = WorkspaceQueryRepository(session_factory=database_client)+ workspace_member_repository = WorkspaceMemberQueryRepository(session_factory=database_client)+ dataset_dependencies = build_dataset_dependencies(+ database_client=database_client,+ workspace_roles=workspace_member_repository,+ )+ datasource_credentials = build_data_source_credentials(database_client=database_client)+ data_sources = build_data_source_services(+ database_client=database_client,+ dataset_access=dataset_dependencies.access,+ datasets=dataset_dependencies.datasets,+ documents=dataset_dependencies.documents,+ actor_credentials=datasource_credentials.actor,+ providers=datasource_credentials.providers,+ )+ oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis)+ apps = build_app_services(+ database_client=database_client,+ oauth=oauth_server,+ recommended_packages=recommended_app_packages,+ )+ tags = TagApplicationService(tags=TagRepository(session_factory=database_client))+ knowledge = build_knowledge_services(+ database_client=database_client,+ dataset_access=dataset_dependencies.access,+ datasets=dataset_dependencies.datasets,+ documents=dataset_dependencies.documents,+ actor_credentials=datasource_credentials.actor,+ stored_credentials=datasource_credentials.stored,+ providers=datasource_credentials.providers,+ redis=redis,+ tags=tags,+ app_queries=apps.queries,+ )app_scoped_end_user_repository = AppScopedEndUserRepo(session_factory=database_client)file_service = FileService(session_factory=database_client)remote_file_service = RemoteFileService(files=file_service)@@ -598,8 +605,8 @@ def build_application_services(workflow_node_execution_repository = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(session_maker=database_client)- oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis)return ApplicationServices(+ credential_queries=CredentialQueryRepository(session_factory=database_client),accounts=AccountServices(access=AccountAccessService(accounts=accounts,@@ -754,11 +761,7 @@ def build_application_services(enabled=dify_config.RBAC_ENABLED,),),- apps=build_app_services(- database_client=database_client,- oauth=oauth_server,- recommended_packages=recommended_app_packages,- ),+ apps=apps,agent_apps=build_agent_app_services(database_client=database_client),advanced_prompt_templates=AdvancedPromptTemplateService(),app_definitions=app_definitions,@@ -774,7 +777,10 @@ def build_application_services(sites=AppSiteCommandRepository(session_factory=database_client),),app_api_keys=build_app_api_key_service(database_client=database_client),- dataset_api_keys=build_dataset_api_key_service(database_client=database_client),+ dataset_api_keys=build_dataset_api_key_service(+ database_client=database_client,+ dataset_access=dataset_dependencies.access,+ ),app_statistics=AppStatisticQueryRepository(session_factory=database_client),app_tracing_configs=AppTracingConfigService(configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),@@ -794,12 +800,8 @@ def build_application_services(redis_client=redis,),),- data_source_api_key_auth=DataSourceApiKeyAuthService(- bindings=data_source_api_key_auth_bindings,- validator=ProviderApiKeyAuthCredentialValidator(),- encryptor=TenantApiKeyAuthCredentialEncryptor(),- ),- data_source_oauth=_build_data_source_oauth_services(database_client=database_client),+ data_sources=data_sources,+ knowledge=knowledge,app_scoped_end_users=AppScopedEndUserServices(commands=AppScopedEndUserService(end_users=app_scoped_end_user_repository),queries=AppScopedEndUserQueryService(end_users=app_scoped_end_user_repository),@@ -905,9 +907,7 @@ def build_application_services(plans=DeploymentWorkspacePlanGateway(),),workspace_member_queries=WorkspaceMemberQueryService(- members=WorkspaceMemberQueryRepository(- session_factory=database_client,- ),+ members=workspace_member_repository,roles=DeploymentWorkspaceMemberRoleResolver(),),workflow_app_logs=WorkflowAppLogQueryService(@@ -927,9 +927,7 @@ def build_application_services(now=lambda: datetime.now(UTC),access_token_expire_minutes=dify_config.ACCESS_TOKEN_EXPIRE_MINUTES,),- tags=TagApplicationService(- tags=TagRepository(session_factory=database_client),- ),+ tags=tags,workflow_statistics=WorkflowStatisticQueryService(workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker=database_client,diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.pyindex f4eff8a1012..a91009b4884 100644--- a/api/extensions/ext_celery.py+++ b/api/extensions/ext_celery.py@@ -176,6 +176,8 @@ def init_app(app: DifyApp) -> Celery:"tasks.trigger_processing_tasks", # async trigger processing"tasks.generate_summary_index_task", # summary index generation"tasks.regenerate_summary_index_task", # summary index regeneration+ "tasks.rag_pipeline.rag_pipeline_run_task", # regular RAG pipeline execution+ "tasks.rag_pipeline.priority_rag_pipeline_run_task", # priority RAG pipeline execution"tasks.initialize_created_app_rbac_access_task", # app access initialization"tasks.install_default_plugins_task", # tenant default plugin installation"tasks.new_agent_beta_task", # New Agent Beta eligibility checksdiff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.pyindex d70cea6d694..e6902f1cab2 100644--- a/api/fields/dataset_fields.py+++ b/api/fields/dataset_fields.py@@ -1,26 +1,21 @@-from collections.abc import Sequence-from dataclasses import dataclass, fieldfrom datetime import datetime-from typing import Any-import sqlalchemy as sa-from pydantic import Field, field_validator-from sqlalchemy import func, select-from sqlalchemy.orm import Session+from pydantic import BaseModel, Field, field_validatorfrom fields.base import ResponseModelfrom libs.helper import to_timestamp-from models.account import Account-from models.dataset import (- AppDatasetJoin,- Dataset,- DatasetMetadata,- Document,- ExternalKnowledgeApis,- ExternalKnowledgeBindings,- Pipeline,-)-from models.model import App, Tag, TagBinding+from services.data_source.entities.notion_import import NotionPageType+++class NotionEstimatePagePayload(BaseModel):+ page_id: str = Field(min_length=1)+ page_type: NotionPageType = Field(alias="type")+++class NotionEstimateWorkspacePayload(BaseModel):+ workspace_id: str = Field(min_length=1)+ credential_id: str = Field(min_length=1)+ pages: list[NotionEstimatePagePayload] = Field(min_length=1)class DatasetMetadataResponse(ResponseModel):@@ -206,270 +201,3 @@ class DatasetDetailResponse(ResponseModel):@classmethoddef _expand_null_nested(cls, value: object) -> object:return {} if value is None else value---@dataclass(frozen=True)-class DatasetDetailPrefetch:- """Batch-loaded values backing the session-scoped fields of a page of datasets.-- Building a detail response for a single dataset issues about ten queries for- that row alone, so a list endpoint costs one query per dataset per field.- Loading the same values for every dataset on the page up front keeps the- query count bounded regardless of the page size.- """-- app_counts: dict[str, int] = field(default_factory=dict)- document_counts: dict[str, int] = field(default_factory=dict)- word_counts: dict[str, int] = field(default_factory=dict)- available_document_counts: dict[str, int] = field(default_factory=dict)- author_names: dict[str, str] = field(default_factory=dict)- tags: dict[tuple[str, str], list[Tag]] = field(default_factory=dict)- doc_forms: dict[tuple[str, str], str | None] = field(default_factory=dict)- external_knowledge_infos: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)- doc_metadata: dict[str, list[DatasetMetadata]] = field(default_factory=dict)- pipeline_published: dict[str, bool] = field(default_factory=dict)---def build_dataset_detail_prefetch(datasets: Sequence[Dataset], *, session: Session) -> DatasetDetailPrefetch:- """Load every session-scoped detail field for the given datasets in a fixed number of queries."""- dataset_ids = [dataset.id for dataset in datasets]- if not dataset_ids:- return DatasetDetailPrefetch()-- tenant_ids = {dataset.tenant_id for dataset in datasets}-- app_counts = dict(- session.execute(- select(AppDatasetJoin.dataset_id, func.count(AppDatasetJoin.id))- .where(AppDatasetJoin.dataset_id.in_(dataset_ids), App.id == AppDatasetJoin.app_id)- .group_by(AppDatasetJoin.dataset_id)- )- .tuples()- .all()- )-- document_counts: dict[str, int] = {}- word_counts: dict[str, int] = {}- available_document_counts: dict[str, int] = {}- available_document_case = sa.case(- (- sa.and_(- Document.indexing_status == "completed",- Document.enabled == True,- Document.archived == False,- ),- 1,- ),- else_=0,- )- document_rows = session.execute(- select(- Document.dataset_id,- func.count(Document.id),- func.coalesce(func.sum(Document.word_count), 0),- func.coalesce(func.sum(available_document_case), 0),- )- .where(Document.dataset_id.in_(dataset_ids))- .group_by(Document.dataset_id)- ).all()- for dataset_id, document_count, word_count, available_count in document_rows:- document_counts[dataset_id] = document_count- word_counts[dataset_id] = word_count- available_document_counts[dataset_id] = available_count-- # doc_form only falls back to the documents table for datasets without a- # chunk structure of their own.- doc_forms: dict[tuple[str, str], str | None] = {}- doc_form_dataset_ids = [dataset.id for dataset in datasets if not dataset.chunk_structure]- if doc_form_dataset_ids:- doc_forms = {- (dataset_id, tenant_id): doc_form- for dataset_id, tenant_id, doc_form in session.execute(- select(Document.dataset_id, Document.tenant_id, func.min(Document.doc_form))- .where(Document.dataset_id.in_(doc_form_dataset_ids), Document.tenant_id.in_(tenant_ids))- .group_by(Document.dataset_id, Document.tenant_id)- ).all()- }-- author_ids = {dataset.created_by for dataset in datasets if dataset.created_by}- author_names = dict(- session.execute(select(Account.id, Account.name).where(Account.id.in_(author_ids))).tuples().all()- )-- tags: dict[tuple[str, str], list[Tag]] = {}- tag_rows = session.execute(- select(TagBinding.target_id, TagBinding.tenant_id, Tag)- .join(TagBinding, Tag.id == TagBinding.tag_id)- .where(- TagBinding.target_id.in_(dataset_ids),- TagBinding.tenant_id.in_(tenant_ids),- Tag.tenant_id == TagBinding.tenant_id,- Tag.type == "knowledge",- )- ).all()- for target_id, tenant_id, tag in tag_rows:- tags.setdefault((target_id, tenant_id), []).append(tag)-- doc_metadata: dict[str, list[DatasetMetadata]] = {}- for dataset_metadata in session.scalars(- select(DatasetMetadata).where(DatasetMetadata.dataset_id.in_(dataset_ids))- ).all():- doc_metadata.setdefault(dataset_metadata.dataset_id, []).append(dataset_metadata)-- pipeline_ids = {dataset.pipeline_id for dataset in datasets if dataset.pipeline_id}- pipeline_published: dict[str, bool] = {}- if pipeline_ids:- pipeline_published = dict(- session.execute(select(Pipeline.id, Pipeline.is_published).where(Pipeline.id.in_(pipeline_ids)))- .tuples()- .all()- )-- return DatasetDetailPrefetch(- app_counts=app_counts,- document_counts=document_counts,- word_counts=word_counts,- available_document_counts=available_document_counts,- author_names=author_names,- tags=tags,- doc_forms=doc_forms,- external_knowledge_infos=_load_external_knowledge_infos(datasets, session=session),- doc_metadata=doc_metadata,- pipeline_published=pipeline_published,- )---def _load_external_knowledge_infos(- datasets: Sequence[Dataset], *, session: Session-) -> dict[tuple[str, str], dict[str, Any]]:- external_datasets = [dataset for dataset in datasets if dataset.provider == "external"]- if not external_datasets:- return {}-- external_dataset_ids = [dataset.id for dataset in external_datasets]- external_tenant_ids = {dataset.tenant_id for dataset in external_datasets}- bindings = session.scalars(- select(ExternalKnowledgeBindings).where(- ExternalKnowledgeBindings.dataset_id.in_(external_dataset_ids),- ExternalKnowledgeBindings.tenant_id.in_(external_tenant_ids),- )- ).all()- if not bindings:- return {}-- apis = {- (api.id, api.tenant_id): api- for api in session.scalars(- select(ExternalKnowledgeApis).where(- ExternalKnowledgeApis.id.in_({binding.external_knowledge_api_id for binding in bindings}),- ExternalKnowledgeApis.tenant_id.in_(external_tenant_ids),- )- ).all()- }-- infos: dict[tuple[str, str], dict[str, Any]] = {}- for binding in bindings:- info = Dataset.build_external_knowledge_info(- binding, apis.get((binding.external_knowledge_api_id, binding.tenant_id))- )- if info is not None:- infos[(binding.dataset_id, binding.tenant_id)] = info- return infos---@dataclass(frozen=True)-class DatasetDetailResponseSource:- """Expose session-backed dataset fields during response validation.-- With a prefetch the values come from the batch load for the whole page;- without one each field is resolved against the session on access.- """-- dataset: Any- session: Session- prefetch: DatasetDetailPrefetch | None = None-- @property- def _prefetch_key(self) -> tuple[str, str]:- return (self.dataset.id, self.dataset.tenant_id)-- @property- def app_count(self) -> int:- if self.prefetch is not None:- return self.prefetch.app_counts.get(self.dataset.id, 0)- return self.dataset.get_app_count(session=self.session)-- @property- def document_count(self) -> int:- if self.prefetch is not None:- return self.prefetch.document_counts.get(self.dataset.id, 0)- return self.dataset.get_document_count(session=self.session)-- @property- def word_count(self) -> int:- if self.prefetch is not None:- return self.prefetch.word_counts.get(self.dataset.id, 0)- return self.dataset.get_word_count(session=self.session)-- @property- def author_name(self) -> str | None:- if self.prefetch is not None:- return self.prefetch.author_names.get(self.dataset.created_by)- return self.dataset.get_author_name(session=self.session)-- @property- def tags(self) -> Any:- if self.prefetch is not None:- return self.prefetch.tags.get(self._prefetch_key, [])- return self.dataset.get_tags(session=self.session)-- @property- def doc_form(self) -> str | None:- if self.prefetch is not None:- if self.dataset.chunk_structure:- return self.dataset.chunk_structure- return self.prefetch.doc_forms.get(self._prefetch_key)- return self.dataset.get_doc_form(session=self.session)-- @property- def external_knowledge_info(self) -> Any:- if self.prefetch is not None:- if self.dataset.provider != "external":- return None- return self.prefetch.external_knowledge_infos.get(self._prefetch_key)- return self.dataset.get_external_knowledge_info(session=self.session)-- @property- def doc_metadata(self) -> Any:- if self.prefetch is not None:- return self.dataset.build_doc_metadata(self.prefetch.doc_metadata.get(self.dataset.id, []))- return self.dataset.get_doc_metadata(session=self.session)-- @property- def is_published(self) -> bool:- if self.prefetch is not None:- if not self.dataset.pipeline_id:- return False- return self.prefetch.pipeline_published.get(self.dataset.pipeline_id, False)- return self.dataset.get_is_published(session=self.session)-- @property- def total_documents(self) -> int:- if self.prefetch is not None:- return self.prefetch.document_counts.get(self.dataset.id, 0)- return self.dataset.get_total_documents(session=self.session)-- @property- def total_available_documents(self) -> int:- if self.prefetch is not None:- return self.prefetch.available_document_counts.get(self.dataset.id, 0)- return self.dataset.get_total_available_documents(session=self.session)-- def __getattr__(self, name: str) -> Any:- return getattr(self.dataset, name) # guard-ignore: no-new-getattr -- delegates model fields---def dataset_detail_response_source(- dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None-) -> DatasetDetailResponseSource:- return DatasetDetailResponseSource(dataset=dataset, session=session, prefetch=prefetch)diff --git a/api/fields/document_fields.py b/api/fields/document_fields.pyindex 06a69422cf6..d0122c36e08 100644--- a/api/fields/document_fields.py+++ b/api/fields/document_fields.py@@ -1,16 +1,12 @@"""Response schemas for dataset document endpoints."""-from collections.abc import Iterable-from dataclasses import dataclassfrom datetime import datetimefrom typing import Anyfrom pydantic import Field, field_validator-from sqlalchemy.orm import Sessionfrom fields.base import ResponseModelfrom libs.helper import to_timestamp-from models.dataset import DocMetadataDetailItem, Documentdef normalize_enum(value: Any) -> Any:@@ -70,39 +66,6 @@ class DocumentResponse(ResponseModel):return to_timestamp(value)-@dataclass(frozen=True)-class DocumentWithSession:- """Expose session-backed document fields during response validation."""-- document: Document- session: Session-- @property- def data_source_detail_dict(self) -> dict[str, Any]:- return self.document.get_data_source_detail_dict(session=self.session)-- @property- def hit_count(self) -> int:- return self.document.get_hit_count(session=self.session)-- @property- def doc_metadata_details(self) -> list[DocMetadataDetailItem] | None:- return self.document.get_doc_metadata_details(session=self.session)-- def __getattr__(self, name: str) -> Any:- return getattr(self.document, name) # guard-ignore: no-new-getattr -- delegates model fields---def document_response(document: Document, *, session: Session) -> DocumentResponse:- return DocumentResponse.model_validate(- DocumentWithSession(document=document, session=session), from_attributes=True- )---def document_responses(documents: Iterable[Document], *, session: Session) -> list[DocumentResponse]:- return [document_response(document, session=session) for document in documents]--class DocumentListResponse(ResponseModel):data: list[DocumentResponse]has_more: booldiff --git a/api/fields/segment_fields.py b/api/fields/segment_fields.pyindex ecee5335e53..c4f73c3c08c 100644--- a/api/fields/segment_fields.py+++ b/api/fields/segment_fields.py@@ -1,10 +1,6 @@-from collections.abc import Iterable, Mapping-from dataclasses import dataclassfrom datetime import datetime-from typing import Anyfrom pydantic import field_serializer-from sqlalchemy.orm import Sessionfrom fields.base import ResponseModelfrom libs.helper import to_timestamp@@ -72,40 +68,6 @@ class SegmentResponse(ResponseModel):return to_timestamp(value)-@dataclass(frozen=True)-class SegmentWithSummary:- """Expose session-backed segment relations during response validation."""-- segment: Any- summary: str | None- session: Session-- @property- def child_chunks(self) -> Any:- return self.segment.get_child_chunks(session=self.session, include_full_doc=False)-- @property- def attachments(self) -> Any:- return self.segment.get_attachments(session=self.session)-- def __getattr__(self, name: str) -> Any:- return getattr(self.segment, name)---def segment_response_with_summary(segment: Any, summary: str | None, *, session: Session) -> SegmentResponse:- response_source = SegmentWithSummary(segment=segment, summary=summary, session=session)- return SegmentResponse.model_validate(response_source, from_attributes=True)---def segment_responses_with_summaries(- segments: Iterable[Any],- summaries: Mapping[str, str | None],- *,- session: Session,-) -> list[SegmentResponse]:- return [segment_response_with_summary(segment, summaries.get(segment.id), session=session) for segment in segments]--class SegmentDetailResponse(ResponseModel):data: SegmentResponsedoc_form: strdiff --git a/api/models/dataset.py b/api/models/dataset.pyindex 796e3d50ab6..093606796e8 100644--- a/api/models/dataset.py+++ b/api/models/dataset.py@@ -1,12 +1,5 @@-import base64-import hashlib-import hmacimport json-import logging-import osimport pickle-import re-import timefrom collections.abc import Sequencefrom datetime import datetimefrom json import JSONDecodeError@@ -14,22 +7,17 @@ from typing import Any, ClassVar, TypedDict, cast, overridefrom uuid import uuid4import sqlalchemy as sa-from sqlalchemy import DateTime, String, func, select-from sqlalchemy.orm import Mapped, Session, mapped_column, scoped_session+from sqlalchemy import DateTime, String, func+from sqlalchemy.orm import Mapped, mapped_columnfrom configs import dify_config-from core.rag.entities import ParentMode, Rule-from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource+from core.rag.entities import PreProcessingRuleKey+from core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from core.rag.index_processor.constant.query_type import QueryTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethod-from core.tools.signature import sign_upload_file_preview_url-from extensions.ext_storage import storagefrom libs.uuid_utils import uuidv7--from .account import Account-from .base import Base, TypeBase-from .enums import (+from models.base import Base, TypeBase+from models.enums import (CollectionBindingType,CreatorUserRole,DatasetMetadataType,@@ -45,14 +33,11 @@ from .enums import (SummaryStatus,TidbAuthBindingStatus,)-from .model import App, Tag, TagBinding, UploadFile-from .types import AdjustedJSON, BinaryData, EnumText, LongText, StringUUID, adjusted_json_index--logger = logging.getLogger(__name__)+from models.types import AdjustedJSON, BinaryData, EnumText, LongText, StringUUID, adjusted_json_indexclass PreProcessingRuleItem(TypedDict):- id: str+ id: PreProcessingRuleKeyenabled: bool@@ -95,6 +80,67 @@ class DatasetBindingItem(TypedDict):name: str+class ExternalKnowledgeApiDict(TypedDict):+ id: str+ tenant_id: str+ name: str+ description: str+ settings: dict[str, Any] | None+ dataset_bindings: list[DatasetBindingItem]+ created_by: str+ created_at: str+++class DocumentDict(TypedDict):+ id: str+ tenant_id: str+ dataset_id: str+ position: int+ data_source_type: str+ data_source_info: str | None+ dataset_process_rule_id: str | None+ batch: str+ name: str+ created_from: str+ created_by: str+ created_api_request_id: str | None+ created_at: datetime+ processing_started_at: datetime | None+ file_id: str | None+ word_count: int | None+ parsing_completed_at: datetime | None+ cleaning_completed_at: datetime | None+ splitting_completed_at: datetime | None+ tokens: int | None+ indexing_latency: float | None+ completed_at: datetime | None+ is_paused: bool | None+ paused_by: str | None+ paused_at: datetime | None+ error: str | None+ stopped_at: datetime | None+ indexing_status: str+ enabled: bool+ disabled_at: datetime | None+ disabled_by: str | None+ archived: bool+ archived_reason: str | None+ archived_by: str | None+ archived_at: datetime | None+ updated_at: datetime+ doc_type: str | None+ doc_metadata: Any+ doc_form: IndexStructureType+ doc_language: str | None+ display_status: str | None+ data_source_info_dict: dict[str, Any]+ average_segment_length: int+ dataset_process_rule: ProcessRuleDict | None+ dataset: None+ segment_count: int | None+ hit_count: int | None++from models.enums import PermissionEnum# Backward-compatible alias — new code should import PermissionEnum from models.enums@@ -149,25 +195,6 @@ class Dataset(Base):enable_api = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())is_multimodal = mapped_column(sa.Boolean, default=False, nullable=False)- def get_total_documents(self, *, session: Session) -> int:- return self.get_document_count(session=session)-- def get_total_available_documents(self, *, session: Session) -> int:- return (- session.scalar(- select(func.count(Document.id)).where(- Document.dataset_id == self.id,- Document.indexing_status == "completed",- Document.enabled == True,- Document.archived == False,- )- )- or 0- )-- def get_dataset_keyword_table(self, *, session: Session) -> "DatasetKeywordTable | None":- return session.scalar(select(DatasetKeywordTable).where(DatasetKeywordTable.dataset_id == self.id))-@propertydef index_struct_dict(self):return json.loads(self.index_struct) if self.index_struct else None@@ -180,78 +207,6 @@ class Dataset(Base):}return self.retrieval_model or default_retrieval_model- def get_created_by_account(self, *, session: Session) -> Account | None:- return session.get(Account, self.created_by)-- def get_author_name(self, *, session: Session) -> str | None:- account = self.get_created_by_account(session=session)- if account:- return account.name- return None-- def get_latest_process_rule(self, *, session: Session) -> "DatasetProcessRule | None":- return session.scalar(- select(DatasetProcessRule)- .where(DatasetProcessRule.dataset_id == self.id)- .order_by(DatasetProcessRule.created_at.desc())- .limit(1)- )-- def get_app_count(self, *, session: Session) -> int:- return (- session.scalar(- select(func.count(AppDatasetJoin.id)).where(- AppDatasetJoin.dataset_id == self.id, App.id == AppDatasetJoin.app_id- )- )- or 0- )-- def get_document_count(self, *, session: Session) -> int:- return session.scalar(select(func.count(Document.id)).where(Document.dataset_id == self.id)) or 0-- def get_available_document_count(self, *, session: Session) -> int:- return (- session.scalar(- select(func.count(Document.id)).where(- Document.dataset_id == self.id,- Document.indexing_status == "completed",- Document.enabled == True,- Document.archived == False,- )- )- or 0- )-- def get_available_segment_count(self, *, session: Session) -> int:- return (- session.scalar(- select(func.count(DocumentSegment.id)).where(- DocumentSegment.dataset_id == self.id,- DocumentSegment.status == "completed",- DocumentSegment.enabled == True,- )- )- or 0- )-- def get_word_count(self, *, session: Session) -> int:- return (- session.scalar(- select(func.coalesce(func.sum(Document.word_count), 0)).where(Document.dataset_id == self.id)- )- or 0- )-- def get_doc_form(self, *, session: Session) -> str | None:- if self.chunk_structure:- return self.chunk_structure- return session.scalar(- select(Document.doc_form)- .where(Document.dataset_id == self.id, Document.tenant_id == self.tenant_id)- .limit(1)- )-@propertydef retrieval_model_dict(self):"""Return a normalized retrieval model payload for API responses.@@ -272,39 +227,6 @@ class Dataset(Base):return {**default_retrieval_model, **self.retrieval_model}- def get_tags(self, *, session: Session) -> Sequence[Tag]:- tags = session.scalars(- select(Tag)- .join(TagBinding, Tag.id == TagBinding.tag_id)- .where(- TagBinding.target_id == self.id,- TagBinding.tenant_id == self.tenant_id,- Tag.tenant_id == self.tenant_id,- Tag.type == "knowledge",- )- ).all()-- return tags or []-- def get_external_knowledge_info(self, *, session: Session) -> dict[str, Any] | None:- if self.provider != "external":- return None- external_knowledge_binding = session.scalar(- select(ExternalKnowledgeBindings).where(- ExternalKnowledgeBindings.dataset_id == self.id,- ExternalKnowledgeBindings.tenant_id == self.tenant_id,- )- )- if not external_knowledge_binding:- return None- external_knowledge_api = session.scalar(- select(ExternalKnowledgeApis).where(- ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id,- ExternalKnowledgeApis.tenant_id == self.tenant_id,- )- )- return self.build_external_knowledge_info(external_knowledge_binding, external_knowledge_api)-@staticmethoddef build_external_knowledge_info(external_knowledge_binding: "ExternalKnowledgeBindings | None",@@ -322,18 +244,6 @@ class Dataset(Base):"external_knowledge_api_endpoint": json.loads(external_knowledge_api.settings).get("endpoint", ""),}- def get_is_published(self, *, session: Session) -> bool:- if self.pipeline_id:- pipeline = session.scalar(select(Pipeline).where(Pipeline.id == self.pipeline_id))- if pipeline:- return pipeline.is_published- return False-- def get_doc_metadata(self, *, session: Session) -> list[dict[str, str]]:- dataset_metadatas = session.scalars(select(DatasetMetadata).where(DatasetMetadata.dataset_id == self.id)).all()-- return self.build_doc_metadata(dataset_metadatas)-def build_doc_metadata(self, dataset_metadatas: "Sequence[DatasetMetadata]") -> list[dict[str, str]]:"""Format metadata rows, appending the built-in fields when they are enabled."""doc_metadata = [@@ -407,7 +317,6 @@ class DatasetProcessRule(TypeBase):)MODES = ["automatic", "custom", "hierarchical"]- PRE_PROCESSING_RULES = ["remove_stopwords", "remove_extra_spaces", "remove_urls_emails"]AUTOMATIC_RULES: ClassVar[AutomaticRulesConfig] = {"pre_processing_rules": [{"id": "remove_extra_spaces", "enabled": True},@@ -482,11 +391,11 @@ class Document(Base):paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)# error- error = mapped_column(LongText, nullable=True)+ error: Mapped[str | None] = mapped_column(LongText, nullable=True)stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)# basic fields- indexing_status = mapped_column(+ indexing_status: Mapped[str] = mapped_column(EnumText(IndexingStatus, length=255), nullable=False, server_default=sa.text("'waiting'"))enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())@@ -512,10 +421,10 @@ class Document(Base):@propertydef display_status(self):status = None- if self.indexing_status == "waiting":- status = "queuing"- elif self.indexing_status not in {"completed", "error", "waiting"} and self.is_paused:+ if self.indexing_status not in {"completed", "error"} and self.is_paused:status = "paused"+ elif self.indexing_status == "waiting":+ status = "queuing"elif self.indexing_status in {"parsing", "cleaning", "splitting", "indexing"}:status = "indexing"elif self.indexing_status == "error":@@ -539,56 +448,6 @@ class Document(Base):return data_source_info_dictreturn {}- def get_data_source_detail_dict(self, *, session: Session) -> dict[str, Any]:- if self.data_source_info:- if self.data_source_type == "upload_file":- data_source_info_dict: dict[str, Any] = json.loads(self.data_source_info)- file_detail = session.scalar(- select(UploadFile).where(UploadFile.id == data_source_info_dict["upload_file_id"])- )- if file_detail:- return {- "upload_file": {- "id": file_detail.id,- "name": file_detail.name,- "size": file_detail.size,- "extension": file_detail.extension,- "mime_type": file_detail.mime_type,- "created_by": file_detail.created_by,- "created_at": file_detail.created_at.timestamp(),- }- }- elif self.data_source_type in {"notion_import", "website_crawl"}:- result: dict[str, Any] = json.loads(self.data_source_info)- return result- return {}-- def get_dataset_process_rule(self, *, session: Session) -> "DatasetProcessRule | None":- if self.dataset_process_rule_id:- return session.get(DatasetProcessRule, self.dataset_process_rule_id)- return None-- def get_dataset(self, *, session: Session) -> Dataset | None:- """Load the owning dataset with the caller-owned database session."""- return session.get(Dataset, self.dataset_id)-- def get_segment_count(self, *, session: Session) -> int:- return session.scalar(select(func.count(DocumentSegment.id)).where(DocumentSegment.document_id == self.id)) or 0-- def get_hit_count(self, *, session: Session) -> int:- return (- session.scalar(- select(func.coalesce(func.sum(DocumentSegment.hit_count), 0)).where(- DocumentSegment.document_id == self.id- )- )- or 0- )-- def get_uploader(self, *, session: Session) -> str | None:- user = session.scalar(select(Account).where(Account.id == self.created_by))- return user.name if user else None-@propertydef upload_date(self):return self.created_at@@ -597,77 +456,57 @@ class Document(Base):def last_update_date(self):return self.updated_at- def get_doc_metadata_details(self, *, session: Session) -> list[DocMetadataDetailItem] | None:- if self.doc_metadata:- document_metadatas = session.scalars(- select(DatasetMetadata)- .join(DatasetMetadataBinding, DatasetMetadataBinding.metadata_id == DatasetMetadata.id)- .where(- DatasetMetadata.tenant_id == self.tenant_id,- DatasetMetadata.dataset_id == self.dataset_id,- DatasetMetadataBinding.tenant_id == self.tenant_id,- DatasetMetadataBinding.dataset_id == self.dataset_id,- DatasetMetadataBinding.document_id == self.id,- )- ).all()- metadata_list: list[DocMetadataDetailItem] = []- for metadata in document_metadatas:- metadata_dict: DocMetadataDetailItem = {- "id": metadata.id,- "name": metadata.name,- "type": metadata.type,- "value": self.doc_metadata.get(metadata.name),- }- metadata_list.append(metadata_dict)- # deal built-in fields- metadata_list.extend(self.get_built_in_fields(session=session))-- return metadata_list- return None-- def get_built_in_fields(self, *, session: Session) -> list[DocMetadataDetailItem]:- built_in_fields: list[DocMetadataDetailItem] = []- built_in_fields.append(- {- "id": "built-in",- "name": BuiltInField.document_name,- "type": "string",- "value": self.name,- }- )- built_in_fields.append(- {- "id": "built-in",- "name": BuiltInField.uploader,- "type": "string",- "value": self.get_uploader(session=session),- }- )- built_in_fields.append(- {- "id": "built-in",- "name": BuiltInField.upload_date,- "type": "time",- "value": str(self.created_at.timestamp()),- }- )- built_in_fields.append(- {- "id": "built-in",- "name": BuiltInField.last_update_date,- "type": "time",- "value": str(self.updated_at.timestamp()),- }- )- built_in_fields.append(- {- "id": "built-in",- "name": BuiltInField.source,- "type": "string",- "value": MetadataDataSource[self.data_source_type],- }- )- return built_in_fields+ def to_dict(self, *, segment_count: int, hit_count: int, process_rule: DatasetProcessRule | None) -> DocumentDict:+ result: DocumentDict = {+ "id": self.id,+ "tenant_id": self.tenant_id,+ "dataset_id": self.dataset_id,+ "position": self.position,+ "data_source_type": self.data_source_type,+ "data_source_info": self.data_source_info,+ "dataset_process_rule_id": self.dataset_process_rule_id,+ "batch": self.batch,+ "name": self.name,+ "created_from": self.created_from,+ "created_by": self.created_by,+ "created_api_request_id": self.created_api_request_id,+ "created_at": self.created_at,+ "processing_started_at": self.processing_started_at,+ "file_id": self.file_id,+ "word_count": self.word_count,+ "parsing_completed_at": self.parsing_completed_at,+ "cleaning_completed_at": self.cleaning_completed_at,+ "splitting_completed_at": self.splitting_completed_at,+ "tokens": self.tokens,+ "indexing_latency": self.indexing_latency,+ "completed_at": self.completed_at,+ "is_paused": self.is_paused,+ "paused_by": self.paused_by,+ "paused_at": self.paused_at,+ "error": self.error,+ "stopped_at": self.stopped_at,+ "indexing_status": self.indexing_status,+ "enabled": self.enabled,+ "disabled_at": self.disabled_at,+ "disabled_by": self.disabled_by,+ "archived": self.archived,+ "archived_reason": self.archived_reason,+ "archived_by": self.archived_by,+ "archived_at": self.archived_at,+ "updated_at": self.updated_at,+ "doc_type": self.doc_type,+ "doc_metadata": self.doc_metadata,+ "doc_form": self.doc_form,+ "doc_language": self.doc_language,+ "display_status": self.display_status,+ "data_source_info_dict": self.data_source_info_dict,+ "average_segment_length": (self.word_count or 0) // segment_count if segment_count else 0,+ "dataset_process_rule": process_rule.to_dict() if process_rule else None,+ "dataset": None,+ "segment_count": segment_count,+ "hit_count": hit_count,+ }+ return result@classmethoddef from_dict(cls, data: dict[str, Any]):@@ -761,155 +600,6 @@ class DocumentSegment(TypeBase):stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)hit_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)- def get_dataset(self, *, session: Session) -> Dataset | None:- """Load the owning dataset with the caller-owned database session."""- return session.get(Dataset, self.dataset_id)-- def get_document(self, *, session: Session) -> Document | None:- """Load the owning document with the caller-owned database session."""- return session.get(Document, self.document_id)-- def previous_segment(self, session: Session) -> "DocumentSegment | None":- return session.scalar(- select(DocumentSegment).where(- DocumentSegment.document_id == self.document_id, DocumentSegment.position == self.position - 1- )- )-- def next_segment(self, session: Session) -> "DocumentSegment | None":- return session.scalar(- select(DocumentSegment).where(- DocumentSegment.document_id == self.document_id, DocumentSegment.position == self.position + 1- )- )-- def get_child_chunks(self, *, session: Session, include_full_doc: bool = True) -> Sequence["ChildChunk"]:- """Load hierarchical child chunks with the caller-owned database session."""- document = session.get(Document, self.document_id)- if not document:- return []- process_rule = document.get_dataset_process_rule(session=session)- if process_rule and process_rule.mode == "hierarchical":- rules_dict = process_rule.rules_dict- if rules_dict:- rules = Rule.model_validate(rules_dict)- if rules.parent_mode and (include_full_doc or rules.parent_mode != ParentMode.FULL_DOC):- child_chunks = session.scalars(- select(ChildChunk).where(ChildChunk.segment_id == self.id).order_by(ChildChunk.position.asc())- ).all()- return child_chunks or []- return []-- @property- def sign_content(self) -> str:- return self.get_sign_content()-- def get_sign_content(self) -> str:- signed_urls: list[tuple[int, int, str]] = []- text = self.content-- # For data before v0.10.0- pattern = r"/files/([a-f0-9\-]+)/image-preview(?:\?.*?)?"- matches = re.finditer(pattern, text)- for match in matches:- upload_file_id = match.group(1)- nonce = os.urandom(16).hex()- timestamp = str(int(time.time()))- data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"- secret_key = dify_config.SECRET_KEY.encode()- sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()- encoded_sign = base64.urlsafe_b64encode(sign).decode()-- params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"- base_url = f"/files/{upload_file_id}/image-preview"- signed_url = f"{base_url}?{params}"- signed_urls.append((match.start(), match.end(), signed_url))-- # For data after v0.10.0- pattern = r"/files/([a-f0-9\-]+)/file-preview(?:\?.*?)?"- matches = re.finditer(pattern, text)- for match in matches:- upload_file_id = match.group(1)- nonce = os.urandom(16).hex()- timestamp = str(int(time.time()))- data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"- secret_key = dify_config.SECRET_KEY.encode()- sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()- encoded_sign = base64.urlsafe_b64encode(sign).decode()-- params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"- base_url = f"/files/{upload_file_id}/file-preview"- signed_url = f"{base_url}?{params}"- signed_urls.append((match.start(), match.end(), signed_url))-- # For tools directory - direct file formats (e.g., .png, .jpg, etc.)- # Match URL including any query parameters up to common URL boundaries (space, parenthesis, quotes)- pattern = r"/files/tools/([a-f0-9\-]+)\.([a-zA-Z0-9]+)(?:\?[^\s\)\"\']*)?"- matches = re.finditer(pattern, text)- for match in matches:- upload_file_id = match.group(1)- file_extension = match.group(2)- nonce = os.urandom(16).hex()- timestamp = str(int(time.time()))- data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"- secret_key = dify_config.SECRET_KEY.encode()- sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()- encoded_sign = base64.urlsafe_b64encode(sign).decode()-- params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"- base_url = f"/files/tools/{upload_file_id}.{file_extension}"- signed_url = f"{base_url}?{params}"- signed_urls.append((match.start(), match.end(), signed_url))-- # Reconstruct the text with signed URLs- offset = 0- for start, end, signed_url in signed_urls:- text = text[: start + offset] + signed_url + text[end + offset :]- offset += len(signed_url) - (end - start)-- return text-- def get_attachments(self, *, session: Session) -> list[AttachmentItem]:- """Load attachment metadata with the caller-owned database session."""- # Use JOIN to fetch attachments in a single query instead of two separate queries- attachments_with_bindings = session.execute(- select(SegmentAttachmentBinding, UploadFile)- .join(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)- .where(- SegmentAttachmentBinding.tenant_id == self.tenant_id,- SegmentAttachmentBinding.dataset_id == self.dataset_id,- SegmentAttachmentBinding.document_id == self.document_id,- SegmentAttachmentBinding.segment_id == self.id,- )- ).all()- if not attachments_with_bindings:- return []- attachment_list: list[AttachmentItem] = []- for _, attachment in attachments_with_bindings:- upload_file_id = attachment.id- nonce = os.urandom(16).hex()- timestamp = str(int(time.time()))- data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"- secret_key = dify_config.SECRET_KEY.encode()- sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()- encoded_sign = base64.urlsafe_b64encode(sign).decode()-- params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"- reference_url = dify_config.FILES_URL or dify_config.CONSOLE_API_URL or ""- base_url = f"{reference_url}/files/{upload_file_id}/image-preview"- source_url = f"{base_url}?{params}"- attachment_list.append(- {- "id": attachment.id,- "name": attachment.name,- "size": attachment.size,- "extension": attachment.extension,- "mime_type": attachment.mime_type,- "source_url": source_url,- }- )- return attachment_list-class ChildChunk(TypeBase):__tablename__ = "child_chunks"@@ -957,15 +647,6 @@ class ChildChunk(TypeBase):)error: Mapped[str | None] = mapped_column(LongText, nullable=True, init=False)- def dataset(self, session: Session) -> Dataset | None:- return session.scalar(select(Dataset).where(Dataset.id == self.dataset_id))-- def document(self, session: Session) -> Document | None:- return session.scalar(select(Document).where(Document.id == self.document_id))-- def segment(self, session: Session) -> DocumentSegment | None:- return session.scalar(select(DocumentSegment).where(DocumentSegment.id == self.segment_id))-class AppDatasetJoin(TypeBase):__tablename__ = "app_dataset_joins"@@ -988,9 +669,6 @@ class AppDatasetJoin(TypeBase):DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False)- def app(self, session: Session) -> App | None:- return session.get(App, self.app_id)-class DatasetQuery(TypeBase):__tablename__ = "dataset_queries"@@ -1017,37 +695,6 @@ class DatasetQuery(TypeBase):DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False)- def get_queries(self, *, session: Session) -> list[dict[str, Any]]:- try:- queries = json.loads(self.content)- if isinstance(queries, list):- for query in queries:- if query["content_type"] == QueryType.IMAGE_QUERY:- file_info = session.scalar(select(UploadFile).where(UploadFile.id == query["content"]))- if file_info:- query["file_info"] = {- "id": file_info.id,- "name": file_info.name,- "size": file_info.size,- "extension": file_info.extension,- "mime_type": file_info.mime_type,- "source_url": sign_upload_file_preview_url(file_info.id, file_info.extension),- }- else:- query["file_info"] = None-- return queries- else:- return [queries]- except JSONDecodeError:- return [- {- "content_type": QueryType.TEXT_QUERY,- "content": self.content,- "file_info": None,- }- ]-class DatasetKeywordTable(TypeBase):__tablename__ = "dataset_keyword_tables"@@ -1067,40 +714,6 @@ class DatasetKeywordTable(TypeBase):keyword_table: Mapped[str] = mapped_column(LongText, nullable=False)data_source_type: Mapped[str] = mapped_column(String(255), nullable=False, default="database")- def get_keyword_table_dict(self, *, session: Session) -> dict[str, set[Any]] | None:- class SetDecoder(json.JSONDecoder):- def __init__(self, *args: Any, **kwargs: Any) -> None:- def object_hook(dct: Any) -> Any:- if isinstance(dct, dict):- result: dict[str, Any] = {}- items = cast(dict[str, Any], dct).items()- for keyword, node_idxs in items:- if isinstance(node_idxs, list):- result[keyword] = set(node_idxs)- else:- result[keyword] = node_idxs- return result- return dct-- super().__init__(object_hook=object_hook, *args, **kwargs)-- # get dataset- dataset = session.scalar(select(Dataset).where(Dataset.id == self.dataset_id))- if not dataset:- return None- if self.data_source_type == "database":- return json.loads(self.keyword_table, cls=SetDecoder) if self.keyword_table else None- else:- file_key = "keyword_files/" + dataset.tenant_id + "/" + self.dataset_id + ".txt"- try:- keyword_table_text = storage.load_once(file_key)- if keyword_table_text:- return json.loads(keyword_table_text.decode("utf-8"), cls=SetDecoder)- return None- except Exception:- logger.exception("Failed to load keyword table from file: %s", file_key)- return None-class Embedding(TypeBase):__tablename__ = "embeddings"@@ -1263,6 +876,18 @@ class ExternalKnowledgeApis(TypeBase):DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False)+ def to_dict(self, *, dataset_bindings: list[DatasetBindingItem]) -> ExternalKnowledgeApiDict:+ return {+ "id": self.id,+ "tenant_id": self.tenant_id,+ "name": self.name,+ "description": self.description,+ "settings": self.settings_dict,+ "dataset_bindings": dataset_bindings,+ "created_by": self.created_by,+ "created_at": self.created_at.isoformat(),+ }+@propertydef settings_dict(self) -> dict[str, Any] | None:try:@@ -1270,18 +895,6 @@ class ExternalKnowledgeApis(TypeBase):except JSONDecodeError:return None- def get_dataset_bindings(self, *, session: Session) -> list[DatasetBindingItem]:- external_knowledge_bindings = session.scalars(- select(ExternalKnowledgeBindings).where(ExternalKnowledgeBindings.external_knowledge_api_id == self.id)- ).all()- dataset_ids = [binding.dataset_id for binding in external_knowledge_bindings]- datasets = session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()- dataset_bindings: list[DatasetBindingItem] = []- for dataset in datasets:- dataset_bindings.append({"id": dataset.id, "name": dataset.name})-- return dataset_bindings-class ExternalKnowledgeBindings(TypeBase):__tablename__ = "external_knowledge_bindings"@@ -1467,12 +1080,6 @@ class PipelineCustomizedTemplate(TypeBase):init=False,)- def created_user_name(self, session: Session) -> str:- account = session.scalar(select(Account).where(Account.id == self.created_by))- if account:- return account.name- return ""-class Pipeline(TypeBase):__tablename__ = "pipelines"@@ -1500,11 +1107,6 @@ class Pipeline(TypeBase):init=False,)- def retrieve_dataset(self, session: Session | scoped_session):- return session.scalar(- select(Dataset).where(Dataset.pipeline_id == self.id, Dataset.tenant_id == self.tenant_id)- )-class DocumentPipelineExecutionLog(TypeBase):__tablename__ = "document_pipeline_execution_logs"diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.mdindex 4f9147765d1..128060a7bd4 100644--- a/api/openapi/markdown/console-openapi.md+++ b/api/openapi/markdown/console-openapi.md@@ -5083,30 +5083,9 @@ Get compliance document download link### [GET] /data-source/integrates#### Responses-| Code | Description |-| ---- | ----------- |-| 200 | Success |--### [PATCH] /data-source/integrates-#### Responses--| Code | Description |-| ---- | ----------- |-| 200 | Success |--### [GET] /data-source/integrates/{binding_id}/{action}-#### Parameters--| Name | Located in | Description | Required | Schema |-| ---- | ---------- | ----------- | -------- | ------ |-| action | path | | Yes | string |-| binding_id | path | | Yes | string (uuid) |--#### Responses--| Code | Description |-| ---- | ----------- |-| 200 | Success |+| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Success | **application/json**: [DataSourceIntegrateListResponse](#datasourceintegratelistresponse)<br> |### [PATCH] /data-source/integrates/{binding_id}/{action}#### Parameters@@ -5118,9 +5097,9 @@ Get compliance document download link#### Responses-| Code | Description |-| ---- | ----------- |-| 200 | Success |+| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Success | **application/json**: [SimpleResultResponse](#simpleresultresponse)<br> |### [GET] /datasetsGet list of datasets@@ -5155,7 +5134,7 @@ Create a new dataset| Code | Description | Schema || ---- | ----------- | ------ |-| 201 | Dataset created successfully | **application/json**: [DatasetDetailResponse](#datasetdetailresponse)<br> |+| 201 | Dataset created successfully | **application/json**: [DatasetDetailWithPartialMembersResponse](#datasetdetailwithpartialmembersresponse)<br> || 400 | Invalid request parameters | |### [GET] /datasets/api-base-info@@ -5218,25 +5197,6 @@ Delete dataset API key| ---- | ----------- | ------ || 200 | Batch import status | **application/json**: [SegmentBatchImportStatusResponse](#segmentbatchimportstatusresponse)<br> |-### [POST] /datasets/batch_import_status/{job_id}-#### Parameters--| Name | Located in | Description | Required | Schema |-| ---- | ---------- | ----------- | -------- | ------ |-| job_id | path | | Yes | string (uuid) |--#### Request Body--| Required | Schema |-| -------- | ------ |-| Yes | **application/json**: [BatchImportPayload](#batchimportpayload)<br> |--#### Responses--| Code | Description | Schema |-| ---- | ----------- | ------ |-| 200 | Batch import started | **application/json**: [SegmentBatchImportStatusResponse](#segmentbatchimportstatusresponse)<br> |-### [POST] /datasets/externalCreate external knowledge dataset@@ -5970,20 +5930,6 @@ Update document processing status (pause/resume)| ---- | ----------- | ------ || 200 | Segments retrieved successfully | **application/json**: [ConsoleSegmentListResponse](#consolesegmentlistresponse)<br> |-### [GET] /datasets/{dataset_id}/documents/{document_id}/segments/batch_import-#### Parameters--| Name | Located in | Description | Required | Schema |-| ---- | ---------- | ----------- | -------- | ------ |-| dataset_id | path | | Yes | string (uuid) |-| document_id | path | | Yes | string (uuid) |--#### Responses--| Code | Description | Schema |-| ---- | ----------- | ------ |-| 200 | Batch import status | **application/json**: [SegmentBatchImportStatusResponse](#segmentbatchimportstatusresponse)<br> |-### [POST] /datasets/{dataset_id}/documents/{document_id}/segments/batch_import#### Parameters@@ -18102,7 +18048,7 @@ Site token customization strategy| page_id | string | | Yes || page_name | string | | Yes || parent_id | string | | Yes |-| type | string | | Yes |+| type | [NotionPageType](#notionpagetype) | | Yes |#### DataSourceIntegrateResponse@@ -21037,15 +20983,30 @@ Coarse node-level status used by Inspector to pick a banner.| notifications | [ [NotificationItemResponse](#notificationitemresponse) ] | | Yes || should_show | boolean | | Yes |+#### NotionEstimatePagePayload++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| page_id | string | | Yes |+| type | [NotionPageType](#notionpagetype) | | Yes |+#### NotionEstimatePayload| Name | Type | Description | Required || ---- | ---- | ----------- | -------- || doc_form | string, <br>**Default:** text_model | | No || doc_language | string, <br>**Default:** English | | No |-| notion_info_list | [ object ] | | Yes |+| notion_info_list | [ [NotionEstimateWorkspacePayload](#notionestimateworkspacepayload) ] | | Yes || process_rule | object | | Yes |+#### NotionEstimateWorkspacePayload++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| credential_id | string | | Yes |+| pages | [ [NotionEstimatePagePayload](#notionestimatepagepayload) ] | | Yes |+| workspace_id | string | | Yes |+#### NotionIcon| Name | Type | Description | Required |@@ -21077,7 +21038,7 @@ Coarse node-level status used by Inspector to pick a banner.| page_id | string | | Yes || page_name | string | | Yes || parent_id | string | | Yes |-| type | string | | Yes |+| type | [NotionPageType](#notionpagetype) | | Yes |#### NotionIntegrateWorkspaceResponse@@ -21097,6 +21058,12 @@ Coarse node-level status used by Inspector to pick a banner.| page_name | string | | Yes || type | string | | Yes |+#### NotionPageType++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| NotionPageType | string | | |+#### OAuthCallbackQuery| Name | Type | Description | Required |@@ -22232,7 +22199,13 @@ Verification of the plugin.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- || enabled | boolean | Whether this preprocessing rule is enabled. | Yes |-| id | string, <br>**Available values:** "remove_extra_spaces", "remove_stopwords", "remove_urls_emails" | Rule identifier.<br>*Enum:* `"remove_extra_spaces"`, `"remove_stopwords"`, `"remove_urls_emails"` | Yes |+| id | [PreProcessingRuleKey](#preprocessingrulekey) | Rule identifier. | Yes |++#### PreProcessingRuleKey++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| PreProcessingRuleKey | string | | |#### PreviewDetaildiff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.mdindex 9fd290e93dc..a30ee05de6f 100644--- a/api/openapi/markdown/service-openapi.md+++ b/api/openapi/markdown/service-openapi.md@@ -3836,7 +3836,13 @@ JSON result for published runs and draft runs using `response_mode: blocking`.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- || enabled | boolean | Whether this preprocessing rule is enabled. | Yes |-| id | string, <br>**Available values:** "remove_extra_spaces", "remove_stopwords", "remove_urls_emails" | Rule identifier.<br>*Enum:* `"remove_extra_spaces"`, `"remove_stopwords"`, `"remove_urls_emails"` | Yes |+| id | [PreProcessingRuleKey](#preprocessingrulekey) | Rule identifier. | Yes |++#### PreProcessingRuleKey++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| PreProcessingRuleKey | string | | |#### ProcessRulediff --git a/api/repositories/credentials/__init__.py b/api/repositories/credentials/__init__.pynew file mode 100644index 00000000000..944f2df7138--- /dev/null+++ b/api/repositories/credentials/__init__.py@@ -0,0 +1 @@+"""Persistence adapters for the credentials domain."""diff --git a/api/repositories/credentials/query_repository.py b/api/repositories/credentials/query_repository.pynew file mode 100644index 00000000000..8e4d582738a--- /dev/null+++ b/api/repositories/credentials/query_repository.py@@ -0,0 +1,250 @@+"""Persistence-owned credential queries and visibility filtering shared across domains."""++from collections.abc import Sequence+from typing import override++from sqlalchemy import and_, func, or_, select+from sqlalchemy.orm import InstrumentedAttribute, Session, sessionmaker++from models.credential_permission import CredentialPermission, CredentialType+from models.enums import PermissionEnum+from models.oauth import DatasourceProvider+from models.provider import ProviderCredential+from models.provider_ids import ModelProviderID+from models.tools import BuiltinToolProvider+from models.trigger import TriggerSubscription, WorkflowPluginTrigger+from services.credentials.query import (+ CredentialQuery,+ DatasourceCredentialListItem,+ ModelCredentialRecord,+ ToolCredentialRecord,+ TriggerSubscriptionRecord,+)+++def apply_credential_visibility_filter_for_actor(+ query,+ *,+ tenant_id: str,+ model_id_column: InstrumentedAttribute,+ model_user_id_column: InstrumentedAttribute,+ model_visibility_column: InstrumentedAttribute,+ credential_type: str,+ actor_id: str,+):+ """Apply the canonical tenant- and actor-scoped credential visibility policy.++ Team credentials and legacy rows without an owner are visible to all tenant+ members. Personal credentials are visible to their owner, while partially+ shared credentials additionally require an enabled permission row for the+ same tenant and credential type. Administrator roles do not bypass this+ policy.+ """+ partial_member_ids = (+ select(CredentialPermission.credential_id)+ .where(+ CredentialPermission.tenant_id == tenant_id,+ CredentialPermission.credential_type == credential_type,+ CredentialPermission.account_id == actor_id,+ CredentialPermission.has_permission.is_(True),+ )+ .correlate_except(CredentialPermission)+ )+ return query.where(+ or_(+ model_visibility_column == PermissionEnum.ALL_TEAM,+ model_user_id_column.is_(None),+ model_user_id_column == actor_id,+ and_(model_visibility_column == PermissionEnum.PARTIAL_TEAM, model_id_column.in_(partial_member_ids)),+ )+ )+++class CredentialQueryRepository(CredentialQuery):+ """Load complete credential lists without exposing ORM objects or SQL to callers."""++ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ @override+ def list_models(self, *, workspace_id: str, provider: str, actor_id: str | None) -> Sequence[ModelCredentialRecord]:+ names = [provider]+ provider_id = ModelProviderID(provider)+ if provider_id.is_langgenius():+ names.append(provider_id.provider_name if "/" in provider else str(provider_id))+ query = (+ select(ProviderCredential)+ .where(ProviderCredential.tenant_id == workspace_id, ProviderCredential.provider_name.in_(names))+ .order_by(ProviderCredential.created_at.desc())+ )+ if actor_id is not None:+ query = apply_credential_visibility_filter_for_actor(+ query,+ tenant_id=workspace_id,+ model_id_column=ProviderCredential.id,+ model_user_id_column=ProviderCredential.user_id,+ model_visibility_column=ProviderCredential.visibility,+ credential_type=CredentialType.PROVIDER_CREDENTIAL,+ actor_id=actor_id,+ )+ with self._session_factory() as session:+ return [ModelCredentialRecord(row.id, row.credential_name) for row in session.scalars(query)]++ @override+ def list_datasources(+ self, *, workspace_id: str, provider: str, plugin_id: str, actor_id: str | None+ ) -> Sequence[DatasourceCredentialListItem]:+ owner = (+ DatasourceProvider.tenant_id == workspace_id,+ DatasourceProvider.provider == provider,+ DatasourceProvider.plugin_id == plugin_id,+ )+ query = select(DatasourceProvider).where(*owner)+ if actor_id is not None:+ query = apply_credential_visibility_filter_for_actor(+ query,+ tenant_id=workspace_id,+ model_id_column=DatasourceProvider.id,+ model_user_id_column=DatasourceProvider.user_id,+ model_visibility_column=DatasourceProvider.visibility,+ credential_type=CredentialType.DATASOURCE_PROVIDER,+ actor_id=actor_id,+ )+ with self._session_factory() as session:+ rows = session.scalars(query).all()+ if not rows:+ return []+ # Preserve the configured default even when that credential is hidden from this actor.+ default_id = session.scalar(+ select(DatasourceProvider.id)+ .where(*owner)+ .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())+ .limit(1)+ )+ return [+ DatasourceCredentialListItem(+ id=row.id,+ name=row.name,+ auth_type=row.auth_type,+ encrypted_credentials=dict(row.encrypted_credentials),+ avatar_url=row.avatar_url,+ is_default=row.id == default_id,+ )+ for row in rows+ ]++ @override+ def list_tools(+ self,+ *,+ workspace_id: str,+ provider: str,+ actor_id: str | None,+ include_credential_ids: Sequence[str] = (),+ ) -> Sequence[ToolCredentialRecord]:+ owner = (BuiltinToolProvider.tenant_id == workspace_id, BuiltinToolProvider.provider == provider)+ order = (BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())+ query = select(BuiltinToolProvider).where(*owner).order_by(*order)+ if actor_id is not None:+ query = apply_credential_visibility_filter_for_actor(+ query,+ tenant_id=workspace_id,+ model_id_column=BuiltinToolProvider.id,+ model_user_id_column=BuiltinToolProvider.user_id,+ model_visibility_column=BuiltinToolProvider.visibility,+ credential_type=CredentialType.BUILTIN_TOOL_PROVIDER,+ actor_id=actor_id,+ )+ with self._session_factory() as session:+ visible = list(session.scalars(query))+ wanted = set(include_credential_ids) - {row.id for row in visible} - {""}+ borrowed: list[BuiltinToolProvider] = (+ list(+ session.scalars(+ select(BuiltinToolProvider).where(*owner, BuiltinToolProvider.id.in_(wanted)).order_by(*order)+ )+ )+ if wanted+ else []+ )+ rows = visible + borrowed+ partial_ids = [row.id for row in rows if row.visibility == PermissionEnum.PARTIAL_TEAM]+ members: dict[str, list[str]] = {}+ if partial_ids:+ for credential_id, account_id in session.execute(+ select(CredentialPermission.credential_id, CredentialPermission.account_id).where(+ CredentialPermission.tenant_id == workspace_id,+ CredentialPermission.credential_type == CredentialType.BUILTIN_TOOL_PROVIDER,+ CredentialPermission.credential_id.in_(partial_ids),+ CredentialPermission.has_permission.is_(True),+ )+ ):+ members.setdefault(credential_id, []).append(account_id)+ borrowed_ids = {row.id for row in borrowed}+ return [+ ToolCredentialRecord(+ id=row.id,+ name=row.name,+ provider=row.provider,+ credential_type=row.credential_type,+ credentials=dict(row.credentials),+ is_default=(bool(visible) and row.id == visible[0].id) or row.is_default,+ visibility=str(row.visibility),+ created_by=row.user_id or "",+ partial_member_ids=tuple(members.get(row.id, [])),+ from_other_member=row.id in borrowed_ids,+ )+ for row in rows+ ]++ @override+ def list_trigger_subscriptions(+ self, *, workspace_id: str, provider: str, actor_id: str | None+ ) -> Sequence[TriggerSubscriptionRecord]:+ query = (+ select(TriggerSubscription)+ .where(TriggerSubscription.tenant_id == workspace_id, TriggerSubscription.provider_id == provider)+ .order_by(TriggerSubscription.created_at.desc())+ )+ if actor_id is not None:+ query = apply_credential_visibility_filter_for_actor(+ query,+ tenant_id=workspace_id,+ model_id_column=TriggerSubscription.id,+ model_user_id_column=TriggerSubscription.user_id,+ model_visibility_column=TriggerSubscription.visibility,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ actor_id=actor_id,+ )+ with self._session_factory() as session:+ rows = session.scalars(query).all()+ if not rows:+ return []+ counts = dict(+ session.execute(+ select(+ WorkflowPluginTrigger.subscription_id, func.count(func.distinct(WorkflowPluginTrigger.app_id))+ )+ .where(+ WorkflowPluginTrigger.tenant_id == workspace_id,+ WorkflowPluginTrigger.subscription_id.in_([row.id for row in rows]),+ )+ .group_by(WorkflowPluginTrigger.subscription_id)+ )+ .tuples()+ .all()+ )+ return [+ TriggerSubscriptionRecord(+ id=row.id,+ name=row.name,+ provider=row.provider_id,+ credential_type=row.credential_type,+ credentials=dict(row.credentials),+ endpoint_id=row.endpoint_id,+ parameters=dict(row.parameters),+ properties=dict(row.properties),+ workflows_in_use=counts.get(row.id, 0),+ )+ for row in rows+ ]diff --git a/api/repositories/data_source/__init__.py b/api/repositories/data_source/__init__.pynew file mode 100644index 00000000000..5b4f34ada4d--- /dev/null+++ b/api/repositories/data_source/__init__.py@@ -0,0 +1 @@+"""Persistence adapters for the data-source domain."""diff --git a/api/repositories/data_source_api_key_auth_repository.py b/api/repositories/data_source/api_key_auth_repository.pysimilarity index 94%rename from api/repositories/data_source_api_key_auth_repository.pyrename to api/repositories/data_source/api_key_auth_repository.pyindex a28ebc16204..9f514d6b3f0 100644--- a/api/repositories/data_source_api_key_auth_repository.py+++ b/api/repositories/data_source/api_key_auth_repository.py@@ -8,8 +8,8 @@ from sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmakerfrom models.source import DataSourceApiKeyAuthBinding-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthBindingRepository-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthBindingRepository+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingRecord,DataSourceApiKeyAuthCredentials,)diff --git a/api/repositories/data_source/credential_repository.py b/api/repositories/data_source/credential_repository.pynew file mode 100644index 00000000000..9862ada014a--- /dev/null+++ b/api/repositories/data_source/credential_repository.py@@ -0,0 +1,252 @@+"""Actor-aware and trusted-source SQLAlchemy repository for datasource credentials."""++import json+from collections.abc import Mapping+from typing import cast++from pydantic import TypeAdapter, ValidationError+from sqlalchemy import select, update+from sqlalchemy.engine import CursorResult+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from models.credential_permission import CredentialType+from models.dataset import Dataset, Document+from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProvider+from repositories.credentials.query_repository import apply_credential_visibility_filter_for_actor+from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecord++_CREDENTIALS_ADAPTER = TypeAdapter(dict[str, object])+++def _source_mapping(value: object) -> Mapping[str, object]:+ try:+ if isinstance(value, str):+ return _CREDENTIALS_ADAPTER.validate_python(json.loads(value))+ return _CREDENTIALS_ADAPTER.validate_python(value)+ except (TypeError, ValueError, ValidationError):+ return {}+++def _datasource_credential_record(provider: DatasourceProvider) -> DatasourceCredentialRecord:+ return DatasourceCredentialRecord(+ id=provider.id,+ workspace_id=provider.tenant_id,+ owner_id=provider.user_id,+ name=provider.name,+ provider=provider.provider,+ plugin_id=provider.plugin_id,+ auth_type=provider.auth_type,+ encrypted_credentials=_CREDENTIALS_ADAPTER.validate_python(provider.encrypted_credentials),+ expires_at=provider.expires_at,+ updated_at=provider.updated_at,+ )+++class SQLAlchemyDatasourceCredentialRepository:+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def get_provider_credential(+ self, *, workspace_id: str, provider: str, plugin_id: str, credential_id: str | None+ ) -> DatasourceCredentialRecord | None:+ with self._session_factory() as session:+ rows = _read_provider_credentials(+ session,+ workspace_id=workspace_id,+ provider=provider,+ plugin_id=plugin_id,+ credential_id=credential_id,+ first_only=True,+ )+ return _datasource_credential_record(rows[0]) if rows else None++ def list_provider_credentials(+ self, *, workspace_id: str, provider: str, plugin_id: str+ ) -> list[DatasourceCredentialRecord]:+ with self._session_factory() as session:+ return [+ _datasource_credential_record(row)+ for row in _read_provider_credentials(+ session,+ workspace_id=workspace_id,+ provider=provider,+ plugin_id=plugin_id,+ )+ ]++ def get_visible(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None:+ query = select(DatasourceProvider).where(+ DatasourceProvider.tenant_id == workspace_id,+ DatasourceProvider.id == credential_id,+ DatasourceProvider.provider == provider,+ DatasourceProvider.plugin_id == plugin_id,+ )+ query = apply_credential_visibility_filter_for_actor(+ query,+ tenant_id=workspace_id,+ model_id_column=DatasourceProvider.id,+ model_user_id_column=DatasourceProvider.user_id,+ model_visibility_column=DatasourceProvider.visibility,+ credential_type=CredentialType.DATASOURCE_PROVIDER,+ actor_id=actor_id,+ )+ with self._session_factory() as session:+ provider_record = session.scalar(query.limit(1))+ return _datasource_credential_record(provider_record) if provider_record is not None else None++ def get_oauth_client_config(+ self,+ *,+ workspace_id: str,+ provider: str,+ plugin_id: str,+ ) -> DatasourceOAuthClientConfigRecord:+ """Load OAuth client rows in one bounded read transaction."""++ with self._session_factory() as session:+ return _read_oauth_client_config(session, workspace_id=workspace_id, provider=provider, plugin_id=plugin_id)++ def update_if_unchanged(+ self,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+ ) -> bool:+ with self._session_factory.begin() as session:+ return _update_datasource_credentials_if_unchanged(+ session, record=record, encrypted_credentials=encrypted_credentials, expires_at=expires_at+ )++ def get_for_stored_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None:+ with self._session_factory() as session:+ query = (+ select(DatasourceProvider, Document.data_source_info)+ .select_from(Document)+ .join(Dataset, Dataset.id == Document.dataset_id)+ .join(DatasourceProvider, DatasourceProvider.tenant_id == Document.tenant_id)+ .where(+ Dataset.id == dataset_id,+ Dataset.tenant_id == workspace_id,+ Document.id == document_id,+ Document.dataset_id == dataset_id,+ Document.tenant_id == workspace_id,+ Document.data_source_type == "notion_import",+ DatasourceProvider.tenant_id == workspace_id,+ DatasourceProvider.provider == provider,+ DatasourceProvider.plugin_id == plugin_id,+ )+ )+ if credential_id is not None:+ query = query.where(DatasourceProvider.id == credential_id)+ else:+ query = query.where(DatasourceProvider.is_default.is_(True)).order_by(+ DatasourceProvider.created_at.asc()+ )+ row = session.execute(query.limit(1)).one_or_none()+ if row is None:+ return None+ datasource_provider, raw_source_info = row+ source_info = _source_mapping(raw_source_info)+ if credential_id is not None and source_info.get("credential_id") != credential_id:+ return None+ return _datasource_credential_record(datasource_provider)+++def _read_oauth_client_config(+ session: Session, *, workspace_id: str, provider: str, plugin_id: str+) -> DatasourceOAuthClientConfigRecord:+ tenant_config = session.scalar(+ select(DatasourceOauthTenantParamConfig)+ .where(+ DatasourceOauthTenantParamConfig.tenant_id == workspace_id,+ DatasourceOauthTenantParamConfig.provider == provider,+ DatasourceOauthTenantParamConfig.plugin_id == plugin_id,+ DatasourceOauthTenantParamConfig.enabled.is_(True),+ )+ .limit(1)+ )+ system_config = session.scalar(+ select(DatasourceOauthParamConfig)+ .where(+ DatasourceOauthParamConfig.provider == provider,+ DatasourceOauthParamConfig.plugin_id == plugin_id,+ )+ .limit(1)+ )+ return DatasourceOAuthClientConfigRecord(+ encrypted_tenant_params=(dict(tenant_config.client_params) if tenant_config is not None else None),+ system_credentials=(dict(system_config.system_credentials) if system_config is not None else None),+ )+++def _update_datasource_credentials_if_unchanged(+ session: Session,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+) -> bool:+ credentials = _CREDENTIALS_ADAPTER.validate_python(encrypted_credentials)+ result = session.execute(+ update(DatasourceProvider)+ .where(+ DatasourceProvider.id == record.id,+ DatasourceProvider.tenant_id == record.workspace_id,+ DatasourceProvider.provider == record.provider,+ DatasourceProvider.plugin_id == record.plugin_id,+ DatasourceProvider.user_id == record.owner_id,+ DatasourceProvider.encrypted_credentials+ == _CREDENTIALS_ADAPTER.validate_python(record.encrypted_credentials),+ DatasourceProvider.expires_at == record.expires_at,+ )+ .values(+ encrypted_credentials=credentials,+ expires_at=expires_at,+ updated_at=naive_utc_now(),+ )+ )+ return cast(CursorResult[object], result).rowcount == 1+++def _read_provider_credentials(+ session: Session,+ *,+ workspace_id: str,+ provider: str,+ plugin_id: str,+ credential_id: str | None = None,+ first_only: bool = False,+) -> list[DatasourceProvider]:+ statement = (+ select(DatasourceProvider)+ .where(+ DatasourceProvider.tenant_id == workspace_id,+ DatasourceProvider.provider == provider,+ DatasourceProvider.plugin_id == plugin_id,+ )+ .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())+ )+ if credential_id is not None:+ statement = statement.where(DatasourceProvider.id == credential_id)+ if first_only:+ statement = statement.limit(1)+ return list(session.scalars(statement))diff --git a/api/repositories/data_source_oauth_binding_repository.py b/api/repositories/data_source/oauth_binding_repository.pysimilarity index 56%rename from api/repositories/data_source_oauth_binding_repository.pyrename to api/repositories/data_source/oauth_binding_repository.pyindex 5c52140f13a..98cda3c65d3 100644--- a/api/repositories/data_source_oauth_binding_repository.py+++ b/api/repositories/data_source/oauth_binding_repository.py@@ -1,24 +1,27 @@"""SQLAlchemy repository for OAuth data-source bindings."""from collections.abc import Mapping-from typing import override+from typing import cast, overridefrom pydantic import TypeAdapter-from sqlalchemy import select+from sqlalchemy import select, update+from sqlalchemy.engine import CursorResultfrom sqlalchemy.orm import Session, sessionmakerfrom libs.datetime_utils import naive_utc_nowfrom models.source import DataSourceOauthBinding-from services.data_source_oauth_service import DataSourceOAuthBindingRepository-from services.entities.data_source_oauth_entities import (+from services.data_source.binding_application_service import BindingMutationResult, DataSourceBindingStore+from services.data_source.entities.oauth import (+ DataSourceBindingSummary,DataSourceOAuthAuthorization,DataSourceOAuthBindingRecord,)+from services.data_source.oauth_service import DataSourceOAuthBindingRepository_SOURCE_INFO_ADAPTER = TypeAdapter(dict[str, object])-class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepository):+class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepository, DataSourceBindingStore):def __init__(self, session_factory: sessionmaker[Session]) -> None:self._session_factory = session_factory@@ -68,7 +71,7 @@ class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepositorDataSourceOauthBinding.tenant_id == workspace_id,DataSourceOauthBinding.provider == provider,DataSourceOauthBinding.id == binding_id,- DataSourceOauthBinding.disabled.is_(False),+ DataSourceOauthBinding.disabled.is_not(True),))if binding is None:@@ -96,7 +99,7 @@ class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepositorDataSourceOauthBinding.tenant_id == workspace_id,DataSourceOauthBinding.provider == provider,DataSourceOauthBinding.id == binding_id,- DataSourceOauthBinding.disabled.is_(False),+ DataSourceOauthBinding.disabled.is_not(True),))if binding is None:@@ -105,3 +108,59 @@ class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepositorbinding.source_info = validated_source_infobinding.updated_at = naive_utc_now()return True++ @override+ def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:+ with self._session_factory() as session:+ bindings = session.scalars(+ select(DataSourceOauthBinding).where(+ DataSourceOauthBinding.tenant_id == workspace_id,+ DataSourceOauthBinding.disabled.is_not(True),+ )+ ).all()+ return tuple(+ DataSourceBindingSummary(+ id=binding.id,+ provider=binding.provider,+ created_at=binding.created_at,+ disabled=False,+ source_info=_SOURCE_INFO_ADAPTER.validate_python(binding.source_info),+ )+ for binding in bindings+ )++ @override+ def change_disabled_state(+ self,+ *,+ workspace_id: str,+ binding_id: str,+ disabled: bool,+ ) -> BindingMutationResult:+ state_predicate = (+ DataSourceOauthBinding.disabled.is_not(True) if disabled else DataSourceOauthBinding.disabled.is_(True)+ )+ with self._session_factory.begin() as session:+ result = session.execute(+ update(DataSourceOauthBinding)+ .where(+ DataSourceOauthBinding.tenant_id == workspace_id,+ DataSourceOauthBinding.id == binding_id,+ state_predicate,+ )+ .values(disabled=disabled, updated_at=naive_utc_now())+ )+ if cast(CursorResult[object], result).rowcount == 1:+ return "updated"++ binding_exists = session.scalar(+ select(DataSourceOauthBinding.id)+ .where(+ DataSourceOauthBinding.tenant_id == workspace_id,+ DataSourceOauthBinding.id == binding_id,+ )+ .limit(1)+ )+ if binding_exists is None:+ return "not_found"+ return "already_disabled" if disabled else "already_enabled"diff --git a/api/repositories/knowledge/__init__.py b/api/repositories/knowledge/__init__.pyindex 2ba7ed3f45b..6769eeaf0f9 100644--- a/api/repositories/knowledge/__init__.py+++ b/api/repositories/knowledge/__init__.py@@ -1 +1 @@-"""Knowledge base persistence and queries."""+"""Knowledge-domain persistence adapters and queries."""diff --git a/api/repositories/knowledge/dataset_api_key_repository.py b/api/repositories/knowledge/dataset_api_key_repository.pyindex 0dd1264eced..841effc81e5 100644--- a/api/repositories/knowledge/dataset_api_key_repository.py+++ b/api/repositories/knowledge/dataset_api_key_repository.py@@ -5,8 +5,8 @@ from typing import overridefrom sqlalchemy import Select, func, selectfrom sqlalchemy.orm import Session, sessionmaker-from models.account import Tenant, TenantAccountJoin-from models.dataset import Dataset, DatasetPermission+from models.account import Tenant+from models.dataset import Datasetfrom models.enums import ApiTokenTypefrom models.model import ApiTokenfrom repositories.knowledge import dataset_api_key_bindings@@ -17,37 +17,12 @@ from services.auth.api_key_contracts import (ApiKeyResourceNotFoundError,)from services.knowledge.api_key_service import DatasetApiKeyStore, UnknownDatasetIdsError-from services.knowledge.dataset_access import DatasetAccessclass DatasetApiKeyRepository(DatasetApiKeyStore):def __init__(self, *, session_factory: sessionmaker[Session]) -> None:self._session_factory = session_factory- @override- def get_dataset_access(self, workspace_id: str, dataset_id: str, account_id: str) -> DatasetAccess:- with self._session_factory() as session:- dataset = self._get_dataset(session, workspace_id, dataset_id)- role = session.scalar(- select(TenantAccountJoin.role).where(- TenantAccountJoin.tenant_id == workspace_id, TenantAccountJoin.account_id == account_id- )- )- has_permission = (- session.scalar(- select(DatasetPermission.id)- .where(- DatasetPermission.tenant_id == workspace_id,- DatasetPermission.dataset_id == dataset_id,- DatasetPermission.account_id == account_id,- DatasetPermission.has_permission.is_(True),- )- .limit(1)- )- is not None- )- return DatasetAccess(dataset.permission, dataset.maintainer, role, has_permission)-@overridedef list_keys(self, workspace_id: str, dataset_id: str) -> tuple[ApiKeyRecord, ...]:with self._session_factory() as session:diff --git a/api/repositories/knowledge/dataset_read_repository.py b/api/repositories/knowledge/dataset_read_repository.pynew file mode 100644index 00000000000..a690eee798c--- /dev/null+++ b/api/repositories/knowledge/dataset_read_repository.py@@ -0,0 +1,755 @@+"""Explicit reads for knowledge models within the caller's transaction.++These query functions are shared by repositories and existing transaction owners;+they never create, commit, or close a session and perform no external I/O.+"""++from __future__ import annotations++from collections.abc import Sequence+from dataclasses import dataclass, field+from typing import Any++import sqlalchemy as sa+from sqlalchemy import func, select, tuple_+from sqlalchemy.orm import Session, scoped_session++from core.app.file_access import DatabaseFileAccessController+from core.rag.entities import ParentMode, Rule+from models.account import Account+from models.dataset import (+ AppDatasetJoin,+ ChildChunk,+ Dataset,+ DatasetBindingItem,+ DatasetKeywordTable,+ DatasetMetadata,+ DatasetMetadataBinding,+ DatasetProcessRule,+ DatasetQuery,+ Document,+ DocumentSegment,+ ExternalKnowledgeApis,+ ExternalKnowledgeBindings,+ Pipeline,+ PipelineCustomizedTemplate,+ SegmentAttachmentBinding,+)+from models.model import App, Tag, TagBinding, UploadFile+from models.tools import ToolFile+++@dataclass(frozen=True)+class DocumentReadBatch:+ uploads: dict[tuple[str, str], UploadFile] = field(default_factory=dict)+ metadatas: dict[str, list[DatasetMetadata]] = field(default_factory=dict)+ built_in_enabled: dict[tuple[str, str], bool] = field(default_factory=dict)+ uploader_names: dict[str, str] = field(default_factory=dict)+ segment_counts: dict[str, int] = field(default_factory=dict)+ hit_counts: dict[str, int] = field(default_factory=dict)+ process_rules: dict[str, DatasetProcessRule] = field(default_factory=dict)+++def load_document_read_batch(documents: Sequence[Document], *, session: Session) -> DocumentReadBatch:+ """Load a page's document relations with a bounded number of queries."""+ if not documents:+ return DocumentReadBatch()+ owners = [(doc.tenant_id, doc.dataset_id, doc.id) for doc in documents]+ dataset_owners = list({(doc.tenant_id, doc.dataset_id) for doc in documents})+ upload_owners = list(+ {+ (doc.tenant_id, file_id)+ for doc in documents+ if doc.data_source_type == "upload_file" and (file_id := doc.data_source_info_dict.get("upload_file_id"))+ }+ )+ uploads = (+ {+ (file.tenant_id, file.id): file+ for file in session.scalars(+ select(UploadFile).where(tuple_(UploadFile.tenant_id, UploadFile.id).in_(upload_owners))+ )+ }+ if upload_owners+ else {}+ )+ metadatas: dict[str, list[DatasetMetadata]] = {}+ for doc_id, metadata in session.execute(+ select(DatasetMetadataBinding.document_id, DatasetMetadata)+ .join(DatasetMetadata, DatasetMetadata.id == DatasetMetadataBinding.metadata_id)+ .where(+ tuple_(+ DatasetMetadataBinding.tenant_id, DatasetMetadataBinding.dataset_id, DatasetMetadataBinding.document_id+ ).in_(owners),+ DatasetMetadata.tenant_id == DatasetMetadataBinding.tenant_id,+ DatasetMetadata.dataset_id == DatasetMetadataBinding.dataset_id,+ )+ ):+ metadatas.setdefault(doc_id, []).append(metadata)+ built_in_enabled = {+ (tenant_id, dataset_id): enabled+ for tenant_id, dataset_id, enabled in session.execute(+ select(Dataset.tenant_id, Dataset.id, Dataset.built_in_field_enabled).where(+ tuple_(Dataset.tenant_id, Dataset.id).in_(dataset_owners)+ )+ )+ }+ uploader_names = dict(+ session.execute(select(Account.id, Account.name).where(Account.id.in_({doc.created_by for doc in documents})))+ .tuples()+ .all()+ )+ counts = session.execute(+ select(+ DocumentSegment.document_id,+ func.count(DocumentSegment.id),+ func.coalesce(func.sum(DocumentSegment.hit_count), 0),+ )+ .where(tuple_(DocumentSegment.tenant_id, DocumentSegment.dataset_id, DocumentSegment.document_id).in_(owners))+ .group_by(DocumentSegment.document_id)+ ).all()+ rule_ids = {doc.dataset_process_rule_id for doc in documents if doc.dataset_process_rule_id}+ process_rules = (+ {+ rule.id: rule+ for rule in session.scalars(+ select(DatasetProcessRule)+ .join(Dataset, Dataset.id == DatasetProcessRule.dataset_id)+ .where(DatasetProcessRule.id.in_(rule_ids), tuple_(Dataset.tenant_id, Dataset.id).in_(dataset_owners))+ )+ }+ if rule_ids+ else {}+ )+ return DocumentReadBatch(+ uploads=uploads,+ metadatas=metadatas,+ built_in_enabled=built_in_enabled,+ uploader_names=uploader_names,+ segment_counts={doc_id: count for doc_id, count, _ in counts},+ hit_counts={doc_id: hits for doc_id, _, hits in counts},+ process_rules=process_rules,+ )+++def get_segment_content_file_ids(+ segment: DocumentSegment, *, upload_ids: set[str], tool_ids: set[str], session: Session+) -> tuple[set[str], set[str]]:+ access = DatabaseFileAccessController()+ uploads = (+ session.scalars(+ access.apply_upload_file_filters(+ select(UploadFile).where(UploadFile.id.in_(upload_ids), UploadFile.tenant_id == segment.tenant_id)+ )+ ).all()+ if upload_ids+ else []+ )+ tools = (+ session.scalars(+ access.apply_tool_file_filters(+ select(ToolFile).where(ToolFile.id.in_(tool_ids), ToolFile.tenant_id == segment.tenant_id)+ )+ ).all()+ if tool_ids+ else []+ )+ return {file.id for file in uploads}, {file.id for file in tools}+++def get_segment_attachment_files(segment: DocumentSegment, *, session: Session) -> Sequence[UploadFile]:+ return session.scalars(+ select(UploadFile)+ .join(SegmentAttachmentBinding, UploadFile.id == SegmentAttachmentBinding.attachment_id)+ .where(+ SegmentAttachmentBinding.tenant_id == segment.tenant_id,+ SegmentAttachmentBinding.dataset_id == segment.dataset_id,+ SegmentAttachmentBinding.document_id == segment.document_id,+ SegmentAttachmentBinding.segment_id == segment.id,+ UploadFile.tenant_id == segment.tenant_id,+ )+ ).all()+++def is_retrieved_segment_owned(+ segment: DocumentSegment, *, tenant_id: str, dataset_ids: Sequence[str], session: Session+) -> bool:+ """Validate retrieved identifiers against the complete persisted owner chain."""+ if segment.tenant_id != tenant_id or segment.dataset_id not in dataset_ids:+ return False+ return (+ session.scalar(+ select(DocumentSegment.id)+ .join(Document, Document.id == DocumentSegment.document_id)+ .join(Dataset, Dataset.id == Document.dataset_id)+ .where(+ Dataset.tenant_id == tenant_id,+ Dataset.id == segment.dataset_id,+ Document.tenant_id == tenant_id,+ Document.dataset_id == segment.dataset_id,+ Document.id == segment.document_id,+ Document.enabled.is_(True),+ Document.archived.is_(False),+ DocumentSegment.tenant_id == tenant_id,+ DocumentSegment.dataset_id == segment.dataset_id,+ DocumentSegment.id == segment.id,+ DocumentSegment.enabled.is_(True),+ )+ )+ is not None+ )+++def get_document_upload_file(document: Document, *, session: Session) -> UploadFile | None:+ file_id = document.data_source_info_dict.get("upload_file_id")+ if not file_id:+ return None+ return session.scalar(+ select(UploadFile).where(UploadFile.id == file_id, UploadFile.tenant_id == document.tenant_id)+ )+++def get_document_metadata_rows(document: Document, *, session: Session) -> Sequence[DatasetMetadata]:+ return session.scalars(+ select(DatasetMetadata)+ .join(DatasetMetadataBinding, DatasetMetadataBinding.metadata_id == DatasetMetadata.id)+ .where(+ DatasetMetadata.tenant_id == document.tenant_id,+ DatasetMetadata.dataset_id == document.dataset_id,+ DatasetMetadataBinding.tenant_id == document.tenant_id,+ DatasetMetadataBinding.dataset_id == document.dataset_id,+ DatasetMetadataBinding.document_id == document.id,+ )+ ).all()+++def get_query_upload_files(query: DatasetQuery, file_ids: set[str], *, session: Session) -> dict[str, UploadFile]:+ if not file_ids:+ return {}+ rows = session.scalars(+ select(UploadFile)+ .join(Dataset, Dataset.tenant_id == UploadFile.tenant_id)+ .where(Dataset.id == query.dataset_id, UploadFile.id.in_(file_ids))+ ).all()+ return {file.id: file for file in rows}+++def get_external_api_bindings_batch(+ apis: Sequence[ExternalKnowledgeApis], *, session: Session+) -> dict[str, list[DatasetBindingItem]]:+ if not apis:+ return {}+ allowed = {(api.id, api.tenant_id) for api in apis}+ rows = session.execute(+ select(+ ExternalKnowledgeBindings.external_knowledge_api_id,+ ExternalKnowledgeBindings.tenant_id,+ Dataset.id,+ Dataset.name,+ )+ .join(Dataset, Dataset.id == ExternalKnowledgeBindings.dataset_id)+ .where(+ tuple_(ExternalKnowledgeBindings.external_knowledge_api_id, ExternalKnowledgeBindings.tenant_id).in_(+ allowed+ ),+ Dataset.tenant_id == ExternalKnowledgeBindings.tenant_id,+ )+ .distinct()+ ).all()+ bindings: dict[str, list[DatasetBindingItem]] = {}+ for api_id, tenant_id, dataset_id, name in rows:+ if (api_id, tenant_id) in allowed:+ bindings.setdefault(api_id, []).append({"id": dataset_id, "name": name})+ return bindings+++def get_external_api_dataset_bindings(api: ExternalKnowledgeApis, *, session: Session) -> list[DatasetBindingItem]:+ return get_external_api_bindings_batch([api], session=session).get(api.id, [])+++def get_dataset_keyword_table(dataset: Dataset, *, session: Session) -> DatasetKeywordTable | None:+ return session.scalar(select(DatasetKeywordTable).where(DatasetKeywordTable.dataset_id == dataset.id))+++def get_dataset_creator(dataset: Dataset, *, session: Session) -> Account | None:+ return session.get(Account, dataset.created_by)+++def get_dataset_author_name(dataset: Dataset, *, session: Session) -> str | None:+ account = get_dataset_creator(dataset, session=session)+ if account:+ return account.name+ return None+++def get_latest_dataset_process_rule(dataset: Dataset, *, session: Session) -> DatasetProcessRule | None:+ return session.scalar(+ select(DatasetProcessRule)+ .where(DatasetProcessRule.dataset_id == dataset.id)+ .order_by(DatasetProcessRule.created_at.desc())+ .limit(1)+ )+++def get_dataset_app_count(dataset: Dataset, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.count(AppDatasetJoin.id)).where(+ AppDatasetJoin.dataset_id == dataset.id,+ App.id == AppDatasetJoin.app_id,+ App.tenant_id == dataset.tenant_id,+ )+ )+ or 0+ )+++def get_dataset_document_count(dataset: Dataset, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.count(Document.id)).where(+ Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id+ )+ )+ or 0+ )+++def get_dataset_available_document_count(dataset: Dataset, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.count(Document.id)).where(+ Document.dataset_id == dataset.id,+ Document.tenant_id == dataset.tenant_id,+ Document.indexing_status == "completed",+ Document.enabled == True,+ Document.archived == False,+ )+ )+ or 0+ )+++def get_dataset_available_segment_count(dataset: Dataset, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.count(DocumentSegment.id)).where(+ DocumentSegment.dataset_id == dataset.id,+ DocumentSegment.tenant_id == dataset.tenant_id,+ DocumentSegment.status == "completed",+ DocumentSegment.enabled == True,+ )+ )+ or 0+ )+++def get_dataset_word_count(dataset: Dataset, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.coalesce(func.sum(Document.word_count), 0)).where(+ Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id+ )+ )+ or 0+ )+++def get_dataset_doc_form(dataset: Dataset, *, session: Session) -> str | None:+ if dataset.chunk_structure:+ return dataset.chunk_structure+ return session.scalar(+ select(Document.doc_form)+ .where(Document.dataset_id == dataset.id, Document.tenant_id == dataset.tenant_id)+ .limit(1)+ )+++def get_dataset_tags(dataset: Dataset, *, session: Session) -> Sequence[Tag]:+ tags = session.scalars(+ select(Tag)+ .join(TagBinding, Tag.id == TagBinding.tag_id)+ .where(+ TagBinding.target_id == dataset.id,+ TagBinding.tenant_id == dataset.tenant_id,+ Tag.tenant_id == dataset.tenant_id,+ Tag.type == "knowledge",+ )+ ).all()+ return tags or []+++def get_external_knowledge_info(dataset: Dataset, *, session: Session) -> dict[str, Any] | None:+ if dataset.provider != "external":+ return None+ external_knowledge_binding = session.scalar(+ select(ExternalKnowledgeBindings).where(+ ExternalKnowledgeBindings.dataset_id == dataset.id, ExternalKnowledgeBindings.tenant_id == dataset.tenant_id+ )+ )+ if not external_knowledge_binding:+ return None+ external_knowledge_api = session.scalar(+ select(ExternalKnowledgeApis).where(+ ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id,+ ExternalKnowledgeApis.tenant_id == dataset.tenant_id,+ )+ )+ return dataset.build_external_knowledge_info(external_knowledge_binding, external_knowledge_api)+++def get_dataset_is_published(dataset: Dataset, *, session: Session) -> bool:+ if dataset.pipeline_id:+ pipeline = session.scalar(+ select(Pipeline).where(Pipeline.id == dataset.pipeline_id, Pipeline.tenant_id == dataset.tenant_id)+ )+ if pipeline:+ return pipeline.is_published+ return False+++def get_dataset_doc_metadata(dataset: Dataset, *, session: Session) -> list[dict[str, str]]:+ dataset_metadatas = session.scalars(+ select(DatasetMetadata).where(+ DatasetMetadata.dataset_id == dataset.id, DatasetMetadata.tenant_id == dataset.tenant_id+ )+ ).all()+ return dataset.build_doc_metadata(dataset_metadatas)+++def get_document_process_rule(document: Document, *, session: Session) -> DatasetProcessRule | None:+ if document.dataset_process_rule_id:+ return session.scalar(+ select(DatasetProcessRule).where(+ DatasetProcessRule.id == document.dataset_process_rule_id,+ DatasetProcessRule.dataset_id == document.dataset_id,+ )+ )+ return None+++def get_document_dataset(document: Document, *, session: Session) -> Dataset | None:+ """Load the owning dataset with the caller-owned database session."""+ return session.scalar(+ select(Dataset).where(Dataset.id == document.dataset_id, Dataset.tenant_id == document.tenant_id)+ )+++def get_document_segment_count(document: Document, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.count(DocumentSegment.id)).where(+ DocumentSegment.document_id == document.id,+ DocumentSegment.dataset_id == document.dataset_id,+ DocumentSegment.tenant_id == document.tenant_id,+ )+ )+ or 0+ )+++def get_document_hit_count(document: Document, *, session: Session) -> int:+ return (+ session.scalar(+ select(func.coalesce(func.sum(DocumentSegment.hit_count), 0)).where(+ DocumentSegment.document_id == document.id,+ DocumentSegment.dataset_id == document.dataset_id,+ DocumentSegment.tenant_id == document.tenant_id,+ )+ )+ or 0+ )+++def get_document_uploader(document: Document, *, session: Session) -> str | None:+ user = session.scalar(select(Account).where(Account.id == document.created_by))+ return user.name if user else None+++def get_segment_dataset(segment: DocumentSegment, *, session: Session) -> Dataset | None:+ """Load the owning dataset with the caller-owned database session."""+ return session.scalar(+ select(Dataset).where(Dataset.id == segment.dataset_id, Dataset.tenant_id == segment.tenant_id)+ )+++def get_segment_document(segment: DocumentSegment, *, session: Session) -> Document | None:+ """Load the owning document with the caller-owned database session."""+ return session.scalar(+ select(Document).where(+ Document.id == segment.document_id,+ Document.dataset_id == segment.dataset_id,+ Document.tenant_id == segment.tenant_id,+ )+ )+++def get_previous_segment(segment: DocumentSegment, session: Session) -> DocumentSegment | None:+ return session.scalar(+ select(DocumentSegment).where(+ DocumentSegment.document_id == segment.document_id,+ DocumentSegment.dataset_id == segment.dataset_id,+ DocumentSegment.tenant_id == segment.tenant_id,+ DocumentSegment.position == segment.position - 1,+ )+ )+++def get_next_segment(segment: DocumentSegment, session: Session) -> DocumentSegment | None:+ return session.scalar(+ select(DocumentSegment).where(+ DocumentSegment.document_id == segment.document_id,+ DocumentSegment.dataset_id == segment.dataset_id,+ DocumentSegment.tenant_id == segment.tenant_id,+ DocumentSegment.position == segment.position + 1,+ )+ )+++def get_segment_child_chunks(+ segment: DocumentSegment, *, session: Session, include_full_doc: bool = True+) -> Sequence[ChildChunk]:+ """Load hierarchical child chunks with the caller-owned database session."""+ document = get_segment_document(segment, session=session)+ if not document:+ return []+ process_rule = get_document_process_rule(document, session=session)+ if process_rule and process_rule.mode == "hierarchical":+ rules_dict = process_rule.rules_dict+ if rules_dict:+ rules = Rule.model_validate(rules_dict)+ if rules.parent_mode and (include_full_doc or rules.parent_mode != ParentMode.FULL_DOC):+ child_chunks = session.scalars(+ select(ChildChunk)+ .where(+ ChildChunk.segment_id == segment.id,+ ChildChunk.tenant_id == segment.tenant_id,+ ChildChunk.dataset_id == segment.dataset_id,+ ChildChunk.document_id == segment.document_id,+ )+ .order_by(ChildChunk.position.asc())+ ).all()+ return child_chunks or []+ return []+++def get_child_dataset(chunk: ChildChunk, session: Session) -> Dataset | None:+ return session.scalar(select(Dataset).where(Dataset.id == chunk.dataset_id, Dataset.tenant_id == chunk.tenant_id))+++def get_child_document(chunk: ChildChunk, session: Session) -> Document | None:+ return session.scalar(+ select(Document).where(+ Document.id == chunk.document_id,+ Document.dataset_id == chunk.dataset_id,+ Document.tenant_id == chunk.tenant_id,+ )+ )+++def get_child_segment(chunk: ChildChunk, session: Session) -> DocumentSegment | None:+ return session.scalar(+ select(DocumentSegment).where(+ DocumentSegment.id == chunk.segment_id,+ DocumentSegment.document_id == chunk.document_id,+ DocumentSegment.dataset_id == chunk.dataset_id,+ DocumentSegment.tenant_id == chunk.tenant_id,+ )+ )+++def get_joined_app(binding: AppDatasetJoin, session: Session) -> App | None:+ return session.get(App, binding.app_id)+++def get_pipeline_template_creator_name(template: PipelineCustomizedTemplate, session: Session) -> str:+ account = session.scalar(select(Account).where(Account.id == template.created_by))+ if account:+ return account.name+ return ""+++def get_pipeline_dataset(pipeline: Pipeline, session: Session | scoped_session) -> Dataset | None:+ return session.scalar(+ select(Dataset).where(Dataset.pipeline_id == pipeline.id, Dataset.tenant_id == pipeline.tenant_id)+ )+++@dataclass(frozen=True)+class DatasetDetailPrefetch:+ """Batch-loaded values backing the session-scoped fields of a page of datasets.++ Building a detail response for a single dataset issues about ten queries for+ that row alone, so a list endpoint costs one query per dataset per field.+ Loading the same values for every dataset on the page up front keeps the+ query count bounded regardless of the page size.+ """++ app_counts: dict[str, int] = field(default_factory=dict)+ document_counts: dict[str, int] = field(default_factory=dict)+ word_counts: dict[str, int] = field(default_factory=dict)+ available_document_counts: dict[str, int] = field(default_factory=dict)+ author_names: dict[str, str] = field(default_factory=dict)+ tags: dict[tuple[str, str], list[Tag]] = field(default_factory=dict)+ doc_forms: dict[tuple[str, str], str | None] = field(default_factory=dict)+ external_knowledge_infos: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)+ doc_metadata: dict[str, list[DatasetMetadata]] = field(default_factory=dict)+ pipeline_published: dict[str, bool] = field(default_factory=dict)+++def build_dataset_detail_prefetch(datasets: Sequence[Dataset], *, session: Session) -> DatasetDetailPrefetch:+ """Load every session-scoped detail field for the given datasets in a fixed number of queries."""+ dataset_ids = [dataset.id for dataset in datasets]+ if not dataset_ids:+ return DatasetDetailPrefetch()++ tenant_ids = {dataset.tenant_id for dataset in datasets}+ dataset_owners = [(dataset.tenant_id, dataset.id) for dataset in datasets]++ app_counts = dict(+ session.execute(+ select(AppDatasetJoin.dataset_id, func.count(AppDatasetJoin.id))+ .where(+ AppDatasetJoin.dataset_id.in_(dataset_ids),+ App.id == AppDatasetJoin.app_id,+ tuple_(App.tenant_id, AppDatasetJoin.dataset_id).in_(dataset_owners),+ )+ .group_by(AppDatasetJoin.dataset_id)+ )+ .tuples()+ .all()+ )++ document_counts: dict[str, int] = {}+ word_counts: dict[str, int] = {}+ available_document_counts: dict[str, int] = {}+ available_document_case = sa.case(+ (+ sa.and_(+ Document.indexing_status == "completed",+ Document.enabled == True,+ Document.archived == False,+ ),+ 1,+ ),+ else_=0,+ )+ document_rows = session.execute(+ select(+ Document.dataset_id,+ func.count(Document.id),+ func.coalesce(func.sum(Document.word_count), 0),+ func.coalesce(func.sum(available_document_case), 0),+ )+ .where(tuple_(Document.tenant_id, Document.dataset_id).in_(dataset_owners))+ .group_by(Document.dataset_id)+ ).all()+ for dataset_id, document_count, word_count, available_count in document_rows:+ document_counts[dataset_id] = document_count+ word_counts[dataset_id] = word_count+ available_document_counts[dataset_id] = available_count++ # doc_form only falls back to the documents table for datasets without a+ # chunk structure of their own.+ doc_forms: dict[tuple[str, str], str | None] = {}+ doc_form_dataset_ids = [dataset.id for dataset in datasets if not dataset.chunk_structure]+ if doc_form_dataset_ids:+ doc_forms = {+ (dataset_id, tenant_id): doc_form+ for dataset_id, tenant_id, doc_form in session.execute(+ select(Document.dataset_id, Document.tenant_id, func.min(Document.doc_form))+ .where(+ Document.dataset_id.in_(doc_form_dataset_ids),+ tuple_(Document.tenant_id, Document.dataset_id).in_(dataset_owners),+ )+ .group_by(Document.dataset_id, Document.tenant_id)+ ).all()+ }++ author_ids = {dataset.created_by for dataset in datasets if dataset.created_by}+ author_names = dict(+ session.execute(select(Account.id, Account.name).where(Account.id.in_(author_ids))).tuples().all()+ )++ tags: dict[tuple[str, str], list[Tag]] = {}+ tag_rows = session.execute(+ select(TagBinding.target_id, TagBinding.tenant_id, Tag)+ .join(TagBinding, Tag.id == TagBinding.tag_id)+ .where(+ tuple_(TagBinding.tenant_id, TagBinding.target_id).in_(dataset_owners),+ Tag.tenant_id == TagBinding.tenant_id,+ Tag.type == "knowledge",+ )+ ).all()+ for target_id, tenant_id, tag in tag_rows:+ tags.setdefault((target_id, tenant_id), []).append(tag)++ doc_metadata: dict[str, list[DatasetMetadata]] = {}+ for dataset_metadata in session.scalars(+ select(DatasetMetadata).where(tuple_(DatasetMetadata.tenant_id, DatasetMetadata.dataset_id).in_(dataset_owners))+ ).all():+ doc_metadata.setdefault(dataset_metadata.dataset_id, []).append(dataset_metadata)++ pipeline_ids = {dataset.pipeline_id for dataset in datasets if dataset.pipeline_id}+ pipeline_published: dict[str, bool] = {}+ if pipeline_ids:+ pipeline_owners = [(dataset.tenant_id, dataset.pipeline_id) for dataset in datasets if dataset.pipeline_id]+ pipeline_published = dict(+ session.execute(+ select(Pipeline.id, Pipeline.is_published).where(+ tuple_(Pipeline.tenant_id, Pipeline.id).in_(pipeline_owners)+ )+ )+ .tuples()+ .all()+ )++ return DatasetDetailPrefetch(+ app_counts=app_counts,+ document_counts=document_counts,+ word_counts=word_counts,+ available_document_counts=available_document_counts,+ author_names=author_names,+ tags=tags,+ doc_forms=doc_forms,+ external_knowledge_infos=_load_external_knowledge_infos(datasets, session=session),+ doc_metadata=doc_metadata,+ pipeline_published=pipeline_published,+ )+++def _load_external_knowledge_infos(+ datasets: Sequence[Dataset], *, session: Session+) -> dict[tuple[str, str], dict[str, Any]]:+ external_datasets = [dataset for dataset in datasets if dataset.provider == "external"]+ if not external_datasets:+ return {}++ external_owners = [(dataset.tenant_id, dataset.id) for dataset in external_datasets]+ external_tenant_ids = {dataset.tenant_id for dataset in external_datasets}+ bindings = session.scalars(+ select(ExternalKnowledgeBindings).where(+ tuple_(ExternalKnowledgeBindings.tenant_id, ExternalKnowledgeBindings.dataset_id).in_(external_owners),+ )+ ).all()+ if not bindings:+ return {}++ apis = {+ (api.id, api.tenant_id): api+ for api in session.scalars(+ select(ExternalKnowledgeApis).where(+ ExternalKnowledgeApis.id.in_({binding.external_knowledge_api_id for binding in bindings}),+ ExternalKnowledgeApis.tenant_id.in_(external_tenant_ids),+ )+ ).all()+ }++ infos: dict[tuple[str, str], dict[str, Any]] = {}+ for binding in bindings:+ info = Dataset.build_external_knowledge_info(+ binding, apis.get((binding.external_knowledge_api_id, binding.tenant_id))+ )+ if info is not None:+ infos[(binding.dataset_id, binding.tenant_id)] = info+ return infosdiff --git a/api/repositories/knowledge/dataset_repository.py b/api/repositories/knowledge/dataset_repository.pynew file mode 100644index 00000000000..447b4d8a4e8--- /dev/null+++ b/api/repositories/knowledge/dataset_repository.py@@ -0,0 +1,153 @@+"""SQLAlchemy repository for tenant-owned dataset state."""++from sqlalchemy import and_, exists, literal, select+from sqlalchemy.orm import Session, sessionmaker++from models.dataset import Dataset, DatasetPermission, Document+from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessSnapshot+from services.knowledge.indexing.estimate import DatasetEstimateRecord+from services.knowledge.resource_scope import DatasetRef+from services.knowledge.segments.application import SegmentDatasetRecord, SegmentDocumentRecord, SegmentScope+++def _dataset_access_record(dataset: Dataset) -> DatasetAccessRecord:+ return DatasetAccessRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ maintainer_id=dataset.maintainer,+ permission=str(dataset.permission),+ )+++def _dataset_estimate_record(dataset: Dataset) -> DatasetEstimateRecord:+ return DatasetEstimateRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ indexing_technique=str(dataset.indexing_technique) if dataset.indexing_technique is not None else None,+ )+++def _get_dataset(session: Session, dataset_ref: DatasetRef) -> Dataset | None:+ """Shared SQL for implementation adapters that already own a transaction."""+ return session.scalar(+ select(Dataset)+ .where(+ Dataset.id == dataset_ref.dataset_id,+ Dataset.tenant_id == dataset_ref.tenant_id,+ )+ .limit(1)+ )+++class SQLAlchemyDatasetRepository:+ """Own SQL access to tenant-scoped dataset state."""++ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:+ with self._session_factory() as session:+ dataset = _get_dataset(session, dataset_ref)+ return _dataset_estimate_record(dataset) if dataset is not None else None++ def is_notion_dataset(self, dataset_ref: DatasetRef) -> bool:+ """Return whether the owned dataset is configured for Notion imports."""+ with self._session_factory() as session:+ return bool(+ session.scalar(+ select(+ exists().where(+ Dataset.id == dataset_ref.dataset_id,+ Dataset.tenant_id == dataset_ref.tenant_id,+ Dataset.data_source_type == "notion_import",+ )+ )+ )+ )++ def get_access_snapshot(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ actor_id: str,+ ) -> DatasetAccessSnapshot | None:+ with self._session_factory() as session:+ dataset = _get_dataset(session, DatasetRef(workspace_id, dataset_id))+ if dataset is None:+ return None++ actor_has_partial_access = bool(+ session.scalar(+ select(+ exists().where(+ DatasetPermission.tenant_id == workspace_id,+ DatasetPermission.dataset_id == dataset_id,+ DatasetPermission.account_id == actor_id,+ DatasetPermission.has_permission.is_(True),+ )+ )+ )+ )+ return DatasetAccessSnapshot(+ dataset=_dataset_access_record(dataset),+ actor_has_partial_access=actor_has_partial_access,+ )++ def get_segment_scope(+ self,+ *,+ dataset_ref: DatasetRef,+ document_id: str,+ actor_id: str | None,+ ) -> SegmentScope:+ """Read ownership, model configuration and the actor grant in one snapshot."""+ grant = (+ exists().where(+ DatasetPermission.tenant_id == dataset_ref.tenant_id,+ DatasetPermission.dataset_id == dataset_ref.dataset_id,+ DatasetPermission.account_id == actor_id,+ DatasetPermission.has_permission.is_(True),+ )+ if actor_id is not None+ else literal(False)+ )+ with self._session_factory() as session:+ row = session.execute(+ select(Dataset, Document, grant)+ .outerjoin(+ Document,+ and_(+ Document.id == document_id,+ Document.dataset_id == Dataset.id,+ Document.tenant_id == Dataset.tenant_id,+ ),+ )+ .where(Dataset.id == dataset_ref.dataset_id, Dataset.tenant_id == dataset_ref.tenant_id)+ ).one_or_none()+ if row is None:+ return SegmentScope(None, None)+ dataset, document, actor_has_access = row+ return SegmentScope(+ dataset=SegmentDatasetRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ indexing_technique=str(dataset.indexing_technique),+ embedding_model_provider=dataset.embedding_model_provider,+ embedding_model=dataset.embedding_model,+ ),+ document=SegmentDocumentRecord(+ id=document.id,+ dataset_id=document.dataset_id,+ workspace_id=document.tenant_id,+ doc_form=str(document.doc_form),+ )+ if document is not None+ else None,+ access_snapshot=DatasetAccessSnapshot(+ dataset=_dataset_access_record(dataset),+ actor_has_partial_access=bool(actor_has_access),+ )+ if actor_id is not None+ else None,+ )diff --git a/api/repositories/knowledge/document_repository.py b/api/repositories/knowledge/document_repository.pynew file mode 100644index 00000000000..55ba407c8f3--- /dev/null+++ b/api/repositories/knowledge/document_repository.py@@ -0,0 +1,241 @@+"""SQLAlchemy repository for tenant-owned document state."""++import json+from collections.abc import Mapping++from pydantic import TypeAdapter, ValidationError+from sqlalchemy import exists, select, update+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from models import Account+from models.dataset import Dataset, DatasetProcessRule, Document, Pipeline+from models.enums import IndexingStatus+from repositories.knowledge.dataset_read_repository import get_pipeline_dataset+from services.knowledge.document_sync import SyncDocumentRecord+from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError+from services.knowledge.indexing.estimate import EstimateDocumentRecord, StoredSource+from services.knowledge.indexing.execution import IndexingDocument+from services.knowledge.resource_scope import DatasetRef, DocumentRef++_MAPPING_ADAPTER = TypeAdapter(dict[str, object])+++def _mapping(value: object) -> Mapping[str, object] | None:+ if value is None:+ return None+ try:+ if isinstance(value, str):+ return _MAPPING_ADAPTER.validate_python(json.loads(value))+ return _MAPPING_ADAPTER.validate_python(value)+ except (TypeError, ValueError, ValidationError):+ return {}+++def _estimate_document_record(document: Document) -> EstimateDocumentRecord:+ return EstimateDocumentRecord(+ id=document.id,+ workspace_id=document.tenant_id,+ dataset_id=document.dataset_id,+ data_source_type=str(document.data_source_type),+ data_source_info=_mapping(document.data_source_info),+ indexing_status=str(document.indexing_status),+ doc_form=str(document.doc_form),+ doc_language=document.doc_language,+ dataset_process_rule_id=document.dataset_process_rule_id,+ )+++def _get_document(session: Session, document_ref: DocumentRef) -> Document | None:+ """Shared SQL for implementation adapters that already own a transaction."""+ return session.scalar(+ select(Document)+ .where(+ Document.id == document_ref.document_id,+ Document.dataset_id == document_ref.dataset.dataset_id,+ Document.tenant_id == document_ref.dataset.tenant_id,+ )+ .limit(1)+ )+++def require_indexing_document(session: Session, ref: DocumentRef, *, lock: bool = False) -> Document:+ """Validate the persisted owner chain before each indexing phase."""+ statement = (+ select(Document)+ .join(Dataset, Dataset.id == Document.dataset_id)+ .where(+ Document.id == ref.document_id,+ Document.dataset_id == ref.dataset.dataset_id,+ Document.tenant_id == ref.dataset.tenant_id,+ Dataset.tenant_id == ref.dataset.tenant_id,+ )+ )+ if lock:+ statement = statement.with_for_update(of=Document)+ document = session.scalar(statement)+ if document is None:+ raise DocumentIsDeletedPausedError()+ if document.is_paused:+ raise DocumentIsPausedError()+ return document+++class SQLAlchemyDocumentRepository:+ """Own SQL access to documents through their complete owner chain."""++ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def get_pipeline_dataset(self, pipeline: Pipeline, *, session: Session) -> Dataset | None:+ """Read within the pipeline caller's transaction, including uncommitted changes."""+ return get_pipeline_dataset(pipeline, session=session)++ def get_indexing_document(self, ref: DocumentRef) -> IndexingDocument:+ with self._session_factory() as session:+ document = require_indexing_document(session, ref)+ rule = session.scalar(+ select(DatasetProcessRule).where(+ DatasetProcessRule.id == document.dataset_process_rule_id,+ DatasetProcessRule.dataset_id == ref.dataset.dataset_id,+ )+ )+ return IndexingDocument(+ source=StoredSource.from_document(_estimate_document_record(document)),+ processing_rule=rule.to_dict() if rule else {},+ doc_language=document.doc_language or "English",+ need_summary=bool(document.need_summary),+ )++ def get_indexing_models(self, ref: DocumentRef) -> tuple[Dataset, Document]:+ """Detached legacy processor inputs; never pass them to another thread."""+ with self._session_factory() as session:+ document = require_indexing_document(session, ref)+ dataset = session.scalar(+ select(Dataset).where(+ Dataset.id == ref.dataset.dataset_id,+ Dataset.tenant_id == ref.dataset.tenant_id,+ )+ )+ assert dataset is not None+ return dataset, document++ def get_indexing_user(self, ref: DocumentRef) -> Account:+ with self._session_factory() as session:+ document = require_indexing_document(session, ref)+ account = session.get(Account, document.created_by)+ if account is None:+ raise ValueError("no current user found")+ account.set_tenant_id_with_session(ref.dataset.tenant_id, session=session)+ return account++ def mark_splitting(self, ref: DocumentRef) -> None:+ with self._session_factory.begin() as session:+ document = require_indexing_document(session, ref, lock=True)+ document.indexing_status = IndexingStatus.SPLITTING+ document.parsing_completed_at = naive_utc_now()++ def complete_indexing(self, ref: DocumentRef, *, tokens: int, latency: float) -> None:+ with self._session_factory.begin() as session:+ document = require_indexing_document(session, ref, lock=True)+ document.indexing_status = IndexingStatus.COMPLETED+ document.tokens = tokens+ document.indexing_latency = latency+ document.completed_at = naive_utc_now()+ document.error = None++ def fail_indexing(self, ref: DocumentRef, error: str) -> None:+ try:+ with self._session_factory.begin() as session:+ document = require_indexing_document(session, ref, lock=True)+ document.indexing_status = IndexingStatus.ERROR+ document.error = error+ document.stopped_at = naive_utc_now()+ except DocumentIsDeletedPausedError:+ return++ def exists(self, *, workspace_id: str, dataset_id: str, document_id: str) -> bool:+ with self._session_factory() as session:+ return bool(+ session.scalar(+ select(+ exists().where(+ Document.tenant_id == workspace_id,+ Document.dataset_id == dataset_id,+ Document.id == document_id,+ )+ )+ )+ )++ def mark_failed(self, *, workspace_id: str, dataset_id: str, document_id: str, error: str) -> None:+ with self._session_factory.begin() as session:+ session.execute(+ update(Document)+ .where(+ Document.tenant_id == workspace_id,+ Document.dataset_id == dataset_id,+ Document.id == document_id,+ )+ .values(indexing_status=IndexingStatus.ERROR, error=error)+ )++ def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None:+ with self._session_factory() as session:+ document = _get_document(session, document_ref)+ return _estimate_document_record(document) if document is not None else None++ def list_estimate_documents_by_batch(+ self, dataset_ref: DatasetRef, batch: str+ ) -> tuple[EstimateDocumentRecord, ...]:+ with self._session_factory() as session:+ documents = session.scalars(+ select(Document).where(+ Document.tenant_id == dataset_ref.tenant_id,+ Document.dataset_id == dataset_ref.dataset_id,+ Document.batch == batch,+ )+ ).all()+ return tuple(_estimate_document_record(document) for document in documents)++ def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None:+ with self._session_factory() as session:+ document = _get_document(session, document_ref)+ if document is None:+ return None+ return SyncDocumentRecord(id=document.id, data_source_type=str(document.data_source_type))++ def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]:+ with self._session_factory() as session:+ return tuple(+ dataset_ref.document(document_id)+ for document_id in session.scalars(+ select(Document.id).where(+ Document.tenant_id == dataset_ref.tenant_id,+ Document.dataset_id == dataset_ref.dataset_id,+ Document.data_source_type == "notion_import",+ Document.enabled.is_(True),+ Document.archived.is_(False),+ )+ ).all()+ )++ def list_bound_notion_page_ids(self, dataset_ref: DatasetRef) -> frozenset[str]:+ page_ids: set[str] = set()+ with self._session_factory() as session:+ documents = session.scalars(+ select(Document).where(+ Document.tenant_id == dataset_ref.tenant_id,+ Document.dataset_id == dataset_ref.dataset_id,+ Document.data_source_type == "notion_import",+ Document.enabled.is_(True),+ )+ ).all()+ for document in documents:+ source_info = _mapping(document.data_source_info)+ if source_info is None:+ continue+ page_id = source_info.get("notion_page_id")+ if isinstance(page_id, str) and page_id:+ page_ids.add(page_id)+ return frozenset(page_ids)diff --git a/api/repositories/knowledge/keyword_table_repository.py b/api/repositories/knowledge/keyword_table_repository.pynew file mode 100644index 00000000000..e0af43c9be6--- /dev/null+++ b/api/repositories/knowledge/keyword_table_repository.py@@ -0,0 +1,121 @@+"""Keyword table serialization, object storage, and tenant-scoped SQL persistence."""++import json+from collections.abc import Mapping, Sequence+from typing import Any, Protocol++import orjson+from sqlalchemy import select+from sqlalchemy.orm import Session++from models.dataset import Dataset, DatasetKeywordTable, DocumentSegment+from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table+++class _KeywordStorage(Protocol):+ def load_once(self, key: str, /) -> bytes: ...++ def exists(self, key: str, /) -> bool: ...++ def save(self, key: str, data: bytes, /) -> None: ...++ def delete(self, key: str, /) -> None: ...+++def keyword_file_key(tenant_id: str, dataset_id: str) -> str:+ return f"keyword_files/{tenant_id}/{dataset_id}.txt"+++def _decode_sets(values: dict[str, Any]) -> dict[str, Any]:+ return {key: set(value) if isinstance(value, list) else value for key, value in values.items()}+++def _encode_sets(value: object) -> list[object]:+ if isinstance(value, set):+ return list(value)+ raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")+++def load_keyword_table(+ *, storage: _KeywordStorage, tenant_id: str, dataset_id: str, storage_type: str, data: str | None+) -> dict[str, Any] | None:+ if storage_type == "database":+ return json.loads(data, object_hook=_decode_sets) if data else None+ file_key = keyword_file_key(tenant_id, dataset_id)+ try:+ payload = storage.load_once(file_key)+ return json.loads(payload.decode("utf-8"), object_hook=_decode_sets) if payload else None+ except FileNotFoundError:+ return None+++def load_dataset_keyword_table(+ dataset: Dataset, *, storage: _KeywordStorage, session: Session+) -> dict[str, Any] | None:+ """Resolve a dataset's stored keyword payload using the caller's session."""+ row = get_dataset_keyword_table(dataset, session=session)+ if row is None:+ return None+ return load_keyword_table(+ storage=storage,+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ storage_type=row.data_source_type,+ data=row.keyword_table,+ )+++def save_keyword_table(+ *, storage: _KeywordStorage, tenant_id: str, dataset_id: str, storage_type: str, table: dict[str, set[str]]+) -> str:+ data = orjson.dumps(+ {"__type__": "keyword_table", "__data__": {"index_id": dataset_id, "summary": None, "table": table}},+ default=_encode_sets,+ )+ if storage_type != "database":+ file_key = keyword_file_key(tenant_id, dataset_id)+ if storage.exists(file_key):+ storage.delete(file_key)+ storage.save(file_key, data)+ return data.decode("utf-8")+++def persist_keyword_table(+ session: Session,+ *,+ tenant_id: str,+ dataset_id: str,+ storage_type: str,+ data: str,+ keywords: Mapping[str, Sequence[str]],+) -> None:+ """Write a tenant-owned table and segment keywords in the caller's transaction."""+ result = session.execute(+ select(Dataset.id, DatasetKeywordTable)+ .outerjoin(DatasetKeywordTable, DatasetKeywordTable.dataset_id == Dataset.id)+ .where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id)+ ).one_or_none()+ if result is None:+ raise LookupError("Dataset no longer exists")+ _, row = result+ if row is None:+ row = DatasetKeywordTable(+ dataset_id=dataset_id,+ data_source_type=storage_type,+ keyword_table=data if storage_type == "database" else "",+ )+ session.add(row)+ elif storage_type == "database":+ row.keyword_table = data+ if keywords:+ segments = session.scalars(+ select(DocumentSegment).where(+ DocumentSegment.tenant_id == tenant_id,+ DocumentSegment.dataset_id == dataset_id,+ DocumentSegment.index_node_id.in_(keywords),+ )+ )+ for segment in segments:+ assert segment.index_node_id is not None+ segment.keywords = list(keywords[segment.index_node_id])+ session.flush()diff --git a/api/repositories/knowledge/metadata_repository.py b/api/repositories/knowledge/metadata_repository.pynew file mode 100644index 00000000000..1610fd3ab0d--- /dev/null+++ b/api/repositories/knowledge/metadata_repository.py@@ -0,0 +1,212 @@+"""Tenant-scoped metadata operations; each mutation owns one atomic transaction."""++from collections.abc import Sequence++from sqlalchemy import func, select+from sqlalchemy.orm import Session, sessionmaker++from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource+from libs.datetime_utils import naive_utc_now+from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Document+from repositories.knowledge.dataset_read_repository import get_document_uploader+from services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.metadata.application import MetadataList, MetadataOperation, MetadataRecord+from services.knowledge.resource_scope import DatasetRef+++class SQLAlchemyMetadataRepository:+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._sessions = session_factory++ @staticmethod+ def _dataset(session: Session, ref: DatasetRef, *, write: bool = False) -> Dataset:+ query = select(Dataset).where(Dataset.id == ref.dataset_id, Dataset.tenant_id == ref.tenant_id)+ if write:+ query = query.with_for_update()+ dataset = session.scalar(query)+ if dataset is None:+ raise MetadataResourceNotFoundError("Dataset not found.")+ return dataset++ @staticmethod+ def _fields(ref: DatasetRef):+ return select(DatasetMetadata).where(+ DatasetMetadata.tenant_id == ref.tenant_id, DatasetMetadata.dataset_id == ref.dataset_id+ )++ @staticmethod+ def _documents(ref: DatasetRef):+ return select(Document).where(Document.tenant_id == ref.tenant_id, Document.dataset_id == ref.dataset_id)++ @staticmethod+ def _bindings(ref: DatasetRef):+ return select(DatasetMetadataBinding).where(+ DatasetMetadataBinding.tenant_id == ref.tenant_id, DatasetMetadataBinding.dataset_id == ref.dataset_id+ )++ @staticmethod+ def _record(row: DatasetMetadata) -> MetadataRecord:+ return MetadataRecord(+ id=row.id,+ name=row.name,+ type=row.type,+ tenant_id=row.tenant_id,+ dataset_id=row.dataset_id,+ created_by=row.created_by,+ created_at=row.created_at,+ updated_by=row.updated_by,+ updated_at=row.updated_at,+ )++ @classmethod+ def _check_name(cls, session: Session, ref: DatasetRef, name: str) -> None:+ if session.scalar(cls._fields(ref).where(DatasetMetadata.name == name).limit(1)) is not None:+ raise ValueError("Metadata name already exists.")++ def create(self, ref: DatasetRef, *, name: str, field_type: str, actor_id: str) -> MetadataRecord:+ with self._sessions.begin() as session:+ self._dataset(session, ref, write=True)+ self._check_name(session, ref, name)+ row = DatasetMetadata(+ tenant_id=ref.tenant_id, dataset_id=ref.dataset_id, name=name, type=field_type, created_by=actor_id+ )+ session.add(row)+ session.flush()+ return self._record(row)++ def rename(self, ref: DatasetRef, metadata_id: str, *, name: str, actor_id: str) -> MetadataRecord:+ with self._sessions.begin() as session:+ self._dataset(session, ref, write=True)+ self._check_name(session, ref, name)+ row = session.scalar(self._fields(ref).where(DatasetMetadata.id == metadata_id))+ if row is None:+ raise MetadataResourceNotFoundError("Metadata not found.")+ old_name = row.name+ row.name, row.updated_by, row.updated_at = name, actor_id, naive_utc_now()+ bindings = session.scalars(+ self._bindings(ref).where(DatasetMetadataBinding.metadata_id == metadata_id)+ ).all()+ for document in session.scalars(+ self._documents(ref).where(Document.id.in_([b.document_id for b in bindings]))+ ):+ values = dict(document.doc_metadata or {})+ values[name] = values.pop(old_name, None)+ document.doc_metadata = values+ return self._record(row)++ def delete(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord:+ with self._sessions.begin() as session:+ self._dataset(session, ref, write=True)+ row = session.scalar(self._fields(ref).where(DatasetMetadata.id == metadata_id))+ if row is None:+ raise MetadataResourceNotFoundError("Metadata not found.")+ bindings = session.scalars(+ self._bindings(ref).where(DatasetMetadataBinding.metadata_id == metadata_id)+ ).all()+ for document in session.scalars(+ self._documents(ref).where(Document.id.in_([b.document_id for b in bindings]))+ ):+ values = dict(document.doc_metadata or {})+ values.pop(row.name, None)+ document.doc_metadata = values+ for binding in bindings:+ session.delete(binding)+ session.delete(row)+ return self._record(row)++ @staticmethod+ def _built_in(session: Session, document: Document) -> dict[str, str | int | float | None]:+ return {+ BuiltInField.document_name: document.name,+ BuiltInField.uploader: get_document_uploader(document, session=session),+ BuiltInField.upload_date: document.upload_date.timestamp(),+ BuiltInField.last_update_date: document.last_update_date.timestamp(),+ BuiltInField.source: MetadataDataSource[document.data_source_type],+ }++ def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None:+ with self._sessions.begin() as session:+ dataset = self._dataset(session, ref, write=True)+ if dataset.built_in_field_enabled == enabled:+ return+ for document in session.scalars(+ self._documents(ref).where(+ Document.enabled.is_(True), Document.archived.is_(False), Document.indexing_status == "completed"+ )+ ):+ values = dict(document.doc_metadata or {})+ if enabled:+ values.update(self._built_in(session, document))+ else:+ for key in BuiltInField:+ values.pop(key, None)+ document.doc_metadata = values+ dataset.built_in_field_enabled = enabled++ def update_documents(self, ref: DatasetRef, operations: Sequence[MetadataOperation], *, actor_id: str) -> None:+ with self._sessions.begin() as session:+ dataset = self._dataset(session, ref, write=True)+ metadata_ids = {value.id for operation in operations for value in operation.metadata_list}+ fields = {+ row.id: row for row in session.scalars(self._fields(ref).where(DatasetMetadata.id.in_(metadata_ids)))+ }+ if metadata_ids != set(fields):+ raise MetadataResourceNotFoundError("Metadata not found.")+ document_ids = {operation.document_id for operation in operations}+ documents = {+ row.id: row+ for row in session.scalars(self._documents(ref).where(Document.id.in_(document_ids)).with_for_update())+ }+ if document_ids != set(documents):+ raise MetadataResourceNotFoundError("Document not found.")+ for operation in operations:+ document = documents[operation.document_id]+ values = dict(document.doc_metadata or {}) if operation.partial_update else {}+ values.update({fields[value.id].name: value.value for value in operation.metadata_list})+ if dataset.built_in_field_enabled:+ values.update(self._built_in(session, document))+ document.doc_metadata = values+ bindings = session.scalars(+ self._bindings(ref).where(DatasetMetadataBinding.document_id == document.id)+ ).all()+ previous = {binding.metadata_id for binding in bindings}+ if not operation.partial_update:+ for binding in bindings:+ session.delete(binding)+ previous.clear()+ for value in operation.metadata_list:+ if value.id in previous:+ continue+ session.add(+ DatasetMetadataBinding(+ tenant_id=ref.tenant_id,+ dataset_id=ref.dataset_id,+ document_id=document.id,+ metadata_id=value.id,+ created_by=actor_id,+ )+ )+ previous.add(value.id)++ def list_fields(self, ref: DatasetRef) -> MetadataList:+ with self._sessions() as session:+ dataset = self._dataset(session, ref)+ counts = dict(+ session.execute(+ select(DatasetMetadataBinding.metadata_id, func.count())+ .where(+ DatasetMetadataBinding.tenant_id == ref.tenant_id,+ DatasetMetadataBinding.dataset_id == ref.dataset_id,+ )+ .group_by(DatasetMetadataBinding.metadata_id)+ )+ .tuples()+ .all()+ )+ return {+ "doc_metadata": [+ {"id": row.id, "name": row.name, "type": row.type, "count": counts.get(row.id, 0)}+ for row in session.scalars(self._fields(ref))+ ],+ "built_in_field_enabled": dataset.built_in_field_enabled,+ }diff --git a/api/repositories/knowledge/segment_read_adapter.py b/api/repositories/knowledge/segment_read_adapter.pynew file mode 100644index 00000000000..23ba6042b98--- /dev/null+++ b/api/repositories/knowledge/segment_read_adapter.py@@ -0,0 +1,87 @@+"""Read adapter for authorized segment content and attachment preview values."""++import base64+import hashlib+import hmac+import os+import re+import time+from collections.abc import Sequence+from typing import Literal++from sqlalchemy.orm import Session++from configs import dify_config+from core.tools.signature import bind_file_uri, sign_tool_file_uri+from models.dataset import AttachmentItem, DocumentSegment+from models.model import UploadFile+from repositories.knowledge.dataset_read_repository import get_segment_attachment_files, get_segment_content_file_ids++# Consume the entire old query string and replace matches in source order.+_FILE_ID = r"[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}"+_FILE_LINK = re.compile(+ rf"/files/(?:(?P<upload>{_FILE_ID})/(?P<preview>image-preview|file-preview)"+ rf"|tools/(?P<tool>{_FILE_ID})\.(?P<extension>[a-zA-Z0-9]+))"+ r"(?:\?[^\s)\"'<>]*)?"+)+++def _sign_upload_file_uri(upload_file_id: str, *, preview_kind: Literal["image", "file"] = "file") -> str:+ """Sign an upload ID whose ownership was checked by this read adapter."""+ timestamp = str(int(time.time()))+ nonce = os.urandom(16).hex()+ payload = f"{preview_kind}-preview|{upload_file_id}|{timestamp}|{nonce}"+ signature = hmac.new(dify_config.SECRET_KEY.encode(), payload.encode(), hashlib.sha256).digest()+ encoded_sign = base64.urlsafe_b64encode(signature).decode()+ return f"/files/{upload_file_id}/{preview_kind}-preview?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"+++def sign_segment_content(segment: DocumentSegment, *, session: Session) -> str:+ """Refresh links only after checking persisted file ownership.++ The segment must come from an authorized knowledge read. File queries also+ enforce its tenant and the current execution's file-access scope. Inaccessible+ references lose their old signature and never receive a new one.+ """+ matches = list(_FILE_LINK.finditer(segment.content))+ if not matches:+ return segment.content+ upload_ids = {match["upload"].lower() for match in matches if match["upload"]}+ tool_ids = {match["tool"].lower() for match in matches if match["tool"]}+ allowed_uploads, allowed_tools = get_segment_content_file_ids(+ segment, upload_ids=upload_ids, tool_ids=tool_ids, session=session+ )++ def replace(match: re.Match[str]) -> str:+ if upload_id := match["upload"]:+ upload_id = upload_id.lower()+ if upload_id not in allowed_uploads:+ return match.group().split("?", 1)[0]+ return _sign_upload_file_uri(+ upload_id, preview_kind="image" if match["preview"] == "image-preview" else "file"+ )+ tool_id = match["tool"].lower()+ if tool_id not in allowed_tools:+ return match.group().split("?", 1)[0]+ return sign_tool_file_uri(tool_id, "." + match["extension"])++ return _FILE_LINK.sub(replace, segment.content)+++def format_segment_attachments(files: Sequence[UploadFile]) -> list[AttachmentItem]:+ base_url = dify_config.FILES_URL or dify_config.CONSOLE_API_URL or ""+ return [+ {+ "id": file.id,+ "name": file.name,+ "size": file.size,+ "extension": file.extension,+ "mime_type": file.mime_type,+ "source_url": bind_file_uri(_sign_upload_file_uri(file.id, preview_kind="image"), base_url),+ }+ for file in files+ ]+++def get_segment_attachments(segment: DocumentSegment, *, session: Session) -> list[AttachmentItem]:+ return format_segment_attachments(get_segment_attachment_files(segment, session=session))diff --git a/api/repositories/knowledge/segment_repository.py b/api/repositories/knowledge/segment_repository.pynew file mode 100644index 00000000000..47766f46644--- /dev/null+++ b/api/repositories/knowledge/segment_repository.py@@ -0,0 +1,716 @@+"""Infrastructure adapters for dataset segment use cases."""++from __future__ import annotations++from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from typing import cast++from sqlalchemy import String, case, delete, func, literal, or_, select, update+from sqlalchemy import cast as sql_cast+from sqlalchemy.dialects.postgresql import JSONB+from sqlalchemy.orm import Session, sessionmaker++from configs import dify_config+from core.rag.datasource.vdb.vector_factory import Vector+from core.rag.docstore.dataset_docstore import DatasetDocumentStore+from core.rag.index_processor.constant.index_type import IndexStructureType+from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor+from core.rag.models.document import ChildDocument+from core.rag.models.document import Document as IndexDocument+from graphon.file import File+from libs.datetime_utils import naive_utc_now+from libs.helper import escape_like_pattern+from libs.pagination import PaginatedResult, paginate_query+from models.dataset import (+ ChildChunk,+ Dataset,+ DatasetProcessRule,+ Document,+ DocumentSegment,+ DocumentSegmentSummary,+ SegmentAttachmentBinding,+)+from models.enums import IndexingStatus, SegmentStatus, SegmentType, SummaryStatus+from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table, get_segment_child_chunks+from repositories.knowledge.dataset_repository import _get_dataset+from repositories.knowledge.document_repository import _get_document, require_indexing_document+from repositories.knowledge.keyword_table_repository import persist_keyword_table+from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_content+from services.knowledge.entities.segments import ChildChunkRecord, SegmentRecord+from services.knowledge.indexing.execution import IndexingDocument+from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef+from services.knowledge.segments.application import (+ ChildChunkListFilter,+ ChildChunkPage,+ ChildChunkState,+ SegmentDetail,+ SegmentIndexTarget,+ SegmentListFilter,+ SegmentPage,+ SegmentUpdateState,+)+++class SQLAlchemySegmentRepository:+ """Read and write scoped segment rows in bounded database transactions."""++ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def clear_for_indexing(self, document: IndexingDocument) -> None:+ ref = document.ref+ with self._session_factory.begin() as session:+ require_indexing_document(session, ref, lock=True)+ if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:+ session.execute(+ delete(ChildChunk).where(+ ChildChunk.tenant_id == ref.dataset.tenant_id,+ ChildChunk.dataset_id == ref.dataset.dataset_id,+ ChildChunk.document_id == ref.document_id,+ )+ )+ session.execute(delete(DocumentSegment).where(*self._indexing_scope(ref)))++ def save_for_indexing(+ self, document: IndexingDocument, chunks: list[IndexDocument], token_counts: list[int]+ ) -> None:+ ref = document.ref+ with self._session_factory.begin() as session:+ row = require_indexing_document(session, ref, lock=True)+ dataset = _get_dataset(session, ref.dataset)+ assert dataset is not None+ DatasetDocumentStore(dataset=dataset, user_id=row.created_by, document_id=row.id).add_documents(+ session=session,+ docs=chunks,+ token_counts=token_counts,+ save_child=document.doc_form == IndexStructureType.PARENT_CHILD_INDEX,+ )+ now = naive_utc_now()+ row.indexing_status = IndexingStatus.INDEXING+ row.cleaning_completed_at = now+ row.splitting_completed_at = now+ row.word_count = sum(len(chunk.page_content) for chunk in chunks)+ session.execute(+ update(DocumentSegment)+ .where(*self._indexing_scope(ref))+ .values(+ status=SegmentStatus.INDEXING,+ indexing_at=now,+ )+ )++ def resume_indexing(self, document: IndexingDocument) -> tuple[list[IndexDocument], int]:+ ref = document.ref+ with self._session_factory() as session:+ require_indexing_document(session, ref)+ rows = session.scalars(+ select(DocumentSegment).where(*self._indexing_scope(ref)).order_by(DocumentSegment.position)+ ).all()+ chunks = []+ for row in rows:+ if row.status == SegmentStatus.COMPLETED:+ continue+ chunk = IndexDocument(+ page_content=row.content,+ metadata={+ "doc_id": row.index_node_id,+ "doc_hash": row.index_node_hash,+ "document_id": ref.document_id,+ "dataset_id": ref.dataset.dataset_id,+ },+ )+ if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:+ chunk.children = [+ ChildDocument(+ page_content=child.content,+ metadata={+ "doc_id": child.index_node_id,+ "doc_hash": child.index_node_hash,+ "document_id": ref.document_id,+ "dataset_id": ref.dataset.dataset_id,+ },+ )+ for child in get_segment_child_chunks(row, session=session)+ ]+ chunks.append(chunk)+ return chunks, sum(row.tokens for row in rows)++ def complete_indexing_segments(self, ref: DocumentRef, node_ids: Sequence[str]) -> None:+ with self._session_factory.begin() as session:+ require_indexing_document(session, ref, lock=True)+ session.execute(+ update(DocumentSegment)+ .where(+ *self._indexing_scope(ref),+ DocumentSegment.index_node_id.in_(node_ids),+ DocumentSegment.status == SegmentStatus.INDEXING,+ )+ .values(status=SegmentStatus.COMPLETED, enabled=True, completed_at=naive_utc_now())+ )++ @staticmethod+ def _indexing_scope(ref: DocumentRef):+ return (+ DocumentSegment.tenant_id == ref.dataset.tenant_id,+ DocumentSegment.dataset_id == ref.dataset.dataset_id,+ DocumentSegment.document_id == ref.document_id,+ )++ def list_segments(self, document_ref: DocumentRef, query: SegmentListFilter) -> SegmentPage:+ with self._session_factory() as session:+ statement = (+ select(DocumentSegment)+ .where(+ DocumentSegment.tenant_id == document_ref.dataset.tenant_id,+ DocumentSegment.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegment.document_id == document_ref.document_id,+ )+ .order_by(DocumentSegment.position.asc())+ )+ if query.statuses:+ statement = statement.where(DocumentSegment.status.in_(query.statuses))+ if query.hit_count_gte is not None:+ statement = statement.where(DocumentSegment.hit_count >= query.hit_count_gte)+ if query.keyword:+ escaped_keyword = escape_like_pattern(query.keyword)+ if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql":+ keywords_jsonb = sql_cast(DocumentSegment.keywords, JSONB)+ keywords_array = case(+ (func.jsonb_typeof(keywords_jsonb) == "array", keywords_jsonb),+ else_=sql_cast(literal("[]"), JSONB),+ )+ keywords_condition = func.array_to_string(+ func.array(+ select(func.jsonb_array_elements_text(keywords_array))+ .correlate(DocumentSegment)+ .scalar_subquery()+ ),+ ",",+ ).ilike(f"%{escaped_keyword}%", escape="\\")+ else:+ keywords_condition = sql_cast(DocumentSegment.keywords, String).ilike(+ f"%{escaped_keyword}%", escape="\\"+ )+ statement = statement.where(+ or_(+ DocumentSegment.content.ilike(f"%{escaped_keyword}%", escape="\\"),+ keywords_condition,+ )+ )+ if query.enabled.lower() == "true":+ statement = statement.where(DocumentSegment.enabled.is_(True))+ elif query.enabled.lower() == "false":+ statement = statement.where(DocumentSegment.enabled.is_(False))++ limit = min(query.limit, 100)+ segments = paginate_query(statement, session=session, page=query.page, per_page=limit, max_per_page=100)+ items = list(segments.items)+ summaries = (+ {+ summary.chunk_id: summary+ for summary in session.scalars(+ select(DocumentSegmentSummary).where(+ DocumentSegmentSummary.chunk_id.in_([segment.id for segment in items]),+ DocumentSegmentSummary.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegmentSummary.document_id == document_ref.document_id,+ )+ )+ }+ if items+ else {}+ )+ return SegmentPage(+ items=tuple(+ _segment_data(+ segment,+ summaries[segment.id].summary_content if segment.id in summaries else None,+ session,+ )+ for segment in items+ ),+ total=segments.total,+ total_pages=segments.pages,+ page=query.page,+ limit=limit,+ )++ def list_child_chunks(+ self,+ segment_ref: SegmentRef,+ query: ChildChunkListFilter,+ ) -> ChildChunkPage | None:+ with self._session_factory() as session:+ if _get_segment(session, segment_ref) is None:+ return None+ limit = min(query.limit, 100)+ child_chunks = query_child_chunks(+ session,+ segment_ref,+ page=query.page,+ limit=limit,+ keyword=query.keyword,+ )+ return ChildChunkPage(+ items=tuple(_child_chunk_data(chunk) for chunk in child_chunks.items),+ total=child_chunks.total,+ total_pages=child_chunks.pages,+ page=query.page,+ limit=limit,+ )++ def get_segment(self, segment_ref: SegmentRef) -> SegmentDetail | None:+ with self._session_factory() as session:+ segment = _get_segment(session, segment_ref)+ if segment is None:+ return None+ _, document = _require_scope_models(session, segment_ref.document)+ return _detail(session, segment, document)++ def get_segment_update_state(self, segment_ref: SegmentRef) -> SegmentUpdateState | None:+ with self._session_factory() as session:+ segment = _get_segment(session, segment_ref)+ if segment is None:+ return None+ # Legacy rows may contain scalar or malformed JSON keywords. Treat+ # those as absent so a write can repair them without building a DTO.+ keywords = segment.keywords+ return SegmentUpdateState(+ content=segment.content,+ enabled=segment.enabled,+ keywords=tuple(keywords)+ if isinstance(keywords, list) and all(isinstance(keyword, str) for keyword in keywords)+ else None,+ )++ def get_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> tuple[SegmentIndexTarget, ...]:+ if not segment_ids:+ return ()+ with self._session_factory() as session:+ segments = session.scalars(+ select(DocumentSegment).where(+ DocumentSegment.tenant_id == document_ref.dataset.tenant_id,+ DocumentSegment.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegment.document_id == document_ref.document_id,+ DocumentSegment.id.in_(segment_ids),+ )+ ).all()+ return tuple(SegmentIndexTarget(segment.id, segment.index_node_id, segment.enabled) for segment in segments)++ def get_children_for_segments(+ self, document_ref: DocumentRef, segment_ids: Sequence[str]+ ) -> tuple[ChildChunkState, ...]:+ if not segment_ids:+ return ()+ with self._session_factory() as session:+ return tuple(_child_state(child) for child in session.scalars(_children_query(document_ref, segment_ids)))++ def set_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str], values: Mapping[str, object]) -> None:+ if not segment_ids:+ return+ with self._session_factory.begin() as session:+ session.execute(+ update(DocumentSegment)+ .where(+ DocumentSegment.tenant_id == document_ref.dataset.tenant_id,+ DocumentSegment.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegment.document_id == document_ref.document_id,+ DocumentSegment.id.in_(segment_ids),+ )+ .values(**values)+ )++ def save_segment(self, segment_ref: SegmentRef, values: Mapping[str, object], *, create: bool = False) -> None:+ with self._session_factory.begin() as session:+ _, document = _require_scope_models(session, segment_ref.document)+ segment = _get_segment(session, segment_ref)+ previous_words = segment.word_count if segment is not None else 0+ if create:+ if segment is not None:+ raise ValueError("Segment already exists")+ position = (+ session.scalar(+ select(func.max(DocumentSegment.position)).where(+ DocumentSegment.tenant_id == segment_ref.document.dataset.tenant_id,+ DocumentSegment.dataset_id == segment_ref.document.dataset.dataset_id,+ DocumentSegment.document_id == segment_ref.document.document_id,+ )+ )+ or 0+ )+ segment = DocumentSegment(+ tenant_id=segment_ref.document.dataset.tenant_id,+ dataset_id=segment_ref.document.dataset.dataset_id,+ document_id=segment_ref.document.document_id,+ position=position + 1,+ content=cast(str, values["content"]),+ word_count=cast(int, values["word_count"]),+ tokens=cast(int, values["tokens"]),+ created_by=cast(str, values["created_by"]),+ )+ segment.id = segment_ref.segment_id+ session.add(segment)+ if segment is None:+ raise LookupError("Segment no longer exists")+ for key, value in values.items():+ setattr(segment, key, value)+ if "word_count" in values:+ document.word_count = max(0, (document.word_count or 0) + segment.word_count - previous_words)++ def delete_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> None:+ if not segment_ids:+ return+ with self._session_factory.begin() as session:+ _, document = _require_scope_models(session, document_ref)+ rows = session.scalars(+ select(DocumentSegment).where(+ DocumentSegment.tenant_id == document_ref.dataset.tenant_id,+ DocumentSegment.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegment.document_id == document_ref.document_id,+ DocumentSegment.id.in_(segment_ids),+ )+ ).all()+ document.word_count = max(0, (document.word_count or 0) - sum(row.word_count for row in rows))+ for row in rows:+ session.delete(row)++ def get_children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...] | None:+ with self._session_factory() as session:+ if _get_segment(session, segment_ref) is None:+ return None+ children = session.scalars(+ _children_query(segment_ref.document, (segment_ref.segment_id,)).order_by(ChildChunk.position.asc())+ ).all()+ return tuple(_child_state(child) for child in children)++ def save_children(+ self,+ segment_ref: SegmentRef,+ *,+ added: Sequence[ChildChunkState] = (),+ updated: Sequence[ChildChunkState] = (),+ deleted: Sequence[ChildChunkState] = (),+ ) -> None:+ with self._session_factory.begin() as session:+ if _get_segment(session, segment_ref) is None:+ raise LookupError("Segment no longer exists")+ for child in deleted:+ row = _get_child_chunk(session, segment_ref, child.data.id)+ if row is not None:+ session.delete(row)+ for child in updated:+ row = _get_child_chunk(session, segment_ref, child.data.id)+ if row is None:+ raise LookupError("Child chunk no longer exists")+ row.content = child.data.content+ row.word_count = child.data.word_count+ row.type = SegmentType(child.data.type)+ row.updated_by = child.updated_by+ row.updated_at = child.data.updated_at+ for child in added:+ row = ChildChunk(+ **child.data.model_dump(exclude={"id", "created_at", "updated_at"}),+ tenant_id=segment_ref.document.dataset.tenant_id,+ dataset_id=segment_ref.document.dataset.dataset_id,+ document_id=segment_ref.document.document_id,+ index_node_id=child.index_node_id,+ index_node_hash=child.index_node_hash,+ created_by=child.created_by,+ )+ row.id = child.data.id+ row.created_at = child.data.created_at+ row.updated_at = child.data.updated_at+ row.updated_by = child.updated_by+ session.add(row)++ def get_indexing_snapshot(self, segment_ref: SegmentRef) -> SegmentIndexingSnapshot:+ return self.get_indexing_snapshots(segment_ref.document, (segment_ref.segment_id,))[0]++ def get_indexing_snapshots(+ self, document_ref: DocumentRef, segment_ids: Sequence[str]+ ) -> tuple[SegmentIndexingSnapshot, ...]:+ """Load an ordered batch and its owner chain in one bounded read session."""+ if not segment_ids:+ return ()+ with self._session_factory() as session:+ dataset, document = _require_scope_models(session, document_ref)+ segments = {+ segment.id: segment+ for segment in session.scalars(+ select(DocumentSegment).where(+ DocumentSegment.tenant_id == dataset.tenant_id,+ DocumentSegment.dataset_id == dataset.id,+ DocumentSegment.document_id == document.id,+ DocumentSegment.id.in_(segment_ids),+ )+ )+ }+ if set(segments) != set(segment_ids):+ raise LookupError("Segment no longer exists")+ rule = (+ session.scalar(+ select(DatasetProcessRule).where(+ DatasetProcessRule.id == document.dataset_process_rule_id,+ DatasetProcessRule.dataset_id == dataset.id,+ )+ )+ if document.dataset_process_rule_id+ else None+ )+ vector_type = (+ Vector.resolve_vector_type(dataset, session=session)+ if dataset.indexing_technique == "high_quality"+ else None+ )+ attachments: dict[str, list[str]] = {segment_id: [] for segment_id in segment_ids}+ for binding in session.scalars(+ select(SegmentAttachmentBinding).where(+ SegmentAttachmentBinding.tenant_id == dataset.tenant_id,+ SegmentAttachmentBinding.dataset_id == dataset.id,+ SegmentAttachmentBinding.document_id == document.id,+ SegmentAttachmentBinding.segment_id.in_(segment_ids),+ )+ ):+ attachments[binding.segment_id].append(binding.attachment_id)+ summaries = {+ summary.chunk_id: summary+ for summary in session.scalars(+ select(DocumentSegmentSummary).where(+ DocumentSegmentSummary.dataset_id == dataset.id,+ DocumentSegmentSummary.document_id == document.id,+ DocumentSegmentSummary.chunk_id.in_(segment_ids),+ )+ )+ }+ return tuple(+ SegmentIndexingSnapshot(+ dataset,+ document,+ segments[segment_id],+ rule,+ vector_type,+ summaries.get(segment_id),+ tuple(attachments[segment_id]),+ )+ for segment_id in segment_ids+ )++ def replace_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None:+ with self._session_factory.begin() as session:+ if _get_segment(session, segment_ref) is None:+ raise LookupError("Segment no longer exists")+ session.execute(+ delete(SegmentAttachmentBinding).where(+ SegmentAttachmentBinding.tenant_id == segment_ref.document.dataset.tenant_id,+ SegmentAttachmentBinding.dataset_id == segment_ref.document.dataset.dataset_id,+ SegmentAttachmentBinding.document_id == segment_ref.document.document_id,+ SegmentAttachmentBinding.segment_id == segment_ref.segment_id,+ )+ )+ session.add_all(+ [+ SegmentAttachmentBinding(+ tenant_id=segment_ref.document.dataset.tenant_id,+ dataset_id=segment_ref.document.dataset.dataset_id,+ document_id=segment_ref.document.document_id,+ segment_id=segment_ref.segment_id,+ attachment_id=file_id,+ )+ for file_id in attachment_ids+ ]+ )++ def save_summary(+ self, segment_ref: SegmentRef, content: str | None, *, error: str | None = None+ ) -> DocumentSegmentSummary | None:+ with self._session_factory.begin() as session:+ segment = _get_segment(session, segment_ref)+ if segment is None:+ raise LookupError("Segment no longer exists")+ summary = _summary(session, segment)+ if content is None:+ if summary is not None:+ session.delete(summary)+ return None+ if summary is None:+ summary = DocumentSegmentSummary(+ dataset_id=segment.dataset_id, document_id=segment.document_id, chunk_id=segment.id+ )+ session.add(summary)+ summary.summary_content = content+ summary.status = SummaryStatus.ERROR if error else SummaryStatus.GENERATING+ summary.error = error+ session.flush()+ session.expunge(summary)+ return summary++ def get_summary_images(self, segment_ref: SegmentRef) -> list[File]:+ with self._session_factory() as session:+ segment = _get_segment(session, segment_ref)+ if segment is None:+ raise LookupError("Segment no longer exists")+ images = ParagraphIndexProcessor._extract_images_from_segment_attachments(+ segment.tenant_id, segment.id, session+ )+ return images or ParagraphIndexProcessor._extract_images_from_text(+ segment.tenant_id, segment.content, session+ )++ def get_keyword_table(self, dataset_ref: DatasetRef) -> tuple[str, str | None]:+ with self._session_factory() as session:+ dataset = _get_dataset(session, dataset_ref)+ if dataset is None:+ raise LookupError("Dataset no longer exists")+ row = get_dataset_keyword_table(dataset, session=session)+ return (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)++ def save_keyword_table(+ self, dataset_ref: DatasetRef, *, storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]+ ) -> None:+ with self._session_factory.begin() as session:+ persist_keyword_table(+ session,+ tenant_id=dataset_ref.tenant_id,+ dataset_id=dataset_ref.dataset_id,+ storage_type=storage_type,+ data=data,+ keywords=keywords,+ )+++@dataclass(frozen=True)+class SegmentIndexingSnapshot:+ dataset: Dataset+ document: Document+ segment: DocumentSegment+ process_rule: DatasetProcessRule | None+ vector_type: str | None+ summary: DocumentSegmentSummary | None+ attachment_ids: tuple[str, ...]+++def _summary(session: Session, segment: DocumentSegment) -> DocumentSegmentSummary | None:+ return session.scalar(+ select(DocumentSegmentSummary)+ .where(+ DocumentSegmentSummary.dataset_id == segment.dataset_id,+ DocumentSegmentSummary.document_id == segment.document_id,+ DocumentSegmentSummary.chunk_id == segment.id,+ )+ .limit(1)+ )+++def _children_query(document_ref: DocumentRef, segment_ids: Sequence[str]):+ return select(ChildChunk).where(+ ChildChunk.tenant_id == document_ref.dataset.tenant_id,+ ChildChunk.dataset_id == document_ref.dataset.dataset_id,+ ChildChunk.document_id == document_ref.document_id,+ ChildChunk.segment_id.in_(segment_ids),+ )+++def _child_state(child: ChildChunk) -> ChildChunkState:+ return ChildChunkState(+ data=_child_chunk_data(child),+ index_node_id=child.index_node_id,+ index_node_hash=child.index_node_hash,+ created_by=child.created_by,+ updated_by=child.updated_by,+ )+++def _detail(session: Session, segment: DocumentSegment, document: Document) -> SegmentDetail:+ summary = _summary(session, segment)+ return SegmentDetail(+ data=_segment_data(segment, summary.summary_content if summary else None, session),+ doc_form=str(document.doc_form),+ )+++def _require_scope_models(session: Session, document_ref: DocumentRef) -> tuple[Dataset, Document]:+ dataset = _get_dataset(session, document_ref.dataset)+ document = _get_document(session, document_ref)+ if dataset is None or document is None:+ raise LookupError("Dataset document scope no longer exists")+ return dataset, document+++def _get_segment(session: Session, segment_ref: SegmentRef) -> DocumentSegment | None:+ return session.scalar(+ select(DocumentSegment)+ .where(+ DocumentSegment.id == segment_ref.segment_id,+ DocumentSegment.document_id == segment_ref.document.document_id,+ DocumentSegment.dataset_id == segment_ref.document.dataset.dataset_id,+ DocumentSegment.tenant_id == segment_ref.document.dataset.tenant_id,+ )+ .limit(1)+ )+++def _get_child_chunk(session: Session, segment_ref: SegmentRef, child_chunk_id: str) -> ChildChunk | None:+ return session.scalar(+ select(ChildChunk)+ .where(+ ChildChunk.id == child_chunk_id,+ ChildChunk.segment_id == segment_ref.segment_id,+ ChildChunk.document_id == segment_ref.document.document_id,+ ChildChunk.dataset_id == segment_ref.document.dataset.dataset_id,+ ChildChunk.tenant_id == segment_ref.document.dataset.tenant_id,+ )+ .limit(1)+ )+++def _segment_data(segment: DocumentSegment, summary: str | None, session: Session) -> SegmentRecord:+ return SegmentRecord.model_validate(+ {+ "id": segment.id,+ "position": segment.position,+ "document_id": segment.document_id,+ "content": segment.content,+ "answer": segment.answer,+ "word_count": segment.word_count,+ "tokens": segment.tokens,+ "keywords": segment.keywords,+ "index_node_id": segment.index_node_id,+ "index_node_hash": segment.index_node_hash,+ "hit_count": segment.hit_count,+ "enabled": segment.enabled,+ "disabled_at": segment.disabled_at,+ "disabled_by": segment.disabled_by,+ "status": segment.status,+ "created_by": segment.created_by,+ "created_at": segment.created_at,+ "updated_at": segment.updated_at,+ "updated_by": segment.updated_by,+ "indexing_at": segment.indexing_at,+ "completed_at": segment.completed_at,+ "error": segment.error,+ "stopped_at": segment.stopped_at,+ "sign_content": sign_segment_content(segment, session=session),+ "summary": summary,+ "child_chunks": get_segment_child_chunks(segment, session=session, include_full_doc=False),+ "attachments": get_segment_attachments(segment, session=session),+ }+ )+++def _child_chunk_data(child_chunk: ChildChunk) -> ChildChunkRecord:+ return ChildChunkRecord.model_validate(child_chunk)+++def query_child_chunks(+ session: Session,+ segment_ref: SegmentRef,+ *,+ page: int,+ limit: int,+ keyword: str | None,+) -> PaginatedResult[ChildChunk]:+ """Share the query with callers that already own an explicit session."""+ query = _children_query(segment_ref.document, (segment_ref.segment_id,)).order_by(ChildChunk.position.asc())+ if keyword:+ query = query.where(ChildChunk.content.ilike(f"%{escape_like_pattern(keyword)}%", escape="\\"))+ return paginate_query(query, session=session, page=page, per_page=limit, max_per_page=100)diff --git a/api/repositories/knowledge/summary_repository.py b/api/repositories/knowledge/summary_repository.pynew file mode 100644index 00000000000..0c2c96b2a3c--- /dev/null+++ b/api/repositories/knowledge/summary_repository.py@@ -0,0 +1,74 @@+"""Persist summary indexing outcomes using one owned session per attempt."""++from collections.abc import Callable++from sqlalchemy import select+from sqlalchemy.orm import Session++from libs.datetime_utils import naive_utc_now+from models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummary+from models.enums import SummaryStatus+from services.knowledge.summaries.application import SummaryVectorInput, SummaryVectorResult+++class SQLAlchemySummaryRepository:+ def __init__(self, *, new_session: Callable[[], Session]) -> None:+ self._new_session = new_session++ @staticmethod+ def _record(session: Session, command: SummaryVectorInput) -> DocumentSegmentSummary | None:+ ref = command.ref+ segment = session.scalar(+ select(DocumentSegment)+ .join(Document, Document.id == DocumentSegment.document_id)+ .join(Dataset, Dataset.id == Document.dataset_id)+ .where(+ Dataset.id == ref.document.dataset.dataset_id,+ Dataset.tenant_id == ref.document.dataset.tenant_id,+ Document.id == ref.document.document_id,+ Document.tenant_id == ref.document.dataset.tenant_id,+ DocumentSegment.id == ref.segment_id,+ DocumentSegment.dataset_id == ref.document.dataset.dataset_id,+ DocumentSegment.tenant_id == ref.document.dataset.tenant_id,+ )+ )+ if segment is None:+ raise LookupError("Summary segment no longer exists in this dataset")+ return session.scalar(+ select(DocumentSegmentSummary)+ .where(+ DocumentSegmentSummary.dataset_id == ref.document.dataset.dataset_id,+ DocumentSegmentSummary.document_id == ref.document.document_id,+ DocumentSegmentSummary.chunk_id == ref.segment_id,+ )+ .limit(1)+ )++ def completed(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None:+ with self._new_session() as session, session.begin():+ row = self._record(session, command)+ if row is None:+ row = DocumentSegmentSummary(+ dataset_id=command.ref.document.dataset.dataset_id,+ document_id=command.ref.document.document_id,+ chunk_id=command.ref.segment_id,+ summary_content=command.content,+ enabled=True,+ )+ row.id = command.summary_id+ session.add(row)+ row.summary_content = command.content+ row.summary_index_node_id = result.node_id+ row.summary_index_node_hash = result.content_hash+ row.tokens = result.tokens+ row.status = SummaryStatus.COMPLETED+ row.error = None+ row.updated_at = naive_utc_now()++ def failed(self, command: SummaryVectorInput, error: str) -> None:+ with self._new_session() as session, session.begin():+ row = self._record(session, command)+ if row is not None:+ row.status = SummaryStatus.ERROR+ row.error = error+ row.updated_at = naive_utc_now()diff --git a/api/repositories/knowledge/upload_file_repository.py b/api/repositories/knowledge/upload_file_repository.pynew file mode 100644index 00000000000..9b9e05fdd2d--- /dev/null+++ b/api/repositories/knowledge/upload_file_repository.py@@ -0,0 +1,55 @@+"""SQLAlchemy repository for upload files used by knowledge extraction."""++from collections.abc import Sequence++from sqlalchemy import delete, select+from sqlalchemy.orm import Session, sessionmaker++from core.rag.extractor.entity.extract_setting import UploadFileExtractionInput+from models.model import UploadFile+++def _query_files(session: Session, *, workspace_id: str, file_ids: Sequence[str]) -> dict[str, UploadFile]:+ if not file_ids:+ return {}+ uploads = session.scalars(+ select(UploadFile).where(UploadFile.id.in_(file_ids), UploadFile.tenant_id == workspace_id)+ )+ return {upload.id: upload for upload in uploads}+++def query_upload_extraction_inputs(+ session: Session, *, workspace_id: str, file_ids: Sequence[str]+) -> dict[str, UploadFileExtractionInput]:+ """Read detached extraction inputs without changing the caller's session lifecycle."""+ return {+ upload.id: UploadFileExtractionInput.model_validate(upload)+ for upload in _query_files(session, workspace_id=workspace_id, file_ids=file_ids).values()+ }+++class SQLAlchemyKnowledgeUploadRepository:+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def get_by_id(self, *, workspace_id: str, file_id: str) -> UploadFileExtractionInput | None:+ return self.get_by_ids(workspace_id=workspace_id, file_ids=(file_id,)).get(file_id)++ def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> dict[str, UploadFileExtractionInput]:+ with self._session_factory() as session:+ return query_upload_extraction_inputs(session, workspace_id=workspace_id, file_ids=file_ids)++ def get_files(self, *, workspace_id: str, file_ids: Sequence[str]) -> dict[str, UploadFile]:+ with self._session_factory() as session:+ return _query_files(session, workspace_id=workspace_id, file_ids=file_ids)++ def get_file_name(self, *, workspace_id: str, upload_file_id: str) -> str | None:+ upload = self.get_files(workspace_id=workspace_id, file_ids=(upload_file_id,)).get(upload_file_id)+ return upload.name if upload is not None else None++ def delete_files(self, *, workspace_id: str, file_ids: Sequence[str]) -> None:+ """Remove preview records after their storage cleanup has finished."""+ if not file_ids:+ return+ with self._session_factory.begin() as session:+ session.execute(delete(UploadFile).where(UploadFile.tenant_id == workspace_id, UploadFile.id.in_(file_ids)))diff --git a/api/repositories/workspace_member_query_repository.py b/api/repositories/workspace_member_query_repository.pyindex 1c67ae13e7f..f53e6ad1375 100644--- a/api/repositories/workspace_member_query_repository.py+++ b/api/repositories/workspace_member_query_repository.py@@ -60,3 +60,17 @@ class WorkspaceMemberQueryRepository(WorkspaceMemberQuery):)return records++ def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None:+ """Return one member's role without loading the whole workspace roster."""++ with self._session_factory() as session:+ role = session.scalar(+ select(TenantAccountJoin.role)+ .where(+ TenantAccountJoin.tenant_id == workspace_id,+ TenantAccountJoin.account_id == account_id,+ )+ .limit(1)+ )+ return role.value if role is not None else Nonediff --git a/api/schedule/clean_unused_datasets_task.py b/api/schedule/clean_unused_datasets_task.pyindex 02770c83232..ef55110271e 100644--- a/api/schedule/clean_unused_datasets_task.py+++ b/api/schedule/clean_unused_datasets_task.py@@ -14,6 +14,7 @@ from enums import CloudPlanfrom extensions.ext_redis import redis_clientfrom libs.pagination import paginate_queryfrom models.dataset import Dataset, DatasetAutoDisableLog, DatasetQuery, Document+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom services.feature_service import FeatureService@@ -140,7 +141,7 @@ def clean_unused_datasets_task():# Remove indexindex_processor = IndexProcessorFactory(- dataset.get_doc_form(session=session)+ get_dataset_doc_form(dataset, session=session)).init_index_processor()index_processor.clean(dataset, None, session=session)diff --git a/api/services/agent/knowledge_datasets.py b/api/services/agent/knowledge_datasets.pyindex 9db9bbc08c1..a7277e9059c 100644--- a/api/services/agent/knowledge_datasets.py+++ b/api/services/agent/knowledge_datasets.py@@ -35,7 +35,7 @@ def get_tenant_knowledge_dataset_rows(*, session: Session, tenant_id: str, datasa dataset row, so they are treated as missing instead of breaking theUUID-typed dataset lookup."""- from services.dataset_service import DatasetService+ from services.knowledge.dataset_service import DatasetServicevalid_ids: list[str] = []for dataset_id in dataset_ids:diff --git a/api/services/app_service.py b/api/services/app_service.pyindex 7510d6b3e95..00e926adbfc 100644--- a/api/services/app_service.py+++ b/api/services/app_service.py@@ -60,7 +60,7 @@ from services.entities.app_entities import (AppListSortBy,CreateAppParams,)-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServicefrom services.openapi.visibility import apply_openapi_gate, is_openapi_visiblefrom services.rbac_agent_access_service import initialize_agent_rbac_accessfrom services.system_feature_service import SystemFeatureServicediff --git a/api/services/credential_permission_service.py b/api/services/credential_permission_service.pydeleted file mode 100644index d9ce5e7c502..00000000000--- a/api/services/credential_permission_service.py+++ /dev/null@@ -1,70 +0,0 @@-from collections.abc import Sequence--from sqlalchemy import or_, select-from sqlalchemy.orm import InstrumentedAttribute, Session--from models.account import Account-from models.credential_permission import CredentialPermission-from models.enums import PermissionEnum---class CredentialPermissionService:- """- Shared service for per-credential access control.- Mirrors DatasetPermissionService but supports all credential types- via a credential_type discriminator.- """-- @classmethod- def get_partial_member_list(cls, credential_id: str, credential_type: str, *, session: Session) -> Sequence[str]:- """Return account_ids that have partial-member access to a credential."""- return session.scalars(- select(CredentialPermission.account_id).where(- CredentialPermission.credential_id == credential_id,- CredentialPermission.credential_type == credential_type,- )- ).all()-- @classmethod- def apply_visibility_filter(- cls,- query,- *,- model_id_column: InstrumentedAttribute,- model_user_id_column: InstrumentedAttribute,- model_visibility_column: InstrumentedAttribute,- credential_type: str,- user: Account,- ):- """- Add WHERE clauses to a SQLAlchemy query so it only returns credentials- visible to the given user.-- - all_team_members: always visible- - only_me: visible only to the creator (user.id matches)- - partial_members: visible to the creator OR users in credential_permissions- - Legacy rows with NULL user_id are treated as all_team_members- - No admin bypass: personal credentials are private regardless of role- """- # Subquery: credential_ids where user has partial-member permission- partial_subquery = (- select(CredentialPermission.credential_id)- .where(- CredentialPermission.credential_type == credential_type,- CredentialPermission.account_id == user.id,- )- .correlate_except(CredentialPermission)- )-- return query.where(- or_(- # all_team is always visible- model_visibility_column == PermissionEnum.ALL_TEAM,- # legacy rows with NULL user_id treated as all_team- model_user_id_column.is_(None),- # only_me: creator sees their own- (model_user_id_column == user.id),- # partial_members: user is in the permission table- model_id_column.in_(partial_subquery),- )- )diff --git a/api/services/auth/firecrawl/__init__.py b/api/services/credentials/__init__.pysimilarity index 100%rename from api/services/auth/firecrawl/__init__.pyrename to api/services/credentials/__init__.pydiff --git a/api/services/credentials/query.py b/api/services/credentials/query.pynew file mode 100644index 00000000000..54fb193b107--- /dev/null+++ b/api/services/credentials/query.py@@ -0,0 +1,79 @@+"""Consumer-owned read contracts for credential lists and their visibility policy."""++from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from typing import Protocol++from core.plugin.entities.plugin_daemon import CredentialType+++@dataclass(frozen=True, slots=True)+class ModelCredentialRecord:+ id: str+ name: str+++@dataclass(frozen=True, slots=True)+class DatasourceCredentialListItem:+ id: str+ name: str+ auth_type: str+ encrypted_credentials: Mapping[str, object]+ avatar_url: str | None+ is_default: bool+++@dataclass(frozen=True, slots=True)+class ToolCredentialRecord:+ id: str+ name: str+ provider: str+ credential_type: CredentialType+ credentials: Mapping[str, object]+ is_default: bool+ visibility: str+ created_by: str+ partial_member_ids: tuple[str, ...]+ from_other_member: bool+++@dataclass(frozen=True, slots=True)+class TriggerSubscriptionRecord:+ id: str+ name: str+ provider: str+ credential_type: CredentialType+ credentials: Mapping[str, object]+ endpoint_id: str+ parameters: Mapping[str, object]+ properties: Mapping[str, object]+ workflows_in_use: int+++class CredentialQuery(Protocol):+ """Return loaded values; implementations own SQL and short read sessions.++ An absent actor is reserved for trusted internal callers. Account-scoped+ callers must supply their actor ID; administrator roles do not bypass it.+ """++ def list_models(+ self, *, workspace_id: str, provider: str, actor_id: str | None+ ) -> Sequence[ModelCredentialRecord]: ...++ def list_datasources(+ self, *, workspace_id: str, provider: str, plugin_id: str, actor_id: str | None+ ) -> Sequence[DatasourceCredentialListItem]: ...++ def list_tools(+ self,+ *,+ workspace_id: str,+ provider: str,+ actor_id: str | None,+ include_credential_ids: Sequence[str] = (),+ ) -> Sequence[ToolCredentialRecord]: ...++ def list_trigger_subscriptions(+ self, *, workspace_id: str, provider: str, actor_id: str | None+ ) -> Sequence[TriggerSubscriptionRecord]: ...diff --git a/api/services/data_source/__init__.py b/api/services/data_source/__init__.pynew file mode 100644index 00000000000..48969ae2282--- /dev/null+++ b/api/services/data_source/__init__.py@@ -0,0 +1 @@+"""Application services and adapters for data-source use cases."""diff --git a/api/services/auth/jina/__init__.py b/api/services/data_source/auth/__init__.pysimilarity index 100%rename from api/services/auth/jina/__init__.pyrename to api/services/data_source/auth/__init__.pydiff --git a/api/services/auth/data_source_api_key_auth_gateways.py b/api/services/data_source/auth/api_key_gateways.pysimilarity index 78%rename from api/services/auth/data_source_api_key_auth_gateways.pyrename to api/services/data_source/auth/api_key_gateways.pyindex d40490c731c..e6db86fddef 100644--- a/api/services/auth/data_source_api_key_auth_gateways.py+++ b/api/services/data_source/auth/api_key_gateways.py@@ -6,12 +6,12 @@ from typing import Protocol, overrideimport httpxfrom core.helper import encrypter-from services.auth.data_source_api_key_auth_service import ApiKeyAuthCredentialEncryptor, ApiKeyAuthCredentialValidatorfrom services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,UnsupportedDataSourceApiKeyAuthProviderError,)-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.auth.api_key_service import ApiKeyAuthCredentialEncryptor, ApiKeyAuthCredentialValidator+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsclass _ProviderApiKeyAuthValidator(Protocol):@@ -27,15 +27,15 @@ type _ProviderApiKeyAuthValidatorFactory = Callable[def _get_provider_validator_factory(provider: str) -> _ProviderApiKeyAuthValidatorFactory:match provider:case "firecrawl":- from services.auth.firecrawl.firecrawl import FirecrawlAuth+ from services.data_source.auth.firecrawl.firecrawl import FirecrawlAuthreturn FirecrawlAuthcase "watercrawl":- from services.auth.watercrawl.watercrawl import WatercrawlAuth+ from services.data_source.auth.watercrawl.watercrawl import WatercrawlAuthreturn WatercrawlAuthcase "jinareader":- from services.auth.jina.jina import JinaAuth+ from services.data_source.auth.jina.jina import JinaAuthreturn JinaAuthcase _:diff --git a/api/services/auth/data_source_api_key_auth_service.py b/api/services/data_source/auth/api_key_service.pysimilarity index 97%rename from api/services/auth/data_source_api_key_auth_service.pyrename to api/services/data_source/auth/api_key_service.pyindex 585845a4a55..5831177b821 100644--- a/api/services/auth/data_source_api_key_auth_service.py+++ b/api/services/data_source/auth/api_key_service.py@@ -5,7 +5,7 @@ from dataclasses import replacefrom typing import Protocolfrom machinery.context import RequestContext-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthBindingRecord,DataSourceApiKeyAuthCredentials,diff --git a/api/services/auth/watercrawl/__init__.py b/api/services/data_source/auth/firecrawl/__init__.pysimilarity index 100%rename from api/services/auth/watercrawl/__init__.pyrename to api/services/data_source/auth/firecrawl/__init__.pydiff --git a/api/services/auth/firecrawl/firecrawl.py b/api/services/data_source/auth/firecrawl/firecrawl.pysimilarity index 96%rename from api/services/auth/firecrawl/firecrawl.pyrename to api/services/data_source/auth/firecrawl/firecrawl.pyindex 2261208bf08..fb0b8d8e9aa 100644--- a/api/services/auth/firecrawl/firecrawl.py+++ b/api/services/data_source/auth/firecrawl/firecrawl.py@@ -9,7 +9,7 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentials# Explicit bounded timeout for credential-validation requests so a slow or# hanging Firecrawl endpoint cannot block the worker indefinitely.diff --git a/api/services/data_source/auth/jina/__init__.py b/api/services/data_source/auth/jina/__init__.pynew file mode 100644index 00000000000..e69de29bb2ddiff --git a/api/services/auth/jina/jina.py b/api/services/data_source/auth/jina/jina.pysimilarity index 96%rename from api/services/auth/jina/jina.pyrename to api/services/data_source/auth/jina/jina.pyindex cf201ae0c38..52d1cd58bfd 100644--- a/api/services/auth/jina/jina.py+++ b/api/services/data_source/auth/jina/jina.py@@ -10,7 +10,7 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentials_http_client: httpx.Client = get_pooled_http_client("auth:jina",diff --git a/api/services/data_source/auth/watercrawl/__init__.py b/api/services/data_source/auth/watercrawl/__init__.pynew file mode 100644index 00000000000..e69de29bb2ddiff --git a/api/services/auth/watercrawl/watercrawl.py b/api/services/data_source/auth/watercrawl/watercrawl.pysimilarity index 96%rename from api/services/auth/watercrawl/watercrawl.pyrename to api/services/data_source/auth/watercrawl/watercrawl.pyindex e4d0951f6df..fe7a9e85f6b 100644--- a/api/services/auth/watercrawl/watercrawl.py+++ b/api/services/data_source/auth/watercrawl/watercrawl.py@@ -10,7 +10,7 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentials# Explicit bounded timeout for credential-validation requests so a slow or# hanging WaterCrawl endpoint cannot block the worker indefinitely.diff --git a/api/services/data_source/binding_application_service.py b/api/services/data_source/binding_application_service.pynew file mode 100644index 00000000000..3f11d13738b--- /dev/null+++ b/api/services/data_source/binding_application_service.py@@ -0,0 +1,69 @@+"""Application service for managing data-source OAuth bindings."""++from collections.abc import Sequence+from enum import StrEnum+from typing import Literal, Protocol++from machinery.context import RequestContext+from services.data_source.entities.oauth import DataSourceBindingSummary++type BindingMutationResult = Literal["updated", "not_found", "already_enabled", "already_disabled"]++_SUPPORTED_PROVIDERS = frozenset({"notion"})+++class BindingAction(StrEnum):+ ENABLE = "enable"+ DISABLE = "disable"+++class DataSourceBindingStore(Protocol):+ def list_enabled_bindings(self, *, workspace_id: str) -> Sequence[DataSourceBindingSummary]: ...++ def change_disabled_state(+ self,+ *,+ workspace_id: str,+ binding_id: str,+ disabled: bool,+ ) -> BindingMutationResult: ...+++class DataSourceBindingError(Exception):+ """Base class for framework-neutral binding management failures."""+++class DataSourceBindingNotFoundError(DataSourceBindingError):+ def __init__(self) -> None:+ super().__init__("Data source binding not found")+++class DataSourceBindingStateError(DataSourceBindingError):+ def __init__(self, *, disabled: bool) -> None:+ state = "disabled" if disabled else "enabled"+ super().__init__(f"Data source is already {state}")+++class DataSourceBindingApplicationService:+ def __init__(self, *, bindings: DataSourceBindingStore) -> None:+ self._bindings = bindings++ def list_integrations(self, context: RequestContext) -> tuple[DataSourceBindingSummary, ...]:+ return tuple(+ binding+ for binding in self._bindings.list_enabled_bindings(workspace_id=context.active_workspace_id)+ if binding.provider in _SUPPORTED_PROVIDERS+ )++ def change_state(self, context: RequestContext, binding_id: str, action: BindingAction) -> None:+ disabled = action == BindingAction.DISABLE+ result = self._bindings.change_disabled_state(+ workspace_id=context.active_workspace_id,+ binding_id=binding_id,+ disabled=disabled,+ )+ if result == "updated":+ return+ if result == "not_found":+ raise DataSourceBindingNotFoundError()+ raise DataSourceBindingStateError(disabled=disabled)diff --git a/api/services/data_source/credential_adapters.py b/api/services/data_source/credential_adapters.pynew file mode 100644index 00000000000..d58c4ea065d--- /dev/null+++ b/api/services/data_source/credential_adapters.py@@ -0,0 +1,197 @@+"""Plugin and persistence adapters for datasource credential application ports."""++from collections.abc import Callable, Mapping, Sequence+from typing import Protocol++from configs import dify_config+from core.helper import encrypter+from core.helper.provider_cache import NoOpProviderCredentialCache+from core.plugin.entities.plugin_daemon import CredentialType+from core.plugin.impl.datasource import PluginDatasourceManager+from core.plugin.impl.oauth import OAuthHandler+from core.plugin.plugin_service import PluginService+from core.tools.utils.encryption import create_provider_encrypter+from graphon.model_runtime.entities.provider_entities import FormType+from models.provider_ids import DatasourceProviderID+from services.data_source.credential_gateway import (+ DatasourceCredentialError,+ DatasourceCredentialRefreshError,+ RefreshedDatasourceCredential,+)+from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecord+++class DatasourceOAuthClientConfigReader(Protocol):+ def get_oauth_client_config(+ self,+ *,+ workspace_id: str,+ provider: str,+ plugin_id: str,+ ) -> DatasourceOAuthClientConfigRecord: ...+++class DatasourceOAuthClientResolver(Protocol):+ def resolve(self, *, workspace_id: str, provider_id: DatasourceProviderID) -> dict[str, object]: ...+++class PluginDatasourceCredentialCodec:+ def __init__(self, provider_manager: PluginDatasourceManager) -> None:+ self._provider_manager = provider_manager++ def decrypt(self, record: DatasourceCredentialRecord) -> dict[str, object]:+ return transform_secret_fields(+ record.workspace_id, record.encrypted_credentials, self._secret_variables(record), encrypter.decrypt_token+ )++ def encrypt(self, record: DatasourceCredentialRecord, credentials: Mapping[str, object]) -> dict[str, object]:+ return transform_secret_fields(+ record.workspace_id, credentials, self._secret_variables(record), encrypter.encrypt_token+ )++ def _secret_variables(self, record: DatasourceCredentialRecord) -> tuple[str, ...]:+ return datasource_secret_variables(+ self._provider_manager,+ record.workspace_id,+ f"{record.plugin_id}/{record.provider}",+ CredentialType.of(record.auth_type),+ )+++def transform_secret_fields(+ workspace_id: str,+ credentials: Mapping[str, object],+ secret_variables: Sequence[str],+ transform: Callable[[str, str], str],+) -> dict[str, object]:+ """Apply the shared token codec only to declared string secrets."""+ result = dict(credentials)+ for name in secret_variables:+ value = result.get(name)+ if isinstance(value, str):+ result[name] = transform(workspace_id, value)+ return result+++def datasource_secret_variables(+ manager: PluginDatasourceManager, workspace_id: str, provider_id: str, credential_type: CredentialType+) -> tuple[str, ...]:+ provider = manager.fetch_datasource_provider(tenant_id=workspace_id, provider_id=provider_id)+ if credential_type == CredentialType.API_KEY:+ schemas = provider.declaration.credentials_schema+ elif credential_type == CredentialType.OAUTH2:+ if provider.declaration.oauth_schema is None:+ raise DatasourceCredentialError("Datasource provider oauth schema not found")+ schemas = provider.declaration.oauth_schema.credentials_schema+ else:+ raise DatasourceCredentialError(f"Unsupported credential type: {credential_type}")+ return tuple(schema.name for schema in schemas if schema.type.value == FormType.SECRET_INPUT)+++class PluginDatasourceOAuthClientResolver:+ """Turn detached OAuth client rows into plugin-ready credentials."""++ def __init__(+ self,+ *,+ configs: DatasourceOAuthClientConfigReader,+ provider_manager: PluginDatasourceManager,+ ) -> None:+ self._configs = configs+ self._provider_manager = provider_manager++ def resolve(self, *, workspace_id: str, provider_id: DatasourceProviderID) -> dict[str, object]:+ config = self._configs.get_oauth_client_config(+ workspace_id=workspace_id,+ provider=provider_id.provider_name,+ plugin_id=provider_id.plugin_id,+ )+ return resolve_datasource_oauth_client(+ config, workspace_id=workspace_id, provider_id=provider_id, provider_manager=self._provider_manager+ )+++def resolve_datasource_oauth_client(+ config: DatasourceOAuthClientConfigRecord,+ *,+ workspace_id: str,+ provider_id: DatasourceProviderID,+ provider_manager: PluginDatasourceManager,+) -> dict[str, object]:+ provider = provider_manager.fetch_datasource_provider(+ tenant_id=workspace_id,+ provider_id=str(provider_id),+ )+ if config.encrypted_tenant_params is not None:+ oauth_schema = provider.declaration.oauth_schema+ if oauth_schema is None:+ raise DatasourceCredentialError("Datasource provider oauth schema not found")+ credential_encrypter, _ = create_provider_encrypter(+ tenant_id=workspace_id,+ config=[item.to_basic_provider_config() for item in oauth_schema.client_schema],+ cache=NoOpProviderCredentialCache(),+ )+ return dict(credential_encrypter.decrypt(dict(config.encrypted_tenant_params)))++ if config.system_credentials is not None and PluginService.is_plugin_verified(+ workspace_id, provider.plugin_unique_identifier+ ):+ return dict(config.system_credentials)+ raise DatasourceCredentialError(f"OAuth client is not configured for {provider_id}")+++class OAuthDatasourceCredentialRefresher:+ """Refresh datasource credentials through an injected OAuth-client resolver."""++ def __init__(+ self,+ *,+ oauth_clients: DatasourceOAuthClientResolver,+ oauth_handler: OAuthHandler,+ ) -> None:+ self._oauth_clients = oauth_clients+ self._oauth_handler = oauth_handler++ def refresh(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> RefreshedDatasourceCredential:+ provider_id = DatasourceProviderID(f"{record.plugin_id}/{record.provider}")+ try:+ return refresh_datasource_credential(+ self._oauth_handler,+ workspace_id=workspace_id,+ actor_id=actor_id,+ provider_id=provider_id,+ system_credentials=self._oauth_clients.resolve(workspace_id=workspace_id, provider_id=provider_id),+ credentials=credentials,+ )+ except DatasourceCredentialError:+ raise+ except Exception as error:+ raise DatasourceCredentialRefreshError(record.id) from error+++def refresh_datasource_credential(+ handler: OAuthHandler,+ *,+ workspace_id: str,+ actor_id: str,+ provider_id: DatasourceProviderID,+ system_credentials: Mapping[str, object],+ credentials: Mapping[str, object],+) -> RefreshedDatasourceCredential:+ refreshed = handler.refresh_credentials(+ tenant_id=workspace_id,+ user_id=actor_id,+ plugin_id=provider_id.plugin_id,+ provider=provider_id.provider_name,+ redirect_uri=f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_id}/datasource/callback",+ system_credentials=dict(system_credentials),+ credentials=dict(credentials),+ )+ return RefreshedDatasourceCredential(credentials=dict(refreshed.credentials), expires_at=refreshed.expires_at)diff --git a/api/services/data_source/credential_gateway.py b/api/services/data_source/credential_gateway.pynew file mode 100644index 00000000000..75dc5cf7cf4--- /dev/null+++ b/api/services/data_source/credential_gateway.py@@ -0,0 +1,319 @@+"""Application ports and policies for datasource credential resolution."""++import logging+import time+from collections.abc import Callable, Mapping, Sequence+from dataclasses import dataclass+from typing import Protocol++from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecord++logger = logging.getLogger(__name__)+++class NotionCredentialUnavailableError(ValueError):+ pass+++def resolve_stored_notion_access_token(+ *,+ credential_id: str | None,+ load_token: Callable[[str], str],+ integration_token: str | None,+) -> str:+ """Use the saved credential, then the configured legacy integration.++ Stored documents never select a tenant default credential: changing that+ default must not change the integration used to read an existing document.+ Preserve indexing's environment fallback when credential resolution fails,+ including refresh failures. Actor-selected imports do not use this policy.+ """+ credential_error: Exception | None = None+ if credential_id:+ try:+ token = load_token(credential_id)+ if isinstance(token, str) and token:+ return token+ except Exception as error:+ credential_error = error++ if not integration_token:+ raise NotionCredentialUnavailableError(+ "Must specify `integration_token` or set environment variable `NOTION_INTEGRATION_TOKEN`."+ ) from credential_error++ logger.warning("Stored Notion credential %s unavailable; using NOTION_INTEGRATION_TOKEN", credential_id)+ return integration_token+++class DatasourceProviderCredentialStore(Protocol):+ """Persisted provider credentials for already-authorized execution paths."""++ def get_provider_credential(+ self, *, workspace_id: str, provider: str, plugin_id: str, credential_id: str | None+ ) -> DatasourceCredentialRecord | None: ...++ def list_provider_credentials(+ self, *, workspace_id: str, provider: str, plugin_id: str+ ) -> Sequence[DatasourceCredentialRecord]: ...++ def get_oauth_client_config(+ self, *, workspace_id: str, provider: str, plugin_id: str+ ) -> DatasourceOAuthClientConfigRecord: ...++ def update_if_unchanged(+ self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int+ ) -> bool: ...+++class DatasourceCredentialStore(Protocol):+ def get_visible(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None: ...++ def update_if_unchanged(+ self,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+ ) -> bool: ...+++class TrustedDatasourceCredentialStore(Protocol):+ def get_for_stored_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None: ...++ def update_if_unchanged(+ self,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+ ) -> bool: ...+++class DatasourceCredentialCodec(Protocol):+ def decrypt(self, record: DatasourceCredentialRecord) -> dict[str, object]: ...++ def encrypt(+ self,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> dict[str, object]: ...+++@dataclass(frozen=True, slots=True)+class RefreshedDatasourceCredential:+ credentials: Mapping[str, object]+ expires_at: int+++class DatasourceCredentialRefresher(Protocol):+ def refresh(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> RefreshedDatasourceCredential: ...+++class ActorDatasourceCredentialResolver(Protocol):+ """Resolve a credential selected by an authenticated actor."""++ def resolve(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ provider: str,+ plugin_id: str,+ ) -> dict[str, object]: ...+++class StoredDatasourceCredentialResolver(Protocol):+ """Resolve a credential proven by an existing document owner chain."""++ def resolve_for_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> dict[str, object]: ...+++class DatasourceCredentialError(Exception):+ """Base class for credential resolution failures."""+++class DatasourceCredentialNotFoundError(DatasourceCredentialError):+ def __init__(self) -> None:+ super().__init__("Credential not found")+++class DatasourceCredentialConcurrentUpdateError(DatasourceCredentialError):+ def __init__(self) -> None:+ super().__init__("Credential changed while it was being refreshed")+++class DatasourceCredentialRefreshError(DatasourceCredentialError):+ def __init__(self, credential_id: str) -> None:+ super().__init__(f"Failed to refresh datasource credential: {credential_id}")+++class ActorAwareDatasourceCredentialGateway:+ def __init__(+ self,+ *,+ credentials: DatasourceCredentialStore,+ codec: DatasourceCredentialCodec,+ refresher: DatasourceCredentialRefresher,+ now: Callable[[], float] = time.time,+ ) -> None:+ self._credentials = credentials+ self._codec = codec+ self._refresher = refresher+ self._now = now++ def resolve(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ provider: str,+ plugin_id: str,+ ) -> dict[str, object]:+ def load_record() -> DatasourceCredentialRecord:+ record = self._credentials.get_visible(+ workspace_id=workspace_id,+ actor_id=actor_id,+ credential_id=credential_id,+ provider=provider,+ plugin_id=plugin_id,+ )+ if record is None:+ raise DatasourceCredentialNotFoundError()+ return record++ return resolve_credential_snapshot(+ load_record(),+ actor_id=actor_id,+ codec=self._codec,+ refresher=self._refresher,+ update=self._credentials.update_if_unchanged,+ load_latest=load_record,+ now=self._now,+ )+++class TrustedStoredDatasourceCredentialGateway:+ """Resolve a credential only when a stored document proves the full owner chain."""++ def __init__(+ self,+ *,+ credentials: TrustedDatasourceCredentialStore,+ codec: DatasourceCredentialCodec,+ refresher: DatasourceCredentialRefresher,+ now: Callable[[], float] = time.time,+ ) -> None:+ self._credentials = credentials+ self._codec = codec+ self._refresher = refresher+ self._now = now++ def resolve_for_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> dict[str, object]:+ def load_record() -> DatasourceCredentialRecord:+ record = self._credentials.get_for_stored_document(+ workspace_id=workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ credential_id=credential_id,+ provider=provider,+ plugin_id=plugin_id,+ )+ if record is None:+ raise DatasourceCredentialNotFoundError()+ return record++ record = load_record()+ return resolve_credential_snapshot(+ record,+ actor_id=record.owner_id,+ codec=self._codec,+ refresher=self._refresher,+ update=self._credentials.update_if_unchanged,+ load_latest=load_record,+ now=self._now,+ )+++def credential_needs_refresh(expires_at: int, *, now: float) -> bool:+ return expires_at != -1 and expires_at - 60 < int(now)+++class CredentialSnapshotUpdate(Protocol):+ def __call__(+ self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int+ ) -> bool: ...+++def resolve_credential_snapshot(+ record: DatasourceCredentialRecord,+ *,+ actor_id: str | None,+ codec: DatasourceCredentialCodec,+ refresher: DatasourceCredentialRefresher,+ update: CredentialSnapshotUpdate,+ load_latest: Callable[[], DatasourceCredentialRecord],+ now: Callable[[], float],+) -> dict[str, object]:+ decrypted = codec.decrypt(record)+ if not credential_needs_refresh(record.expires_at, now=now()):+ return decrypted+ if actor_id is None:+ raise DatasourceCredentialRefreshError(record.id)+ refreshed = refresher.refresh(+ workspace_id=record.workspace_id, actor_id=actor_id, record=record, credentials=decrypted+ )+ if update(+ record=record,+ encrypted_credentials=codec.encrypt(record, refreshed.credentials),+ expires_at=refreshed.expires_at,+ ):+ return dict(refreshed.credentials)+ latest = load_latest()+ if credential_needs_refresh(latest.expires_at, now=now()):+ raise DatasourceCredentialConcurrentUpdateError()+ return codec.decrypt(latest)diff --git a/api/services/data_source/entities/__init__.py b/api/services/data_source/entities/__init__.pynew file mode 100644index 00000000000..4504a9013aa--- /dev/null+++ b/api/services/data_source/entities/__init__.py@@ -0,0 +1 @@+"""Shared data contracts for data-source use cases."""diff --git a/api/services/entities/data_source_api_key_auth_entities.py b/api/services/data_source/entities/api_key_auth.pysimilarity index 100%rename from api/services/entities/data_source_api_key_auth_entities.pyrename to api/services/data_source/entities/api_key_auth.pydiff --git a/api/services/data_source/entities/credential.py b/api/services/data_source/entities/credential.pynew file mode 100644index 00000000000..4a06a0d8a66--- /dev/null+++ b/api/services/data_source/entities/credential.py@@ -0,0 +1,27 @@+"""Detached data contracts for datasource credentials."""++from collections.abc import Mapping+from dataclasses import dataclass+from datetime import datetime+++@dataclass(frozen=True, slots=True)+class DatasourceCredentialRecord:+ id: str+ workspace_id: str+ owner_id: str | None+ name: str+ provider: str+ plugin_id: str+ auth_type: str+ encrypted_credentials: Mapping[str, object]+ expires_at: int+ updated_at: datetime+++@dataclass(frozen=True, slots=True)+class DatasourceOAuthClientConfigRecord:+ """Raw persisted OAuth client configuration detached from its SQLAlchemy session."""++ encrypted_tenant_params: Mapping[str, object] | None+ system_credentials: Mapping[str, object] | Nonediff --git a/api/services/data_source/entities/notion_import.py b/api/services/data_source/entities/notion_import.pynew file mode 100644index 00000000000..555e1c5a1cb--- /dev/null+++ b/api/services/data_source/entities/notion_import.py@@ -0,0 +1,72 @@+"""Framework-neutral values used by Notion import use cases."""++from collections.abc import Mapping+from dataclasses import dataclass+from enum import StrEnum+++class NotionPageType(StrEnum):+ PAGE = "page"+ DATABASE = "database"+++@dataclass(frozen=True, slots=True)+class NotionPageIcon:+ type: str | None = None+ url: str | None = None+ emoji: str | None = None+++@dataclass(frozen=True, slots=True)+class AuthorizedNotionPage:+ page_id: str+ page_name: str+ page_icon: NotionPageIcon | None+ parent_id: str | None+ page_type: NotionPageType+++@dataclass(frozen=True, slots=True)+class NotionImportPage:+ page_id: str+ page_name: str+ page_icon: NotionPageIcon | None+ parent_id: str | None+ page_type: NotionPageType+ is_bound: bool+++@dataclass(frozen=True, slots=True)+class NotionWorkspace:+ workspace_id: str | None+ workspace_name: str | None+ workspace_icon: str | None+ pages: tuple[AuthorizedNotionPage, ...]+++@dataclass(frozen=True, slots=True)+class NotionImportWorkspace:+ workspace_id: str | None+ workspace_name: str | None+ workspace_icon: str | None+ pages: tuple[NotionImportPage, ...]+++@dataclass(frozen=True, slots=True)+class NotionImportResult:+ workspaces: tuple[NotionImportWorkspace, ...]+++def notion_page_icon(value: Mapping[str, object] | None) -> NotionPageIcon | None:+ """Map a plugin icon payload without leaking the plugin entity type."""++ if value is None:+ return None+ icon_type = value.get("type")+ url = value.get("url")+ emoji = value.get("emoji")+ return NotionPageIcon(+ type=icon_type if isinstance(icon_type, str) else None,+ url=url if isinstance(url, str) else None,+ emoji=emoji if isinstance(emoji, str) else None,+ )diff --git a/api/services/entities/data_source_oauth_entities.py b/api/services/data_source/entities/oauth.pysimilarity index 66%rename from api/services/entities/data_source_oauth_entities.pyrename to api/services/data_source/entities/oauth.pyindex 844cec22847..a65e2126b50 100644--- a/api/services/entities/data_source_oauth_entities.py+++ b/api/services/data_source/entities/oauth.py@@ -2,6 +2,7 @@from collections.abc import Mappingfrom dataclasses import dataclass+from datetime import datetime@dataclass(frozen=True, slots=True)@@ -22,3 +23,14 @@ class DataSourceOAuthBindingRecord:id: straccess_token: strsource_info: Mapping[str, object]+++@dataclass(frozen=True, slots=True)+class DataSourceBindingSummary:+ """Secret-free management view of a data-source OAuth binding."""++ id: str+ provider: str+ created_at: datetime+ disabled: bool+ source_info: Mapping[str, object]diff --git a/api/services/data_source/notion_import_adapters.py b/api/services/data_source/notion_import_adapters.pynew file mode 100644index 00000000000..6d43d3ccc8d--- /dev/null+++ b/api/services/data_source/notion_import_adapters.py@@ -0,0 +1,146 @@+"""Notion plugin and extractor adapter for import use cases."""++import logging+from collections.abc import Callable, Mapping+from typing import Protocol, cast++from core.datasource.entities.datasource_entities import DatasourceProviderType+from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin+from core.rag.extractor.notion_extractor import NotionExtractor+from core.rag.models.document import Document as ExtractedDocument+from services.data_source.credential_gateway import ActorDatasourceCredentialResolver, DatasourceCredentialError+from services.data_source.entities.notion_import import (+ AuthorizedNotionPage,+ NotionPageType,+ NotionWorkspace,+ notion_page_icon,+)+from services.data_source.notion_import_application_service import NotionImportCredentialUnavailableError++_NOTION_PROVIDER = "notion_datasource"+_NOTION_PLUGIN = "langgenius/notion_datasource"+_NOTION_RUNTIME_PROVIDER = f"{_NOTION_PLUGIN}/{_NOTION_PROVIDER}"+logger = logging.getLogger(__name__)+++class _RuntimeLoader(Protocol):+ def __call__(+ self,+ *,+ provider_id: str,+ datasource_name: str,+ tenant_id: str,+ datasource_type: DatasourceProviderType,+ ) -> object: ...+++class _NotionPageExtractor(Protocol):+ def extract(self) -> list[ExtractedDocument]: ...+++class PluginNotionSourceGateway:+ def __init__(+ self,+ *,+ credentials: ActorDatasourceCredentialResolver,+ runtime_loader: _RuntimeLoader,+ extractor_factory: Callable[..., _NotionPageExtractor] = NotionExtractor,+ ) -> None:+ self._credentials = credentials+ self._runtime_loader = runtime_loader+ self._extractor_factory = extractor_factory++ def list_authorized_pages(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ ) -> tuple[NotionWorkspace, ...]:+ credentials = self._resolve_credentials(workspace_id, actor_id, credential_id)+ runtime = cast(+ OnlineDocumentDatasourcePlugin,+ self._runtime_loader(+ provider_id=_NOTION_RUNTIME_PROVIDER,+ datasource_name=_NOTION_PROVIDER,+ tenant_id=workspace_id,+ datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,+ ),+ )+ runtime.runtime.credentials = dict(credentials)++ workspaces: dict[str | None, tuple[str | None, str | None, list[AuthorizedNotionPage]]] = {}+ for message in runtime.get_online_document_pages(+ user_id=actor_id,+ datasource_parameters={},+ provider_type=runtime.datasource_provider_type(),+ ):+ for workspace in message.result:+ workspace_info = workspaces.get(workspace.workspace_id)+ if workspace_info is None:+ workspace_info = (workspace.workspace_name, workspace.workspace_icon, [])+ workspaces[workspace.workspace_id] = workspace_info+ for page in workspace.pages:+ try:+ page_type = NotionPageType(page.type)+ except (TypeError, ValueError):+ logger.warning("Skipping unsupported Notion page type %r for page %s", page.type, page.page_id)+ continue+ workspace_info[2].append(+ AuthorizedNotionPage(+ page_id=page.page_id,+ page_name=page.page_name,+ page_icon=notion_page_icon(page.page_icon),+ parent_id=page.parent_id,+ page_type=page_type,+ )+ )+ return tuple(+ NotionWorkspace(+ workspace_id=workspace_id,+ workspace_name=workspace_name,+ workspace_icon=workspace_icon,+ pages=tuple(pages),+ )+ for workspace_id, (workspace_name, workspace_icon, pages) in workspaces.items()+ )++ def preview_page(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ page_id: str,+ page_type: NotionPageType,+ ) -> str:+ credentials = self._resolve_credentials(workspace_id, actor_id, credential_id)+ integration_secret = credentials.get("integration_secret")+ if not isinstance(integration_secret, str) or not integration_secret:+ raise NotionImportCredentialUnavailableError()+ extractor = self._extractor_factory(+ notion_workspace_id="",+ notion_obj_id=page_id,+ notion_page_type=page_type.value,+ notion_access_token=integration_secret,+ notion_token_loader=lambda: integration_secret,+ tenant_id=workspace_id,+ )+ return "\n".join(document.page_content for document in extractor.extract())++ def _resolve_credentials(+ self,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ ) -> Mapping[str, object]:+ try:+ return self._credentials.resolve(+ workspace_id=workspace_id,+ actor_id=actor_id,+ credential_id=credential_id,+ provider=_NOTION_PROVIDER,+ plugin_id=_NOTION_PLUGIN,+ )+ except DatasourceCredentialError as error:+ raise NotionImportCredentialUnavailableError() from errordiff --git a/api/services/data_source/notion_import_application_service.py b/api/services/data_source/notion_import_application_service.pynew file mode 100644index 00000000000..e1454ed71e4--- /dev/null+++ b/api/services/data_source/notion_import_application_service.py@@ -0,0 +1,129 @@+"""Application service for listing and previewing importable Notion pages."""++from typing import Protocol++from machinery.context import RequestContext+from services.data_source.entities.notion_import import (+ NotionImportPage,+ NotionImportResult,+ NotionImportWorkspace,+ NotionPageType,+ NotionWorkspace,+)+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.resource_scope import DatasetRef+++class NotionDocumentBindingReader(Protocol):+ def list_bound_notion_page_ids(self, dataset_ref: DatasetRef) -> frozenset[str]: ...+++class NotionDatasetReader(Protocol):+ def is_notion_dataset(self, dataset_ref: DatasetRef) -> bool: ...+++class NotionSourceGateway(Protocol):+ def list_authorized_pages(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ ) -> tuple[NotionWorkspace, ...]: ...++ def preview_page(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ page_id: str,+ page_type: NotionPageType,+ ) -> str: ...+++class NotionImportError(Exception):+ """Base class for framework-neutral Notion import failures."""+++class DatasetIsNotNotionSourceError(NotionImportError):+ def __init__(self) -> None:+ super().__init__("Dataset is not notion type")+++class NotionImportCredentialUnavailableError(NotionImportError):+ def __init__(self) -> None:+ super().__init__("Notion credential is unavailable")+++class NotionImportApplicationService:+ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ datasets: NotionDatasetReader,+ documents: NotionDocumentBindingReader,+ source: NotionSourceGateway,+ ) -> None:+ self._dataset_access = dataset_access+ self._datasets = datasets+ self._documents = documents+ self._source = source++ def list_pages(+ self,+ context: RequestContext,+ *,+ credential_id: str,+ dataset_id: str | None = None,+ ) -> NotionImportResult:+ bound_page_ids: frozenset[str] = frozenset()+ if dataset_id is not None:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ dataset_ref = DatasetRef(dataset.workspace_id, dataset.id)+ if not self._datasets.is_notion_dataset(dataset_ref):+ raise DatasetIsNotNotionSourceError()+ bound_page_ids = self._documents.list_bound_notion_page_ids(dataset_ref)++ workspaces = self._source.list_authorized_pages(+ workspace_id=context.active_workspace_id,+ actor_id=context.account_id,+ credential_id=credential_id,+ )+ return NotionImportResult(+ workspaces=tuple(+ NotionImportWorkspace(+ workspace_id=workspace.workspace_id,+ workspace_name=workspace.workspace_name,+ workspace_icon=workspace.workspace_icon,+ pages=tuple(+ NotionImportPage(+ page_id=page.page_id,+ page_name=page.page_name,+ page_icon=page.page_icon,+ parent_id=page.parent_id,+ page_type=page.page_type,+ is_bound=page.page_id in bound_page_ids,+ )+ for page in workspace.pages+ ),+ )+ for workspace in workspaces+ )+ )++ def preview_page(+ self,+ context: RequestContext,+ *,+ credential_id: str,+ page_id: str,+ page_type: NotionPageType,+ ) -> str:+ return self._source.preview_page(+ workspace_id=context.active_workspace_id,+ actor_id=context.account_id,+ credential_id=credential_id,+ page_id=page_id,+ page_type=page_type,+ )diff --git a/api/services/notion_data_source_gateway.py b/api/services/data_source/notion_oauth_gateway.pysimilarity index 95%rename from api/services/notion_data_source_gateway.pyrename to api/services/data_source/notion_oauth_gateway.pyindex 5cadbc82fb9..75405f0529a 100644--- a/api/services/notion_data_source_gateway.py+++ b/api/services/data_source/notion_oauth_gateway.py@@ -2,15 +2,16 @@import urllib.parsefrom collections.abc import Mapping-from typing import Any, Literal, TypedDict, override+from typing import Any, TypedDict, overrideimport httpxfrom pydantic import TypeAdapter, ValidationErrorfrom core.helper.ssrf_proxy import MaxRetriesExceededError, SSRFProxyfrom core.tools.errors import ToolSSRFError-from services.data_source_oauth_service import DataSourceOAuthError, DataSourceProviderGateway-from services.entities.data_source_oauth_entities import DataSourceOAuthAuthorization+from services.data_source.entities.notion_import import NotionPageType+from services.data_source.entities.oauth import DataSourceOAuthAuthorization+from services.data_source.oauth_service import DataSourceOAuthError, DataSourceProviderGatewayclass _NotionPageSummary(TypedDict):@@ -18,7 +19,7 @@ class _NotionPageSummary(TypedDict):page_name: strpage_icon: dict[str, str] | Noneparent_id: str- type: Literal["page", "database"]+ type: NotionPageTypeclass _NotionSourceInfo(TypedDict):@@ -147,7 +148,7 @@ class NotionDataSourceGateway(DataSourceProviderGateway):"page_name": page_name,"page_icon": icon,"parent_id": parent_id,- "type": "page",+ "type": NotionPageType.PAGE,}))@@ -180,19 +181,19 @@ class NotionDataSourceGateway(DataSourceProviderGateway):"page_name": page_name,"page_icon": icon,"parent_id": parent_id,- "type": "database",+ "type": NotionPageType.DATABASE,}))return pagesdef notion_page_search(self, access_token: str) -> list[dict[str, Any]]:- return self._search(access_token, object_type="page")+ return self._search(access_token, object_type=NotionPageType.PAGE)def notion_database_search(self, access_token: str) -> list[dict[str, Any]]:- return self._search(access_token, object_type="database")+ return self._search(access_token, object_type=NotionPageType.DATABASE)- def _search(self, access_token: str, *, object_type: str) -> list[dict[str, Any]]:+ def _search(self, access_token: str, *, object_type: NotionPageType) -> list[dict[str, Any]]:results: list[dict[str, Any]] = []next_cursor: str | None = Nonehas_more = Truediff --git a/api/services/data_source_oauth_service.py b/api/services/data_source/oauth_service.pysimilarity index 98%rename from api/services/data_source_oauth_service.pyrename to api/services/data_source/oauth_service.pyindex 7e91839a9fe..16f5e258957 100644--- a/api/services/data_source_oauth_service.py+++ b/api/services/data_source/oauth_service.py@@ -4,7 +4,7 @@ from collections.abc import Mappingfrom typing import Protocolfrom machinery.context import RequestContext-from services.entities.data_source_oauth_entities import (+from services.data_source.entities.oauth import (DataSourceOAuthAuthorization,DataSourceOAuthBindingRecord,DataSourceOAuthCallback,diff --git a/api/services/datasource_provider_service.py b/api/services/data_source/provider_service.pysimilarity index 79%rename from api/services/datasource_provider_service.pyrename to api/services/data_source/provider_service.pyindex ed394bd0a29..008dc846b31 100644--- a/api/services/datasource_provider_service.py+++ b/api/services/data_source/provider_service.py@@ -1,7 +1,10 @@import loggingimport timefrom collections.abc import Mapping-from typing import TYPE_CHECKING, Any+from dataclasses import replace+from typing import TYPE_CHECKING, Any, cast++from services.credentials.query import CredentialQueryif TYPE_CHECKING:from models.account import Account@@ -17,14 +20,25 @@ from core.helper.provider_cache import NoOpProviderCredentialCachefrom core.plugin.entities.plugin_daemon import CredentialTypefrom core.plugin.impl.datasource import PluginDatasourceManagerfrom core.plugin.impl.oauth import OAuthHandler-from core.plugin.plugin_service import PluginServicefrom core.tools.utils.encryption import ProviderConfigCache, ProviderConfigEncrypter, create_provider_encrypterfrom extensions.ext_database import dbfrom extensions.ext_redis import redis_client-from graphon.model_runtime.entities.provider_entities import FormTypefrom models.enums import PermissionEnumfrom models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProviderfrom models.provider_ids import DatasourceProviderID+from services.data_source.credential_adapters import (+ datasource_secret_variables,+ refresh_datasource_credential,+ resolve_datasource_oauth_client,+ transform_secret_fields,+)+from services.data_source.credential_gateway import (+ DatasourceCredentialError,+ DatasourceProviderCredentialStore,+ credential_needs_refresh,+ resolve_stored_notion_access_token,+)+from services.data_source.entities.credential import DatasourceCredentialRecordlogger = logging.getLogger(__name__)@@ -47,25 +61,47 @@ def get_current_user():class DatasourceProviderService:"""- Model Provider Service+ Datasource provider credentials and authorization management."""- def __init__(self) -> None:+ def __init__(self, *, credentials: DatasourceProviderCredentialStore) -> None:+ self._credentials = credentialsself.provider_manager = PluginDatasourceManager()+ def get_stored_notion_access_token(self, *, tenant_id: str, credential_id: str | None) -> str:+ """Resolve a stored Notion credential with the configured integration fallback."""++ def load_token(saved_id: str) -> str:+ credentials = self.get_datasource_credentials(+ tenant_id=tenant_id,+ credential_id=saved_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ token = credentials.get("integration_secret") if credentials else None+ if not isinstance(token, str) or not token:+ raise ValueError(f"No notion credential found for tenant {tenant_id} and credential {saved_id}")+ return token++ return resolve_stored_notion_access_token(+ credential_id=credential_id,+ load_token=load_token,+ integration_token=dify_config.NOTION_INTEGRATION_TOKEN,+ )+@staticmethod- def _should_refresh_credentials(datasource_provider: DatasourceProvider, now: int | None = None) -> bool:+ def _should_refresh_credentials(+ datasource_provider: DatasourceProvider | DatasourceCredentialRecord, now: int | None = None+ ) -> bool:current_time = int(time.time()) if now is None else now- if datasource_provider.expires_at == -1:- return False- return (datasource_provider.expires_at - 60) < current_time+ return credential_needs_refresh(datasource_provider.expires_at, now=current_time)def _refresh_datasource_credentials(self,tenant_id: str,provider: str,plugin_id: str,- datasource_provider: DatasourceProvider,+ datasource_provider: DatasourceProvider | DatasourceCredentialRecord,current_user: Any,) -> tuple[dict[str, Any], int]:datasource_provider_id = DatasourceProviderID(f"{plugin_id}/{provider}")@@ -90,17 +126,13 @@ class DatasourceProviderService:plugin_id=plugin_id,provider=provider,)- redirect_uri = (- f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{datasource_provider_id}/datasource/callback"- )system_credentials = self.get_oauth_client(tenant_id, datasource_provider_id)try:- refreshed_credentials = OAuthHandler().refresh_credentials(- tenant_id=tenant_id,- user_id=current_user.id,- plugin_id=datasource_provider_id.plugin_id,- provider=provider_name,- redirect_uri=redirect_uri,+ refreshed_credentials = refresh_datasource_credential(+ OAuthHandler(),+ workspace_id=tenant_id,+ actor_id=current_user.id,+ provider_id=datasource_provider_id,system_credentials=system_credentials or {},credentials=decrypted_credentials,)@@ -157,7 +189,7 @@ class DatasourceProviderService:def decrypt_datasource_provider_credentials(self,tenant_id: str,- datasource_provider: DatasourceProvider,+ datasource_provider: DatasourceProvider | DatasourceCredentialRecord,plugin_id: str,provider: str,) -> dict[str, Any]:@@ -167,11 +199,9 @@ class DatasourceProviderService:provider_id=f"{plugin_id}/{provider}",credential_type=CredentialType.of(datasource_provider.auth_type),)- decrypted_credentials = encrypted_credentials.copy()- for key, value in decrypted_credentials.items():- if key in credential_secret_variables:- decrypted_credentials[key] = encrypter.decrypt_token(tenant_id, value)- return decrypted_credentials+ return transform_secret_fields(+ tenant_id, encrypted_credentials, credential_secret_variables, encrypter.decrypt_token+ )def encrypt_datasource_provider_credentials(self,@@ -179,18 +209,16 @@ class DatasourceProviderService:provider: str,plugin_id: str,raw_credentials: Mapping[str, Any],- datasource_provider: DatasourceProvider,+ datasource_provider: DatasourceProvider | DatasourceCredentialRecord,) -> dict[str, Any]:provider_credential_secret_variables = self.extract_secret_variables(tenant_id=tenant_id,provider_id=f"{plugin_id}/{provider}",credential_type=CredentialType.of(datasource_provider.auth_type),)- encrypted_credentials = dict(raw_credentials)- for key, value in encrypted_credentials.items():- if key in provider_credential_secret_variables:- encrypted_credentials[key] = encrypter.encrypt_token(tenant_id, value)- return encrypted_credentials+ return transform_secret_fields(+ tenant_id, raw_credentials, provider_credential_secret_variables, encrypter.encrypt_token+ )def get_datasource_credentials(self,@@ -205,44 +233,44 @@ class DatasourceProviderService:If the stored credential is expired or about to expire, this method refreshesit through plugin-daemon and persists the refreshed credential before returning."""- with sessionmaker(bind=db.engine).begin() as session:- if credential_id:- datasource_provider = session.scalar(- select(DatasourceProvider)- .where(DatasourceProvider.tenant_id == tenant_id, DatasourceProvider.id == credential_id)- .limit(1)- )+ record = self._credentials.get_provider_credential(+ workspace_id=tenant_id, provider=provider, plugin_id=plugin_id, credential_id=credential_id+ )+ if record is None:+ return {}+ return self._resolve_provider_credentials(record)++ def _resolve_provider_credentials(self, record: DatasourceCredentialRecord) -> dict[str, Any]:+ """Refresh a snapshot; the store owns each read and conditional write transaction."""+ if self._should_refresh_credentials(record):+ encrypted, expires_at = self._refresh_datasource_credentials(+ tenant_id=record.workspace_id,+ provider=record.provider,+ plugin_id=record.plugin_id,+ datasource_provider=record,+ current_user=get_current_user(),+ )+ saved = self._credentials.update_if_unchanged(+ record=record, encrypted_credentials=encrypted, expires_at=expires_at+ )+ if saved:+ record = replace(record, encrypted_credentials=encrypted, expires_at=expires_at)else:- datasource_provider = session.scalar(- select(DatasourceProvider)- .where(- DatasourceProvider.tenant_id == tenant_id,- DatasourceProvider.provider == provider,- DatasourceProvider.plugin_id == plugin_id,- )- .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())- .limit(1)- )- if not datasource_provider:- return {}- if self._should_refresh_credentials(datasource_provider):- current_user = get_current_user()- encrypted_credentials, expires_at = self._refresh_datasource_credentials(- tenant_id=tenant_id,- provider=provider,- plugin_id=plugin_id,- datasource_provider=datasource_provider,- current_user=current_user,+ latest = self._credentials.get_provider_credential(+ workspace_id=record.workspace_id,+ provider=record.provider,+ plugin_id=record.plugin_id,+ credential_id=record.id,)- datasource_provider.encrypted_credentials = encrypted_credentials- datasource_provider.expires_at = expires_at-- return self.decrypt_datasource_provider_credentials(- tenant_id=tenant_id,- datasource_provider=datasource_provider,- plugin_id=plugin_id,- provider=provider,- )+ if latest is None or self._should_refresh_credentials(latest):+ raise ValueError("Credential changed while it was being refreshed")+ record = latest+ return self.decrypt_datasource_provider_credentials(+ tenant_id=record.workspace_id,+ datasource_provider=record,+ plugin_id=record.plugin_id,+ provider=record.provider,+ )def get_all_datasource_credentials_by_provider(self,@@ -256,54 +284,20 @@ class DatasourceProviderService:Expired credentials are refreshed independently. A failed credential refresh islogged and skipped so one broken authorization does not block other credentials."""- with sessionmaker(bind=db.engine).begin() as session:- datasource_providers = session.scalars(- select(DatasourceProvider)- .where(- DatasourceProvider.tenant_id == tenant_id,- DatasourceProvider.provider == provider,- DatasourceProvider.plugin_id == plugin_id,+ datasource_providers = self._credentials.list_provider_credentials(+ workspace_id=tenant_id, provider=provider, plugin_id=plugin_id+ )+ credentials = []+ for datasource_provider in datasource_providers:+ try:+ credentials.append(self._resolve_provider_credentials(datasource_provider))+ except Exception:+ logger.exception(+ "Skipping datasource credentials for provider %s after refresh or decrypt failure",+ provider,+ extra={"tenant_id": tenant_id, "plugin_id": plugin_id, "credential_id": datasource_provider.id},)- .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())- ).all()- if not datasource_providers:- return []- current_user = get_current_user()- real_credentials_list = []- for datasource_provider in datasource_providers:- try:- if self._should_refresh_credentials(datasource_provider):- encrypted_credentials, expires_at = self._refresh_datasource_credentials(- tenant_id=tenant_id,- provider=provider,- plugin_id=plugin_id,- datasource_provider=datasource_provider,- current_user=current_user,- )- datasource_provider.encrypted_credentials = encrypted_credentials- datasource_provider.expires_at = expires_at- real_credentials = self.decrypt_datasource_provider_credentials(- tenant_id=tenant_id,- datasource_provider=datasource_provider,- plugin_id=plugin_id,- provider=provider,- )- real_credentials_list.append(real_credentials)- except Exception:- logger.exception(- "Skipping datasource credentials for provider %s after refresh or decrypt failure",- provider,- extra={- "tenant_id": tenant_id,- "plugin_id": plugin_id,- "provider": provider,- "credential_id": getattr(datasource_provider, "id", None),- "credential_name": getattr(datasource_provider, "name", None),- "expires_at": getattr(datasource_provider, "expires_at", None),- },- )-- return real_credentials_list+ return credentialsdef update_datasource_provider_name(self, tenant_id: str, datasource_provider_id: DatasourceProviderID, name: str, credential_id: str@@ -516,42 +510,22 @@ class DatasourceProviderService:"""get oauth client"""- provider = datasource_provider_id.provider_name- plugin_id = datasource_provider_id.plugin_id- with Session(db.engine).no_autoflush as session:- # get tenant oauth client params- tenant_oauth_client_params = session.scalar(- select(DatasourceOauthTenantParamConfig)- .where(- DatasourceOauthTenantParamConfig.tenant_id == tenant_id,- DatasourceOauthTenantParamConfig.provider == provider,- DatasourceOauthTenantParamConfig.plugin_id == plugin_id,- DatasourceOauthTenantParamConfig.enabled.is_(True),- )- .limit(1)+ config = self._credentials.get_oauth_client_config(+ workspace_id=tenant_id,+ provider=datasource_provider_id.provider_name,+ plugin_id=datasource_provider_id.plugin_id,+ )+ try:+ return resolve_datasource_oauth_client(+ config,+ workspace_id=tenant_id,+ provider_id=datasource_provider_id,+ provider_manager=self.provider_manager,)- if tenant_oauth_client_params:- encrypter, _ = self.get_oauth_encrypter(tenant_id, datasource_provider_id)- return dict(encrypter.decrypt(tenant_oauth_client_params.client_params))-- provider_controller = self.provider_manager.fetch_datasource_provider(- tenant_id=tenant_id, provider_id=str(datasource_provider_id)- )- is_verified = PluginService.is_plugin_verified(tenant_id, provider_controller.plugin_unique_identifier)- if is_verified:- # fallback to system oauth client params- oauth_client_params = session.scalar(- select(DatasourceOauthParamConfig)- .where(- DatasourceOauthParamConfig.provider == provider,- DatasourceOauthParamConfig.plugin_id == plugin_id,- )- .limit(1)- )- if oauth_client_params:- return oauth_client_params.system_credentials-- raise ValueError(f"Please configure oauth client params(system/tenant) for {plugin_id}/{provider}")+ except DatasourceCredentialError as error:+ raise ValueError(+ f"Please configure oauth client params(system/tenant) for {datasource_provider_id}"+ ) from error@staticmethoddef generate_next_datasource_provider_name(@@ -800,25 +774,12 @@ class DatasourceProviderService::param credential_form_schemas::return:"""- datasource_provider = self.provider_manager.fetch_datasource_provider(- tenant_id=tenant_id, provider_id=provider_id- )- credential_form_schemas = []- if credential_type == CredentialType.API_KEY:- credential_form_schemas = list(datasource_provider.declaration.credentials_schema)- elif credential_type == CredentialType.OAUTH2:- if not datasource_provider.declaration.oauth_schema:- raise ValueError("Datasource provider oauth schema not found")- credential_form_schemas = list(datasource_provider.declaration.oauth_schema.credentials_schema)- else:+ if credential_type not in (CredentialType.API_KEY, CredentialType.OAUTH2):raise ValueError(f"Invalid credential type: {credential_type}")-- secret_input_form_variables = []- for credential_form_schema in credential_form_schemas:- if credential_form_schema.type.value == FormType.SECRET_INPUT:- secret_input_form_variables.append(credential_form_schema.name)-- return secret_input_form_variables+ try:+ return list(datasource_secret_variables(self.provider_manager, tenant_id, provider_id, credential_type))+ except DatasourceCredentialError as error:+ raise ValueError(str(error)) from errordef list_datasource_credentials(self,@@ -827,7 +788,7 @@ class DatasourceProviderService:plugin_id: str,user: "Account | None" = None,*,- session: Session,+ credential_query: CredentialQuery,) -> list[dict]:"""list datasource credentials with obfuscated sensitive fields,@@ -836,41 +797,13 @@ class DatasourceProviderService::param tenant_id: workspace id:param provider: provider name:param plugin_id: plugin id- :param user: current user (id + admin flag drive the visibility filter)+ :param user: current viewer; administrator roles do not bypass credential visibility:return:"""- from models.credential_permission import CredentialType as CredPermType- from services.credential_permission_service import CredentialPermissionService-- # Get all provider configurations of the current workspace- query = select(DatasourceProvider).where(- DatasourceProvider.tenant_id == tenant_id,- DatasourceProvider.provider == provider,- DatasourceProvider.plugin_id == plugin_id,+ datasource_providers = credential_query.list_datasources(+ workspace_id=tenant_id, provider=provider, plugin_id=plugin_id, actor_id=user.id if user else None)- if user is not None:- query = CredentialPermissionService.apply_visibility_filter(- query,- model_id_column=DatasourceProvider.id,- model_user_id_column=DatasourceProvider.user_id,- model_visibility_column=DatasourceProvider.visibility,- credential_type=CredPermType.DATASOURCE_PROVIDER,- user=user,- )- datasource_providers: list[DatasourceProvider] = list(session.scalars(query).all())- if not datasource_providers:- return []copy_credentials_list = []- default_provider = session.execute(- select(DatasourceProvider.id)- .where(- DatasourceProvider.tenant_id == tenant_id,- DatasourceProvider.provider == provider,- DatasourceProvider.plugin_id == plugin_id,- )- .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())- ).first()- default_provider_id = default_provider.id if default_provider else Nonefor datasource_provider in datasource_providers:encrypted_credentials = datasource_provider.encrypted_credentials# Get provider credential secret variables@@ -881,10 +814,10 @@ class DatasourceProviderService:)# Obfuscate provider credentials- copy_credentials = encrypted_credentials.copy()+ copy_credentials = dict(encrypted_credentials)for key, value in copy_credentials.items():if key in credential_secret_variables:- copy_credentials[key] = encrypter.obfuscated_token(value)+ copy_credentials[key] = encrypter.obfuscated_token(cast(str, value))copy_credentials_list.append({"credential": copy_credentials,@@ -892,14 +825,14 @@ class DatasourceProviderService:"name": datasource_provider.name,"avatar_url": datasource_provider.avatar_url,"id": datasource_provider.id,- "is_default": default_provider_id and datasource_provider.id == default_provider_id,+ "is_default": datasource_provider.is_default,})return copy_credentials_listdef get_all_datasource_credentials(- self, tenant_id: str, *, session: Session, user: "Account | None" = None+ self, tenant_id: str, *, session: Session, credential_query: CredentialQuery, user: "Account | None" = None) -> list[dict]:"""get datasource credentials.@@ -921,7 +854,7 @@ class DatasourceProviderService:provider=datasource.provider,plugin_id=datasource.plugin_id,user=user,- session=session,+ credential_query=credential_query,)redirect_uri = (f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{datasource_provider_id}/datasource/callback"@@ -965,7 +898,7 @@ class DatasourceProviderService:return datasource_credentialsdef get_hard_code_datasource_credentials(- self, tenant_id: str, *, session: Session, user: "Account | None" = None+ self, tenant_id: str, *, session: Session, credential_query: CredentialQuery, user: "Account | None" = None) -> list[dict]:"""get hard code datasource credentials.@@ -990,7 +923,7 @@ class DatasourceProviderService:provider=datasource.provider,plugin_id=datasource.plugin_id,user=user,- session=session,+ credential_query=credential_query,)redirect_uri = "{}/console/api/oauth/plugin/{}/datasource/callback".format(dify_config.CONSOLE_API_URL, datasource_provider_iddiff --git a/api/services/website_service.py b/api/services/data_source/website_service.pysimilarity index 88%rename from api/services/website_service.pyrename to api/services/data_source/website_service.pyindex b833c146216..049ae1bb4dc 100644--- a/api/services/website_service.py+++ b/api/services/data_source/website_service.py@@ -14,7 +14,7 @@ from core.rag.extractor.firecrawl.firecrawl_app import CrawlStatusResponse, Firefrom core.rag.extractor.watercrawl.provider import WaterCrawlProviderfrom extensions.ext_redis import redis_clientfrom extensions.ext_storage import storage-from services.datasource_provider_service import DatasourceProviderService+from services.data_source.provider_service import DatasourceProviderService# Reuse pooled HTTP clients to avoid creating new connections per request and ease testing.# Both clients carry a bounded read/connect timeout so a stalled Jina or@@ -147,8 +147,10 @@ class CrawlStatusDict(TypedDict):class WebsiteService:"""Service class for website crawling operations using different providers."""- @classmethod- def _get_credentials_and_config(cls, tenant_id: str, provider: str) -> tuple[Any, Any]:+ def __init__(self, *, providers: DatasourceProviderService) -> None:+ self._providers = providers++ def _get_credentials_and_config(self, tenant_id: str, provider: str) -> tuple[Any, Any]:"""Get and validate credentials for a provider."""if provider == "firecrawl":plugin_id = "langgenius/firecrawl_datasource"@@ -158,8 +160,7 @@ class WebsiteService:plugin_id = "langgenius/jina_datasource"else:raise ValueError("Invalid provider")- datasource_provider_service = DatasourceProviderService()- credential = datasource_provider_service.get_datasource_credentials(+ credential = self._providers.get_datasource_credentials(tenant_id=tenant_id,provider=provider,plugin_id=plugin_id,@@ -187,19 +188,18 @@ class WebsiteService:except ValueError as e:raise ValueError(f"Invalid arguments: {e}")- @classmethod- def crawl_url(cls, api_request: WebsiteCrawlApiRequest) -> dict[str, Any]:+ def crawl_url(self, api_request: WebsiteCrawlApiRequest) -> dict[str, Any]:"""Crawl a URL using the specified provider with typed request."""request = api_request.to_crawl_request()- api_key, config = cls._get_credentials_and_config(current_user.current_tenant_id, request.provider)+ api_key, config = self._get_credentials_and_config(current_user.current_tenant_id, request.provider)if request.provider == "firecrawl":- return cls._crawl_with_firecrawl(request=request, api_key=api_key, config=config)+ return self._crawl_with_firecrawl(request=request, api_key=api_key, config=config)elif request.provider == "watercrawl":- return cls._crawl_with_watercrawl(request=request, api_key=api_key, config=config)+ return self._crawl_with_watercrawl(request=request, api_key=api_key, config=config)elif request.provider == "jinareader":- return cls._crawl_with_jinareader(request=request, api_key=api_key)+ return self._crawl_with_jinareader(request=request, api_key=api_key)else:raise ValueError("Invalid provider")@@ -278,23 +278,21 @@ class WebsiteService:raise ValueError("Failed to crawl")return {"status": "active", "job_id": response.json().get("data", {}).get("taskId")}- @classmethod- def get_crawl_status(cls, job_id: str, provider: str) -> CrawlStatusDict:+ def get_crawl_status(self, job_id: str, provider: str) -> CrawlStatusDict:"""Get crawl status using string parameters."""api_request = WebsiteCrawlStatusApiRequest(provider=provider, job_id=job_id)- return cls.get_crawl_status_typed(api_request)+ return self.get_crawl_status_typed(api_request)- @classmethod- def get_crawl_status_typed(cls, api_request: WebsiteCrawlStatusApiRequest) -> CrawlStatusDict:+ def get_crawl_status_typed(self, api_request: WebsiteCrawlStatusApiRequest) -> CrawlStatusDict:"""Get crawl status using typed request."""- api_key, config = cls._get_credentials_and_config(current_user.current_tenant_id, api_request.provider)+ api_key, config = self._get_credentials_and_config(current_user.current_tenant_id, api_request.provider)if api_request.provider == "firecrawl":- return cls._get_firecrawl_status(api_request.job_id, api_key, config)+ return self._get_firecrawl_status(api_request.job_id, api_key, config)elif api_request.provider == "watercrawl":- return cls._get_watercrawl_status(api_request.job_id, api_key, config)+ return self._get_watercrawl_status(api_request.job_id, api_key, config)elif api_request.provider == "jinareader":- return cls._get_jinareader_status(api_request.job_id, api_key)+ return self._get_jinareader_status(api_request.job_id, api_key)else:raise ValueError("Invalid provider")@@ -359,16 +357,15 @@ class WebsiteService:crawl_status_data["data"] = formatted_datareturn crawl_status_data- @classmethod- def get_crawl_url_data(cls, job_id: str, provider: str, url: str, tenant_id: str) -> dict[str, Any] | None:- api_key, config = cls._get_credentials_and_config(tenant_id, provider)+ def get_crawl_url_data(self, job_id: str, provider: str, url: str, tenant_id: str) -> dict[str, Any] | None:+ api_key, config = self._get_credentials_and_config(tenant_id, provider)if provider == "firecrawl":- return cls._get_firecrawl_url_data(job_id, url, api_key, config)+ return self._get_firecrawl_url_data(job_id, url, api_key, config)elif provider == "watercrawl":- return cls._get_watercrawl_url_data(job_id, url, api_key, config)+ return self._get_watercrawl_url_data(job_id, url, api_key, config)elif provider == "jinareader":- return cls._get_jinareader_url_data(job_id, url, api_key)+ return self._get_jinareader_url_data(job_id, url, api_key)else:raise ValueError("Invalid provider")@@ -435,16 +432,15 @@ class WebsiteService:return dict(item.get("data", {}))return None- @classmethod- def get_scrape_url_data(cls, provider: str, url: str, tenant_id: str, only_main_content: bool) -> dict[str, Any]:+ def get_scrape_url_data(self, provider: str, url: str, tenant_id: str, only_main_content: bool) -> dict[str, Any]:request = ScrapeRequest(provider=provider, url=url, tenant_id=tenant_id, only_main_content=only_main_content)- api_key, config = cls._get_credentials_and_config(tenant_id=request.tenant_id, provider=request.provider)+ api_key, config = self._get_credentials_and_config(tenant_id=request.tenant_id, provider=request.provider)if request.provider == "firecrawl":- return cls._scrape_with_firecrawl(request=request, api_key=api_key, config=config)+ return self._scrape_with_firecrawl(request=request, api_key=api_key, config=config)elif request.provider == "watercrawl":- return cls._scrape_with_watercrawl(request=request, api_key=api_key, config=config)+ return self._scrape_with_watercrawl(request=request, api_key=api_key, config=config)else:raise ValueError("Invalid provider")diff --git a/api/services/dataset_ref_service.py b/api/services/dataset_ref_service.pydeleted file mode 100644index 6bd5cf444ff..00000000000--- a/api/services/dataset_ref_service.py+++ /dev/null@@ -1,65 +0,0 @@-"""Typed resource references for dataset ownership chains."""--from typing import NamedTuple--from sqlalchemy import select-from sqlalchemy.orm import Session--from models.dataset import Dataset, Document---class DatasetRef(NamedTuple):- """Dataset identifiers used to scope downstream resource lookups."""-- tenant_id: str- dataset_id: str---class DocumentRef(NamedTuple):- """Owner-bound lookup coordinates, not proof that a document is authorized or exists."""-- dataset: DatasetRef- document_id: str---class SegmentRef(NamedTuple):- """Segment identifiers used to scope downstream resource lookups."""-- document: DocumentRef- segment_id: str---class DatasetRefService:- """Build child locators from validated dataset roots and resolve them with owner predicates."""-- @staticmethod- def create_dataset_ref(dataset: Dataset) -> DatasetRef:- """Create a root ref from a dataset already validated by the caller."""- return DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)-- @staticmethod- def create_document_ref(dataset_ref: DatasetRef, document: Document) -> DocumentRef | None:- if document.tenant_id != dataset_ref.tenant_id or document.dataset_id != dataset_ref.dataset_id:- return None- return DatasetRefService.create_document_ref_from_id(dataset_ref, document.id)-- @staticmethod- def create_document_ref_from_id(dataset_ref: DatasetRef, document_id: str) -> DocumentRef:- """Bind a candidate document ID; ownership is enforced when the ref is consumed."""- return DocumentRef(dataset=dataset_ref, document_id=document_id)-- @staticmethod- def create_segment_ref(document_ref: DocumentRef, segment_id: str) -> SegmentRef:- """Bind a candidate segment ID; ownership is enforced when the ref is consumed."""- return SegmentRef(document=document_ref, segment_id=segment_id)-- @staticmethod- def get_document_by_ref(document_ref: DocumentRef, *, session: Session) -> Document | None:- """Resolve a document through its complete tenant and dataset ownership chain."""- return session.scalar(- select(Document).where(- Document.id == document_ref.document_id,- Document.dataset_id == document_ref.dataset.dataset_id,- Document.tenant_id == document_ref.dataset.tenant_id,- )- )diff --git a/api/services/knowledge/__init__.py b/api/services/knowledge/__init__.pyindex b1ff0725e7a..1ae0c3b5d8b 100644--- a/api/services/knowledge/__init__.py+++ b/api/services/knowledge/__init__.py@@ -1 +1 @@-"""Knowledge base application services."""+"""Knowledge-domain application services and adapters."""diff --git a/api/services/knowledge/api_key_service.py b/api/services/knowledge/api_key_service.pyindex f03aad1f1ff..875028fab9c 100644--- a/api/services/knowledge/api_key_service.py+++ b/api/services/knowledge/api_key_service.py@@ -4,14 +4,16 @@ from collections.abc import Callablefrom typing import Protocolfrom machinery.context import RequestContext-from services.auth.api_key_contracts import ApiKeyCache, ApiKeyRecord+from services.auth.api_key_contracts import ApiKeyCache, ApiKeyRecord, ApiKeyResourceNotFoundErrorfrom services.errors.account import NoPermissionError-from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.dataset_access import (+ DatasetAccess,+ DatasetAccessDeniedError,+ DatasetNotFoundError,+)class DatasetApiKeyStore(Protocol):- def get_dataset_access(self, workspace_id: str, dataset_id: str, account_id: str) -> DatasetAccess: ...-def list_keys(self, workspace_id: str, dataset_id: str) -> tuple[ApiKeyRecord, ...]: ...def create_key(self, workspace_id: str, dataset_id: str, *, max_keys: int, prefix: str) -> ApiKeyRecord: ...@@ -35,9 +37,17 @@ class UnknownDatasetIdsError(Exception):class DatasetApiKeyService:MAX_KEYS = 10- def __init__(self, *, keys: DatasetApiKeyStore, cache: ApiKeyCache, rbac_enabled: Callable[[], bool]) -> None:+ def __init__(+ self,+ *,+ keys: DatasetApiKeyStore,+ cache: ApiKeyCache,+ access: DatasetAccess,+ rbac_enabled: Callable[[], bool],+ ) -> None:self._keys = keysself._cache = cache+ self._access = accessself._rbac_enabled = rbac_enableddef list_keys(self, context: RequestContext, dataset_id: str) -> tuple[ApiKeyRecord, ...]:@@ -56,9 +66,12 @@ class DatasetApiKeyService:def _check_access(self, context: RequestContext, dataset_id: str) -> None:# Enterprise RBAC is enforced at admission; community deployments use the dataset ACL.if not self._rbac_enabled():- access = self._keys.get_dataset_access(context.active_workspace_id, dataset_id, context.account_id)- if not access.allows(context.account_id):- raise NoPermissionError("You do not have permission to access this dataset.")+ try:+ self._access.require_accessible(context, dataset_id)+ except DatasetNotFoundError as error:+ raise ApiKeyResourceNotFoundError("Dataset not found.") from error+ except DatasetAccessDeniedError as error:+ raise NoPermissionError("You do not have permission to access this dataset.") from errordef list_workspace_keys(self, context: RequestContext) -> tuple[ApiKeyRecord, ...]:return self._keys.list_workspace_keys(context.active_workspace_id)diff --git a/api/services/knowledge/dataset_access.py b/api/services/knowledge/dataset_access.pyindex c79f40ac513..4d7fd436293 100644--- a/api/services/knowledge/dataset_access.py+++ b/api/services/knowledge/dataset_access.py@@ -1,22 +1,123 @@-"""Dataset ACL policy independent of persistence and request globals."""+"""Dataset access use case, read model, errors, and consumer-owned ports."""-from dataclasses import dataclass+from typing import NamedTuple, Protocol+from machinery.context import RequestContext-@dataclass(frozen=True, slots=True)-class DatasetAccess:- permission: str+_OWNER_ROLE = "owner"+_ALL_TEAM_MEMBERS = "all_team_members"+_PARTIAL_MEMBERS = "partial_members"+++class DatasetAccessRecord(NamedTuple):+ """Dataset state consumed only by the access policy."""++ id: str+ workspace_id: strmaintainer_id: str | None- role: str | None- has_explicit_permission: bool-- def allows(self, account_id: str) -> bool:- if self.role is None:- return False- if self.role == "owner" or self.maintainer_id == account_id:- return True- if self.permission == "only_me":- return False- if self.permission == "partial_members":- return self.has_explicit_permission- return self.permission == "all_team_members"+ permission: str+++class AccessibleDataset(NamedTuple):+ """Minimal result proving that the request may operate on a dataset."""++ id: str+ workspace_id: str+++class DatasetAccessSnapshot(NamedTuple):+ """Dataset state plus the requesting account's partial-member grant."""++ dataset: DatasetAccessRecord+ actor_has_partial_access: bool+++class DatasetAccessReader(Protocol):+ """Load tenant-scoped dataset state needed by the access policy."""++ def get_access_snapshot(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ actor_id: str,+ ) -> DatasetAccessSnapshot | None: ...+++class WorkspaceRoleReader(Protocol):+ """Read the account's legacy role in a workspace."""++ def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None: ...+++class DatasetAccessError(Exception):+ """Base class for framework-neutral dataset access failures."""+++class DatasetNotFoundError(DatasetAccessError):+ def __init__(self) -> None:+ super().__init__("Dataset not found")+++class DatasetAccessDeniedError(DatasetAccessError):+ def __init__(self) -> None:+ super().__init__("You do not have permission to access this dataset")+++def can_access_dataset(snapshot: DatasetAccessSnapshot, *, actor_id: str, workspace_role: str | None) -> bool:+ """Evaluate legacy dataset visibility without database or framework access."""++ if workspace_role is None:+ return False+ dataset = snapshot.dataset+ if workspace_role == _OWNER_ROLE or dataset.maintainer_id == actor_id:+ return True+ if dataset.permission == _ALL_TEAM_MEMBERS:+ return True+ if dataset.permission == _PARTIAL_MEMBERS:+ return snapshot.actor_has_partial_access+ return False+++class DatasetAccessService:+ """Resolve an owned dataset and enforce legacy permissions when configured."""++ def __init__(+ self,+ *,+ datasets: DatasetAccessReader,+ workspace_roles: WorkspaceRoleReader,+ legacy_permissions_enabled: bool,+ ) -> None:+ self._datasets = datasets+ self._workspace_roles = workspace_roles+ self._legacy_permissions_enabled = legacy_permissions_enabled++ def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset:+ snapshot = self._datasets.get_access_snapshot(+ workspace_id=context.active_workspace_id,+ dataset_id=dataset_id,+ actor_id=context.account_id,+ )+ if snapshot is None:+ raise DatasetNotFoundError()+ return self.check_access(context, snapshot)++ def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset:+ """Authorize already-loaded state without querying the dataset again."""+ if snapshot.dataset.workspace_id != context.active_workspace_id:+ raise DatasetAccessDeniedError()+ if self._legacy_permissions_enabled:+ workspace_role = self._workspace_roles.get_legacy_role(+ workspace_id=context.active_workspace_id,+ account_id=context.account_id,+ )+ if not can_access_dataset(snapshot, actor_id=context.account_id, workspace_role=workspace_role):+ raise DatasetAccessDeniedError()+ return AccessibleDataset(id=snapshot.dataset.id, workspace_id=snapshot.dataset.workspace_id)+++class DatasetAccess(Protocol):+ def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset: ...++ def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset: ...diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.pynew file mode 100644index 00000000000..90b076d4400--- /dev/null+++ b/api/services/knowledge/dataset_read_service.py@@ -0,0 +1,318 @@+"""Load knowledge details and materialize data before response serialization."""++from __future__ import annotations++import json+from collections.abc import Iterable, Mapping, Sequence+from typing import Any, cast++from sqlalchemy.orm import Session++from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource+from core.rag.index_processor.constant.query_type import QueryType+from core.tools.signature import sign_upload_file_preview_url+from models.dataset import Dataset, DatasetMetadata, DatasetQuery, DocMetadataDetailItem, Document, DocumentSegment+from models.model import UploadFile+from repositories.knowledge.dataset_read_repository import (+ DatasetDetailPrefetch,+ DocumentReadBatch,+ build_dataset_detail_prefetch,+ get_document_dataset,+ get_document_metadata_rows,+ get_document_upload_file,+ get_document_uploader,+ get_query_upload_files,+ get_segment_child_chunks,+ load_document_read_batch,+)+from repositories.knowledge.segment_read_adapter import get_segment_attachments, sign_segment_content+from services.knowledge.entities.datasets import DatasetDetailRecord+++def load_dataset_detail(+ dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None+) -> DatasetDetailRecord:+ values = prefetch if prefetch is not None else build_dataset_detail_prefetch([dataset], session=session)+ return _dataset_detail_values(dataset, values)+++def load_dataset_details(datasets: Sequence[Dataset], *, session: Session) -> list[DatasetDetailRecord]:+ """Read a page in one batch and return values with no ORM or session references."""+ prefetch = build_dataset_detail_prefetch(datasets, session=session)+ return [_dataset_detail_values(dataset, prefetch) for dataset in datasets]+++def _dataset_detail_values(dataset: Dataset, prefetch: DatasetDetailPrefetch) -> DatasetDetailRecord:+ owner = (dataset.id, dataset.tenant_id)+ return {+ "id": dataset.id,+ "name": dataset.name,+ "description": dataset.description,+ "provider": dataset.provider,+ "permission": dataset.permission,+ "data_source_type": dataset.data_source_type,+ "indexing_technique": dataset.indexing_technique,+ "app_count": prefetch.app_counts.get(dataset.id, 0),+ "document_count": prefetch.document_counts.get(dataset.id, 0),+ "word_count": prefetch.word_counts.get(dataset.id, 0),+ "created_by": dataset.created_by,+ "author_name": prefetch.author_names.get(dataset.created_by),+ "created_at": dataset.created_at,+ "updated_by": dataset.updated_by,+ "updated_at": dataset.updated_at,+ "embedding_model": dataset.embedding_model,+ "embedding_model_provider": dataset.embedding_model_provider,+ "embedding_available": vars(dataset).get("embedding_available"),+ "retrieval_model_dict": dataset.retrieval_model_dict,+ "summary_index_setting": dataset.summary_index_setting,+ "tags": [{"id": tag.id, "name": tag.name, "type": tag.type} for tag in prefetch.tags.get(owner, [])],+ "doc_form": dataset.chunk_structure or prefetch.doc_forms.get(owner),+ "external_knowledge_info": prefetch.external_knowledge_infos.get(owner)+ if dataset.provider == "external"+ else None,+ "external_retrieval_model": dataset.external_retrieval_model,+ "doc_metadata": dataset.build_doc_metadata(prefetch.doc_metadata.get(dataset.id, [])),+ "built_in_field_enabled": dataset.built_in_field_enabled,+ "pipeline_id": dataset.pipeline_id,+ "runtime_mode": dataset.runtime_mode,+ "chunk_structure": dataset.chunk_structure,+ "icon_info": dataset.icon_info,+ "is_published": prefetch.pipeline_published.get(dataset.pipeline_id, False) if dataset.pipeline_id else False,+ "total_documents": prefetch.document_counts.get(dataset.id, 0),+ "total_available_documents": prefetch.available_document_counts.get(dataset.id, 0),+ "enable_api": dataset.enable_api,+ "is_multimodal": dataset.is_multimodal,+ "permission_keys": vars(dataset).get("permission_keys", []),+ "maintainer": dataset.maintainer,+ }+++def load_document_detail(document: Document, *, session: Session) -> dict[str, Any]:+ return load_document_details([document], session=session)[0]+++def load_document_details(documents: Iterable[Document], *, session: Session) -> list[dict[str, Any]]:+ """Materialize document attributes and relations while the caller's session is open."""+ items = list(documents)+ batch = load_document_read_batch(items, session=session)+ return [+ {+ "id": document.id,+ "position": document.position,+ "data_source_type": document.data_source_type,+ "data_source_info_dict": document.data_source_info_dict,+ "dataset_process_rule_id": document.dataset_process_rule_id,+ "name": document.name,+ "created_from": document.created_from,+ "created_by": document.created_by,+ "created_at": document.created_at,+ "tokens": document.tokens,+ "indexing_status": document.indexing_status,+ "error": document.error,+ "enabled": document.enabled,+ "disabled_at": document.disabled_at,+ "disabled_by": document.disabled_by,+ "archived": document.archived,+ "display_status": document.display_status,+ "word_count": document.word_count,+ "doc_form": document.doc_form,+ "summary_index_status": vars(document).get("summary_index_status"),+ "need_summary": document.need_summary,+ "completed_segments": vars(document).get("completed_segments"),+ "total_segments": vars(document).get("total_segments"),+ **document_response_values(document, batch),+ }+ for document in items+ ]+++def load_segment_detail(segment: DocumentSegment, summary: str | None, *, session: Session) -> dict[str, Any]:+ """Load relations and authorize file links before handing plain values to the response model."""+ child_chunks = get_segment_child_chunks(segment, session=session, include_full_doc=False)+ return {+ "id": segment.id,+ "position": segment.position,+ "document_id": segment.document_id,+ "content": segment.content,+ "sign_content": sign_segment_content(segment, session=session),+ "answer": segment.answer,+ "word_count": segment.word_count,+ "tokens": segment.tokens,+ "keywords": segment.keywords,+ "index_node_id": segment.index_node_id,+ "index_node_hash": segment.index_node_hash,+ "hit_count": segment.hit_count,+ "enabled": segment.enabled,+ "disabled_at": segment.disabled_at,+ "disabled_by": segment.disabled_by,+ "status": segment.status,+ "created_by": segment.created_by,+ "created_at": segment.created_at,+ "updated_at": segment.updated_at,+ "updated_by": segment.updated_by,+ "indexing_at": segment.indexing_at,+ "completed_at": segment.completed_at,+ "error": segment.error,+ "stopped_at": segment.stopped_at,+ "child_chunks": [+ {+ "id": chunk.id,+ "segment_id": chunk.segment_id,+ "content": chunk.content,+ "position": chunk.position,+ "word_count": chunk.word_count,+ "type": chunk.type,+ "created_at": chunk.created_at,+ "updated_at": chunk.updated_at,+ }+ for chunk in child_chunks+ ],+ "attachments": get_segment_attachments(segment, session=session),+ "summary": summary,+ }+++def load_segment_details(+ segments: Iterable[DocumentSegment], summaries: Mapping[str, str | None], *, session: Session+) -> list[dict[str, Any]]:+ return [load_segment_detail(segment, summaries.get(segment.id), session=session) for segment in segments]+++def format_document_source_detail(document: Document, file: UploadFile | None) -> dict[str, Any]:+ if document.data_source_type == "upload_file" and file is not None:+ return {+ "upload_file": {+ "id": file.id,+ "name": file.name,+ "size": file.size,+ "extension": file.extension,+ "mime_type": file.mime_type,+ "created_by": file.created_by,+ "created_at": file.created_at.timestamp(),+ }+ }+ if document.data_source_type in {"notion_import", "website_crawl"}:+ return document.data_source_info_dict+ return {}+++def format_document_metadata(+ document: Document, metadatas: Sequence[DatasetMetadata], *, built_in_enabled: bool, uploader_name: str | None+) -> list[DocMetadataDetailItem] | None:+ values = document.doc_metadata or {}+ details: list[DocMetadataDetailItem] = [+ {"id": metadata.id, "name": metadata.name, "type": metadata.type, "value": values.get(metadata.name)}+ for metadata in metadatas+ ]+ if built_in_enabled:+ details.extend(format_document_built_in_fields(document, uploader_name=uploader_name))+ return details or None+++def document_response_values(document: Document, batch: DocumentReadBatch) -> dict[str, Any]:+ """Build response-only values from a loaded snapshot without database access."""+ file_id = document.data_source_info_dict.get("upload_file_id")+ process_rule = batch.process_rules.get(document.dataset_process_rule_id)+ if process_rule is not None and process_rule.dataset_id != document.dataset_id:+ process_rule = None+ return {+ "data_source_detail_dict": format_document_source_detail(+ document, batch.uploads.get((document.tenant_id, file_id)) if isinstance(file_id, str) else None+ ),+ "hit_count": batch.hit_counts.get(document.id, 0),+ "doc_metadata_details": format_document_metadata(+ document,+ batch.metadatas.get(document.id, []),+ built_in_enabled=batch.built_in_enabled.get((document.tenant_id, document.dataset_id), False),+ uploader_name=batch.uploader_names.get(document.created_by),+ ),+ "process_rule_dict": process_rule.to_dict() if process_rule else None,+ }+++def get_document_source_detail(document: Document, *, session: Session) -> dict[str, Any]:+ file = get_document_upload_file(document, session=session) if document.data_source_type == "upload_file" else None+ return format_document_source_detail(document, file)+++def get_document_metadata_details(document: Document, *, session: Session) -> list[DocMetadataDetailItem] | None:+ dataset = get_document_dataset(document, session=session)+ built_in_enabled = bool(dataset is not None and dataset.built_in_field_enabled)+ return format_document_metadata(+ document,+ get_document_metadata_rows(document, session=session) if document.doc_metadata else [],+ built_in_enabled=built_in_enabled,+ uploader_name=get_document_uploader(document, session=session) if built_in_enabled else None,+ )+++def get_document_built_in_fields(document: Document, *, session: Session) -> list[DocMetadataDetailItem]:+ return format_document_built_in_fields(document, uploader_name=get_document_uploader(document, session=session))+++def format_document_built_in_fields(document: Document, *, uploader_name: str | None) -> list[DocMetadataDetailItem]:+ built_in_fields: list[DocMetadataDetailItem] = []+ built_in_fields.append(+ {"id": "built-in", "name": BuiltInField.document_name, "type": "string", "value": document.name}+ )+ built_in_fields.append({"id": "built-in", "name": BuiltInField.uploader, "type": "string", "value": uploader_name})+ built_in_fields.append(+ {+ "id": "built-in",+ "name": BuiltInField.upload_date,+ "type": "time",+ "value": str(document.created_at.timestamp()),+ }+ )+ built_in_fields.append(+ {+ "id": "built-in",+ "name": BuiltInField.last_update_date,+ "type": "time",+ "value": str(document.updated_at.timestamp()),+ }+ )+ built_in_fields.append(+ {+ "id": "built-in",+ "name": BuiltInField.source,+ "type": "string",+ "value": MetadataDataSource[document.data_source_type],+ }+ )+ return built_in_fields+++def get_dataset_queries(query: DatasetQuery, *, session: Session) -> list[dict[str, Any]]:+ """Read structured audit content, preserving legacy plain-text queries."""+ try:+ decoded = json.loads(query.content)+ except (ValueError, TypeError):+ decoded = None+ raw_records = decoded if isinstance(decoded, list) else [decoded]+ if not raw_records or not all(+ isinstance(item, dict)+ and item.get("content_type") in {QueryType.TEXT_QUERY, QueryType.IMAGE_QUERY}+ and isinstance(item.get("content"), str)+ for item in raw_records+ ):+ return [{"content_type": QueryType.TEXT_QUERY, "content": query.content, "file_info": None}]+ records = cast(list[dict[str, Any]], raw_records)+ files = get_query_upload_files(+ query, {item["content"] for item in records if item["content_type"] == QueryType.IMAGE_QUERY}, session=session+ )+ result: list[dict[str, Any]] = []+ for item in records:+ record: dict[str, Any] = {"content_type": item["content_type"], "content": item["content"], "file_info": None}+ file = files.get(item["content"]) if item["content_type"] == QueryType.IMAGE_QUERY else None+ if file is not None:+ record["file_info"] = {+ "id": file.id,+ "name": file.name,+ "size": file.size,+ "extension": file.extension,+ "mime_type": file.mime_type,+ "source_url": sign_upload_file_preview_url(file.id, file.extension),+ }+ result.append(record)+ return resultdiff --git a/api/services/dataset_service.py b/api/services/knowledge/dataset_service.pysimilarity index 74%rename from api/services/dataset_service.pyrename to api/services/knowledge/dataset_service.pyindex 2c6628f6c43..65e5152fd64 100644--- a/api/services/dataset_service.py+++ b/api/services/knowledge/dataset_service.py@@ -5,12 +5,12 @@ import loggingimport secretsimport timeimport uuid-from collections import Counterfrom collections.abc import Sequence-from typing import Annotated, Any, Literal, TypedDict, cast+from contextlib import nullcontext+from dataclasses import replace+from typing import Any, Literal, TypedDict, castimport sqlalchemy as sa-from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validatorfrom redis.exceptions import LockNotOwnedErrorfrom sqlalchemy import ColumnElement, case, delete, exists, func, select, tuple_, updatefrom sqlalchemy.orm import Session@@ -49,7 +49,6 @@ from models.dataset import (DocumentSegment,ExternalKnowledgeBindings,Pipeline,- SegmentAttachmentBinding,)from models.enums import (DatasetRuntimeMode,@@ -58,25 +57,18 @@ from models.enums import (IndexingStatus,ProcessRuleMode,SegmentStatus,- SegmentType,)from models.model import UploadFilefrom models.provider_ids import ModelProviderIDfrom models.source import DataSourceOauthBindingfrom models.workflow import Workflowfrom repositories.knowledge import dataset_api_key_bindings-from services.dataset_ref_service import DatasetRef, DatasetRefService, SegmentRef+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, get_latest_dataset_process_rule+from repositories.knowledge.segment_repository import query_child_chunksfrom services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxyfrom services.document_indexing_proxy.duplicate_document_indexing_task_proxy import DuplicateDocumentIndexingTaskProxyfrom services.enterprise import rbac_service as enterprise_rbac_servicefrom services.entities.feature_entities import FeatureModel-from services.entities.knowledge_entities.knowledge_entities import (- ChildChunkUpdateArgs,- KnowledgeConfig,- RerankingModel,- RetrievalModel,- SegmentUpdateArgs,-)from services.entities.knowledge_entities.rag_pipeline_entities import (KnowledgeConfiguration,RagPipelineDatasetCreateEntity,@@ -86,23 +78,35 @@ from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingfrom services.errors.dataset import DatasetNameDuplicateErrorfrom services.errors.document import DocumentIndexingErrorfrom services.errors.file import FileNotExistsError-from services.external_knowledge_service import ExternalDatasetServicefrom services.feature_service import FeatureServicefrom services.file_service import FileService-from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessSnapshot, can_access_dataset+from services.knowledge.entities.document_creation import DocumentCreationDataset, DocumentIndexingJobs+from services.knowledge.entities.knowledge_entities import (+ KnowledgeConfig,+ RerankingModel,+ RetrievalModel,+)+from services.knowledge.entities.segments import SegmentUpdateArgs+from services.knowledge.external.service import ExternalDatasetService+from services.knowledge.resource_scope import DatasetRef, SegmentRef+from services.knowledge.segments.application import (+ ChildChunkDeleteIndexApplicationError,+ ChildChunkIndexingApplicationError,+ SegmentDatasetRecord,+ SegmentDocumentRecord,+ SegmentMutationScope,+ SegmentMutationService,+ validate_segment_values,+)from services.rag_pipeline.rag_pipeline import RagPipelineServicefrom services.tag_application_service import TagTargetQuery-from services.vector_service import VectorServicefrom tasks.add_document_to_index_task import add_document_to_index_taskfrom tasks.batch_clean_document_task import batch_clean_document_taskfrom tasks.clean_notion_document_task import clean_notion_document_taskfrom tasks.deal_dataset_index_update_task import deal_dataset_index_update_taskfrom tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task-from tasks.delete_segment_from_index_task import delete_segment_from_index_task-from tasks.disable_segment_from_index_task import disable_segment_from_index_task-from tasks.disable_segments_from_index_task import disable_segments_from_index_taskfrom tasks.document_indexing_update_task import document_indexing_update_task-from tasks.enable_segments_to_index_task import enable_segments_to_index_taskfrom tasks.recover_document_indexing_task import recover_document_indexing_taskfrom tasks.regenerate_summary_index_task import regenerate_summary_index_taskfrom tasks.remove_document_from_index_task import remove_document_from_index_task@@ -122,121 +126,6 @@ class AutoDisableLogsDict(TypedDict):count: int-class _EstimatePreProcessingRule(BaseModel):- id: str = Field(min_length=1)- enabled: bool-- @field_validator("id")- @classmethod- def _validate_id(cls, v: str) -> str:- if v not in DatasetProcessRule.PRE_PROCESSING_RULES:- raise ValueError("Process rule pre_processing_rules id is invalid")- return v---class _EstimateSegmentation(BaseModel):- separator: str = Field(min_length=1)- max_tokens: int = Field(gt=0)---class _EstimateRules(BaseModel):- pre_processing_rules: list[_EstimatePreProcessingRule]- segmentation: _EstimateSegmentation-- @field_validator("pre_processing_rules")- @classmethod- def _deduplicate(cls, v: list[_EstimatePreProcessingRule]) -> list[_EstimatePreProcessingRule]:- seen: dict[str, _EstimatePreProcessingRule] = {}- for rule in v:- seen[rule.id] = rule- return list(seen.values())---class _EstimateHierarchicalRules(_EstimateRules):- parent_mode: Literal["full-doc", "paragraph"] | None = None- subchunk_segmentation: _EstimateSegmentation | None = None---class _SummaryIndexSettingDisabled(BaseModel):- enable: Literal[False] = False---class _SummaryIndexSettingEnabled(BaseModel):- enable: Literal[True]- model_name: str = Field(min_length=1)- model_provider_name: str = Field(min_length=1)---_SummaryIndexSetting = Annotated[- _SummaryIndexSettingDisabled | _SummaryIndexSettingEnabled,- Field(discriminator="enable"),-]---class _AutomaticProcessRule(BaseModel):- model_config = ConfigDict(extra="allow")-- mode: Literal[ProcessRuleMode.AUTOMATIC]- summary_index_setting: _SummaryIndexSetting | None = None-- @field_validator("summary_index_setting", mode="before")- @classmethod- def _normalize_summary_index_setting(cls, v: Any) -> Any:- """Treat dicts with enable=None (or missing enable) as None (#36602)."""- if v is None:- return None- if isinstance(v, dict) and v.get("enable") is None:- return None- return v---class _CustomProcessRule(BaseModel):- model_config = ConfigDict(extra="allow")-- mode: Literal[ProcessRuleMode.CUSTOM]- rules: _EstimateRules- summary_index_setting: _SummaryIndexSetting | None = None-- @field_validator("summary_index_setting", mode="before")- @classmethod- def _normalize_summary_index_setting(cls, v: Any) -> Any:- """Treat dicts with enable=None (or missing enable) as None (#36602)."""- if v is None:- return None- if isinstance(v, dict) and v.get("enable") is None:- return None- return v---class _HierarchicalProcessRule(BaseModel):- model_config = ConfigDict(extra="allow")-- mode: Literal[ProcessRuleMode.HIERARCHICAL]- rules: _EstimateHierarchicalRules- summary_index_setting: _SummaryIndexSetting | None = None-- @field_validator("summary_index_setting", mode="before")- @classmethod- def _normalize_summary_index_setting(cls, v: Any) -> Any:- """Treat dicts with enable=None (or missing enable) as None (#36602)."""- if v is None:- return None- if isinstance(v, dict) and v.get("enable") is None:- return None- return v---_EstimateProcessRule = Annotated[- _AutomaticProcessRule | _CustomProcessRule | _HierarchicalProcessRule,- Field(discriminator="mode"),-]---class _EstimateArgs(BaseModel):- info_list: dict[str, Any]- process_rule: _EstimateProcessRule--class DatasetService:@staticmethoddef _can_manage_all_datasets(tenant_id: str, account_id: str, *, session: Session) -> bool:@@ -561,7 +450,7 @@ class DatasetService:@staticmethoddef check_doc_form(dataset: Dataset, doc_form: str, *, session: Session):- dataset_doc_form = dataset.get_doc_form(session=session)+ dataset_doc_form = get_dataset_doc_form(dataset, session=session)if dataset_doc_form and doc_form != dataset_doc_form:raise ValueError("doc_form is different from the dataset doc_form.")@@ -1403,8 +1292,16 @@ class DatasetService:)is not None)- access = DatasetAccess(dataset.permission, dataset.maintainer, user.current_role, has_permission)- if not access.allows(user.id):+ snapshot = DatasetAccessSnapshot(+ dataset=DatasetAccessRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ maintainer_id=dataset.maintainer,+ permission=dataset.permission,+ ),+ actor_has_partial_access=has_permission,+ )+ if not can_access_dataset(snapshot, actor_id=user.id, workspace_role=user.current_role):logger.debug("User %s does not have permission to access dataset %s", user.id, dataset.id)raise NoPermissionError("You do not have permission to access this dataset.")@@ -1775,9 +1672,9 @@ class DocumentService:# Calculate summary_index_status for documents that need summary (only if dataset summary index is enabled)summary_status_map: dict[str, str | None] = {}if has_summary_index and document_ids_need_summary:- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdapter- summary_status_map = SummaryIndexService.get_documents_summary_index_status(+ summary_status_map = SummaryIndexAdapter.get_documents_summary_index_status(document_ids=document_ids_need_summary,dataset_id=dataset.id,tenant_id=tenant_id,@@ -2122,7 +2019,7 @@ class DocumentService:return document@staticmethod- def pause_document(document, session: Session):+ def pause_document(document, session: Session, *, actor_id: str):if document.indexing_status not in {IndexingStatus.WAITING,IndexingStatus.PARSING,@@ -2132,9 +2029,8 @@ class DocumentService:}:raise DocumentIndexingError()# update document to be paused- assert current_user is not Nonedocument.is_paused = True- document.paused_by = current_user.id+ document.paused_by = actor_iddocument.paused_at = naive_utc_now()session.add(document)@@ -2161,13 +2057,13 @@ class DocumentService:recover_document_indexing_task.delay(document.dataset_id, document.id)@staticmethod- def retry_document(dataset_id: str, documents: list[Document], session: Session):+ def retry_document(dataset_id: str, documents: list[Document], session: Session, *, actor_id: str):"""Reserve the whole retry batch before changing any document state.Redis lock acquisition is intentionally coupled to this bounded statustransaction so a concurrent request cannot partially admit the batch."""- if not current_user or not current_user.id:+ if not actor_id:raise ValueError("Current user or current user id not found")unique_documents = list({document.id: document for document in documents}.values())@@ -2202,12 +2098,11 @@ class DocumentService:raisedocument_ids = [document.id for document in unique_documents]- retry_document_indexing_task.delay(dataset_id, document_ids, current_user.id)+ retry_document_indexing_task.delay(dataset_id, document_ids, actor_id)@staticmethoddef sync_website_document(dataset: Dataset, document: Document, session: Session):- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- if DatasetRefService.create_document_ref(dataset_ref, document) is None:+ if document.tenant_id != dataset.tenant_id or document.dataset_id != dataset.id:raise ValueError("Document not found.")# add sync flag@@ -2248,14 +2143,47 @@ class DocumentService:*,session: Session,) -> tuple[list[Document], str]:- # check doc_form+ # Compatibility entry point for existing callers that own their session.DatasetService.check_doc_form(dataset, knowledge_config.doc_form, session=session)- # check document limit- assert isinstance(current_user, Account)- assert current_user.current_tenant_id is not None+ assert isinstance(account, Account)+ assert account.current_tenant_id is not None+ config = DocumentService.prepare_document_creation(+ DocumentCreationDataset(+ dataset.tenant_id, dataset.indexing_technique, dataset.embedding_model, dataset.embedding_model_provider+ ),+ knowledge_config,+ )+ documents: list[Document] = []+ batch = ""+ lock = (+ nullcontext()+ if config.original_document_id+ else redis_client.lock(f"add_document_lock_dataset_id_{dataset.id}", timeout=600)+ )+ try:+ with lock:+ documents, batch, jobs = DocumentService.save_prepared_documents(+ dataset, config, account, dataset_process_rule, created_from, session=session+ )+ DocumentService.dispatch_document_indexing(jobs)+ except LockNotOwnedError:+ if config.original_document_id:+ raise+ if not batch:+ batch = DocumentService.generate_document_batch()+ return documents, batch++ @staticmethod+ def generate_document_batch() -> str:+ return time.strftime("%Y%m%d%H%M%S") + str(100000 + secrets.randbelow(exclusive_upper_bound=900000))+ @staticmethod+ def prepare_document_creation(+ dataset: DocumentCreationDataset, knowledge_config: KnowledgeConfig+ ) -> KnowledgeConfig:+ """Resolve remote quotas and models from values; new callers close their read transaction first."""if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:- features = FeatureService.get_features(current_user.current_tenant_id, exclude_vector_space=True)+ features = FeatureService.get_features(dataset.tenant_id, exclude_vector_space=True)if not knowledge_config.original_document_id:count = 0if knowledge_config.data_source:@@ -2274,6 +2202,59 @@ class DocumentService:count = len(website_info.urls)DocumentService.check_document_creation_limits(count, features)+ config = knowledge_config+ effective_dataset = dataset+ if not dataset.indexing_technique:+ if config.indexing_technique not in Dataset.INDEXING_TECHNIQUE_LIST:+ raise ValueError("Indexing technique is invalid")+ if config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ if not config.embedding_model or not config.embedding_model_provider:+ model = ModelManager.for_tenant(tenant_id=dataset.tenant_id).get_default_model_instance(+ tenant_id=dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING+ )+ config = config.model_copy(+ update={"embedding_model": model.model_name, "embedding_model_provider": model.provider}+ )+ effective_dataset = replace(+ dataset,+ indexing_technique=config.indexing_technique,+ embedding_model=config.embedding_model,+ embedding_model_provider=config.embedding_model_provider,+ )+ if config.original_document_id:+ DatasetService.check_dataset_model_setting(effective_dataset)+ return config++ @staticmethod+ def dispatch_document_indexing(jobs: DocumentIndexingJobs) -> None:+ """Publish only identifiers after the corresponding database writes commit."""+ ref = jobs.dataset+ if jobs.removed_notion:+ clean_notion_document_task.delay(list(jobs.removed_notion), ref.dataset_id)+ if jobs.created:+ DocumentIndexingTaskProxy(ref.tenant_id, ref.dataset_id, list(jobs.created)).delay()+ if jobs.duplicated:+ DuplicateDocumentIndexingTaskProxy(ref.tenant_id, ref.dataset_id, list(jobs.duplicated)).delay()+ if jobs.updated:+ document_indexing_update_task.delay(ref.dataset_id, jobs.updated)++ @staticmethod+ def save_prepared_documents(+ dataset: Dataset,+ knowledge_config: KnowledgeConfig,+ account: Account | Any,+ dataset_process_rule: DatasetProcessRule | None = None,+ created_from: str = DocumentCreatedFrom.WEB,+ *,+ session: Session,+ ) -> tuple[list[Document], str, DocumentIndexingJobs]:+ """Persist prepared input without remote calls, Redis locks, or task dispatch.++ The caller must resolve quotas/models with prepare_document_creation and+ hold the dataset's creation lock for new documents. ORM values stay within+ this persistence adapter; publish the returned jobs after closing its session.+ """+ DatasetService.check_doc_form(dataset, knowledge_config.doc_form, session=session)# if dataset is empty, update dataset data_source_typeif not dataset.data_source_type and knowledge_config.data_source:dataset.data_source_type = knowledge_config.data_source.info_list.data_source_type@@ -2284,16 +2265,10 @@ class DocumentService:dataset.indexing_technique = IndexTechniqueType(knowledge_config.indexing_technique)if knowledge_config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id)- if knowledge_config.embedding_model and knowledge_config.embedding_model_provider:- dataset_embedding_model = knowledge_config.embedding_model- dataset_embedding_model_provider = knowledge_config.embedding_model_provider- else:- embedding_model = model_manager.get_default_model_instance(- tenant_id=current_user.current_tenant_id, model_type=ModelType.TEXT_EMBEDDING- )- dataset_embedding_model = embedding_model.model_name- dataset_embedding_model_provider = embedding_model.provider+ assert knowledge_config.embedding_model+ assert knowledge_config.embedding_model_provider+ dataset_embedding_model = knowledge_config.embedding_model+ dataset_embedding_model_provider = knowledge_config.embedding_model_providerdataset.embedding_model = dataset_embedding_modeldataset.embedding_model_provider = dataset_embedding_model_providerdataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding(@@ -2315,19 +2290,19 @@ class DocumentService:else default_retrieval_model)- documents = []+ documents: list[Document] = []+ jobs = DocumentIndexingJobs(DatasetRef(dataset.tenant_id, dataset.id))if knowledge_config.original_document_id:- document = DocumentService.update_document_with_dataset_id(- dataset, knowledge_config, account, session=session- )+ document = DocumentService.update_prepared_document(dataset, knowledge_config, account, session=session)documents.append(document)batch = document.batch+ jobs = replace(jobs, updated=document.id)else:# When creating new documents, data_source must be providedif not knowledge_config.data_source:raise ValueError("Data source is required when creating new documents")- batch = time.strftime("%Y%m%d%H%M%S") + str(100000 + secrets.randbelow(exclusive_upper_bound=900000))+ batch = DocumentService.generate_document_batch()# save process ruleif not dataset_process_rule:process_rule = knowledge_config.process_rule@@ -2341,7 +2316,7 @@ class DocumentService:created_by=account.id,)else:- dataset_process_rule = dataset.get_latest_process_rule(session=session)+ dataset_process_rule = get_latest_dataset_process_rule(dataset, session=session)if not dataset_process_rule:raise ValueError("No process rule found.")elif process_rule.mode == ProcessRuleMode.AUTOMATIC:@@ -2356,14 +2331,14 @@ class DocumentService:"Invalid process rule mode: %s, can not find dataset process rule",process_rule.mode,)- return [], ""+ return [], "", jobssession.add(dataset_process_rule)session.flush()else:# Fallback when no process_rule provided in knowledge_config:# 1) reuse the dataset's latest process rule if present# 2) otherwise create an automatic rule- dataset_process_rule = dataset.get_latest_process_rule(session=session)+ dataset_process_rule = get_latest_dataset_process_rule(dataset, session=session)if not dataset_process_rule:dataset_process_rule = DatasetProcessRule(dataset_id=dataset.id,@@ -2373,170 +2348,127 @@ class DocumentService:)session.add(dataset_process_rule)session.flush()- lock_name = f"add_document_lock_dataset_id_{dataset.id}"- try:- with redis_client.lock(lock_name, timeout=600):- assert dataset_process_rule- position = DocumentService.get_documents_position(dataset.id, session)- document_ids = []- duplicate_document_ids = []- if knowledge_config.data_source.info_list.data_source_type == "upload_file":- if not knowledge_config.data_source.info_list.file_info_list:- raise ValueError("File source info is required")- upload_file_list = knowledge_config.data_source.info_list.file_info_list.file_ids- # Issue #41735: under MySQL's default- # ``REPEATABLE READ`` isolation, the SELECT snapshot- # for this transaction is pinned by the first- # read done earlier in ``save_document_with_dataset_id``- # (e.g. ``check_doc_form``). The caller has just- # committed the upload in a different session- # (``FileService.upload_text`` /- # ``FileService.upload_file``), so the rows exist- # but are invisible to *this* session until its- # snapshot is refreshed. End the current transaction- # so the next SELECT starts with a fresh snapshot.- # The pending changes on ``dataset`` are also- # flushed, which is what we want — they must be- # persisted by this point anyway. PostgreSQL's- # default ``READ COMMITTED`` rebuilds the read view- # per statement so it isn't affected.- session.commit()- files = list(- session.scalars(- select(UploadFile).where(- UploadFile.tenant_id == dataset.tenant_id,- UploadFile.id.in_(upload_file_list),- )- ).all()+ assert dataset_process_rule+ position = DocumentService.get_documents_position(dataset.id, session)+ document_ids = []+ duplicate_document_ids = []+ removed_notion_ids: list[str] = []+ if knowledge_config.data_source.info_list.data_source_type == "upload_file":+ if not knowledge_config.data_source.info_list.file_info_list:+ raise ValueError("File source info is required")+ upload_file_list = knowledge_config.data_source.info_list.file_info_list.file_ids+ # Issue #41735: under MySQL's default+ # ``REPEATABLE READ`` isolation, the SELECT snapshot+ # for this transaction is pinned by the first+ # read done earlier in ``save_document_with_dataset_id``+ # (e.g. ``check_doc_form``). The caller has just+ # committed the upload in a different session+ # (``FileService.upload_text`` /+ # ``FileService.upload_file``), so the rows exist+ # but are invisible to *this* session until its+ # snapshot is refreshed. End the current transaction+ # so the next SELECT starts with a fresh snapshot.+ # The pending changes on ``dataset`` are also+ # flushed, which is what we want — they must be+ # persisted by this point anyway. PostgreSQL's+ # default ``READ COMMITTED`` rebuilds the read view+ # per statement so it isn't affected.+ session.commit()+ files = list(+ session.scalars(+ select(UploadFile).where(+ UploadFile.tenant_id == dataset.tenant_id,+ UploadFile.id.in_(upload_file_list),)- if len(files) != len(set(upload_file_list)):- raise FileNotExistsError("One or more files not found.")-- file_names = [file.name for file in files]- db_documents = list(- session.scalars(- select(Document).where(- Document.dataset_id == dataset.id,- Document.tenant_id == current_user.current_tenant_id,- Document.data_source_type == DataSourceType.UPLOAD_FILE,- Document.enabled == True,- Document.name.in_(file_names),- )- ).all()+ ).all()+ )+ if len(files) != len(set(upload_file_list)):+ raise FileNotExistsError("One or more files not found.")++ file_names = [file.name for file in files]+ db_documents = list(+ session.scalars(+ select(Document).where(+ Document.dataset_id == dataset.id,+ Document.tenant_id == account.current_tenant_id,+ Document.data_source_type == DataSourceType.UPLOAD_FILE,+ Document.enabled == True,+ Document.name.in_(file_names),)- documents_map = {document.name: document for document in db_documents}- for file in files:- data_source_info: dict[str, object] = {- "upload_file_id": file.id,- }- document = documents_map.get(file.name)- if knowledge_config.duplicate and document:- document.dataset_process_rule_id = dataset_process_rule.id- document.updated_at = naive_utc_now()- document.created_from = created_from- document.doc_form = IndexStructureType(knowledge_config.doc_form)- document.doc_language = knowledge_config.doc_language- document.data_source_info = json.dumps(data_source_info)- document.batch = batch- document.indexing_status = IndexingStatus.WAITING- session.add(document)- documents.append(document)- duplicate_document_ids.append(document.id)- continue- else:- document = DocumentService.build_document(- dataset,- dataset_process_rule.id,- knowledge_config.data_source.info_list.data_source_type,- knowledge_config.doc_form,- knowledge_config.doc_language,- data_source_info,- created_from,- position,- account,- file.name,- batch,- )- session.add(document)- session.flush()- document_ids.append(document.id)- documents.append(document)- position += 1- elif knowledge_config.data_source.info_list.data_source_type == "notion_import":- notion_info_list = knowledge_config.data_source.info_list.notion_info_list # type: ignore- if not notion_info_list:- raise ValueError("No notion info list found.")- exist_page_ids = []- exist_document = {}- documents = list(- session.scalars(- select(Document).where(- Document.dataset_id == dataset.id,- Document.tenant_id == current_user.current_tenant_id,- Document.data_source_type == DataSourceType.NOTION_IMPORT,- Document.enabled == True,- )- ).all()+ ).all()+ )+ documents_map = {document.name: document for document in db_documents}+ for file in files:+ data_source_info: dict[str, object] = {+ "upload_file_id": file.id,+ }+ document = documents_map.get(file.name)+ if knowledge_config.duplicate and document:+ document.dataset_process_rule_id = dataset_process_rule.id+ document.updated_at = naive_utc_now()+ document.created_from = created_from+ document.doc_form = IndexStructureType(knowledge_config.doc_form)+ document.doc_language = knowledge_config.doc_language+ document.data_source_info = json.dumps(data_source_info)+ document.batch = batch+ document.indexing_status = IndexingStatus.WAITING+ session.add(document)+ documents.append(document)+ duplicate_document_ids.append(document.id)+ continue+ else:+ document = DocumentService.build_document(+ dataset,+ dataset_process_rule.id,+ knowledge_config.data_source.info_list.data_source_type,+ knowledge_config.doc_form,+ knowledge_config.doc_language,+ data_source_info,+ created_from,+ position,+ account,+ file.name,+ batch,)- if documents:- for document in documents:- data_source_info = json.loads(document.data_source_info)- exist_page_ids.append(data_source_info["notion_page_id"])- exist_document[data_source_info["notion_page_id"]] = document.id- for notion_info in notion_info_list:- workspace_id = notion_info.workspace_id- for page in notion_info.pages:- if page.page_id not in exist_page_ids:- data_source_info = {- "credential_id": notion_info.credential_id,- "notion_workspace_id": workspace_id,- "notion_page_id": page.page_id,- "notion_page_icon": page.page_icon.model_dump() if page.page_icon else None, # type: ignore- "type": page.type,- }- # Truncate page name to 255 characters to prevent DB field length errors- truncated_page_name = page.page_name[:255] if page.page_name else "nopagename"- document = DocumentService.build_document(- dataset,- dataset_process_rule.id,- knowledge_config.data_source.info_list.data_source_type,- knowledge_config.doc_form,- knowledge_config.doc_language,- data_source_info,- created_from,- position,- account,- truncated_page_name,- batch,- )- document.id = str(uuid.uuid4())- session.add(document)- document_ids.append(document.id)- documents.append(document)- position += 1- else:- exist_document.pop(page.page_id)+ session.add(document)session.flush()- # delete not selected documents- if len(exist_document) > 0:- clean_notion_document_task.delay(list(exist_document.values()), dataset.id)- elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":- website_info = knowledge_config.data_source.info_list.website_info_list- if not website_info:- raise ValueError("No website info list found.")- urls = website_info.urls- for url in urls:+ document_ids.append(document.id)+ documents.append(document)+ position += 1+ elif knowledge_config.data_source.info_list.data_source_type == "notion_import":+ notion_info_list = knowledge_config.data_source.info_list.notion_info_list # type: ignore+ if not notion_info_list:+ raise ValueError("No notion info list found.")+ exist_page_ids = []+ exist_document = {}+ documents = list(+ session.scalars(+ select(Document).where(+ Document.dataset_id == dataset.id,+ Document.tenant_id == account.current_tenant_id,+ Document.data_source_type == DataSourceType.NOTION_IMPORT,+ Document.enabled == True,+ )+ ).all()+ )+ if documents:+ for document in documents:+ data_source_info = json.loads(document.data_source_info)+ exist_page_ids.append(data_source_info["notion_page_id"])+ exist_document[data_source_info["notion_page_id"]] = document.id+ for notion_info in notion_info_list:+ workspace_id = notion_info.workspace_id+ for page in notion_info.pages:+ if page.page_id not in exist_page_ids:data_source_info = {- "url": url,- "provider": website_info.provider,- "job_id": website_info.job_id,- "only_main_content": website_info.only_main_content,- "mode": "crawl",+ "credential_id": notion_info.credential_id,+ "notion_workspace_id": workspace_id,+ "notion_page_id": page.page_id,+ "notion_page_icon": page.page_icon.model_dump() if page.page_icon else None, # type: ignore+ "type": page.type,}- if len(url) > 255:- document_name = url[:200] + "..."- else:- document_name = url+ # Truncate page name to 255 characters to prevent DB field length errors+ truncated_page_name = page.page_name[:255] if page.page_name else "nopagename"document = DocumentService.build_document(dataset,dataset_process_rule.id,@@ -2547,29 +2479,64 @@ class DocumentService:created_from,position,account,- document_name,+ truncated_page_name,batch,)+ document.id = str(uuid.uuid4())session.add(document)- session.flush()document_ids.append(document.id)documents.append(document)position += 1- session.commit()-- # trigger async task- if document_ids:- DocumentIndexingTaskProxy(dataset.tenant_id, dataset.id, document_ids).delay()- if duplicate_document_ids:- DuplicateDocumentIndexingTaskProxy(- dataset.tenant_id, dataset.id, duplicate_document_ids- ).delay()- # Note: Summary index generation is triggered in document_indexing_task after indexing completes- # to ensure segments are available. See tasks/document_indexing_task.py- except LockNotOwnedError:- pass+ else:+ exist_document.pop(page.page_id)+ session.flush()+ # delete not selected documents+ if len(exist_document) > 0:+ removed_notion_ids = list(exist_document.values())+ elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":+ website_info = knowledge_config.data_source.info_list.website_info_list+ if not website_info:+ raise ValueError("No website info list found.")+ urls = website_info.urls+ for url in urls:+ data_source_info = {+ "url": url,+ "provider": website_info.provider,+ "job_id": website_info.job_id,+ "only_main_content": website_info.only_main_content,+ "mode": "crawl",+ }+ if len(url) > 255:+ document_name = url[:200] + "..."+ else:+ document_name = url+ document = DocumentService.build_document(+ dataset,+ dataset_process_rule.id,+ knowledge_config.data_source.info_list.data_source_type,+ knowledge_config.doc_form,+ knowledge_config.doc_language,+ data_source_info,+ created_from,+ position,+ account,+ document_name,+ batch,+ )+ session.add(document)+ session.flush()+ document_ids.append(document.id)+ documents.append(document)+ position += 1+ session.commit()- return documents, batch+ jobs = replace(+ jobs,+ created=tuple(document_ids),+ duplicated=tuple(duplicate_document_ids),+ removed_notion=tuple(removed_notion_ids),+ )+ return documents, batch, jobs# @staticmethod# def save_document_with_dataset_id(@@ -2945,9 +2912,27 @@ class DocumentService:*,session: Session,):- assert isinstance(current_user, Account)-DatasetService.check_dataset_model_setting(dataset)+ document = DocumentService.update_prepared_document(+ dataset, document_data, account, dataset_process_rule, created_from, session=session+ )+ DocumentService.dispatch_document_indexing(+ DocumentIndexingJobs(DatasetRef(dataset.tenant_id, dataset.id), updated=document.id)+ )+ return document++ @staticmethod+ def update_prepared_document(+ dataset: Dataset,+ document_data: KnowledgeConfig,+ account: Account,+ dataset_process_rule: DatasetProcessRule | None = None,+ created_from: str = DocumentCreatedFrom.WEB,+ *,+ session: Session,+ ):+ assert isinstance(account, Account)+document = DocumentService.get_document(dataset.id, document_data.original_document_id, session=session)if document is None:raise NotFound("Document not found")@@ -3007,7 +2992,7 @@ class DocumentService:select(DataSourceOauthBinding).where(sa.and_(- DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,+ DataSourceOauthBinding.tenant_id == account.current_tenant_id,DataSourceOauthBinding.provider == "notion",DataSourceOauthBinding.disabled == False,DataSourceOauthBinding.source_info["workspace_id"] == f'"{workspace_id}"',@@ -3064,20 +3049,18 @@ class DocumentService:.values(status=SegmentStatus.RE_SEGMENT))session.commit()- # trigger async task- document_indexing_update_task.delay(document.dataset_id, document.id)return document@staticmethoddef save_document_without_dataset_id(tenant_id: str, knowledge_config: KnowledgeConfig, account: Account, session: Session):- assert isinstance(current_user, Account)- assert current_user.current_tenant_id is not None+ assert isinstance(account, Account)+ assert account.current_tenant_id is not Noneassert knowledge_config.data_sourceif dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:- features = FeatureService.get_features(current_user.current_tenant_id, exclude_vector_space=True)+ features = FeatureService.get_features(account.current_tenant_id, exclude_vector_space=True)count = 0if knowledge_config.data_source.info_list.data_source_type == "upload_file":upload_file_list = (@@ -3097,6 +3080,28 @@ class DocumentService:count = len(website_info.urls)DocumentService.check_document_creation_limits(count, features)+ dataset = DocumentService.build_dataset_for_documents(+ DatasetRef(tenant_id, str(uuid.uuid4())), knowledge_config, account, session+ )++ documents, batch = DocumentService.save_document_with_dataset_id(+ dataset, knowledge_config, account, session=session+ )++ cut_length = 18+ cut_name = documents[0].name[:cut_length]+ dataset.name = cut_name + "..."+ dataset.description = "useful for when you want to answer queries about the " + documents[0].name+ session.flush()++ return dataset, documents, batch++ @staticmethod+ def build_dataset_for_documents(+ ref: DatasetRef, knowledge_config: KnowledgeConfig, account: Account, session: Session+ ) -> Dataset:+ """Create a dataset inside the caller's SQL-only persistence phase."""+ assert knowledge_config.data_sourcedataset_collection_binding_id = Noneretrieval_model = Noneif knowledge_config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:@@ -3120,7 +3125,8 @@ class DocumentService:)# save datasetdataset = Dataset(- tenant_id=tenant_id,+ id=ref.dataset_id,+ tenant_id=ref.tenant_id,name="",data_source_type=knowledge_config.data_source.info_list.data_source_type,indexing_technique=IndexTechniqueType(knowledge_config.indexing_technique),@@ -3137,17 +3143,7 @@ class DocumentService:session.add(dataset)session.flush()- documents, batch = DocumentService.save_document_with_dataset_id(- dataset, knowledge_config, account, session=session- )-- cut_length = 18- cut_name = documents[0].name[:cut_length]- dataset.name = cut_name + "..."- dataset.description = "useful for when you want to answer queries about the " + documents[0].name- session.flush()-- return dataset, documents, batch+ return dataset@classmethoddef document_create_args_validate(cls, knowledge_config: KnowledgeConfig):@@ -3235,23 +3231,6 @@ class DocumentService:if not isinstance(knowledge_config.process_rule.rules.segmentation.max_tokens, int):raise ValueError("Process rule segmentation max_tokens is invalid")- @classmethod- def estimate_args_validate(cls, args: dict[str, Any]):- try:- validated = _EstimateArgs.model_validate(args)- except ValidationError as e:- first = e.errors()[0]- original = first.get("ctx", {}).get("error")- raise ValueError(str(original) if isinstance(original, ValueError) else first["msg"]) from e- process_rule_dict = validated.process_rule.model_dump(exclude_none=True)- if validated.process_rule.mode == ProcessRuleMode.AUTOMATIC:- process_rule_dict["rules"] = {}- elif validated.process_rule.mode == ProcessRuleMode.HIERARCHICAL:- rules = process_rule_dict.get("rules")- if isinstance(rules, dict) and not rules.get("parent_mode"):- rules["parent_mode"] = "paragraph"- args["process_rule"] = process_rule_dict-@staticmethoddef batch_update_document_status(dataset: Dataset,@@ -3448,147 +3427,125 @@ class DocumentService:return update_info-class SegmentService:- @classmethod- def segment_create_args_validate(cls, args: dict[str, Any], document: Document):- if document.doc_form == IndexStructureType.QA_INDEX:- if "answer" not in args or not args["answer"]:- raise ValueError("Answer is required")- if not args["answer"].strip():- raise ValueError("Answer is empty")- if "content" not in args or not args["content"] or not args["content"].strip():- raise ValueError("Content is empty")-- if args.get("attachment_ids"):- if not isinstance(args["attachment_ids"], list):- raise ValueError("Attachment IDs is invalid")- single_chunk_attachment_limit = dify_config.SINGLE_CHUNK_ATTACHMENT_LIMIT- if len(args["attachment_ids"]) > single_chunk_attachment_limit:- raise ValueError(f"Exceeded maximum attachment limit of {single_chunk_attachment_limit}")+def _segment_mutation_scope(dataset: Dataset, document: Document) -> SegmentMutationScope:+ """Materialize already-authorized rows before ending their read transaction."""+ if document.tenant_id != dataset.tenant_id or document.dataset_id != dataset.id:+ raise ValueError("Document does not belong to the dataset")+ return SegmentMutationScope(+ dataset=SegmentDatasetRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ indexing_technique=dataset.indexing_technique or "economy",+ embedding_model_provider=dataset.embedding_model_provider,+ embedding_model=dataset.embedding_model,+ ),+ document=SegmentDocumentRecord(+ id=document.id,+ dataset_id=document.dataset_id,+ workspace_id=document.tenant_id,+ doc_form=document.doc_form,+ ),+ )- @classmethod- def create_segment(cls, args: dict[str, Any], document: Document, dataset: Dataset, session: Session):- assert isinstance(current_user, Account)- assert current_user.current_tenant_id is not None- content = args["content"]- doc_id = str(uuid.uuid4())- segment_hash = helper.generate_text_hash(content)- tokens = 0- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id)- embedding_model = model_manager.get_model_instance(- tenant_id=current_user.current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- # calc embedding use tokens- tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]- lock_name = f"add_segment_lock_document_id_{document.id}"- try:- with redis_client.lock(lock_name, timeout=600):- max_position = session.scalar(- select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == document.id)- )- segment_document = DocumentSegment(- tenant_id=current_user.current_tenant_id,- dataset_id=document.dataset_id,- document_id=document.id,- index_node_id=doc_id,- index_node_hash=segment_hash,- position=max_position + 1 if max_position else 1,- content=content,- word_count=len(content),- tokens=tokens,- status=SegmentStatus.COMPLETED,- indexing_at=naive_utc_now(),- completed_at=naive_utc_now(),- created_by=current_user.id,- )- if document.doc_form == IndexStructureType.QA_INDEX:- segment_document.word_count += len(args["answer"])- segment_document.answer = args["answer"]-- session.add(segment_document)- # update document word count- assert document.word_count is not None- document.word_count += segment_document.word_count- session.add(document)- session.commit()+def _segment_ref_for_mutation(scope: SegmentMutationScope, segment: DocumentSegment) -> SegmentRef:+ document_ref = scope.document.ref+ if (+ segment.tenant_id != document_ref.dataset.tenant_id+ or segment.dataset_id != document_ref.dataset.dataset_id+ or segment.document_id != document_ref.document_id+ ):+ raise ValueError("Segment does not belong to the document")+ return document_ref.segment(segment.id)- if args["attachment_ids"]:- for attachment_id in args["attachment_ids"]:- binding = SegmentAttachmentBinding(- tenant_id=current_user.current_tenant_id,- dataset_id=document.dataset_id,- document_id=document.id,- segment_id=segment_document.id,- attachment_id=attachment_id,- )- session.add(binding)- session.commit()- # save vector index- try:- keywords = args.get("keywords")- keywords_list = [keywords] if keywords is not None else None- VectorService.create_segments_vector(- keywords_list, [segment_document], dataset, document.doc_form, session=session- )- except Exception as e:- logger.exception("create segment index failed")- segment_document.enabled = False- segment_document.disabled_at = naive_utc_now()- segment_document.status = SegmentStatus.ERROR- segment_document.error = str(e)- session.commit()- segment = session.get(DocumentSegment, segment_document.id)- return segment- except LockNotOwnedError:- pass+def _check_child_chunk_scope(child: ChildChunk, segment_ref: SegmentRef) -> None:+ if (+ child.tenant_id != segment_ref.document.dataset.tenant_id+ or child.dataset_id != segment_ref.document.dataset.dataset_id+ or child.document_id != segment_ref.document.document_id+ or child.segment_id != segment_ref.segment_id+ ):+ raise ValueError("Child chunk does not belong to the segment")++class SegmentService:@classmethod- def multi_create_segment(cls, segments: list, document: Document, dataset: Dataset, session: Session):+ def segment_create_args_validate(cls, args: dict[str, Any], document: Document):+ validate_segment_values(+ args, doc_form=document.doc_form, attachment_limit=dify_config.SINGLE_CHUNK_ATTACHMENT_LIMIT+ )++ @classmethod+ def multi_create_segment(+ cls,+ segments: list,+ document: Document,+ dataset: Dataset,+ session: Session,+ *,+ mutations: SegmentMutationService,+ ):+ """Commit a Service API batch before dispatching its scoped index update."""assert isinstance(current_user, Account)assert current_user.current_tenant_id is not None-- lock_name = f"multi_add_segment_lock_document_id_{document.id}"+ scope = _segment_mutation_scope(dataset, document)+ if current_user.current_tenant_id != scope.dataset.workspace_id:+ raise ValueError("Dataset does not belong to the current workspace")+ actor_id = current_user.id+ document_ref = scope.document.ref+ session.commit()+ lock_name = f"multi_add_segment_lock_document_id_{document_ref.document_id}"increment_word_count = 0try:with redis_client.lock(lock_name, timeout=600):embedding_model = None- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id)+ if scope.dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ model_manager = ModelManager.for_tenant(tenant_id=scope.dataset.workspace_id)embedding_model = model_manager.get_model_instance(- tenant_id=current_user.current_tenant_id,- provider=dataset.embedding_model_provider,+ tenant_id=scope.dataset.workspace_id,+ provider=scope.dataset.embedding_model_provider or "",model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,+ model=scope.dataset.embedding_model or "",)+ token_counts = [+ embedding_model.get_text_embedding_num_tokens(+ texts=[+ item["content"] + item["answer"]+ if scope.document.doc_form == IndexStructureType.QA_INDEX+ else item["content"]+ ]+ )[0]+ if embedding_model+ else 0+ for item in segments+ ]+ persisted_document = session.scalar(+ select(Document).where(+ Document.id == document_ref.document_id,+ Document.dataset_id == document_ref.dataset.dataset_id,+ Document.tenant_id == document_ref.dataset.tenant_id,+ )+ )+ if persisted_document is None:+ raise ValueError("Document no longer exists")+ document = persisted_documentmax_position = session.scalar(- select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == document.id)+ select(func.max(DocumentSegment.position)).where(+ DocumentSegment.tenant_id == document_ref.dataset.tenant_id,+ DocumentSegment.dataset_id == document_ref.dataset.dataset_id,+ DocumentSegment.document_id == document_ref.document_id,+ ))- pre_segment_data_list = []segment_data_list = []keywords_list = []position = max_position + 1 if max_position else 1- for segment_item in segments:+ for segment_item, tokens in zip(segments, token_counts, strict=True):content = segment_item["content"]doc_id = str(uuid.uuid4())segment_hash = helper.generate_text_hash(content)- tokens = 0- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY and embedding_model:- # calc embedding use tokens- if document.doc_form == IndexStructureType.QA_INDEX:- tokens = embedding_model.get_text_embedding_num_tokens(- texts=[content + segment_item["answer"]]- )[0]- else:- tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]-segment_document = DocumentSegment(- tenant_id=current_user.current_tenant_id,+ tenant_id=document_ref.dataset.tenant_id,dataset_id=document.dataset_id,document_id=document.id,index_node_id=doc_id,@@ -3601,7 +3558,7 @@ class SegmentService:status=SegmentStatus.COMPLETED,indexing_at=naive_utc_now(),completed_at=naive_utc_now(),- created_by=current_user.id,+ created_by=actor_id,)if document.doc_form == IndexStructureType.QA_INDEX:segment_document.answer = segment_item["answer"]@@ -3611,7 +3568,6 @@ class SegmentService:segment_data_list.append(segment_document)position += 1- pre_segment_data_list.append(segment_document)if "keywords" in segment_item:keywords_list.append(segment_item["keywords"])else:@@ -3620,19 +3576,16 @@ class SegmentService:assert document.word_count is not Nonedocument.word_count += increment_word_countsession.add(document)+ session.flush()+ segment_ids = [segment.id for segment in segment_data_list]+ session.commit()try:- # save vector index- VectorService.create_segments_vector(- keywords_list, pre_segment_data_list, dataset, document.doc_form, session=session- )- except Exception as e:+ mutations.index_segments(document_ref, segment_ids=segment_ids, keywords_list=keywords_list)+ except Exception:logger.exception("create segment index failed")- for segment_document in segment_data_list:- segment_document.enabled = False- segment_document.disabled_at = naive_utc_now()- segment_document.status = SegmentStatus.ERROR- segment_document.error = str(e)- session.commit()+ session.expire_all()+ for segment in segment_data_list:+ session.refresh(segment)return segment_data_listexcept LockNotOwnedError:pass@@ -3645,446 +3598,37 @@ class SegmentService:document: Document,dataset: Dataset,session: Session,- ):- assert isinstance(current_user, Account)- assert current_user.current_tenant_id is not None-- indexing_cache_key = f"segment_{segment.id}_indexing"- cache_result = redis_client.get(indexing_cache_key)- if cache_result is not None:- raise ValueError("Segment is indexing, please try again later")- if args.enabled is not None:- action = args.enabled- if segment.enabled != action:- if not action:- segment.enabled = action- segment.disabled_at = naive_utc_now()- segment.disabled_by = current_user.id- session.add(segment)- session.commit()- # Set cache to prevent indexing the same segment multiple times- redis_client.setex(indexing_cache_key, 600, 1)- disable_segment_from_index_task.delay(segment.id)- return segment- if not segment.enabled:- if args.enabled is not None:- if not args.enabled:- raise ValueError("Can't update disabled segment")- else:- raise ValueError("Can't update disabled segment")- try:- word_count_change = segment.word_count- content = args.content or segment.content- if segment.content == content:- segment.word_count = len(content)- if document.doc_form == IndexStructureType.QA_INDEX:- segment.answer = args.answer- segment.word_count += len(args.answer) if args.answer else 0- word_count_change = segment.word_count - word_count_change- keyword_changed = False- if args.keywords:- if Counter(segment.keywords) != Counter(args.keywords):- segment.keywords = args.keywords- keyword_changed = True- segment.enabled = True- segment.disabled_at = None- segment.disabled_by = None- session.add(segment)- session.commit()- # update document word count- if word_count_change != 0:- assert document.word_count is not None- document.word_count = max(0, document.word_count + word_count_change)- session.add(document)- # update segment index task- if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX and args.regenerate_child_chunks:- # regenerate child chunks- # get embedding model instance- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # check embedding model setting- model_manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)-- if dataset.embedding_model_provider:- embedding_model_instance = model_manager.get_model_instance(- tenant_id=dataset.tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- else:- embedding_model_instance = model_manager.get_default_model_instance(- tenant_id=dataset.tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )- else:- raise ValueError("The knowledge base index technique is not high quality!")- # get the process rule- processing_rule = session.get(DatasetProcessRule, document.dataset_process_rule_id)- if processing_rule:- VectorService.generate_child_chunks(- segment, document, dataset, embedding_model_instance, processing_rule, True, session=session- )- elif document.doc_form in (IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.QA_INDEX):- if args.enabled or keyword_changed:- # update segment vector index- VectorService.update_segment_vector(args.keywords, segment, dataset, session=session)- # update summary index if summary is provided and has changed- if args.summary is not None:- # When user manually provides summary, allow saving even if summary_index_setting doesn't exist- # summary_index_setting is only needed for LLM generation, not for manual summary vectorization- # Vectorization uses dataset.embedding_model, which doesn't require summary_index_setting- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # Query existing summary from database- from models.dataset import DocumentSegmentSummary-- existing_summary = session.scalar(- select(DocumentSegmentSummary)- .where(- DocumentSegmentSummary.chunk_id == segment.id,- DocumentSegmentSummary.dataset_id == dataset.id,- )- .limit(1)- )-- # Check if summary has changed- existing_summary_content = existing_summary.summary_content if existing_summary else None- if existing_summary_content != args.summary:- # Summary has changed, update it- from services.summary_index_service import SummaryIndexService-- try:- SummaryIndexService.update_summary_for_segment(- segment,- dataset,- args.summary,- session=session,- )- except Exception:- logger.exception("Failed to update summary for segment %s", segment.id)- # Don't fail the entire update if summary update fails- else:- segment_hash = helper.generate_text_hash(content)- tokens = 0- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id)- embedding_model = model_manager.get_model_instance(- tenant_id=current_user.current_tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )-- # calc embedding use tokens- if document.doc_form == IndexStructureType.QA_INDEX:- segment.answer = args.answer- tokens = embedding_model.get_text_embedding_num_tokens(texts=[content + segment.answer])[0] # type: ignore- else:- tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]- segment.content = content- segment.index_node_hash = segment_hash- segment.word_count = len(content)- segment.tokens = tokens- segment.status = SegmentStatus.COMPLETED- segment.indexing_at = naive_utc_now()- segment.completed_at = naive_utc_now()- segment.updated_by = current_user.id- segment.updated_at = naive_utc_now()- segment.enabled = True- segment.disabled_at = None- segment.disabled_by = None- if document.doc_form == IndexStructureType.QA_INDEX:- segment.answer = args.answer- segment.word_count += len(args.answer) if args.answer else 0- word_count_change = segment.word_count - word_count_change- # update document word count- if word_count_change != 0:- assert document.word_count is not None- document.word_count = max(0, document.word_count + word_count_change)- session.add(document)- session.add(segment)- session.commit()- if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX and args.regenerate_child_chunks:- # get embedding model instance- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # check embedding model setting- model_manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)-- if dataset.embedding_model_provider:- embedding_model_instance = model_manager.get_model_instance(- tenant_id=dataset.tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- else:- embedding_model_instance = model_manager.get_default_model_instance(- tenant_id=dataset.tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )- else:- raise ValueError("The knowledge base index technique is not high quality!")- # get the process rule- processing_rule = session.get(DatasetProcessRule, document.dataset_process_rule_id)- if processing_rule:- VectorService.generate_child_chunks(- segment, document, dataset, embedding_model_instance, processing_rule, True, session=session- )- elif document.doc_form in (IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.QA_INDEX):- # update segment vector index- VectorService.update_segment_vector(args.keywords, segment, dataset, session=session)- # Handle summary index when content changed- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- from models.dataset import DocumentSegmentSummary-- existing_summary = session.scalar(- select(DocumentSegmentSummary)- .where(- DocumentSegmentSummary.chunk_id == segment.id,- DocumentSegmentSummary.dataset_id == dataset.id,- )- .limit(1)- )-- if args.summary is None:- # User didn't provide summary, auto-regenerate if segment previously had summary- # Auto-regeneration only happens if summary_index_setting exists and enable is True- if (- existing_summary- and dataset.summary_index_setting- and dataset.summary_index_setting.get("enable") is True- ):- # Segment previously had summary, regenerate it with new content- from services.summary_index_service import SummaryIndexService-- try:- SummaryIndexService.generate_and_vectorize_summary(- segment,- dataset,- dataset.summary_index_setting,- session=session,- )- logger.info("Auto-regenerated summary for segment %s after content change", segment.id)- except Exception:- logger.exception("Failed to auto-regenerate summary for segment %s", segment.id)- # Don't fail the entire update if summary regeneration fails- else:- # User provided summary, check if it has changed- # Manual summary updates are allowed even if summary_index_setting doesn't exist- existing_summary_content = existing_summary.summary_content if existing_summary else None- if existing_summary_content != args.summary:- # Summary has changed, use user-provided summary- from services.summary_index_service import SummaryIndexService-- try:- SummaryIndexService.update_summary_for_segment(- segment,- dataset,- args.summary,- session=session,- )- logger.info("Updated summary for segment %s with user-provided content", segment.id)- except Exception:- logger.exception("Failed to update summary for segment %s", segment.id)- # Don't fail the entire update if summary update fails- else:- # Summary hasn't changed, regenerate based on new content- # Auto-regeneration only happens if summary_index_setting exists and enable is True- if (- existing_summary- and dataset.summary_index_setting- and dataset.summary_index_setting.get("enable") is True- ):- from services.summary_index_service import SummaryIndexService-- try:- SummaryIndexService.generate_and_vectorize_summary(- segment,- dataset,- dataset.summary_index_setting,- session=session,- )- logger.info(- "Regenerated summary for segment %s after content change (summary unchanged)",- segment.id,- )- except Exception:- logger.exception("Failed to regenerate summary for segment %s", segment.id)- # Don't fail the entire update if summary regeneration fails- # update multimodel vector index- if args.attachment_ids is not None:- VectorService.update_multimodel_vector(segment, args.attachment_ids, dataset, session=session)- except Exception as e:- logger.exception("update segment index failed")- segment.enabled = False- segment.disabled_at = naive_utc_now()- segment.status = SegmentStatus.ERROR- segment.error = str(e)- session.commit()- new_segment = session.get(DocumentSegment, segment.id)- if not new_segment:- raise ValueError("new_segment is not found")- return new_segment-- @classmethod- def delete_segment(cls, segment: DocumentSegment, document: Document, dataset: Dataset, session: Session):- indexing_cache_key = f"segment_{segment.id}_delete_indexing"- cache_result = redis_client.get(indexing_cache_key)- if cache_result is not None:- raise ValueError("Segment is deleting.")-- # enabled segment need to delete index- if segment.enabled:- # send delete segment index task- redis_client.setex(indexing_cache_key, 600, 1)-- # Get child chunk IDs before parent segment is deleted- child_node_ids = []- if segment.index_node_id:- child_node_ids = list(- session.scalars(- select(ChildChunk.index_node_id).where(- ChildChunk.segment_id == segment.id,- ChildChunk.dataset_id == dataset.id,- )- ).all()- )-- delete_segment_from_index_task.delay(- [segment.index_node_id], dataset.id, document.id, [segment.id], child_node_ids- )-- session.delete(segment)- # update document word count- assert document.word_count is not None- document.word_count -= segment.word_count- session.add(document)- session.commit()-- @classmethod- def delete_segments(cls, segment_ids: list, document: Document, dataset: Dataset, session: Session):- assert current_user is not None- # Check if segment_ids is not empty to avoid WHERE false condition- if not segment_ids or len(segment_ids) == 0:- return- segments_info = session.execute(- select(DocumentSegment.index_node_id, DocumentSegment.id, DocumentSegment.word_count).where(- DocumentSegment.id.in_(segment_ids),- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == document.id,- DocumentSegment.tenant_id == current_user.current_tenant_id,- )- ).all()-- if not segments_info:- return-- index_node_ids = [info[0] for info in segments_info]- segment_db_ids = [info[1] for info in segments_info]- total_words = sum(info[2] for info in segments_info if info[2] is not None)-- # Get child chunk IDs before parent segments are deleted- child_node_ids = []- if index_node_ids:- child_node_ids = [- nid- for nid in session.scalars(- select(ChildChunk.index_node_id).where(- ChildChunk.segment_id.in_(segment_db_ids),- ChildChunk.dataset_id == dataset.id,- )- ).all()- if nid- ]-- # Start async cleanup with both parent and child node IDs- if index_node_ids or child_node_ids:- delete_segment_from_index_task.delay(- index_node_ids, dataset.id, document.id, segment_db_ids, child_node_ids- )-- if document.word_count is None:- document.word_count = 0- else:- document.word_count = max(0, document.word_count - total_words)-- session.add(document)-- # Delete database records- session.execute(- delete(DocumentSegment).where(- DocumentSegment.id.in_(segment_db_ids),- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == document.id,- DocumentSegment.tenant_id == current_user.current_tenant_id,- )- )+ *,+ actor_id: str,+ mutations: SegmentMutationService,+ ) -> DocumentSegment:+ """Delegate an authorized Service API patch to the shared mutation owner."""+ scope = _segment_mutation_scope(dataset, document)+ segment_ref = _segment_ref_for_mutation(scope, segment)+ # Finish the caller's read transaction before the mutation owner performs+ # external indexing I/O in between its own bounded write transactions.session.commit()+ mutations.update_segment(scope, actor_id=actor_id, segment_id=segment_ref.segment_id, args=args)+ session.expire_all()+ updated = cls.get_segment_by_ref(segment_ref, session=session)+ if updated is None:+ raise ValueError("new_segment is not found")+ return updated@classmethod- def update_segments_status(+ def delete_segment(cls,- segment_ids: list,- action: Literal["enable", "disable"],- dataset: Dataset,+ segment: DocumentSegment,document: Document,+ dataset: Dataset,session: Session,- ):- assert current_user is not None-- # Check if segment_ids is not empty to avoid WHERE false condition- if not segment_ids or len(segment_ids) == 0:- return- match action:- case "enable":- segments = session.scalars(- select(DocumentSegment).where(- DocumentSegment.id.in_(segment_ids),- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == document.id,- DocumentSegment.enabled == False,- )- ).all()- if not segments:- return- real_deal_segment_ids = []- for segment in segments:- indexing_cache_key = f"segment_{segment.id}_indexing"- cache_result = redis_client.get(indexing_cache_key)- if cache_result is not None:- continue- segment.enabled = True- segment.disabled_at = None- segment.disabled_by = None- session.add(segment)- real_deal_segment_ids.append(segment.id)- session.commit()-- enable_segments_to_index_task.delay(real_deal_segment_ids, dataset.id, document.id)- case "disable":- segments = session.scalars(- select(DocumentSegment).where(- DocumentSegment.id.in_(segment_ids),- DocumentSegment.dataset_id == dataset.id,- DocumentSegment.document_id == document.id,- DocumentSegment.enabled == True,- )- ).all()- if not segments:- return- real_deal_segment_ids = []- for segment in segments:- indexing_cache_key = f"segment_{segment.id}_indexing"- cache_result = redis_client.get(indexing_cache_key)- if cache_result is not None:- continue- segment.enabled = False- segment.disabled_at = naive_utc_now()- segment.disabled_by = current_user.id- session.add(segment)- real_deal_segment_ids.append(segment.id)- session.commit()-- disable_segments_from_index_task.delay(real_deal_segment_ids, dataset.id, document.id)+ *,+ mutations: SegmentMutationService,+ ) -> None:+ segment_ref = _segment_ref_for_mutation(_segment_mutation_scope(dataset, document), segment)+ session.commit()+ mutations.delete_segment(segment_ref)+ session.expire_all()@classmethoddef create_child_chunk(@@ -4094,126 +3638,21 @@ class SegmentService:document: Document,dataset: Dataset,session: Session,+ *,+ actor_id: str,+ mutations: SegmentMutationService,) -> ChildChunk:- assert isinstance(current_user, Account)-- lock_name = f"add_child_lock_{segment.id}"- with redis_client.lock(lock_name, timeout=20):- index_node_id = str(uuid.uuid4())- index_node_hash = helper.generate_text_hash(content)- max_position = session.scalar(- select(func.max(ChildChunk.position)).where(- ChildChunk.tenant_id == current_user.current_tenant_id,- ChildChunk.dataset_id == dataset.id,- ChildChunk.document_id == document.id,- ChildChunk.segment_id == segment.id,- )- )- assert current_user.current_tenant_id- child_chunk = ChildChunk(- tenant_id=current_user.current_tenant_id,- dataset_id=dataset.id,- document_id=document.id,- segment_id=segment.id,- position=max_position + 1 if max_position else 1,- index_node_id=index_node_id,- index_node_hash=index_node_hash,- content=content,- word_count=len(content),- type=SegmentType.CUSTOMIZED,- created_by=current_user.id,- )- session.add(child_chunk)- # save vector index- try:- VectorService.create_child_chunk_vector(child_chunk, dataset, session=session)- except Exception as e:- logger.exception("create child chunk index failed")- session.rollback()- raise ChildChunkIndexingError(str(e))- session.commit()-- return child_chunk-- @classmethod- def update_child_chunks(- cls,- child_chunks_update_args: list[ChildChunkUpdateArgs],- segment: DocumentSegment,- document: Document,- dataset: Dataset,- session: Session,- ) -> list[ChildChunk]:- assert isinstance(current_user, Account)- child_chunks = session.scalars(- select(ChildChunk).where(- ChildChunk.dataset_id == dataset.id,- ChildChunk.document_id == document.id,- ChildChunk.segment_id == segment.id,- )- ).all()- child_chunks_map = {chunk.id: chunk for chunk in child_chunks}-- new_child_chunks, update_child_chunks, delete_child_chunks, new_child_chunks_args = [], [], [], []-- for child_chunk_update_args in child_chunks_update_args:- if child_chunk_update_args.id:- child_chunk = child_chunks_map.pop(child_chunk_update_args.id, None)- if child_chunk:- if child_chunk.content != child_chunk_update_args.content:- child_chunk.content = child_chunk_update_args.content- child_chunk.word_count = len(child_chunk.content)- child_chunk.updated_by = current_user.id- child_chunk.updated_at = naive_utc_now()- child_chunk.type = SegmentType.CUSTOMIZED- update_child_chunks.append(child_chunk)- else:- new_child_chunks_args.append(child_chunk_update_args)- if child_chunks_map:- delete_child_chunks = list(child_chunks_map.values())+ segment_ref = _segment_ref_for_mutation(_segment_mutation_scope(dataset, document), segment)+ session.commit()try:- if update_child_chunks:- session.bulk_save_objects(update_child_chunks)-- if delete_child_chunks:- for child_chunk in delete_child_chunks:- session.delete(child_chunk)- if new_child_chunks_args:- child_chunk_count = len(child_chunks)- for position, args in enumerate(new_child_chunks_args, start=child_chunk_count + 1):- assert current_user.current_tenant_id- index_node_id = str(uuid.uuid4())- index_node_hash = helper.generate_text_hash(args.content)- child_chunk = ChildChunk(- tenant_id=current_user.current_tenant_id,- dataset_id=dataset.id,- document_id=document.id,- segment_id=segment.id,- position=position,- index_node_id=index_node_id,- index_node_hash=index_node_hash,- content=args.content,- word_count=len(args.content),- type=SegmentType.CUSTOMIZED,- created_by=current_user.id,- )-- session.add(child_chunk)- session.flush()- new_child_chunks.append(child_chunk)- VectorService.update_child_chunk_vector(- new_child_chunks,- update_child_chunks,- delete_child_chunks,- dataset,- session=session,- )- session.commit()- except Exception as e:- logger.exception("update child chunk index failed")- session.rollback()- raise ChildChunkIndexingError(str(e))- return sorted(new_child_chunks + update_child_chunks, key=lambda x: x.position)+ created = mutations.create_child_chunk(segment_ref, content=content, actor_id=actor_id)+ except ChildChunkIndexingApplicationError as error:+ raise ChildChunkIndexingError(str(error)) from error+ session.expire_all()+ child = cls.get_child_chunk_by_segment_ref(created.id, segment_ref, session=session)+ if child is None:+ raise ValueError("Child chunk no longer exists")+ return child@classmethoddef update_child_chunk(@@ -4224,34 +3663,46 @@ class SegmentService:document: Document,dataset: Dataset,session: Session,+ *,+ actor_id: str,+ mutations: SegmentMutationService,) -> ChildChunk:- assert current_user is not None-+ segment_ref = _segment_ref_for_mutation(_segment_mutation_scope(dataset, document), segment)+ _check_child_chunk_scope(child_chunk, segment_ref)+ child_id = child_chunk.id+ session.commit()try:- child_chunk.content = content- child_chunk.word_count = len(content)- child_chunk.updated_by = current_user.id- child_chunk.updated_at = naive_utc_now()- child_chunk.type = SegmentType.CUSTOMIZED- session.add(child_chunk)- VectorService.update_child_chunk_vector([], [child_chunk], [], dataset, session=session)- session.commit()- except Exception as e:- logger.exception("update child chunk index failed")- session.rollback()- raise ChildChunkIndexingError(str(e))- return child_chunk+ mutations.update_child_chunk(segment_ref, child_chunk_id=child_id, content=content, actor_id=actor_id)+ except ChildChunkIndexingApplicationError as error:+ raise ChildChunkIndexingError(str(error)) from error+ session.expire_all()+ updated = cls.get_child_chunk_by_segment_ref(child_id, segment_ref, session=session)+ if updated is None:+ raise ValueError("Child chunk no longer exists")+ return updated@classmethod- def delete_child_chunk(cls, child_chunk: ChildChunk, dataset: Dataset, session: Session):- session.delete(child_chunk)- try:- VectorService.delete_child_chunk_vector(child_chunk, dataset, session=session)- except Exception as e:- logger.exception("delete child chunk index failed")- session.rollback()- raise ChildChunkDeleteIndexError(str(e))+ def delete_child_chunk(+ cls,+ child_chunk: ChildChunk,+ dataset: Dataset,+ session: Session,+ *,+ mutations: SegmentMutationService,+ ) -> None:+ segment_ref = (+ DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ .document(child_chunk.document_id)+ .segment(child_chunk.segment_id)+ )+ _check_child_chunk_scope(child_chunk, segment_ref)+ child_id = child_chunk.idsession.commit()+ try:+ mutations.delete_child_chunk(segment_ref, child_chunk_id=child_id)+ except ChildChunkDeleteIndexApplicationError as error:+ raise ChildChunkDeleteIndexError(str(error)) from error+ session.expire_all()@classmethoddef get_child_chunks(@@ -4267,20 +3718,14 @@ class SegmentService:):assert isinstance(current_user, Account)- query = (- select(ChildChunk)- .filter_by(- tenant_id=current_user.current_tenant_id,- dataset_id=dataset_id,- document_id=document_id,- segment_id=segment_id,- )- .order_by(ChildChunk.position.asc())+ assert current_user.current_tenant_id is not None+ return query_child_chunks(+ session,+ DatasetRef(current_user.current_tenant_id, dataset_id).document(document_id).segment(segment_id),+ page=page,+ limit=limit,+ keyword=keyword,)- if keyword:- escaped_keyword = helper.escape_like_pattern(keyword)- query = query.where(ChildChunk.content.ilike(f"%{escaped_keyword}%", escape="\\"))- return paginate_query(query, session=session, page=page, per_page=limit, max_per_page=100)@classmethoddef get_child_chunk_by_id(cls, child_chunk_id: str, tenant_id: str, session: Session) -> ChildChunk | None:@@ -4371,13 +3816,16 @@ class SegmentService:session: Session,status: str | None = None,enabled: bool | None = None,+ *,+ tenant_id: str,) -> Sequence[DocumentSegment]:"""- Get segments for a document in a dataset with optional filtering.+ Get segments through their tenant, dataset, and document owner chain.Args:document_id: Document IDdataset_id: Dataset ID+ tenant_id: Owning tenant IDstatus: Optional status filter (e.g., "completed")enabled: Optional enabled filter (True/False)@@ -4385,6 +3833,7 @@ class SegmentService:Sequence of DocumentSegment instances"""query = select(DocumentSegment).where(+ DocumentSegment.tenant_id == tenant_id,DocumentSegment.document_id == document_id,DocumentSegment.dataset_id == dataset_id,)diff --git a/api/services/knowledge/datasets/__init__.py b/api/services/knowledge/datasets/__init__.pynew file mode 100644index 00000000000..697c750e8a8--- /dev/null+++ b/api/services/knowledge/datasets/__init__.py@@ -0,0 +1 @@+"""Dataset administration use cases and infrastructure adapters."""diff --git a/api/services/knowledge/datasets/adapters.py b/api/services/knowledge/datasets/adapters.pynew file mode 100644index 00000000000..592b43ed1c4--- /dev/null+++ b/api/services/knowledge/datasets/adapters.py@@ -0,0 +1,328 @@+"""Session-owning dataset administration adapters."""++from collections.abc import Generator, Mapping+from contextlib import contextmanager+from typing import Any++from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from configs import dify_config+from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager+from graphon.model_runtime.entities.model_entities import ModelType+from libs.pagination import clamp_pagination+from machinery.context import RequestContext+from models import Account, Dataset, Document+from models.dataset import DatasetPermission, DatasetPermissionEnum+from models.provider_ids import ModelProviderID+from repositories.knowledge.dataset_repository import _get_dataset+from services.app.query_service import AppQueryService+from services.enterprise import rbac_service+from services.errors.account import NoPermissionError+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.dataset_read_service import get_dataset_queries, load_dataset_detail, load_dataset_details+from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService+from services.knowledge.datasets.application import (+ DatasetListFilter,+ DatasetVisibility,+)+from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage+from services.knowledge.resource_scope import DatasetRef+from services.tag_application_service import TagTargetQuery+++@contextmanager+def _translate_permissions() -> Generator[None, None, None]:+ try:+ yield+ except NoPermissionError as error:+ raise DatasetAccessDeniedError() from error+++def require_dataset(session: Session, ref: DatasetRef) -> Dataset:+ dataset = _get_dataset(session, ref)+ if dataset is None:+ raise DatasetNotFoundError()+ return dataset+++def load_actor(session: Session, context: RequestContext) -> Account:+ account = session.get(Account, context.account_id)+ if account is None:+ raise DatasetAccessDeniedError()+ account.set_tenant_id_with_session(context.active_workspace_id, session=session)+ return account+++def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:+ if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:+ item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))+ return item+++def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:+ result = {+ "id": document.id,+ "indexing_status": document.indexing_status,+ "processing_started_at": document.processing_started_at,+ "parsing_completed_at": document.parsing_completed_at,+ "cleaning_completed_at": document.cleaning_completed_at,+ "splitting_completed_at": document.splitting_completed_at,+ "completed_at": document.completed_at,+ "paused_at": document.paused_at,+ "error": document.error,+ "stopped_at": document.stopped_at,+ }+ if counts is not None:+ result.update(completed_segments=counts[0], total_segments=counts[1])+ return result+++class SQLAlchemyDatasetOperations:+ def __init__(+ self, *, session_factory: sessionmaker[Session], tags: TagTargetQuery, app_queries: AppQueryService+ ) -> None:+ self._sessions = session_factory+ self._tags = tags+ self._app_queries = app_queries++ def visibility(self, context: RequestContext) -> DatasetVisibility:+ with self._sessions() as session:+ permissions = rbac_service.RBACService.MyPermissions.get(+ context.active_workspace_id, context.account_id, session=session+ )+ scope = (+ rbac_service.RBACService.DatasetAccess.whitelist_resources(context.active_workspace_id, context.account_id)+ if dify_config.RBAC_ENABLED+ else None+ )+ return DatasetVisibility(+ default_permissions=permissions.dataset.default_permission_keys,+ workspace_permissions=permissions.workspace.permission_keys,+ overrides={item.resource_id: item.permission_keys for item in permissions.dataset.overrides},+ unrestricted=scope is None or scope.unrestricted,+ whitelist_ids=scope.resource_ids if scope is not None else (),+ )++ def embedding_models(self, workspace_id: str) -> set[str]:+ manager = create_plugin_provider_manager(tenant_id=workspace_id)+ models = manager.get_configurations(tenant_id=workspace_id).get_models(+ model_type=ModelType.TEXT_EMBEDDING, only_active=True+ )+ return {f"{model.model}:{model.provider.provider}" for model in models}++ def list_datasets(+ self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None, include_own: bool+ ) -> DatasetPage:+ page, limit = clamp_pagination(query.page, query.limit, 100)+ with self._sessions() as session:+ account = load_actor(session, context)+ if query.ids:+ datasets, total = DatasetService.get_datasets_by_ids(+ list(query.ids),+ context.active_workspace_id,+ user=account,+ accessible_dataset_ids=accessible_ids,+ include_own_datasets=include_own,+ session=session,+ )+ else:+ datasets, total = DatasetService.get_datasets(+ page,+ limit,+ session,+ context.active_workspace_id,+ account,+ query.keyword,+ list(query.tag_ids),+ query.include_all,+ tags=self._tags,+ accessible_dataset_ids=accessible_ids,+ include_own_datasets=include_own,+ )+ data = load_dataset_details(datasets, session=session)+ partial_ids = [item["id"] for item in data if item["permission"] == "partial_members"]+ members: dict[str, list[str]] = {}+ if partial_ids:+ for dataset_id, account_id in session.execute(+ select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(+ DatasetPermission.dataset_id.in_(partial_ids),+ DatasetPermission.tenant_id == context.active_workspace_id,+ )+ ):+ members.setdefault(dataset_id, []).append(account_id)+ for item in data:+ _normalize_provider(item)+ item["partial_member_list"] = members.get(item["id"], [])+ return {+ "data": data,+ "has_more": not query.ids and page * limit < total,+ "total": total,+ "page": page,+ "limit": limit,+ }++ @staticmethod+ def _detail(session: Session, context: RequestContext, dataset: Dataset) -> DatasetDetailRecord:+ result = load_dataset_detail(dataset, session=session)+ permissions = rbac_service.RBACService.DatasetPermissions.batch_get(+ context.active_workspace_id, context.account_id, [dataset.id], session=session+ )+ result["permission_keys"] = permissions.get(dataset.id, [])+ return result++ def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord:+ with self._sessions() as session:+ dataset = require_dataset(session, ref)+ result = load_dataset_detail(dataset, session=session)+ permissions = rbac_service.RBACService.MyPermissions.get(+ context.active_workspace_id, context.account_id, dataset_id=ref.dataset_id, session=session+ )+ result["permission_keys"] = permissions.dataset.permission_keys_by_resource_ids([dataset.id]).get(+ dataset.id, []+ )+ if dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:+ result["partial_member_list"] = self._members(session, ref)+ return _normalize_provider(result)++ def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord:+ with _translate_permissions(), self._sessions() as session:+ dataset = DatasetService.create_empty_dataset(+ session=session,+ tenant_id=context.active_workspace_id,+ account=load_actor(session, context),+ **dict(values),+ )+ result = self._detail(session, context, dataset)+ result["partial_member_list"] = None+ dataset_id = dataset.id+ session.commit()+ if dify_config.RBAC_ENABLED:+ rbac_service.RBACService.DatasetAccess.replace_whitelist(+ context.active_workspace_id,+ context.account_id,+ dataset_id,+ rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),+ )+ return result++ def update_dataset(+ self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]+ ) -> DatasetDetailRecord:+ data = dict(values)+ if (+ data.get("indexing_technique") == "high_quality"+ and data.get("embedding_model_provider") is not None+ and data.get("embedding_model") is not None+ ):+ data["is_multimodal"] = DatasetService.check_is_multimodal_model(+ ref.tenant_id, data["embedding_model_provider"], data["embedding_model"]+ )+ with _translate_permissions(), self._sessions() as session:+ dataset = require_dataset(session, ref)+ account = load_actor(session, context)+ if not dify_config.RBAC_ENABLED:+ DatasetPermissionService.check_permission(+ account, dataset, data.get("permission"), data.get("partial_member_list"), session=session+ )+ # The legacy updater consumes fields from its input; retain the+ # member selection for the permission update in this transaction.+ dataset = DatasetService.update_dataset(ref.dataset_id, dict(data), account, session=session)+ if dataset is None:+ raise DatasetNotFoundError()+ result = self._detail(session, context, dataset)+ if (+ data.get("partial_member_list") is not None+ and data.get("permission") == DatasetPermissionEnum.PARTIAL_TEAM+ ):+ DatasetPermissionService.update_partial_member_list(+ ref.tenant_id, ref.dataset_id, data["partial_member_list"], session+ )+ elif data.get("permission") in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:+ DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)+ result["partial_member_list"] = self._members(session, ref)+ session.commit()+ return result++ def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None:+ with _translate_permissions(), self._sessions() as session:+ require_dataset(session, ref)+ if not DatasetService.delete_dataset(ref.dataset_id, load_actor(session, context), session):+ raise DatasetNotFoundError()+ DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)+ session.commit()++ def is_in_use(self, ref: DatasetRef) -> bool:+ with self._sessions() as session:+ require_dataset(session, ref)+ return DatasetService.dataset_use_check(ref, session)++ def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]:+ page, limit = clamp_pagination(page, limit, 100)+ with self._sessions() as session:+ require_dataset(session, ref)+ queries, total = DatasetService.get_dataset_queries(+ dataset_id=ref.dataset_id, page=page, per_page=limit, session=session+ )+ data = [+ {+ "id": query.id,+ "queries": get_dataset_queries(query, session=session),+ "source": query.source,+ "source_app_id": query.source_app_id,+ "created_by_role": query.created_by_role,+ "created_by": query.created_by,+ "created_at": query.created_at,+ }+ for query in queries+ ]+ return {"data": data, "has_more": page * limit < total, "total": total, "page": page, "limit": limit}++ def related_apps(self, ref: DatasetRef) -> dict[str, Any]:+ with self._sessions() as session:+ require_dataset(session, ref)+ joins = DatasetService.get_related_apps(ref.dataset_id, session)+ app_ids = [join.app_id for join in joins]+ apps = self._app_queries.related_apps(ref.tenant_id, app_ids)+ return {"data": apps, "total": len(apps)}++ def indexing_status(self, ref: DatasetRef) -> dict[str, Any]:+ with self._sessions() as session:+ require_dataset(session, ref)+ documents = session.scalars(+ select(Document).where(Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id)+ ).all()+ counts = DocumentService.get_document_segment_counts(documents, session=session)+ return {"data": [_status(document, counts.get(document.id, (0, 0))) for document in documents]}++ def error_documents(self, ref: DatasetRef) -> dict[str, Any]:+ with self._sessions() as session:+ require_dataset(session, ref)+ documents = DocumentService.get_error_documents_by_dataset_ref(ref, session)+ return {"data": [_status(document) for document in documents], "total": len(documents)}++ @staticmethod+ def _members(session: Session, ref: DatasetRef) -> list[str]:+ return list(+ session.scalars(+ select(DatasetPermission.account_id).where(+ DatasetPermission.dataset_id == ref.dataset_id, DatasetPermission.tenant_id == ref.tenant_id+ )+ ).all()+ )++ def partial_members(self, ref: DatasetRef) -> list[str]:+ with self._sessions() as session:+ require_dataset(session, ref)+ return self._members(session, ref)++ def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]:+ with self._sessions() as session:+ return dict(DatasetService.get_dataset_auto_disable_logs(ref, session))++ def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None:+ with self._sessions() as session:+ DatasetService.update_dataset_api_status(+ require_dataset(session, ref), enabled, load_actor(session, context), session+ )+ session.commit()diff --git a/api/services/knowledge/datasets/application.py b/api/services/knowledge/datasets/application.pynew file mode 100644index 00000000000..279bac2438d--- /dev/null+++ b/api/services/knowledge/datasets/application.py@@ -0,0 +1,167 @@+"""Dataset administration policies independent of HTTP and persistence."""++from collections.abc import Mapping, Sequence+from dataclasses import dataclass, field+from typing import Any, Protocol++from libs.url_utils import normalize_api_base_url+from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.datasets.retrieval import retrieval_methods+from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage+from services.knowledge.resource_scope import DatasetRef+++@dataclass(frozen=True)+class DatasetListFilter:+ page: int = 1+ limit: int = 20+ keyword: str | None = None+ include_all: bool = False+ ids: Sequence[str] = ()+ tag_ids: Sequence[str] = ()+++@dataclass(frozen=True)+class DatasetVisibility:+ default_permissions: Sequence[str] = ()+ workspace_permissions: Sequence[str] = ()+ overrides: Mapping[str, Sequence[str]] = field(default_factory=dict)+ unrestricted: bool = True+ whitelist_ids: Sequence[str] = ()++ def list_scope(self, *, rbac_enabled: bool) -> tuple[list[str] | None, bool]:+ if not rbac_enabled:+ return None, False+ # A restricted whitelist always takes precedence over default grants,+ # resource overrides and the creator's own-dataset fallback.+ if not self.unrestricted:+ return sorted(set(self.whitelist_ids)), False+ read_keys = {"dataset.preview", "dataset.acl.preview", "dataset.full_access"}+ default_read = bool(+ read_keys.intersection(self.default_permissions) or read_keys.intersection(self.workspace_permissions)+ )+ ids = (+ None+ if default_read+ else sorted(key for key, grants in self.overrides.items() if read_keys.intersection(grants))+ )+ return ids, "dataset.create_and_management" in self.workspace_permissions+++class DatasetOperations(Protocol):+ """Materialize owned values and complete writes before returning."""++ def visibility(self, context: RequestContext) -> DatasetVisibility: ...+ def list_datasets(+ self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None, include_own: bool+ ) -> DatasetPage: ...+ def embedding_models(self, workspace_id: str) -> set[str]: ...+ def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...+ def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...+ def update_dataset(+ self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]+ ) -> DatasetDetailRecord: ...+ def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None: ...+ def is_in_use(self, ref: DatasetRef) -> bool: ...+ def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]: ...+ def related_apps(self, ref: DatasetRef) -> dict[str, Any]: ...+ def indexing_status(self, ref: DatasetRef) -> dict[str, Any]: ...+ def error_documents(self, ref: DatasetRef) -> dict[str, Any]: ...+ def partial_members(self, ref: DatasetRef) -> list[str]: ...+ def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]: ...+ def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None: ...+++class DatasetApplicationService:+ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ operations: DatasetOperations,+ rbac_enabled: bool,+ service_api_url: str,+ vector_store: str | None,+ tidb_fulltext: bool,+ ) -> None:+ self._dataset_access = dataset_access+ self._operations = operations+ self._rbac_enabled = rbac_enabled+ self._service_api_url = service_api_url+ self._vector_store = vector_store+ self._tidb_fulltext = tidb_fulltext++ def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ return DatasetRef(dataset.workspace_id, dataset.id)++ @staticmethod+ def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing: bool) -> None:+ high_quality = item["indexing_technique"] == "high_quality"+ item["embedding_available"] = (+ not high_quality+ or (listing and not item["embedding_model_provider"])+ or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models+ )++ def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:+ visibility = self._operations.visibility(context)+ ids, include_own = visibility.list_scope(rbac_enabled=self._rbac_enabled)+ result = self._operations.list_datasets(context, query, ids, include_own)+ models = self._operations.embedding_models(context.active_workspace_id)+ for item in result["data"]:+ item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))+ self._embedding_available(item, models, listing=True)+ return result++ def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:+ result = self._operations.get_dataset(context, self._dataset(context, dataset_id))+ self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)+ return result++ def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:+ settings = dict(values)+ settings["permission"] = "all_team_members" if self._rbac_enabled else settings.get("permission") or "only_me"+ return self._operations.create_dataset(context, settings)++ def update_dataset(+ self, context: RequestContext, *, dataset_id: str, values: Mapping[str, Any]+ ) -> DatasetDetailRecord:+ return self._operations.update_dataset(context, self._dataset(context, dataset_id), values)++ def delete_dataset(self, context: RequestContext, *, dataset_id: str) -> None:+ self._operations.delete_dataset(context, self._dataset(context, dataset_id))++ def is_in_use(self, context: RequestContext, *, dataset_id: str) -> bool:+ return self._operations.is_in_use(self._dataset(context, dataset_id))++ def queries(self, context: RequestContext, *, dataset_id: str, page: int, limit: int) -> dict[str, Any]:+ return self._operations.queries(self._dataset(context, dataset_id), page=page, limit=limit)++ def related_apps(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:+ return self._operations.related_apps(self._dataset(context, dataset_id))++ def indexing_status(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:+ return self._operations.indexing_status(self._dataset(context, dataset_id))++ def error_documents(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:+ return self._operations.error_documents(self._dataset(context, dataset_id))++ def partial_members(self, context: RequestContext, *, dataset_id: str) -> list[str]:+ return self._operations.partial_members(self._dataset(context, dataset_id))++ def auto_disable_logs(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:+ return self._operations.auto_disable_logs(self._dataset(context, dataset_id))++ def set_api_enabled(self, context: RequestContext, *, dataset_id: str, status: str) -> None:+ self._operations.set_api_enabled(context, self._dataset(context, dataset_id), status == "enable")++ def api_base_url(self, context: RequestContext, *, request_base_url: str) -> str:+ return normalize_api_base_url(self._service_api_url or request_base_url)++ def retrieval_settings(+ self, context: RequestContext, *, vector_type: str | None = None, is_mock: bool = False+ ) -> dict[str, list[str]]:+ return retrieval_methods(+ vector_type if is_mock else self._vector_store, is_mock=is_mock, tidb_fulltext=self._tidb_fulltext+ )diff --git a/api/services/knowledge/datasets/retrieval.py b/api/services/knowledge/datasets/retrieval.pynew file mode 100644index 00000000000..f07fc477a5d--- /dev/null+++ b/api/services/knowledge/datasets/retrieval.py@@ -0,0 +1,85 @@+"""Supported dataset retrieval methods for each vector store."""++from core.rag.datasource.vdb.vector_type import VectorType+from core.rag.retrieval.retrieval_methods import RetrievalMethod+++def retrieval_methods(+ vector_type: str | None, *, is_mock: bool = False, tidb_fulltext: bool = False+) -> dict[str, list[str]]:+ """+ Get supported retrieval methods based on vector database type.++ Args:+ vector_type: Vector database type, can be None+ is_mock: Whether this is a Mock API, affects MILVUS handling+ tidb_fulltext: Whether TiDB full-text retrieval is enabled++ Returns:+ Dictionary containing supported retrieval methods++ Raises:+ ValueError: If vector_type is None or unsupported+ """+ if vector_type is None:+ raise ValueError("Vector store type is not configured.")++ # Define vector database types that only support semantic search+ semantic_only_types = {+ VectorType.RELYT,+ VectorType.CHROMA,+ VectorType.PGVECTO_RS,+ VectorType.VIKINGDB,+ VectorType.UPSTASH,+ }++ # Define vector database types that support all retrieval methods+ full_search_types = {+ VectorType.QDRANT,+ VectorType.WEAVIATE,+ VectorType.OPENSEARCH,+ VectorType.ANALYTICDB,+ VectorType.MYSCALE,+ VectorType.ORACLE,+ VectorType.ELASTICSEARCH,+ VectorType.ELASTICSEARCH_JA,+ VectorType.PGVECTOR,+ VectorType.VASTBASE,+ VectorType.TIDB_ON_QDRANT,+ VectorType.LINDORM,+ VectorType.COUCHBASE,+ VectorType.OPENGAUSS,+ VectorType.OCEANBASE,+ VectorType.SEEKDB,+ VectorType.TABLESTORE,+ VectorType.HUAWEI_CLOUD,+ VectorType.TENCENT,+ VectorType.MATRIXONE,+ VectorType.CLICKZETTA,+ VectorType.BAIDU,+ VectorType.ALIBABACLOUD_MYSQL,+ VectorType.IRIS,+ VectorType.HOLOGRES,+ }++ semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}+ full_methods = {+ "retrieval_method": [+ RetrievalMethod.SEMANTIC_SEARCH.value,+ RetrievalMethod.FULL_TEXT_SEARCH.value,+ RetrievalMethod.HYBRID_SEARCH.value,+ ]+ }++ if vector_type == VectorType.MILVUS:+ return semantic_methods if is_mock else full_methods++ if vector_type == VectorType.TIDB_VECTOR:+ return full_methods if tidb_fulltext else semantic_methods++ if vector_type in semantic_only_types:+ return semantic_methods+ elif vector_type in full_search_types:+ return full_methods+ else:+ raise ValueError(f"Unsupported vector db type {vector_type}.")diff --git a/api/services/knowledge/document_sync.py b/api/services/knowledge/document_sync.pynew file mode 100644index 00000000000..a22bf695f08--- /dev/null+++ b/api/services/knowledge/document_sync.py@@ -0,0 +1,71 @@+"""Request-side document synchronization use cases."""++from typing import NamedTuple, Protocol++from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.resource_scope import DatasetRef, DocumentRef+++class SyncDocumentRecord(NamedTuple):+ """Document state required to authorize and dispatch a sync request."""++ id: str+ data_source_type: str+++class DocumentSyncReader(Protocol):+ def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None: ...++ def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]: ...+++class DocumentSyncDispatcher(Protocol):+ def dispatch(self, *, dataset_id: str, document_id: str) -> None: ...+++class DocumentSyncError(Exception):+ """Base class for document synchronization failures."""+++class SyncDocumentNotFoundError(DocumentSyncError):+ def __init__(self) -> None:+ super().__init__("Document not found")+++class SyncDocumentSourceError(DocumentSyncError):+ def __init__(self) -> None:+ super().__init__("Document is not notion type")+++class DocumentSyncApplicationService:+ """Authorize synchronization requests and dispatch the worker use case."""++ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ documents: DocumentSyncReader,+ dispatcher: DocumentSyncDispatcher,+ ) -> None:+ self._dataset_access = dataset_access+ self._documents = documents+ self._dispatcher = dispatcher++ def sync_dataset(self, context: RequestContext, dataset_id: str) -> int:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ dataset_ref = DatasetRef(dataset.workspace_id, dataset.id)+ document_refs = self._documents.list_active_notion_refs(dataset_ref)+ for document_ref in document_refs:+ self._dispatcher.dispatch(dataset_id=dataset_ref.dataset_id, document_id=document_ref.document_id)+ return len(document_refs)++ def sync_document(self, context: RequestContext, dataset_id: str, document_id: str) -> None:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ document_ref = DatasetRef(dataset.workspace_id, dataset.id).document(document_id)+ document = self._documents.get_sync_document(document_ref)+ if document is None:+ raise SyncDocumentNotFoundError()+ if document.data_source_type != "notion_import":+ raise SyncDocumentSourceError()+ self._dispatcher.dispatch(dataset_id=dataset.id, document_id=document.id)diff --git a/api/services/knowledge/document_sync_adapters.py b/api/services/knowledge/document_sync_adapters.pynew file mode 100644index 00000000000..c110fe22faf--- /dev/null+++ b/api/services/knowledge/document_sync_adapters.py@@ -0,0 +1,11 @@+"""Infrastructure adapters for document-sync requests."""++from collections.abc import Callable+++class CeleryDocumentSyncDispatcher:+ def __init__(self, *, delay: Callable[[str, str], object]) -> None:+ self._delay = delay++ def dispatch(self, *, dataset_id: str, document_id: str) -> None:+ self._delay(dataset_id, document_id)diff --git a/api/services/knowledge/documents/__init__.py b/api/services/knowledge/documents/__init__.pynew file mode 100644index 00000000000..5c5a2feed80--- /dev/null+++ b/api/services/knowledge/documents/__init__.py@@ -0,0 +1 @@+"""Document application use cases and their infrastructure adapters."""diff --git a/api/services/knowledge/documents/adapters.py b/api/services/knowledge/documents/adapters.pynew file mode 100644index 00000000000..1c2bc398dd5--- /dev/null+++ b/api/services/knowledge/documents/adapters.py@@ -0,0 +1,580 @@+"""Session-owning adapters for document queries and existing document operations."""++import copy+import json+from collections.abc import Generator, Mapping, Sequence+from contextlib import contextmanager, nullcontext+from typing import Any, Literal+from uuid import uuid4++from redis.exceptions import LockNotOwnedError+from sqlalchemy import asc, desc, func, select, update+from sqlalchemy.orm import Session, sessionmaker+from werkzeug.exceptions import NotFound++from configs import dify_config+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError+from core.model_manager import ModelManager+from core.rag.index_processor.constant.built_in_field import BuiltInField+from core.rag.index_processor.constant.index_type import IndexTechniqueType+from extensions.ext_redis import redis_client+from graphon.model_runtime.entities.model_entities import ModelType+from graphon.model_runtime.errors.invoke import InvokeAuthorizationError+from libs.pagination import paginate_query+from machinery.context import RequestContext+from models import Account, Dataset, Document, DocumentSegment, UploadFile+from models.dataset import DatasetPermissionEnum, DocumentPipelineExecutionLog+from models.enums import IndexingStatus+from repositories.knowledge.dataset_read_repository import (+ get_dataset_doc_form,+ get_document_hit_count,+ get_document_process_rule,+ get_document_segment_count,+ get_latest_dataset_process_rule,+)+from repositories.knowledge.dataset_repository import _get_dataset+from repositories.knowledge.document_repository import _get_document+from services.enterprise import rbac_service+from services.errors.account import NoPermissionError+from services.errors.document import DocumentIndexingError+from services.file_service import FileService+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.dataset_read_service import (+ get_document_metadata_details,+ get_document_source_detail,+ load_document_detail,+ load_document_details,+)+from services.knowledge.dataset_service import DatasetService, DocumentService+from services.knowledge.documents.application import (+ DocumentIndexingStateError,+ DocumentInvalidActionError,+ DocumentListFilter,+ DocumentNotFoundError,+ DocumentProviderError,+ DocumentState,+ DocumentZip,+)+from services.knowledge.entities.document_creation import DocumentCreationDataset, DocumentIndexingJobs+from services.knowledge.entities.knowledge_entities import KnowledgeConfig+from services.knowledge.resource_scope import DatasetRef, DocumentRef+from services.knowledge.summaries.adapters import SummaryIndexAdapter+from services.vector_space_admission_service import get_vector_space_admission_error_fields+from tasks.generate_summary_index_task import generate_summary_index_task+++@contextmanager+def _translate_errors() -> Generator[None, None, None]:+ try:+ yield+ except NotFound as error:+ raise DocumentNotFoundError(error.description) from error+ except NoPermissionError as error:+ raise DatasetAccessDeniedError() from error+ except DocumentIndexingError as error:+ raise DocumentIndexingStateError(str(error)) from error+ except (ProviderTokenNotInitError, InvokeAuthorizationError) as error:+ raise DocumentProviderError(str(error), kind="uninitialized") from error+ except QuotaExceededError as error:+ raise DocumentProviderError(str(error), kind="quota") from error+ except ModelCurrentlyNotSupportError as error:+ raise DocumentProviderError(str(error), kind="unsupported") from error+++def _require_dataset(session: Session, ref: DatasetRef) -> Dataset:+ dataset = _get_dataset(session, ref)+ if dataset is None:+ raise DatasetNotFoundError()+ return dataset+++def _require_document(session: Session, ref: DocumentRef) -> Document:+ document = _get_document(session, ref)+ if document is None:+ raise DocumentNotFoundError("Document not found.")+ return document+++def _state(document: Document) -> DocumentState:+ return DocumentState(+ id=document.id,+ archived=bool(document.archived),+ indexing_status=str(document.indexing_status),+ is_paused=bool(document.is_paused),+ data_source_type=str(document.data_source_type),+ doc_form=str(document.doc_form),+ need_summary=bool(document.need_summary),+ )+++def _created_response(dataset: Dataset, documents: Sequence[Document], batch: str, session: Session) -> dict[str, Any]:+ return {+ "dataset": {+ "id": dataset.id,+ "name": dataset.name,+ "description": dataset.description,+ "permission": dataset.permission,+ "data_source_type": dataset.data_source_type,+ "indexing_technique": dataset.indexing_technique,+ "created_by": dataset.created_by,+ "created_at": dataset.created_at,+ },+ "documents": load_document_details(documents, session=session),+ "batch": batch,+ }+++def _indexing_status(document: Document, counts: tuple[int, int]) -> dict[str, Any]:+ return {+ "id": document.id,+ "processing_started_at": document.processing_started_at,+ "parsing_completed_at": document.parsing_completed_at,+ "cleaning_completed_at": document.cleaning_completed_at,+ "splitting_completed_at": document.splitting_completed_at,+ "completed_at": document.completed_at,+ "paused_at": document.paused_at,+ "error": document.error,+ "stopped_at": document.stopped_at,+ "indexing_status": IndexingStatus.PAUSED if document.is_paused else document.indexing_status,+ **get_vector_space_admission_error_fields(document.error),+ "completed_segments": counts[0],+ "total_segments": counts[1],+ }+++class SQLAlchemyDocumentOperations:+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._sessions = session_factory++ def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None:+ with self._sessions() as session:+ document = session.scalar(+ select(Document).where(Document.id == document_id, Document.tenant_id == workspace_id)+ )+ return DatasetRef(workspace_id, document.dataset_id).document(document.id) if document else None++ def get_state(self, ref: DocumentRef) -> DocumentState | None:+ with self._sessions() as session:+ document = _get_document(session, ref)+ return _state(document) if document else None++ def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]:+ with self._sessions() as session:+ return tuple(+ _state(document) for document in DocumentService.get_documents_by_ids(ref, document_ids, session)+ )++ def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]:+ result = copy.deepcopy(DocumentService.DEFAULT_RULES)+ if ref is not None:+ with self._sessions() as session:+ rule = get_latest_dataset_process_rule(_require_dataset(session, ref), session=session)+ if rule is not None:+ result.update(mode=rule.mode, rules=rule.rules_dict)+ return result++ def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]:+ with self._sessions() as session:+ dataset = _require_dataset(session, ref)+ statement = select(Document).where(+ Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id+ )+ statement = DocumentService.apply_display_status_filter(statement, query.status)+ if query.search:+ statement = statement.where(Document.name.like(f"%{query.search}%"))+ direction = desc if query.sort.startswith("-") else asc+ sort = query.sort.removeprefix("-")+ if sort == "hit_count":+ hits = (+ select(DocumentSegment.document_id, func.sum(DocumentSegment.hit_count).label("total_hit_count"))+ .where(DocumentSegment.dataset_id == ref.dataset_id, DocumentSegment.tenant_id == ref.tenant_id)+ .group_by(DocumentSegment.document_id)+ .subquery()+ )+ statement = statement.outerjoin(hits, hits.c.document_id == Document.id).order_by(+ direction(func.coalesce(hits.c.total_hit_count, 0)), direction(Document.position)+ )+ elif sort == "created_at":+ statement = statement.order_by(direction(Document.created_at), direction(Document.position))+ else:+ statement = statement.order_by(desc(Document.created_at), desc(Document.position))+ page = paginate_query(statement, session=session, page=query.page, per_page=query.limit, max_per_page=100)+ DocumentService.enrich_documents_with_summary_index_status(+ documents=page.items, dataset=dataset, tenant_id=ref.tenant_id, session=session+ )+ if query.fetch:+ counts = DocumentService.get_document_segment_counts(page.items, session=session)+ for document in page.items:+ document.completed_segments, document.total_segments = counts.get(document.id, (0, 0))+ return {+ "data": load_document_details(page.items, session=session),+ "has_more": page.has_next,+ "limit": page.per_page,+ "total": page.total,+ "page": page.page,+ }++ def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]:+ with self._sessions() as session:+ document = _require_document(session, ref)+ result = {+ "id": document.id,+ "doc_type": document.doc_type,+ "doc_metadata": get_document_metadata_details(document, session=session),+ }+ if metadata_only:+ return result+ rule = get_document_process_rule(document, session=session)+ segment_count = get_document_segment_count(document, session=session)+ result.update(+ {+ "position": document.position,+ "data_source_type": document.data_source_type,+ "dataset_process_rule_id": document.dataset_process_rule_id,+ "name": document.name,+ "created_from": document.created_from,+ "created_by": document.created_by,+ "tokens": document.tokens,+ "indexing_status": document.indexing_status,+ "indexing_latency": document.indexing_latency,+ "error": document.error,+ "enabled": document.enabled,+ "disabled_by": document.disabled_by,+ "archived": document.archived,+ "display_status": document.display_status,+ "doc_form": document.doc_form,+ "doc_language": document.doc_language,+ }+ )+ result.update(+ {+ "created_at": int(document.created_at.timestamp()) if document.created_at else None,+ "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,+ "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,+ "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,+ }+ )+ result.update(+ data_source_info=document.data_source_info_dict,+ data_source_detail_dict=get_document_source_detail(document, session=session),+ dataset_process_rule=DatasetService.get_process_rules(ref.dataset.dataset_id, session),+ document_process_rule=dict(rule.to_dict()) if rule else {},+ segment_count=segment_count,+ average_segment_length=(document.word_count or 0) // segment_count if segment_count else 0,+ hit_count=get_document_hit_count(document, session=session),+ need_summary=bool(document.need_summary),+ )+ return result++ def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]:+ with self._sessions() as session:+ document = _require_document(session, ref)+ counts = DocumentService.get_document_segment_counts([document], session=session)+ return _indexing_status(document, counts.get(document.id, (0, 0)))++ def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]:+ with self._sessions() as session:+ documents = session.scalars(+ select(Document).where(+ Document.tenant_id == ref.tenant_id, Document.dataset_id == ref.dataset_id, Document.batch == batch+ )+ ).all()+ if not documents:+ raise DocumentNotFoundError("Documents not found.")+ counts = DocumentService.get_document_segment_counts(documents, session=session)+ return {"data": [_indexing_status(document, counts.get(document.id, (0, 0))) for document in documents]}++ def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]:+ with self._sessions() as session:+ _require_document(session, ref)+ log = session.scalar(+ select(DocumentPipelineExecutionLog)+ .where(DocumentPipelineExecutionLog.document_id == ref.document_id)+ .order_by(DocumentPipelineExecutionLog.created_at.desc())+ .limit(1)+ )+ if log is None:+ return {}+ return {+ "datasource_info": json.loads(log.datasource_info),+ "datasource_type": log.datasource_type,+ "input_data": log.input_data,+ "datasource_node_id": log.datasource_node_id,+ }++ def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]:+ with self._sessions() as session:+ _require_document(session, ref)+ return dict(+ SummaryIndexAdapter.get_document_summary_status_detail(+ tenant_id=ref.dataset.tenant_id,+ document_id=ref.document_id,+ dataset_id=ref.dataset.dataset_id,+ session=session,+ )+ )++ def get_download_url(self, ref: DocumentRef) -> str:+ with _translate_errors(), self._sessions() as session:+ return DocumentService.get_document_download_url(_require_document(session, ref), session)++ @contextmanager+ def build_download_zip(self, ref: DatasetRef, document_ids: Sequence[str]) -> Generator[DocumentZip, None, None]:+ with _translate_errors(), self._sessions() as session:+ files_by_document = DocumentService._get_upload_files_by_document_id_for_zip_download(+ dataset_id=ref.dataset_id, document_ids=document_ids, tenant_id=ref.tenant_id, session=session+ )+ files = [files_by_document[document_id] for document_id in document_ids]+ filename = DocumentService._generate_document_batch_download_zip_filename()+ # The read transaction is closed before reading storage or streaming the response.+ with FileService.build_upload_files_zip_tempfile(upload_files=files) as path:+ yield DocumentZip(path=path, filename=filename)++ @staticmethod+ def _actor(session: Session, context: RequestContext) -> Account:+ account = session.get(Account, context.account_id)+ if account is None:+ raise DatasetAccessDeniedError()+ account.set_tenant_id_with_session(context.active_workspace_id, session=session)+ return account++ def _creation_dataset(+ self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig+ ) -> DocumentCreationDataset:+ with self._sessions() as session:+ dataset = _require_dataset(session, ref)+ if not dataset.indexing_technique and not config.indexing_technique:+ raise ValueError("indexing_technique is required.")+ self._actor(session, context)+ DatasetService.check_doc_form(dataset, config.doc_form, session=session)+ return DocumentCreationDataset(+ dataset.tenant_id, dataset.indexing_technique, dataset.embedding_model, dataset.embedding_model_provider+ )++ def _persist_creation(+ self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool+ ) -> tuple[dict[str, Any], DocumentIndexingJobs]:+ """Keep ORM values local and materialize the response before closing the session."""+ jobs: DocumentIndexingJobs | None = None+ try:+ with self._sessions() as session:+ account = self._actor(session, context)+ dataset = (+ DocumentService.build_dataset_for_documents(ref, config, account, session)+ if initialize+ else _require_dataset(session, ref)+ )+ documents, batch, jobs = DocumentService.save_prepared_documents(+ dataset, config, account, session=session+ )+ if initialize:+ dataset.name = documents[0].name[:18] + "..."+ dataset.description = "useful for when you want to answer queries about the " + documents[0].name+ dataset.permission = (+ DatasetPermissionEnum.ALL_TEAM if dify_config.RBAC_ENABLED else DatasetPermissionEnum.ONLY_ME+ )+ session.flush()+ result = _created_response(dataset, documents, batch, session)+ session.commit()+ return result, jobs+ except Exception:+ # The persistence helper commits documents before loading response+ # details. Preserve their indexing even if that later read fails.+ if jobs is not None:+ DocumentService.dispatch_document_indexing(jobs)+ raise++ def _run_creation(+ self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool+ ) -> dict[str, Any]:+ lock = (+ nullcontext()+ if config.original_document_id+ else redis_client.lock(f"add_document_lock_dataset_id_{ref.dataset_id}", timeout=600)+ )+ result = None+ try:+ with lock:+ result, jobs = self._persist_creation(context, ref, config, initialize=initialize)+ # No session or ORM entity crosses the task boundary. Keep the+ # creation lock until committed documents have been dispatched.+ DocumentService.dispatch_document_indexing(jobs)+ except LockNotOwnedError:+ if config.original_document_id:+ raise+ # Preserve successful creation when the Redis lease expires on release.+ if result is None:+ if initialize:+ raise+ with self._sessions() as session:+ result = _created_response(+ _require_dataset(session, ref), [], DocumentService.generate_document_batch(), session+ )+ assert result is not None+ return result++ def create_documents(self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]) -> dict[str, Any]:+ config = KnowledgeConfig.model_validate(settings)+ DocumentService.document_create_args_validate(config)+ with _translate_errors():+ snapshot = self._creation_dataset(context, ref, config)+ config = DocumentService.prepare_document_creation(snapshot, config)+ return self._run_creation(context, ref, config, initialize=False)++ def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]:+ config = KnowledgeConfig.model_validate(settings)+ with _translate_errors():+ if config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ if config.embedding_model is None or config.embedding_model_provider is None:+ raise ValueError(+ "embedding model and embedding model provider are required for high quality indexing."+ )+ ModelManager.for_tenant(tenant_id=context.active_workspace_id).get_model_instance(+ tenant_id=context.active_workspace_id,+ provider=config.embedding_model_provider,+ model_type=ModelType.TEXT_EMBEDDING,+ model=config.embedding_model,+ )+ config = config.model_copy(+ update={+ "is_multimodal": DatasetService.check_is_multimodal_model(+ context.active_workspace_id, config.embedding_model_provider, config.embedding_model+ )+ }+ )+ DocumentService.document_create_args_validate(config)+ with self._sessions() as session:+ self._actor(session, context)+ config = DocumentService.prepare_document_creation(+ DocumentCreationDataset(+ context.active_workspace_id,+ config.indexing_technique,+ config.embedding_model,+ config.embedding_model_provider,+ ),+ config,+ )+ dataset_id = str(uuid4())+ result = self._run_creation(+ context, DatasetRef(context.active_workspace_id, dataset_id), config, initialize=True+ )+ if dify_config.RBAC_ENABLED:+ rbac_service.RBACService.DatasetAccess.replace_whitelist(+ context.active_workspace_id,+ context.account_id,+ dataset_id,+ rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),+ )+ rbac_service.try_sync_creator_access_policy_member_bindings(+ context.active_workspace_id,+ context.account_id,+ rbac_service.RBACResourceType.DATASET,+ dataset_id,+ )+ return result++ def _check_model(self, ref: DatasetRef) -> None:+ with self._sessions() as session:+ dataset = _require_dataset(session, ref)+ DatasetService.check_dataset_model_setting(dataset)++ def delete_document(self, ref: DocumentRef) -> None:+ self._check_model(ref.dataset)+ with _translate_errors(), self._sessions() as session:+ DocumentService.delete_document(_require_document(session, ref), session)++ def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:+ with _translate_errors(), self._sessions() as session:+ dataset = _require_dataset(session, ref)+ DocumentService.delete_documents(+ ref, list(document_ids), get_dataset_doc_form(dataset, session=session), session+ )++ def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None:+ with self._sessions.begin() as session:+ document = _require_document(session, ref)+ for name, value in values.items():+ setattr(document, name, value)++ def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]:+ with self._sessions.begin() as session:+ dataset = _require_dataset(session, ref.dataset)+ document = _require_document(session, ref)+ if dataset.built_in_field_enabled and document.doc_metadata:+ document.doc_metadata = {**document.doc_metadata, BuiltInField.document_name: name}+ document.name = name+ if document.data_source_info_dict and (file_id := document.data_source_info_dict.get("upload_file_id")):+ session.execute(+ update(UploadFile)+ .where(UploadFile.id == file_id, UploadFile.tenant_id == ref.dataset.tenant_id)+ .values(name=name)+ )+ session.flush()+ return load_document_detail(document, session=session)++ def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None:+ with _translate_errors(), self._sessions() as session:+ DocumentService.pause_document(_require_document(session, ref), session, actor_id=actor_id)++ def recover_document(self, ref: DocumentRef) -> None:+ with _translate_errors(), self._sessions() as session:+ DocumentService.recover_document(_require_document(session, ref), session)++ def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None:+ with _translate_errors(), self._sessions() as session:+ documents = DocumentService.get_documents_by_ids(ref, document_ids, session)+ DocumentService.retry_document(ref.dataset_id, list(documents), session, actor_id=actor_id)++ def change_status(+ self,+ context: RequestContext,+ ref: DatasetRef,+ document_ids: Sequence[str],+ action: Literal["enable", "disable", "archive", "un_archive"],+ ) -> None:+ self._check_model(ref)+ with _translate_errors(), self._sessions() as session:+ dataset = _require_dataset(session, ref)+ account = self._actor(session, context)+ documents = DocumentService.get_documents_by_ids(ref, document_ids, session)+ try:+ DocumentService.batch_update_document_status(+ dataset, [doc.id for doc in documents], action, account, session+ )+ except (ValueError, DocumentIndexingError) as error:+ raise DocumentInvalidActionError(str(error)) from error++ def sync_website(self, ref: DocumentRef) -> None:+ with _translate_errors(), self._sessions() as session:+ DocumentService.sync_website_document(+ _require_dataset(session, ref.dataset), _require_document(session, ref), session+ )++ def require_summary_enabled(self, ref: DatasetRef) -> None:+ with self._sessions() as session:+ dataset = _require_dataset(session, ref)+ if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:+ raise ValueError(+ "Summary generation is only available for 'high_quality' indexing technique. "+ f"Current indexing technique: {dataset.indexing_technique}"+ )+ if not dataset.summary_index_setting or not dataset.summary_index_setting.get("enable"):+ raise ValueError(+ "Summary index is not enabled for this dataset. Please enable it in the dataset settings."+ )++ def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:+ if document_ids:+ with self._sessions.begin() as session:+ session.execute(+ update(Document)+ .where(+ Document.tenant_id == ref.tenant_id,+ Document.dataset_id == ref.dataset_id,+ Document.id.in_(document_ids),+ )+ .values(need_summary=True)+ )++ def dispatch_summary(self, ref: DocumentRef) -> None:+ generate_summary_index_task.delay(ref.dataset.dataset_id, ref.document_id)diff --git a/api/services/knowledge/documents/application.py b/api/services/knowledge/documents/application.pynew file mode 100644index 00000000000..604193de7bc--- /dev/null+++ b/api/services/knowledge/documents/application.py@@ -0,0 +1,310 @@+"""Console document use cases, independent of HTTP and persistence frameworks."""++from collections.abc import Mapping, Sequence+from contextlib import AbstractContextManager+from dataclasses import dataclass+from typing import Any, Literal, Protocol++from libs.datetime_utils import naive_utc_now+from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.resource_scope import DatasetRef, DocumentRef+++@dataclass(frozen=True)+class DocumentListFilter:+ page: int = 1+ limit: int = 20+ search: str = ""+ sort: str = "-created_at"+ status: str | None = None+ fetch: bool = False+++@dataclass(frozen=True)+class DocumentState:+ id: str+ archived: bool+ indexing_status: str+ is_paused: bool+ data_source_type: str+ doc_form: str+ need_summary: bool+++@dataclass(frozen=True)+class DocumentZip:+ path: str+ filename: str+++class DocumentApplicationError(Exception):+ """Base class for document use-case failures mapped by the transport."""+++class DocumentNotFoundError(DocumentApplicationError):+ pass+++class DocumentArchivedError(DocumentApplicationError):+ pass+++class DocumentIndexingStateError(DocumentApplicationError):+ pass+++class DocumentInvalidActionError(DocumentApplicationError):+ pass+++class DocumentProviderError(DocumentApplicationError):+ def __init__(self, description: str, *, kind: Literal["uninitialized", "quota", "unsupported"]) -> None:+ super().__init__(description)+ self.kind = kind+++class DocumentOperations(Protocol):+ """Owned document operations; adapters return materialized values and own their sessions."""++ def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None: ...+ def get_state(self, ref: DocumentRef) -> DocumentState | None: ...+ def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]: ...+ def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]: ...+ def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]: ...+ def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]: ...+ def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]: ...+ def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]: ...+ def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]: ...+ def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]: ...+ def get_download_url(self, ref: DocumentRef) -> str: ...+ def build_download_zip(+ self, ref: DatasetRef, document_ids: Sequence[str]+ ) -> AbstractContextManager[DocumentZip]: ...+ def create_documents(+ self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]+ ) -> dict[str, Any]: ...+ def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]: ...+ def delete_document(self, ref: DocumentRef) -> None: ...+ def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...+ def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None: ...+ def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]: ...+ def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None: ...+ def recover_document(self, ref: DocumentRef) -> None: ...+ def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None: ...+ def change_status(+ self,+ context: RequestContext,+ ref: DatasetRef,+ document_ids: Sequence[str],+ action: Literal["enable", "disable", "archive", "un_archive"],+ ) -> None: ...+ def sync_website(self, ref: DocumentRef) -> None: ...+ def require_summary_enabled(self, ref: DatasetRef) -> None: ...+ def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...+ def dispatch_summary(self, ref: DocumentRef) -> None: ...+++class DatasetDocumentApplicationService:+ """Authorize document use cases and apply document state transitions."""++ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ operations: DocumentOperations,+ metadata_schema: Mapping[str, Mapping[str, type]],+ ) -> None:+ self._dataset_access = dataset_access+ self._operations = operations+ self._metadata_schema = metadata_schema++ def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ return DatasetRef(tenant_id=dataset.workspace_id, dataset_id=dataset.id)++ def _document(+ self, context: RequestContext, dataset_id: str, document_id: str+ ) -> tuple[DocumentRef, DocumentState]:+ ref = self._dataset(context, dataset_id).document(document_id)+ state = self._operations.get_state(ref)+ if state is None:+ raise DocumentNotFoundError("Document not found.")+ return ref, state++ @staticmethod+ def _require_mutable(state: DocumentState) -> None:+ if state.archived:+ raise DocumentArchivedError()++ def get_process_rule(self, context: RequestContext, *, document_id: str | None) -> dict[str, Any]:+ ref = None+ if document_id:+ document_ref = self._operations.find_document(+ workspace_id=context.active_workspace_id, document_id=document_id+ )+ if document_ref is None:+ raise DocumentNotFoundError("Document not found.")+ ref = self._dataset(context, document_ref.dataset.dataset_id)+ return self._operations.get_process_rule(ref)++ def list_documents(self, context: RequestContext, *, dataset_id: str, query: DocumentListFilter) -> dict[str, Any]:+ return self._operations.list_documents(self._dataset(context, dataset_id), query)++ def get_document(+ self, context: RequestContext, *, dataset_id: str, document_id: str, metadata_only: bool = False+ ) -> dict[str, Any]:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.get_detail(ref, metadata_only=metadata_only)++ def get_indexing_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.get_indexing_status(ref)++ def get_batch_indexing_status(self, context: RequestContext, *, dataset_id: str, batch: str) -> dict[str, Any]:+ return self._operations.get_batch_indexing_status(self._dataset(context, dataset_id), batch)++ def get_execution_log(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.get_execution_log(ref)++ def get_summary_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.get_summary_status(ref)++ def get_download_url(self, context: RequestContext, *, dataset_id: str, document_id: str) -> str:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.get_download_url(ref)++ def build_download_zip(+ self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]+ ) -> AbstractContextManager[DocumentZip]:+ return self._operations.build_download_zip(self._dataset(context, dataset_id), document_ids)++ def create_documents(+ self, context: RequestContext, *, dataset_id: str, settings: Mapping[str, Any]+ ) -> dict[str, Any]:+ return self._operations.create_documents(context, self._dataset(context, dataset_id), settings)++ def initialize_dataset(self, context: RequestContext, *, settings: Mapping[str, Any]) -> dict[str, Any]:+ return self._operations.initialize_dataset(context, settings)++ def delete_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:+ ref, _ = self._document(context, dataset_id, document_id)+ self._operations.delete_document(ref)++ def delete_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:+ self._operations.delete_documents(self._dataset(context, dataset_id), document_ids)++ def update_processing(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ action: Literal["pause", "resume"],+ ) -> None:+ ref, state = self._document(context, dataset_id, document_id)+ if action == "pause":+ if state.indexing_status != "indexing":+ raise DocumentInvalidActionError("Document not in indexing state.")+ values: dict[str, object] = {+ "paused_by": context.account_id,+ "paused_at": naive_utc_now(),+ "is_paused": True,+ }+ elif action == "resume":+ if state.indexing_status not in {"paused", "error"}:+ raise DocumentInvalidActionError("Document not in paused or error state.")+ values = {"paused_by": None, "paused_at": None, "is_paused": False}+ else:+ raise DocumentInvalidActionError("Invalid action.")+ self._operations.update_document(ref, values)++ def update_metadata(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ doc_type: str | None,+ doc_metadata: dict[str, Any] | None,+ ) -> None:+ ref, _ = self._document(context, dataset_id, document_id)+ if doc_type is None or doc_metadata is None:+ raise ValueError("Both doc_type and doc_metadata must be provided.")+ if doc_type not in self._metadata_schema:+ raise ValueError("Invalid doc_type.")+ metadata = (+ doc_metadata+ if doc_type == "others"+ else {+ key: doc_metadata[key]+ for key, value_type in self._metadata_schema[doc_type].items()+ if doc_metadata.get(key) is not None and isinstance(doc_metadata[key], value_type)+ }+ )+ self._operations.update_document(+ ref, {"doc_type": doc_type, "doc_metadata": metadata, "updated_at": naive_utc_now()}+ )++ def change_status(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_ids: Sequence[str],+ action: Literal["enable", "disable", "archive", "un_archive"],+ ) -> None:+ self._operations.change_status(context, self._dataset(context, dataset_id), document_ids, action)++ def pause_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:+ ref, state = self._document(context, dataset_id, document_id)+ self._require_mutable(state)+ if state.indexing_status not in {"waiting", "parsing", "cleaning", "splitting", "indexing"}:+ raise DocumentIndexingStateError("Cannot pause completed document.")+ self._operations.pause_document(ref, actor_id=context.account_id)++ def recover_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:+ ref, state = self._document(context, dataset_id, document_id)+ self._require_mutable(state)+ if not state.is_paused:+ raise DocumentIndexingStateError("Document is not in paused status.")+ self._operations.recover_document(ref)++ def retry_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:+ ref = self._dataset(context, dataset_id)+ states = {state.id: state for state in self._operations.get_states(ref, document_ids)}+ retry_ids = [+ document_id+ for document_id in dict.fromkeys(document_ids)+ if (state := states.get(document_id)) is not None+ and not state.archived+ and state.indexing_status != "completed"+ ]+ self._operations.retry_documents(ref, retry_ids, actor_id=context.account_id)++ def rename_document(+ self, context: RequestContext, *, dataset_id: str, document_id: str, name: str+ ) -> dict[str, Any]:+ ref, _ = self._document(context, dataset_id, document_id)+ return self._operations.rename_document(ref, name)++ def sync_website(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:+ ref, state = self._document(context, dataset_id, document_id)+ if state.data_source_type != "website_crawl":+ raise ValueError("Document is not a website document.")+ self._require_mutable(state)+ self._operations.sync_website(ref)++ def generate_summary(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:+ ref = self._dataset(context, dataset_id)+ self._operations.require_summary_enabled(ref)+ states = self._operations.get_states(ref, document_ids)+ if len(states) != len(document_ids):+ missing_ids = set(document_ids) - {state.id for state in states}+ raise DocumentNotFoundError(f"Some documents not found: {sorted(missing_ids)}")+ targets = [state for state in states if state.doc_form != "qa_model"]+ self._operations.enable_summary(ref, [state.id for state in targets if not state.need_summary])+ for state in targets:+ self._operations.dispatch_summary(ref.document(state.id))diff --git a/api/services/knowledge/entities/__init__.py b/api/services/knowledge/entities/__init__.pynew file mode 100644index 00000000000..bb315666d2c--- /dev/null+++ b/api/services/knowledge/entities/__init__.py@@ -0,0 +1 @@+"""Shared data contracts for knowledge use cases."""diff --git a/api/services/knowledge/entities/datasets.py b/api/services/knowledge/entities/datasets.pynew file mode 100644index 00000000000..21d5d659b99--- /dev/null+++ b/api/services/knowledge/entities/datasets.py@@ -0,0 +1,69 @@+"""Materialized dataset read contracts, independent of persistence and HTTP.++Timestamps remain datetimes until transport serialization. JSON configuration+objects retain their stored keys; this boundary does not apply response defaults.+"""++from collections.abc import Mapping, Sequence+from datetime import datetime+from typing import NotRequired, TypedDict++from pydantic import JsonValue+++class DatasetTagRecord(TypedDict):+ id: str+ name: str+ type: str+++class DatasetDetailRecord(TypedDict):+ id: str+ name: str+ description: str | None+ provider: str+ permission: str+ data_source_type: str | None+ indexing_technique: str | None+ app_count: int+ document_count: int+ word_count: int+ created_by: str+ author_name: str | None+ created_at: datetime+ updated_by: str | None+ updated_at: datetime+ embedding_model: str | None+ embedding_model_provider: str | None+ # None means availability has not been evaluated by the application service.+ embedding_available: bool | None+ retrieval_model_dict: Mapping[str, JsonValue]+ summary_index_setting: Mapping[str, JsonValue] | None+ tags: list[DatasetTagRecord]+ doc_form: str | None+ external_knowledge_info: Mapping[str, JsonValue] | None+ external_retrieval_model: Mapping[str, JsonValue] | None+ doc_metadata: Sequence[Mapping[str, str]]+ built_in_field_enabled: bool+ pipeline_id: str | None+ runtime_mode: str | None+ chunk_structure: str | None+ icon_info: Mapping[str, JsonValue] | None+ is_published: bool+ total_documents: int+ total_available_documents: int+ enable_api: bool+ is_multimodal: bool+ permission_keys: list[str]+ maintainer: str | None+ # Detail/create responses may omit membership or explicitly return null;+ # listing and updates materialize a list, including an empty one.+ partial_member_list: NotRequired[list[str] | None]+++class DatasetPage(TypedDict):+ data: list[DatasetDetailRecord]+ has_more: bool+ total: int+ page: int+ limit: intdiff --git a/api/services/knowledge/entities/document_creation.py b/api/services/knowledge/entities/document_creation.pynew file mode 100644index 00000000000..fb7bb474a61--- /dev/null+++ b/api/services/knowledge/entities/document_creation.py@@ -0,0 +1,22 @@+"""Owned values passed between document creation's database and I/O phases."""++from dataclasses import dataclass++from services.knowledge.resource_scope import DatasetRef+++@dataclass(frozen=True)+class DocumentCreationDataset:+ tenant_id: str+ indexing_technique: str | None+ embedding_model: str | None+ embedding_model_provider: str | None+++@dataclass(frozen=True)+class DocumentIndexingJobs:+ dataset: DatasetRef+ created: tuple[str, ...] = ()+ duplicated: tuple[str, ...] = ()+ removed_notion: tuple[str, ...] = ()+ updated: str | None = Nonediff --git a/api/services/knowledge/entities/indexing_estimate.py b/api/services/knowledge/entities/indexing_estimate.pynew file mode 100644index 00000000000..437a16b7671--- /dev/null+++ b/api/services/knowledge/entities/indexing_estimate.py@@ -0,0 +1,159 @@+"""Commands and validation shared by knowledge indexing-estimate use cases."""++from collections.abc import Mapping+from dataclasses import dataclass, field+from typing import Annotated, Literal++from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator++from core.rag.entities import PreProcessingRuleKey+from services.data_source.entities.notion_import import NotionPageType+++class EstimateValidationError(ValueError):+ """Raised when an estimate process rule is invalid."""+++class _PreProcessingRule(BaseModel):+ id: PreProcessingRuleKey+ enabled: bool+++class _Segmentation(BaseModel):+ separator: str = Field(min_length=1)+ max_tokens: int = Field(gt=0)+++class _Rules(BaseModel):+ pre_processing_rules: list[_PreProcessingRule]+ segmentation: _Segmentation++ @field_validator("pre_processing_rules")+ @classmethod+ def _deduplicate(cls, value: list[_PreProcessingRule]) -> list[_PreProcessingRule]:+ by_id = {rule.id: rule for rule in value}+ return list(by_id.values())+++class _HierarchicalRules(_Rules):+ parent_mode: Literal["full-doc", "paragraph"] | None = None+ subchunk_segmentation: _Segmentation | None = None+++class _SummaryIndexSettingDisabled(BaseModel):+ enable: Literal[False] = False+++class _SummaryIndexSettingEnabled(BaseModel):+ enable: Literal[True]+ model_name: str = Field(min_length=1)+ model_provider_name: str = Field(min_length=1)+++_SummaryIndexSetting = Annotated[+ _SummaryIndexSettingDisabled | _SummaryIndexSettingEnabled,+ Field(discriminator="enable"),+]+++class _ProcessRuleBase(BaseModel):+ model_config = ConfigDict(extra="allow")++ summary_index_setting: _SummaryIndexSetting | None = None++ @field_validator("summary_index_setting", mode="before")+ @classmethod+ def _normalize_summary_index_setting(cls, value: object) -> object:+ """Treat mappings with a missing/None enable flag as absent (#36602)."""++ if value is None:+ return None+ if isinstance(value, Mapping) and value.get("enable") is None:+ return None+ return value+++class _AutomaticProcessRule(_ProcessRuleBase):+ mode: Literal["automatic"]+++class _CustomProcessRule(_ProcessRuleBase):+ mode: Literal["custom"]+ rules: _Rules+++class _HierarchicalProcessRule(_ProcessRuleBase):+ mode: Literal["hierarchical"]+ rules: _HierarchicalRules+++_ProcessRule = Annotated[+ _AutomaticProcessRule | _CustomProcessRule | _HierarchicalProcessRule,+ Field(discriminator="mode"),+]+++class _ProcessRuleEnvelope(BaseModel):+ process_rule: _ProcessRule+++def _validation_message(error: ValidationError) -> str:+ first = error.errors()[0]+ original = first.get("ctx", {}).get("error")+ return str(original) if isinstance(original, ValueError) else str(first["msg"])+++def normalize_process_rule(process_rule: Mapping[str, object]) -> dict[str, object]:+ """Validate and normalize a process rule without mutating caller-owned input."""++ try:+ validated = _ProcessRuleEnvelope.model_validate({"process_rule": process_rule})+ except ValidationError as error:+ raise EstimateValidationError(_validation_message(error)) from error++ result = validated.process_rule.model_dump(exclude_none=True)+ if validated.process_rule.mode == "automatic":+ result["rules"] = {}+ elif validated.process_rule.mode == "hierarchical":+ rules = result.get("rules")+ if isinstance(rules, dict) and not rules.get("parent_mode"):+ rules["parent_mode"] = "paragraph"+ return result+++@dataclass(frozen=True, slots=True)+class UploadFileEstimateSource:+ file_id: str+ source_type: Literal["upload_file"] = field(default="upload_file", init=False)+++@dataclass(frozen=True, slots=True)+class NotionEstimateSource:+ workspace_id: str+ page_id: str+ page_type: NotionPageType+ credential_id: str+ source_type: Literal["notion_import"] = field(default="notion_import", init=False)+++@dataclass(frozen=True, slots=True)+class WebsiteEstimateSource:+ provider: str+ job_id: str+ url: str+ mode: str = "crawl"+ only_main_content: bool = False+ source_type: Literal["website_crawl"] = field(default="website_crawl", init=False)+++type NewEstimateSource = UploadFileEstimateSource | NotionEstimateSource | WebsiteEstimateSource+++@dataclass(frozen=True, slots=True)+class NewSourcesEstimateCommand:+ sources: tuple[NewEstimateSource, ...]+ process_rule: Mapping[str, object]+ doc_form: str = "text_model"+ doc_language: str = "English"+ dataset_id: str | None = None+ indexing_technique: str = "economy"diff --git a/api/services/entities/knowledge_entities/knowledge_entities.py b/api/services/knowledge/entities/knowledge_entities.pysimilarity index 91%rename from api/services/entities/knowledge_entities/knowledge_entities.pyrename to api/services/knowledge/entities/knowledge_entities.pyindex f24283073b9..759fa7328be 100644--- a/api/services/entities/knowledge_entities/knowledge_entities.py+++ b/api/services/knowledge/entities/knowledge_entities.py@@ -2,8 +2,7 @@ from typing import Annotated, Any, Literalfrom pydantic import BaseModel, Field, WithJsonSchema, field_validator-from core.rag.entities import Rule-from core.rag.entities.metadata_entities import MetadataFilteringCondition+from core.rag.entities import MetadataFilteringCondition, Rulefrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom libs.helper import UUIDStr@@ -250,24 +249,6 @@ class SegmentCreateArgs(BaseModel):attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")-class SegmentUpdateArgs(BaseModel):- content: str | None = Field(default=None, description="Updated chunk text content.")- answer: str | None = Field(default=None, description="Updated answer content for QA mode.")- keywords: list[str] | None = Field(default=None, description="Updated keywords for the chunk.")- regenerate_child_chunks: bool = Field(- default=False,- description="Whether to regenerate child chunks after updating a parent chunk.",- )- enabled: bool | None = Field(default=None, description="Whether the chunk is enabled.")- attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")- summary: str | None = Field(default=None, description="Summary content for summary index.")---class ChildChunkUpdateArgs(BaseModel):- id: str | None = Field(default=None, description="Existing child chunk ID. Omit to create a new child chunk.")- content: str = Field(description="Child chunk text content.")--class MetadataArgs(BaseModel):type: Literal["string", "number", "time"] = Field(description="`string` for text values, `number` for numeric values, `time` for date/time values."diff --git a/api/services/knowledge/entities/segments.py b/api/services/knowledge/entities/segments.pynew file mode 100644index 00000000000..8431265a4b7--- /dev/null+++ b/api/services/knowledge/entities/segments.py@@ -0,0 +1,79 @@+"""Detached knowledge segment values shared by application ports."""++from datetime import datetime++from pydantic import BaseModel, ConfigDict, Field+++class ChildChunkUpdateArgs(BaseModel):+ id: str | None = Field(default=None, description="Existing child chunk ID. Omit to create a new child chunk.")+ content: str = Field(description="Child chunk text content.")+++class SegmentAttachmentRecord(BaseModel):+ model_config = ConfigDict(from_attributes=True, frozen=True)++ id: str+ name: str+ size: int+ extension: str+ mime_type: str | None+ source_url: str+++class ChildChunkRecord(BaseModel):+ model_config = ConfigDict(from_attributes=True, frozen=True)++ id: str+ segment_id: str+ content: str+ position: int+ word_count: int+ type: str+ created_at: datetime+ updated_at: datetime+++class SegmentRecord(BaseModel):+ model_config = ConfigDict(from_attributes=True, frozen=True)++ id: str+ position: int+ document_id: str+ content: str+ sign_content: str+ answer: str | None+ word_count: int+ tokens: int+ keywords: tuple[str, ...] | None+ index_node_id: str | None+ index_node_hash: str | None+ hit_count: int+ enabled: bool+ disabled_at: datetime | None+ disabled_by: str | None+ status: str+ created_by: str+ created_at: datetime+ updated_at: datetime+ updated_by: str | None+ indexing_at: datetime | None+ completed_at: datetime | None+ error: str | None+ stopped_at: datetime | None+ child_chunks: tuple[ChildChunkRecord, ...]+ attachments: tuple[SegmentAttachmentRecord, ...]+ summary: str | None+++class SegmentUpdateArgs(BaseModel):+ content: str | None = Field(default=None, description="Updated chunk text content.")+ answer: str | None = Field(default=None, description="Updated answer content for QA mode.")+ keywords: list[str] | None = Field(default=None, description="Updated keywords for the chunk.")+ regenerate_child_chunks: bool = Field(+ default=False,+ description="Whether to regenerate child chunks after updating a parent chunk.",+ )+ enabled: bool | None = Field(default=None, description="Whether the chunk is enabled.")+ attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")+ summary: str | None = Field(default=None, description="Summary content for summary index.")diff --git a/api/services/knowledge/external/__init__.py b/api/services/knowledge/external/__init__.pynew file mode 100644index 00000000000..ed656469c7a--- /dev/null+++ b/api/services/knowledge/external/__init__.py@@ -0,0 +1 @@+"""External knowledge connection and retrieval use cases."""diff --git a/api/services/knowledge/external/adapters.py b/api/services/knowledge/external/adapters.pynew file mode 100644index 00000000000..3add04e9425--- /dev/null+++ b/api/services/knowledge/external/adapters.py@@ -0,0 +1,153 @@+"""External knowledge persistence and network adapters with bounded sessions."""++from collections.abc import Mapping+from typing import Any++from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.rag.entities import MetadataFilteringCondition+from libs.pagination import clamp_pagination+from machinery.context import RequestContext+from models.dataset import DatasetQuery, ExternalKnowledgeApis+from models.enums import CreatorUserRole, DatasetQuerySource+from repositories.knowledge.dataset_read_repository import (+ get_external_api_bindings_batch,+ get_external_api_dataset_bindings,+)+from services.enterprise import rbac_service+from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload+from services.hit_testing_service import HitTestingService+from services.knowledge.dataset_read_service import load_dataset_detail+from services.knowledge.datasets.adapters import require_dataset+from services.knowledge.entities.datasets import DatasetDetailRecord+from services.knowledge.external.application import ExternalTemplateNotFoundError+from services.knowledge.external.service import ExternalDatasetService+from services.knowledge.resource_scope import DatasetRef+++class SQLAlchemyExternalKnowledgeOperations:+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._sessions = session_factory++ @staticmethod+ def _template(session: Session, workspace_id: str, template_id: str) -> ExternalKnowledgeApis:+ template = session.scalar(+ select(ExternalKnowledgeApis).where(+ ExternalKnowledgeApis.id == template_id, ExternalKnowledgeApis.tenant_id == workspace_id+ )+ )+ if template is None:+ raise ExternalTemplateNotFoundError("API template not found.")+ return template++ @staticmethod+ def _values(session: Session, template: ExternalKnowledgeApis) -> dict[str, Any]:+ return dict(template.to_dict(dataset_bindings=get_external_api_dataset_bindings(template, session=session)))++ def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:+ page, limit = clamp_pagination(page, limit, 100)+ with self._sessions() as session:+ templates, total = ExternalDatasetService.get_external_knowledge_apis(+ page, limit, workspace_id, keyword, session=session+ )+ bindings = get_external_api_bindings_batch(templates, session=session)+ return {+ "data": [item.to_dict(dataset_bindings=bindings.get(item.id, [])) for item in templates],+ "has_more": page * limit < total,+ "total": total,+ "page": page,+ "limit": limit,+ }++ def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]:+ with self._sessions() as session:+ return self._values(session, self._template(session, workspace_id, template_id))++ def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:+ # The legacy creator probes the endpoint before its first SQL statement.+ # An unopened session keeps that external request outside a transaction.+ with self._sessions() as session:+ template = ExternalDatasetService.create_external_knowledge_api(+ context.active_workspace_id,+ context.account_id,+ {"name": name, "settings": dict(settings)},+ session=session,+ )+ result = self._values(session, template)+ session.commit()+ return result++ def update_template(+ self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]+ ) -> dict[str, Any]:+ with self._sessions.begin() as session:+ self._template(session, context.active_workspace_id, template_id)+ template = ExternalDatasetService.update_external_knowledge_api(+ context.active_workspace_id,+ context.account_id,+ template_id,+ {"name": name, "settings": dict(settings)},+ session=session,+ )+ return self._values(session, template)++ def delete_template(self, workspace_id: str, template_id: str) -> None:+ with self._sessions.begin() as session:+ session.delete(self._template(session, workspace_id, template_id))++ def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]:+ with self._sessions() as session:+ return ExternalDatasetService.external_knowledge_api_use_check(template_id, workspace_id, session=session)++ def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:+ with self._sessions() as session:+ self._template(session, context.active_workspace_id, payload.external_knowledge_api_id)+ dataset = ExternalDatasetService.create_external_dataset(+ context.active_workspace_id, context.account_id, payload, session=session+ )+ permissions = rbac_service.RBACService.DatasetPermissions.batch_get(+ context.active_workspace_id, context.account_id, [dataset.id], session=session+ )+ result = load_dataset_detail(dataset, session=session)+ result["permission_keys"] = permissions.get(dataset.id, [])+ session.commit()+ return result++ def retrieve(+ self,+ context: RequestContext,+ ref: DatasetRef,+ *,+ query: str,+ retrieval_model: dict[str, Any] | None,+ metadata_filters: dict[str, Any] | None,+ ) -> dict[str, Any]:+ with self._sessions() as session:+ dataset = require_dataset(session, ref)+ if dataset.provider != "external":+ return {"query": {"content": query}, "records": []}+ request = ExternalDatasetService.prepare_external_knowledge_retrieval(+ tenant_id=ref.tenant_id,+ dataset_id=ref.dataset_id,+ query=HitTestingService.escape_query_for_search(query),+ external_retrieval_parameters=retrieval_model or {},+ metadata_condition=MetadataFilteringCondition.model_validate(metadata_filters)+ if metadata_filters+ else None,+ session=session,+ )+ documents = ExternalDatasetService.execute_external_knowledge_retrieval(request)+ with self._sessions.begin() as session:+ require_dataset(session, ref)+ session.add(+ DatasetQuery(+ dataset_id=ref.dataset_id,+ content=query,+ source=DatasetQuerySource.HIT_TESTING,+ source_app_id=None,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=context.account_id,+ )+ )+ return dict(HitTestingService.compact_external_retrieve_response(dataset, query, documents))diff --git a/api/services/knowledge/external/application.py b/api/services/knowledge/external/application.pynew file mode 100644index 00000000000..8fa758e1a54--- /dev/null+++ b/api/services/knowledge/external/application.py@@ -0,0 +1,104 @@+"""External knowledge use cases without transport or persistence dependencies."""++from collections.abc import Mapping+from typing import Any, Protocol++from machinery.context import RequestContext+from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.datasets import DatasetDetailRecord+from services.knowledge.resource_scope import DatasetRef+++class ExternalTemplateNotFoundError(Exception):+ pass+++class ExternalHitTestingError(Exception):+ pass+++class ExternalKnowledgeOperations(Protocol):+ def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]: ...+ def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]: ...+ def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]: ...+ def update_template(+ self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]+ ) -> dict[str, Any]: ...+ def delete_template(self, workspace_id: str, template_id: str) -> None: ...+ def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]: ...+ def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord: ...+ def retrieve(+ self,+ context: RequestContext,+ ref: DatasetRef,+ *,+ query: str,+ retrieval_model: dict[str, Any] | None,+ metadata_filters: dict[str, Any] | None,+ ) -> dict[str, Any]: ...+++class ExternalKnowledgeApplicationService:+ def __init__(self, *, dataset_access: DatasetAccess, operations: ExternalKnowledgeOperations) -> None:+ self._dataset_access = dataset_access+ self._operations = operations++ @staticmethod+ def _validate_settings(settings: Mapping[str, Any]) -> None:+ if not settings:+ raise ValueError("api list is empty")+ if not settings.get("endpoint"):+ raise ValueError("endpoint is required")+ if not settings.get("api_key"):+ raise ValueError("api_key is required")++ def list_templates(self, context: RequestContext, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:+ return self._operations.list_templates(context.active_workspace_id, page=page, limit=limit, keyword=keyword)++ def get_template(self, context: RequestContext, *, template_id: str) -> dict[str, Any]:+ return self._operations.get_template(context.active_workspace_id, template_id)++ def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:+ self._validate_settings(settings)+ return self._operations.create_template(context, name=name, settings=settings)++ def update_template(+ self, context: RequestContext, *, template_id: str, name: str, settings: Mapping[str, Any]+ ) -> dict[str, Any]:+ self._validate_settings(settings)+ return self._operations.update_template(context, template_id, name=name, settings=settings)++ def delete_template(self, context: RequestContext, *, template_id: str) -> None:+ self._operations.delete_template(context.active_workspace_id, template_id)++ def template_usage(self, context: RequestContext, *, template_id: str) -> tuple[bool, int]:+ return self._operations.template_usage(context.active_workspace_id, template_id)++ def create_dataset(self, context: RequestContext, *, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:+ return self._operations.create_dataset(context, payload)++ def hit_testing(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ query: str,+ retrieval_model: dict[str, Any] | None,+ metadata_filters: dict[str, Any] | None,+ ) -> dict[str, Any]:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ if not query:+ raise ValueError("Query or attachment_ids is required")+ if len(query) > 250:+ raise ValueError("Query cannot exceed 250 characters")+ try:+ return self._operations.retrieve(+ context,+ DatasetRef(dataset.workspace_id, dataset.id),+ query=query,+ retrieval_model=retrieval_model,+ metadata_filters=metadata_filters,+ )+ except Exception as error:+ raise ExternalHitTestingError(str(error)) from errordiff --git a/api/services/external_knowledge_service.py b/api/services/knowledge/external/service.pysimilarity index 93%rename from api/services/external_knowledge_service.pyrename to api/services/knowledge/external/service.pyindex 76620c48f75..8d7d5855800 100644--- a/api/services/external_knowledge_service.py+++ b/api/services/knowledge/external/service.py@@ -358,6 +358,22 @@ class ExternalDatasetService:the inner knowledge retrieval API—can consistently expose``502 external_knowledge_failed``."""+ request = ExternalDatasetService.prepare_external_knowledge_retrieval(+ tenant_id, dataset_id, query, external_retrieval_parameters, metadata_condition, session=session+ )+ return ExternalDatasetService.execute_external_knowledge_retrieval(request)++ @staticmethod+ def prepare_external_knowledge_retrieval(+ tenant_id: str,+ dataset_id: str,+ query: str,+ external_retrieval_parameters: dict[str, Any],+ metadata_condition: MetadataFilteringCondition | None = None,+ *,+ session: Session,+ ) -> ExternalKnowledgeApiSetting:+ """Resolve owned configuration into a request that outlives the read session."""external_knowledge_binding = session.scalar(select(ExternalKnowledgeBindings).where(ExternalKnowledgeBindings.dataset_id == dataset_id, ExternalKnowledgeBindings.tenant_id == tenant_id)@@ -393,14 +409,19 @@ class ExternalDatasetService:"metadata_condition": metadata_condition.model_dump() if metadata_condition else None,}+ return ExternalKnowledgeApiSetting(+ url=f"{settings.get('endpoint')}/retrieval",+ request_method="post",+ headers=headers,+ params=request_params,+ )++ @staticmethod+ def execute_external_knowledge_retrieval(request: ExternalKnowledgeApiSetting) -> list[Any]:+ """Execute and validate a prepared request without holding a database session."""try:response = ExternalDatasetService.process_external_api(- ExternalKnowledgeApiSetting(- url=f"{settings.get('endpoint')}/retrieval",- request_method="post",- headers=headers,- params=request_params,- ),+ request,None,)except ExternalKnowledgeRetrievalError:diff --git a/api/services/knowledge/indexing/__init__.py b/api/services/knowledge/indexing/__init__.pynew file mode 100644index 00000000000..0629046e587--- /dev/null+++ b/api/services/knowledge/indexing/__init__.py@@ -0,0 +1 @@+"""Use-case-centered stored-document indexing slice."""diff --git a/api/services/knowledge/indexing/adapters/__init__.py b/api/services/knowledge/indexing/adapters/__init__.pynew file mode 100644index 00000000000..f8596026c1d--- /dev/null+++ b/api/services/knowledge/indexing/adapters/__init__.py@@ -0,0 +1 @@+"""Infrastructure adapters for the knowledge indexing slice."""diff --git a/api/services/knowledge/indexing/adapters/cleanup.py b/api/services/knowledge/indexing/adapters/cleanup.pynew file mode 100644index 00000000000..e6ef60a5cb4--- /dev/null+++ b/api/services/knowledge/indexing/adapters/cleanup.py@@ -0,0 +1,120 @@+"""Delete document indexes between bounded database transactions."""++from collections.abc import Callable, Mapping, Sequence++from sqlalchemy import delete, select+from sqlalchemy.orm import Session++from configs import dify_config+from core.rag.datasource.keyword.jieba.jieba import Jieba+from core.rag.datasource.vdb.vector_factory import Vector+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from extensions.ext_redis import redis_client+from models.dataset import ChildChunk, Dataset, DocumentSegment, DocumentSegmentSummary+from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table+from repositories.knowledge.keyword_table_repository import persist_keyword_table+++def clean_document_indexes(+ *, dataset_id: str, document_ids: Sequence[str], doc_form: str | None, new_session: Callable[[], Session]+) -> str | None:+ """Clean indexes even when the owning document rows have already been deleted.++ Snapshot the dataset-owned rows, close the read session, remove external+ indexes, then delete the selected summary and child rows in one transaction.+ A failed vector deletion leaves those rows available for a retry.+ Empty document selections never mean deleting the entire dataset index.+ """+ if not document_ids:+ return None+ with new_session() as session:+ dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id))+ if dataset is None:+ return None+ tenant_id = dataset.tenant_id+ segments = session.scalars(+ select(DocumentSegment).where(+ DocumentSegment.tenant_id == tenant_id,+ DocumentSegment.dataset_id == dataset_id,+ DocumentSegment.document_id.in_(document_ids),+ )+ ).all()+ summaries = session.scalars(+ select(DocumentSegmentSummary).where(+ DocumentSegmentSummary.dataset_id == dataset_id,+ DocumentSegmentSummary.document_id.in_(document_ids),+ )+ ).all()+ children = session.scalars(+ select(ChildChunk).where(+ ChildChunk.tenant_id == tenant_id,+ ChildChunk.dataset_id == dataset_id,+ ChildChunk.document_id.in_(document_ids),+ )+ ).all()+ summary_ids = [summary.id for summary in summaries]+ child_ids = [child.id for child in children]+ summary_node_ids = [summary.summary_index_node_id for summary in summaries if summary.summary_index_node_id]+ body_node_ids = (+ [child.index_node_id for child in children if child.index_node_id]+ if doc_form == IndexStructureType.PARENT_CHILD_INDEX+ else [segment.index_node_id for segment in segments if segment.index_node_id]+ )+ high_quality = dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY+ node_ids = list(dict.fromkeys([*summary_node_ids, *body_node_ids]))+ vector_type = Vector.resolve_vector_type(dataset, session=session) if high_quality and node_ids else None+ session.expunge(dataset)++ if not node_ids and not summary_ids and not child_ids:+ return None++ if high_quality:+ if node_ids:+ Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(node_ids)+ elif body_node_ids:+ # The keyword adapter owns its lock and storage I/O; callbacks only do SQL.+ def read() -> tuple[str, str | None]:+ with new_session() as session:+ row = get_dataset_keyword_table(dataset, session=session)+ return (+ (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)+ )++ def write(storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]) -> None:+ with new_session() as session, session.begin():+ persist_keyword_table(+ session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ storage_type=storage_type,+ data=data,+ keywords=keywords,+ )++ Jieba(dataset).update_texts(+ [],+ read=read,+ write=write,+ lock=redis_client.lock(f"keyword_indexing_lock_{dataset_id}", timeout=600),+ deleted_ids=body_node_ids,+ )++ with new_session() as session, session.begin():+ if summary_ids:+ session.execute(+ delete(DocumentSegmentSummary).where(+ DocumentSegmentSummary.dataset_id == dataset_id,+ DocumentSegmentSummary.document_id.in_(document_ids),+ DocumentSegmentSummary.id.in_(summary_ids),+ )+ )+ if child_ids:+ session.execute(+ delete(ChildChunk).where(+ ChildChunk.tenant_id == tenant_id,+ ChildChunk.dataset_id == dataset_id,+ ChildChunk.document_id.in_(document_ids),+ ChildChunk.id.in_(child_ids),+ )+ )+ return tenant_iddiff --git a/api/services/knowledge/indexing/adapters/estimate.py b/api/services/knowledge/indexing/adapters/estimate.pynew file mode 100644index 00000000000..c09b8c42438--- /dev/null+++ b/api/services/knowledge/indexing/adapters/estimate.py@@ -0,0 +1,187 @@+"""Infrastructure adapters for indexing estimates."""++import json+import logging+from collections.abc import Mapping+from typing import cast++from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from configs import dify_config+from core.credit_usage import CreditUsageCreatedBy+from core.entities.knowledge_entities import IndexingEstimate, PreviewDetail, QAPreviewDetail+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError+from core.model_context import with_credit_usage_created_by+from core.model_manager import ModelInstance, ModelManager+from core.plugin.impl.exc import PluginDaemonClientSideError+from core.rag.extractor.entity.extract_setting import ExtractSetting+from core.rag.index_processor.constant.index_type import IndexTechniqueType+from core.rag.index_processor.index_processor_base import SummaryIndexSettingDict+from core.rag.index_processor.index_processor_factory import IndexProcessorFactory+from core.tools.utils.web_reader_tool import get_image_upload_file_ids+from enums import DeploymentEdition+from extensions.ext_storage import storage+from graphon.model_runtime.entities.model_entities import ModelType+from models.dataset import Dataset, DatasetProcessRule+from repositories.knowledge.dataset_repository import _get_dataset+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.indexing.estimate import (+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+)+from services.knowledge.resource_scope import DatasetRef++logger = logging.getLogger(__name__)+++class SQLAlchemyProcessRuleReader:+ """Load one tenant-owned process rule in a bounded read session."""++ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None:+ with self._session_factory() as session:+ process_rule = session.scalar(+ select(DatasetProcessRule)+ .join(Dataset, Dataset.id == DatasetProcessRule.dataset_id)+ .where(+ DatasetProcessRule.id == process_rule_id,+ DatasetProcessRule.dataset_id == dataset_ref.dataset_id,+ Dataset.tenant_id == dataset_ref.tenant_id,+ )+ .limit(1)+ )+ if process_rule is None:+ return None+ rules = json.loads(process_rule.rules) if process_rule.rules else {}+ return {"mode": str(process_rule.mode), "rules": rules}+++class IndexingEstimateAdapter:+ """Extract previews without depending on the document indexing execution flow."""++ def __init__(+ self,+ *,+ session_factory: sessionmaker[Session],+ ) -> None:+ self._session_factory = session_factory+ self._uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def run(+ self,+ tenant_id: str,+ extract_settings: list[ExtractSetting],+ tmp_processing_rule: Mapping[str, object],+ doc_form: str | None = None,+ doc_language: str = "English",+ dataset_id: str | None = None,+ indexing_technique: str = "economy",+ ) -> IndexingEstimate:+ try:+ return self._estimate(+ tenant_id, extract_settings, tmp_processing_rule, doc_form, doc_language, dataset_id, indexing_technique+ )+ except (LLMBadRequestError, ProviderTokenNotInitError, PluginDaemonClientSideError) as error:+ raise IndexingEstimateProviderUnavailableError(error.description) from error+ except Exception as error:+ raise IndexingEstimateExecutionError(str(error)) from error++ def _estimate(+ self,+ tenant_id: str,+ extract_settings: list[ExtractSetting],+ processing_rule: Mapping[str, object],+ doc_form: str | None,+ doc_language: str,+ dataset_id: str | None,+ indexing_technique: str,+ ) -> IndexingEstimate:+ if (+ dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD+ and len(extract_settings) > dify_config.BATCH_UPLOAD_LIMIT+ ):+ raise ValueError(f"You have reached the batch upload limit of {dify_config.BATCH_UPLOAD_LIMIT}.")++ model = self._embedding_model(tenant_id, dataset_id, indexing_technique)+ processor = IndexProcessorFactory(doc_form).init_index_processor()+ previews: list[PreviewDetail] = []+ qa_previews: list[QAPreviewDetail] = []+ image_ids: set[str] = set()+ total_segments = 0+ # Legacy processors need a local session for extraction artifacts. No+ # document-state transaction is shared with this preview operation.+ with self._session_factory() as session:+ for setting in extract_settings:+ texts = processor.extract(setting, process_rule_mode=processing_rule["mode"], session=session)+ documents = processor.transform(+ texts,+ current_user=None,+ embedding_model_instance=model,+ process_rule={"mode": processing_rule["mode"], "rules": processing_rule.get("rules")},+ tenant_id=tenant_id,+ doc_language=doc_language,+ preview=True,+ session=session,+ )+ total_segments += len(documents)+ for document in documents:+ if doc_form == "qa_model":+ if len(qa_previews) < 10:+ qa_previews.append(+ QAPreviewDetail(+ question=document.page_content, answer=document.metadata.get("answer") or ""+ )+ )+ elif len(previews) < 10:+ preview = PreviewDetail(content=document.page_content)+ if document.children:+ preview.child_chunks = [child.page_content for child in document.children]+ previews.append(preview)+ image_ids.update(get_image_upload_file_ids(document.page_content))+ session.commit()++ # Storage I/O takes place after extraction commits and outside the+ # upload-record deletion transaction. Foreign tenant files are excluded.+ uploads = self._uploads.get_files(workspace_id=tenant_id, file_ids=tuple(image_ids))+ for upload in uploads.values():+ try:+ storage.delete(upload.key)+ except Exception:+ logger.exception("Delete image file failed while estimating indexing, upload_file_id: %s", upload.id)+ self._uploads.delete_files(workspace_id=tenant_id, file_ids=tuple(uploads))++ if doc_form == "qa_model":+ return IndexingEstimate(total_segments=total_segments * 20, qa_preview=qa_previews, preview=[])+ summary = processing_rule.get("summary_index_setting")+ if isinstance(summary, Mapping) and summary.get("enable") and previews:+ with self._session_factory() as session:+ previews = processor.generate_summary_preview(+ tenant_id, previews, cast(SummaryIndexSettingDict, summary), doc_language, session=session+ )+ return IndexingEstimate(total_segments=total_segments, preview=previews)++ def _embedding_model(self, tenant_id: str, dataset_id: str | None, technique: str) -> ModelInstance | None:+ provider = None+ model_name = None+ if dataset_id:+ with self._session_factory() as session:+ dataset = _get_dataset(session, DatasetRef(tenant_id, dataset_id))+ if dataset is None:+ raise ValueError("Dataset not found.")+ if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ technique = IndexTechniqueType.HIGH_QUALITY+ provider, model_name = dataset.embedding_model_provider, dataset.embedding_model+ if technique != IndexTechniqueType.HIGH_QUALITY:+ return None+ manager = ModelManager.for_tenant(tenant_id=tenant_id)+ if provider:+ if not model_name:+ raise ValueError("Embedding model is not configured.")+ return manager.get_model_instance(+ tenant_id=tenant_id, provider=provider, model_type=ModelType.TEXT_EMBEDDING, model=model_name+ )+ return manager.get_default_model_instance(tenant_id=tenant_id, model_type=ModelType.TEXT_EMBEDDING)diff --git a/api/services/knowledge/indexing/adapters/execution.py b/api/services/knowledge/indexing/adapters/execution.pynew file mode 100644index 00000000000..c5578cb97ef--- /dev/null+++ b/api/services/knowledge/indexing/adapters/execution.py@@ -0,0 +1,247 @@+"""Adapt legacy processors to committed document-indexing phases."""++from concurrent.futures import ThreadPoolExecutor+from typing import cast++from flask import Flask, current_app+from sqlalchemy.orm import Session, sessionmaker+from werkzeug.exceptions import HTTPException+from werkzeug.local import LocalProxy++from core.errors.error import (+ AppInvokeQuotaExceededError,+ InvokeRateLimitError,+ LLMError,+ ModelCurrentlyNotSupportError,+ ProviderTokenNotInitError,+ QuotaExceededError,+)+from core.model_manager import ModelManager+from core.plugin.impl.exc import PluginDaemonError+from core.rag.datasource.keyword.jieba.jieba import Jieba+from core.rag.datasource.vdb.vector_factory import Vector+from core.rag.embedding.token_counter import calculate_segment_token_counts+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from core.rag.index_processor.index_processor_factory import IndexProcessorFactory+from core.rag.models.document import Document+from extensions.application_services.data_sources import build_data_source_credentials+from extensions.ext_redis import redis_client+from extensions.otel import propagate_context+from graphon.model_runtime.entities.model_entities import ModelType+from libs import helper+from models.enums import DataSourceType+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.indexing.adapters.sources import (+ CompositeStoredSourceResolver,+ FileSourceAdapter,+ NotionSourceResolver,+ WebsiteSourceAdapter,+)+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument+from services.knowledge.resource_scope import DocumentRef+from services.vector_space_admission_service import VectorSpaceAdmissionService+++class IndexingExecutionAdapter:+ def __init__(+ self,+ *,+ session_factory: sessionmaker[Session],+ documents: SQLAlchemyDocumentRepository,+ segments: SQLAlchemySegmentRepository,+ uploads: SQLAlchemyKnowledgeUploadRepository,+ sources: CompositeStoredSourceResolver,+ ) -> None:+ self._session_factory = session_factory+ self._documents = documents+ self._segments = segments+ self._uploads = uploads+ self._sources = sources++ def describe_error(self, error: Exception) -> str:+ """Preserve provider and extractor descriptions without their transport prefixes."""+ if isinstance(+ error,+ (+ LLMError,+ ProviderTokenNotInitError,+ QuotaExceededError,+ AppInvokeQuotaExceededError,+ ModelCurrentlyNotSupportError,+ InvokeRateLimitError,+ PluginDaemonError,+ HTTPException,+ ),+ ):+ return str(error.description)+ return str(error)++ def check_paused(self, ref: DocumentRef) -> None:+ if redis_client.get(f"document_{ref.document_id}_is_paused"):+ raise DocumentIsPausedError(f"Document paused, document id: {ref.document_id}")++ def extract(self, document: IndexingDocument) -> list[Document]:+ if not document.processing_rule:+ raise ValueError("no process rule found")+ setting = self._sources.resolve_for_indexing(document.source)+ processor = IndexProcessorFactory(document.doc_form).init_index_processor()+ # Legacy extractors persist downloaded images and Notion source metadata.+ # Their session is independent of document-state write transactions.+ with self._session_factory() as session:+ texts = processor.extract(setting, process_rule_mode=document.processing_rule["mode"], session=session)+ session.commit()+ for text in texts:+ text.metadata.update(document_id=document.ref.document_id, dataset_id=document.ref.dataset.dataset_id)+ return texts++ def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]:+ dataset, _ = self._documents.get_indexing_models(document.ref)+ user = self._documents.get_indexing_user(document.ref)+ model = None+ if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)+ if dataset.embedding_model_provider:+ model = manager.get_model_instance(+ tenant_id=dataset.tenant_id,+ provider=dataset.embedding_model_provider,+ model_type=ModelType.TEXT_EMBEDDING,+ model=dataset.embedding_model,+ )+ else:+ model = manager.get_default_model_instance(+ tenant_id=dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING+ )+ processor = IndexProcessorFactory(document.doc_form).init_index_processor()+ with self._session_factory() as session:+ chunks = processor.transform(+ texts,+ user,+ embedding_model_instance=model,+ process_rule=document.processing_rule,+ tenant_id=dataset.tenant_id,+ doc_language=document.doc_language,+ session=session,+ )+ session.commit()+ return chunks++ def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]:+ dataset, _ = self._documents.get_indexing_models(document.ref)+ return calculate_segment_token_counts(dataset=dataset, documents=chunks)++ def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None:+ dataset, _ = self._documents.get_indexing_models(document.ref)+ with self._session_factory() as session:+ VectorSpaceAdmissionService().ensure_document_can_be_indexed(+ dataset=dataset,+ document_id=document.ref.document_id,+ doc_form=document.doc_form,+ documents=chunks,+ include_summaries=document.need_summary,+ session=session,+ )++ def load(self, document: IndexingDocument, chunks: list[Document]) -> None:+ dataset, _ = self._documents.get_indexing_models(document.ref)+ app = cast(LocalProxy[Flask], current_app)._get_current_object()+ if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ groups: list[list[Document]] = [[] for _ in range(10)]+ for chunk in chunks:+ groups[int(helper.generate_text_hash(chunk.page_content), 16) % 10].append(chunk)+ with ThreadPoolExecutor(max_workers=10) as executor:+ futures = [+ executor.submit(+ propagate_context(self._process_chunk),+ app,+ document.ref,+ group,+ False,+ )+ for group in groups+ if group+ ]+ for future in futures:+ future.result()+ elif (+ dataset.indexing_technique == IndexTechniqueType.ECONOMY+ and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX+ ):+ # A future propagates keyword failures; Thread.join() used to hide+ # them and incorrectly let the document become completed.+ with ThreadPoolExecutor(max_workers=1) as executor:+ executor.submit(propagate_context(self._process_chunk), app, document.ref, chunks, True).result()++ def _process_chunk(self, app: Flask, ref: DocumentRef, chunks: list[Document], keywords: bool) -> None:+ with app.app_context():+ self.check_paused(ref)+ # Workers load their own inputs and own their sessions. No ORM+ # instance or session crosses the executor boundary.+ dataset, document = self._documents.get_indexing_models(ref)+ if keywords:+ # Keep the read/modify/write lock while each repository call+ # closes its transaction before keyword extraction or storage I/O.+ Jieba(dataset).update_texts(+ chunks,+ read=lambda: self._segments.get_keyword_table(ref.dataset),+ write=lambda storage_type, data, selected: self._segments.save_keyword_table(+ ref.dataset, storage_type=storage_type, data=data, keywords=selected+ ),+ lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),+ )+ elif dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ with self._session_factory() as session:+ vector_type = Vector.resolve_vector_type(dataset, session=session)+ attachments = (+ [attachment for chunk in chunks for attachment in chunk.attachments or []]+ if dataset.is_multimodal+ else []+ )+ files = self._uploads.get_files(+ workspace_id=ref.dataset.tenant_id,+ file_ids=[attachment.metadata["doc_id"] for attachment in attachments],+ )+ # All database inputs are materialized before initializing the+ # vector backend, embedding text/images, or reading object storage.+ vector = Vector(dataset, session=None, vector_type=vector_type)+ if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:+ for chunk in chunks:+ if chunk.children:+ vector.create([Document.model_validate(child.model_dump()) for child in chunk.children])+ else:+ vector.create(chunks)+ if attachments:+ vector.create_multimodal(attachments, upload_files=files)+ self.check_paused(ref)+ self._segments.complete_indexing_segments(ref, [chunk.metadata["doc_id"] for chunk in chunks])+++def build_document_indexing_service(+ *,+ session_factory: sessionmaker[Session],+ enforce_vector_space_admission: bool = False,+) -> DocumentIndexingService:+ """Assemble a fresh indexing use case from explicit database dependencies."""+ documents = SQLAlchemyDocumentRepository(session_factory=session_factory)+ segments = SQLAlchemySegmentRepository(session_factory=session_factory)+ uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)+ credentials = build_data_source_credentials(database_client=session_factory)+ sources = CompositeStoredSourceResolver(+ adapters={+ DataSourceType.UPLOAD_FILE: FileSourceAdapter(uploads=uploads),+ DataSourceType.NOTION_IMPORT: NotionSourceResolver(+ actor_credentials=credentials.actor, stored_credentials=credentials.stored+ ),+ DataSourceType.WEBSITE_CRAWL: WebsiteSourceAdapter(),+ }+ )+ return DocumentIndexingService(+ documents=documents,+ segments=segments,+ backend=IndexingExecutionAdapter(+ session_factory=session_factory, documents=documents, segments=segments, uploads=uploads, sources=sources+ ),+ enforce_vector_space_admission=enforce_vector_space_admission,+ )diff --git a/api/services/knowledge/indexing/adapters/sources.py b/api/services/knowledge/indexing/adapters/sources.pynew file mode 100644index 00000000000..08a2966829a--- /dev/null+++ b/api/services/knowledge/indexing/adapters/sources.py@@ -0,0 +1,282 @@+"""Datasource-specific extraction input resolvers."""++from collections.abc import Mapping, Sequence+from typing import Protocol++from configs import dify_config+from core.rag.extractor.entity.datasource_type import DatasourceType+from core.rag.extractor.entity.extract_setting import (+ ExtractSetting,+ NotionInfo,+ StoredDocumentExtractionInput,+ UploadFileExtractionInput,+ WebsiteInfo,+)+from services.data_source.credential_gateway import (+ ActorDatasourceCredentialResolver,+ NotionCredentialUnavailableError,+ StoredDatasourceCredentialResolver,+ resolve_stored_notion_access_token,+)+from services.data_source.entities.notion_import import NotionPageType+from services.knowledge.indexing.errors import (+ IndexingInputSourceError,+ SourceCredentialUnavailableError,+ UnsupportedStoredSourceError,+)+from services.knowledge.indexing.estimate import StoredSource++_NOTION_PROVIDER = "notion_datasource"+_NOTION_PLUGIN = "langgenius/notion_datasource"+++class StoredSourceAdapter(Protocol):+ """Resolve one persisted datasource kind."""++ def resolve(self, source: StoredSource) -> ExtractSetting: ...+++class KnowledgeUploadCatalog(Protocol):+ def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> Mapping[str, UploadFileExtractionInput]: ...+++class FileSourceAdapter:+ def __init__(self, *, uploads: KnowledgeUploadCatalog) -> None:+ self._uploads = uploads++ def resolve(self, source: StoredSource) -> ExtractSetting:+ info = _required_info(source, "no upload file found")+ file_id = _required_string(info, "upload_file_id", "no upload file found")+ return self.resolve_selection(+ workspace_id=source.document_ref.dataset.tenant_id,+ file_id=file_id,+ document_model=source.document_model,+ )++ def resolve_selection(+ self,+ *,+ workspace_id: str,+ file_id: str,+ document_model: str,+ ) -> ExtractSetting:+ return self.resolve_selections(workspace_id=workspace_id, file_ids=(file_id,), document_model=document_model)[+ file_id+ ]++ def resolve_selections(+ self, *, workspace_id: str, file_ids: Sequence[str], document_model: str+ ) -> dict[str, ExtractSetting]:+ if not file_ids:+ return {}+ uploads = self._uploads.get_by_ids(workspace_id=workspace_id, file_ids=file_ids)+ settings: dict[str, ExtractSetting] = {}+ for file_id in file_ids:+ upload = uploads.get(file_id)+ if upload is None:+ raise IndexingInputSourceError(f"no upload file found: {file_id}")+ settings[file_id] = ExtractSetting(+ datasource_type=DatasourceType.FILE, upload_file=upload, document_model=document_model+ )+ return settings+++class WebsiteSourceAdapter:+ def resolve(self, source: StoredSource) -> ExtractSetting:+ info = _required_info(source, "no website crawl info found")+ return self.resolve_selection(+ workspace_id=source.document_ref.dataset.tenant_id,+ provider=_required_string(info, "provider"),+ job_id=_required_string(info, "job_id"),+ url=_required_string(info, "url"),+ mode=_required_string(info, "mode"),+ only_main_content=bool(info.get("only_main_content", False)),+ document_model=source.document_model,+ )++ def resolve_selection(+ self,+ *,+ workspace_id: str,+ provider: str,+ job_id: str,+ url: str,+ mode: str,+ only_main_content: bool,+ document_model: str,+ ) -> ExtractSetting:+ return ExtractSetting(+ datasource_type=DatasourceType.WEBSITE,+ website_info=WebsiteInfo(+ provider=provider,+ job_id=job_id,+ url=url,+ mode=mode,+ tenant_id=workspace_id,+ only_main_content=only_main_content,+ ),+ document_model=document_model,+ )+++class NotionSourceResolver:+ """Resolve actor-selected and trusted-stored Notion extraction inputs."""++ def __init__(+ self,+ *,+ actor_credentials: ActorDatasourceCredentialResolver,+ stored_credentials: StoredDatasourceCredentialResolver,+ integration_token: str | None = None,+ ) -> None:+ self._actor_credentials = actor_credentials+ self._stored_credentials = stored_credentials+ self._integration_token = integration_token++ def resolve(self, source: StoredSource) -> ExtractSetting:+ info = _required_info(source, "no notion import info found")+ document_ref = source.document_ref+ credential_id = _string_value(info, "credential_id")+ token = self._stored_token(source, credential_id)+ return self._prepared(+ tenant_id=document_ref.dataset.tenant_id,+ notion_workspace_id=_required_string(info, "notion_workspace_id"),+ page_id=_required_string(info, "notion_page_id"),+ page_type=_notion_page_type(info),+ document_model=source.document_model,+ credential_id=credential_id,+ token=token,+ )++ def _stored_token(self, source: StoredSource, credential_id: str | None) -> str:+ document_ref = source.document_ref++ def load_token(saved_id: str) -> str:+ credentials = self._stored_credentials.resolve_for_document(+ workspace_id=document_ref.dataset.tenant_id,+ dataset_id=document_ref.dataset.dataset_id,+ document_id=document_ref.document_id,+ credential_id=saved_id,+ provider=_NOTION_PROVIDER,+ plugin_id=_NOTION_PLUGIN,+ )+ return self._required_token(credentials)++ try:+ return resolve_stored_notion_access_token(+ credential_id=credential_id,+ load_token=load_token,+ integration_token=self._integration_token or dify_config.NOTION_INTEGRATION_TOKEN,+ )+ except NotionCredentialUnavailableError as error:+ raise SourceCredentialUnavailableError("Notion credential is unavailable") from error++ def resolve_selection(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ notion_workspace_id: str,+ page_id: str,+ page_type: NotionPageType,+ document_model: str,+ ) -> ExtractSetting:+ credentials = self._actor_credentials.resolve(+ workspace_id=workspace_id,+ actor_id=actor_id,+ credential_id=credential_id,+ provider=_NOTION_PROVIDER,+ plugin_id=_NOTION_PLUGIN,+ )+ return self._prepared(+ tenant_id=workspace_id,+ notion_workspace_id=notion_workspace_id,+ page_id=page_id,+ page_type=page_type,+ document_model=document_model,+ credential_id=credential_id,+ token=self._required_token(credentials),+ )++ def _prepared(+ self,+ *,+ tenant_id: str,+ notion_workspace_id: str,+ page_id: str,+ page_type: NotionPageType,+ document_model: str,+ credential_id: str | None,+ token: str,+ ) -> ExtractSetting:+ return ExtractSetting(+ datasource_type=DatasourceType.NOTION,+ notion_info=NotionInfo(+ credential_id=credential_id,+ notion_workspace_id=notion_workspace_id,+ notion_obj_id=page_id,+ notion_page_type=page_type.value,+ tenant_id=tenant_id,+ notion_access_token=token,+ ),+ document_model=document_model,+ )++ @staticmethod+ def _required_token(credentials: Mapping[str, object]) -> str:+ token = credentials.get("integration_secret")+ if isinstance(token, str) and token:+ return token+ raise SourceCredentialUnavailableError("Notion credential is unavailable")+++class CompositeStoredSourceResolver:+ """Route persisted sources to datasource-specific adapters."""++ def __init__(self, *, adapters: Mapping[str, StoredSourceAdapter]) -> None:+ self._adapters = dict(adapters)++ def resolve(self, source: StoredSource) -> ExtractSetting:+ adapter = self._adapters.get(source.source_type)+ if adapter is None:+ raise UnsupportedStoredSourceError(source.source_type)+ return adapter.resolve(source)++ def resolve_for_indexing(self, source: StoredSource) -> ExtractSetting:+ setting = self.resolve(source)+ if setting.notion_info is not None:+ ref = source.document_ref+ setting.notion_info.document = StoredDocumentExtractionInput(+ id=ref.document_id,+ tenant_id=ref.dataset.tenant_id,+ dataset_id=ref.dataset.dataset_id,+ data_source_info_dict=dict(source.source_info) if source.source_info is not None else None,+ )+ return setting+++def _required_info(source: StoredSource, message: str) -> Mapping[str, object]:+ if source.source_info is None:+ raise IndexingInputSourceError(message)+ return source.source_info+++def _string_value(info: Mapping[str, object], key: str) -> str | None:+ value = info.get(key)+ return value if isinstance(value, str) and value else None+++def _required_string(info: Mapping[str, object], key: str, message: str | None = None) -> str:+ value = _string_value(info, key)+ if value is None:+ raise IndexingInputSourceError(message or f"invalid data source field: {key}")+ return value+++def _notion_page_type(info: Mapping[str, object]) -> NotionPageType:+ value = _required_string(info, "type")+ try:+ return NotionPageType(value)+ except ValueError as error:+ raise IndexingInputSourceError("invalid data source field: type") from errordiff --git a/api/services/knowledge/indexing/errors.py b/api/services/knowledge/indexing/errors.pynew file mode 100644index 00000000000..ab16e980ae0--- /dev/null+++ b/api/services/knowledge/indexing/errors.py@@ -0,0 +1,27 @@+"""Application failures raised while preparing or executing indexing."""+++class IndexingInputError(Exception):+ """Base class for failures while resolving stored extraction inputs."""+++class IndexingInputSourceError(IndexingInputError):+ """Raised when a persisted source cannot form a valid extraction input."""+++class UnsupportedStoredSourceError(IndexingInputSourceError):+ def __init__(self, source_type: str) -> None:+ self.source_type = source_type+ super().__init__(f"unsupported data source type: {source_type}")+++class SourceCredentialUnavailableError(IndexingInputSourceError):+ """Raised when a datasource cannot resolve its required credential policy."""+++class DocumentIsPausedError(Exception):+ """Indexing stopped because the owned document was paused."""+++class DocumentIsDeletedPausedError(Exception):+ """Indexing stopped because its document owner chain no longer exists."""diff --git a/api/services/knowledge/indexing/estimate.py b/api/services/knowledge/indexing/estimate.pynew file mode 100644index 00000000000..6e75705e27c--- /dev/null+++ b/api/services/knowledge/indexing/estimate.py@@ -0,0 +1,345 @@+"""Indexing-estimate application service and its consumer-owned ports."""++from collections.abc import Mapping, Sequence+from typing import NamedTuple, Never, Protocol++from core.entities.knowledge_entities import IndexingEstimate+from core.rag.extractor.entity.extract_setting import ExtractSetting+from machinery.context import RequestContext+from services.data_source.credential_gateway import DatasourceCredentialError+from services.data_source.entities.notion_import import NotionPageType+from services.knowledge.dataset_access import DatasetAccess, DatasetNotFoundError+from services.knowledge.entities.indexing_estimate import (+ NewEstimateSource,+ NewSourcesEstimateCommand,+ NotionEstimateSource,+ UploadFileEstimateSource,+ WebsiteEstimateSource,+ normalize_process_rule,+)+from services.knowledge.indexing.errors import (+ IndexingInputSourceError,+ SourceCredentialUnavailableError,+ UnsupportedStoredSourceError,+)+from services.knowledge.resource_scope import DatasetRef, DocumentRef+++class DatasetEstimateRecord(NamedTuple):+ """Dataset state required to estimate stored documents."""++ id: str+ workspace_id: str+ indexing_technique: str | None+++class EstimateDocumentRecord(NamedTuple):+ """Document state required by the stored-document estimate use case."""++ id: str+ workspace_id: str+ dataset_id: str+ data_source_type: str+ data_source_info: Mapping[str, object] | None+ indexing_status: str+ doc_form: str+ doc_language: str | None+ dataset_process_rule_id: str | None++ @property+ def ref(self) -> DocumentRef:+ return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)+++class StoredSource(NamedTuple):+ """A persisted extraction source bound to its owning document."""++ document_ref: DocumentRef+ source_type: str+ source_info: Mapping[str, object] | None+ document_model: str++ @classmethod+ def from_document(cls, document: EstimateDocumentRecord) -> "StoredSource":+ return cls(+ document_ref=document.ref,+ source_type=document.data_source_type,+ source_info=document.data_source_info,+ document_model=document.doc_form,+ )+++class EstimateDocumentReader(Protocol):+ def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None: ...++ def list_estimate_documents_by_batch(+ self, dataset_ref: DatasetRef, batch: str+ ) -> tuple[EstimateDocumentRecord, ...]: ...+++class EstimateDatasetReader(Protocol):+ def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None: ...+++class UploadEstimateSourceResolver(Protocol):+ def resolve_selections(+ self, *, workspace_id: str, file_ids: Sequence[str], document_model: str+ ) -> Mapping[str, ExtractSetting]: ...+++class NotionEstimateSourceResolver(Protocol):+ def resolve_selection(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ notion_workspace_id: str,+ page_id: str,+ page_type: NotionPageType,+ document_model: str,+ ) -> ExtractSetting: ...+++class WebsiteEstimateSourceResolver(Protocol):+ def resolve_selection(+ self,+ *,+ workspace_id: str,+ provider: str,+ job_id: str,+ url: str,+ mode: str,+ only_main_content: bool,+ document_model: str,+ ) -> ExtractSetting: ...+++class StoredEstimateSourceResolver(Protocol):+ def resolve(self, source: StoredSource) -> ExtractSetting: ...+++class ProcessRuleReader(Protocol):+ def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None: ...+++class IndexingEstimateRunner(Protocol):+ def run(+ self,+ tenant_id: str,+ extract_settings: list[ExtractSetting],+ tmp_processing_rule: Mapping[str, object],+ doc_form: str | None = None,+ doc_language: str = "English",+ dataset_id: str | None = None,+ indexing_technique: str = "economy",+ ) -> IndexingEstimate: ...+++class IndexingEstimateApplicationError(Exception):+ """Base class for framework-neutral estimate failures."""+++class EstimateSourceNotFoundError(IndexingEstimateApplicationError):+ def __init__(self, source_id: str) -> None:+ super().__init__(f"Estimate source not found: {source_id}")+++class UnsupportedEstimateSourceError(IndexingEstimateApplicationError):+ def __init__(self, source_type: str) -> None:+ super().__init__(f"Data source type not supported: {source_type}")+++class IndexingEstimateCredentialUnavailableError(IndexingEstimateApplicationError):+ """Raised when a datasource credential required by an estimate is unavailable."""+++class IndexingEstimateProviderUnavailableError(IndexingEstimateApplicationError):+ """Raised when a configured model or plugin provider is unavailable."""+++class IndexingEstimateExecutionError(IndexingEstimateApplicationError):+ """Raised when the estimate processor cannot complete an estimate."""+++class EstimateDocumentNotFoundError(IndexingEstimateApplicationError):+ def __init__(self) -> None:+ super().__init__("Document not found")+++class EstimateDocumentAlreadyFinishedError(IndexingEstimateApplicationError):+ def __init__(self) -> None:+ super().__init__("Document already finished")+++class IndexingEstimateApplicationService:+ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ datasets: EstimateDatasetReader,+ documents: EstimateDocumentReader,+ files: UploadEstimateSourceResolver,+ notion: NotionEstimateSourceResolver,+ websites: WebsiteEstimateSourceResolver,+ stored_sources: StoredEstimateSourceResolver,+ process_rules: ProcessRuleReader,+ runner: IndexingEstimateRunner,+ ) -> None:+ self._dataset_access = dataset_access+ self._datasets = datasets+ self._documents = documents+ self._files = files+ self._notion = notion+ self._websites = websites+ self._stored_sources = stored_sources+ self._process_rules = process_rules+ self._runner = runner++ def estimate_new_sources(+ self,+ context: RequestContext,+ command: NewSourcesEstimateCommand,+ ) -> IndexingEstimate:+ dataset_id = command.dataset_id or None+ if dataset_id is not None:+ dataset = self._dataset_access.require_accessible(context, dataset_id)+ dataset_id = dataset.id+ process_rule = normalize_process_rule(command.process_rule)++ try:+ file_settings = self._files.resolve_selections(+ workspace_id=context.active_workspace_id,+ file_ids=tuple(+ dict.fromkeys(+ source.file_id for source in command.sources if isinstance(source, UploadFileEstimateSource)+ )+ ),+ document_model=command.doc_form,+ )+ extract_settings = [+ file_settings[source.file_id]+ if isinstance(source, UploadFileEstimateSource)+ else self._resolve_new_source(context, source, document_model=command.doc_form)+ for source in command.sources+ ]+ if not extract_settings:+ return IndexingEstimate(total_segments=0, preview=[])+ return self._runner.run(+ tenant_id=context.active_workspace_id,+ extract_settings=extract_settings,+ tmp_processing_rule=process_rule,+ doc_form=command.doc_form,+ doc_language=command.doc_language,+ dataset_id=dataset_id,+ indexing_technique=command.indexing_technique,+ )+ except Exception as error:+ self._translate_error(error)++ def estimate_document(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ ) -> IndexingEstimate:+ accessible = self._dataset_access.require_accessible(context, dataset_id)+ dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)+ dataset = self._require_estimate_dataset(dataset_ref)+ document = self._documents.get_estimate_document(dataset_ref.document(document_id))+ if document is None:+ raise EstimateDocumentNotFoundError()+ return self._estimate_document(context, dataset, document)++ def estimate_batch(self, context: RequestContext, *, dataset_id: str, batch: str) -> IndexingEstimate:+ accessible = self._dataset_access.require_accessible(context, dataset_id)+ dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)+ dataset = self._require_estimate_dataset(dataset_ref)+ documents = self._documents.list_estimate_documents_by_batch(dataset_ref, batch)+ if not documents:+ raise EstimateDocumentNotFoundError()+ # A batch preview uses its first document, as the existing API does.+ return self._estimate_document(context, dataset, documents[0])++ def _estimate_document(+ self,+ context: RequestContext,+ dataset: DatasetEstimateRecord,+ document: EstimateDocumentRecord,+ ) -> IndexingEstimate:+ if document.indexing_status in {"completed", "error"}:+ raise EstimateDocumentAlreadyFinishedError()+ try:+ extract_setting = self._stored_sources.resolve(StoredSource.from_document(document))+ process_rule = self._load_process_rule(dataset, document)+ return self._runner.run(+ tenant_id=context.active_workspace_id,+ extract_settings=[extract_setting],+ tmp_processing_rule=process_rule,+ doc_form=document.doc_form,+ doc_language=document.doc_language or "English",+ dataset_id=dataset.id,+ indexing_technique=dataset.indexing_technique or "economy",+ )+ except Exception as error:+ self._translate_error(error)++ def _resolve_new_source(+ self, context: RequestContext, source: NewEstimateSource, *, document_model: str+ ) -> ExtractSetting:+ if isinstance(source, NotionEstimateSource):+ return self._notion.resolve_selection(+ workspace_id=context.active_workspace_id,+ actor_id=context.account_id,+ credential_id=source.credential_id,+ notion_workspace_id=source.workspace_id,+ page_id=source.page_id,+ page_type=source.page_type,+ document_model=document_model,+ )+ if isinstance(source, WebsiteEstimateSource):+ return self._websites.resolve_selection(+ workspace_id=context.active_workspace_id,+ provider=source.provider,+ job_id=source.job_id,+ url=source.url,+ mode=source.mode,+ only_main_content=source.only_main_content,+ document_model=document_model,+ )+ raise UnsupportedEstimateSourceError(source.source_type)++ def _load_process_rule(+ self,+ dataset: DatasetEstimateRecord,+ document: EstimateDocumentRecord,+ ) -> Mapping[str, object]:+ process_rule_id = document.dataset_process_rule_id+ if not process_rule_id:+ return {"mode": "automatic", "rules": {}}+ process_rule = self._process_rules.get_by_id(+ dataset_ref=DatasetRef(dataset.workspace_id, dataset.id),+ process_rule_id=process_rule_id,+ )+ if process_rule is None:+ raise EstimateSourceNotFoundError(process_rule_id)+ return process_rule++ @staticmethod+ def _translate_error(error: Exception) -> Never:+ if isinstance(error, IndexingEstimateApplicationError):+ raise error+ if isinstance(error, UnsupportedStoredSourceError):+ raise UnsupportedEstimateSourceError(error.source_type) from error+ if isinstance(error, (DatasourceCredentialError, SourceCredentialUnavailableError)):+ raise IndexingEstimateCredentialUnavailableError(str(error)) from error+ if isinstance(error, IndexingInputSourceError):+ raise EstimateSourceNotFoundError(str(error)) from error+ raise IndexingEstimateExecutionError(str(error)) from error++ def _require_estimate_dataset(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord:+ dataset = self._datasets.get_estimate_record(dataset_ref)+ if dataset is None:+ raise DatasetNotFoundError()+ return datasetdiff --git a/api/services/knowledge/indexing/execution.py b/api/services/knowledge/indexing/execution.pynew file mode 100644index 00000000000..dba53de6dbb--- /dev/null+++ b/api/services/knowledge/indexing/execution.py@@ -0,0 +1,138 @@+"""Document indexing phases and their application-owned persistence and execution ports."""++import logging+import time+from collections.abc import Callable, Mapping, Sequence+from dataclasses import dataclass+from typing import Literal, Protocol++from core.credit_usage import CreditUsageCreatedBy+from core.model_context import with_credit_usage_created_by+from core.rag.models.document import Document+from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError+from services.knowledge.indexing.estimate import StoredSource+from services.knowledge.resource_scope import DocumentRef++logger = logging.getLogger(__name__)+++@dataclass(frozen=True)+class IndexingDocument:+ source: StoredSource+ processing_rule: Mapping[str, object]+ doc_language: str+ need_summary: bool++ @property+ def ref(self) -> DocumentRef:+ return self.source.document_ref++ @property+ def doc_form(self) -> str:+ return self.source.document_model+++class IndexingDocumentStore(Protocol):+ def get_indexing_document(self, ref: DocumentRef) -> IndexingDocument | None: ...++ def mark_splitting(self, ref: DocumentRef) -> None: ...++ def complete_indexing(self, ref: DocumentRef, *, tokens: int, latency: float) -> None: ...++ def fail_indexing(self, ref: DocumentRef, error: str) -> None: ...+++class IndexingSegmentStore(Protocol):+ def clear_for_indexing(self, document: IndexingDocument) -> None: ...++ def save_for_indexing(+ self, document: IndexingDocument, chunks: list[Document], token_counts: list[int]+ ) -> None: ...++ def resume_indexing(self, document: IndexingDocument) -> tuple[list[Document], int]: ...+++class IndexingBackend(Protocol):+ def describe_error(self, error: Exception) -> str:+ """Translate backend-specific exceptions into the persisted error message."""+ ...++ def check_paused(self, ref: DocumentRef) -> None: ...++ def extract(self, document: IndexingDocument) -> list[Document]: ...++ def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]: ...++ def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None: ...++ def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]: ...++ def load(self, document: IndexingDocument, chunks: list[Document]) -> None: ...+++class DocumentIndexingService:+ """Coordinate committed phases; no caller session is held across execution."""++ def __init__(+ self,+ *,+ documents: IndexingDocumentStore,+ segments: IndexingSegmentStore,+ backend: IndexingBackend,+ enforce_vector_space_admission: bool = False,+ clock: Callable[[], float] = time.perf_counter,+ ) -> None:+ self._documents = documents+ self._segments = segments+ self._backend = backend+ self._enforce_vector_space_admission = enforce_vector_space_admission+ self._clock = clock++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def run(self, document_refs: Sequence[DocumentRef]) -> None:+ for ref in document_refs:+ self._run(ref, start="parsing")++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def run_in_splitting_status(self, ref: DocumentRef) -> None:+ self._run(ref, start="splitting")++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def run_in_indexing_status(self, ref: DocumentRef) -> None:+ self._run(ref, start="indexing")++ def _run(self, ref: DocumentRef, *, start: Literal["parsing", "splitting", "indexing"]) -> None:+ try:+ document = self._documents.get_indexing_document(ref)+ if document is None:+ logger.info("Document not found, skipping indexing: %s", ref)+ return+ self._backend.check_paused(ref)+ if start == "indexing":+ chunks, total_tokens = self._segments.resume_indexing(document)+ else:+ if start == "splitting":+ self._segments.clear_for_indexing(document)+ texts = self._backend.extract(document)+ self._documents.mark_splitting(ref)+ chunks = self._backend.transform(document, texts)+ # Recovery retains its existing admission policy; only normal+ # indexing callers that explicitly opt in reserve vector space.+ if start == "parsing" and self._enforce_vector_space_admission:+ self._backend.ensure_admission(document, chunks)+ token_counts = self._backend.count_tokens(document, chunks)+ total_tokens = sum(token_counts)+ self._segments.save_for_indexing(document, chunks, token_counts)++ started = self._clock()+ self._backend.check_paused(ref)+ self._backend.load(document, chunks)+ self._backend.check_paused(ref)+ self._documents.complete_indexing(ref, tokens=total_tokens, latency=self._clock() - started)+ except DocumentIsPausedError:+ raise+ except DocumentIsDeletedPausedError:+ logger.info("Document deleted during indexing: %s", ref)+ except Exception as error:+ logger.exception("Document indexing failed: %s", ref)+ self._documents.fail_indexing(ref, self._backend.describe_error(error))diff --git a/api/services/knowledge/metadata/__init__.py b/api/services/knowledge/metadata/__init__.pynew file mode 100644index 00000000000..e69de29bb2ddiff --git a/api/services/knowledge/metadata/application.py b/api/services/knowledge/metadata/application.pynew file mode 100644index 00000000000..6a39918cea6--- /dev/null+++ b/api/services/knowledge/metadata/application.py@@ -0,0 +1,125 @@+"""Metadata use cases with explicit ownership and persistence contracts."""++from collections.abc import Sequence+from dataclasses import dataclass+from datetime import datetime+from typing import Protocol, TypedDict++from core.rag.index_processor.constant.built_in_field import BuiltInField+from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.resource_scope import DatasetRef+++class MetadataInput(Protocol):+ @property+ def name(self) -> str: ...+ @property+ def type(self) -> str: ...+++class MetadataValue(Protocol):+ @property+ def id(self) -> str: ...+ @property+ def value(self) -> str | int | float | None: ...+++class MetadataOperation(Protocol):+ @property+ def document_id(self) -> str: ...+ @property+ def metadata_list(self) -> Sequence[MetadataValue]: ...+ @property+ def partial_update(self) -> bool: ...+++class MetadataOperationsInput(Protocol):+ @property+ def operation_data(self) -> Sequence[MetadataOperation]: ...+++@dataclass(frozen=True)+class MetadataRecord:+ id: str+ name: str+ type: str+ tenant_id: str+ dataset_id: str+ created_by: str+ created_at: datetime+ updated_by: str | None+ updated_at: datetime | None+++class MetadataFieldSummary(TypedDict):+ id: str+ name: str+ type: str+ count: int+++class MetadataList(TypedDict):+ doc_metadata: list[MetadataFieldSummary]+ built_in_field_enabled: bool+++class MetadataStore(Protocol):+ """Own bounded transactions and recheck the complete dataset owner chain."""++ def create(self, ref: DatasetRef, *, name: str, field_type: str, actor_id: str) -> MetadataRecord: ...+ def rename(self, ref: DatasetRef, metadata_id: str, *, name: str, actor_id: str) -> MetadataRecord: ...+ def delete(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord: ...+ def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None: ...+ def update_documents(self, ref: DatasetRef, operations: Sequence[MetadataOperation], *, actor_id: str) -> None: ...+ def list_fields(self, ref: DatasetRef) -> MetadataList: ...+++class MetadataService:+ def __init__(self, *, store: MetadataStore, dataset_access: DatasetAccess) -> None:+ self._store = store+ self._access = dataset_access++ def require_dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:+ dataset = self._access.require_accessible(context, dataset_id)+ return DatasetRef(dataset.workspace_id, dataset.id)++ @staticmethod+ def _validate_name(name: str) -> None:+ if len(name) > 255:+ raise ValueError("Metadata name cannot exceed 255 characters.")+ if name in BuiltInField:+ raise ValueError("Metadata name already exists in Built-in fields.")++ def create_metadata(self, ref: DatasetRef, args: MetadataInput, *, actor_id: str) -> MetadataRecord:+ self._validate_name(args.name)+ return self._store.create(ref, name=args.name, field_type=args.type, actor_id=actor_id)++ def update_metadata_name(self, ref: DatasetRef, metadata_id: str, name: str, *, actor_id: str) -> MetadataRecord:+ self._validate_name(name)+ return self._store.rename(ref, metadata_id, name=name, actor_id=actor_id)++ def delete_metadata(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord:+ return self._store.delete(ref, metadata_id)++ @staticmethod+ def get_built_in_fields() -> list[dict[str, str]]:+ return [+ {+ "name": field.value,+ "type": "time" if field in (BuiltInField.upload_date, BuiltInField.last_update_date) else "string",+ }+ for field in BuiltInField+ ]++ def enable_built_in_field(self, ref: DatasetRef) -> None:+ self._store.set_built_in(ref, enabled=True)++ def disable_built_in_field(self, ref: DatasetRef) -> None:+ self._store.set_built_in(ref, enabled=False)++ def update_documents_metadata(self, ref: DatasetRef, args: MetadataOperationsInput, *, actor_id: str) -> None:+ self._store.update_documents(ref, args.operation_data, actor_id=actor_id)++ def get_dataset_metadatas(self, ref: DatasetRef) -> MetadataList:+ return self._store.list_fields(ref)diff --git a/api/services/knowledge/resource_scope.py b/api/services/knowledge/resource_scope.pynew file mode 100644index 00000000000..2a29f9dedba--- /dev/null+++ b/api/services/knowledge/resource_scope.py@@ -0,0 +1,32 @@+"""Typed owner chains used by knowledge application and persistence ports."""++from __future__ import annotations++from typing import NamedTuple+++class DatasetRef(NamedTuple):+ """Identify a dataset inside its owning workspace."""++ tenant_id: str+ dataset_id: str++ def document(self, document_id: str) -> DocumentRef:+ return DocumentRef(dataset=self, document_id=document_id)+++class DocumentRef(NamedTuple):+ """Identify a document through its complete dataset owner chain."""++ dataset: DatasetRef+ document_id: str++ def segment(self, segment_id: str) -> SegmentRef:+ return SegmentRef(document=self, segment_id=segment_id)+++class SegmentRef(NamedTuple):+ """Identify a segment through its complete document owner chain."""++ document: DocumentRef+ segment_id: strdiff --git a/api/services/knowledge/retrieval/__init__.py b/api/services/knowledge/retrieval/__init__.pynew file mode 100644index 00000000000..675f7c2640b--- /dev/null+++ b/api/services/knowledge/retrieval/__init__.py@@ -0,0 +1 @@+"""Knowledge retrieval authorization and application operations."""diff --git a/api/services/knowledge/retrieval/attachments.py b/api/services/knowledge/retrieval/attachments.pynew file mode 100644index 00000000000..9c5ae792bec--- /dev/null+++ b/api/services/knowledge/retrieval/attachments.py@@ -0,0 +1,33 @@+"""Grant execution-local file access only for validated knowledge retrieval results."""++from collections.abc import Sequence++from sqlalchemy.orm import Session++from core.app.file_access import (+ get_current_file_access_scope,+ grant_retriever_segment_access,+ grant_upload_file_access,+)+from models.dataset import DocumentSegment+from models.model import UploadFile+from repositories.knowledge.dataset_read_repository import get_segment_attachment_files, is_retrieved_segment_owned+++def authorize_retrieved_segment(+ segment: DocumentSegment, *, tenant_id: str, dataset_ids: Sequence[str], session: Session+) -> Sequence[UploadFile] | None:+ """Authorize the owner chain and attachments before content links are signed.++ A valid segment with no attachments returns an empty sequence; an invalid+ retrieval result returns None and must not be included in the response.+ """+ scope = get_current_file_access_scope()+ if scope is not None and scope.tenant_id != tenant_id:+ return None+ if not is_retrieved_segment_owned(segment, tenant_id=tenant_id, dataset_ids=dataset_ids, session=session):+ return None+ attachments = get_segment_attachment_files(segment, session=session)+ grant_retriever_segment_access([segment.id])+ grant_upload_file_access(file.id for file in attachments)+ return attachmentsdiff --git a/api/services/knowledge/segments/__init__.py b/api/services/knowledge/segments/__init__.pynew file mode 100644index 00000000000..45c7098779b--- /dev/null+++ b/api/services/knowledge/segments/__init__.py@@ -0,0 +1 @@+"""Dataset segment use cases."""diff --git a/api/services/knowledge/segments/adapters.py b/api/services/knowledge/segments/adapters.pynew file mode 100644index 00000000000..ae501e3b8d3--- /dev/null+++ b/api/services/knowledge/segments/adapters.py@@ -0,0 +1,82 @@+"""Infrastructure adapters for dataset segment use cases."""++from __future__ import annotations++from collections.abc import Callable+from typing import Any, Protocol++from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError+from core.model_manager import ModelManager+from graphon.model_runtime.entities.model_entities import ModelType+from services.knowledge.segments.application import (+ SegmentDatasetRecord,+ SegmentModelProviderError,+)+++class RedisSegmentClient(Protocol):+ def get(self, name: str | bytes) -> Any: ...++ def setnx(self, name: str | bytes, value: Any) -> Any: ...+ def setex(self, name: str, time: int, value: Any) -> Any: ...+ def lock(self, name: str, timeout: int) -> Any: ...+++class ModelManagerSegmentGuard:+ def check(self, dataset: SegmentDatasetRecord) -> None:+ try:+ ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(+ tenant_id=dataset.workspace_id,+ provider=dataset.embedding_model_provider or "",+ model_type=ModelType.TEXT_EMBEDDING,+ model=dataset.embedding_model or "",+ )+ except LLMBadRequestError as error:+ raise SegmentModelProviderError(kind="bad_request", description=str(error)) from error+ except ProviderTokenNotInitError as error:+ raise SegmentModelProviderError(kind="token", description=error.description) from error+++class RedisSegmentIndexingState:+ def __init__(self, redis: RedisSegmentClient) -> None:+ self._redis = redis++ def lock(self, name: str, *, timeout: int):+ return self._redis.lock(name, timeout=timeout)++ def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool:+ suffix = "delete_indexing" if deleting else "indexing"+ return self._redis.get(f"segment_{segment_id}_{suffix}") is not None++ def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None:+ suffix = "delete_indexing" if deleting else "indexing"+ self._redis.setex(f"segment_{segment_id}_{suffix}", 600, 1)++ def is_document_indexing(self, document_id: str) -> bool:+ return self._redis.get(f"document_{document_id}_indexing") is not None++ def set_batch_waiting(self, job_id: str) -> None:+ self._redis.setnx(f"segment_batch_import_{job_id}", "waiting")++ def get_batch_status(self, job_id: str) -> str | None:+ value = self._redis.get(f"segment_batch_import_{job_id}")+ if value is None:+ return None+ return value.decode() if isinstance(value, bytes) else value+++class CelerySegmentBatchImportDispatcher:+ def __init__(self, *, delay: Callable[..., object]) -> None:+ self._delay = delay++ def dispatch(+ self,+ *,+ job_id: str,+ upload_file_id: str,+ dataset_id: str,+ document_id: str,+ workspace_id: str,+ actor_id: str,+ ) -> None:+ self._delay(job_id, upload_file_id, dataset_id, document_id, workspace_id, actor_id)diff --git a/api/services/knowledge/segments/application.py b/api/services/knowledge/segments/application.pynew file mode 100644index 00000000000..3600468c16c--- /dev/null+++ b/api/services/knowledge/segments/application.py@@ -0,0 +1,958 @@+"""Application service for dataset segment and child-chunk HTTP use cases."""++from __future__ import annotations++import logging+from collections import Counter+from collections.abc import Callable, Mapping, Sequence+from contextlib import AbstractContextManager+from dataclasses import dataclass, replace+from typing import Literal, Protocol+from uuid import uuid4++from libs.datetime_utils import naive_utc_now+from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccess, DatasetAccessDeniedError, DatasetAccessSnapshot+from services.knowledge.entities.segments import (+ ChildChunkRecord,+ ChildChunkUpdateArgs,+ SegmentRecord,+ SegmentUpdateArgs,+)+from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef++_HIGH_QUALITY = "high_quality"+logger = logging.getLogger(__name__)+++@dataclass(frozen=True, slots=True)+class SegmentDatasetRecord:+ id: str+ workspace_id: str+ indexing_technique: str+ embedding_model_provider: str | None+ embedding_model: str | None+++@dataclass(frozen=True, slots=True)+class SegmentDocumentRecord:+ id: str+ dataset_id: str+ workspace_id: str+ doc_form: str++ @property+ def ref(self) -> DocumentRef:+ return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)+++@dataclass(frozen=True, slots=True)+class SegmentScope:+ dataset: SegmentDatasetRecord | None+ document: SegmentDocumentRecord | None+ access_snapshot: DatasetAccessSnapshot | None = None+++@dataclass(frozen=True, slots=True)+class SegmentListFilter:+ page: int = 1+ limit: int = 20+ statuses: tuple[str, ...] = ()+ hit_count_gte: int | None = None+ enabled: str = "all"+ keyword: str | None = None+++@dataclass(frozen=True, slots=True)+class ChildChunkListFilter:+ page: int = 1+ limit: int = 20+ keyword: str | None = None+++@dataclass(frozen=True, slots=True)+class SegmentPage:+ items: tuple[SegmentRecord, ...]+ total: int+ total_pages: int+ page: int+ limit: int+++@dataclass(frozen=True, slots=True)+class SegmentDetail:+ data: SegmentRecord+ doc_form: str+++@dataclass(frozen=True, slots=True)+class ChildChunkPage:+ items: tuple[ChildChunkRecord, ...]+ total: int+ total_pages: int+ page: int+ limit: int+++@dataclass(frozen=True, slots=True)+class SegmentBatchImport:+ job_id: str+ job_status: str+++class SegmentScopeReader(Protocol):+ def get_segment_scope(+ self,+ *,+ dataset_ref: DatasetRef,+ document_id: str,+ actor_id: str | None,+ ) -> SegmentScope: ...+++@dataclass(frozen=True, slots=True)+class ChildChunkState:+ data: ChildChunkRecord+ index_node_id: str | None+ index_node_hash: str | None+ created_by: str+ updated_by: str | None = None+++@dataclass(frozen=True, slots=True)+class SegmentIndexTarget:+ id: str+ index_node_id: str | None+ enabled: bool+++@dataclass(frozen=True, slots=True)+class SegmentUpdateState:+ """Persisted fields needed to decide a segment update, without presentation data."""++ content: str+ enabled: bool+ keywords: tuple[str, ...] | None+++class SegmentStore(Protocol):+ """Owner-scoped persistence; each operation owns a short session."""++ def list_segments(self, document_ref: DocumentRef, query: SegmentListFilter) -> SegmentPage: ...+ def get_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> tuple[SegmentIndexTarget, ...]: ...+ def get_segment(self, segment_ref: SegmentRef) -> SegmentDetail | None: ...+ def get_segment_update_state(self, segment_ref: SegmentRef) -> SegmentUpdateState | None: ...+ def save_segment(self, segment_ref: SegmentRef, values: Mapping[str, object], *, create: bool = False) -> None: ...+ def delete_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> None: ...+ def list_child_chunks(self, segment_ref: SegmentRef, query: ChildChunkListFilter) -> ChildChunkPage | None: ...+ def get_children_for_segments(+ self, document_ref: DocumentRef, segment_ids: Sequence[str]+ ) -> tuple[ChildChunkState, ...]: ...+ def set_segments(+ self, document_ref: DocumentRef, segment_ids: Sequence[str], values: Mapping[str, object]+ ) -> None: ...+ def get_children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...] | None: ...+ def save_children(+ self,+ segment_ref: SegmentRef,+ *,+ added: Sequence[ChildChunkState] = (),+ updated: Sequence[ChildChunkState] = (),+ deleted: Sequence[ChildChunkState] = (),+ ) -> None: ...+++class SegmentIndex(Protocol):+ def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int: ...+ def create_many(+ self,+ document_ref: DocumentRef,+ *,+ segment_ids: Sequence[str],+ keywords_list: Sequence[Sequence[str] | None] | None,+ ) -> None: ...+ def create(+ self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]+ ) -> None: ...+ def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None: ...+ def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None: ...+ def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None: ...+ def update_children(+ self,+ segment_ref: SegmentRef,+ *,+ added: Sequence[ChildChunkState] = (),+ updated: Sequence[ChildChunkState] = (),+ deleted: Sequence[ChildChunkState] = (),+ ) -> None: ...+ def delete(+ self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]+ ) -> None: ...+ def change_status(+ self, document_ref: DocumentRef, segment_ids: Sequence[str], action: Literal["enable", "disable"]+ ) -> None: ...+++class SegmentLimits(Protocol):+ SINGLE_CHUNK_ATTACHMENT_LIMIT: int+++class SegmentModelGuard(Protocol):+ def check(self, dataset: SegmentDatasetRecord) -> None: ...+++class SegmentUploadCatalog(Protocol):+ def get_file_name(self, *, workspace_id: str, upload_file_id: str) -> str | None: ...+++class SegmentIndexingState(Protocol):+ def lock(self, name: str, *, timeout: int) -> AbstractContextManager[object]: ...+ def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool: ...+ def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None: ...++ def is_document_indexing(self, document_id: str) -> bool: ...++ def set_batch_waiting(self, job_id: str) -> None: ...++ def get_batch_status(self, job_id: str) -> str | None: ...+++class SegmentBatchImportDispatcher(Protocol):+ def dispatch(+ self,+ *,+ job_id: str,+ upload_file_id: str,+ dataset_id: str,+ document_id: str,+ workspace_id: str,+ actor_id: str,+ ) -> None: ...+++class SegmentApplicationError(Exception):+ """Base class for framework-neutral segment use-case errors."""+++class SegmentDatasetNotFoundError(SegmentApplicationError):+ pass+++class SegmentDocumentNotFoundError(SegmentApplicationError):+ pass+++class SegmentNotFoundError(SegmentApplicationError):+ pass+++class ChildChunkNotFoundError(SegmentApplicationError):+ pass+++class SegmentUploadFileNotFoundError(SegmentApplicationError):+ pass+++class SegmentPermissionDeniedError(SegmentApplicationError):+ pass+++class SegmentInvalidFileTypeError(SegmentApplicationError):+ pass+++class SegmentDocumentIndexingError(SegmentApplicationError):+ pass+++class SegmentStatusUpdateError(SegmentApplicationError):+ pass+++class SegmentBatchImportDispatchError(SegmentApplicationError):+ pass+++class SegmentBatchImportNotFoundError(SegmentApplicationError):+ pass+++class SegmentModelProviderError(Exception):+ """Adapter error raised when the configured embedding model cannot be loaded."""++ def __init__(self, *, kind: Literal["bad_request", "token"], description: str) -> None:+ super().__init__(description)+ self.kind = kind+ self.description = description+++class SegmentDatasetModelUnavailableError(SegmentApplicationError):+ pass+++class SegmentEmbeddingModelUnavailableError(SegmentApplicationError):+ pass+++class ChildChunkIndexingApplicationError(SegmentApplicationError):+ pass+++class ChildChunkDeleteIndexApplicationError(SegmentApplicationError):+ pass+++class DatasetSegmentApplicationService:+ def __init__(+ self,+ *,+ dataset_access: DatasetAccess,+ scopes: SegmentScopeReader,+ store: SegmentStore,+ index: SegmentIndex,+ limits: SegmentLimits,+ text_hash: Callable[[str], str],+ uploads: SegmentUploadCatalog,+ model_guard: SegmentModelGuard,+ indexing_state: SegmentIndexingState,+ batch_dispatcher: SegmentBatchImportDispatcher,+ job_id_factory: Callable[[], str],+ ) -> None:+ self._dataset_access = dataset_access+ self._scopes = scopes+ self._store = store+ self._index = index+ self._limits = limits+ self._uploads = uploads+ self._model_guard = model_guard+ self._indexing_state = indexing_state+ self._batch_dispatcher = batch_dispatcher+ self._job_id_factory = job_id_factory+ self.mutations = SegmentMutationService(+ store=store, index=index, text_hash=text_hash, indexing_state=indexing_state+ )++ def list_segments(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ query: SegmentListFilter,+ ) -> SegmentPage:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ return self._store.list_segments(scope.document.ref, query)++ def delete_segments(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_ids: Sequence[str],+ ) -> None:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ segments = self._store.get_segments(scope.document.ref, segment_ids)+ children = self._store.get_children_for_segments(scope.document.ref, [segment.id for segment in segments])+ if segments:+ self._index.delete(scope.document.ref, segments, children)+ self._store.delete_segments(scope.document.ref, [segment.id for segment in segments])++ def change_segment_status(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_ids: Sequence[str],+ action: Literal["enable", "disable"],+ ) -> None:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ self._check_embedding_model(scope.dataset)+ if self._indexing_state.is_document_indexing(scope.document.id):+ raise SegmentDocumentIndexingError("Document is being indexed, please try again later")+ try:+ segments = self._store.get_segments(scope.document.ref, segment_ids)+ ids = []+ for segment in segments:+ if segment.enabled == (action == "enable") or self._indexing_state.is_segment_indexing(segment.id):+ continue+ ids.append(segment.id)+ if ids:+ self._store.set_segments(+ scope.document.ref,+ ids,+ {+ "enabled": action == "enable",+ "disabled_at": naive_utc_now() if action == "disable" else None,+ "disabled_by": context.account_id if action == "disable" else None,+ },+ )+ self._index.change_status(scope.document.ref, ids, action)+ except Exception as error:+ raise SegmentStatusUpdateError(str(error)) from error++ def create_segment(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ values: Mapping[str, object],+ ) -> SegmentDetail:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_embedding_model(scope.dataset)+ args = self._validate_values(values, scope.document.doc_form)+ segment_ref = self.mutations.create_segment(scope, actor_id=context.account_id, args=args)+ return self._detail(segment_ref)++ def update_segment(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ values: Mapping[str, object],+ ) -> SegmentDetail:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ self._check_embedding_model(scope.dataset)+ args = self._validate_values(values, scope.document.doc_form)+ self.mutations.update_segment(scope, actor_id=context.account_id, segment_id=segment_id, args=args)+ return self._detail(scope.document.ref.segment(segment_id))++ def _detail(self, segment_ref: SegmentRef) -> SegmentDetail:+ result = self._store.get_segment(segment_ref)+ if result is None:+ raise SegmentNotFoundError("Segment not found")+ return result++ def delete_segment(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ ) -> None:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ return self.mutations.delete_segment(scope.document.ref.segment(segment_id))++ def start_batch_import(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ upload_file_id: str,+ ) -> SegmentBatchImport:+ scope = self._require_scope(+ context,+ dataset_id=dataset_id,+ document_id=document_id,+ enforce_permission=False,+ )+ filename = self._uploads.get_file_name(+ workspace_id=scope.dataset.workspace_id,+ upload_file_id=upload_file_id,+ )+ if filename is None:+ raise SegmentUploadFileNotFoundError("UploadFile not found")+ if not filename.lower().endswith(".csv"):+ raise SegmentInvalidFileTypeError("Invalid file type. Only CSV files are allowed")++ job_id = self._job_id_factory()+ try:+ self._indexing_state.set_batch_waiting(job_id)+ self._batch_dispatcher.dispatch(+ job_id=job_id,+ upload_file_id=upload_file_id,+ dataset_id=scope.dataset.id,+ document_id=scope.document.id,+ workspace_id=scope.dataset.workspace_id,+ actor_id=context.account_id,+ )+ except Exception as error:+ raise SegmentBatchImportDispatchError(str(error)) from error+ return SegmentBatchImport(job_id=job_id, job_status="waiting")++ def get_batch_import_status(self, job_id: str) -> SegmentBatchImport:+ status = self._indexing_state.get_batch_status(job_id)+ if status is None:+ raise SegmentBatchImportNotFoundError("The job does not exist.")+ return SegmentBatchImport(job_id=job_id, job_status=status)++ def create_child_chunk(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ content: str,+ ) -> ChildChunkRecord:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_embedding_model(scope.dataset)+ return self.mutations.create_child_chunk(+ scope.document.ref.segment(segment_id), content=content, actor_id=context.account_id+ )++ def list_child_chunks(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ query: ChildChunkListFilter,+ ) -> ChildChunkPage:+ scope = self._require_scope_without_actor(+ workspace_id=workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ )+ self._check_dataset_model(scope.dataset)+ result = self._store.list_child_chunks(scope.document.ref.segment(segment_id), query)+ if result is None:+ raise SegmentNotFoundError("Segment not found")+ return result++ def update_child_chunks(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ chunks: Sequence[ChildChunkUpdateArgs],+ ) -> tuple[ChildChunkRecord, ...]:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ return self.mutations.update_child_chunks(+ scope.document.ref.segment(segment_id), chunks=chunks, actor_id=context.account_id+ )++ def delete_child_chunk(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ child_chunk_id: str,+ ) -> None:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ return self.mutations.delete_child_chunk(scope.document.ref.segment(segment_id), child_chunk_id=child_chunk_id)++ def update_child_chunk(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ child_chunk_id: str,+ content: str,+ ) -> ChildChunkRecord:+ scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)+ self._check_dataset_model(scope.dataset)+ return self.mutations.update_child_chunk(+ scope.document.ref.segment(segment_id),+ child_chunk_id=child_chunk_id,+ content=content,+ actor_id=context.account_id,+ )++ def _validate_values(self, values: Mapping[str, object], doc_form: str) -> SegmentUpdateArgs:+ return validate_segment_values(+ values, doc_form=doc_form, attachment_limit=self._limits.SINGLE_CHUNK_ATTACHMENT_LIMIT+ )++ def _require_scope(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ enforce_permission: bool = True,+ ) -> SegmentMutationScope:+ scope = self._scopes.get_segment_scope(+ dataset_ref=DatasetRef(context.active_workspace_id, dataset_id),+ document_id=document_id,+ actor_id=context.account_id if enforce_permission else None,+ )+ dataset = scope.dataset+ if dataset is None:+ raise SegmentDatasetNotFoundError("Dataset not found")+ if enforce_permission:+ if scope.access_snapshot is None:+ raise SegmentPermissionDeniedError("Dataset access state is missing")+ try:+ self._dataset_access.check_access(context, scope.access_snapshot)+ except DatasetAccessDeniedError as error:+ raise SegmentPermissionDeniedError(str(error)) from error+ document = scope.document+ if document is None:+ raise SegmentDocumentNotFoundError("Document not found")+ return SegmentMutationScope(dataset=dataset, document=document)++ def _require_scope_without_actor(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ ) -> SegmentMutationScope:+ scope = self._scopes.get_segment_scope(+ dataset_ref=DatasetRef(workspace_id, dataset_id),+ document_id=document_id,+ actor_id=None,+ )+ if scope.dataset is None:+ raise SegmentDatasetNotFoundError("Dataset not found")+ if scope.document is None:+ raise SegmentDocumentNotFoundError("Document not found")+ return SegmentMutationScope(dataset=scope.dataset, document=scope.document)++ def _check_dataset_model(self, dataset: SegmentDatasetRecord) -> None:+ if dataset.indexing_technique != _HIGH_QUALITY:+ return+ try:+ self._model_guard.check(dataset)+ except SegmentModelProviderError as error:+ if error.kind == "token":+ message = f"The dataset is unavailable, due to: {error.description}"+ else:+ message = (+ "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."+ )+ raise SegmentDatasetModelUnavailableError(message) from error++ def _check_embedding_model(self, dataset: SegmentDatasetRecord) -> None:+ if dataset.indexing_technique != _HIGH_QUALITY:+ return+ try:+ self._model_guard.check(dataset)+ except SegmentModelProviderError as error:+ if error.kind == "bad_request":+ message = (+ "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."+ )+ else:+ message = error.description+ raise SegmentEmbeddingModelUnavailableError(message) from error+++class SegmentMutationService:+ """Shared writes for Console and Service API after their own admission checks.++ Callers supply an owner chain reconstructed from authorized database rows.+ Persistence ports recheck that chain and own their transactions. Partial+ update arguments are accepted here; transport validation stays at admission.+ """++ def __init__(+ self,+ *,+ store: SegmentStore,+ index: SegmentIndex,+ text_hash: Callable[[str], str],+ indexing_state: SegmentIndexingState,+ ) -> None:+ self._store = store+ self._index = index+ self._text_hash = text_hash+ self._indexing_state = indexing_state++ def index_segments(+ self,+ document_ref: DocumentRef,+ *,+ segment_ids: Sequence[str],+ keywords_list: Sequence[Sequence[str] | None] | None = None,+ ) -> None:+ """Index a committed batch and persist failures through the scoped store.++ Callers must finish their write transaction before invoking this method.+ The index adapter reloads the complete owner chain before external I/O.+ """+ if keywords_list is not None and len(keywords_list) != len(segment_ids):+ raise ValueError("Keywords must correspond to the segment batch")+ if not segment_ids:+ return+ try:+ self._index.create_many(document_ref, segment_ids=segment_ids, keywords_list=keywords_list)+ except Exception as error:+ logger.exception("Batch indexing failed for document %s", document_ref.document_id)+ self._store.set_segments(+ document_ref,+ segment_ids,+ {"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)},+ )+ raise++ def create_segment(+ self,+ scope: SegmentMutationScope,+ *,+ actor_id: str,+ args: SegmentUpdateArgs,+ ) -> SegmentRef:+ now = naive_utc_now()+ segment_ref = scope.document.ref.segment(str(uuid4()))+ saved = False+ try:+ with self._indexing_state.lock(f"add_segment_lock_document_id_{scope.document.id}", timeout=600):+ self._store.save_segment(+ segment_ref,+ {+ "content": args.content,+ "answer": args.answer if scope.document.doc_form == "qa_model" else None,+ "word_count": len(args.content or "")+ + (len(args.answer or "") if scope.document.doc_form == "qa_model" else 0),+ "tokens": self._index.count_tokens(scope.dataset, args.content or ""),+ "index_node_id": str(uuid4()),+ "index_node_hash": self._text_hash(args.content or ""),+ "created_by": actor_id,+ "status": "completed",+ "indexing_at": now,+ "completed_at": now,+ },+ create=True,+ )+ saved = True+ except Exception as error:+ if saved:+ self._mark_error(segment_ref, error)+ raise+ try:+ self._index.create(segment_ref, keywords=args.keywords, attachment_ids=args.attachment_ids or ())+ except Exception as error:+ self._mark_error(segment_ref, error)+ return segment_ref++ def update_segment(+ self,+ scope: SegmentMutationScope,+ *,+ actor_id: str,+ segment_id: str,+ args: SegmentUpdateArgs,+ ) -> None:+ segment_ref = scope.document.ref.segment(segment_id)+ previous = self._store.get_segment_update_state(segment_ref)+ if previous is None:+ raise SegmentNotFoundError("Segment not found")+ if self._indexing_state.is_segment_indexing(segment_id):+ raise ValueError("Segment is indexing, please try again later")+ if args.enabled is False and previous.enabled:+ self._store.save_segment(+ segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "disabled_by": actor_id}+ )+ self._indexing_state.mark_segment_indexing(segment_id)+ self._index.change_status(scope.document.ref, [segment_id], "disable")+ return+ if not previous.enabled and args.enabled is not True:+ raise ValueError("Can't update disabled segment")+ content = args.content or previous.content+ content_changed = content != previous.content+ keyword_changed = bool(args.keywords) and Counter(previous.keywords or ()) != Counter(args.keywords or ())+ changes: dict[str, object] = {+ "content": content,+ "word_count": len(content) + (len(args.answer or "") if scope.document.doc_form == "qa_model" else 0),+ "enabled": True,+ "disabled_at": None,+ "disabled_by": None,+ }+ if scope.document.doc_form == "qa_model":+ changes["answer"] = args.answer+ if keyword_changed and not content_changed:+ changes["keywords"] = args.keywords+ try:+ if content_changed:+ now = naive_utc_now()+ changes.update(+ index_node_hash=self._text_hash(content),+ tokens=self._index.count_tokens(+ scope.dataset,+ content + (args.answer or "") if scope.document.doc_form == "qa_model" else content,+ ),+ status="completed",+ indexing_at=now,+ completed_at=now,+ updated_at=now,+ updated_by=actor_id,+ )+ self._store.save_segment(segment_ref, changes)+ if scope.document.doc_form == "hierarchical_model":+ if args.regenerate_child_chunks:+ self._index.update(segment_ref, keywords=args.keywords, regenerate_children=True)+ elif content_changed or args.enabled or keyword_changed:+ self._index.update(segment_ref, keywords=args.keywords, regenerate_children=False)+ if scope.dataset.indexing_technique == _HIGH_QUALITY:+ try:+ self._index.update_summary(segment_ref, summary=args.summary, content_changed=content_changed)+ except Exception:+ logger.exception("Failed to update summary for segment %s", segment_id)+ if args.attachment_ids is not None:+ self._index.update_attachments(segment_ref, args.attachment_ids)+ except Exception as error:+ self._mark_error(segment_ref, error)++ def delete_segment(self, segment_ref: SegmentRef) -> None:+ segments = self._store.get_segments(segment_ref.document, (segment_ref.segment_id,))+ if not segments:+ raise SegmentNotFoundError("Segment not found")+ segment = segments[0]+ if self._indexing_state.is_segment_indexing(segment_ref.segment_id, deleting=True):+ raise ValueError("Segment is deleting.")+ if segment.enabled:+ self._indexing_state.mark_segment_indexing(segment_ref.segment_id, deleting=True)+ self._index.delete(+ segment_ref.document,+ (segment,),+ self._children(segment_ref),+ )+ self._store.delete_segments(segment_ref.document, (segment_ref.segment_id,))++ def create_child_chunk(self, segment_ref: SegmentRef, *, content: str, actor_id: str) -> ChildChunkRecord:+ with self._indexing_state.lock(f"add_child_lock_{segment_ref.segment_id}", timeout=20):+ children = self._children(segment_ref)+ child = self._new_child(+ segment_ref, content, max((c.data.position for c in children), default=0) + 1, actor_id+ )+ self._write_children(segment_ref, added=(child,))+ return child.data++ def update_child_chunks(+ self, segment_ref: SegmentRef, *, chunks: Sequence[ChildChunkUpdateArgs], actor_id: str+ ) -> tuple[ChildChunkRecord, ...]:+ existing = self._children(segment_ref)+ remaining = {child.data.id: child for child in existing}+ added: list[ChildChunkState] = []+ updated: list[ChildChunkState] = []+ for chunk in chunks:+ if chunk.id:+ previous = remaining.pop(chunk.id, None)+ if previous is not None and previous.data.content != chunk.content:+ updated.append(self._updated_child(previous, chunk.content, actor_id))+ else:+ added.append(self._new_child(segment_ref, chunk.content, len(existing) + len(added) + 1, actor_id))+ self._write_children(segment_ref, added=added, updated=updated, deleted=tuple(remaining.values()))+ return tuple(child.data for child in sorted(added + updated, key=lambda child: child.data.position))++ def delete_child_chunk(self, segment_ref: SegmentRef, *, child_chunk_id: str) -> None:+ child = self._child(segment_ref, child_chunk_id)+ try:+ self._index.update_children(segment_ref, deleted=(child,))+ except Exception as error:+ raise ChildChunkDeleteIndexApplicationError(str(error)) from error+ self._store.save_children(segment_ref, deleted=(child,))++ def update_child_chunk(+ self, segment_ref: SegmentRef, *, child_chunk_id: str, content: str, actor_id: str+ ) -> ChildChunkRecord:+ child = self._updated_child(self._child(segment_ref, child_chunk_id), content, actor_id)+ self._write_children(segment_ref, updated=(child,))+ return child.data++ def _mark_error(self, segment_ref: SegmentRef, error: Exception) -> None:+ logger.exception("Segment indexing failed: %s", segment_ref.segment_id)+ self._store.save_segment(+ segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)}+ )++ def _children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...]:+ children = self._store.get_children(segment_ref)+ if children is None:+ raise SegmentNotFoundError("Segment not found")+ return children++ def _child(self, segment_ref: SegmentRef, child_id: str) -> ChildChunkState:+ for child in self._children(segment_ref):+ if child.data.id == child_id:+ return child+ raise ChildChunkNotFoundError("Child chunk not found")++ def _new_child(self, segment_ref: SegmentRef, content: str, position: int, actor_id: str) -> ChildChunkState:+ now = naive_utc_now()+ return ChildChunkState(+ data=ChildChunkRecord(+ id=str(uuid4()),+ segment_id=segment_ref.segment_id,+ content=content,+ position=position,+ word_count=len(content),+ type="customized",+ created_at=now,+ updated_at=now,+ ),+ index_node_id=str(uuid4()),+ index_node_hash=self._text_hash(content),+ created_by=actor_id,+ )++ def _updated_child(self, child: ChildChunkState, content: str, actor_id: str) -> ChildChunkState:+ return replace(+ child,+ data=child.data.model_copy(+ update={+ "content": content,+ "word_count": len(content),+ "type": "customized",+ "updated_at": naive_utc_now(),+ }+ ),+ updated_by=actor_id,+ )++ def _write_children(+ self,+ segment_ref: SegmentRef,+ *,+ added: Sequence[ChildChunkState] = (),+ updated: Sequence[ChildChunkState] = (),+ deleted: Sequence[ChildChunkState] = (),+ ) -> None:+ try:+ self._index.update_children(segment_ref, added=added, updated=updated, deleted=deleted)+ self._store.save_children(segment_ref, added=added, updated=updated, deleted=deleted)+ except Exception as error:+ raise ChildChunkIndexingApplicationError(str(error)) from error+++@dataclass(frozen=True, slots=True)+class SegmentMutationScope:+ """Authorized dataset and document values for shared mutation policy."""++ dataset: SegmentDatasetRecord+ document: SegmentDocumentRecord++ def __post_init__(self) -> None:+ if self.document.dataset_id != self.dataset.id or self.document.workspace_id != self.dataset.workspace_id:+ raise ValueError("Document does not belong to the dataset")+++def validate_segment_values(values: Mapping[str, object], *, doc_form: str, attachment_limit: int) -> SegmentUpdateArgs:+ answer = values.get("answer")+ if doc_form == "qa_model":+ if not answer:+ raise ValueError("Answer is required")+ if not isinstance(answer, str) or not answer.strip():+ raise ValueError("Answer is empty")+ content = values.get("content")+ if not isinstance(content, str) or not content.strip():+ raise ValueError("Content is empty")+ attachments = values.get("attachment_ids")+ if attachments:+ if not isinstance(attachments, list):+ raise ValueError("Attachment IDs is invalid")+ if len(attachments) > attachment_limit:+ raise ValueError(f"Exceeded maximum attachment limit of {attachment_limit}")+ return SegmentUpdateArgs.model_validate(values)diff --git a/api/services/knowledge/segments/indexing.py b/api/services/knowledge/segments/indexing.pynew file mode 100644index 00000000000..462e19f5ece--- /dev/null+++ b/api/services/knowledge/segments/indexing.py@@ -0,0 +1,356 @@+"""Indexing adapters that materialize database inputs before external I/O."""++import logging+from collections.abc import Callable, Mapping, Sequence+from uuid import uuid4++from sqlalchemy.orm import Session++from core.credit_usage import CreditUsageCreatedBy+from core.model_context import with_credit_usage_created_by+from core.model_manager import ModelManager+from core.rag.datasource.keyword.jieba.jieba import Jieba+from core.rag.datasource.vdb.vector_factory import Vector+from core.rag.entities import Rule+from core.rag.index_processor.constant.doc_type import DocType+from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor+from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor+from core.rag.models.document import Document+from graphon.model_runtime.entities.model_entities import ModelType+from libs.datetime_utils import naive_utc_now+from models.enums import SegmentType+from repositories.knowledge.segment_repository import SegmentIndexingSnapshot, SQLAlchemySegmentRepository+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.entities.segments import ChildChunkRecord+from services.knowledge.resource_scope import DocumentRef, SegmentRef+from services.knowledge.segments.adapters import RedisSegmentClient+from services.knowledge.segments.application import ChildChunkState, SegmentDatasetRecord, SegmentIndexTarget+from services.knowledge.summaries.adapters import SummaryIndexAdapter++logger = logging.getLogger(__name__)+++class SegmentIndexingGateway:+ def __init__(+ self,+ *,+ segments: SQLAlchemySegmentRepository,+ uploads: SQLAlchemyKnowledgeUploadRepository,+ redis: RedisSegmentClient,+ new_session: Callable[[], Session],+ delete_task: Callable[..., object],+ enable_task: Callable[..., object],+ disable_task: Callable[..., object],+ ) -> None:+ self._segments = segments+ self._uploads = uploads+ self._redis = redis+ self._new_session = new_session+ self._delete_task = delete_task+ self._enable_task = enable_task+ self._disable_task = disable_task++ def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int:+ if dataset.indexing_technique != "high_quality":+ return 0+ model = ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(+ tenant_id=dataset.workspace_id,+ provider=dataset.embedding_model_provider or "",+ model_type=ModelType.TEXT_EMBEDDING,+ model=dataset.embedding_model or "",+ )+ return model.get_text_embedding_num_tokens(texts=[text])[0]++ @staticmethod+ def _vector(snapshot: SegmentIndexingSnapshot) -> Vector:+ return Vector(snapshot.dataset, session=None, vector_type=snapshot.vector_type)++ @staticmethod+ def _document(snapshot: SegmentIndexingSnapshot) -> Document:+ segment = snapshot.segment+ return Document(+ page_content=segment.content,+ metadata={+ "doc_id": segment.index_node_id,+ "doc_hash": segment.index_node_hash,+ "document_id": segment.document_id,+ "dataset_id": segment.dataset_id,+ "doc_type": DocType.TEXT,+ },+ )++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def create_many(+ self,+ document_ref: DocumentRef,+ *,+ segment_ids: Sequence[str],+ keywords_list: Sequence[Sequence[str] | None] | None,+ ) -> None:+ """Materialize the whole scoped batch before indexing any of its records."""+ snapshots = self._segments.get_indexing_snapshots(document_ref, segment_ids)+ if not snapshots:+ return+ first = snapshots[0]+ documents = [self._document(snapshot) for snapshot in snapshots]+ if first.document.doc_form == "hierarchical_model":+ for snapshot in snapshots:+ self._regenerate_children(document_ref.segment(snapshot.segment.id), snapshot, replace_existing=False)+ elif first.dataset.indexing_technique == "high_quality":+ self._vector(first).create(documents)+ elif first.document.doc_form != "qa_model":+ self._write_keywords(document_ref, first, documents, keywords_list, replace_existing=False)+ if first.dataset.is_multimodal:+ files = self._uploads.get_files(+ workspace_id=document_ref.dataset.tenant_id,+ file_ids=tuple(attachment_id for snapshot in snapshots for attachment_id in snapshot.attachment_ids),+ )+ attachments = [+ document+ for snapshot in snapshots+ for document in self._attachment_documents(+ snapshot, [files[file_id] for file_id in snapshot.attachment_ids if file_id in files]+ )+ ]+ if attachments and first.dataset.indexing_technique == "high_quality":+ self._vector(first).create_multimodal(attachments, upload_files=files)++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def create(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]) -> None:+ snapshot = self._segments.get_indexing_snapshot(segment_ref)+ files = self._uploads.get_files(workspace_id=segment_ref.document.dataset.tenant_id, file_ids=attachment_ids)+ self._segments.replace_attachments(segment_ref, tuple(files))+ if snapshot.document.doc_form == "hierarchical_model":+ self._regenerate_children(segment_ref, snapshot, replace_existing=False)+ elif snapshot.dataset.indexing_technique == "high_quality":+ self._vector(snapshot).create([self._document(snapshot)])+ else:+ self._update_keywords(segment_ref, snapshot, keywords, replace_existing=False)+ if files and snapshot.dataset.is_multimodal:+ self._vector(snapshot).create_multimodal(+ self._attachment_documents(snapshot, files.values()), upload_files=files+ )++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None:+ snapshot = self._segments.get_indexing_snapshot(segment_ref)+ if regenerate_children:+ self._regenerate_children(segment_ref, snapshot, replace_existing=True)+ elif snapshot.dataset.indexing_technique == "high_quality":+ vector = self._vector(snapshot)+ assert snapshot.segment.index_node_id is not None+ vector.delete_by_ids([snapshot.segment.index_node_id])+ vector.add_texts([self._document(snapshot)], duplicate_check=True)+ else:+ self._update_keywords(segment_ref, snapshot, keywords, replace_existing=True)++ def _regenerate_children(+ self, segment_ref: SegmentRef, snapshot: SegmentIndexingSnapshot, *, replace_existing: bool+ ) -> None:+ if snapshot.dataset.indexing_technique != "high_quality":+ raise ValueError("The knowledge base index technique is not high quality!")+ rule = snapshot.process_rule+ if rule is None:+ if replace_existing:+ return+ raise ValueError("No processing rule found.")+ manager = ModelManager.for_tenant(tenant_id=snapshot.dataset.tenant_id)+ model = (+ manager.get_model_instance(+ tenant_id=snapshot.dataset.tenant_id,+ provider=snapshot.dataset.embedding_model_provider,+ model_type=ModelType.TEXT_EMBEDDING,+ model=snapshot.dataset.embedding_model,+ )+ if snapshot.dataset.embedding_model_provider+ else manager.get_default_model_instance(+ tenant_id=snapshot.dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING+ )+ )+ children = ParentChildIndexProcessor().split_child_nodes(+ self._document(snapshot), Rule.model_validate(rule.to_dict()["rules"]), rule.mode, model+ )+ previous = self._segments.get_children(segment_ref) or ()+ vector = self._vector(snapshot)+ if replace_existing and previous:+ vector.delete_by_ids([child.index_node_id for child in previous if child.index_node_id])+ if children:+ vector.create([Document.model_validate(child.model_dump()) for child in children])+ now = naive_utc_now()+ added = tuple(+ ChildChunkState(+ data=ChildChunkRecord(+ id=str(uuid4()),+ segment_id=segment_ref.segment_id,+ content=child.page_content,+ position=position,+ word_count=len(child.page_content),+ type=SegmentType.AUTOMATIC,+ created_at=now,+ updated_at=now,+ ),+ index_node_id=child.metadata["doc_id"],+ index_node_hash=child.metadata["doc_hash"],+ created_by=snapshot.document.created_by,+ )+ for position, child in enumerate(children, start=1)+ )+ self._segments.save_children(segment_ref, added=added, deleted=previous if replace_existing else ())++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def update_children(+ self,+ segment_ref: SegmentRef,+ *,+ added: Sequence[ChildChunkState] = (),+ updated: Sequence[ChildChunkState] = (),+ deleted: Sequence[ChildChunkState] = (),+ ) -> None:+ snapshot = self._segments.get_indexing_snapshot(segment_ref)+ if snapshot.dataset.indexing_technique != "high_quality":+ return+ documents = [+ Document(+ page_content=child.data.content,+ metadata={+ "doc_id": child.index_node_id,+ "doc_hash": child.index_node_hash,+ "document_id": segment_ref.document.document_id,+ "dataset_id": segment_ref.document.dataset.dataset_id,+ },+ )+ for child in (*added, *updated)+ ]+ vector = self._vector(snapshot)+ deleted_ids = []+ for child in (*updated, *deleted):+ assert child.index_node_id is not None+ deleted_ids.append(child.index_node_id)+ if deleted_ids:+ vector.delete_by_ids(deleted_ids)+ if documents:+ vector.add_texts(documents, duplicate_check=True)++ @staticmethod+ def _attachment_documents(snapshot: SegmentIndexingSnapshot, files) -> list[Document]:+ return [+ Document(+ page_content=file.name,+ metadata={+ "doc_id": file.id,+ "doc_hash": "",+ "dataset_id": snapshot.dataset.id,+ "document_id": snapshot.document.id,+ "doc_type": DocType.IMAGE,+ },+ )+ for file in files+ ]++ @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)+ def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None:+ snapshot = self._segments.get_indexing_snapshot(segment_ref)+ if snapshot.dataset.indexing_technique != "high_quality" or set(snapshot.attachment_ids) == set(attachment_ids):+ return+ files = self._uploads.get_files(workspace_id=snapshot.dataset.tenant_id, file_ids=attachment_ids)+ if snapshot.dataset.is_multimodal:+ vector = self._vector(snapshot)+ if snapshot.attachment_ids:+ vector.delete_by_ids(list(snapshot.attachment_ids))+ if files:+ vector.create_multimodal(self._attachment_documents(snapshot, files.values()), upload_files=files)+ self._segments.replace_attachments(segment_ref, tuple(files))++ def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None:+ if summary is None and not content_changed:+ return+ snapshot = self._segments.get_indexing_snapshot(segment_ref)+ if snapshot.dataset.indexing_technique != "high_quality" or snapshot.document.doc_form == "qa_model":+ return+ setting = snapshot.dataset.summary_index_setting+ previous = snapshot.summary+ if summary is None or (previous is not None and summary == previous.summary_content):+ if not (content_changed and previous and setting and setting.get("enable") is True):+ return+ try:+ self._segments.save_summary(segment_ref, previous.summary_content or "")+ summary, _ = ParagraphIndexProcessor.generate_summary_from_inputs(+ snapshot.dataset.tenant_id,+ snapshot.segment.content,+ setting,+ document_language=snapshot.document.doc_language,+ image_loader=lambda: self._segments.get_summary_images(segment_ref),+ )+ if not summary.strip():+ raise ValueError("Generated summary is empty")+ except Exception as error:+ self._segments.save_summary(segment_ref, previous.summary_content or "", error=str(error))+ raise+ if not summary.strip():+ if previous and previous.summary_index_node_id:+ try:+ self._vector(snapshot).delete_by_ids([previous.summary_index_node_id])+ except Exception:+ logger.exception("Failed to delete summary vector for %s", segment_ref.segment_id)+ self._segments.save_summary(segment_ref, None)+ return+ record = self._segments.save_summary(segment_ref, summary)+ assert record is not None+ SummaryIndexAdapter.vectorize_summary(+ record,+ snapshot.segment,+ snapshot.dataset,+ new_session=self._new_session,+ vector_factory=lambda: self._vector(snapshot),+ )++ def _update_keywords(+ self,+ segment_ref: SegmentRef,+ snapshot: SegmentIndexingSnapshot,+ keywords: Sequence[str] | None,+ *,+ replace_existing: bool,+ ) -> None:+ assert snapshot.segment.index_node_id is not None+ self._write_keywords(+ segment_ref.document, snapshot, [self._document(snapshot)], [keywords], replace_existing=replace_existing+ )++ def _write_keywords(+ self,+ document_ref: DocumentRef,+ snapshot: SegmentIndexingSnapshot,+ documents: list[Document],+ keywords_list: Sequence[Sequence[str] | None] | None,+ *,+ replace_existing: bool,+ ) -> None:+ dataset_ref = document_ref.dataset++ def persist(storage_type: str, data: str, selected: Mapping[str, Sequence[str]]) -> None:+ self._segments.save_keyword_table(dataset_ref, storage_type=storage_type, data=data, keywords=selected)++ Jieba(snapshot.dataset).update_texts(+ documents,+ read=lambda: self._segments.get_keyword_table(dataset_ref),+ write=persist,+ lock=self._redis.lock(f"keyword_indexing_lock_{snapshot.dataset.id}", timeout=600),+ keywords_list=keywords_list,+ replace_existing=replace_existing,+ )++ def delete(+ self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]+ ) -> None:+ self._delete_task(+ [segment.index_node_id for segment in segments],+ document_ref.dataset.dataset_id,+ document_ref.document_id,+ [segment.id for segment in segments],+ [child.index_node_id for child in children if child.index_node_id],+ )++ def change_status(self, document_ref: DocumentRef, segment_ids: Sequence[str], action: str) -> None:+ task = self._enable_task if action == "enable" else self._disable_task+ task(list(segment_ids), document_ref.dataset.dataset_id, document_ref.document_id)diff --git a/api/services/knowledge/summaries/__init__.py b/api/services/knowledge/summaries/__init__.pynew file mode 100644index 00000000000..e69de29bb2ddiff --git a/api/services/summary_index_service.py b/api/services/knowledge/summaries/adapters.pysimilarity index 63%rename from api/services/summary_index_service.pyrename to api/services/knowledge/summaries/adapters.pyindex 2f5d832c0e1..f85bce12d77 100644--- a/api/services/summary_index_service.py+++ b/api/services/knowledge/summaries/adapters.py@@ -1,13 +1,14 @@-"""Summary index service for generating and managing document segment summaries."""+"""Adapters for existing ORM-based summary callers and indexing infrastructure."""import loggingimport timeimport uuid-from datetime import UTC, datetime+from collections.abc import Callablefrom typing import TypedDict, castfrom sqlalchemy import select, updatefrom sqlalchemy.orm import Session+from sqlalchemy.orm.attributes import set_committed_valuefrom core.credit_usage import CreditUsageCreatedByfrom core.db.session_factory import session_factory@@ -24,10 +25,21 @@ from libs import helperfrom models.dataset import Dataset, DocumentSegment, DocumentSegmentSummaryfrom models.dataset import Document as DatasetDocumentfrom models.enums import SummaryStatus+from repositories.knowledge.dataset_read_repository import get_segment_document+from repositories.knowledge.summary_repository import SQLAlchemySummaryRepository+from services.knowledge.resource_scope import DatasetRef+from services.knowledge.summaries.application import SummaryIndexService, SummaryVectorInput, SummaryVectorResultlogger = logging.getLogger(__name__)+def _delete_summary_vectors(dataset: Dataset, node_ids: list[str], *, session: Session) -> None:+ # Legacy callers own this transaction; committing here breaks session.begin().+ # Document cleanup tasks materialize their inputs before deleting external indexes.+ vector_type = Vector.resolve_vector_type(dataset, session=session)+ Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(node_ids)++class SummaryEntryDict(TypedDict):segment_id: strsegment_position: int@@ -44,8 +56,8 @@ class DocumentSummaryStatusDetailDict(TypedDict):summaries: list[SummaryEntryDict]-class SummaryIndexService:- """Service for generating and managing summary indexes."""+class SummaryIndexAdapter:+ """Adapt existing indexing callers to summary generation, storage and vectors."""@staticmethod@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)@@ -65,7 +77,8 @@ class SummaryIndexService:summary_index_setting: Summary index configurationKeyword Args:- session: SQLAlchemy session used to load the segment's document.+ session: Caller session, committed after loading generation inputs and+ before invoking the model. Image reads use a separate bounded session.Returns:Tuple of (summary_content, llm_usage) where llm_usage is LLMUsage object@@ -73,24 +86,34 @@ class SummaryIndexService:Raises:ValueError: If summary_index_setting is invalid or generation fails"""- # Reuse the existing generate_summary method from ParagraphIndexProcessor- # Use lazy import to avoid circular import+ # Lazy import avoids the index processor's circular dependency on this adapter.from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor# Get document language to ensure summary is generated in the correct language# This is especially important for image-only chunks where text is empty or minimaldocument_language = None- document = segment.get_document(session=session)+ document = get_segment_document(segment, session=session)if document and document.doc_language:document_language = document.doc_language- summary_content, usage = ParagraphIndexProcessor.generate_summary(- tenant_id=dataset.tenant_id,- text=segment.content,+ tenant_id, text, segment_id = dataset.tenant_id, segment.content, segment.id+ if segment.tenant_id != tenant_id or segment.dataset_id != dataset.id:+ raise ValueError("Segment does not belong to the dataset")+ session.commit()++ def load_images():+ with session_factory.create_session() as image_session:+ images = ParagraphIndexProcessor._extract_images_from_segment_attachments(+ tenant_id, segment_id, image_session+ )+ return images or ParagraphIndexProcessor._extract_images_from_text(tenant_id, text, image_session)++ summary_content, usage = ParagraphIndexProcessor.generate_summary_from_inputs(+ tenant_id=tenant_id,+ text=text,summary_index_setting=summary_index_setting,- segment_id=segment.id,document_language=document_language,- session=session,+ image_loader=load_images,)if not summary_content:@@ -167,416 +190,79 @@ class SummaryIndexService:segment: DocumentSegment,dataset: Dataset,session: Session | None = None,+ *,+ new_session: Callable[[], Session] | None = None,+ vector_factory: Callable[[], Vector] | None = None,) -> None:- """- Vectorize summary and store in vector database.+ """Bridge already-authorized ORM callers to the detached summary use case.- Args:- summary_record: DocumentSegmentSummary record- segment: Original DocumentSegment- dataset: Dataset containing the segment- session: Optional SQLAlchemy session. If provided, uses this session instead of creating a new one.- If not provided, creates a new session and commits automatically.+ Pending summary writes are committed before external I/O. Each outcome+ attempt then owns a separate transaction, including retries and failures."""if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:- logger.warning(- "Summary vectorization skipped for dataset %s: indexing_technique is not high_quality",- dataset.id,- )return-- # Get summary_record_id for later session queries- summary_record_id = summary_record.id- # Save the original session parameter for use in error handling- original_session = session-- def create_vector() -> Vector:- if original_session is not None:- return Vector(dataset, session=original_session)- with session_factory.create_session() as vector_session:- return Vector(dataset, session=vector_session)-- logger.debug(- "Starting vectorization for segment %s, summary_record_id=%s, using_provided_session=%s",- segment.id,- summary_record_id,- original_session is not None,+ ref = DatasetRef(dataset.tenant_id, dataset.id).document(segment.document_id).segment(segment.id)+ if segment.dataset_id != dataset.id or segment.tenant_id != dataset.tenant_id:+ raise ValueError("Segment does not belong to the dataset")+ if (+ summary_record.dataset_id != dataset.id+ or summary_record.document_id != segment.document_id+ or summary_record.chunk_id != segment.id+ ):+ raise ValueError("Summary does not belong to the segment")+ command = SummaryVectorInput(+ ref, summary_record.id, summary_record.summary_content or "", summary_record.summary_index_node_id)-- # Reuse existing index_node_id if available (like segment does), otherwise generate new one- old_summary_node_id = summary_record.summary_index_node_id- if old_summary_node_id:- # Reuse existing index_node_id (like segment behavior)- summary_index_node_id = old_summary_node_id- logger.debug("Reusing existing index_node_id %s for segment %s", summary_index_node_id, segment.id)- else:- # Generate new index node ID only for new summaries- summary_index_node_id = str(uuid.uuid4())- logger.debug("Generated new index_node_id %s for segment %s", summary_index_node_id, segment.id)-- # Always regenerate hash (in case summary content changed)- summary_content = summary_record.summary_content- if not summary_content or not summary_content.strip():+ if not command.content.strip():raise ValueError(f"Summary content is empty for segment {segment.id}, cannot vectorize")- summary_hash = helper.generate_text_hash(summary_content)-- # Delete old vector only if we're reusing the same index_node_id (to overwrite)- # If index_node_id changed, the old vector should have been deleted elsewhere- if old_summary_node_id and old_summary_node_id == summary_index_node_id:- try:- vector = create_vector()- vector.delete_by_ids([old_summary_node_id])- except Exception as e:- logger.warning(- "Failed to delete old summary vector for segment %s: %s. Continuing with new vectorization.",- segment.id,- str(e),+ if session is not None:+ session.commit()+ create_session = new_session or session_factory.create_session+ with create_session() as read_session:+ owned_dataset = read_session.scalar(+ select(Dataset).where(+ Dataset.id == ref.document.dataset.dataset_id, Dataset.tenant_id == ref.document.dataset.tenant_id)-- # Calculate embedding tokens for summary (for logging and statistics)- embedding_tokens = 0- try:- model_manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)- embedding_model = model_manager.get_model_instance(- tenant_id=dataset.tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,)- if embedding_model:- tokens_list = embedding_model.get_text_embedding_num_tokens([summary_content])- raw_embedding_tokens = tokens_list[0] if tokens_list else 0- embedding_tokens = raw_embedding_tokens if isinstance(raw_embedding_tokens, int) else 0- except Exception:- logger.warning("Failed to calculate embedding tokens for summary", exc_info=True)-- # Create document with summary content and metadata- summary_document = Document(- page_content=summary_content,- metadata={- "doc_id": summary_index_node_id,- "doc_hash": summary_hash,- "dataset_id": dataset.id,- "document_id": segment.document_id,- "original_chunk_id": segment.id, # Key: link to original chunk- "doc_type": DocType.TEXT,- "is_summary": True, # Identifier for summary documents- },- )-- # Vectorize and store with retry mechanism for connection errors- max_retries = 3- retry_delay = 2.0-- for attempt in range(max_retries):- try:- logger.debug(- "Attempting to vectorize summary for segment %s (attempt %s/%s)",- segment.id,- attempt + 1,- max_retries,- )- vector = create_vector()- # Use duplicate_check=False to ensure re-vectorization even if old vector still exists- # The old vector should have been deleted above, but if deletion failed,- # we still want to re-vectorize (upsert will overwrite)- vector.add_texts([summary_document], duplicate_check=False)- logger.debug(- "Successfully added summary vector to database for segment %s (attempt %s/%s)",- segment.id,- attempt + 1,- max_retries,+ owned_segment = read_session.scalar(+ select(DocumentSegment).where(+ DocumentSegment.id == ref.segment_id,+ DocumentSegment.tenant_id == ref.document.dataset.tenant_id,+ DocumentSegment.dataset_id == ref.document.dataset.dataset_id,+ DocumentSegment.document_id == ref.document.document_id,)-- # Log embedding token usage- if embedding_tokens > 0:- logger.info(- "Summary embedding for segment %s used %s tokens",- segment.id,- embedding_tokens,- )-- # Success - update summary record with index node info- # Use provided session if available, otherwise create a new one- use_provided_session = session is not None- if not use_provided_session:- logger.debug("Creating new session for vectorization of segment %s", segment.id)- session_context = session_factory.create_session()- session = session_context.__enter__()- else:- logger.debug("Using provided session for vectorization of segment %s", segment.id)- session_context = None # Don't use context manager for provided session-- # At this point, session is guaranteed to be not None- # Type narrowing: session is definitely not None after the if/else above- if session is None:- raise RuntimeError("Session should not be None at this point")-- try:- # Declare summary_record_in_session variable- summary_record_in_session: DocumentSegmentSummary | None-- # If using provided session, merge the summary_record into it- if use_provided_session:- # Merge the summary_record into the provided session- logger.debug(- "Merging summary_record (id=%s) into provided session for segment %s",- summary_record_id,- segment.id,- )- summary_record_in_session = session.merge(summary_record)- logger.debug(- "Successfully merged summary_record for segment %s, merged_id=%s",- segment.id,- summary_record_in_session.id,- )- else:- # Query the summary record in the new session- logger.debug(- "Querying summary_record by id=%s for segment %s in new session",- summary_record_id,- segment.id,- )- summary_record_in_session = session.scalar(- select(DocumentSegmentSummary)- .where(DocumentSegmentSummary.id == summary_record_id)- .limit(1)- )-- if not summary_record_in_session:- # Record not found - try to find by chunk_id and dataset_id instead- logger.debug(- "Summary record not found by id=%s, trying chunk_id=%s and dataset_id=%s "- "for segment %s",- summary_record_id,- segment.id,- dataset.id,- segment.id,- )- summary_record_in_session = session.scalar(- select(DocumentSegmentSummary)- .where(- DocumentSegmentSummary.chunk_id == segment.id,- DocumentSegmentSummary.dataset_id == dataset.id,- )- .limit(1)- )-- if not summary_record_in_session:- # Still not found - create a new one using the parameter data- logger.warning(- "Summary record not found in database for segment %s (id=%s), creating new one. "- "This may indicate a session isolation issue.",- segment.id,- summary_record_id,- )- summary_record_in_session = DocumentSegmentSummary(- dataset_id=dataset.id,- document_id=segment.document_id,- chunk_id=segment.id,- summary_content=summary_content,- summary_index_node_id=summary_index_node_id,- summary_index_node_hash=summary_hash,- tokens=embedding_tokens,- status=SummaryStatus.COMPLETED,- enabled=True,- )- if summary_record_in_session is None:- raise RuntimeError("summary_record_in_session should not be None at this point")- summary_record_in_session.id = summary_record_id- session.add(summary_record_in_session)- logger.info(- "Created new summary record (id=%s) for segment %s after vectorization",- summary_record_id,- segment.id,- )- else:- # Found by chunk_id - update it- logger.info(- "Found summary record for segment %s by chunk_id "- "(id mismatch: expected %s, found %s). "- "This may indicate the record was created in a different session.",- segment.id,- summary_record_id,- summary_record_in_session.id,- )- else:- logger.debug(- "Found summary_record (id=%s) for segment %s in new session",- summary_record_id,- segment.id,- )-- # At this point, summary_record_in_session is guaranteed to be not None- if summary_record_in_session is None:- raise RuntimeError("summary_record_in_session should not be None at this point")-- # Update all fields including summary_content- # Always use the summary_content from the parameter (which is the latest from outer session)- # rather than relying on what's in the database, in case outer session hasn't committed yet- summary_record_in_session.summary_index_node_id = summary_index_node_id- summary_record_in_session.summary_index_node_hash = summary_hash- summary_record_in_session.tokens = embedding_tokens # Save embedding tokens- summary_record_in_session.status = SummaryStatus.COMPLETED- # Ensure summary_content is preserved (use the latest from summary_record parameter)- # This is critical: use the parameter value, not the database value- summary_record_in_session.summary_content = summary_content- # Explicitly update updated_at to ensure it's refreshed even if other fields haven't changed- summary_record_in_session.updated_at = datetime.now(UTC).replace(tzinfo=None)- session.add(summary_record_in_session)-- # Only commit if we created the session ourselves- if not use_provided_session:- logger.debug("Committing session for segment %s (self-created session)", segment.id)- session.commit()- logger.debug("Successfully committed session for segment %s", segment.id)- else:- # When using provided session, flush to ensure changes are written to database- # This prevents refresh() from overwriting our changes- logger.debug(- "Flushing session for segment %s (using provided session, caller will commit)",- segment.id,- )- session.flush()- logger.debug("Successfully flushed session for segment %s", segment.id)- # If using provided session, let the caller handle commit-- logger.info(- "Successfully vectorized summary for segment %s, index_node_id=%s, index_node_hash=%s, "- "tokens=%s, summary_record_id=%s, use_provided_session=%s",- segment.id,- summary_index_node_id,- summary_hash,- embedding_tokens,- summary_record_in_session.id,- use_provided_session,- )- # Update the original object for consistency- summary_record.summary_index_node_id = summary_index_node_id- summary_record.summary_index_node_hash = summary_hash- summary_record.tokens = embedding_tokens- summary_record.status = SummaryStatus.COMPLETED- summary_record.summary_content = summary_content- if summary_record_in_session.updated_at:- summary_record.updated_at = summary_record_in_session.updated_at- finally:- # Only close session if we created it ourselves- if not use_provided_session and session_context:- session_context.__exit__(None, None, None)- # Success, exit function- return-- except (ConnectionError, Exception) as e:- error_str = str(e).lower()- # Check if it's a connection-related error that might be transient- is_connection_error = any(- keyword in error_str- for keyword in [- "connection",- "disconnected",- "timeout",- "network",- "could not connect",- "server disconnected",- "weaviate",- ]+ )+ owned_document = read_session.scalar(+ select(DatasetDocument).where(+ DatasetDocument.id == ref.document.document_id,+ DatasetDocument.dataset_id == ref.document.dataset.dataset_id,+ DatasetDocument.tenant_id == ref.document.dataset.tenant_id,)-- if is_connection_error and attempt < max_retries - 1:- # Retry for connection errors- wait_time = retry_delay * (2**attempt) # Exponential backoff- logger.warning(- "Vectorization attempt %s/%s failed for segment %s (connection error): %s. "- "Retrying in %.1f seconds...",- attempt + 1,- max_retries,- segment.id,- str(e),- wait_time,- )- time.sleep(wait_time)- continue- else:- # Final attempt failed or non-connection error - log and update status- logger.error(- "Failed to vectorize summary for segment %s after %s attempts: %s. "- "summary_record_id=%s, index_node_id=%s, use_provided_session=%s",- segment.id,- attempt + 1,- str(e),- summary_record_id,- summary_index_node_id,- session is not None,- exc_info=True,- )- # Update error status in session- # Use the original_session saved at function start (the function parameter)- logger.debug(- "Updating error status for segment %s, summary_record_id=%s, has_original_session=%s",- segment.id,- summary_record_id,- original_session is not None,- )- if original_session is not None:- # Keep the error update in the caller-owned transaction. Opening another- # session here can deadlock when the caller has already flushed this row.- summary_record.status = SummaryStatus.ERROR- summary_record.error = f"Vectorization failed: {str(e)}"- summary_record.updated_at = datetime.now(UTC).replace(tzinfo=None)- original_session.add(summary_record)- raise-- # Standalone callers still need this method to persist the error itself.- with session_factory.create_session() as error_session:- # Try to find the record by id first- # Note: Using assignment only (no type annotation) to avoid redeclaration error- summary_record_in_session = error_session.scalar(- select(DocumentSegmentSummary)- .where(DocumentSegmentSummary.id == summary_record_id)- .limit(1)- )- if not summary_record_in_session:- # Try to find by chunk_id and dataset_id- logger.debug(- "Summary record not found by id=%s, trying chunk_id=%s and dataset_id=%s "- "for segment %s",- summary_record_id,- segment.id,- dataset.id,- segment.id,- )- summary_record_in_session = error_session.scalar(- select(DocumentSegmentSummary)- .where(- DocumentSegmentSummary.chunk_id == segment.id,- DocumentSegmentSummary.dataset_id == dataset.id,- )- .limit(1)- )-- if summary_record_in_session:- summary_record_in_session.status = SummaryStatus.ERROR- summary_record_in_session.error = f"Vectorization failed: {str(e)}"- summary_record_in_session.updated_at = datetime.now(UTC).replace(tzinfo=None)- error_session.add(summary_record_in_session)- error_session.commit()- logger.info(- "Updated error status in new session for segment %s, record_id=%s",- segment.id,- summary_record_in_session.id,- )- # Update the original object for consistency- summary_record.status = SummaryStatus.ERROR- summary_record.error = summary_record_in_session.error- summary_record.updated_at = summary_record_in_session.updated_at- else:- logger.warning(- "Could not update error status: summary record not found for segment %s (id=%s). "- "This may indicate a session isolation issue.",- segment.id,- summary_record_id,- )- raise+ )+ if owned_dataset is None or owned_segment is None or owned_document is None:+ raise LookupError("Summary segment no longer exists in this dataset")+ vector_type = (+ Vector.resolve_vector_type(owned_dataset, session=read_session) if vector_factory is None else None+ )+ factory = vector_factory or (lambda: Vector(owned_dataset, session=None, vector_type=vector_type))+ service = SummaryIndexService(+ store=SQLAlchemySummaryRepository(new_session=create_session),+ vectors=SummaryVectorGateway(owned_dataset, factory),+ new_id=lambda: str(uuid.uuid4()),+ text_hash=helper.generate_text_hash,+ sleep=time.sleep,+ )+ try:+ result = service.vectorize(command)+ except Exception as error:+ set_committed_value(summary_record, "status", SummaryStatus.ERROR)+ set_committed_value(summary_record, "error", f"Vectorization failed: {error}")+ raise+ set_committed_value(summary_record, "summary_index_node_id", result.node_id)+ set_committed_value(summary_record, "summary_index_node_hash", result.content_hash)+ set_committed_value(summary_record, "tokens", result.tokens)+ set_committed_value(summary_record, "status", SummaryStatus.COMPLETED)+ set_committed_value(summary_record, "error", None)@staticmethoddef batch_create_summary_records(@@ -726,7 +412,7 @@ class SummaryIndexService:session.commit()# Generate summary (returns summary_content and llm_usage)- summary_content, llm_usage = SummaryIndexService.generate_summary_for_segment(+ summary_content, llm_usage = SummaryIndexAdapter.generate_summary_for_segment(segment, dataset, summary_index_setting, session=session)@@ -746,7 +432,7 @@ class SummaryIndexService:llm_usage.completion_tokens,)- SummaryIndexService.vectorize_summary(summary_record_in_session, segment, dataset, session=session)+ SummaryIndexAdapter.vectorize_summary(summary_record_in_session, segment, dataset, session=session)# vectorize_summary mutates status and token fields; refresh before returning the ORM object.session.refresh(summary_record_in_session)session.commit()@@ -837,9 +523,10 @@ class SummaryIndexService:logger.info("No segments found for document %s", document.id)return []+ session.commit()# Batch create summary records with "not_started" status before processing# This ensures all records exist upfront, allowing status tracking- SummaryIndexService.batch_create_summary_records(+ SummaryIndexAdapter.batch_create_summary_records(segments=segments,dataset=dataset,status=SummaryStatus.NOT_STARTED,@@ -860,14 +547,14 @@ class SummaryIndexService:passtry:- summary_record = SummaryIndexService.generate_and_vectorize_summary(+ summary_record = SummaryIndexAdapter.generate_and_vectorize_summary(segment, dataset, summary_index_setting, session=session)summary_records.append(summary_record)except Exception as e:logger.exception("Failed to generate summary for segment %s", segment.id)# Update summary record with error status- SummaryIndexService.update_summary_record_error(+ SummaryIndexAdapter.update_summary_record_error(segment=segment,dataset=dataset,error=str(e),@@ -925,8 +612,7 @@ class SummaryIndexService:summary_node_ids = [s.summary_index_node_id for s in summaries if s.summary_index_node_id]if summary_node_ids:try:- vector = Vector(dataset, session=session)- vector.delete_by_ids(summary_node_ids)+ _delete_summary_vectors(dataset, summary_node_ids, session=session)except Exception:logger.warning("Failed to remove summary vectors", exc_info=True)@@ -1006,7 +692,7 @@ class SummaryIndexService:try:# Re-vectorize summary (this will update status and tokens in its own session)# Pass the session to vectorize_summary to avoid session isolation issues- SummaryIndexService.vectorize_summary(summary, segment, dataset, session=session)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset, session=session)# Refresh the object from database to get the updated status and tokens from vectorize_summarysession.refresh(summary)@@ -1050,14 +736,13 @@ class SummaryIndexService:if not summaries:return- # Delete from vector database+ # Preserve the caller's transaction until it decides to commit or roll back.if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:summary_node_ids = [s.summary_index_node_id for s in summaries if s.summary_index_node_id]if summary_node_ids:- vector = Vector(dataset, session=session)- vector.delete_by_ids(summary_node_ids)+ _delete_summary_vectors(dataset, summary_node_ids, session=session)- # Delete summary records+ # Delete summary records in the caller's existing transaction.for summary in summaries:session.delete(summary)@@ -1095,7 +780,7 @@ class SummaryIndexService:# Vectorization uses dataset.embedding_model, which doesn't require summary_index_setting# Skip qa_model documents- document = segment.get_document(session=session)+ document = get_segment_document(segment, session=session)if document and document.doc_form == "qa_model":return None@@ -1117,8 +802,7 @@ class SummaryIndexService:old_summary_node_id = summary_record.summary_index_node_idif old_summary_node_id:try:- vector = Vector(dataset, session=session)- vector.delete_by_ids([old_summary_node_id])+ _delete_summary_vectors(dataset, [old_summary_node_id], session=session)except Exception as e:logger.warning("Failed to delete old summary vector for segment %s: %s",@@ -1137,9 +821,6 @@ class SummaryIndexService:return Noneif summary_record:- # Update existing summary- old_summary_node_id = summary_record.summary_index_node_id-# Update summary contentsummary_record.summary_content = summary_contentsummary_record.status = SummaryStatus.GENERATING@@ -1148,20 +829,9 @@ class SummaryIndexService:# Flush to ensure summary_content is saved before vectorize_summary queries itsession.flush()- # Delete old vector if exists (before vectorization)- if old_summary_node_id:- try:- vector = Vector(dataset, session=session)- vector.delete_by_ids([old_summary_node_id])- except Exception as e:- logger.warning(- "Failed to delete old summary vector for segment %s: %s",- segment.id,- str(e),- )else:# Create new summary record if doesn't exist- summary_record = SummaryIndexService.create_summary_record(+ summary_record = SummaryIndexAdapter.create_summary_record(segment,dataset,summary_content,@@ -1171,7 +841,7 @@ class SummaryIndexService:try:# Vectorization must finish here so the manual summary is searchable immediately.- SummaryIndexService.vectorize_summary(summary_record, segment, dataset, session=session)+ SummaryIndexAdapter.vectorize_summary(summary_record, segment, dataset, session=session)session.refresh(summary_record)session.commit()logger.info("Successfully updated and re-vectorized summary for segment %s", segment.id)@@ -1340,7 +1010,7 @@ class SummaryIndexService:return None# Get all summary records for these segments- summaries = SummaryIndexService.get_segments_summaries(segment_ids, dataset_id, session=session)+ summaries = SummaryIndexAdapter.get_segments_summaries(segment_ids, dataset_id, session=session)summary_status_map = {chunk_id: summary.status for chunk_id, summary in summaries.items()}# Check if there are any "not_started" or "generating" status summaries@@ -1396,7 +1066,7 @@ class SummaryIndexService:# Get all summary records for these segmentsall_segment_ids = [seg.id for seg in segments]- summaries = SummaryIndexService.get_segments_summaries(all_segment_ids, dataset_id, session=session)+ summaries = SummaryIndexAdapter.get_segments_summaries(all_segment_ids, dataset_id, session=session)summary_status_map = {chunk_id: summary.status for chunk_id, summary in summaries.items()}# Calculate summary_index_status for each document@@ -1433,6 +1103,8 @@ class SummaryIndexService:document_id: str,dataset_id: str,session: Session,+ *,+ tenant_id: str,) -> DocumentSummaryStatusDetailDict:"""Get detailed summary status for a document.@@ -1440,6 +1112,7 @@ class SummaryIndexService:Args:document_id: Document IDdataset_id: Dataset ID+ tenant_id: Owning tenant IDsession: SQLAlchemy session used for segment lookupReturns:@@ -1453,10 +1126,11 @@ class SummaryIndexService:- timeout: Number of summaries that timed out- summaries: List of summary records with status and content preview"""- from services.dataset_service import SegmentService+ from services.knowledge.dataset_service import SegmentService# Get all segments for this documentsegments = SegmentService.get_segments_by_document_and_dataset(+ tenant_id=tenant_id,document_id=document_id,dataset_id=dataset_id,session=session,@@ -1470,7 +1144,7 @@ class SummaryIndexService:segment_ids = [segment.id for segment in segments]summaries = []if segment_ids:- summaries = SummaryIndexService.get_document_summaries(+ summaries = SummaryIndexAdapter.get_document_summaries(document_id=document_id,dataset_id=dataset_id,segment_ids=segment_ids,@@ -1528,3 +1202,37 @@ class SummaryIndexService:summary_status=cast(dict[str, int], status_counts),summaries=summary_list,)+++class SummaryVectorGateway:+ def __init__(self, dataset: Dataset, factory: Callable[[], Vector]) -> None:+ self._dataset, self._factory = dataset, factory++ def delete(self, node_id: str) -> None:+ self._factory().delete_by_ids([node_id])++ def count_tokens(self, content: str) -> int:+ dataset = self._dataset+ model = ModelManager.for_tenant(tenant_id=dataset.tenant_id).get_model_instance(+ tenant_id=dataset.tenant_id,+ provider=dataset.embedding_model_provider or "",+ model_type=ModelType.TEXT_EMBEDDING,+ model=dataset.embedding_model or "",+ )+ tokens = model.get_text_embedding_num_tokens([content])+ return tokens[0] if tokens and isinstance(tokens[0], int) else 0++ def write(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None:+ document = Document(+ page_content=command.content,+ metadata={+ "doc_id": result.node_id,+ "doc_hash": result.content_hash,+ "dataset_id": command.ref.document.dataset.dataset_id,+ "document_id": command.ref.document.document_id,+ "original_chunk_id": command.ref.segment_id,+ "doc_type": DocType.TEXT,+ "is_summary": True,+ },+ )+ self._factory().add_texts([document], duplicate_check=False)diff --git a/api/services/knowledge/summaries/application.py b/api/services/knowledge/summaries/application.pynew file mode 100644index 00000000000..538bc465a3d--- /dev/null+++ b/api/services/knowledge/summaries/application.py@@ -0,0 +1,82 @@+"""Summary vectorization policy independent of ORM, model providers, and vector stores."""++import logging+from collections.abc import Callable+from dataclasses import dataclass+from typing import Protocol++from services.knowledge.resource_scope import SegmentRef++logger = logging.getLogger(__name__)+++@dataclass(frozen=True)+class SummaryVectorInput:+ ref: SegmentRef+ summary_id: str+ content: str+ previous_node_id: str | None+++@dataclass(frozen=True)+class SummaryVectorResult:+ node_id: str+ content_hash: str+ tokens: int+++class SummaryStore(Protocol):+ def completed(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...+ def failed(self, command: SummaryVectorInput, error: str) -> None: ...+++class SummaryVectorBackend(Protocol):+ def delete(self, node_id: str) -> None: ...+ def count_tokens(self, content: str) -> int: ...+ def write(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...+++class SummaryIndexService:+ def __init__(+ self,+ *,+ store: SummaryStore,+ vectors: SummaryVectorBackend,+ new_id: Callable[[], str],+ text_hash: Callable[[str], str],+ sleep: Callable[[float], None],+ ) -> None:+ self._store, self._vectors = store, vectors+ self._new_id, self._hash, self._sleep = new_id, text_hash, sleep++ def vectorize(self, command: SummaryVectorInput) -> SummaryVectorResult:+ if not command.content.strip():+ raise ValueError(f"Summary content is empty for segment {command.ref.segment_id}, cannot vectorize")+ node_id = command.previous_node_id or self._new_id()+ if command.previous_node_id:+ try:+ self._vectors.delete(command.previous_node_id)+ except Exception:+ logger.warning("Failed to delete previous summary vector for %s", command.ref.segment_id, exc_info=True)+ tokens = 0+ try:+ tokens = self._vectors.count_tokens(command.content)+ except Exception:+ logger.warning("Failed to calculate summary embedding tokens", exc_info=True)+ result = SummaryVectorResult(node_id, self._hash(command.content), tokens)+ for attempt in range(3):+ try:+ self._vectors.write(command, result)+ self._store.completed(command, result)+ return result+ except Exception as error:+ transient = any(+ word in str(error).lower()+ for word in ("connection", "disconnected", "timeout", "network", "weaviate")+ )+ if transient and attempt < 2:+ self._sleep(2.0 * 2**attempt)+ continue+ self._store.failed(command, f"Vectorization failed: {error}")+ raise+ raise AssertionError("Unreachable vectorization retry state")diff --git a/api/services/metadata_service.py b/api/services/metadata_service.pydeleted file mode 100644index 317e371fc2a..00000000000--- a/api/services/metadata_service.py+++ /dev/null@@ -1,399 +0,0 @@-import copy-import logging--from sqlalchemy import delete, func, select-from sqlalchemy.orm import Session--from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource-from extensions.ext_redis import redis_client-from libs.datetime_utils import naive_utc_now-from libs.login import resolve_account_fallback-from models import Account-from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Document-from models.enums import DatasetMetadataType-from services.dataset_ref_service import DatasetRefService-from services.dataset_service import DocumentService-from services.entities.knowledge_entities.knowledge_entities import (- MetadataArgs,- MetadataOperationData,-)-from services.errors.metadata import MetadataResourceNotFoundError--logger = logging.getLogger(__name__)---class MetadataService:- @staticmethod- def create_metadata(- dataset_id: str,- metadata_args: MetadataArgs,- current_user: Account | None = None, # TODO: the service_api is not migrated yet- current_tenant_id: str | None = None,- *,- session: Session,- ) -> DatasetMetadata:- # check if metadata name is too long- if len(metadata_args.name) > 255:- raise ValueError("Metadata name cannot exceed 255 characters.")- current_user, current_tenant_id = resolve_account_fallback(current_user, current_tenant_id)- # check if metadata name already exists- if session.scalar(- select(DatasetMetadata)- .where(- DatasetMetadata.tenant_id == current_tenant_id,- DatasetMetadata.dataset_id == dataset_id,- DatasetMetadata.name == metadata_args.name,- )- .limit(1)- ):- raise ValueError("Metadata name already exists.")- for field in BuiltInField:- if field.value == metadata_args.name:- raise ValueError("Metadata name already exists in Built-in fields.")- metadata = DatasetMetadata(- tenant_id=current_tenant_id,- dataset_id=dataset_id,- type=metadata_args.type,- name=metadata_args.name,- created_by=current_user.id,- )- session.add(metadata)- session.flush()- return metadata-- @staticmethod- def update_metadata_name(- dataset: Dataset,- metadata_id: str,- name: str,- current_user: Account,- *,- session: Session,- ) -> DatasetMetadata | None:- # check if metadata name is too long- if len(name) > 255:- raise ValueError("Metadata name cannot exceed 255 characters.")-- lock_key = f"dataset_metadata_lock_{dataset.id}"- # check if metadata name already exists- if session.scalar(- select(DatasetMetadata)- .where(- DatasetMetadata.tenant_id == dataset.tenant_id,- DatasetMetadata.dataset_id == dataset.id,- DatasetMetadata.name == name,- )- .limit(1)- ):- raise ValueError("Metadata name already exists.")- for field in BuiltInField:- if field.value == name:- raise ValueError("Metadata name already exists in Built-in fields.")- try:- MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)- metadata = session.scalar(- select(DatasetMetadata)- .where(- DatasetMetadata.id == metadata_id,- DatasetMetadata.tenant_id == dataset.tenant_id,- DatasetMetadata.dataset_id == dataset.id,- )- .limit(1)- )- if metadata is None:- raise ValueError("Metadata not found.")- old_name = metadata.name- metadata.name = name- metadata.updated_by = current_user.id- metadata.updated_at = naive_utc_now()-- # update related documents- dataset_metadata_bindings = session.scalars(- select(DatasetMetadataBinding).where(- DatasetMetadataBinding.metadata_id == metadata_id,- DatasetMetadataBinding.tenant_id == dataset.tenant_id,- DatasetMetadataBinding.dataset_id == dataset.id,- )- ).all()- if dataset_metadata_bindings:- document_ids = [binding.document_id for binding in dataset_metadata_bindings]- documents = DocumentService.get_document_by_ids(- DatasetRefService.create_dataset_ref(dataset), document_ids, session- )- for document in documents:- if not document.doc_metadata:- doc_metadata = {}- else:- doc_metadata = copy.deepcopy(document.doc_metadata)- value = doc_metadata.pop(old_name, None)- doc_metadata[name] = value- document.doc_metadata = doc_metadata- session.add(document)- session.commit()- return metadata- except Exception:- logger.exception("Update metadata name failed")- return None- finally:- redis_client.delete(lock_key)-- @staticmethod- def delete_metadata(dataset: Dataset, metadata_id: str, session: Session):- lock_key = f"dataset_metadata_lock_{dataset.id}"- try:- MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)- metadata = session.scalar(- select(DatasetMetadata)- .where(- DatasetMetadata.id == metadata_id,- DatasetMetadata.tenant_id == dataset.tenant_id,- DatasetMetadata.dataset_id == dataset.id,- )- .limit(1)- )- if metadata is None:- raise ValueError("Metadata not found.")- session.delete(metadata)-- # deal related documents- dataset_metadata_bindings = session.scalars(- select(DatasetMetadataBinding).where(- DatasetMetadataBinding.metadata_id == metadata_id,- DatasetMetadataBinding.tenant_id == dataset.tenant_id,- DatasetMetadataBinding.dataset_id == dataset.id,- )- ).all()- if dataset_metadata_bindings:- document_ids = [binding.document_id for binding in dataset_metadata_bindings]- documents = DocumentService.get_document_by_ids(- DatasetRefService.create_dataset_ref(dataset), document_ids, session- )- for document in documents:- if not document.doc_metadata:- doc_metadata = {}- else:- doc_metadata = copy.deepcopy(document.doc_metadata)- doc_metadata.pop(metadata.name, None)- document.doc_metadata = doc_metadata- session.add(document)- session.commit()- return metadata- except Exception:- logger.exception("Delete metadata failed")- finally:- redis_client.delete(lock_key)-- @staticmethod- def get_built_in_fields():- return [- {"name": BuiltInField.document_name, "type": DatasetMetadataType.STRING},- {"name": BuiltInField.uploader, "type": DatasetMetadataType.STRING},- {"name": BuiltInField.upload_date, "type": DatasetMetadataType.TIME},- {"name": BuiltInField.last_update_date, "type": DatasetMetadataType.TIME},- {"name": BuiltInField.source, "type": DatasetMetadataType.STRING},- ]-- @staticmethod- def enable_built_in_field(dataset: Dataset, session: Session):- if dataset.built_in_field_enabled:- return- lock_key = f"dataset_metadata_lock_{dataset.id}"- try:- MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)- session.add(dataset)- documents = DocumentService.get_working_documents_by_dataset_id(dataset.id, session)- if documents:- for document in documents:- if not document.doc_metadata:- doc_metadata = {}- else:- doc_metadata = copy.deepcopy(document.doc_metadata)- doc_metadata[BuiltInField.document_name] = document.name- doc_metadata[BuiltInField.uploader] = document.get_uploader(session=session)- doc_metadata[BuiltInField.upload_date] = document.upload_date.timestamp()- doc_metadata[BuiltInField.last_update_date] = document.last_update_date.timestamp()- doc_metadata[BuiltInField.source] = MetadataDataSource[document.data_source_type]- document.doc_metadata = doc_metadata- session.add(document)- dataset.built_in_field_enabled = True- session.commit()- except Exception:- logger.exception("Enable built-in field failed")- finally:- redis_client.delete(lock_key)-- @staticmethod- def disable_built_in_field(dataset: Dataset, session: Session):- if not dataset.built_in_field_enabled:- return- lock_key = f"dataset_metadata_lock_{dataset.id}"- try:- MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)- session.add(dataset)- documents = DocumentService.get_working_documents_by_dataset_id(dataset.id, session)- document_ids = []- if documents:- for document in documents:- if not document.doc_metadata:- doc_metadata = {}- else:- doc_metadata = copy.deepcopy(document.doc_metadata)- doc_metadata.pop(BuiltInField.document_name, None)- doc_metadata.pop(BuiltInField.uploader, None)- doc_metadata.pop(BuiltInField.upload_date, None)- doc_metadata.pop(BuiltInField.last_update_date, None)- doc_metadata.pop(BuiltInField.source, None)- document.doc_metadata = doc_metadata- session.add(document)- document_ids.append(document.id)- dataset.built_in_field_enabled = False- session.commit()- except Exception:- logger.exception("Disable built-in field failed")- finally:- redis_client.delete(lock_key)-- @staticmethod- def update_documents_metadata(- dataset: Dataset,- metadata_args: MetadataOperationData,- current_user: Account,- *,- session: Session,- ):- metadata_ids = {- metadata_value.id- for operation in metadata_args.operation_data- for metadata_value in operation.metadata_list- }- metadatas = session.scalars(- select(DatasetMetadata).where(- DatasetMetadata.id.in_(metadata_ids),- DatasetMetadata.tenant_id == dataset.tenant_id,- DatasetMetadata.dataset_id == dataset.id,- )- ).all()- metadata_by_id = {metadata.id: metadata for metadata in metadatas}- if metadata_ids != set(metadata_by_id):- raise MetadataResourceNotFoundError("Metadata not found.")-- document_ids = {operation.document_id for operation in metadata_args.operation_data}- owned_document_ids = set(- session.scalars(- select(Document.id).where(- Document.id.in_(document_ids),- Document.tenant_id == dataset.tenant_id,- Document.dataset_id == dataset.id,- )- ).all()- )- if document_ids != owned_document_ids:- raise MetadataResourceNotFoundError("Document not found.")-- for operation in metadata_args.operation_data:- lock_key = f"document_metadata_lock_{operation.document_id}"- try:- MetadataService.knowledge_base_metadata_lock_check(None, operation.document_id)- document = session.scalar(- select(Document)- .where(- Document.id == operation.document_id,- Document.tenant_id == dataset.tenant_id,- Document.dataset_id == dataset.id,- )- .with_for_update()- .execution_options(populate_existing=True)- )- if document is None:- raise MetadataResourceNotFoundError("Document not found.")- if operation.partial_update:- doc_metadata = copy.deepcopy(document.doc_metadata) if document.doc_metadata else {}- else:- doc_metadata = {}- for metadata_value in operation.metadata_list:- doc_metadata[metadata_by_id[metadata_value.id].name] = metadata_value.value- if dataset.built_in_field_enabled:- doc_metadata[BuiltInField.document_name] = document.name- doc_metadata[BuiltInField.uploader] = document.get_uploader(session=session)- doc_metadata[BuiltInField.upload_date] = document.upload_date.timestamp()- doc_metadata[BuiltInField.last_update_date] = document.last_update_date.timestamp()- doc_metadata[BuiltInField.source] = MetadataDataSource[document.data_source_type]- document.doc_metadata = doc_metadata- session.add(document)-- # deal metadata binding (in the same transaction as the doc_metadata update)- if not operation.partial_update:- session.execute(- delete(DatasetMetadataBinding).where(- DatasetMetadataBinding.tenant_id == dataset.tenant_id,- DatasetMetadataBinding.dataset_id == dataset.id,- DatasetMetadataBinding.document_id == document.id,- )- )-- for metadata_value in operation.metadata_list:- # check if binding already exists- if operation.partial_update:- existing_binding = session.scalar(- select(DatasetMetadataBinding)- .where(- DatasetMetadataBinding.tenant_id == dataset.tenant_id,- DatasetMetadataBinding.dataset_id == dataset.id,- DatasetMetadataBinding.document_id == document.id,- DatasetMetadataBinding.metadata_id == metadata_value.id,- )- .limit(1)- )- if existing_binding:- continue-- dataset_metadata_binding = DatasetMetadataBinding(- tenant_id=dataset.tenant_id,- dataset_id=dataset.id,- document_id=document.id,- metadata_id=metadata_value.id,- created_by=current_user.id,- )- session.add(dataset_metadata_binding)- session.commit()- except Exception:- session.rollback()- logger.exception("Update documents metadata failed")- raise- finally:- redis_client.delete(lock_key)-- @staticmethod- def knowledge_base_metadata_lock_check(dataset_id: str | None, document_id: str | None):- if dataset_id:- lock_key = f"dataset_metadata_lock_{dataset_id}"- if redis_client.get(lock_key):- raise ValueError("Another knowledge base metadata operation is running, please wait a moment.")- redis_client.set(lock_key, 1, ex=3600)- if document_id:- lock_key = f"document_metadata_lock_{document_id}"- if redis_client.get(lock_key):- raise ValueError("Another document metadata operation is running, please wait a moment.")- redis_client.set(lock_key, 1, ex=3600)-- @staticmethod- def get_dataset_metadatas(dataset: Dataset, session: Session):- return {- "doc_metadata": [- {- "id": item.get("id"),- "name": item.get("name"),- "type": item.get("type"),- "count": session.scalar(- select(func.count(DatasetMetadataBinding.id)).where(- DatasetMetadataBinding.metadata_id == item.get("id"),- DatasetMetadataBinding.dataset_id == dataset.id,- )- )- or 0,- }- for item in dataset.get_doc_metadata(session=session)- if item.get("id") != "built-in"- ],- "built_in_field_enabled": dataset.built_in_field_enabled,- }diff --git a/api/services/model_provider/__init__.py b/api/services/model_provider/__init__.pynew file mode 100644index 00000000000..e69de29bb2ddiff --git a/api/services/model_provider_service.py b/api/services/model_provider/service.pysimilarity index 98%rename from api/services/model_provider_service.pyrename to api/services/model_provider/service.pyindex 9aee8eb87a5..a1e218f8cca 100644--- a/api/services/model_provider_service.py+++ b/api/services/model_provider/service.py@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Anyfrom sqlalchemy import and_, selectfrom sqlalchemy.orm import Session+from services.credentials.query import CredentialQuery+if TYPE_CHECKING:from models.account import Account@@ -459,12 +461,15 @@ class ModelProviderService:for model in provider_configurations.get_models(provider=provider)]- def get_provider_available_credentials(self, tenant_id: str, provider: str, user: "Account | None" = None):- return self._get_provider_manager(tenant_id).get_provider_available_credentials(- tenant_id=tenant_id,- provider_name=provider,- user=user,- )+ def get_provider_available_credentials(+ self, tenant_id: str, provider: str, user: "Account | None" = None, *, credential_query: CredentialQuery+ ) -> list[CredentialConfiguration]:+ return [+ CredentialConfiguration(credential_id=record.id, credential_name=record.name)+ for record in credential_query.list_models(+ workspace_id=tenant_id, provider=provider, actor_id=user.id if user else None+ )+ ]def get_provider_model_available_credentials(self,diff --git a/api/services/rag_pipeline/pipeline_generate_service.py b/api/services/rag_pipeline/pipeline_generate_service.pyindex 182e3fb283e..12e85611b04 100644--- a/api/services/rag_pipeline/pipeline_generate_service.py+++ b/api/services/rag_pipeline/pipeline_generate_service.py@@ -10,7 +10,9 @@ from models.dataset import Pipelinefrom models.enums import IndexingStatusfrom models.model import Account, App, EndUserfrom models.workflow import Workflow-from services.dataset_ref_service import DatasetRefService, DocumentRef+from repositories.knowledge.dataset_read_repository import get_pipeline_dataset+from services.knowledge.dataset_service import DocumentService+from services.knowledge.resource_scope import DatasetRef, DocumentReffrom services.rag_pipeline.rag_pipeline import RagPipelineService@@ -25,6 +27,7 @@ class PipelineGenerateService:streaming: bool = True,*,session: Session,+ generator: PipelineGenerator,):"""Pipeline Content Generate@@ -38,14 +41,14 @@ class PipelineGenerateService:try:workflow = cls._get_workflow(pipeline, invoke_from, session)if original_document_id := args.get("original_document_id"):- dataset = pipeline.retrieve_dataset(session)+ dataset = get_pipeline_dataset(pipeline, session)if dataset is None or dataset.tenant_id != pipeline.tenant_id:raise ValueError("Pipeline dataset is required")- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, original_document_id)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ document_ref = dataset_ref.document(original_document_id)cls.update_document_status(document_ref, session=session)return PipelineGenerator.convert_to_event_stream(- PipelineGenerator().generate(+ generator.generate(session=session,pipeline=pipeline,workflow=workflow,@@ -71,11 +74,19 @@ class PipelineGenerateService:@classmethoddef generate_single_iteration(- cls, pipeline: Pipeline, user: Account, node_id: str, args: Any, session: Session, streaming: bool = True+ cls,+ pipeline: Pipeline,+ user: Account,+ node_id: str,+ args: Any,+ session: Session,+ streaming: bool = True,+ *,+ generator: PipelineGenerator,):workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session)return PipelineGenerator.convert_to_event_stream(- PipelineGenerator().single_iteration_generate(+ generator.single_iteration_generate(pipeline=pipeline,workflow=workflow,node_id=node_id,@@ -88,11 +99,19 @@ class PipelineGenerateService:@classmethoddef generate_single_loop(- cls, pipeline: Pipeline, user: Account, node_id: str, args: Any, session: Session, streaming: bool = True+ cls,+ pipeline: Pipeline,+ user: Account,+ node_id: str,+ args: Any,+ session: Session,+ streaming: bool = True,+ *,+ generator: PipelineGenerator,):workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session)return PipelineGenerator.convert_to_event_stream(- PipelineGenerator().single_loop_generate(+ generator.single_loop_generate(pipeline=pipeline,workflow=workflow,node_id=node_id,@@ -130,7 +149,9 @@ class PipelineGenerateService:@classmethoddef update_document_status(cls, document_ref: DocumentRef, *, session: Session) -> None:"""Set a document in the owner-bound dataset to waiting."""- document = DatasetRefService.get_document_by_ref(document_ref, session=session)+ document = next(+ iter(DocumentService.get_documents_by_ids(document_ref.dataset, [document_ref.document_id], session)), None+ )if document is None:raise ValueError("Pipeline document not found")document.indexing_status = IndexingStatus.WAITINGdiff --git a/api/services/rag_pipeline/pipeline_template/customized/customized_retrieval.py b/api/services/rag_pipeline/pipeline_template/customized/customized_retrieval.pyindex 2aa39cd5d4e..268dd51ab68 100644--- a/api/services/rag_pipeline/pipeline_template/customized/customized_retrieval.py+++ b/api/services/rag_pipeline/pipeline_template/customized/customized_retrieval.py@@ -6,6 +6,7 @@ from sqlalchemy.orm import Sessionfrom libs.login import resolve_tenant_id_fallbackfrom models.dataset import PipelineCustomizedTemplate+from repositories.knowledge.dataset_read_repository import get_pipeline_template_creator_namefrom services.rag_pipeline.pipeline_template.pipeline_template_base import PipelineTemplateRetrievalBasefrom services.rag_pipeline.pipeline_template.pipeline_template_type import PipelineTemplateType@@ -119,5 +120,5 @@ class CustomizedPipelineTemplateRetrieval(PipelineTemplateRetrievalBase):"chunk_structure": pipeline_template.chunk_structure,"export_data": pipeline_template.yaml_content,"graph": graph_data,- "created_by": pipeline_template.created_user_name(session=session),+ "created_by": get_pipeline_template_creator_name(pipeline_template, session=session),}diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.pyindex c5f1e1935ca..35d973e24a6 100644--- a/api/services/rag_pipeline/rag_pipeline.py+++ b/api/services/rag_pipeline/rag_pipeline.py@@ -15,6 +15,7 @@ import contextsfrom configs import dify_configfrom core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFrom+from core.datasource.datasource_manager import DatasourceManagerfrom core.datasource.entities.datasource_entities import (DatasourceMessage,DatasourceProviderType,@@ -75,15 +76,18 @@ from models.workflow import (WorkflowType,)from repositories.factory import DifyAPIRepositoryFactory-from services.dataset_ref_service import DatasetRefService-from services.datasource_provider_service import DatasourceProviderService+from repositories.knowledge.dataset_read_repository import get_pipeline_dataset+from services.credentials.query import CredentialQuery+from services.data_source.provider_service import DatasourceProviderServicefrom services.entities.knowledge_entities.rag_pipeline_entities import (KnowledgeConfiguration,PipelineTemplateInfoEntity,)from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundErrorfrom services.errors.rag_pipeline import RagPipelineResourceNotFoundError+from services.knowledge.resource_scope import DatasetReffrom services.rag_pipeline.pipeline_template.pipeline_template_factory import PipelineTemplateRetrievalFactory+from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslServicefrom services.tools.builtin_tools_manage_service import BuiltinToolManageServicefrom services.workflow_draft_variable_service import DraftVariableSaver, DraftVarLoaderfrom services.workflow_node_execution_trace_service import (@@ -491,7 +495,8 @@ class RagPipelineService:graph = workflow.graph_dictnodes = graph.get("nodes", [])- from services.dataset_service import DatasetService+ # dataset_service imports RagPipelineService, so defer this reverse dependency.+ from services.knowledge.dataset_service import DatasetServicefor node in nodes:if node.get("data", {}).get("type") == "knowledge-index":@@ -499,7 +504,7 @@ class RagPipelineService:knowledge_configuration = KnowledgeConfiguration.model_validate(knowledge_configuration)# update dataset- dataset = pipeline.retrieve_dataset(session=session)+ dataset = get_pipeline_dataset(pipeline, session=session)if not dataset:raise ValueError("Dataset not found")DatasetService.update_rag_pipeline_dataset_settings(@@ -667,6 +672,8 @@ class RagPipelineService:datasource_type: str,is_published: bool,credential_id: str | None = None,+ *,+ datasource_providers: DatasourceProviderService,) -> Generator[Mapping[str, Any], None, None]:"""Run published workflow datasource@@ -718,16 +725,13 @@ class RagPipelineService:# other type directly use original valuevariables_map[key] = param_value- from core.datasource.datasource_manager import DatasourceManager-datasource_runtime = DatasourceManager.get_datasource_runtime(provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}",datasource_name=datasource_node_data.get("datasource_name"),tenant_id=pipeline.tenant_id,datasource_type=DatasourceProviderType(datasource_type),)- datasource_provider_service = DatasourceProviderService()- credentials = datasource_provider_service.get_datasource_credentials(+ credentials = datasource_providers.get_datasource_credentials(tenant_id=pipeline.tenant_id,provider=datasource_node_data.get("provider_name"),plugin_id=datasource_node_data.get("plugin_id"),@@ -835,6 +839,8 @@ class RagPipelineService:datasource_type: str,is_published: bool,credential_id: str | None = None,+ *,+ datasource_providers: DatasourceProviderService,) -> Mapping[str, Any]:"""Run published workflow datasource@@ -863,16 +869,13 @@ class RagPipelineService:if not user_inputs.get(key):user_inputs[key] = value["value"]- from core.datasource.datasource_manager import DatasourceManager-datasource_runtime = DatasourceManager.get_datasource_runtime(provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}",datasource_name=datasource_node_data.get("datasource_name"),tenant_id=pipeline.tenant_id,datasource_type=DatasourceProviderType(datasource_type),)- datasource_provider_service = DatasourceProviderService()- credentials = datasource_provider_service.get_datasource_credentials(+ credentials = datasource_providers.get_datasource_credentials(tenant_id=pipeline.tenant_id,provider=datasource_node_data.get("provider_name"),plugin_id=datasource_node_data.get("plugin_id"),@@ -1065,9 +1068,19 @@ class RagPipelineService:.limit(1))if dataset:- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, document_id.value)- document = DatasetRefService.get_document_by_ref(document_ref, session=self._session)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ document_ref = dataset_ref.document(document_id.value)+ # dataset_service imports RagPipelineService; importing here avoids the cycle.+ from services.knowledge.dataset_service import DocumentService++ document = next(+ iter(+ DocumentService.get_documents_by_ids(+ document_ref.dataset, [document_ref.document_id], self._session+ )+ ),+ None,+ )if document:document.indexing_status = IndexingStatus.ERRORdocument.error = error@@ -1319,8 +1332,6 @@ class RagPipelineService:))- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService-rag_pipeline_dsl_service = RagPipelineDslService(session)dsl = rag_pipeline_dsl_service.export_rag_pipeline_dsl(pipeline=pipeline, include_secret=True)pipeline_customized_template = PipelineCustomizedTemplate(@@ -1499,7 +1510,9 @@ class RagPipelineService:"uninstalled_recommended_plugins": uninstalled_plugin_list,}- def retry_error_document(self, dataset: Dataset, document: Document, user: Account | EndUser):+ def retry_error_document(+ self, dataset: Dataset, document: Document, user: Account | EndUser, *, generator: PipelineGenerator+ ):"""Retry error document"""@@ -1515,7 +1528,7 @@ class RagPipelineService:workflow = self.get_published_workflow(pipeline)if not workflow:raise ValueError("Workflow not found")- PipelineGenerator().generate(+ generator.generate(session=self._session,pipeline=pipeline,workflow=workflow,@@ -1534,7 +1547,15 @@ class RagPipelineService:is_retry=True,)- def get_datasource_plugins(self, tenant_id: str, dataset_id: str, is_published: bool) -> list[dict]:+ def get_datasource_plugins(+ self,+ tenant_id: str,+ dataset_id: str,+ is_published: bool,+ *,+ credential_query: CredentialQuery,+ datasource_providers: DatasourceProviderService,+ ) -> list[dict]:"""Get datasource plugins"""@@ -1601,12 +1622,11 @@ class RagPipelineService:user_input_variables.append(value)# get credentials- datasource_provider_service: DatasourceProviderService = DatasourceProviderService()- credentials: list[dict[Any, Any]] = datasource_provider_service.list_datasource_credentials(+ credentials: list[dict[Any, Any]] = datasource_providers.list_datasource_credentials(tenant_id=tenant_id,provider=datasource_node_data.get("provider_name"),plugin_id=datasource_node_data.get("plugin_id"),- session=self._session,+ credential_query=credential_query,)credential_info_list: list[Any] = []for credential in credentials:diff --git a/api/services/rag_pipeline/rag_pipeline_dsl_service.py b/api/services/rag_pipeline/rag_pipeline_dsl_service.pyindex 088cade8dfe..5090b89aca5 100644--- a/api/services/rag_pipeline/rag_pipeline_dsl_service.py+++ b/api/services/rag_pipeline/rag_pipeline_dsl_service.py@@ -42,6 +42,7 @@ from models import Accountfrom models.dataset import Dataset, DatasetCollectionBinding, Pipelinefrom models.enums import CollectionBindingType, DatasetRuntimeModefrom models.workflow import Workflow, WorkflowType+from repositories.knowledge.dataset_read_repository import get_pipeline_datasetfrom services.dsl_content import DSL_MAX_SIZE, dsl_content_sizefrom services.dsl_version import check_version_compatibilityfrom services.entities.dsl_entities import CheckDependenciesResult, ImportMode, ImportStatus, PendingImportOwner@@ -217,7 +218,7 @@ class RagPipelineDslService:status=ImportStatus.FAILED,error="Pipeline not found",)- dataset = pipeline.retrieve_dataset(session=self._session)+ dataset = get_pipeline_dataset(pipeline, session=self._session)# If major version mismatch, store import info in Redisif status == ImportStatus.PENDING:@@ -414,7 +415,7 @@ class RagPipelineDslService:data=data,account=account,)- dataset = pipeline.retrieve_dataset(session=self._session)+ dataset = get_pipeline_dataset(pipeline, session=self._session)# create datasetname = pipeline.name@@ -656,7 +657,7 @@ class RagPipelineDslService::param include_secret: Whether include secret variable:return:"""- dataset = pipeline.retrieve_dataset(session=self._session)+ dataset = get_pipeline_dataset(pipeline, session=self._session)if not dataset:raise ValueError("Missing dataset for rag pipeline")icon_info = dataset.icon_infodiff --git a/api/services/rag_pipeline/rag_pipeline_manage_service.py b/api/services/rag_pipeline/rag_pipeline_manage_service.pyindex c01140ff934..0278292abb3 100644--- a/api/services/rag_pipeline/rag_pipeline_manage_service.py+++ b/api/services/rag_pipeline/rag_pipeline_manage_service.py@@ -2,7 +2,7 @@ import loggingfrom core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntityfrom core.plugin.impl.datasource import PluginDatasourceManager-from services.datasource_provider_service import DatasourceProviderService+from services.data_source.provider_service import DatasourceProviderServicelogger = logging.getLogger(__name__)@@ -11,6 +11,8 @@ class RagPipelineManageService:@staticmethoddef list_rag_pipeline_datasources(tenant_id: str,+ *,+ datasource_providers: DatasourceProviderService,) -> list[PluginDatasourceProviderEntity]:"""list rag pipeline datasources@@ -24,9 +26,8 @@ class RagPipelineManageService:# built-in providers that declare neither credentials nor OAuth never require authorizationdatasource.is_authorized = Truecontinue- datasource_provider_service = DatasourceProviderService()try:- credentials = datasource_provider_service.get_datasource_credentials(+ credentials = datasource_providers.get_datasource_credentials(tenant_id=tenant_id,provider=datasource.provider,plugin_id=datasource.plugin_id,diff --git a/api/services/rag_pipeline/rag_pipeline_task_proxy.py b/api/services/rag_pipeline/rag_pipeline_task_proxy.pyindex ab3b7338c08..3db7820d71f 100644--- a/api/services/rag_pipeline/rag_pipeline_task_proxy.py+++ b/api/services/rag_pipeline/rag_pipeline_task_proxy.py@@ -1,8 +1,11 @@import jsonimport logging-from collections.abc import Callable, Sequence+from collections.abc import Sequencefrom functools import cached_property+from celery import signature+from celery.canvas import Signature+from configs import dify_configfrom core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntityfrom core.rag.pipeline.queue import TenantIsolatedTaskQueue@@ -10,8 +13,6 @@ from enums import CloudPlan, DeploymentEditionfrom extensions.ext_database import dbfrom services.feature_service import FeatureServicefrom services.file_service import FileService-from tasks.rag_pipeline.priority_rag_pipeline_run_task import priority_rag_pipeline_run_task-from tasks.rag_pipeline.rag_pipeline_run_task import rag_pipeline_run_tasklogger = logging.getLogger(__name__)@@ -44,14 +45,14 @@ class RagPipelineTaskProxy:)return upload_file.id- def _send_to_direct_queue(self, upload_file_id: str, task_func: Callable[[str, str], None]):+ def _send_to_direct_queue(self, upload_file_id: str, task_func: Signature):logger.info("tenant %s send file %s to direct queue", self._dataset_tenant_id, upload_file_id)- task_func.delay( # type: ignore+ task_func.delay(rag_pipeline_invoke_entities_file_id=upload_file_id,tenant_id=self._dataset_tenant_id,)- def _send_to_tenant_queue(self, upload_file_id: str, task_func: Callable[[str, str], None]):+ def _send_to_tenant_queue(self, upload_file_id: str, task_func: Signature):logger.info("tenant %s send file %s to tenant queue", self._dataset_tenant_id, upload_file_id)if self._tenant_isolated_task_queue.get_task_key():# Add to waiting queue using List operations (lpush)@@ -60,20 +61,35 @@ class RagPipelineTaskProxy:else:# Set flag and execute taskself._tenant_isolated_task_queue.set_task_waiting_time()- task_func.delay( # type: ignore+ task_func.delay(rag_pipeline_invoke_entities_file_id=upload_file_id,tenant_id=self._dataset_tenant_id,)logger.info("tenant %s init tasks: %s", self._dataset_tenant_id, upload_file_id)def _send_to_default_tenant_queue(self, upload_file_id: str):- self._send_to_tenant_queue(upload_file_id, rag_pipeline_run_task)+ self._send_to_tenant_queue(+ upload_file_id,+ signature("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task", queue="pipeline"),+ )def _send_to_priority_tenant_queue(self, upload_file_id: str):- self._send_to_tenant_queue(upload_file_id, priority_rag_pipeline_run_task)+ self._send_to_tenant_queue(+ upload_file_id,+ signature(+ "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task",+ queue="priority_pipeline",+ ),+ )def _send_to_priority_direct_queue(self, upload_file_id: str):- self._send_to_direct_queue(upload_file_id, priority_rag_pipeline_run_task)+ self._send_to_direct_queue(+ upload_file_id,+ signature(+ "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task",+ queue="priority_pipeline",+ ),+ )def _dispatch(self):upload_file_id = self._upload_invoke_entities()diff --git a/api/services/rag_pipeline/rag_pipeline_transform_service.py b/api/services/rag_pipeline/rag_pipeline_transform_service.pyindex 9304257de9a..fbf4dd526c3 100644--- a/api/services/rag_pipeline/rag_pipeline_transform_service.py+++ b/api/services/rag_pipeline/rag_pipeline_transform_service.py@@ -19,6 +19,7 @@ from factories import variable_factoryfrom models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipelinefrom models.enums import DatasetRuntimeMode, DataSourceTypefrom models.workflow import Workflow, WorkflowType+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom services.entities.knowledge_entities.rag_pipeline_entities import KnowledgeConfiguration, RetrievalSettingfrom services.errors.rag_pipeline import RagPipelineResourceNotFoundErrorfrom services.file_service import FileService@@ -56,7 +57,7 @@ class RagPipelineTransformService:if not datasource_type and not indexing_technique:return self._transform_to_empty_pipeline(dataset, account_id=account_id, session=session)- doc_form = dataset.get_doc_form(session=session)+ doc_form = get_dataset_doc_form(dataset, session=session)if not doc_form:return self._transform_to_empty_pipeline(dataset, account_id=account_id, session=session)retrieval_model = RetrievalSetting.model_validate(dataset.retrieval_model) if dataset.retrieval_model else Nonediff --git a/api/services/rbac_resource_service.py b/api/services/rbac_resource_service.pyindex fa34638f0c6..65026008267 100644--- a/api/services/rbac_resource_service.py+++ b/api/services/rbac_resource_service.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKINGfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, scoped_session-from models.dataset import Dataset+from models.dataset import Dataset, Documentfrom models.model import Appif TYPE_CHECKING:@@ -32,6 +32,15 @@ class RBACResourceService:select(Dataset.maintainer).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))+ @staticmethod+ def get_dataset_id_by_document(session: Session | scoped_session, tenant_id: str, document_id: str) -> str | None:+ dataset_id = session.scalar(+ select(Dataset.id)+ .join(Document, Document.dataset_id == Dataset.id)+ .where(Dataset.tenant_id == tenant_id, Document.tenant_id == tenant_id, Document.id == document_id)+ )+ return None if dataset_id is None else str(dataset_id)+@staticmethoddef get_dataset_id_by_pipeline(session: Session | scoped_session, tenant_id: str, pipeline_id: str) -> str | None:dataset_id = session.scalar(diff --git a/api/services/tools/builtin_tools_manage_service.py b/api/services/tools/builtin_tools_manage_service.pyindex dc82f91d3c1..dd7fd0f2152 100644--- a/api/services/tools/builtin_tools_manage_service.py+++ b/api/services/tools/builtin_tools_manage_service.py@@ -34,6 +34,7 @@ from models.account import Accountfrom models.enums import PermissionEnumfrom models.provider_ids import ToolProviderIDfrom models.tools import BuiltinToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient+from services.credentials.query import CredentialQuery, ToolCredentialRecordfrom services.tools.tools_transform_service import ToolTransformServicelogger = logging.getLogger(__name__)@@ -310,7 +311,7 @@ class BuiltinToolManageService:@staticmethoddef create_tool_encrypter(tenant_id: str,- db_provider: BuiltinToolProvider,+ db_provider: BuiltinToolProvider | ToolCredentialRecord,provider: str,provider_controller: BuiltinToolProviderController,):@@ -346,9 +347,10 @@ class BuiltinToolManageService:def get_builtin_tool_provider_credentials(tenant_id: str,provider_name: str,- session: Session,user: Account | None = None,include_credential_ids: list[str] | None = None,+ *,+ credential_query: CredentialQuery,) -> list[ToolProviderCredentialApiEntity]:"""get builtin tool provider credentials, filtered by visibility.@@ -364,84 +366,45 @@ class BuiltinToolManageService:rows are marked with ``from_other_member=True`` so the UI can render them asborrowed-from-teammate (selectable but not editable)."""- from models.credential_permission import CredentialType as CredPermType- from services.credential_permission_service import CredentialPermissionService-- with session.no_autoflush:- base_filter = (- BuiltinToolProvider.tenant_id == tenant_id,- BuiltinToolProvider.provider == provider_name,+ providers = credential_query.list_tools(+ workspace_id=tenant_id,+ provider=provider_name,+ actor_id=user.id if user else None,+ include_credential_ids=include_credential_ids or (),+ )+ if not providers:+ return []+ provider_controller = ToolManager.get_builtin_provider(providers[0].provider, tenant_id)+ credentials: list[ToolProviderCredentialApiEntity] = []+ for provider in providers:+ encrypter, _ = BuiltinToolManageService.create_tool_encrypter(+ tenant_id, provider, provider.provider, provider_controller)- order = (BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())- visible_query = select(BuiltinToolProvider).where(*base_filter).order_by(*order)- if user is not None:- visible_query = CredentialPermissionService.apply_visibility_filter(- visible_query,- model_id_column=BuiltinToolProvider.id,- model_user_id_column=BuiltinToolProvider.user_id,- model_visibility_column=BuiltinToolProvider.visibility,- credential_type=CredPermType.BUILTIN_TOOL_PROVIDER,- user=user,- )- visible_providers = list(session.scalars(visible_query).all())-- # Fetch any explicitly-included IDs that the visibility filter excluded.- borrowed_ids: set[str] = set()- borrowed_providers: list[BuiltinToolProvider] = []- if include_credential_ids:- visible_id_set = {p.id for p in visible_providers}- wanted_ids = [cid for cid in include_credential_ids if cid and cid not in visible_id_set]- if wanted_ids:- borrowed_query = (- select(BuiltinToolProvider)- .where(*base_filter, BuiltinToolProvider.id.in_(wanted_ids))- .order_by(*order)- )- borrowed_providers = list(session.scalars(borrowed_query).all())- borrowed_ids = {p.id for p in borrowed_providers}-- providers = visible_providers + borrowed_providers- if not providers:- return []-- # Only the first visible row should be flagged is_default in the response.- if visible_providers:- visible_providers[0].is_default = True- provider_controller = ToolManager.get_builtin_provider(providers[0].provider, tenant_id)-- credentials: list[ToolProviderCredentialApiEntity] = []- for provider in providers:- encrypter, _ = BuiltinToolManageService.create_tool_encrypter(- tenant_id, provider, provider.provider, provider_controller- )- decrypt_credential = encrypter.mask_plugin_credentials(encrypter.decrypt(provider.credentials))- credential_entity = ToolTransformService.convert_builtin_provider_to_credential_entity(- provider=provider,- credentials=dict(decrypt_credential),+ decrypted = encrypter.mask_plugin_credentials(encrypter.decrypt(provider.credentials))+ credentials.append(+ ToolProviderCredentialApiEntity(+ id=provider.id,+ name=provider.name,+ provider=provider.provider,+ credential_type=provider.credential_type,+ is_default=provider.is_default,+ credentials=dict(decrypted),+ visibility=provider.visibility,+ created_by=provider.created_by,+ partial_member_list=list(provider.partial_member_ids),+ from_other_member=provider.from_other_member,)- # Attach visibility, creator, and partial member list to the response entity- vis = getattr(provider, "visibility", "all_team_members")- vis_str = vis.value if hasattr(vis, "value") else str(vis)- credential_entity.visibility = vis_str- credential_entity.created_by = getattr(provider, "user_id", "") or ""- if vis_str == "partial_members":- credential_entity.partial_member_list = list(- CredentialPermissionService.get_partial_member_list(- provider.id, CredPermType.BUILTIN_TOOL_PROVIDER, session=session- )- )- if provider.id in borrowed_ids:- credential_entity.from_other_member = True- credentials.append(credential_entity)- return credentials+ )+ return credentials@staticmethoddef get_builtin_tool_provider_credential_info(tenant_id: str,provider: str,- session: Session,user: Account | None = None,include_credential_ids: list[str] | None = None,+ *,+ credential_query: CredentialQuery,) -> ToolProviderCredentialInfoApiEntity:"""get builtin tool provider credential info@@ -451,9 +414,9 @@ class BuiltinToolManageService:credentials = BuiltinToolManageService.get_builtin_tool_provider_credentials(tenant_id,provider,- session=session,user=user,include_credential_ids=include_credential_ids,+ credential_query=credential_query,)credential_info = ToolProviderCredentialInfoApiEntity(supported_credential_types=supported_credential_types,diff --git a/api/services/trigger/trigger_provider_service.py b/api/services/trigger/trigger_provider_service.pyindex 8506c523a61..58112bcc273 100644--- a/api/services/trigger/trigger_provider_service.py+++ b/api/services/trigger/trigger_provider_service.py@@ -5,10 +5,12 @@ import uuidfrom collections.abc import Mappingfrom typing import TYPE_CHECKING, Any, TypedDict+from services.credentials.query import CredentialQuery+if TYPE_CHECKING:from models.account import Account-from sqlalchemy import delete, desc, func, select+from sqlalchemy import delete, func, selectfrom sqlalchemy.orm import Session, sessionmakerfrom configs import dify_config@@ -39,7 +41,6 @@ from models.trigger import (TriggerOAuthSystemClient,TriggerOAuthTenantClient,TriggerSubscription,- WorkflowPluginTrigger,)logger = logging.getLogger(__name__)@@ -73,48 +74,28 @@ class TriggerProviderService:tenant_id: str,provider_id: TriggerProviderID,user: "Account | None" = None,+ *,+ credential_query: CredentialQuery,) -> list[TriggerProviderSubscriptionApiEntity]:"""List all trigger subscriptions for the current tenant, filtered by visibility."""- from models.credential_permission import CredentialType as CredPermType- from services.credential_permission_service import CredentialPermissionService-- subscriptions: list[TriggerProviderSubscriptionApiEntity] = []- workflows_in_use_map: dict[str, int] = {}- with Session(db.engine, expire_on_commit=False) as session:- # Get all subscriptions with visibility filtering- query = (- select(TriggerSubscription)- .where(- TriggerSubscription.tenant_id == tenant_id,- TriggerSubscription.provider_id == str(provider_id),- )- .order_by(desc(TriggerSubscription.created_at))- )- if user is not None:- query = CredentialPermissionService.apply_visibility_filter(- query,- model_id_column=TriggerSubscription.id,- model_user_id_column=TriggerSubscription.user_id,- model_visibility_column=TriggerSubscription.visibility,- credential_type=CredPermType.TRIGGER_SUBSCRIPTION,- user=user,- )- subscriptions_db = session.scalars(query).all()- subscriptions = [subscription.to_api_entity() for subscription in subscriptions_db]- if not subscriptions:- return []- usage_counts = session.execute(- select(- WorkflowPluginTrigger.subscription_id,- func.count(func.distinct(WorkflowPluginTrigger.app_id)).label("app_count"),- )- .where(- WorkflowPluginTrigger.tenant_id == tenant_id,- WorkflowPluginTrigger.subscription_id.in_([s.id for s in subscriptions]),- )- .group_by(WorkflowPluginTrigger.subscription_id)- ).all()- workflows_in_use_map = {str(row.subscription_id): int(row.app_count) for row in usage_counts}+ subscriptions = [+ TriggerProviderSubscriptionApiEntity(+ id=row.id,+ name=row.name,+ provider=row.provider,+ credential_type=row.credential_type,+ credentials=dict(row.credentials),+ endpoint=generate_plugin_trigger_endpoint_url(row.endpoint_id),+ parameters=dict(row.parameters),+ properties=dict(row.properties),+ workflows_in_use=row.workflows_in_use,+ )+ for row in credential_query.list_trigger_subscriptions(+ workspace_id=tenant_id, provider=str(provider_id), actor_id=user.id if user else None+ )+ ]+ if not subscriptions:+ return []provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)for subscription in subscriptions:@@ -135,8 +116,6 @@ class TriggerProviderService:properties_encrypter.mask_credentials(dict(properties_encrypter.decrypt(subscription.properties))))subscription.parameters = dict(subscription.parameters)- count = workflows_in_use_map.get(subscription.id)- subscription.workflows_in_use = count if count is not None else 0return subscriptionsdiff --git a/api/services/vector_service.py b/api/services/vector_service.pydeleted file mode 100644index 9d0505ed78e..00000000000--- a/api/services/vector_service.py+++ /dev/null@@ -1,379 +0,0 @@-import logging--from sqlalchemy import delete, select-from sqlalchemy.orm import Session--from core.credit_usage import CreditUsageCreatedBy-from core.model_context import with_credit_usage_created_by-from core.model_manager import ModelInstance, ModelManager-from core.rag.datasource.keyword.keyword_factory import Keyword-from core.rag.datasource.vdb.vector_factory import Vector-from core.rag.entities import ParentMode-from core.rag.index_processor.constant.doc_type import DocType-from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from core.rag.index_processor.index_processor_base import BaseIndexProcessor-from core.rag.index_processor.index_processor_factory import IndexProcessorFactory-from core.rag.models.document import AttachmentDocument, Document-from graphon.model_runtime.entities.model_entities import ModelType-from models import UploadFile-from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment, SegmentAttachmentBinding-from models.dataset import Document as DatasetDocument-from models.enums import SegmentType--logger = logging.getLogger(__name__)---class VectorService:- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def create_segments_vector(- cls,- keywords_list: list[list[str]] | None,- segments: list[DocumentSegment],- dataset: Dataset,- doc_form: str,- session: Session,- ):- """Create vector records for document segments using the caller's active DB session."""- documents: list[Document] = []- multimodal_documents: list[AttachmentDocument] = []-- for segment in segments:- if doc_form == IndexStructureType.PARENT_CHILD_INDEX:- dataset_document = session.get(DatasetDocument, segment.document_id)- if not dataset_document:- logger.warning(- "Expected DatasetDocument record to exist, but none was found, document_id=%s, segment_id=%s",- segment.document_id,- segment.id,- )- continue- # get the process rule- processing_rule = session.get(DatasetProcessRule, dataset_document.dataset_process_rule_id)- if not processing_rule:- raise ValueError("No processing rule found.")- # get embedding model instance- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # check embedding model setting- model_manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)-- if dataset.embedding_model_provider:- embedding_model_instance = model_manager.get_model_instance(- tenant_id=dataset.tenant_id,- provider=dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=dataset.embedding_model,- )- else:- embedding_model_instance = model_manager.get_default_model_instance(- tenant_id=dataset.tenant_id,- model_type=ModelType.TEXT_EMBEDDING,- )- else:- raise ValueError("The knowledge base index technique is not high quality!")- cls.generate_child_chunks(- segment,- dataset_document,- dataset,- embedding_model_instance,- processing_rule,- False,- session=session,- )- else:- rag_document = Document(- page_content=segment.content,- metadata={- "doc_id": segment.index_node_id,- "doc_hash": segment.index_node_hash,- "document_id": segment.document_id,- "dataset_id": segment.dataset_id,- "doc_type": DocType.TEXT,- },- )- documents.append(rag_document)- if dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):- multimodal_document: AttachmentDocument = AttachmentDocument(- page_content=attachment["name"],- metadata={- "doc_id": attachment["id"],- "doc_hash": "",- "document_id": segment.document_id,- "dataset_id": segment.dataset_id,- "doc_type": DocType.IMAGE,- },- )- multimodal_documents.append(multimodal_document)- index_processor: BaseIndexProcessor = IndexProcessorFactory(doc_form).init_index_processor()-- if len(documents) > 0:- index_processor.load(- dataset, documents, None, with_keywords=True, keywords_list=keywords_list, session=session- )- if len(multimodal_documents) > 0:- index_processor.load(dataset, [], multimodal_documents, with_keywords=False, session=session)-- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def update_segment_vector(- cls, keywords: list[str] | None, segment: DocumentSegment, dataset: Dataset, session: Session- ):- # update segment index task-- # format new index- document = Document(- page_content=segment.content,- metadata={- "doc_id": segment.index_node_id,- "doc_hash": segment.index_node_hash,- "document_id": segment.document_id,- "dataset_id": segment.dataset_id,- },- )- assert segment.index_node_id- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # update vector index- vector = Vector(dataset=dataset, session=session)- vector.delete_by_ids([segment.index_node_id])- vector.add_texts([document], duplicate_check=True)- else:- # update keyword index- keyword = Keyword(dataset)- keyword.delete_by_ids([segment.index_node_id], session)-- # save keyword index- if keywords and len(keywords) > 0:- keyword.add_texts([document], session, keywords_list=[keywords])- else:- keyword.add_texts([document], session)-- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def generate_child_chunks(- cls,- segment: DocumentSegment,- dataset_document: DatasetDocument,- dataset: Dataset,- embedding_model_instance: ModelInstance,- processing_rule: DatasetProcessRule,- regenerate: bool = False,- *,- session: Session,- ):- """Generate child chunks and persist them with the caller's active DB session."""- index_processor = IndexProcessorFactory(dataset.get_doc_form(session=session)).init_index_processor()- assert segment.index_node_id- if regenerate:- # delete child chunks- index_processor.clean(- dataset,- [segment.index_node_id],- with_keywords=True,- delete_child_chunks=True,- session=session,- )-- # generate child chunks- document = Document(- page_content=segment.content,- metadata={- "doc_id": segment.index_node_id,- "doc_hash": segment.index_node_hash,- "document_id": segment.document_id,- "dataset_id": segment.dataset_id,- "doc_type": DocType.TEXT,- },- )- # use full doc mode to generate segment's child chunk- processing_rule_dict = processing_rule.to_dict()- if processing_rule_dict["rules"] is not None:- processing_rule_dict["rules"]["parent_mode"] = ParentMode.FULL_DOC- documents = index_processor.transform(- [document],- embedding_model_instance=embedding_model_instance,- process_rule=processing_rule_dict,- tenant_id=dataset.tenant_id,- doc_language=dataset_document.doc_language,- session=session,- )- # save child chunks- if documents and documents[0].children:- index_processor.load(dataset, documents, session=session)-- for position, child_chunk in enumerate(documents[0].children, start=1):- child_segment = ChildChunk(- tenant_id=dataset.tenant_id,- dataset_id=dataset.id,- document_id=dataset_document.id,- segment_id=segment.id,- position=position,- index_node_id=child_chunk.metadata["doc_id"],- index_node_hash=child_chunk.metadata["doc_hash"],- content=child_chunk.page_content,- word_count=len(child_chunk.page_content),- type=SegmentType.AUTOMATIC,- created_by=dataset_document.created_by,- )- session.add(child_segment)- session.flush()-- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def create_child_chunk_vector(cls, child_segment: ChildChunk, dataset: Dataset, *, session: Session):- child_document = Document(- page_content=child_segment.content,- metadata={- "doc_id": child_segment.index_node_id,- "doc_hash": child_segment.index_node_hash,- "document_id": child_segment.document_id,- "dataset_id": child_segment.dataset_id,- },- )- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # save vector index- vector = Vector(dataset=dataset, session=session)- vector.add_texts([child_document], duplicate_check=True)-- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def update_child_chunk_vector(- cls,- new_child_chunks: list[ChildChunk],- update_child_chunks: list[ChildChunk],- delete_child_chunks: list[ChildChunk],- dataset: Dataset,- *,- session: Session,- ):- documents = []- delete_node_ids = []- for new_child_chunk in new_child_chunks:- new_child_document = Document(- page_content=new_child_chunk.content,- metadata={- "doc_id": new_child_chunk.index_node_id,- "doc_hash": new_child_chunk.index_node_hash,- "document_id": new_child_chunk.document_id,- "dataset_id": new_child_chunk.dataset_id,- },- )- documents.append(new_child_document)- for update_child_chunk in update_child_chunks:- assert update_child_chunk.index_node_id- child_document = Document(- page_content=update_child_chunk.content,- metadata={- "doc_id": update_child_chunk.index_node_id,- "doc_hash": update_child_chunk.index_node_hash,- "document_id": update_child_chunk.document_id,- "dataset_id": update_child_chunk.dataset_id,- },- )- documents.append(child_document)- delete_node_ids.append(update_child_chunk.index_node_id)- for delete_child_chunk in delete_child_chunks:- assert delete_child_chunk.index_node_id- delete_node_ids.append(delete_child_chunk.index_node_id)- if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:- # update vector index- vector = Vector(dataset=dataset, session=session)- if delete_node_ids:- vector.delete_by_ids(delete_node_ids)- if documents:- vector.add_texts(documents, duplicate_check=True)-- @classmethod- def delete_child_chunk_vector(cls, child_chunk: ChildChunk, dataset: Dataset, *, session: Session):- vector = Vector(dataset=dataset, session=session)- assert child_chunk.index_node_id- vector.delete_by_ids([child_chunk.index_node_id])-- @classmethod- @with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)- def update_multimodel_vector(- cls, segment: DocumentSegment, attachment_ids: list[str], dataset: Dataset, session: Session- ):- if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:- return-- attachments = segment.get_attachments(session=session)- old_attachment_ids = [attachment["id"] for attachment in attachments] if attachments else []-- # Check if there's any actual change needed- if set(attachment_ids) == set(old_attachment_ids):- return-- try:- vector = Vector(dataset=dataset, session=session)- if dataset.is_multimodal:- # Delete old vectors if they exist- if old_attachment_ids:- vector.delete_by_ids(old_attachment_ids)-- # Delete existing segment attachment bindings in one operation- session.execute(delete(SegmentAttachmentBinding).where(SegmentAttachmentBinding.segment_id == segment.id))-- if not attachment_ids:- session.flush()- return-- # Bulk fetch upload files - only fetch needed fields- upload_file_list = session.scalars(select(UploadFile).where(UploadFile.id.in_(attachment_ids))).all()-- if not upload_file_list:- session.flush()- return-- # Create a mapping for quick lookup- upload_file_map = {upload_file.id: upload_file for upload_file in upload_file_list}-- # Prepare batch operations- bindings = []- documents = []-- # Create common metadata base to avoid repetition- base_metadata = {- "doc_hash": "",- "document_id": segment.document_id,- "dataset_id": segment.dataset_id,- "doc_type": DocType.IMAGE,- }-- # Process attachments in the order specified by attachment_ids- for attachment_id in attachment_ids:- upload_file = upload_file_map.get(attachment_id)- if not upload_file:- logger.warning("Upload file not found for attachment_id: %s", attachment_id)- continue-- # Create segment attachment binding- bindings.append(- SegmentAttachmentBinding(- tenant_id=segment.tenant_id,- dataset_id=segment.dataset_id,- document_id=segment.document_id,- segment_id=segment.id,- attachment_id=upload_file.id,- )- )-- # Create document for vector indexing- documents.append(- Document(page_content=upload_file.name, metadata={**base_metadata, "doc_id": upload_file.id})- )-- # Bulk insert all bindings at once- if bindings:- session.add_all(bindings)-- # Add documents to vector store if any- if documents and dataset.is_multimodal:- vector.create_multimodal(documents)-- session.flush()-- except Exception:- logger.exception("Failed to update multimodal vector for segment %s", segment.id)- session.rollback()- raisediff --git a/api/tasks/add_document_to_index_task.py b/api/tasks/add_document_to_index_task.pyindex 9bd08b7daa8..76486cb51e2 100644--- a/api/tasks/add_document_to_index_task.py+++ b/api/tasks/add_document_to_index_task.py@@ -15,6 +15,12 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import DatasetAutoDisableLog, DocumentSegmentfrom models.dataset import Document as DatasetDocumentfrom models.enums import IndexingStatus, SegmentStatus+from repositories.knowledge.dataset_read_repository import (+ get_dataset_doc_form,+ get_document_dataset,+ get_segment_child_chunks,+)+from repositories.knowledge.segment_read_adapter import get_segment_attachmentslogger = logging.getLogger(__name__)@@ -44,7 +50,7 @@ def add_document_to_index_task(dataset_document_id: str):indexing_cache_key = f"document_{dataset_document.id}_indexing"try:- dataset = dataset_document.get_dataset(session=session)+ dataset = get_document_dataset(dataset_document, session=session)if not dataset:raise Exception(f"Document {dataset_document.id} dataset {dataset_document.dataset_id} doesn't exist.")@@ -70,7 +76,7 @@ def add_document_to_index_task(dataset_document_id: str):},)if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=session)+ child_chunks = get_segment_child_chunks(segment, session=session)if child_chunks:child_documents = []for child_chunk in child_chunks:@@ -86,7 +92,7 @@ def add_document_to_index_task(dataset_document_id: str):child_documents.append(child_document)document.children = child_documentsif dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):+ for attachment in get_segment_attachments(segment, session=session):multimodal_documents.append(AttachmentDocument(page_content=attachment["name"],@@ -101,7 +107,7 @@ def add_document_to_index_task(dataset_document_id: str):)documents.append(document)- index_type = dataset.get_doc_form(session=session)+ index_type = get_dataset_doc_form(dataset, session=session)index_processor = IndexProcessorFactory(index_type).init_index_processor()index_processor.load(dataset, documents, multimodal_documents=multimodal_documents, session=session)@@ -119,12 +125,12 @@ def add_document_to_index_task(dataset_document_id: str):session.commit()# Enable summary indexes for all segments in this document- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptersegment_ids_list = [segment.id for segment in segments]if segment_ids_list:try:- SummaryIndexService.enable_summaries_for_segments(+ SummaryIndexAdapter.enable_summaries_for_segments(dataset=dataset,segment_ids=segment_ids_list,)diff --git a/api/tasks/annotation/add_annotation_to_index_task.py b/api/tasks/annotation/add_annotation_to_index_task.pyindex 9e0a7d83cc1..509b9a0355c 100644--- a/api/tasks/annotation/add_annotation_to_index_task.py+++ b/api/tasks/annotation/add_annotation_to_index_task.py@@ -9,7 +9,7 @@ from core.rag.datasource.vdb.vector_factory import Vectorfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.models.document import Documentfrom models.dataset import Dataset-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/annotation/batch_import_annotations_task.py b/api/tasks/annotation/batch_import_annotations_task.pyindex 12af1a13df8..41656e1ab11 100644--- a/api/tasks/annotation/batch_import_annotations_task.py+++ b/api/tasks/annotation/batch_import_annotations_task.py@@ -13,7 +13,7 @@ from core.rag.models.document import Documentfrom extensions.ext_redis import redis_clientfrom models.dataset import Datasetfrom models.model import App, AppAnnotationSetting, MessageAnnotation-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/annotation/delete_annotation_index_task.py b/api/tasks/annotation/delete_annotation_index_task.pyindex 4e467e2e0ba..f86d06e153f 100644--- a/api/tasks/annotation/delete_annotation_index_task.py+++ b/api/tasks/annotation/delete_annotation_index_task.py@@ -8,7 +8,7 @@ from core.db.session_factory import session_factoryfrom core.rag.datasource.vdb.vector_factory import Vectorfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom models.dataset import Dataset-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/annotation/enable_annotation_reply_task.py b/api/tasks/annotation/enable_annotation_reply_task.pyindex 1247cdf32b7..bf4410f10aa 100644--- a/api/tasks/annotation/enable_annotation_reply_task.py+++ b/api/tasks/annotation/enable_annotation_reply_task.py@@ -14,7 +14,7 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Datasetfrom models.enums import CollectionBindingTypefrom models.model import App, AppAnnotationSetting, MessageAnnotation-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/annotation/update_annotation_to_index_task.py b/api/tasks/annotation/update_annotation_to_index_task.pyindex 1d4c90155e2..611934c5cb1 100644--- a/api/tasks/annotation/update_annotation_to_index_task.py+++ b/api/tasks/annotation/update_annotation_to_index_task.py@@ -9,7 +9,7 @@ from core.rag.datasource.vdb.vector_factory import Vectorfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.models.document import Documentfrom models.dataset import Dataset-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/batch_clean_document_task.py b/api/tasks/batch_clean_document_task.pyindex 040b9b4f349..e158be32df5 100644--- a/api/tasks/batch_clean_document_task.py+++ b/api/tasks/batch_clean_document_task.py@@ -8,11 +8,11 @@ from sqlalchemy import delete, selectfrom sqlalchemy.engine import CursorResultfrom core.db.session_factory import session_factory-from core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom core.tools.utils.web_reader_tool import get_image_upload_file_idsfrom extensions.ext_storage import storage-from models.dataset import Dataset, DatasetMetadataBinding, DocumentSegment, SegmentAttachmentBinding+from models.dataset import DatasetMetadataBinding, DocumentSegment, SegmentAttachmentBindingfrom models.model import UploadFile+from services.knowledge.indexing.adapters.cleanup import clean_document_indexesfrom tasks.refresh_billing_vector_space_task import schedule_billing_vector_space_refreshlogger = logging.getLogger(__name__)@@ -88,32 +88,21 @@ def batch_clean_document_task(files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()storage_keys_to_delete.extend([f.key for f in files if f and f.key])- # ============ Step 2: Clean vector index (external service, fresh session for dataset) ============- if index_node_ids:- try:- # Fetch dataset in a fresh session to avoid DetachedInstanceError- with session_factory.create_session() as session, session.begin():- dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))- if not dataset:- logger.warning("Dataset not found for vector index cleanup, dataset_id: %s", dataset_id)- else:- index_processor = IndexProcessorFactory(doc_form).init_index_processor()- index_processor.clean(- dataset,- index_node_ids,- with_keywords=True,- delete_child_chunks=True,- delete_summaries=True,- session=session,- )- dataset_tenant_id = dataset.tenant_id- except Exception:- logger.exception(- "Failed to clean vector index for dataset_id: %s, document_ids: %s, index_node_ids count: %d",- dataset_id,- document_ids,- len(index_node_ids),- )+ # A retry can still have summaries even after its segments were deleted.+ try:+ dataset_tenant_id = clean_document_indexes(+ dataset_id=dataset_id,+ document_ids=document_ids,+ doc_form=doc_form,+ new_session=session_factory.create_session,+ )+ except Exception:+ logger.exception(+ "Failed to clean vector index for dataset_id: %s, document_ids: %s, index_node_ids count: %d",+ dataset_id,+ document_ids,+ len(index_node_ids),+ )# ============ Step 3: Delete metadata binding (separate short transaction) ============try:diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.pyindex a4a5feed649..1abdffc02d8 100644--- a/api/tasks/batch_create_segment_to_index_task.py+++ b/api/tasks/batch_create_segment_to_index_task.py@@ -23,7 +23,7 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import SegmentStatusfrom models.model import UploadFile-from services.vector_service import VectorService+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -61,11 +61,15 @@ def batch_create_segment_to_index_task(with session_factory.create_session() as session:try:- dataset = session.get(Dataset, dataset_id)+ dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))if not dataset:raise ValueError("Dataset not exist.")- dataset_document = session.get(Document, document_id)+ dataset_document = session.scalar(+ select(Document).where(+ Document.id == document_id, Document.dataset_id == dataset_id, Document.tenant_id == tenant_id+ )+ )if not dataset_document:raise ValueError("Document not exist.")@@ -76,7 +80,9 @@ def batch_create_segment_to_index_task():raise ValueError("Document is not available.")- upload_file = session.get(UploadFile, upload_file_id)+ upload_file = session.scalar(+ select(UploadFile).where(UploadFile.id == upload_file_id, UploadFile.tenant_id == tenant_id)+ )if not upload_file:raise ValueError("UploadFile not found.")@@ -123,7 +129,7 @@ def batch_create_segment_to_index_task(if len(content) == 0:raise ValueError("The CSV file is empty.")- document_segments = []+ segment_ids: list[str] = []embedding_model = Noneif dataset_config["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY:model_manager = ModelManager.for_tenant(tenant_id=dataset_config["tenant_id"])@@ -143,7 +149,11 @@ def batch_create_segment_to_index_task(with session_factory.create_session() as session, session.begin():max_position = (session.scalar(- select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == document_config["id"])+ select(func.max(DocumentSegment.position)).where(+ DocumentSegment.tenant_id == tenant_id,+ DocumentSegment.dataset_id == dataset_id,+ DocumentSegment.document_id == document_id,+ ))or 0)@@ -172,21 +182,28 @@ def batch_create_segment_to_index_task(segment_document.word_count += len(segment["answer"])word_count_change += segment_document.word_countsession.add(segment_document)- document_segments.append(segment_document)+ segment_ids.append(segment_document.id)- with session_factory.create_session() as session, session.begin():- dataset_document = session.get(Document, document_id)- if dataset_document:- assert dataset_document.word_count is not None- dataset_document.word_count += word_count_change- session.add(dataset_document)-- with session_factory.create_session() as session, session.begin():- dataset = session.get(Dataset, dataset_id)- if dataset:- VectorService.create_segments_vector(- None, document_segments, dataset, document_config["doc_form"], session=session+ dataset_document = session.scalar(+ select(Document).where(+ Document.id == document_id, Document.dataset_id == dataset_id, Document.tenant_id == tenant_id)+ )+ if dataset_document is None:+ raise ValueError("Document no longer exists.")+ dataset_document.word_count = (dataset_document.word_count or 0) + word_count_change+ session.add(dataset_document)++ # The registry imports this task while assembling the batch dispatcher.+ from extensions.ext_application_services import application_services++ try:+ application_services().knowledge.segments.mutations.index_segments(+ DatasetRef(tenant_id, dataset_id).document(document_id), segment_ids=segment_ids+ )+ except Exception:+ redis_client.setex(indexing_cache_key, 600, "error")+ raiseredis_client.setex(indexing_cache_key, 600, "completed")end_at = time.perf_counter()diff --git a/api/tasks/clean_document_task.py b/api/tasks/clean_document_task.pyindex e09743a0018..42d0d108df3 100644--- a/api/tasks/clean_document_task.py+++ b/api/tasks/clean_document_task.py@@ -6,11 +6,11 @@ from celery import shared_taskfrom sqlalchemy import delete, selectfrom core.db.session_factory import session_factory-from core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom core.tools.utils.web_reader_tool import get_image_upload_file_idsfrom extensions.ext_storage import storagefrom models.dataset import Dataset, DatasetMetadataBinding, DocumentSegment, SegmentAttachmentBindingfrom models.model import UploadFile+from services.knowledge.indexing.adapters.cleanup import clean_document_indexesfrom tasks.refresh_billing_vector_space_task import schedule_billing_vector_space_refreshlogger = logging.getLogger(__name__)@@ -67,39 +67,27 @@ def clean_document_task(logger.exception("Cleaned document when document deleted failed")return- # check segment is exist- if index_node_ids:- # Wrap vector / keyword index cleanup in try/except so that a transient- # failure here (e.g. billing API hiccup propagated via FeatureService when- # ModelManager is initialized inside ``Vector(dataset)``) does not abort- # the entire task and leave document_segments / child_chunks / image_files- # / metadata bindings stranded in PG. Mirrors the pattern already used in- # ``clean_dataset_task`` so the document row's hard delete (already- # committed by the caller) does not produce orphan PG rows just because- # the vector backend or one of its transitive dependencies was unhappy.- try:- index_processor = IndexProcessorFactory(doc_form).init_index_processor()- with session_factory.create_session() as session, session.begin():- dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))- if dataset:- index_processor.clean(- dataset,- index_node_ids,- with_keywords=True,- delete_child_chunks=True,- delete_summaries=True,- session=session,- )- vector_cleanup_succeeded = True- except Exception:- logger.exception(- "Failed to clean vector / keyword index in clean_document_task, "- "document_id=%s, dataset_id=%s, index_node_ids_count=%d. "- "Continuing with PG / storage cleanup; vector orphans can be reaped later.",- document_id,- dataset_id,- len(index_node_ids),+ # A retry must also find summaries whose segments were already removed.+ # Preserve relational cleanup if the external index is temporarily unavailable.+ try:+ vector_cleanup_succeeded = (+ clean_document_indexes(+ dataset_id=dataset_id,+ document_ids=[document_id],+ doc_form=doc_form,+ new_session=session_factory.create_session,)+ is not None+ )+ except Exception:+ logger.exception(+ "Failed to clean vector / keyword index in clean_document_task, "+ "document_id=%s, dataset_id=%s, index_node_ids_count=%d. "+ "Continuing with PG / storage cleanup; vector orphans can be reaped later.",+ document_id,+ dataset_id,+ len(index_node_ids),+ )total_image_files = []with session_factory.create_session() as session, session.begin():diff --git a/api/tasks/clean_notion_document_task.py b/api/tasks/clean_notion_document_task.pyindex e7500b9a8d7..7e7d1a7718c 100644--- a/api/tasks/clean_notion_document_task.py+++ b/api/tasks/clean_notion_document_task.py@@ -6,8 +6,9 @@ from celery import shared_taskfrom sqlalchemy import delete, selectfrom core.db.session_factory import session_factory-from core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom models.dataset import Dataset, Document, DocumentSegment+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form+from services.knowledge.indexing.adapters.cleanup import clean_document_indexeslogger = logging.getLogger(__name__)@@ -30,8 +31,7 @@ def clean_notion_document_task(document_ids: list[str], dataset_id: str):if not dataset:raise Exception("Document has no dataset")- index_type = dataset.get_doc_form(session=session)- index_processor = IndexProcessorFactory(index_type).init_index_processor()+ index_type = get_dataset_doc_form(dataset, session=session)document_delete_stmt = delete(Document).where(Document.id.in_(document_ids))session.execute(document_delete_stmt)@@ -48,17 +48,12 @@ def clean_notion_document_task(document_ids: list[str], dataset_id: str):# exception escaping this task would produce orphans that no later request# can reference back. Mirrors the pattern in ``clean_dataset_task``.try:- with session_factory.create_session() as session, session.begin():- dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))- if dataset:- index_processor.clean(- dataset,- total_index_node_ids,- with_keywords=True,- delete_child_chunks=True,- delete_summaries=True,- session=session,- )+ clean_document_indexes(+ dataset_id=dataset_id,+ document_ids=document_ids,+ doc_form=index_type,+ new_session=session_factory.create_session,+ )except Exception:logger.exception("Failed to clean vector / keyword index in clean_notion_document_task, "diff --git a/api/tasks/create_segment_to_index_task.py b/api/tasks/create_segment_to_index_task.pyindex dfa1fa7fd2b..eef7d0d2d50 100644--- a/api/tasks/create_segment_to_index_task.py+++ b/api/tasks/create_segment_to_index_task.py@@ -12,6 +12,11 @@ from extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import DocumentSegmentfrom models.enums import IndexingStatus, SegmentStatus+from repositories.knowledge.dataset_read_repository import (+ get_dataset_doc_form,+ get_segment_dataset,+ get_segment_document,+)logger = logging.getLogger(__name__)@@ -56,13 +61,13 @@ def create_segment_to_index_task(segment_id: str, keywords: list[str] | None = N},)- dataset = segment.get_dataset(session=session)+ dataset = get_segment_dataset(segment, session=session)if not dataset:logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))return- dataset_document = segment.get_document(session=session)+ dataset_document = get_segment_document(segment, session=session)if not dataset_document:logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))@@ -76,7 +81,7 @@ def create_segment_to_index_task(segment_id: str, keywords: list[str] | None = Nlogger.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))return- index_type = dataset.get_doc_form(session=session)+ index_type = get_dataset_doc_form(dataset, session=session)index_processor = IndexProcessorFactory(index_type).init_index_processor()index_processor.load(dataset, [document], session=session)diff --git a/api/tasks/deal_dataset_index_update_task.py b/api/tasks/deal_dataset_index_update_task.pyindex dfbbe3de065..02eae8ab6cf 100644--- a/api/tasks/deal_dataset_index_update_task.py+++ b/api/tasks/deal_dataset_index_update_task.py@@ -12,6 +12,8 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFactofrom core.rag.models.document import AttachmentDocument, ChildDocument, Documentfrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocument+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, get_segment_child_chunks+from repositories.knowledge.segment_read_adapter import get_segment_attachments@shared_task(queue="dataset")@@ -31,7 +33,7 @@ def deal_dataset_index_update_task(dataset_id: str, action: str):if not dataset:raise Exception("Dataset not found")- index_type = dataset.get_doc_form(session=session) or IndexStructureType.PARAGRAPH_INDEX+ index_type = get_dataset_doc_form(dataset, session=session) or IndexStructureType.PARAGRAPH_INDEXindex_processor = IndexProcessorFactory(index_type).init_index_processor()if action == "upgrade":dataset_documents = session.scalars(@@ -150,7 +152,7 @@ def deal_dataset_index_update_task(dataset_id: str, action: str):},)if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=session)+ child_chunks = get_segment_child_chunks(segment, session=session)if child_chunks:child_documents = []for child_chunk in child_chunks:@@ -166,7 +168,7 @@ def deal_dataset_index_update_task(dataset_id: str, action: str):child_documents.append(child_document)document.children = child_documentsif dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):+ for attachment in get_segment_attachments(segment, session=session):multimodal_documents.append(AttachmentDocument(page_content=attachment["name"],diff --git a/api/tasks/deal_dataset_vector_index_task.py b/api/tasks/deal_dataset_vector_index_task.pyindex 928134fb7b7..4cb8ac6b84c 100644--- a/api/tasks/deal_dataset_vector_index_task.py+++ b/api/tasks/deal_dataset_vector_index_task.py@@ -12,6 +12,8 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFactofrom core.rag.models.document import AttachmentDocument, ChildDocument, Documentfrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocument+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, get_segment_child_chunks+from repositories.knowledge.segment_read_adapter import get_segment_attachmentslogger = logging.getLogger(__name__)@@ -33,7 +35,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):if not dataset:raise Exception("Dataset not found")- index_type = dataset.get_doc_form(session=session) or IndexStructureType.PARAGRAPH_INDEX+ index_type = get_dataset_doc_form(dataset, session=session) or IndexStructureType.PARAGRAPH_INDEXindex_processor = IndexProcessorFactory(index_type).init_index_processor()if action == "remove":index_processor.clean(dataset, None, with_keywords=False, session=session)@@ -146,7 +148,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):},)if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=session)+ child_chunks = get_segment_child_chunks(segment, session=session)if child_chunks:child_documents = []for child_chunk in child_chunks:@@ -162,7 +164,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):child_documents.append(child_document)document.children = child_documentsif dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):+ for attachment in get_segment_attachments(segment, session=session):multimodal_documents.append(AttachmentDocument(page_content=attachment["name"],diff --git a/api/tasks/disable_segment_from_index_task.py b/api/tasks/disable_segment_from_index_task.pyindex 6db3837e0cd..75c52b7d9f4 100644--- a/api/tasks/disable_segment_from_index_task.py+++ b/api/tasks/disable_segment_from_index_task.py@@ -10,6 +10,7 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFactofrom extensions.ext_redis import redis_clientfrom models.dataset import DocumentSegmentfrom models.enums import SegmentStatus+from repositories.knowledge.dataset_read_repository import get_segment_dataset, get_segment_documentlogger = logging.getLogger(__name__)@@ -38,13 +39,13 @@ def disable_segment_from_index_task(segment_id: str):indexing_cache_key = f"segment_{segment.id}_indexing"try:- dataset = segment.get_dataset(session=session)+ dataset = get_segment_dataset(segment, session=session)if not dataset:logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))return- dataset_document = segment.get_document(session=session)+ dataset_document = get_segment_document(segment, session=session)if not dataset_document:logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))@@ -65,10 +66,10 @@ def disable_segment_from_index_task(segment_id: str):session.commit()# Disable summary index for this segment- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptertry:- SummaryIndexService.disable_summaries_for_segments(+ SummaryIndexAdapter.disable_summaries_for_segments(dataset=dataset,segment_ids=[segment.id],disabled_by=segment.disabled_by,diff --git a/api/tasks/disable_segments_from_index_task.py b/api/tasks/disable_segments_from_index_task.pyindex da536e210e2..c6e5aa76db5 100644--- a/api/tasks/disable_segments_from_index_task.py+++ b/api/tasks/disable_segments_from_index_task.py@@ -70,13 +70,13 @@ def disable_segments_from_index_task(segment_ids: list, dataset_id: str, documensession.commit()# Disable summary indexes for these segments- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptersegment_ids_list = [segment.id for segment in segments]try:# Get disabled_by from first segment (they should all have the same disabled_by)disabled_by = segments[0].disabled_by if segments else None- SummaryIndexService.disable_summaries_for_segments(+ SummaryIndexAdapter.disable_summaries_for_segments(dataset=dataset,segment_ids=segment_ids_list,disabled_by=disabled_by,diff --git a/api/tasks/document_indexing_sync_task.py b/api/tasks/document_indexing_sync_task.pyindex 7b12c045c50..f0e46c25bdf 100644--- a/api/tasks/document_indexing_sync_task.py+++ b/api/tasks/document_indexing_sync_task.py@@ -6,14 +6,16 @@ import clickfrom celery import shared_taskfrom sqlalchemy import delete, select-from core.db.session_factory import session_factory-from core.indexing_runner import DocumentIsPausedError, IndexingRunner+from core.db.session_factory import get_session_maker, session_factoryfrom core.rag.extractor.notion_extractor import NotionExtractorfrom core.rag.index_processor.index_processor_factory import IndexProcessorFactory+from extensions.application_services.data_sources import build_data_source_credentialsfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import IndexingStatus-from services.datasource_provider_service import DatasourceProviderService+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -72,7 +74,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):index_node_ids = [segment.index_node_id for segment in segments if segment.index_node_id]# Get credentials from datasource provider- datasource_provider_service = DatasourceProviderService()+ datasource_provider_service = build_data_source_credentials(database_client=get_session_maker()).providerscredential = datasource_provider_service.get_datasource_credentials(tenant_id=tenant_id,credential_id=credential_id,@@ -97,6 +99,9 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):returnloader = NotionExtractor(+ notion_token_loader=lambda: datasource_provider_service.get_stored_notion_access_token(+ tenant_id=tenant_id, credential_id=credential_id+ ),notion_workspace_id=workspace_id,notion_obj_id=page_id,notion_page_type=page_type,@@ -112,7 +117,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):logger.info(click.style(f"Document {document_id} content changed, starting sync", fg="green"))try:- indexing_runner = IndexingRunner()+ indexing_service = build_document_indexing_service(session_factory=session_factory.get_session_maker())with session_factory.create_session() as session:document = session.scalar(select(Document).where(Document.id == document_id).limit(1))if not document:@@ -151,13 +156,13 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.document_id == document_id)session.execute(segment_delete_stmt)+ document_refs = [DatasetRef(document.tenant_id, document.dataset_id).document(document.id)]# Make the source update and segment deletion visible before extraction.session.commit()logger.info(click.style(f"Deleted segments for document {document_id}", fg="green"))- indexing_runner.run([document], session)- session.commit()+ indexing_service.run(document_refs)end_at = time.perf_counter()logger.info(click.style(f"Sync completed for document {document_id} latency: {end_at - start_at}", fg="green"))except DocumentIsPausedError as ex:diff --git a/api/tasks/document_indexing_task.py b/api/tasks/document_indexing_task.pyindex 7d41c542bba..c9af37a1fcb 100644--- a/api/tasks/document_indexing_task.py+++ b/api/tasks/document_indexing_task.py@@ -10,7 +10,6 @@ from sqlalchemy import selectfrom configs import dify_configfrom core.db.session_factory import session_factoryfrom core.entities.document_task import DocumentTask-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom enums import CloudPlan, DeploymentEdition@@ -18,6 +17,9 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, Documentfrom models.enums import IndexingStatusfrom services.feature_service import FeatureService+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReffrom tasks.generate_summary_index_task import generate_summary_index_tasklogger = logging.getLogger(__name__)@@ -107,7 +109,9 @@ def _document_indexing(dataset_id: str, document_ids: Sequence[str]):# Phase 2: Execute indexing without holding locks from the parsing-status update.has_error = Falsetry:- indexing_runner = IndexingRunner(enforce_vector_space_admission=True)+ indexing_service = build_document_indexing_service(+ session_factory=session_factory.get_session_maker(), enforce_vector_space_admission=True+ )with session_factory.create_session() as session:dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))if not dataset:@@ -120,8 +124,9 @@ def _document_indexing(dataset_id: str, document_ids: Sequence[str]):).all())- indexing_runner.run(documents, session)+ document_refs = [DatasetRef(doc.tenant_id, doc.dataset_id).document(doc.id) for doc in documents]session.commit()+ indexing_service.run(document_refs)end_at = time.perf_counter()logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))diff --git a/api/tasks/document_indexing_update_task.py b/api/tasks/document_indexing_update_task.pyindex 18e2338e97a..73929fd54d0 100644--- a/api/tasks/document_indexing_update_task.py+++ b/api/tasks/document_indexing_update_task.py@@ -6,7 +6,6 @@ from celery import shared_taskfrom sqlalchemy import delete, selectfrom core.db.session_factory import session_factory-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom extensions.ext_storage import storage@@ -14,6 +13,9 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBindingfrom models.enums import IndexingStatusfrom models.model import UploadFile+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReffrom tasks.generate_summary_index_task import generate_summary_index_tasklogger = logging.getLogger(__name__)@@ -164,9 +166,10 @@ def document_indexing_update_task(dataset_id: str, document_id: str):storage_key,)- indexing_runner = IndexingRunner()- indexing_runner.run([document], session)+ indexing_service = build_document_indexing_service(session_factory=session_factory.get_session_maker())+ document_refs = [DatasetRef(document.tenant_id, document.dataset_id).document(document.id)]session.commit()+ indexing_service.run(document_refs)end_at = time.perf_counter()logger.info(click.style(f"update document: {document.id} latency: {end_at - start_at}", fg="green"))diff --git a/api/tasks/duplicate_document_indexing_task.py b/api/tasks/duplicate_document_indexing_task.pyindex c083dbdbbad..5afa6582811 100644--- a/api/tasks/duplicate_document_indexing_task.py+++ b/api/tasks/duplicate_document_indexing_task.py@@ -9,7 +9,6 @@ from sqlalchemy import delete, selectfrom configs import dify_configfrom core.db.session_factory import session_factoryfrom core.entities.document_task import DocumentTask-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom enums import CloudPlan, DeploymentEdition@@ -17,6 +16,9 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import IndexingStatusfrom services.feature_service import FeatureService+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -158,12 +160,11 @@ def _duplicate_document_indexing_task(dataset_id: str, document_ids: Sequence[stdocument.indexing_status = IndexingStatus.PARSINGdocument.processing_started_at = naive_utc_now()session.add(document)+ indexing_service = build_document_indexing_service(session_factory=session_factory.get_session_maker())+ document_refs = [DatasetRef(doc.tenant_id, doc.dataset_id).document(doc.id) for doc in documents]# Do not keep segment deletions or parsing status changes open during extraction.session.commit()-- indexing_runner = IndexingRunner()- indexing_runner.run(list(documents), session)- session.commit()+ indexing_service.run(document_refs)end_at = time.perf_counter()logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))except DocumentIsPausedError as ex:diff --git a/api/tasks/enable_segment_to_index_task.py b/api/tasks/enable_segment_to_index_task.pyindex 782dc82dde3..f4075b9df86 100644--- a/api/tasks/enable_segment_to_index_task.py+++ b/api/tasks/enable_segment_to_index_task.py@@ -14,6 +14,12 @@ from extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import DocumentSegmentfrom models.enums import IndexingStatus, SegmentStatus+from repositories.knowledge.dataset_read_repository import (+ get_segment_child_chunks,+ get_segment_dataset,+ get_segment_document,+)+from repositories.knowledge.segment_read_adapter import get_segment_attachmentslogger = logging.getLogger(__name__)@@ -52,13 +58,13 @@ def enable_segment_to_index_task(segment_id: str):},)- dataset = segment.get_dataset(session=session)+ dataset = get_segment_dataset(segment, session=session)if not dataset:logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))return- dataset_document = segment.get_document(session=session)+ dataset_document = get_segment_document(segment, session=session)if not dataset_document:logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))@@ -74,7 +80,7 @@ def enable_segment_to_index_task(segment_id: str):index_processor = IndexProcessorFactory(dataset_document.doc_form).init_index_processor()if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=session)+ child_chunks = get_segment_child_chunks(segment, session=session)if child_chunks:child_documents = []for child_chunk in child_chunks:@@ -91,7 +97,7 @@ def enable_segment_to_index_task(segment_id: str):document.children = child_documentsmultimodel_documents = []if dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):+ for attachment in get_segment_attachments(segment, session=session):multimodel_documents.append(AttachmentDocument(page_content=attachment["name"],@@ -110,10 +116,10 @@ def enable_segment_to_index_task(segment_id: str):session.commit()# Enable summary index for this segment- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptertry:- SummaryIndexService.enable_summaries_for_segments(+ SummaryIndexAdapter.enable_summaries_for_segments(dataset=dataset,segment_ids=[segment.id],)diff --git a/api/tasks/enable_segments_to_index_task.py b/api/tasks/enable_segments_to_index_task.pyindex 0de5effcd47..6a95d564728 100644--- a/api/tasks/enable_segments_to_index_task.py+++ b/api/tasks/enable_segments_to_index_task.py@@ -15,6 +15,8 @@ from libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocumentfrom models.enums import SegmentStatus+from repositories.knowledge.dataset_read_repository import get_segment_child_chunks+from repositories.knowledge.segment_read_adapter import get_segment_attachmentslogger = logging.getLogger(__name__)@@ -73,7 +75,7 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_i)if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:- child_chunks = segment.get_child_chunks(session=session)+ child_chunks = get_segment_child_chunks(segment, session=session)if child_chunks:child_documents = []for child_chunk in child_chunks:@@ -90,7 +92,7 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_idocument.children = child_documentsif dataset.is_multimodal:- for attachment in segment.get_attachments(session=session):+ for attachment in get_segment_attachments(segment, session=session):multimodal_documents.append(AttachmentDocument(page_content=attachment["name"],@@ -110,11 +112,11 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_isession.commit()# Enable summary indexes for these segments- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptersegment_ids_list = [segment.id for segment in segments]try:- SummaryIndexService.enable_summaries_for_segments(+ SummaryIndexAdapter.enable_summaries_for_segments(dataset=dataset,segment_ids=segment_ids_list,)diff --git a/api/tasks/generate_summary_index_task.py b/api/tasks/generate_summary_index_task.pyindex 9eda5716b8c..7beadb806f2 100644--- a/api/tasks/generate_summary_index_task.py+++ b/api/tasks/generate_summary_index_task.py@@ -11,7 +11,7 @@ from core.db.session_factory import session_factoryfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocument-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterlogger = logging.getLogger(__name__)@@ -45,7 +45,13 @@ def generate_summary_index_task(dataset_id: str, document_id: str, segment_ids:logger.error(click.style(f"Dataset not found: {dataset_id}", fg="red"))return- document = session.scalar(select(DatasetDocument).where(DatasetDocument.id == document_id).limit(1))+ document = session.scalar(+ select(DatasetDocument).where(+ DatasetDocument.id == document_id,+ DatasetDocument.dataset_id == dataset.id,+ DatasetDocument.tenant_id == dataset.tenant_id,+ )+ )if not document:logger.error(click.style(f"Document not found: {document_id}", fg="red"))return@@ -85,8 +91,9 @@ def generate_summary_index_task(dataset_id: str, document_id: str, segment_ids:# Determine if only parent chunks should be processedonly_parent_chunks = dataset.chunk_structure == "parent_child_index"- # Generate summaries- summary_records = SummaryIndexService.generate_summaries_for_document(+ # Release the task's read transaction before the adapter invokes models.+ session.commit()+ summary_records = SummaryIndexAdapter.generate_summaries_for_document(dataset=dataset,document=document,summary_index_setting=summary_index_setting,diff --git a/api/tasks/install_default_plugins_task.py b/api/tasks/install_default_plugins_task.pyindex b3ab5515d85..67b2a79b016 100644--- a/api/tasks/install_default_plugins_task.py+++ b/api/tasks/install_default_plugins_task.py@@ -8,7 +8,7 @@ from configs import dify_configfrom core.helper import marketplacefrom core.plugin.entities.plugin_daemon import PluginInstallTaskStatusfrom core.plugin.plugin_service import PluginService-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServicelogger = logging.getLogger(__name__)diff --git a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.pyindex c0e447f5afe..c76edab1b4f 100644--- a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py@@ -14,15 +14,19 @@ from sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmakerfrom configs import dify_config+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntityfrom core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity+from core.db.session_factory import get_session_makerfrom core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom core.repositories.factory import DifyCoreRepositoryFactory+from extensions.application_services.data_sources import build_data_source_credentialsfrom extensions.ext_database import dbfrom models import Account, Tenantfrom models.dataset import Pipelinefrom models.enums import WorkflowRunTriggeredFromfrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.file_service import FileServicelogger = logging.getLogger(__name__)@@ -170,9 +174,11 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any],# Direct execution without creating another thread# Since we're already in a thread pool, no need for nested threading- from core.app.apps.pipeline.pipeline_generator import PipelineGenerator- pipeline_generator = PipelineGenerator()+ pipeline_generator = PipelineGenerator(+ documents=SQLAlchemyDocumentRepository(session_factory=session_factory),+ datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers,+ )# Using protected method intentionally for async executionwith Session(db.engine, expire_on_commit=False) as session:pipeline_generator._generate( # type: ignore[attr-defined]diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.pyindex 1e8ab2b79f0..9774be64e9f 100644--- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py@@ -15,15 +15,19 @@ from sqlalchemy import selectfrom sqlalchemy.orm import sessionmakerfrom configs import dify_config+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntityfrom core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity+from core.db.session_factory import get_session_makerfrom core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom core.repositories.factory import DifyCoreRepositoryFactory+from extensions.application_services.data_sources import build_data_source_credentialsfrom extensions.ext_database import dbfrom models import Account, Tenantfrom models.dataset import Pipelinefrom models.enums import WorkflowRunTriggeredFromfrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.file_service import FileServicelogger = logging.getLogger(__name__)@@ -184,9 +188,11 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any],# Direct execution without creating another thread# Since we're already in a thread pool, no need for nested threading- from core.app.apps.pipeline.pipeline_generator import PipelineGenerator- pipeline_generator = PipelineGenerator()+ pipeline_generator = PipelineGenerator(+ documents=SQLAlchemyDocumentRepository(session_factory=session_factory),+ datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers,+ )# Using protected method intentionally for async executionpipeline_generator._generate( # type: ignore[attr-defined]session=session,diff --git a/api/tasks/recover_document_indexing_task.py b/api/tasks/recover_document_indexing_task.pyindex 4606981c44b..b081e1bdf8b 100644--- a/api/tasks/recover_document_indexing_task.py+++ b/api/tasks/recover_document_indexing_task.py@@ -6,8 +6,10 @@ from celery import shared_taskfrom sqlalchemy import selectfrom core.db.session_factory import session_factory-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom models.dataset import Document+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -34,14 +36,16 @@ def recover_document_indexing_task(dataset_id: str, document_id: str):returntry:- indexing_runner = IndexingRunner()- if document.indexing_status in {"waiting", "parsing", "cleaning"}:- indexing_runner.run([document], session)- elif document.indexing_status == "splitting":- indexing_runner.run_in_splitting_status(document, session)- elif document.indexing_status == "indexing":- indexing_runner.run_in_indexing_status(document, session)+ indexing_service = build_document_indexing_service(session_factory=session_factory.get_session_maker())+ document_ref = DatasetRef(document.tenant_id, document.dataset_id).document(document.id)+ status = document.indexing_statussession.commit()+ if status in {"waiting", "parsing", "cleaning"}:+ indexing_service.run([document_ref])+ elif status == "splitting":+ indexing_service.run_in_splitting_status(document_ref)+ elif status == "indexing":+ indexing_service.run_in_indexing_status(document_ref)end_at = time.perf_counter()logger.info(click.style(f"Processed document: {document.id} latency: {end_at - start_at}", fg="green"))except DocumentIsPausedError as ex:diff --git a/api/tasks/regenerate_summary_index_task.py b/api/tasks/regenerate_summary_index_task.pyindex 69aeba389bf..2d89aca647a 100644--- a/api/tasks/regenerate_summary_index_task.py+++ b/api/tasks/regenerate_summary_index_task.py@@ -15,7 +15,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Indfrom models.dataset import Dataset, DocumentSegment, DocumentSegmentSummaryfrom models.dataset import Document as DatasetDocumentfrom models.enums import SummaryStatus-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterlogger = logging.getLogger(__name__)@@ -150,22 +150,8 @@ def regenerate_summary_index_task(for segment, summary_record in segment_summary_pairs:try:- # Delete old vector- if summary_record.summary_index_node_id:- try:- from core.rag.datasource.vdb.vector_factory import Vector-- vector = Vector(dataset, session=session)- vector.delete_by_ids([summary_record.summary_index_node_id])- except Exception as e:- logger.warning(- "Failed to delete old summary vector for segment %s: %s",- segment.id,- str(e),- )-- # Re-vectorize with new embedding model- SummaryIndexService.vectorize_summary(summary_record, segment, dataset, session=session)+ # The summary use case replaces the old vector after releasing this transaction.+ SummaryIndexAdapter.vectorize_summary(summary_record, segment, dataset, session=session)session.commit()total_segments_processed += 1@@ -261,7 +247,7 @@ def regenerate_summary_index_task(continue# Regenerate both summary content and vectors (for summary_model change)- SummaryIndexService.generate_and_vectorize_summary(+ SummaryIndexAdapter.generate_and_vectorize_summary(segment, dataset, summary_index_setting, session=session)total_segments_processed += 1diff --git a/api/tasks/remove_document_from_index_task.py b/api/tasks/remove_document_from_index_task.pyindex ebffe41db0c..de50e196e8e 100644--- a/api/tasks/remove_document_from_index_task.py+++ b/api/tasks/remove_document_from_index_task.py@@ -10,6 +10,7 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFactofrom extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Document, DocumentSegment+from repositories.knowledge.dataset_read_repository import get_document_datasetlogger = logging.getLogger(__name__)@@ -38,7 +39,7 @@ def remove_document_from_index_task(document_id: str):indexing_cache_key = f"document_{document.id}_indexing"try:- dataset = document.get_dataset(session=session)+ dataset = get_document_dataset(document, session=session)if not dataset:raise Exception("Document has no dataset")@@ -48,12 +49,12 @@ def remove_document_from_index_task(document_id: str):segments = session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all()# Disable summary indexes for all segments in this document- from services.summary_index_service import SummaryIndexService+ from services.knowledge.summaries.adapters import SummaryIndexAdaptersegment_ids_list = [segment.id for segment in segments]if segment_ids_list:try:- SummaryIndexService.disable_summaries_for_segments(+ SummaryIndexAdapter.disable_summaries_for_segments(dataset=dataset,segment_ids=segment_ids_list,disabled_by=document.disabled_by,diff --git a/api/tasks/retry_document_indexing_task.py b/api/tasks/retry_document_indexing_task.pyindex 08e0aaea5ab..744723eae44 100644--- a/api/tasks/retry_document_indexing_task.py+++ b/api/tasks/retry_document_indexing_task.py@@ -6,16 +6,20 @@ from celery import shared_taskfrom sqlalchemy import delete, selectfrom configs import dify_config-from core.db.session_factory import session_factory-from core.indexing_runner import IndexingRunner+from core.app.apps.pipeline.pipeline_generator import PipelineGenerator+from core.db.session_factory import get_session_maker, session_factoryfrom core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom enums import DeploymentEdition+from extensions.application_services.data_sources import build_data_source_credentialsfrom extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models import Account, Tenantfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import IndexingStatus+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.feature_service import FeatureService+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.resource_scope import DatasetReffrom services.rag_pipeline.rag_pipeline import RagPipelineServicelogger = logging.getLogger(__name__)@@ -109,17 +113,32 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str], user_document.indexing_status = IndexingStatus.PARSINGdocument.processing_started_at = naive_utc_now()session.add(document)- # The runner performs slow extraction/indexing in a separate transaction phase.+ document_ref = DatasetRef(document.tenant_id, document.dataset_id).document(document.id)+ runtime_mode = dataset.runtime_mode+ # Persist preparation before handing off to the selected execution service.session.commit()- if dataset.runtime_mode == "rag_pipeline":+ if runtime_mode == "rag_pipeline":with session_factory.create_session() as rag_session:rag_pipeline_service = RagPipelineService(rag_session)- rag_pipeline_service.retry_error_document(dataset, document, user)+ rag_pipeline_service.retry_error_document(+ dataset,+ document,+ user,+ generator=PipelineGenerator(+ documents=SQLAlchemyDocumentRepository(+ session_factory=session_factory.get_session_maker()+ ),+ datasource_providers=build_data_source_credentials(+ database_client=get_session_maker()+ ).providers,+ ),+ )else:- indexing_runner = IndexingRunner(enforce_vector_space_admission=True)- indexing_runner.run([document], session)- session.commit()+ indexing_service = build_document_indexing_service(+ session_factory=session_factory.get_session_maker(), enforce_vector_space_admission=True+ )+ indexing_service.run([document_ref])redis_client.delete(retry_indexing_cache_key)except Exception as ex:session.rollback()diff --git a/api/tasks/sync_website_document_indexing_task.py b/api/tasks/sync_website_document_indexing_task.pyindex cceafa9623c..30dc205b44c 100644--- a/api/tasks/sync_website_document_indexing_task.py+++ b/api/tasks/sync_website_document_indexing_task.py@@ -7,15 +7,16 @@ from sqlalchemy import delete, selectfrom configs import dify_configfrom core.db.session_factory import session_factory-from core.indexing_runner import IndexingRunnerfrom core.rag.index_processor.index_processor_factory import IndexProcessorFactoryfrom enums import DeploymentEditionfrom extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, DocumentSegmentfrom models.enums import IndexingStatus-from services.dataset_ref_service import DatasetRefService+from repositories.knowledge.document_repository import _get_documentfrom services.feature_service import FeatureService+from services.knowledge.indexing.adapters.execution import build_document_indexing_service+from services.knowledge.resource_scope import DatasetReflogger = logging.getLogger(__name__)@@ -36,9 +37,9 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):if dataset is None:raise ValueError("Dataset not found")tenant_id = dataset.tenant_id- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, document_id)- document = DatasetRefService.get_document_by_ref(document_ref, session=session)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ document_ref = dataset_ref.document(document_id)+ document = _get_document(session, document_ref)if document is None:logger.info(click.style(f"Document not found: {document_id}", fg="yellow"))return@@ -100,16 +101,14 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):document.indexing_status = IndexingStatus.PARSINGdocument.processing_started_at = naive_utc_now()session.add(document)+ indexing_service = build_document_indexing_service(session_factory=session_factory.get_session_maker())# Release document/segment locks before extraction starts.session.commit()-- indexing_runner = IndexingRunner()- indexing_runner.run([document], session)- session.commit()+ indexing_service.run([document_ref])redis_client.delete(sync_indexing_cache_key)except Exception as ex:session.rollback()- document = DatasetRefService.get_document_by_ref(document_ref, session=session)+ document = _get_document(session, document_ref)if document:document.indexing_status = IndexingStatus.ERRORdocument.error = str(ex)diff --git a/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_bearer_auth.py b/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_bearer_auth.pyindex f51bf8f13e1..1723f00bcb2 100644--- a/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_bearer_auth.py+++ b/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_bearer_auth.py@@ -9,7 +9,7 @@ from sqlalchemy.orm import Sessionfrom models.source import DataSourceApiKeyAuthBindingfrom services.auth.errors import DataSourceApiKeyAuthCredentialValidationError-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthCredentials,)@@ -82,9 +82,7 @@ def test_create_binding_successful("credentials": {"auth_type": "bearer", "config": {"api_key": "secret"}},}- with patch(- "services.auth.data_source_api_key_auth_service.DataSourceApiKeyAuthService.create_binding"- ) as create_auth:+ with patch("services.data_source.auth.api_key_service.DataSourceApiKeyAuthService.create_binding") as create_auth:response = test_client_with_containers.post("/console/api/api-key-auth/data-source/binding",json=payload,@@ -111,7 +109,7 @@ def test_create_binding_failure(account, _tenant = create_console_account_and_tenant(db_session_with_containers)with patch(- "services.auth.data_source_api_key_auth_service.DataSourceApiKeyAuthService.create_binding",+ "services.data_source.auth.api_key_service.DataSourceApiKeyAuthService.create_binding",side_effect=DataSourceApiKeyAuthCredentialValidationError("Credentials rejected"),):response = test_client_with_containers.post(diff --git a/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_oauth.py b/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_oauth.pyindex c9e800235b7..307f80d62cd 100644--- a/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_oauth.py+++ b/api/tests/test_containers_integration_tests/controllers/console/auth/test_data_source_oauth.py@@ -7,8 +7,8 @@ from sqlalchemy.orm import Sessionfrom extensions.ext_application_services import ApplicationServicesfrom models.source import DataSourceOauthBinding-from services.data_source_oauth_service import InvalidDataSourceOAuthProviderError-from services.entities.data_source_oauth_entities import DataSourceOAuthCallback+from services.data_source.entities.oauth import DataSourceOAuthCallback+from services.data_source.oauth_service import InvalidDataSourceOAuthProviderErrorfrom tests.test_containers_integration_tests.controllers.console.helpers import (authenticate_console_client,create_console_account_and_tenant,@@ -17,7 +17,7 @@ from tests.test_containers_integration_tests.controllers.console.helpers importdef _services(service: MagicMock) -> MagicMock:services = create_autospec(ApplicationServices, instance=True, spec_set=True)- services.resolve_data_source_oauth.return_value = service+ services.data_sources.resolve_oauth.return_value = servicereturn services@@ -54,7 +54,7 @@ def test_get_oauth_url_invalid_provider(service = MagicMock()services = _services(service)- services.resolve_data_source_oauth.side_effect = InvalidDataSourceOAuthProviderError+ services.data_sources.resolve_oauth.side_effect = InvalidDataSourceOAuthProviderErrorwith patch("controllers.console.auth.data_source_oauth.application_services",return_value=services,@@ -99,7 +99,7 @@ def test_oauth_callback_missing_code(test_client_with_containers: FlaskClient) -def test_oauth_callback_invalid_provider(test_client_with_containers: FlaskClient) -> None:service = MagicMock()services = _services(service)- services.resolve_data_source_oauth.side_effect = InvalidDataSourceOAuthProviderError+ services.data_sources.resolve_oauth.side_effect = InvalidDataSourceOAuthProviderErrorwith patch("controllers.console.auth.data_source_oauth.application_services",return_value=services,diff --git a/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py b/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.pyindex bc0a205ecb1..1a9389d299b 100644--- a/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py+++ b/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py@@ -1,26 +1,51 @@"""Integration coverage for Notion page bindings backed by persisted documents."""from inspect import unwrap+from types import SimpleNamespacefrom unittest.mock import MagicMock, patchfrom uuid import uuid4from flask import Flask-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom controllers.console.datasets.data_source import DataSourceNotionListApi, DataSourceNotionListQuery-from models import Account-from models.dataset import Document+from machinery.context import RequestContext+from models import Account, Tenant, TenantAccountJoin, TenantAccountRole+from models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from repositories.workspace_member_query_repository import WorkspaceMemberQueryRepository+from services.data_source.credential_gateway import ActorDatasourceCredentialResolver+from services.data_source.notion_import_adapters import PluginNotionSourceGateway+from services.data_source.notion_import_application_service import NotionImportApplicationService+from services.knowledge.dataset_access import DatasetAccessServicedef test_notion_page_is_marked_bound_from_persisted_document(flask_app_with_containers: Flask,db_session_with_containers: Session,) -> None:- tenant_id = str(uuid4())+ tenant = Tenant(name="Workspace")+ tenant_id = tenant.iddataset_id = str(uuid4())account = Account(name="Test User", email="[email protected]")account.id = str(uuid4())+ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ name="Notion dataset",+ created_by=account.id,+ data_source_type=DataSourceType.NOTION_IMPORT,+ )+ db_session_with_containers.add_all(+ [+ account,+ tenant,+ TenantAccountJoin(tenant_id=tenant_id, account_id=account.id, role=TenantAccountRole.OWNER, current=True),+ dataset,+ ]+ )document = Document(tenant_id=tenant_id,dataset_id=dataset_id,@@ -30,7 +55,7 @@ def test_notion_page_is_marked_bound_from_persisted_document(batch=f"batch-{uuid4()}",name="Notion Page",created_from=DocumentCreatedFrom.WEB,- created_by=str(uuid4()),+ created_by=account.id,indexing_status=IndexingStatus.COMPLETED,enabled=True,)@@ -62,28 +87,43 @@ def test_notion_page_is_marked_bound_from_persisted_document(datasource_provider_type=lambda: None,)+ sessions = sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False)+ datasets = SQLAlchemyDatasetRepository(session_factory=sessions)+ credentials = MagicMock(spec=ActorDatasourceCredentialResolver)+ credentials.resolve.return_value = {"token": "token"}+ notion_imports = NotionImportApplicationService(+ dataset_access=DatasetAccessService(+ datasets=datasets,+ workspace_roles=WorkspaceMemberQueryRepository(session_factory=sessions),+ legacy_permissions_enabled=True,+ ),+ datasets=datasets,+ documents=SQLAlchemyDocumentRepository(session_factory=sessions),+ source=PluginNotionSourceGateway(credentials=credentials, runtime_loader=MagicMock(return_value=runtime)),+ )+ context = RequestContext(+ request_id="notion-binding-test", trace_id=None, account_id=account.id, active_workspace_id=tenant_id+ )+with (flask_app_with_containers.test_request_context(f"/?credential_id=c1&dataset_id={dataset_id}"),patch(- "controllers.console.datasets.data_source.DatasourceProviderService.get_datasource_credentials",- return_value={"token": "token"},- ),- patch(- "controllers.console.datasets.data_source.DatasetService.get_dataset",- return_value=MagicMock(data_source_type="notion_import"),- ),- patch(- "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime",- return_value=runtime,+ "controllers.console.datasets.data_source.application_services",+ return_value=SimpleNamespace(data_sources=SimpleNamespace(notion_imports=notion_imports)),),):- response, status = unwrap(DataSourceNotionListApi().get)(+ response, status = unwrap(DataSourceNotionListApi.get)(DataSourceNotionListApi(),DataSourceNotionListQuery(credential_id="c1", dataset_id=dataset_id),- db_session_with_containers,- tenant_id,- account,+ context,)assert status == 200assert response["notion_info"][0]["pages"][0]["is_bound"] is True+ credentials.resolve.assert_called_once_with(+ workspace_id=tenant_id,+ actor_id=account.id,+ credential_id="c1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )diff --git a/api/tests/test_containers_integration_tests/models/test_dataset_models.py b/api/tests/test_containers_integration_tests/models/test_dataset_models.pyindex 52d9b42c356..1d3df7aa58a 100644--- a/api/tests/test_containers_integration_tests/models/test_dataset_models.py+++ b/api/tests/test_containers_integration_tests/models/test_dataset_models.py@@ -1,3 +1,9 @@+from repositories.knowledge.dataset_read_repository import (+ get_dataset_available_segment_count,+ get_document_hit_count,+ get_document_segment_count,+)+"""Integration tests for Dataset and Document model properties using testcontainers.@@ -13,6 +19,15 @@ from sqlalchemy.orm import Sessionfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from repositories.knowledge.dataset_read_repository import (+ get_dataset_available_document_count,+ get_dataset_document_count,+ get_dataset_word_count,+ get_next_segment,+ get_previous_segment,+ get_segment_dataset,+ get_segment_document,+)class TestDatasetDocumentProperties:@@ -49,7 +64,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add(doc)db_session_with_containers.flush()- assert dataset.get_total_documents(session=db_session_with_containers) == 3+ assert get_dataset_document_count(dataset, session=db_session_with_containers) == 3def test_dataset_available_documents_count(self, db_session_with_containers: Session) -> None:"""Test dataset can count available documents."""@@ -104,7 +119,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add_all([doc_available, doc_pending, doc_disabled])db_session_with_containers.flush()- assert dataset.get_total_available_documents(session=db_session_with_containers) == 1+ assert get_dataset_available_document_count(dataset, session=db_session_with_containers) == 1def test_dataset_word_count_aggregation(self, db_session_with_containers: Session) -> None:"""Test dataset can aggregate word count from documents."""@@ -132,7 +147,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add(doc)db_session_with_containers.flush()- assert dataset.get_word_count(session=db_session_with_containers) == 5000+ assert get_dataset_word_count(dataset, session=db_session_with_containers) == 5000def test_dataset_available_segment_count(self, db_session_with_containers: Session) -> None:"""Test Dataset.available_segment_count counts completed and enabled segments."""@@ -188,7 +203,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add(seg_waiting)db_session_with_containers.flush()- assert dataset.get_available_segment_count(session=db_session_with_containers) == 2+ assert get_dataset_available_segment_count(dataset, session=db_session_with_containers) == 2def test_document_segment_count_property(self, db_session_with_containers: Session) -> None:"""Test document can count its segments."""@@ -228,7 +243,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add(seg)db_session_with_containers.flush()- assert doc.get_segment_count(session=db_session_with_containers) == 3+ assert get_document_segment_count(doc, session=db_session_with_containers) == 3def test_document_hit_count_aggregation(self, db_session_with_containers: Session) -> None:"""Test document can aggregate hit count from segments."""@@ -269,7 +284,7 @@ class TestDatasetDocumentProperties:db_session_with_containers.add(seg)db_session_with_containers.flush()- assert doc.get_hit_count(session=db_session_with_containers) == 25+ assert get_document_hit_count(doc, session=db_session_with_containers) == 25class TestDocumentSegmentNavigationProperties:@@ -322,7 +337,7 @@ class TestDocumentSegmentNavigationProperties:db_session_with_containers.flush()# Act- related_dataset = segment.get_dataset(session=db_session_with_containers)+ related_dataset = get_segment_dataset(segment, session=db_session_with_containers)# Assertassert related_dataset is not None@@ -369,7 +384,7 @@ class TestDocumentSegmentNavigationProperties:db_session_with_containers.flush()# Act- related_document = segment.get_document(session=db_session_with_containers)+ related_document = get_segment_document(segment, session=db_session_with_containers)# Assertassert related_document is not None@@ -426,7 +441,7 @@ class TestDocumentSegmentNavigationProperties:db_session_with_containers.flush()# Act- prev_seg = segment.previous_segment(session=db_session_with_containers)+ prev_seg = get_previous_segment(segment, session=db_session_with_containers)# Assertassert prev_seg is not None@@ -483,7 +498,7 @@ class TestDocumentSegmentNavigationProperties:db_session_with_containers.flush()# Act- next_seg = segment.next_segment(session=db_session_with_containers)+ next_seg = get_next_segment(segment, session=db_session_with_containers)# Assertassert next_seg is not Nonediff --git a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.pyindex ec2c013459e..12ac010819f 100644--- a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py+++ b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py@@ -12,14 +12,14 @@ from sqlalchemy.orm import Session, sessionmakerfrom machinery.context import RequestContextfrom models.source import DataSourceApiKeyAuthBinding-from repositories.data_source_api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository-from services.auth.data_source_api_key_auth_gateways import (+from repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository+from services.auth.errors import DataSourceApiKeyAuthProviderUnavailableError+from services.data_source.auth.api_key_gateways import (ProviderApiKeyAuthCredentialValidator,TenantApiKeyAuthCredentialEncryptor,)-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthService-from services.auth.errors import DataSourceApiKeyAuthProviderUnavailableError-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthCredentials,)@@ -67,8 +67,8 @@ class TestAuthIntegration:encryptor=TenantApiKeyAuthCredentialEncryptor(),)- @patch("services.auth.firecrawl.firecrawl.httpx.post")- @patch("services.auth.data_source_api_key_auth_gateways.encrypter.encrypt_token")+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")def test_end_to_end_auth_flow(self,mock_encrypt,@@ -96,9 +96,9 @@ class TestAuthIntegration:assert len(bindings) == 1assert bindings[0].provider == _FIRECRAWL_PROVIDER- @patch("services.auth.data_source_api_key_auth_gateways.encrypter.encrypt_token")- @patch("services.auth.firecrawl.firecrawl.httpx.post")- @patch("services.auth.jina.jina._http_client.post")+ @patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.jina.jina._http_client.post")def test_multi_tenant_isolation(self,mock_jina_http,@@ -132,8 +132,8 @@ class TestAuthIntegration:assert len(result2) == 1assert result2[0].provider == _JINA_PROVIDER- @patch("services.auth.firecrawl.firecrawl.httpx.post")- @patch("services.auth.data_source_api_key_auth_gateways.encrypter.encrypt_token", return_value="encrypted_key")+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token", return_value="encrypted_key")def test_concurrent_creation_safety(self,mock_encrypt,@@ -168,7 +168,7 @@ class TestAuthIntegration:assert len(results) == 5assert len(exceptions) == 0- @patch("services.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")def test_network_failure_recovery(self,mock_http,diff --git a/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py b/api/tests/test_containers_integration_tests/services/dataset_collection_binding.pyindex d180da9555e..706382b4b9f 100644--- a/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py+++ b/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py@@ -13,7 +13,7 @@ from sqlalchemy.orm import Sessionfrom models.dataset import DatasetCollectionBindingfrom models.enums import CollectionBindingType-from services.dataset_service import DatasetCollectionBindingService+from services.knowledge.dataset_service import DatasetCollectionBindingServiceclass DatasetCollectionBindingTestDataFactory:diff --git a/api/tests/test_containers_integration_tests/services/dataset_service_update_delete.py b/api/tests/test_containers_integration_tests/services/dataset_service_update_delete.pyindex 2d6545bc6af..587258f5864 100644--- a/api/tests/test_containers_integration_tests/services/dataset_service_update_delete.py+++ b/api/tests/test_containers_integration_tests/services/dataset_service_update_delete.py@@ -17,9 +17,9 @@ from models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccofrom models.dataset import AppDatasetJoin, Dataset, DatasetPermissionEnumfrom models.enums import DataSourceTypefrom models.model import App-from services.dataset_ref_service import DatasetRefService-from services.dataset_service import DatasetServicefrom services.errors.account import NoPermissionError+from services.knowledge.dataset_service import DatasetService+from services.knowledge.resource_scope import DatasetRefclass DatasetUpdateDeleteTestDataFactory:@@ -140,7 +140,7 @@ class TestDatasetServiceDeleteDataset:dataset = DatasetUpdateDeleteTestDataFactory.create_dataset(db_session_with_containers, tenant.id, owner.id)# Act- with patch("services.dataset_service.dataset_was_deleted") as mock_dataset_was_deleted:+ with patch("services.knowledge.dataset_service.dataset_was_deleted") as mock_dataset_was_deleted:result = DatasetService.delete_dataset(dataset.id, owner, session=db_session_with_containers)# Assert@@ -228,7 +228,7 @@ class TestDatasetServiceDatasetUseCheck:dataset = DatasetUpdateDeleteTestDataFactory.create_dataset(db_session_with_containers, tenant.id, owner.id)app = DatasetUpdateDeleteTestDataFactory.create_app(db_session_with_containers, tenant.id, owner.id)DatasetUpdateDeleteTestDataFactory.create_app_dataset_join(db_session_with_containers, app.id, dataset.id)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)# Actresult = DatasetService.dataset_use_check(dataset_ref, session=db_session_with_containers)@@ -253,7 +253,7 @@ class TestDatasetServiceDatasetUseCheck:db_session_with_containers, role=TenantAccountRole.OWNER)dataset = DatasetUpdateDeleteTestDataFactory.create_dataset(db_session_with_containers, tenant.id, owner.id)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)# Actresult = DatasetService.dataset_use_check(dataset_ref, session=db_session_with_containers)@@ -290,7 +290,7 @@ class TestDatasetServiceUpdateDatasetApiStatus:current_time = datetime.datetime(2023, 1, 1, 12, 0, 0)# Act- with patch("services.dataset_service.naive_utc_now", return_value=current_time):+ with patch("services.knowledge.dataset_service.naive_utc_now", return_value=current_time):DatasetService.update_dataset_api_status(dataset, True, owner, session=db_session_with_containers)# Assert@@ -322,7 +322,7 @@ class TestDatasetServiceUpdateDatasetApiStatus:current_time = datetime.datetime(2023, 1, 1, 12, 0, 0)# Act- with patch("services.dataset_service.naive_utc_now", return_value=current_time):+ with patch("services.knowledge.dataset_service.naive_utc_now", return_value=current_time):DatasetService.update_dataset_api_status(dataset, False, owner, session=db_session_with_containers)# Assertdiff --git a/api/tests/test_containers_integration_tests/services/document_service_status.py b/api/tests/test_containers_integration_tests/services/document_service_status.pyindex 6ffaf49d947..67427ef50e7 100644--- a/api/tests/test_containers_integration_tests/services/document_service_status.py+++ b/api/tests/test_containers_integration_tests/services/document_service_status.py@@ -20,8 +20,8 @@ from models import Accountfrom models.dataset import Dataset, Documentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import UploadFile-from services.dataset_service import DocumentServicefrom services.errors.document import DocumentIndexingError+from services.knowledge.dataset_service import DocumentServiceFIXED_TIME = datetime.datetime(2023, 1, 1, 12, 0, 0)@@ -256,10 +256,10 @@ class TestDocumentServicePauseDocument:"""with (patch(- "services.dataset_service.current_user", create_autospec(Account, instance=True)+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)) as mock_current_user,- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.naive_utc_now") as mock_naive_utc_now,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.naive_utc_now") as mock_naive_utc_now,):current_time = datetime.datetime(2023, 1, 1, 12, 0, 0)user_id = str(uuid4())@@ -301,7 +301,9 @@ class TestDocumentServicePauseDocument:)# Act- DocumentService.pause_document(document, session=db_session_with_containers)+ DocumentService.pause_document(+ document, session=db_session_with_containers, actor_id=mock_document_service_dependencies["user_id"]+ )# Assertdb_session_with_containers.refresh(document)@@ -336,7 +338,9 @@ class TestDocumentServicePauseDocument:)# Act- DocumentService.pause_document(document, session=db_session_with_containers)+ DocumentService.pause_document(+ document, session=db_session_with_containers, actor_id=mock_document_service_dependencies["user_id"]+ )# Assertdb_session_with_containers.refresh(document)@@ -366,7 +370,9 @@ class TestDocumentServicePauseDocument:)# Act- DocumentService.pause_document(document, session=db_session_with_containers)+ DocumentService.pause_document(+ document, session=db_session_with_containers, actor_id=mock_document_service_dependencies["user_id"]+ )# Assertdb_session_with_containers.refresh(document)@@ -398,7 +404,9 @@ class TestDocumentServicePauseDocument:# Act & Assertwith pytest.raises(DocumentIndexingError):- DocumentService.pause_document(document, session=db_session_with_containers)+ DocumentService.pause_document(+ document, session=db_session_with_containers, actor_id=mock_document_service_dependencies["user_id"]+ )db_session_with_containers.refresh(document)assert document.is_paused is False@@ -429,7 +437,9 @@ class TestDocumentServicePauseDocument:# Act & Assertwith pytest.raises(DocumentIndexingError):- DocumentService.pause_document(document, session=db_session_with_containers)+ DocumentService.pause_document(+ document, session=db_session_with_containers, actor_id=mock_document_service_dependencies["user_id"]+ )db_session_with_containers.refresh(document)assert document.is_paused is False@@ -468,8 +478,8 @@ class TestDocumentServiceRecoverDocument:- Recovery task"""with (- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.recover_document_indexing_task") as mock_task,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.recover_document_indexing_task") as mock_task,):yield {"redis_client": mock_redis,@@ -588,10 +598,10 @@ class TestDocumentServiceRetryDocument:"""with (patch(- "services.dataset_service.current_user", create_autospec(Account, instance=True)+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)) as mock_current_user,- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.retry_document_indexing_task") as mock_task,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.retry_document_indexing_task") as mock_task,):user_id = str(uuid4())mock_current_user.id = user_id@@ -640,7 +650,12 @@ class TestDocumentServiceRetryDocument:)# Act- DocumentService.retry_document(dataset.id, [document], session=db_session_with_containers)+ DocumentService.retry_document(+ dataset.id,+ [document],+ session=db_session_with_containers,+ actor_id=mock_document_service_dependencies["user_id"],+ )# Assertdb_session_with_containers.refresh(document)@@ -689,7 +704,12 @@ class TestDocumentServiceRetryDocument:)# Act- DocumentService.retry_document(dataset.id, [document1, document2], session=db_session_with_containers)+ DocumentService.retry_document(+ dataset.id,+ [document1, document2],+ session=db_session_with_containers,+ actor_id=mock_document_service_dependencies["user_id"],+ )# Assertdb_session_with_containers.refresh(document1)@@ -732,7 +752,12 @@ class TestDocumentServiceRetryDocument:# Act & Assertwith pytest.raises(ValueError, match="Document is being retried, please try again later"):- DocumentService.retry_document(dataset.id, [document], session=db_session_with_containers)+ DocumentService.retry_document(+ dataset.id,+ [document],+ session=db_session_with_containers,+ actor_id=mock_document_service_dependencies["user_id"],+ )db_session_with_containers.refresh(document)assert document.indexing_status == IndexingStatus.ERROR@@ -763,7 +788,12 @@ class TestDocumentServiceRetryDocument:mock_document_service_dependencies["redis_client"].lock.side_effect = [first_retry_lock, second_retry_lock]with pytest.raises(ValueError, match="Document is being retried, please try again later"):- DocumentService.retry_document(dataset.id, [document1, document2], session=db_session_with_containers)+ DocumentService.retry_document(+ dataset.id,+ [document1, document2],+ session=db_session_with_containers,+ actor_id=mock_document_service_dependencies["user_id"],+ )db_session_with_containers.refresh(document1)db_session_with_containers.refresh(document2)@@ -773,13 +803,13 @@ class TestDocumentServiceRetryDocument:second_retry_lock.release.assert_not_called()mock_document_service_dependencies["retry_task"].delay.assert_not_called()- def test_retry_document_missing_current_user_error(+ def test_retry_document_missing_actor_id_error(self, db_session_with_containers: Session, mock_document_service_dependencies):"""- Test error when current_user is missing.+ Test error when the explicit actor ID is missing.- Verifies that when current_user is None or has no ID, a ValueError+ Verifies that when the actor ID is empty, a ValueErroris raised.This test ensures:@@ -797,11 +827,9 @@ class TestDocumentServiceRetryDocument:indexing_status=IndexingStatus.ERROR,)- mock_document_service_dependencies["current_user"].id = None-# Act & Assertwith pytest.raises(ValueError, match="Current user or current user id not found"):- DocumentService.retry_document(dataset.id, [document], session=db_session_with_containers)+ DocumentService.retry_document(dataset.id, [document], session=db_session_with_containers, actor_id="")db_session_with_containers.refresh(document)assert document.indexing_status == IndexingStatus.ERROR@@ -846,10 +874,10 @@ class TestDocumentServiceBatchUpdateDocumentStatus:- Async tasks"""with (- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.add_document_to_index_task") as mock_add_task,- patch("services.dataset_service.remove_document_from_index_task") as mock_remove_task,- patch("services.dataset_service.naive_utc_now") as mock_naive_utc_now,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.add_document_to_index_task") as mock_add_task,+ patch("services.knowledge.dataset_service.remove_document_from_index_task") as mock_remove_task,+ patch("services.knowledge.dataset_service.naive_utc_now") as mock_naive_utc_now,):current_time = datetime.datetime(2023, 1, 1, 12, 0, 0)mock_naive_utc_now.return_value = current_time@@ -1146,7 +1174,7 @@ class TestDocumentServiceRenameDocument:- Database session"""with patch(- "services.dataset_service.current_user", create_autospec(Account, instance=True)+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)) as mock_current_user:mock_current_user.current_tenant_id = str(uuid4())diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_permission_service.py b/api/tests/test_containers_integration_tests/services/test_dataset_permission_service.pyindex 3f7db13c41e..4285108c054 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_permission_service.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_permission_service.py@@ -19,8 +19,8 @@ from models.dataset import (DatasetPermissionEnum,)from models.enums import DataSourceType-from services.dataset_service import DatasetPermissionService, DatasetServicefrom services.errors.account import NoPermissionError+from services.knowledge.dataset_service import DatasetPermissionService, DatasetServiceclass DatasetPermissionTestDataFactory:diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service.py b/api/tests/test_containers_integration_tests/services/test_dataset_service.pyindex 912e00b0b7d..81199f29705 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service.py@@ -17,10 +17,10 @@ from graphon.model_runtime.entities.model_entities import ModelTypefrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, DatasetPermissionEnum, Document, ExternalKnowledgeBindings, Pipelinefrom models.enums import DatasetRuntimeMode, DataSourceType, DocumentCreatedFrom, IndexingStatus-from services.dataset_service import DatasetService-from services.entities.knowledge_entities.knowledge_entities import RerankingModel, RetrievalModelfrom services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, RagPipelineDatasetCreateEntityfrom services.errors.dataset import DatasetNameDuplicateError+from services.knowledge.dataset_service import DatasetService+from services.knowledge.entities.knowledge_entities import RerankingModel, RetrievalModelclass DatasetServiceIntegrationDataFactory:@@ -176,7 +176,7 @@ class TestDatasetServiceCreateDataset:embedding_model = DatasetServiceIntegrationDataFactory.create_embedding_model()# Act- with patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager:+ with patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager:mock_model_manager.return_value.get_default_model_instance.return_value = embedding_modelresult = DatasetService.create_empty_dataset(@@ -229,7 +229,9 @@ class TestDatasetServiceCreateDataset:external_knowledge_id = "knowledge-123"# Act- with patch("services.dataset_service.ExternalDatasetService.get_external_knowledge_api") as mock_get_api:+ with patch(+ "services.knowledge.dataset_service.ExternalDatasetService.get_external_knowledge_api"+ ) as mock_get_api:mock_get_api.return_value = Mock(id=external_knowledge_api_id)result = DatasetService.create_empty_dataset(tenant_id=tenant.id,@@ -269,8 +271,10 @@ class TestDatasetServiceCreateDataset:# Actwith (- patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager,- patch("services.dataset_service.DatasetService.check_reranking_model_setting") as mock_check_reranking,+ patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager,+ patch(+ "services.knowledge.dataset_service.DatasetService.check_reranking_model_setting"+ ) as mock_check_reranking,):mock_model_manager.return_value.get_default_model_instance.return_value = embedding_model@@ -303,8 +307,10 @@ class TestDatasetServiceCreateDataset:# Actwith (- patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager,- patch("services.dataset_service.DatasetService.check_embedding_model_setting") as mock_check_embedding,+ patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager,+ patch(+ "services.knowledge.dataset_service.DatasetService.check_embedding_model_setting"+ ) as mock_check_embedding,):mock_model_manager.return_value.get_model_instance.return_value = embedding_model@@ -386,7 +392,9 @@ class TestDatasetServiceCreateDataset:external_knowledge_api_id = str(uuid4())# Act / Assert- with patch("services.dataset_service.ExternalDatasetService.get_external_knowledge_api") as mock_get_api:+ with patch(+ "services.knowledge.dataset_service.ExternalDatasetService.get_external_knowledge_api"+ ) as mock_get_api:mock_get_api.return_value = Nonewith pytest.raises(ValueError, match=r"External API template not found\.?"):DatasetService.create_empty_dataset(@@ -408,7 +416,9 @@ class TestDatasetServiceCreateDataset:external_knowledge_api_id = str(uuid4())# Act / Assert- with patch("services.dataset_service.ExternalDatasetService.get_external_knowledge_api") as mock_get_api:+ with patch(+ "services.knowledge.dataset_service.ExternalDatasetService.get_external_knowledge_api"+ ) as mock_get_api:mock_get_api.return_value = Mock(id=external_knowledge_api_id)with pytest.raises(ValueError, match="external_knowledge_id is required"):DatasetService.create_empty_dataset(@@ -440,7 +450,7 @@ class TestDatasetServiceCreateRagPipelineDataset:)# Act- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):result = DatasetService.create_empty_rag_pipeline_dataset(tenant_id=tenant.id,rag_pipeline_dataset_create_entity=entity,@@ -475,8 +485,8 @@ class TestDatasetServiceCreateRagPipelineDataset:# Actwith (- patch("services.dataset_service.current_user", account),- patch("services.dataset_service.generate_incremental_name") as mock_generate_name,+ patch("services.knowledge.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.generate_incremental_name") as mock_generate_name,):mock_generate_name.return_value = generated_nameresult = DatasetService.create_empty_rag_pipeline_dataset(@@ -516,7 +526,7 @@ class TestDatasetServiceCreateRagPipelineDataset:# Act / Assertwith (- patch("services.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.current_user", account),pytest.raises(DatasetNameDuplicateError, match=f"Dataset with name {duplicate_name} already exists"),):DatasetService.create_empty_rag_pipeline_dataset(@@ -538,7 +548,7 @@ class TestDatasetServiceCreateRagPipelineDataset:)# Act- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):result = DatasetService.create_empty_rag_pipeline_dataset(tenant_id=tenant.id,rag_pipeline_dataset_create_entity=entity,@@ -567,7 +577,7 @@ class TestDatasetServiceCreateRagPipelineDataset:)# Act- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):result = DatasetService.create_empty_rag_pipeline_dataset(tenant_id=tenant.id,rag_pipeline_dataset_create_entity=entity,@@ -621,7 +631,7 @@ class TestDatasetServiceUpdateAndDeleteDataset:)# Act- with patch("services.dataset_service.dataset_was_deleted") as dataset_deleted_signal:+ with patch("services.knowledge.dataset_service.dataset_was_deleted") as dataset_deleted_signal:result = DatasetService.delete_dataset(dataset.id, account, session=db_session_with_containers)# Assert@@ -642,7 +652,7 @@ class TestDatasetServiceUpdateAndDeleteDataset:)# Act- with patch("services.dataset_service.dataset_was_deleted") as dataset_deleted_signal:+ with patch("services.knowledge.dataset_service.dataset_was_deleted") as dataset_deleted_signal:result = DatasetService.delete_dataset(dataset.id, account, session=db_session_with_containers)# Assert@@ -663,7 +673,7 @@ class TestDatasetServiceUpdateAndDeleteDataset:)# Act- with patch("services.dataset_service.dataset_was_deleted") as dataset_deleted_signal:+ with patch("services.knowledge.dataset_service.dataset_was_deleted") as dataset_deleted_signal:result = DatasetService.delete_dataset(dataset.id, account, session=db_session_with_containers)# Assert@@ -747,13 +757,13 @@ class TestDocumentServicePauseRecoverRetry:def test_pause_document_success(self, db_session_with_containers: Session):from extensions.ext_redis import redis_client- from services.dataset_service import DocumentService+ from services.knowledge.dataset_service import DocumentServicedoc, account = self._create_indexing_document(db_session_with_containers, indexing_status="indexing")- with patch("services.dataset_service.current_user") as mock_user:+ with patch("services.knowledge.dataset_service.current_user") as mock_user:mock_user.id = account.id- DocumentService.pause_document(doc, session=db_session_with_containers)+ DocumentService.pause_document(doc, session=db_session_with_containers, actor_id=account.id)db_session_with_containers.refresh(doc)assert doc.is_paused is True@@ -765,29 +775,29 @@ class TestDocumentServicePauseRecoverRetry:redis_client.delete(cache_key)def test_pause_document_invalid_status_error(self, db_session_with_containers: Session):- from services.dataset_service import DocumentServicefrom services.errors.document import DocumentIndexingError+ from services.knowledge.dataset_service import DocumentServicedoc, account = self._create_indexing_document(db_session_with_containers, indexing_status="completed")- with patch("services.dataset_service.current_user") as mock_user:+ with patch("services.knowledge.dataset_service.current_user") as mock_user:mock_user.id = account.idwith pytest.raises(DocumentIndexingError):- DocumentService.pause_document(doc, session=db_session_with_containers)+ DocumentService.pause_document(doc, session=db_session_with_containers, actor_id=account.id)def test_recover_document_success(self, db_session_with_containers: Session):from extensions.ext_redis import redis_client- from services.dataset_service import DocumentService+ from services.knowledge.dataset_service import DocumentServicedoc, account = self._create_indexing_document(db_session_with_containers, indexing_status="indexing")# Pause first- with patch("services.dataset_service.current_user") as mock_user:+ with patch("services.knowledge.dataset_service.current_user") as mock_user:mock_user.id = account.id- DocumentService.pause_document(doc, session=db_session_with_containers)+ DocumentService.pause_document(doc, session=db_session_with_containers, actor_id=account.id)# Recover- with patch("services.dataset_service.recover_document_indexing_task") as recover_task:+ with patch("services.knowledge.dataset_service.recover_document_indexing_task") as recover_task:DocumentService.recover_document(doc, session=db_session_with_containers)db_session_with_containers.refresh(doc)@@ -801,7 +811,7 @@ class TestDocumentServicePauseRecoverRetry:def test_retry_document_indexing_success(self, db_session_with_containers: Session):from extensions.ext_redis import redis_client- from services.dataset_service import DocumentService+ from services.knowledge.dataset_service import DocumentServicefactory = DatasetServiceIntegrationDataFactoryaccount, tenant = factory.create_account_with_tenant(db_session_with_containers)@@ -814,11 +824,13 @@ class TestDocumentServicePauseRecoverRetry:db_session_with_containers.commit()with (- patch("services.dataset_service.current_user") as mock_user,- patch("services.dataset_service.retry_document_indexing_task") as retry_task,+ patch("services.knowledge.dataset_service.current_user") as mock_user,+ patch("services.knowledge.dataset_service.retry_document_indexing_task") as retry_task,):mock_user.id = account.id- DocumentService.retry_document(dataset.id, [doc1, doc2], session=db_session_with_containers)+ DocumentService.retry_document(+ dataset.id, [doc1, doc2], session=db_session_with_containers, actor_id=account.id+ )db_session_with_containers.refresh(doc1)db_session_with_containers.refresh(doc2)diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_batch_update_document_status.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_batch_update_document_status.pyindex 2f1022a47ca..698a4220be5 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_batch_update_document_status.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_batch_update_document_status.py@@ -16,8 +16,8 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus-from services.dataset_service import DocumentServicefrom services.errors.document import DocumentIndexingError+from services.knowledge.dataset_service import DocumentServiceFIXED_TIME = datetime.datetime(2023, 1, 1, 12, 0, 0)@@ -140,10 +140,10 @@ class TestDatasetServiceBatchUpdateDocumentStatus:def patched_dependencies(self):"""Patch non-DB collaborators only."""with (- patch("services.dataset_service.redis_client") as redis_client,- patch("services.dataset_service.add_document_to_index_task") as add_task,- patch("services.dataset_service.remove_document_from_index_task") as remove_task,- patch("services.dataset_service.naive_utc_now") as naive_utc_now,+ patch("services.knowledge.dataset_service.redis_client") as redis_client,+ patch("services.knowledge.dataset_service.add_document_to_index_task") as add_task,+ patch("services.knowledge.dataset_service.remove_document_from_index_task") as remove_task,+ patch("services.knowledge.dataset_service.naive_utc_now") as naive_utc_now,):naive_utc_now.return_value = FIXED_TIMEredis_client.get.return_value = Nonediff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_create_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_create_dataset.pyindex 292ae46190a..623a9bea386 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_create_dataset.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_create_dataset.py@@ -9,8 +9,8 @@ import pytestfrom sqlalchemy.orm import Sessionfrom models.account import Account, Tenant, TenantAccountJoin-from services.dataset_service import DatasetServicefrom services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, RagPipelineDatasetCreateEntity+from services.knowledge.dataset_service import DatasetServiceclass TestDatasetServiceCreateRagPipelineDataset:@@ -53,7 +53,7 @@ class TestDatasetServiceCreateRagPipelineDataset:tenant, _ = self._create_tenant_and_account(db_session_with_containers)mock_user = Mock(id=None)- with patch("services.dataset_service.current_user", mock_user):+ with patch("services.knowledge.dataset_service.current_user", mock_user):with pytest.raises(ValueError, match="Current user or current user id not found"):DatasetService.create_empty_rag_pipeline_dataset(tenant_id=tenant.id,diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.pyindex b3ee73721d4..dc57992a999 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py@@ -9,7 +9,8 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Indfrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom-from services.dataset_service import DatasetService+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form+from services.knowledge.dataset_service import DatasetServiceclass DatasetDeleteIntegrationDataFactory:@@ -142,7 +143,7 @@ class TestDatasetServiceDeleteDataset:dataset.indexing_technique,dataset.index_struct,dataset.collection_binding_id,- dataset.get_doc_form(session=db_session_with_containers),+ get_dataset_doc_form(dataset, session=db_session_with_containers),dataset.pipeline_id,)diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_document.pyindex da7b0416a3d..28dc5d2f583 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py@@ -16,9 +16,10 @@ from models.account import TenantAccountRolefrom models.dataset import Dataset, Documentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import UploadFile-from services.dataset_ref_service import DatasetRefService-from services.dataset_service import DocumentService+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom services.errors.account import NoPermissionError+from services.knowledge.dataset_service import DocumentService+from services.knowledge.resource_scope import DatasetRefFIXED_UPLOAD_CREATED_AT = datetime.datetime(2024, 1, 1, 0, 0, 0)@@ -117,7 +118,9 @@ class DocumentServiceIntegrationFactory:@pytest.fixturedef current_user_mock():- with patch("services.dataset_service.current_user", create_autospec(Account, instance=True)) as current_user:+ with patch(+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)+ ) as current_user:current_user.id = str(uuid4())current_user.current_tenant_id = str(uuid4())current_user.current_role = TenantAccountRole.EDITOR@@ -144,7 +147,7 @@ def test_get_documents_by_ids_returns_empty_for_empty_input(db_session_with_contdataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers)result = DocumentService.get_documents_by_ids(- DatasetRefService.create_dataset_ref(dataset), [], session=db_session_with_containers+ DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id), [], session=db_session_with_containers)assert result == []@@ -161,7 +164,7 @@ def test_get_documents_by_ids_uses_single_batch_query(db_session_with_containers)result = DocumentService.get_documents_by_ids(- DatasetRefService.create_dataset_ref(dataset), [doc_a.id, doc_b.id], db_session_with_containers+ DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id), [doc_a.id, doc_b.id], db_session_with_containers)assert {document.id for document in result} == {doc_a.id, doc_b.id}@@ -218,7 +221,9 @@ def test_get_document_download_url_uses_signed_url_helper(db_session_with_contaidata_source_info={"upload_file_id": upload_file.id},)- with patch("services.dataset_service.file_helpers.get_signed_file_url", return_value="signed-url") as get_url:+ with patch(+ "services.knowledge.dataset_service.file_helpers.get_signed_file_url", return_value="signed-url"+ ) as get_url:result = DocumentService.get_document_download_url(document, session=db_session_with_containers)assert result == "signed-url"@@ -287,7 +292,7 @@ def test_get_upload_file_for_upload_file_document_raises_when_file_service_returdata_source_info={"upload_file_id": "missing-file"},)- with patch("services.dataset_service.FileService.get_upload_files_by_ids", return_value={}):+ with patch("services.knowledge.dataset_service.FileService.get_upload_files_by_ids", return_value={}):with pytest.raises(NotFound, match="Uploaded file not found"):DocumentService._get_upload_file_for_upload_file_document(document, session=db_session_with_containers)@@ -432,7 +437,7 @@ def test_prepare_document_batch_download_zip_translates_permission_error_to_forb)with patch(- "services.dataset_service.DatasetService.check_dataset_permission",+ "services.knowledge.dataset_service.DatasetService.check_dataset_permission",side_effect=NoPermissionError("denied"),):with pytest.raises(Forbidden, match="denied"):@@ -552,7 +557,7 @@ def test_get_error_documents_by_dataset_ref_returns_error_and_paused_documents(dindexing_status=IndexingStatus.COMPLETED,)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)result = DocumentService.get_error_documents_by_dataset_ref(dataset_ref, session=db_session_with_containers)assert {document.id for document in result} == {error_document.id, paused_document.id}@@ -574,7 +579,9 @@ def test_get_batch_documents_filters_by_current_user_tenant(db_session_with_contbatch=batch,)- with patch("services.dataset_service.current_user", create_autospec(Account, instance=True)) as current_user:+ with patch(+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)+ ) as current_user:current_user.current_tenant_id = dataset.tenant_idresult = DocumentService.get_batch_documents(dataset.id, batch, session=db_session_with_containers)@@ -608,7 +615,7 @@ def test_delete_document_emits_signal_and_commits(db_session_with_containers: Sedata_source_info={"upload_file_id": upload_file.id},)- with patch("services.dataset_service.document_was_deleted.send") as signal_send:+ with patch("services.knowledge.dataset_service.document_was_deleted.send") as signal_send:DocumentService.delete_document(document, session=db_session_with_containers)assert db_session_with_containers.get(Document, document.id) is None@@ -622,13 +629,13 @@ def test_delete_document_emits_signal_and_commits(db_session_with_containers: Sedef test_delete_documents_ignores_empty_input(db_session_with_containers: Session):dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)- with patch("services.dataset_service.batch_clean_document_task.delay") as delay:+ with patch("services.knowledge.dataset_service.batch_clean_document_task.delay") as delay:DocumentService.delete_documents(dataset_ref,[],- dataset.get_doc_form(session=db_session_with_containers),+ get_dataset_doc_form(dataset, session=db_session_with_containers),session=db_session_with_containers,)@@ -662,13 +669,13 @@ def test_delete_documents_deletes_rows_and_dispatches_cleanup_task(db_session_wiposition=2,data_source_info={"upload_file_id": upload_file_b.id},)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)- with patch("services.dataset_service.batch_clean_document_task.delay") as delay:+ with patch("services.knowledge.dataset_service.batch_clean_document_task.delay") as delay:DocumentService.delete_documents(dataset_ref,[document_a.id, document_b.id],- dataset.get_doc_form(session=db_session_with_containers),+ get_dataset_doc_form(dataset, session=db_session_with_containers),session=db_session_with_containers,)diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_get_segments.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_get_segments.pyindex 84b9bcd7c5d..c4546648314 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_get_segments.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_get_segments.py@@ -16,7 +16,7 @@ from core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, DatasetPermissionEnum, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus-from services.dataset_service import SegmentService+from services.knowledge.dataset_service import SegmentServiceclass SegmentServiceTestDataFactory:diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.pyindex e0dd574a175..9c7e2d48269 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py@@ -21,9 +21,9 @@ from models.dataset import (DatasetPermissionEnum,)from models.enums import DataSourceType-from services.dataset_ref_service import DatasetRef, DatasetRefService-from services.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetServicefrom services.errors.account import NoPermissionError+from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetService+from services.knowledge.resource_scope import DatasetReffrom tests.unit_tests.config_override import config_overrides_context@@ -196,7 +196,7 @@ class TestDatasetServicePermissionsAndLifecycle:created_by=owner.id,)- with patch("services.dataset_service.dataset_was_deleted.send") as send_deleted_signal:+ with patch("services.knowledge.dataset_service.dataset_was_deleted.send") as send_deleted_signal:result = DatasetService.delete_dataset(dataset.id, user=owner, session=db_session_with_containers)assert result is True@@ -215,7 +215,7 @@ class TestDatasetServicePermissionsAndLifecycle:dataset_id=dataset.id,)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)assert DatasetService.dataset_use_check(dataset_ref, session=db_session_with_containers) is True@@ -227,7 +227,7 @@ class TestDatasetServicePermissionsAndLifecycle:created_by=owner.id,)- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)assert DatasetService.dataset_use_check(dataset_ref, session=db_session_with_containers) is False@@ -400,7 +400,7 @@ class TestDatasetServicePermissionsAndLifecycle:)now = datetime(2026, 4, 14, 18, 0, 0)- with patch("services.dataset_service.naive_utc_now", return_value=now):+ with patch("services.knowledge.dataset_service.naive_utc_now", return_value=now):DatasetService.update_dataset_api_status(dataset, True, owner, session=db_session_with_containers)db_session_with_containers.refresh(dataset)@@ -414,7 +414,7 @@ class TestDatasetServicePermissionsAndLifecycle:features = SimpleNamespace(billing=SimpleNamespace(subscription=SimpleNamespace(plan="professional")))dataset_ref = DatasetRef(tenant_id=tenant.id, dataset_id=str(uuid4()))- with patch("services.dataset_service.FeatureService.get_features", return_value=features):+ with patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=features):result = DatasetService.get_dataset_auto_disable_logs(dataset_ref, session=db_session_with_containers)assert result == {"document_ids": [], "count": 0}@@ -440,9 +440,9 @@ class TestDatasetServicePermissionsAndLifecycle:document_id=str(uuid4()),)features = SimpleNamespace(billing=SimpleNamespace(subscription=SimpleNamespace(plan="professional")))- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)- with patch("services.dataset_service.FeatureService.get_features", return_value=features):+ with patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=features):result = DatasetService.get_dataset_auto_disable_logs(dataset_ref, session=db_session_with_containers)assert result["count"] == 2diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_retrieval.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_retrieval.pyindex 5643bfbeba7..2da05037f2a 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_retrieval.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_retrieval.py@@ -29,7 +29,7 @@ from models.dataset import ()from models.enums import DatasetQuerySource, DataSourceType, ProcessRuleMode, TagTypefrom models.model import Tag, TagBinding-from services.dataset_service import DatasetService, DocumentService+from services.knowledge.dataset_service import DatasetService, DocumentServiceclass DatasetRetrievalTestDataFactory:diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.pyindex 8f4099fa484..cfa8b74d466 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py@@ -17,8 +17,8 @@ from models.account import ()from models.dataset import Dataset, ExternalKnowledgeApis, ExternalKnowledgeBindingsfrom models.enums import DataSourceType-from services.dataset_service import DatasetServicefrom services.errors.account import NoPermissionError+from services.knowledge.dataset_service import DatasetServiceclass DatasetUpdateTestDataFactory:@@ -370,7 +370,7 @@ class TestDatasetServiceUpdateDataset:"retrieval_model": "new_model",}- with patch("services.dataset_service.deal_dataset_vector_index_task") as mock_task:+ with patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task:result = DatasetService.update_dataset(dataset.id, update_data, user, session=db_session_with_containers)mock_task.delay.assert_called_once_with(dataset.id, "remove")@@ -408,12 +408,12 @@ class TestDatasetServiceUpdateDataset:}with (- patch("services.dataset_service.current_user", user),- patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager,+ patch("services.knowledge.dataset_service.current_user", user),+ patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager,patch(- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding"+ "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding") as mock_get_binding,- patch("services.dataset_service.deal_dataset_vector_index_task") as mock_task,+ patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task,):mock_model_manager.return_value.get_model_instance.return_value = embedding_modelmock_get_binding.return_value = binding@@ -503,13 +503,13 @@ class TestDatasetServiceUpdateDataset:}with (- patch("services.dataset_service.current_user", user),- patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager,+ patch("services.knowledge.dataset_service.current_user", user),+ patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager,patch(- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding"+ "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding") as mock_get_binding,- patch("services.dataset_service.deal_dataset_vector_index_task") as mock_task,- patch("services.dataset_service.regenerate_summary_index_task") as mock_regenerate_task,+ patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task,+ patch("services.knowledge.dataset_service.regenerate_summary_index_task") as mock_regenerate_task,):mock_model_manager.return_value.get_model_instance.return_value = embedding_modelmock_get_binding.return_value = binding@@ -589,8 +589,8 @@ class TestDatasetServiceUpdateDataset:}with (- patch("services.dataset_service.current_user", user),- patch("services.dataset_service.ModelManager.for_tenant") as mock_model_manager,+ patch("services.knowledge.dataset_service.current_user", user),+ patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager,):mock_model_manager.return_value.get_model_instance.side_effect = Exception("No Embedding Model available")diff --git a/api/tests/test_containers_integration_tests/services/test_document_service_display_status.py b/api/tests/test_containers_integration_tests/services/test_document_service_display_status.pyindex 383a5f6374f..46b32dbaca9 100644--- a/api/tests/test_containers_integration_tests/services/test_document_service_display_status.py+++ b/api/tests/test_containers_integration_tests/services/test_document_service_display_status.py@@ -7,7 +7,7 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus-from services.dataset_service import DocumentService+from services.knowledge.dataset_service import DocumentServicedef _create_dataset(db_session_with_containers) -> Dataset:diff --git a/api/tests/test_containers_integration_tests/services/test_document_service_rename_document.py b/api/tests/test_containers_integration_tests/services/test_document_service_rename_document.pyindex 6a0ea17560a..773452527d0 100644--- a/api/tests/test_containers_integration_tests/services/test_document_service_rename_document.py+++ b/api/tests/test_containers_integration_tests/services/test_document_service_rename_document.py@@ -13,7 +13,7 @@ from models import Accountfrom models.dataset import Dataset, Documentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFromfrom models.model import UploadFile-from services.dataset_service import DocumentService+from services.knowledge.dataset_service import DocumentServiceFIXED_UPLOAD_CREATED_AT = datetime.datetime(2024, 1, 1, 0, 0, 0)@@ -21,7 +21,9 @@ FIXED_UPLOAD_CREATED_AT = datetime.datetime(2024, 1, 1, 0, 0, 0)@pytest.fixturedef mock_env():"""Patch only non-SQL dependency used by rename_document: current_user context."""- with patch("services.dataset_service.current_user", create_autospec(Account, instance=True)) as current_user:+ with patch(+ "services.knowledge.dataset_service.current_user", create_autospec(Account, instance=True)+ ) as current_user:current_user.current_tenant_id = str(uuid4())current_user.id = str(uuid4())yield {"current_user": current_user}diff --git a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.pyindex b38bc6f4862..016c85e9dad 100644--- a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py+++ b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom concurrent.futures import ThreadPoolExecutorfrom threading import Barrier-from unittest.mock import patch+from unittest.mock import create_autospecfrom uuid import uuid4import pytest@@ -13,12 +13,24 @@ from sqlalchemy.orm import Session, sessionmakerfrom models import Account, Tenantfrom models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Documentfrom models.enums import DatasetMetadataType, DataSourceType, DocumentCreatedFrom-from services.entities.knowledge_entities.knowledge_entities import (+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.knowledge_entities import (DocumentMetadataOperation,MetadataDetail,MetadataOperationData,)-from services.metadata_service import MetadataService+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetRef+++def _metadata_service(session: Session) -> MetadataService:+ return MetadataService(+ store=SQLAlchemyMetadataRepository(+ session_factory=sessionmaker(bind=session.get_bind(), expire_on_commit=False)+ ),+ dataset_access=create_autospec(DatasetAccess, instance=True),+ )def _create_dataset(db_session: Session, *, tenant_id: str, built_in_field_enabled: bool = False) -> Dataset:@@ -112,8 +124,8 @@ class TestMetadataPartialUpdate:)metadata_args = MetadataOperationData(operation_data=[operation])- MetadataService.update_documents_metadata(- dataset, metadata_args, current_account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata_args, actor_id=current_account.id)db_session_with_containers.expire_all()@@ -171,7 +183,7 @@ class TestMetadataPartialUpdate:_context: object,_executemany: bool,) -> None:- if "FROM documents" in statement and "FOR UPDATE" in statement:+ if "FROM datasets" in statement and "FOR UPDATE" in statement:locked_selects.append(None)locked_select_barrier.wait(timeout=10)@@ -179,15 +191,13 @@ class TestMetadataPartialUpdate:with session_factory() as session:owned_dataset = session.get(Dataset, dataset_id)assert owned_dataset is not None- MetadataService.update_documents_metadata(- owned_dataset, metadata_args, current_account, session=session+ _metadata_service(session).update_documents_metadata(+ DatasetRef(owned_dataset.tenant_id, owned_dataset.id), metadata_args, actor_id=current_account.id)event.listen(engine, "before_cursor_execute", synchronize_locked_select)try:with (- patch.object(MetadataService, "knowledge_base_metadata_lock_check"),- patch("services.metadata_service.redis_client.delete"),ThreadPoolExecutor(max_workers=2) as executor,):futures = [executor.submit(update, operation) for operation in operations]@@ -227,8 +237,8 @@ class TestMetadataPartialUpdate:)metadata_args = MetadataOperationData(operation_data=[operation])- MetadataService.update_documents_metadata(- dataset, metadata_args, current_account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata_args, actor_id=current_account.id)db_session_with_containers.expire_all()@@ -273,8 +283,8 @@ class TestMetadataPartialUpdate:)metadata_args = MetadataOperationData(operation_data=[operation])- MetadataService.update_documents_metadata(- dataset, metadata_args, current_account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata_args, actor_id=current_account.id)db_session_with_containers.expire_all()@@ -311,8 +321,20 @@ class TestMetadataPartialUpdate:)metadata_args = MetadataOperationData(operation_data=[operation])- with patch.object(db_session_with_containers, "commit", side_effect=RuntimeError("database connection lost")):+ service = _metadata_service(db_session_with_containers)++ def fail_commit(_session: Session) -> None:+ raise RuntimeError("database connection lost")++ event.listen(Session, "before_commit", fail_commit)+ try:with pytest.raises(RuntimeError, match="database connection lost"):- MetadataService.update_documents_metadata(- dataset, metadata_args, current_account, session=db_session_with_containers+ service.update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata_args, actor_id=current_account.id)+ finally:+ event.remove(Session, "before_commit", fail_commit)+ db_session_with_containers.expire_all()+ unchanged = db_session_with_containers.get(Document, document.id)+ assert unchanged is not None+ assert unchanged.doc_metadata == {"existing_key": "existing_value"}diff --git a/api/tests/test_containers_integration_tests/services/test_metadata_service.py b/api/tests/test_containers_integration_tests/services/test_metadata_service.pyindex 6f7bf625563..6caa5fd757d 100644--- a/api/tests/test_containers_integration_tests/services/test_metadata_service.py+++ b/api/tests/test_containers_integration_tests/services/test_metadata_service.py@@ -1,115 +1,68 @@-from collections.abc import Generator-from typing import TypedDict-from unittest.mock import Mock, patch+from unittest.mock import create_autospecimport pytestfrom faker import Faker-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus-from services.entities.knowledge_entities.knowledge_entities import (+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.knowledge_entities import (DocumentMetadataOperation,MetadataArgs,MetadataDetail,MetadataOperationData,)-from services.errors.metadata import MetadataResourceNotFoundError-from services.metadata_service import MetadataService+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetRef-class MetadataServiceDeps(TypedDict):- redis_client: Mock- document_service: Mock+def _metadata_service(session: Session) -> MetadataService:+ return MetadataService(+ store=SQLAlchemyMetadataRepository(+ session_factory=sessionmaker(bind=session.get_bind(), expire_on_commit=False)+ ),+ dataset_access=create_autospec(DatasetAccess, instance=True),+ )class TestMetadataService:"""Integration tests for MetadataService using testcontainers."""- @pytest.fixture- def mock_external_service_dependencies(self) -> Generator[MetadataServiceDeps, None, None]:- """Mock setup for external service dependencies."""- with (- patch("services.metadata_service.redis_client") as mock_redis_client,- patch("services.dataset_service.DocumentService") as mock_document_service,- ):- # Setup default mock returns- mock_redis_client.get.return_value = None- mock_redis_client.set.return_value = True- mock_redis_client.delete.return_value = 1-- yield {- "redis_client": mock_redis_client,- "document_service": mock_document_service,- }-- def _create_test_account_and_tenant(- self,- db_session_with_containers: Session,- mock_external_service_dependencies: MetadataServiceDeps,- ) -> tuple[Account, Tenant]:+ def _create_test_account_and_tenant(self, db_session_with_containers: Session) -> tuple[Account, Tenant]:"""Helper method to create a test account and tenant for testing.Args:db_session_with_containers: Database session from testcontainers infrastructure- mock_external_service_dependencies: Mock dependenciesReturns:tuple: (account, tenant) - Created account and tenant instances"""fake = Faker()-- # Create account- account = Account(- email=fake.email(),- name=fake.name(),- interface_language="en-US",- status=AccountStatus.ACTIVE,- )-+ account = Account(email=fake.email(), name=fake.name(), interface_language="en-US", status=AccountStatus.ACTIVE)db_session_with_containers.add(account)db_session_with_containers.commit()-- # Create tenant for the account- tenant = Tenant(- name=fake.company(),- status=TenantStatus.NORMAL,- )+ tenant = Tenant(name=fake.company(), status=TenantStatus.NORMAL)db_session_with_containers.add(tenant)db_session_with_containers.commit()-- # Create tenant-account join- join = TenantAccountJoin(- tenant_id=tenant.id,- account_id=account.id,- role=TenantAccountRole.OWNER,- current=True,- )+ join = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER, current=True)db_session_with_containers.add(join)db_session_with_containers.commit()-- # Set current tenant for accountaccount.set_current_tenant_with_session(tenant, session=db_session_with_containers)-return account, tenant- def _create_test_dataset(- self,- db_session_with_containers: Session,- mock_external_service_dependencies: MetadataServiceDeps,- account: Account,- tenant: Tenant,- ) -> Dataset:+ def _create_test_dataset(self, db_session_with_containers: Session, account: Account, tenant: Tenant) -> Dataset:"""Helper method to create a test dataset for testing.Args:db_session_with_containers: Database session from testcontainers infrastructure- mock_external_service_dependencies: Mock dependenciesaccount: Account instancetenant: Tenant instance@@ -117,7 +70,6 @@ class TestMetadataService:Dataset: Created dataset instance"""fake = Faker()-dataset = Dataset(tenant_id=tenant.id,name=fake.company(),@@ -126,25 +78,18 @@ class TestMetadataService:created_by=account.id,built_in_field_enabled=False,)-db_session_with_containers.add(dataset)db_session_with_containers.commit()-return datasetdef _create_test_document(- self,- db_session_with_containers: Session,- mock_external_service_dependencies: MetadataServiceDeps,- dataset: Dataset,- account: Account,+ self, db_session_with_containers: Session, dataset: Dataset, account: Account) -> Document:"""Helper method to create a test document for testing.Args:db_session_with_containers: Database session from testcontainers infrastructure- mock_external_service_dependencies: Mock dependenciesdataset: Dataset instanceaccount: Account instance@@ -152,7 +97,6 @@ class TestMetadataService:Document: Created document instance"""fake = Faker()-document = Document(tenant_id=dataset.tenant_id,dataset_id=dataset.id,@@ -166,357 +110,221 @@ class TestMetadataService:doc_form=IndexStructureType.PARAGRAPH_INDEX,doc_language="en",)-db_session_with_containers.add(document)db_session_with_containers.commit()-return document- def test_create_metadata_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_create_metadata_success(self, db_session_with_containers: Session) -> None:"""Test successful metadata creation with valid parameters."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="test_metadata")-- # Act: Execute the method under test- result = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ result = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Assert: Verify the expected outcomes+ db_session_with_containers.expire_all()assert result is not Noneassert result.name == "test_metadata"assert result.type == "string"assert result.dataset_id == dataset.idassert result.tenant_id == tenant.idassert result.created_by == account.id-- # Verify database state-- db_session_with_containers.refresh(result)+ db_session_with_containers.expire_all()assert result.id is not Noneassert result.created_at is not None- def test_create_metadata_name_too_long(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_create_metadata_name_too_long(self, db_session_with_containers: Session) -> None:"""Test metadata creation fails when name exceeds 255 characters."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- long_name = "a" * 256 # 256 characters, exceeding 255 limit+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ long_name = "a" * 256metadata_args = MetadataArgs(type="string", name=long_name)-- # Act & Assert: Verify proper error handlingwith pytest.raises(ValueError, match="Metadata name cannot exceed 255 characters."):- MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_create_metadata_name_already_exists(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_create_metadata_name_already_exists(self, db_session_with_containers: Session) -> None:"""Test metadata creation fails when name already exists in the same dataset."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create first metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)first_metadata_args = MetadataArgs(type="string", name="duplicate_name")- MetadataService.create_metadata(- dataset.id, first_metadata_args, account, tenant.id, session=db_session_with_containers+ _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), first_metadata_args, actor_id=account.id)-- # Try to create second metadata with same name+ db_session_with_containers.expire_all()second_metadata_args = MetadataArgs(type="number", name="duplicate_name")-- # Act & Assert: Verify proper error handlingwith pytest.raises(ValueError, match="Metadata name already exists."):- MetadataService.create_metadata(- dataset.id, second_metadata_args, account, tenant.id, session=db_session_with_containers+ _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), second_metadata_args, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_create_metadata_name_conflicts_with_built_in_field(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_create_metadata_name_conflicts_with_built_in_field(self, db_session_with_containers: Session) -> None:"""Test metadata creation fails when name conflicts with built-in field names."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Try to create metadata with built-in field name+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)built_in_field_name = BuiltInField.document_namemetadata_args = MetadataArgs(type="string", name=built_in_field_name)-- # Act & Assert: Verify proper error handlingwith pytest.raises(ValueError, match="Metadata name already exists in Built-in fields."):- MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_update_metadata_name_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_metadata_name_success(self, db_session_with_containers: Session) -> None:"""Test successful metadata name update with valid parameters."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="old_name")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Act: Execute the method under test+ db_session_with_containers.expire_all()new_name = "new_name"- result = MetadataService.update_metadata_name(- dataset, metadata.id, new_name, account, session=db_session_with_containers+ result = _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id, new_name, actor_id=account.id)-- # Assert: Verify the expected outcomes+ db_session_with_containers.expire_all()assert result is not Noneassert result.name == new_nameassert result.updated_by == account.idassert result.updated_at is not None-- # Verify database state-- db_session_with_containers.refresh(result)+ db_session_with_containers.expire_all()assert result.name == new_name- def test_update_metadata_name_too_long(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_metadata_name_too_long(self, db_session_with_containers: Session) -> None:"""Test metadata name update fails when new name exceeds 255 characters."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="old_name")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Try to update with too long name- long_name = "a" * 256 # 256 characters, exceeding 255 limit-- # Act & Assert: Verify proper error handling+ db_session_with_containers.expire_all()+ long_name = "a" * 256with pytest.raises(ValueError, match="Metadata name cannot exceed 255 characters."):- MetadataService.update_metadata_name(- dataset, metadata.id, long_name, account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id, long_name, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_update_metadata_name_already_exists(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_metadata_name_already_exists(self, db_session_with_containers: Session) -> None:"""Test metadata name update fails when new name already exists in the same dataset."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create two metadata entries+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)first_metadata_args = MetadataArgs(type="string", name="first_metadata")- first_metadata = MetadataService.create_metadata(- dataset.id, first_metadata_args, account, tenant.id, session=db_session_with_containers+ first_metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), first_metadata_args, actor_id=account.id)-+ db_session_with_containers.expire_all()second_metadata_args = MetadataArgs(type="number", name="second_metadata")- second_metadata = MetadataService.create_metadata(- dataset.id, second_metadata_args, account, tenant.id, session=db_session_with_containers+ second_metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), second_metadata_args, actor_id=account.id)-- # Try to update first metadata with second metadata's name+ db_session_with_containers.expire_all()with pytest.raises(ValueError, match="Metadata name already exists."):- MetadataService.update_metadata_name(- dataset, first_metadata.id, "second_metadata", account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), first_metadata.id, "second_metadata", actor_id=account.id)+ db_session_with_containers.expire_all()- def test_update_metadata_name_conflicts_with_built_in_field(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_metadata_name_conflicts_with_built_in_field(self, db_session_with_containers: Session) -> None:"""Test metadata name update fails when new name conflicts with built-in field names."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="old_name")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Try to update with built-in field name+ db_session_with_containers.expire_all()built_in_field_name = BuiltInField.document_name-with pytest.raises(ValueError, match="Metadata name already exists in Built-in fields."):- MetadataService.update_metadata_name(- dataset, metadata.id, built_in_field_name, account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id, built_in_field_name, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_update_metadata_name_not_found(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_metadata_name_not_found(self, db_session_with_containers: Session) -> None:"""Test metadata name update fails when metadata ID does not exist."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Try to update non-existent metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)import uuid- fake_metadata_id = str(uuid.uuid4()) # Use valid UUID format+ fake_metadata_id = str(uuid.uuid4())new_name = "new_name"+ with pytest.raises(MetadataResourceNotFoundError, match="Metadata not found"):+ _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), fake_metadata_id, new_name, actor_id=account.id+ )+ db_session_with_containers.expire_all()- # Act: Execute the method under test- result = MetadataService.update_metadata_name(- dataset, fake_metadata_id, new_name, account, session=db_session_with_containers- )-- # Assert: Verify the method returns None when metadata is not found- assert result is None-- def test_delete_metadata_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_delete_metadata_success(self, db_session_with_containers: Session) -> None:"""Test successful metadata deletion with valid parameters."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="to_be_deleted")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Act: Execute the method under test- result = MetadataService.delete_metadata(dataset, metadata.id, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ db_session_with_containers.expire_all()+ result = _metadata_service(db_session_with_containers).delete_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id+ )+ db_session_with_containers.expire_all()assert result is not Noneassert result.id == metadata.id-- # Verify metadata was deleted from database-deleted_metadata = db_session_with_containers.query(DatasetMetadata).filter_by(id=metadata.id).first()assert deleted_metadata is None- def test_delete_metadata_not_found(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_delete_metadata_not_found(self, db_session_with_containers: Session) -> None:"""Test metadata deletion fails when metadata ID does not exist."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Try to delete non-existent metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)import uuid- fake_metadata_id = str(uuid.uuid4()) # Use valid UUID format-- # Act: Execute the method under test- result = MetadataService.delete_metadata(dataset, fake_metadata_id, session=db_session_with_containers)-- # Assert: Verify the method returns None when metadata is not found- assert result is None+ fake_metadata_id = str(uuid.uuid4())+ with pytest.raises(MetadataResourceNotFoundError, match="Metadata not found"):+ _metadata_service(db_session_with_containers).delete_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), fake_metadata_id+ )+ db_session_with_containers.expire_all()- def test_delete_metadata_with_document_bindings(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_delete_metadata_with_document_bindings(self, db_session_with_containers: Session) -> None:"""Test metadata deletion successfully removes document metadata bindings."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )-- # Create metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)metadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Create metadata binding+ db_session_with_containers.expire_all()binding = DatasetMetadataBinding(tenant_id=tenant.id,dataset_id=dataset.id,@@ -524,70 +332,37 @@ class TestMetadataService:document_id=document.id,created_by=account.id,)-db_session_with_containers.add(binding)db_session_with_containers.commit()-- # Set document metadatadocument.doc_metadata = {"test_metadata": "test_value"}db_session_with_containers.add(document)db_session_with_containers.commit()-- # Act: Execute the method under test- result = MetadataService.delete_metadata(dataset, metadata.id, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ result = _metadata_service(db_session_with_containers).delete_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id+ )+ db_session_with_containers.expire_all()assert result is not None-- # Verify metadata was deleted from databasedeleted_metadata = db_session_with_containers.query(DatasetMetadata).filter_by(id=metadata.id).first()assert deleted_metadata is None- # Note: The service attempts to update document metadata but may not succeed- # due to mock configuration. The main functionality (metadata deletion) is verified.-@pytest.mark.parametrize("operation", ["rename", "delete"])@pytest.mark.parametrize("binding_owner", ["metadata", "document"])def test_metadata_changes_ignore_historical_foreign_document_binding(- self,- operation: str,- binding_owner: str,- db_session_with_containers: Session,- mock_external_service_dependencies: MetadataServiceDeps,+ self, operation: str, binding_owner: str, db_session_with_containers: Session) -> None:- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- foreign_account, foreign_tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- foreign_dataset = self._create_test_dataset(- db_session_with_containers,- mock_external_service_dependencies,- foreign_account,- foreign_tenant,- )- foreign_document = self._create_test_document(- db_session_with_containers,- mock_external_service_dependencies,- foreign_dataset,- foreign_account,- )+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ foreign_account, foreign_tenant = self._create_test_account_and_tenant(db_session_with_containers)+ foreign_dataset = self._create_test_dataset(db_session_with_containers, foreign_account, foreign_tenant)+ foreign_document = self._create_test_document(db_session_with_containers, foreign_dataset, foreign_account)foreign_document.enabled = Trueforeign_document.archived = Falseforeign_document.indexing_status = IndexingStatus.COMPLETEDforeign_document.doc_metadata = {"old_name": "foreign-value"}-- metadata = MetadataService.create_metadata(- dataset.id,- MetadataArgs(type="string", name="old_name"),- account,- tenant.id,- session=db_session_with_containers,+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), MetadataArgs(type="string", name="old_name"), actor_id=account.id)+ db_session_with_containers.expire_all()db_session_with_containers.add(DatasetMetadataBinding(tenant_id=dataset.tenant_id if binding_owner == "metadata" else foreign_dataset.tenant_id,@@ -598,163 +373,83 @@ class TestMetadataService:))db_session_with_containers.commit()-if operation == "rename":- MetadataService.update_metadata_name(- dataset, metadata.id, "new_name", account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_metadata_name(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id, "new_name", actor_id=account.id)else:- MetadataService.delete_metadata(dataset, metadata.id, session=db_session_with_containers)-+ _metadata_service(db_session_with_containers).delete_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), metadata.id+ )+ db_session_with_containers.expire_all()db_session_with_containers.refresh(foreign_document)assert foreign_document.doc_metadata == {"old_name": "foreign-value"}- def test_get_built_in_fields_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_get_built_in_fields_success(self, db_session_with_containers: Session) -> None:"""Test successful retrieval of built-in metadata fields."""- # Act: Execute the method under testresult = MetadataService.get_built_in_fields()-- # Assert: Verify the expected outcomesassert result is not Noneassert len(result) == 5-- # Verify all expected built-in fields are presentfield_names = [field["name"] for field in result]field_types = [field["type"] for field in result]-assert BuiltInField.document_name in field_namesassert BuiltInField.uploader in field_namesassert BuiltInField.upload_date in field_namesassert BuiltInField.last_update_date in field_namesassert BuiltInField.source in field_names-- # Verify field typesassert "string" in field_typesassert "time" in field_types- def test_enable_built_in_field_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_enable_built_in_field_success(self, db_session_with_containers: Session) -> None:"""Test successful enabling of built-in fields for a dataset."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )-- # Mock DocumentService.get_working_documents_by_dataset_id- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = [- document- ]-- # Verify dataset starts with built-in fields disabled+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)assert dataset.built_in_field_enabled is False-- # Act: Execute the method under test- MetadataService.enable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes-+ _metadata_service(db_session_with_containers).enable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert dataset.built_in_field_enabled- # Note: Document metadata update depends on DocumentService mock working correctly- # The main functionality (enabling built-in fields) is verified-- def test_enable_built_in_field_already_enabled(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_enable_built_in_field_already_enabled(self, db_session_with_containers: Session) -> None:"""Test enabling built-in fields when they are already enabled."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Enable built-in fields first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)dataset.built_in_field_enabled = True-db_session_with_containers.add(dataset)db_session_with_containers.commit()-- # Mock DocumentService.get_working_documents_by_dataset_id- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = list[- Document- ]()-- # Act: Execute the method under test- MetadataService.enable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the method returns early without changes+ _metadata_service(db_session_with_containers).enable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert dataset.built_in_field_enabled is True- def test_enable_built_in_field_with_no_documents(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_enable_built_in_field_with_no_documents(self, db_session_with_containers: Session) -> None:"""Test enabling built-in fields for a dataset with no documents."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Mock DocumentService.get_working_documents_by_dataset_id to return empty list- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = list[- Document- ]()-- # Act: Execute the method under test- MetadataService.enable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes-+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ _metadata_service(db_session_with_containers).enable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert dataset.built_in_field_enabled- def test_disable_built_in_field_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_disable_built_in_field_success(self, db_session_with_containers: Session) -> None:"""Test successful disabling of built-in fields for a dataset."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )-- # Enable built-in fields first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)dataset.built_in_field_enabled = True-db_session_with_containers.add(dataset)db_session_with_containers.commit()-- # Set document metadata with built-in fieldsdocument.doc_metadata = {BuiltInField.document_name: document.name,BuiltInField.uploader: "test_uploader",@@ -764,128 +459,60 @@ class TestMetadataService:}db_session_with_containers.add(document)db_session_with_containers.commit()-- # Mock DocumentService.get_working_documents_by_dataset_id- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = [- document- ]-- # Act: Execute the method under test- MetadataService.disable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ _metadata_service(db_session_with_containers).disable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert dataset.built_in_field_enabled is False- # Note: Document metadata update depends on DocumentService mock working correctly- # The main functionality (disabling built-in fields) is verified-- def test_disable_built_in_field_already_disabled(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_disable_built_in_field_already_disabled(self, db_session_with_containers: Session) -> None:"""Test disabling built-in fields when they are already disabled."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Verify dataset starts with built-in fields disabled+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)assert dataset.built_in_field_enabled is False-- # Mock DocumentService.get_working_documents_by_dataset_id- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = list[- Document- ]()-- # Act: Execute the method under test- MetadataService.disable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the method returns early without changes-+ _metadata_service(db_session_with_containers).disable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert not dataset.built_in_field_enabled- def test_disable_built_in_field_with_no_documents(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_disable_built_in_field_with_no_documents(self, db_session_with_containers: Session) -> None:"""Test disabling built-in fields for a dataset with no documents."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Enable built-in fields first+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)dataset.built_in_field_enabled = True-db_session_with_containers.add(dataset)db_session_with_containers.commit()-- # Mock DocumentService.get_working_documents_by_dataset_id to return empty list- mock_external_service_dependencies["document_service"].get_working_documents_by_dataset_id.return_value = list[- Document- ]()-- # Act: Execute the method under test- MetadataService.disable_built_in_field(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ _metadata_service(db_session_with_containers).disable_built_in_field(DatasetRef(dataset.tenant_id, dataset.id))+ db_session_with_containers.expire_all()db_session_with_containers.refresh(dataset)assert dataset.built_in_field_enabled is False- def test_update_documents_metadata_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_documents_metadata_success(self, db_session_with_containers: Session) -> None:"""Test successful update of documents metadata."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )-- # Create metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)metadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Mock DocumentService.get_document- mock_external_service_dependencies["document_service"].get_document.return_value = document-+ db_session_with_containers.expire_all()metadata_detail = MetadataDetail(id=metadata.id, name=metadata.name, value="test_value")-operation = DocumentMetadataOperation(document_id=document.id, metadata_list=[metadata_detail])-operation_data = MetadataOperationData(operation_data=[operation])-- # Act: Execute the method under test- MetadataService.update_documents_metadata(dataset, operation_data, account, session=db_session_with_containers)-- # Assert: Verify the expected outcomes-- # Verify document metadata was updated+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), operation_data, actor_id=account.id+ )+ db_session_with_containers.expire_all()db_session_with_containers.refresh(document)assert document.doc_metadata is not Noneassert "test_metadata" in document.doc_metadataassert document.doc_metadata["test_metadata"] == "test_value"-- # Verify metadata binding was createdbinding = (db_session_with_containers.query(DatasetMetadataBinding).filter_by(metadata_id=metadata.id, document_id=document.id)@@ -897,50 +524,24 @@ class TestMetadataService:@pytest.mark.parametrize("foreign_resource", ["metadata", "document"])def test_update_documents_metadata_rejects_foreign_owner_before_writes(- self,- foreign_resource: str,- db_session_with_containers: Session,- mock_external_service_dependencies: MetadataServiceDeps,+ self, foreign_resource: str, db_session_with_containers: Session) -> None:- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )- metadata = MetadataService.create_metadata(- dataset.id,- MetadataArgs(type="string", name="owned"),- account,- tenant.id,- session=db_session_with_containers,- )-- foreign_account, foreign_tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- foreign_dataset = self._create_test_dataset(- db_session_with_containers,- mock_external_service_dependencies,- foreign_account,- foreign_tenant,- )- foreign_document = self._create_test_document(- db_session_with_containers,- mock_external_service_dependencies,- foreign_dataset,- foreign_account,- )- foreign_metadata = MetadataService.create_metadata(- foreign_dataset.id,+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), MetadataArgs(type="string", name="owned"), actor_id=account.id+ )+ db_session_with_containers.expire_all()+ foreign_account, foreign_tenant = self._create_test_account_and_tenant(db_session_with_containers)+ foreign_dataset = self._create_test_dataset(db_session_with_containers, foreign_account, foreign_tenant)+ foreign_document = self._create_test_document(db_session_with_containers, foreign_dataset, foreign_account)+ foreign_metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(foreign_tenant.id, foreign_dataset.id),MetadataArgs(type="string", name="foreign"),- foreign_account,- foreign_tenant.id,- session=db_session_with_containers,+ actor_id=foreign_account.id,)+ db_session_with_containers.expire_all()operation = DocumentMetadataOperation(document_id=foreign_document.id if foreign_resource == "document" else document.id,metadata_list=[@@ -951,209 +552,80 @@ class TestMetadataService:)],)-with pytest.raises(MetadataResourceNotFoundError, match=f"{foreign_resource.capitalize()} not found"):- MetadataService.update_documents_metadata(- dataset,+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id),MetadataOperationData(operation_data=[operation]),- account,- session=db_session_with_containers,+ actor_id=account.id,)-+ db_session_with_containers.expire_all()db_session_with_containers.refresh(document)db_session_with_containers.refresh(foreign_document)assert document.doc_metadata is Noneassert foreign_document.doc_metadata is Noneassert db_session_with_containers.query(DatasetMetadataBinding).count() == 0- def test_update_documents_metadata_with_built_in_fields_enabled(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_documents_metadata_with_built_in_fields_enabled(self, db_session_with_containers: Session) -> None:"""Test update of documents metadata when built-in fields are enabled."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account- )-- # Enable built-in fields+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ document = self._create_test_document(db_session_with_containers, dataset, account)dataset.built_in_field_enabled = True-db_session_with_containers.add(dataset)db_session_with_containers.commit()-- # Create metadatametadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Mock DocumentService.get_document- mock_external_service_dependencies["document_service"].get_document.return_value = document-+ db_session_with_containers.expire_all()metadata_detail = MetadataDetail(id=metadata.id, name=metadata.name, value="test_value")-operation = DocumentMetadataOperation(document_id=document.id, metadata_list=[metadata_detail])-operation_data = MetadataOperationData(operation_data=[operation])-- # Act: Execute the method under test- MetadataService.update_documents_metadata(dataset, operation_data, account, session=db_session_with_containers)-- # Assert: Verify the expected outcomes- # Verify document metadata was updated with both custom and built-in fields+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), operation_data, actor_id=account.id+ )+ db_session_with_containers.expire_all()db_session_with_containers.refresh(document)assert document.doc_metadata is not Noneassert "test_metadata" in document.doc_metadataassert document.doc_metadata["test_metadata"] == "test_value"- # Note: Built-in fields would be added if DocumentService mock works correctly- # The main functionality (custom metadata update) is verified-- def test_update_documents_metadata_document_not_found(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_update_documents_metadata_document_not_found(self, db_session_with_containers: Session) -> None:"""Test update of documents metadata when document is not found."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-+ db_session_with_containers.expire_all()metadata_detail = MetadataDetail(id=metadata.id, name=metadata.name, value="test_value")-- # Use a valid UUID format that does not exist in the databaseoperation = DocumentMetadataOperation(document_id="00000000-0000-0000-0000-000000000000", metadata_list=[metadata_detail])-operation_data = MetadataOperationData(operation_data=[operation])-with pytest.raises(MetadataResourceNotFoundError, match="Document not found"):- MetadataService.update_documents_metadata(- dataset, operation_data, account, session=db_session_with_containers+ _metadata_service(db_session_with_containers).update_documents_metadata(+ DatasetRef(dataset.tenant_id, dataset.id), operation_data, actor_id=account.id)+ db_session_with_containers.expire_all()- def test_knowledge_base_metadata_lock_check_dataset_id(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:- """- Test metadata lock check for dataset operations.- """- # Arrange: Setup mocks- mock_external_service_dependencies["redis_client"].get.return_value = None- mock_external_service_dependencies["redis_client"].set.return_value = True-- dataset_id = "test-dataset-id"-- # Act: Execute the method under test- MetadataService.knowledge_base_metadata_lock_check(dataset_id, None)-- # Assert: Verify the expected outcomes- # Verify Redis lock was set- mock_external_service_dependencies["redis_client"].set.assert_called_once()-- # Verify lock key format- call_args = mock_external_service_dependencies["redis_client"].set.call_args- assert call_args[0][0] == f"dataset_metadata_lock_{dataset_id}"-- def test_knowledge_base_metadata_lock_check_document_id(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:- """- Test metadata lock check for document operations.- """- # Arrange: Setup mocks- mock_external_service_dependencies["redis_client"].get.return_value = None- mock_external_service_dependencies["redis_client"].set.return_value = True-- document_id = "test-document-id"-- # Act: Execute the method under test- MetadataService.knowledge_base_metadata_lock_check(None, document_id)-- # Assert: Verify the expected outcomes- # Verify Redis lock was set- mock_external_service_dependencies["redis_client"].set.assert_called_once()-- # Verify lock key format- call_args = mock_external_service_dependencies["redis_client"].set.call_args- assert call_args[0][0] == f"document_metadata_lock_{document_id}"-- def test_knowledge_base_metadata_lock_check_lock_exists(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:- """- Test metadata lock check when lock already exists.- """- # Arrange: Setup mocks to simulate existing lock- mock_external_service_dependencies["redis_client"].get.return_value = "1" # Lock exists-- dataset_id = "test-dataset-id"-- # Act & Assert: Verify proper error handling- with pytest.raises(- ValueError, match="Another knowledge base metadata operation is running, please wait a moment."- ):- MetadataService.knowledge_base_metadata_lock_check(dataset_id, None)-- def test_knowledge_base_metadata_lock_check_document_lock_exists(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:- """- Test metadata lock check when document lock already exists.- """- # Arrange: Setup mocks to simulate existing lock- mock_external_service_dependencies["redis_client"].get.return_value = "1" # Lock exists-- document_id = "test-document-id"-- # Act & Assert: Verify proper error handling- with pytest.raises(ValueError, match="Another document metadata operation is running, please wait a moment."):- MetadataService.knowledge_base_metadata_lock_check(None, document_id)-- def test_get_dataset_metadatas_success(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_get_dataset_metadatas_success(self, db_session_with_containers: Session) -> None:"""Test successful retrieval of dataset metadata information."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Create metadata+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)metadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers- )-- # Create document and metadata binding- document = self._create_test_document(- db_session_with_containers, mock_external_service_dependencies, dataset, account+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-+ db_session_with_containers.expire_all()+ document = self._create_test_document(db_session_with_containers, dataset, account)binding = DatasetMetadataBinding(tenant_id=tenant.id,dataset_id=dataset.id,@@ -1161,95 +633,61 @@ class TestMetadataService:document_id=document.id,created_by=account.id,)-db_session_with_containers.add(binding)db_session_with_containers.commit()-- # Act: Execute the method under test- result = MetadataService.get_dataset_metadatas(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ result = _metadata_service(db_session_with_containers).get_dataset_metadatas(+ DatasetRef(dataset.tenant_id, dataset.id)+ )+ db_session_with_containers.expire_all()assert result is not Noneassert "doc_metadata" in resultassert "built_in_field_enabled" in result-- # Verify metadata informationdoc_metadata = result["doc_metadata"]assert len(doc_metadata) == 1assert doc_metadata[0]["id"] == metadata.idassert doc_metadata[0]["name"] == metadata.nameassert doc_metadata[0]["type"] == metadata.type- assert doc_metadata[0]["count"] == 1 # One document bound to this metadata-- # Verify built-in field status+ assert doc_metadata[0]["count"] == 1assert result["built_in_field_enabled"] is False- def test_get_dataset_metadatas_with_built_in_fields_enabled(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_get_dataset_metadatas_with_built_in_fields_enabled(self, db_session_with_containers: Session) -> None:"""Test retrieval of dataset metadata when built-in fields are enabled."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant- )-- # Enable built-in fields+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)dataset.built_in_field_enabled = True-db_session_with_containers.add(dataset)db_session_with_containers.commit()-- # Create metadatametadata_args = MetadataArgs(type="string", name="test_metadata")- metadata = MetadataService.create_metadata(- dataset.id, metadata_args, account, tenant.id, session=db_session_with_containers+ metadata = _metadata_service(db_session_with_containers).create_metadata(+ DatasetRef(tenant.id, dataset.id), metadata_args, actor_id=account.id)-- # Act: Execute the method under test- result = MetadataService.get_dataset_metadatas(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ db_session_with_containers.expire_all()+ result = _metadata_service(db_session_with_containers).get_dataset_metadatas(+ DatasetRef(dataset.tenant_id, dataset.id)+ )+ db_session_with_containers.expire_all()assert result is not Noneassert "doc_metadata" in resultassert "built_in_field_enabled" in result-- # Verify metadata informationdoc_metadata = result["doc_metadata"]- assert len(doc_metadata) == 1 # Only custom metadata, built-in fields are not included in this list-- # Verify built-in field status+ assert len(doc_metadata) == 1assert result["built_in_field_enabled"] is True- def test_get_dataset_metadatas_no_metadata(- self, db_session_with_containers: Session, mock_external_service_dependencies: MetadataServiceDeps- ) -> None:+ def test_get_dataset_metadatas_no_metadata(self, db_session_with_containers: Session) -> None:"""Test retrieval of dataset metadata when no metadata exists."""- # Arrange: Create test data- account, tenant = self._create_test_account_and_tenant(- db_session_with_containers, mock_external_service_dependencies- )- dataset = self._create_test_dataset(- db_session_with_containers, mock_external_service_dependencies, account, tenant+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)+ result = _metadata_service(db_session_with_containers).get_dataset_metadatas(+ DatasetRef(dataset.tenant_id, dataset.id))-- # Act: Execute the method under test- result = MetadataService.get_dataset_metadatas(dataset, session=db_session_with_containers)-- # Assert: Verify the expected outcomes+ db_session_with_containers.expire_all()assert result is not Noneassert "doc_metadata" in resultassert "built_in_field_enabled" in result-- # Verify metadata informationdoc_metadata = result["doc_metadata"]- assert len(doc_metadata) == 0 # No metadata exists-- # Verify built-in field status+ assert len(doc_metadata) == 0assert result["built_in_field_enabled"] is Falsediff --git a/api/tests/test_containers_integration_tests/services/test_model_provider_service.py b/api/tests/test_containers_integration_tests/services/test_model_provider_service.pyindex f250488bbd7..356bf77dbd2 100644--- a/api/tests/test_containers_integration_tests/services/test_model_provider_service.py+++ b/api/tests/test_containers_integration_tests/services/test_model_provider_service.py@@ -8,7 +8,7 @@ from core.entities.model_entities import ModelStatusfrom graphon.model_runtime.entities.model_entities import FetchFrom, ModelTypefrom models import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.provider import Provider, ProviderModel, ProviderModelSetting, ProviderType-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceclass TestModelProviderService:@@ -19,10 +19,10 @@ class TestModelProviderService:"""Mock setup for external service dependencies."""with (patch(- "services.model_provider_service.create_plugin_provider_manager", autospec=True+ "services.model_provider.service.create_plugin_provider_manager", autospec=True) as mock_provider_manager,patch(- "services.model_provider_service.create_plugin_model_provider_factory", autospec=True+ "services.model_provider.service.create_plugin_model_provider_factory", autospec=True) as mock_model_provider_factory,):# Setup default mock returnsdiff --git a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.pyindex 94fd7602f5e..30857e22c0e 100644--- a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py@@ -8,8 +8,8 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.ext_redis import redis_clientfrom models import Account, Tenant, TenantAccountJoin, TenantAccountRole-from models.dataset import Dataset, DatasetAutoDisableLog, Document, DocumentSegment-from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from models.dataset import ChildChunk, Dataset, DatasetAutoDisableLog, DatasetProcessRule, Document, DocumentSegment+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatusfrom tasks.add_document_to_index_task import add_document_to_index_task@@ -377,56 +377,75 @@ class TestAddDocumentToIndexTask:# Refresh dataset to ensure doc_form property reflects the updated documentdb_session_with_containers.refresh(dataset)- # Create segments with mock child chunks+ # Persist each segment and its own child chunkssegments = self._create_test_segments(db_session_with_containers, document, dataset)# Set up Redis cache keyindexing_cache_key = f"document_{document.id}_indexing"redis_client.set(indexing_cache_key, "processing", ex=300)- # Mock the get_child_chunks method for each segment- with patch.object(DocumentSegment, "get_child_chunks", autospec=True) as mock_get_child_chunks:- # Setup mock to return child chunks for each segment- mock_child_chunks = []- for i in range(2): # Each segment has 2 child chunks- mock_child = MagicMock()- mock_child.content = f"child_content_{i}"- mock_child.index_node_id = f"child_node_{i}"- mock_child.index_node_hash = f"child_hash_{i}"- mock_child_chunks.append(mock_child)-- mock_get_child_chunks.return_value = mock_child_chunks+ rule = DatasetProcessRule(+ dataset_id=dataset.id,+ mode=ProcessRuleMode.HIERARCHICAL,+ rules='{"parent_mode": "paragraph"}',+ created_by=document.created_by,+ )+ db_session_with_containers.add(rule)+ db_session_with_containers.flush()+ document.dataset_process_rule_id = rule.id+ for segment in segments:+ db_session_with_containers.add_all(+ [+ ChildChunk(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ segment_id=segment.id,+ position=i,+ content=f"child_content_{i}",+ word_count=1,+ created_by=document.created_by,+ index_node_id=f"{segment.index_node_id}-child-{i}",+ index_node_hash=f"{segment.index_node_id}-hash-{i}",+ )+ for i in range(2)+ ]+ )+ db_session_with_containers.commit()- # Act: Execute the task- add_document_to_index_task(document.id)+ # Act: Execute the task+ add_document_to_index_task(document.id)- # Assert: Verify parent-child index processing- mock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- IndexStructureType.PARENT_CHILD_INDEX- )- mock_external_service_dependencies["index_processor"].load.assert_called_once()+ # Assert: Verify parent-child index processing+ mock_external_service_dependencies["index_processor_factory"].assert_called_once_with(+ IndexStructureType.PARENT_CHILD_INDEX+ )+ mock_external_service_dependencies["index_processor"].load.assert_called_once()- # Verify the load method was called with correct parameters- call_args = mock_external_service_dependencies["index_processor"].load.call_args- assert call_args is not None- documents = call_args[0][1] # Second argument should be documents list- assert len(documents) == 3 # 3 segments+ # Verify the load method was called with correct parameters+ call_args = mock_external_service_dependencies["index_processor"].load.call_args+ assert call_args is not None+ documents = call_args[0][1] # Second argument should be documents list+ assert len(documents) == 3 # 3 segments- # Verify each document has children- for doc in documents:- assert hasattr(doc, "children")- assert len(doc.children) == 2 # Each document has 2 children+ # Each parent receives only its own persisted children, in position order.+ for doc in documents:+ assert doc.children is not None+ assert [child.page_content for child in doc.children] == ["child_content_0", "child_content_1"]+ assert [child.metadata["doc_id"] for child in doc.children] == [+ f"{doc.metadata['doc_id']}-child-{i}" for i in range(2)+ ]- # Verify database state changes- db_session_with_containers.refresh(document)- for segment in segments:- db_session_with_containers.refresh(segment)- assert segment.enabled is True- assert segment.disabled_at is None- assert segment.disabled_by is None+ # Verify database state changes+ db_session_with_containers.refresh(document)+ for segment in segments:+ db_session_with_containers.refresh(segment)+ assert segment.enabled is True+ assert segment.disabled_at is None+ assert segment.disabled_by is None- # Verify redis cache was cleared- assert redis_client.exists(indexing_cache_key) == 0+ # Verify redis cache was cleared+ assert redis_client.exists(indexing_cache_key) == 0def test_add_document_to_index_with_already_enabled_segments(self, db_session_with_containers: Session, mock_external_service_dependenciesdiff --git a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.pyindex adac71a2edf..574ba58b742 100644--- a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py@@ -7,7 +7,7 @@ and storage containers to ensure proper cleanup of documents, segments, and fileimport jsonimport uuid-from unittest.mock import Mock, patch+from unittest.mock import patchimport pytestfrom faker import Faker@@ -21,6 +21,7 @@ from models import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatusfrom models.model import UploadFile+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom tasks.batch_clean_document_task import batch_clean_document_task@@ -31,25 +32,20 @@ class TestBatchCleanDocumentTask:def mock_external_service_dependencies(self):"""Mock setup for external service dependencies."""with (- patch("extensions.ext_storage.storage") as mock_storage,- patch("core.rag.index_processor.index_processor_factory.IndexProcessorFactory") as mock_index_factory,- patch("core.tools.utils.web_reader_tool.get_image_upload_file_ids") as mock_get_image_ids,+ patch("tasks.batch_clean_document_task.storage") as mock_storage,+ patch("services.knowledge.indexing.adapters.cleanup.Vector", autospec=True) as mock_vector,+ patch("tasks.batch_clean_document_task.get_image_upload_file_ids") as mock_get_image_ids,+ patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),):# Setup default mock returnsmock_storage.delete.return_value = None- # Mock index processor- mock_index_processor = Mock()- mock_index_processor.clean.return_value = None- mock_index_factory.return_value.init_index_processor.return_value = mock_index_processor-# Mock image file ID extractionmock_get_image_ids.return_value = []yield {"storage": mock_storage,- "index_factory": mock_index_factory,- "index_processor": mock_index_processor,+ "vector": mock_vector.return_value,"get_image_ids": mock_get_image_ids,}@@ -121,6 +117,7 @@ class TestBatchCleanDocumentTask:created_by=account.id,embedding_model="text-embedding-ada-002",embedding_model_provider="openai",+ indexing_technique="high_quality",)db_session_with_containers.add(dataset)@@ -251,18 +248,18 @@ class TestBatchCleanDocumentTask:# Store original IDs for verificationdocument_id = document.idsegment_id = segment.id+ index_node_id = segment.index_node_idfile_id = upload_file.id# Execute the taskbatch_clean_document_task(document_ids=[document_id],dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=[file_id],)- # Verify that the task completed successfully- # The task should have processed the segment and cleaned up the database+ mock_external_service_dependencies["vector"].delete_by_ids.assert_called_once_with([index_node_id])# Verify database cleanupdb_session_with_containers.commit() # Ensure all changes are committed@@ -316,7 +313,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=[document_id],dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=[],)@@ -359,7 +356,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=[document_id],dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=[file_id],)@@ -411,7 +408,7 @@ class TestBatchCleanDocumentTask:)# Verify that no index processing occurred- mock_external_service_dependencies["index_processor"].clean.assert_not_called()+ mock_external_service_dependencies["vector"].delete_by_ids.assert_not_called()# Verify that no storage operations occurredmock_external_service_dependencies["storage"].delete.assert_not_called()@@ -457,7 +454,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=[document_id],dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=[file_id],)@@ -518,7 +515,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=document_ids,dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=file_ids,)@@ -658,7 +655,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=document_ids,dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=file_ids,)@@ -754,7 +751,7 @@ class TestBatchCleanDocumentTask:batch_clean_document_task(document_ids=[document_id],dataset_id=dataset.id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),file_ids=[file_id],)diff --git a/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.pyindex b0f57dba084..a99a7301b12 100644--- a/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py@@ -59,7 +59,7 @@ class TestBatchCreateSegmentToIndexTask:"tasks.batch_create_segment_to_index_task.ModelManager.for_tenant",autospec=True,) as mock_model_manager,- patch("tasks.batch_create_segment_to_index_task.VectorService", autospec=True) as mock_vector_service,+ patch("extensions.ext_application_services.application_services") as mock_services,):# Setup default mock returnsmock_storage.download.return_value = None@@ -71,13 +71,13 @@ class TestBatchCreateSegmentToIndexTask:mock_model_manager_instance.get_model_instance.return_value = mock_embedding_modelmock_model_manager.return_value = mock_model_manager_instance- # Mock vector service- mock_vector_service.create_segments_vector.return_value = None+ mutations = mock_services.return_value.knowledge.segments.mutations+ mutations.index_segments.return_value = Noneyield {"storage": mock_storage,"model_manager": mock_model_manager,- "vector_service": mock_vector_service,+ "segment_mutations": mutations,"embedding_model": mock_embedding_model,}@@ -314,8 +314,8 @@ class TestBatchCreateSegmentToIndexTask:assert document.word_count > 0# Verify vector service was called- mock_vector_service = mock_external_service_dependencies["vector_service"]- mock_vector_service.create_segments_vector.assert_called_once()+ mutations = mock_external_service_dependencies["segment_mutations"]+ mutations.index_segments.assert_called_once()# Check Redis cache was setfrom extensions.ext_redis import redis_client@@ -750,8 +750,8 @@ class TestBatchCreateSegmentToIndexTask:assert document.word_count > 0# Verify vector service was called- mock_vector_service = mock_external_service_dependencies["vector_service"]- mock_vector_service.create_segments_vector.assert_called_once()+ mutations = mock_external_service_dependencies["segment_mutations"]+ mutations.index_segments.assert_called_once()# Check Redis cache was setfrom extensions.ext_redis import redis_clientdiff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.pyindex e801597e434..ee47693ee66 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py@@ -41,6 +41,7 @@ from models.enums import (SegmentStatus,)from models.model import UploadFile+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom tasks.clean_dataset_task import clean_dataset_task@@ -295,7 +296,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify results@@ -419,7 +420,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify results@@ -552,7 +553,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify results - even with vector cleanup failure, documents and segments should be deleted@@ -638,7 +639,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify results@@ -751,7 +752,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)end_time = time.time()@@ -845,7 +846,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify results@@ -999,7 +1000,7 @@ class TestCleanDatasetTask:indexing_technique=dataset.indexing_technique,index_struct=dataset.index_struct,collection_binding_id=dataset.collection_binding_id,- doc_form=dataset.get_doc_form(session=db_session_with_containers),+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),)# Verify resultsdiff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.pyindex b2007608e42..d9cdd9aef79 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py@@ -7,7 +7,7 @@ containers to ensure proper cleanup of Notion documents, segments, and vector inimport jsonimport uuid-from unittest.mock import Mock, patch+from unittest.mock import patchimport pytestfrom faker import Faker@@ -47,31 +47,13 @@ class TestCleanNotionDocumentTask:}@pytest.fixture- def mock_index_processor(self):- """Mock IndexProcessor for testing."""- mock_processor = Mock()- mock_processor.clean = Mock()- return mock_processor-- @pytest.fixture- def mock_index_processor_factory(self, mock_index_processor):- """Mock IndexProcessorFactory for testing."""- # Mock the actual IndexProcessorFactory class- with patch("tasks.clean_notion_document_task.IndexProcessorFactory") as mock_factory:- # Create a mock instance that will be returned when IndexProcessorFactory() is called- mock_instance = Mock()- mock_instance.init_index_processor.return_value = mock_index_processor-- # Set the mock_factory to return our mock_instance when called- mock_factory.return_value = mock_instance-- # Ensure the mock_index_processor has the clean method properly set- mock_index_processor.clean = Mock()-- yield mock_factory+ def mock_vector(self):+ """Keep database cleanup real while isolating the external vector backend."""+ with patch("services.knowledge.indexing.adapters.cleanup.Vector", autospec=True) as vector:+ yield vector.return_valuedef test_clean_notion_document_task_success(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test successful cleanup of Notion documents with proper database operations.@@ -79,7 +61,7 @@ class TestCleanNotionDocumentTask:This test verifies that the task correctly:1. Deletes Document records from database2. Deletes DocumentSegment records from database- 3. Calls index processor to clean vector and keyword indices+ 3. Deletes the documents' vector entries4. Commits all changes to database"""fake = Faker()@@ -97,6 +79,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -164,19 +147,19 @@ class TestCleanNotionDocumentTask:# Verify segments are deletedassert _count_segments(db_session_with_containers, DocumentSegment.document_id.in_(document_ids)) == 0- # Verify index processor was called- mock_processor = mock_index_processor_factory.return_value.init_index_processor.return_value- mock_processor.clean.assert_called_once()+ assert _count_documents(db_session_with_containers, Document.id.in_(document_ids)) == 0+ mock_vector.delete_by_ids.assert_called_once()+ assert set(mock_vector.delete_by_ids.call_args.args[0]) == set(index_node_ids)# This test successfully verifies:# 1. Document records are properly deleted from the database# 2. DocumentSegment records are properly deleted from the database- # 3. The index processor's clean method is called+ # 3. The external vector deletion receives the selected node IDs# 4. Database transaction handling works correctly# 5. The task completes without errorsdef test_clean_notion_document_task_dataset_not_found(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task behavior when dataset is not found.@@ -192,11 +175,10 @@ class TestCleanNotionDocumentTask:with pytest.raises(Exception, match="Document has no dataset"):clean_notion_document_task(document_ids, non_existent_dataset_id)- # Verify that the index processor factory was not used- mock_index_processor_factory.return_value.init_index_processor.assert_not_called()+ mock_vector.delete_by_ids.assert_not_called()def test_clean_notion_document_task_empty_document_list(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task behavior with empty document list.@@ -219,6 +201,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -232,23 +215,16 @@ class TestCleanNotionDocumentTask:# Execute cleanup task with empty document listclean_notion_document_task([], dataset.id)- # Verify that the index processor was called once with empty node list- mock_processor = mock_index_processor_factory.return_value.init_index_processor.return_value- assert mock_processor.clean.call_count == 1- args, kwargs = mock_processor.clean.call_args- # args: (dataset, total_index_node_ids)- assert isinstance(args[0], Dataset)- assert args[1] == []- assert kwargs["session"] is not None+ # An empty document selection must never trigger a dataset-wide vector deletion.+ mock_vector.delete_by_ids.assert_not_called()def test_clean_notion_document_task_with_different_index_types(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with different dataset index types.- This test verifies that the task correctly initializes different types- of index processors based on the dataset's doc_form configuration.+ This test verifies that cleanup uses the dataset's document form."""fake = Faker()@@ -270,6 +246,7 @@ class TestCleanNotionDocumentTask:for index_type in index_types:# Create dataset (doc_form will be set via document creation)dataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=f"{fake.company()}_{index_type}",@@ -320,17 +297,15 @@ class TestCleanNotionDocumentTask:# Execute cleanup taskclean_notion_document_task([document.id], dataset.id)- # Note: This test successfully verifies cleanup with different document types.- # The task properly handles various index types and document configurations.-# Verify segments are deletedassert _count_segments(db_session_with_containers, DocumentSegment.document_id == document.id) == 0+ mock_vector.delete_by_ids.assert_called_once_with(["test_node"])# Reset mock for next iteration- mock_index_processor_factory.reset_mock()+ mock_vector.reset_mock()def test_clean_notion_document_task_with_segments_no_index_node_ids(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with segments that have no index_node_ids.@@ -353,6 +328,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -409,11 +385,10 @@ class TestCleanNotionDocumentTask:# Verify segments are deletedassert _count_segments(db_session_with_containers, DocumentSegment.document_id == document.id) == 0- # Note: This test successfully verifies that segments without index_node_ids- # are properly deleted from the database.+ mock_vector.delete_by_ids.assert_not_called()def test_clean_notion_document_task_partial_document_cleanup(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with partial document cleanup scenario.@@ -436,6 +411,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -515,7 +491,7 @@ class TestCleanNotionDocumentTask:# The database operations work correctly, isolating only the specified documents.def test_clean_notion_document_task_with_mixed_segment_statuses(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with segments in different statuses.@@ -538,6 +514,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -601,21 +578,21 @@ class TestCleanNotionDocumentTask:# Verify all segments are deleted regardless of statusassert _count_segments(db_session_with_containers, DocumentSegment.document_id == document.id) == 0- # Note: This test successfully verifies database operations.- # IndexProcessor verification would require more sophisticated mocking.+ mock_vector.delete_by_ids.assert_called_once()+ assert set(mock_vector.delete_by_ids.call_args.args[0]) == set(index_node_ids)- def test_clean_notion_document_task_continues_when_index_processor_fails(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ def test_clean_notion_document_task_continues_when_vector_cleanup_fails(+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""- Index processor failure (e.g. transient billing API error propagated via+ Vector cleanup failure (e.g. transient billing API error propagated via``FeatureService`` when ``Vector(dataset)`` lazily resolves the embeddingmodel) must NOT abort the cleanup task. The Document rows have alreadybeen hard-deleted in the first session block before vector cleanup runs,so any uncaught exception escaping the task would strand``DocumentSegment`` rows in PG with no parent ``Document``.- Contract: the task swallows the index_processor exception, logs it, and+ Contract: the task swallows the vector cleanup exception, logs it, andproceeds to delete the segments — leaving PG consistent. (Vector orphans,if any, can be reaped later by an offline scanner.)@@ -639,6 +616,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -685,23 +663,22 @@ class TestCleanNotionDocumentTask:db_session_with_containers.add(segment)db_session_with_containers.commit()- # Simulate the production failure mode: index_processor.clean() raises a+ # Simulate the production failure mode: vector deletion raises a# ValueError mirroring ``BillingService._send_request`` returning non-200.- mock_index_processor = mock_index_processor_factory.return_value.init_index_processor.return_value- mock_index_processor.clean.side_effect = ValueError(+ mock_vector.delete_by_ids.side_effect = ValueError("Unable to retrieve billing information. Please try again later or contact support.")- # Execute cleanup task — must NOT raise even though clean() raises.+ # Execute cleanup task — must NOT raise even though vector deletion raises.# Before the safety-net wrapper this would have re-raised the ValueError,# aborting the task and leaving DocumentSegment stranded in PG.clean_notion_document_task([document.id], dataset.id)# Vector cleanup was attempted exactly once.- mock_index_processor.clean.assert_called_once()+ mock_vector.delete_by_ids.assert_called_once_with(["test_node"])- # The crucial assertion: despite the index processor failure, the- # final session block (line 51-52, ``DELETE FROM document_segments``)+ # The crucial assertion: despite the vector cleanup failure, the+ # final session block (``DELETE FROM document_segments``)# still ran and committed. This is what the wrapper buys us — without# it the production incident left tens of thousands of orphan segments# per affected tenant. Aligns with the assertion shape used by the@@ -709,7 +686,7 @@ class TestCleanNotionDocumentTask:assert _count_segments(db_session_with_containers, DocumentSegment.document_id == document.id) == 0def test_clean_notion_document_task_with_large_number_of_documents(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with a large number of documents and segments.@@ -732,6 +709,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -808,7 +786,7 @@ class TestCleanNotionDocumentTask:# The database efficiently handles large-scale deletions.def test_clean_notion_document_task_with_documents_from_different_tenants(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with documents from different tenants.@@ -840,6 +818,7 @@ class TestCleanNotionDocumentTask:# Create dataset for each tenantdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=f"{fake.company()}_{i}",@@ -924,7 +903,7 @@ class TestCleanNotionDocumentTask:# Only documents from the target dataset are affected, maintaining tenant separation.def test_clean_notion_document_task_with_documents_in_different_states(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with documents in different indexing states.@@ -947,6 +926,7 @@ class TestCleanNotionDocumentTask:# Create datasetdataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),@@ -1030,7 +1010,7 @@ class TestCleanNotionDocumentTask:# All documents are deleted regardless of their indexing status.def test_clean_notion_document_task_with_documents_having_metadata(- self, db_session_with_containers: Session, mock_index_processor_factory, mock_external_service_dependencies+ self, db_session_with_containers: Session, mock_vector, mock_external_service_dependencies):"""Test cleanup task with documents that have rich metadata.@@ -1053,6 +1033,7 @@ class TestCleanNotionDocumentTask:# Create dataset with built-in fields enableddataset = Dataset(+ indexing_technique="high_quality",id=str(uuid.uuid4()),tenant_id=tenant.id,name=fake.company(),diff --git a/api/tests/test_containers_integration_tests/tasks/test_create_segment_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_create_segment_to_index_task.pyindex 40bffa766b1..aa4257bdb05 100644--- a/api/tests/test_containers_integration_tests/tasks/test_create_segment_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_create_segment_to_index_task.py@@ -19,6 +19,7 @@ from extensions.ext_redis import redis_clientfrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from repositories.knowledge.dataset_read_repository import get_dataset_doc_formfrom tasks.create_segment_to_index_task import create_segment_to_index_task@@ -227,7 +228,7 @@ class TestCreateSegmentToIndexTask:# Verify index processor was calledmock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- dataset.get_doc_form(session=db_session_with_containers)+ get_dataset_doc_form(dataset, session=db_session_with_containers))mock_external_service_dependencies["index_processor"].load.assert_called_once()@@ -555,7 +556,7 @@ class TestCreateSegmentToIndexTask:# Verify index processor was calledmock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- dataset.get_doc_form(session=db_session_with_containers)+ get_dataset_doc_form(dataset, session=db_session_with_containers))mock_external_service_dependencies["index_processor"].load.assert_called_once()@@ -988,7 +989,7 @@ class TestCreateSegmentToIndexTask:# Verify index processor was calledmock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- dataset.get_doc_form(session=db_session_with_containers)+ get_dataset_doc_form(dataset, session=db_session_with_containers))mock_external_service_dependencies["index_processor"].load.assert_called_once()@@ -1064,7 +1065,7 @@ class TestCreateSegmentToIndexTask:# Verify index processor was calledmock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- dataset.get_doc_form(session=db_session_with_containers)+ get_dataset_doc_form(dataset, session=db_session_with_containers))mock_external_service_dependencies["index_processor"].load.assert_called_once()diff --git a/api/tests/test_containers_integration_tests/tasks/test_dataset_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_dataset_indexing_task.pyindex eeb54ec5c6f..0e18308b391 100644--- a/api/tests/test_containers_integration_tests/tasks/test_dataset_indexing_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_dataset_indexing_task.py@@ -9,12 +9,13 @@ from faker import Fakerfrom sqlalchemy import selectfrom sqlalchemy.orm import Session-from core.indexing_runner import DocumentIsPausedErrorfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom enums import CloudPlan, DeploymentEditionfrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DocumentReffrom tasks.document_indexing_task import (_document_indexing,_document_indexing_with_tenant_queue,@@ -84,7 +85,7 @@ def session_close_tracker():def patched_external_dependencies():"""Patch non-DB collaborators while keeping database behavior real."""with (- patch("tasks.document_indexing_task.IndexingRunner", autospec=True) as mock_indexing_runner,+ patch("tasks.document_indexing_task.build_document_indexing_service", autospec=True) as mock_indexing_runner,patch("tasks.document_indexing_task.FeatureService", autospec=True) as mock_feature_service,patch("tasks.document_indexing_task.generate_summary_index_task", autospec=True) as mock_summary_task,):@@ -213,7 +214,7 @@ class TestDatasetIndexingTaskIntegration:assert len(opened) >= 2assert opened_ids <= closed_ids- def _runner_documents_arg(self, patched_external_dependencies) -> Sequence[Document]:+ def _runner_documents_arg(self, patched_external_dependencies) -> Sequence[DocumentRef]:"""Return the document batch passed to the runner."""return patched_external_dependencies["indexing_runner_instance"].run.call_args.args[0]diff --git a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_sync_task.py b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_sync_task.pyindex 77cd2598333..bd3393c510d 100644--- a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_sync_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_sync_task.py@@ -15,11 +15,13 @@ import pytestfrom sqlalchemy import delete, func, select, updatefrom sqlalchemy.orm import Session-from core.indexing_runner import DocumentIsPausedError, IndexingRunnerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.indexing.execution import DocumentIndexingService+from services.knowledge.resource_scope import DatasetReffrom tasks.document_indexing_sync_task import document_indexing_sync_task@@ -133,14 +135,14 @@ class TestDocumentIndexingSyncTask:def mock_external_dependencies(self):"""Patch only external collaborators; keep DB access real."""with (- patch("tasks.document_indexing_sync_task.DatasourceProviderService") as mock_datasource_service_class,+ patch("tasks.document_indexing_sync_task.build_data_source_credentials") as mock_credentials,patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_notion_extractor_class,patch("tasks.document_indexing_sync_task.IndexProcessorFactory") as mock_index_processor_factory,- patch("tasks.document_indexing_sync_task.IndexingRunner") as mock_indexing_runner_class,+ patch("tasks.document_indexing_sync_task.build_document_indexing_service") as mock_indexing_runner_class,):datasource_service = Mock()datasource_service.get_datasource_credentials.return_value = {"integration_secret": "test_token"}- mock_datasource_service_class.return_value = datasource_service+ mock_credentials.return_value.providers = datasource_servicenotion_extractor = Mock()notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z"@@ -150,7 +152,7 @@ class TestDocumentIndexingSyncTask:index_processor.clean = Mock()mock_index_processor_factory.return_value.init_index_processor.return_value = index_processor- indexing_runner = Mock(spec=IndexingRunner)+ indexing_runner = Mock(spec=DocumentIndexingService)indexing_runner.run = Mock()mock_indexing_runner_class.return_value = indexing_runner@@ -339,16 +341,14 @@ class TestDocumentIndexingSyncTask:clean_call_args = mock_external_dependencies["index_processor"].clean.call_argsassert clean_call_args is not Noneclean_args, clean_kwargs = clean_call_args- assert getattr(clean_args[0], "id", None) == context["dataset"].id+ assert clean_args[0].id == context["dataset"].idassert set(clean_args[1]) == set(context["node_ids"])assert clean_kwargs.get("with_keywords") is Trueassert clean_kwargs.get("delete_child_chunks") is True- run_call_args = mock_external_dependencies["indexing_runner"].run.call_args- assert run_call_args is not None- run_documents = run_call_args[0][0]- assert len(run_documents) == 1- assert getattr(run_documents[0], "id", None) == context["document"].id+ mock_external_dependencies["indexing_runner"].run.assert_called_once_with(+ [DatasetRef(context["tenant"].id, context["dataset"].id).document(context["document"].id)]+ )def test_dataset_not_found_during_cleaning(self, db_session_with_containers: Session, mock_external_dependencies):"""Test that task still updates document and reindexes if dataset vanishes before clean."""@@ -458,7 +458,7 @@ class TestDocumentIndexingSyncTask:clean_call_args = mock_external_dependencies["index_processor"].clean.call_argsassert clean_call_args is not Noneclean_args, clean_kwargs = clean_call_args- assert getattr(clean_args[0], "id", None) == context["dataset"].id+ assert clean_args[0].id == context["dataset"].idassert set(clean_args[1]) == set(context["node_ids"])assert clean_kwargs.get("with_keywords") is Trueassert clean_kwargs.get("delete_child_chunks") is Truediff --git a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.pyindex 85b3993b1fd..613bb04e012 100644--- a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py@@ -11,6 +11,7 @@ from enums import CloudPlan, DeploymentEditionfrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus+from services.knowledge.resource_scope import DocumentReffrom tasks.document_indexing_task import (_document_indexing, # Core function_document_indexing_with_tenant_queue, # Tenant queue wrapper function@@ -36,7 +37,9 @@ class TestDocumentIndexingTasks:def mock_external_service_dependencies(self):"""Mock setup for external service dependencies."""with (- patch("tasks.document_indexing_task.IndexingRunner", autospec=True) as mock_indexing_runner,+ patch(+ "tasks.document_indexing_task.build_document_indexing_service", autospec=True+ ) as mock_indexing_runner,patch("tasks.document_indexing_task.FeatureService", autospec=True) as mock_feature_service,):# Setup mock indexing runner@@ -51,7 +54,7 @@ class TestDocumentIndexingTasks:"features": mock_features,}- def _runner_documents_arg(self, mock_external_service_dependencies) -> list[Document]:+ def _runner_documents_arg(self, mock_external_service_dependencies) -> list[DocumentRef]:"""Return the document batch passed to the runner."""return mock_external_service_dependencies["indexing_runner_instance"].run.call_args.args[0]@@ -243,7 +246,7 @@ class TestDocumentIndexingTasks:This test verifies:- Proper dataset retrieval from database- Correct document processing and status updates- - IndexingRunner integration+ - build_document_indexing_service integration- Database state updates"""# Arrange: Create test data@@ -350,10 +353,10 @@ class TestDocumentIndexingTasks:self, db_session_with_containers: Session, mock_external_service_dependencies):"""- Test handling of IndexingRunner exceptions.+ Test handling of build_document_indexing_service exceptions.This test verifies:- - Exceptions from IndexingRunner are properly caught+ - Exceptions from build_document_indexing_service are properly caught- Task completes without raising exceptions- Database session is properly closed- Error logging occurs@@ -364,7 +367,7 @@ class TestDocumentIndexingTasks:)document_ids = [doc.id for doc in documents]- # Mock IndexingRunner to raise an exception+ # Mock build_document_indexing_service to raise an exceptionmock_external_service_dependencies["indexing_runner_instance"].run.side_effect = Exception("Indexing runner failed")@@ -393,7 +396,7 @@ class TestDocumentIndexingTasks:- Documents with different initial states are handled correctly- Only valid documents are processed- Database state updates are consistent- - IndexingRunner receives correct documents+ - build_document_indexing_service receives correct documents"""# Arrange: Create test datadataset, base_documents = self._create_test_dataset_and_documents(@@ -541,7 +544,7 @@ class TestDocumentIndexingTasks:- Processing continues normally outside Cloud- No billing validation occurs- Documents are processed successfully- - IndexingRunner is called correctly+ - build_document_indexing_service is called correctly"""# Arrange: Create test data with billing disableddataset, documents = self._create_test_dataset_with_billing_features(@@ -572,7 +575,7 @@ class TestDocumentIndexingTasks:self, db_session_with_containers: Session, mock_external_service_dependencies):"""- Test handling of DocumentIsPausedError from IndexingRunner.+ Test handling of DocumentIsPausedError from build_document_indexing_service.This test verifies:- DocumentIsPausedError is properly caught and handled@@ -586,8 +589,8 @@ class TestDocumentIndexingTasks:)document_ids = [doc.id for doc in documents]- # Mock IndexingRunner to raise DocumentIsPausedError- from core.indexing_runner import DocumentIsPausedError+ # Mock build_document_indexing_service to raise DocumentIsPausedError+ from services.knowledge.indexing.errors import DocumentIsPausedErrormock_external_service_dependencies["indexing_runner_instance"].run.side_effect = DocumentIsPausedError("Document indexing is paused"@@ -809,7 +812,7 @@ class TestDocumentIndexingTasks:tenant_id = dataset.tenant_iddataset_id = dataset.id- # Mock IndexingRunner to raise an exception+ # Mock build_document_indexing_service to raise an exceptionmock_external_service_dependencies["indexing_runner_instance"].run.side_effect = Exception("Test error")# Mock the task functiondiff --git a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_update_task.py b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_update_task.pyindex bda54389419..55aa5cacfa9 100644--- a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_update_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_update_task.py@@ -9,6 +9,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Indfrom models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from services.knowledge.resource_scope import DatasetReffrom tasks.document_indexing_update_task import document_indexing_update_task@@ -17,11 +18,11 @@ class TestDocumentIndexingUpdateTask:def mock_external_dependencies(self):"""Patch external collaborators used by the update task.- IndexProcessorFactory.init_index_processor().clean(...)- - IndexingRunner.run([...])+ - build_document_indexing_service.run([...])"""with (patch("tasks.document_indexing_update_task.IndexProcessorFactory", autospec=True) as mock_factory,- patch("tasks.document_indexing_update_task.IndexingRunner", autospec=True) as mock_runner,+ patch("tasks.document_indexing_update_task.build_document_indexing_service", autospec=True) as mock_runner,):processor_instance = MagicMock()mock_factory.return_value.init_index_processor.return_value = processor_instance@@ -140,19 +141,16 @@ class TestDocumentIndexingUpdateTask:assert clean_call is not Noneargs, kwargs = clean_call# args[0] is a Dataset instance (from another session), so validate by id.- assert getattr(args[0], "id", None) == dataset.id+ assert args[0].id == dataset.id# args[1] should contain our node_ids.assert set(args[1]) == set(node_ids)assert kwargs.get("with_keywords") is Trueassert kwargs.get("delete_child_chunks") is True- # Assert indexing runner invoked with the updated document- run_call = mock_external_dependencies["runner_instance"].run.call_args- assert run_call is not None- run_docs = run_call[0][0]- assert len(run_docs) == 1- first = run_docs[0]- assert getattr(first, "id", None) == document.id+ # Reindexing receives the detached reference with its complete ownership chain.+ mock_external_dependencies["runner_instance"].run.assert_called_once_with(+ [DatasetRef(dataset.tenant_id, dataset.id).document(document.id)]+ )def test_clean_error_is_logged_and_indexing_continues(self, db_session_with_containers: Session, mock_external_dependenciesdiff --git a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.pyindex 1dacc2bba10..c2fe023cc38 100644--- a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py@@ -5,12 +5,13 @@ from faker import Fakerfrom sqlalchemy import selectfrom sqlalchemy.orm import Session-from core.indexing_runner import DocumentIsPausedErrorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom enums import CloudPlan, DeploymentEditionfrom models import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DocumentReffrom tasks.duplicate_document_indexing_task import (_duplicate_document_indexing_task, # Core function_duplicate_document_indexing_task_with_tenant_queue, # Tenant queue wrapper function@@ -37,7 +38,9 @@ class TestDuplicateDocumentIndexingTasks:def mock_external_service_dependencies(self):"""Mock setup for external service dependencies."""with (- patch("tasks.duplicate_document_indexing_task.IndexingRunner", autospec=True) as mock_indexing_runner,+ patch(+ "tasks.duplicate_document_indexing_task.build_document_indexing_service", autospec=True+ ) as mock_indexing_runner,patch("tasks.duplicate_document_indexing_task.FeatureService", autospec=True) as mock_feature_service,patch("tasks.duplicate_document_indexing_task.IndexProcessorFactory", autospec=True@@ -62,7 +65,7 @@ class TestDuplicateDocumentIndexingTasks:"index_processor": mock_processor,}- def _runner_documents_arg(self, mock_external_service_dependencies) -> list[Document]:+ def _runner_documents_arg(self, mock_external_service_dependencies) -> list[DocumentRef]:"""Return the document batch passed to the runner."""return mock_external_service_dependencies["indexing_runner_instance"].run.call_args.args[0]@@ -311,7 +314,7 @@ class TestDuplicateDocumentIndexingTasks:This test verifies:- Proper dataset retrieval from database- Correct document processing and status updates- - IndexingRunner integration+ - build_document_indexing_service integration- Database state updates"""# Arrange: Create test data@@ -469,10 +472,10 @@ class TestDuplicateDocumentIndexingTasks:self, db_session_with_containers: Session, mock_external_service_dependencies):"""- Test handling of IndexingRunner exceptions.+ Test handling of build_document_indexing_service exceptions.This test verifies:- - Exceptions from IndexingRunner are properly caught+ - Exceptions from build_document_indexing_service are properly caught- Task completes without raising exceptions- Database session is properly closed- Error logging occurs@@ -483,7 +486,7 @@ class TestDuplicateDocumentIndexingTasks:)document_ids = [doc.id for doc in documents]- # Mock IndexingRunner to raise an exception+ # Mock build_document_indexing_service to raise an exceptionmock_external_service_dependencies["indexing_runner_instance"].run.side_effect = Exception("Indexing runner failed")@@ -630,7 +633,7 @@ class TestDuplicateDocumentIndexingTasks:# Act: Execute the task with empty document list_duplicate_document_indexing_task(dataset.id, document_ids)- # Assert: Verify IndexingRunner was called with empty list+ # Assert: Verify build_document_indexing_service was called with empty list# Note: The actual implementation does call run([]) with an empty list.mock_external_service_dependencies["indexing_runner"].assert_called_once()mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()@@ -840,7 +843,7 @@ class TestDuplicateDocumentIndexingTasks:def test_duplicate_document_indexing_runner_error(self, db_session_with_containers: Session, mock_external_service_dependencies):- """Test duplicate document indexing when IndexingRunner raises an error."""+ """Test duplicate document indexing when build_document_indexing_service raises an error."""self._test_duplicate_document_indexing_task_indexing_runner_exception(db_session_with_containers, mock_external_service_dependencies)diff --git a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.pyindex 6d3b90d41c7..525deeca421 100644--- a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py@@ -7,8 +7,8 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.ext_redis import redis_clientfrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole-from models.dataset import Dataset, Document, DocumentSegment-from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, Document, DocumentSegment+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatusfrom tasks.enable_segments_to_index_task import enable_segments_to_index_task@@ -370,7 +370,7 @@ class TestEnableSegmentsToIndexTask:# Refresh dataset to ensure doc_form property reflects the updated documentdb_session_with_containers.refresh(dataset)- # Create segments with mock child chunks+ # Persist each segment and its own child chunkssegments = self._create_test_segments(db_session_with_containers, document, dataset)# Set up Redis cache keys@@ -379,43 +379,62 @@ class TestEnableSegmentsToIndexTask:indexing_cache_key = f"segment_{segment.id}_indexing"redis_client.set(indexing_cache_key, "processing", ex=300)- # Mock the get_child_chunks method for each segment- with patch.object(DocumentSegment, "get_child_chunks", autospec=True) as mock_get_child_chunks:- # Setup mock to return child chunks for each segment- mock_child_chunks = []- for i in range(2): # Each segment has 2 child chunks- mock_child = MagicMock()- mock_child.content = f"child_content_{i}"- mock_child.index_node_id = f"child_node_{i}"- mock_child.index_node_hash = f"child_hash_{i}"- mock_child_chunks.append(mock_child)-- mock_get_child_chunks.return_value = mock_child_chunks+ rule = DatasetProcessRule(+ dataset_id=dataset.id,+ mode=ProcessRuleMode.HIERARCHICAL,+ rules='{"parent_mode": "paragraph"}',+ created_by=document.created_by,+ )+ db_session_with_containers.add(rule)+ db_session_with_containers.flush()+ document.dataset_process_rule_id = rule.id+ for segment in segments:+ db_session_with_containers.add_all(+ [+ ChildChunk(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ segment_id=segment.id,+ position=i,+ content=f"child_content_{i}",+ word_count=1,+ created_by=document.created_by,+ index_node_id=f"{segment.index_node_id}-child-{i}",+ index_node_hash=f"{segment.index_node_id}-hash-{i}",+ )+ for i in range(2)+ ]+ )+ db_session_with_containers.commit()- # Act: Execute the task- enable_segments_to_index_task(segment_ids, dataset.id, document.id)+ # Act: Execute the task+ enable_segments_to_index_task(segment_ids, dataset.id, document.id)- # Assert: Verify parent-child index processing- mock_external_service_dependencies["index_processor_factory"].assert_called_once_with(- IndexStructureType.PARENT_CHILD_INDEX- )- mock_external_service_dependencies["index_processor"].load.assert_called_once()+ # Assert: Verify parent-child index processing+ mock_external_service_dependencies["index_processor_factory"].assert_called_once_with(+ IndexStructureType.PARENT_CHILD_INDEX+ )+ mock_external_service_dependencies["index_processor"].load.assert_called_once()- # Verify the load method was called with correct parameters- call_args = mock_external_service_dependencies["index_processor"].load.call_args- assert call_args is not None- documents = call_args[0][1] # Second argument should be documents list- assert len(documents) == 3 # 3 segments+ # Verify the load method was called with correct parameters+ call_args = mock_external_service_dependencies["index_processor"].load.call_args+ assert call_args is not None+ documents = call_args[0][1] # Second argument should be documents list+ assert len(documents) == 3 # 3 segments- # Verify each document has children- for doc in documents:- assert hasattr(doc, "children")- assert len(doc.children) == 2 # Each document has 2 children+ # Each parent receives only its own persisted children, in position order.+ for doc in documents:+ assert doc.children is not None+ assert [child.page_content for child in doc.children] == ["child_content_0", "child_content_1"]+ assert [child.metadata["doc_id"] for child in doc.children] == [+ f"{doc.metadata['doc_id']}-child-{i}" for i in range(2)+ ]- # Verify Redis cache keys were deleted- for segment in segments:- indexing_cache_key = f"segment_{segment.id}_indexing"- assert redis_client.exists(indexing_cache_key) == 0+ # Verify Redis cache keys were deleted+ for segment in segments:+ indexing_cache_key = f"segment_{segment.id}_indexing"+ assert redis_client.exists(indexing_cache_key) == 0def test_enable_segments_to_index_general_exception_handling(self, db_session_with_containers: Session, mock_external_service_dependenciesdiff --git a/api/tests/unit_tests/controllers/conftest.py b/api/tests/unit_tests/controllers/conftest.pyindex c24a06ff520..d0cdf0fa25f 100644--- a/api/tests/unit_tests/controllers/conftest.py+++ b/api/tests/unit_tests/controllers/conftest.py@@ -1,11 +1,15 @@-"""Explicit App service composition for controller tests that use app queries."""+"""Application dependencies shared by controller tests."""+import sysfrom dataclasses import dataclass+from types import SimpleNamespace+from unittest.mock import create_autospecimport pytestfrom flask import Flaskfrom sqlalchemy.orm import Session, sessionmaker+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom extensions.application_services.app import AppServicesfrom extensions.ext_application_services import (_batch_get_enterprise_webapp_access_modes,@@ -18,10 +22,62 @@ from repositories.webapp_access_query_repository import WebAppAccessQueryRepositfrom services.app.console_service import ConsoleAppServicefrom services.app.import_service import AppImportServicefrom services.app.query_service import AppQueryService+from services.credentials.query import CredentialQuery+from services.data_source.provider_service import DatasourceProviderServicefrom services.tag_application_service import TagApplicationServicefrom services.webapp_access_query_service import WebAppAccessQueryService[email protected](autouse=True)+def datasource_application_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:+ registry = SimpleNamespace(+ data_sources=SimpleNamespace(providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True))+ )+ for name in (+ "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow",+ "controllers.console.datasets.rag_pipeline.datasource_content_preview",+ "controllers.console.datasets.website",+ ):+ module = sys.modules.get(name)+ if module is not None:+ monkeypatch.setattr(module, "application_services", lambda: registry)+++@dataclass(frozen=True)+class PipelineKnowledgeStub:+ pipeline_generator: PipelineGenerator+++@dataclass(frozen=True)+class PipelineDataSourceStub:+ providers: DatasourceProviderService+++@dataclass(frozen=True)+class PipelineApplicationStub:+ knowledge: PipelineKnowledgeStub+ credential_queries: CredentialQuery+ data_sources: PipelineDataSourceStub+++def pipeline_application(monkeypatch: pytest.MonkeyPatch) -> PipelineGenerator:+ from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as console_workflow+ from controllers.service_api.dataset.rag_pipeline import rag_pipeline_workflow as service_api_workflow++ generator = create_autospec(PipelineGenerator, instance=True, spec_set=True)+ registry = PipelineApplicationStub(+ knowledge=PipelineKnowledgeStub(pipeline_generator=generator),+ credential_queries=create_autospec(CredentialQuery, instance=True, spec_set=True),+ data_sources=PipelineDataSourceStub(+ providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True)+ ),+ )+ monkeypatch.setattr(console_workflow, "application_services", lambda: registry)+ monkeypatch.setattr(service_api_workflow, "application_services", lambda: registry)+ return generator++@dataclassclass AppQueryTestServices:console: ConsoleAppServicediff --git a/api/tests/unit_tests/controllers/console/auth/test_data_source_bearer_auth.py b/api/tests/unit_tests/controllers/console/auth/test_data_source_bearer_auth.pyindex 47f25498f3a..b04f8d51451 100644--- a/api/tests/unit_tests/controllers/console/auth/test_data_source_bearer_auth.py+++ b/api/tests/unit_tests/controllers/console/auth/test_data_source_bearer_auth.py@@ -1,8 +1,8 @@from __future__ import annotations+from dataclasses import dataclassfrom datetime import UTC, datetimefrom inspect import unwrap-from types import SimpleNamespacefrom unittest.mock import create_autospec, patchfrom uuid import UUID@@ -23,14 +23,14 @@ from controllers.console.auth.error import ()from libs.exception import BaseHTTPExceptionfrom machinery.context import RequestContext-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthServicefrom services.auth.errors import (DataSourceApiKeyAuthCredentialValidationError,DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,UnsupportedDataSourceApiKeyAuthProviderError,)-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthBindingRecord,DataSourceApiKeyAuthCredentials,@@ -46,8 +46,18 @@ def _request_context() -> RequestContext:)-def _application_services(service: DataSourceApiKeyAuthService) -> SimpleNamespace:- return SimpleNamespace(data_source_api_key_auth=service)+@dataclass(frozen=True)+class _DataSources:+ api_key_auth: DataSourceApiKeyAuthService+++@dataclass(frozen=True)+class _ApplicationServices:+ data_sources: _DataSources+++def _application_services(service: DataSourceApiKeyAuthService) -> _ApplicationServices:+ return _ApplicationServices(data_sources=_DataSources(api_key_auth=service))def test_list_data_source_auth_passes_context_and_serializes_application_result() -> None:diff --git a/api/tests/unit_tests/controllers/console/auth/test_data_source_oauth.py b/api/tests/unit_tests/controllers/console/auth/test_data_source_oauth.pyindex 72d28c0385b..e47876a7c2e 100644--- a/api/tests/unit_tests/controllers/console/auth/test_data_source_oauth.py+++ b/api/tests/unit_tests/controllers/console/auth/test_data_source_oauth.py@@ -13,12 +13,12 @@ from controllers.console.auth.data_source_oauth import ()from extensions.ext_application_services import ApplicationServicesfrom machinery.context import RequestContext-from services.data_source_oauth_service import (+from services.data_source.entities.oauth import DataSourceOAuthCallback+from services.data_source.oauth_service import (DataSourceOAuthError,InvalidDataSourceOAuthCodeError,InvalidDataSourceOAuthProviderError,)-from services.entities.data_source_oauth_entities import DataSourceOAuthCallbackfrom tests.unit_tests.config_override import config_overrides_context@@ -33,7 +33,7 @@ def _request_context() -> RequestContext:def _services(service: MagicMock) -> MagicMock:services = create_autospec(ApplicationServices, instance=True, spec_set=True)- services.resolve_data_source_oauth.return_value = service+ services.data_sources.resolve_oauth.return_value = servicereturn services@@ -55,7 +55,7 @@ def test_get_delegates_to_application_service_and_serializes_response() -> None:def test_get_maps_unknown_provider_to_bad_request() -> None:service = MagicMock()services = _services(service)- services.resolve_data_source_oauth.side_effect = InvalidDataSourceOAuthProviderError+ services.data_sources.resolve_oauth.side_effect = InvalidDataSourceOAuthProviderErrorwith patch("controllers.console.auth.data_source_oauth.application_services",diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_auth.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_auth.pyindex c1cbbaa8682..4874145451d 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_auth.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_auth.py@@ -1,5 +1,7 @@import inspectfrom datetime import UTC, datetime+from importlib import import_module+from types import SimpleNamespacefrom unittest.mock import ANY, MagicMock, patchimport pytest@@ -28,13 +30,26 @@ from controllers.console.datasets.rag_pipeline.datasource_auth import (from core.plugin.impl.oauth import OAuthHandlerfrom graphon.model_runtime.errors.validate import CredentialsValidateFailedErrorfrom models.account import Account-from services.datasource_provider_service import DatasourceProviderService+from services.credentials.query import CredentialQuery+from services.data_source.credential_gateway import DatasourceProviderCredentialStore+from services.data_source.provider_service import DatasourceProviderServicefrom services.plugin.oauth_service import OAuthProxyServicefrom tests.unit_tests.model_factories import make_account_PROVIDER_ID = "langgenius/notion_datasource/notion"[email protected](autouse=True)+def credential_query_dependency(monkeypatch: pytest.MonkeyPatch) -> None:+ query = MagicMock(spec=CredentialQuery)+ providers = DatasourceProviderService(credentials=MagicMock(spec=DatasourceProviderCredentialStore))+ monkeypatch.setattr(+ import_module("controllers.console.datasets.rag_pipeline.datasource_auth"),+ "application_services",+ lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)),+ )++def _account() -> Account:return make_account(account_id="user-1", name="Datasource Auth Tester", email="[email protected]")diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_catalog.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_catalog.pyindex 08fcc368967..cc2029a4a75 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_catalog.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_catalog.py@@ -5,6 +5,7 @@ import pytestfrom flask import Flaskfrom pydantic import ValidationError+from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as workflow_modulefrom controllers.console.datasets.rag_pipeline.rag_pipeline_workflow import (DatasourceListApi,RagPipelineDatasourceListResponse,@@ -14,7 +15,6 @@ from core.datasource.entities.datasource_entities import DatasourceParameterfrom core.plugin.entities import OAuthSchemafrom core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntityfrom core.plugin.impl.datasource import PluginDatasourceManager-from services.datasource_provider_service import DatasourceProviderServicefrom services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService@@ -89,7 +89,9 @@ def test_catalog_serializes_real_plugin_and_local_file_providers(PluginDatasourceManager, "_request_with_plugin_daemon_response", return_value=[provider]) as request,patch.object(- DatasourceProviderService, "get_datasource_credentials", return_value=credentials+ workflow_module.application_services().data_sources.providers,+ "get_datasource_credentials",+ return_value=credentials,) as get_credentials,):result = unwrap(api.get)(api, "tenant-1")@@ -138,7 +140,11 @@ def test_catalog_keeps_oauth_only_provider_unauthorized_without_credentials(app:with (app.test_request_context("/rag/pipelines/datasource-plugins"),patch.object(PluginDatasourceManager, "fetch_datasource_providers", return_value=[provider]),- patch.object(DatasourceProviderService, "get_datasource_credentials", return_value=None) as get_credentials,+ patch.object(+ workflow_module.application_services().data_sources.providers,+ "get_datasource_credentials",+ return_value=None,+ ) as get_credentials,):result = unwrap(api.get)(api, "tenant-1")@@ -154,7 +160,11 @@ def test_catalog_keeps_provider_visible_after_credential_failure(app: Flask) ->with (app.test_request_context("/rag/pipelines/datasource-plugins"),patch.object(PluginDatasourceManager, "fetch_datasource_providers", return_value=[provider]),- patch.object(DatasourceProviderService, "get_datasource_credentials", side_effect=ValueError("decrypt failed")),+ patch.object(+ workflow_module.application_services().data_sources.providers,+ "get_datasource_credentials",+ side_effect=ValueError("decrypt failed"),+ ),):result = unwrap(api.get)(api, "tenant-1")diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.pyindex b3aed7be87c..daae29fd75c 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py@@ -1,5 +1,5 @@from inspect import unwrap-from unittest.mock import MagicMock, patch+from unittest.mock import ANY, MagicMock, patchimport pytestfrom flask import Flask@@ -61,6 +61,7 @@ class TestDataSourceContentPreviewApi:datasource_type=payload["datasource_type"],is_published=True,credential_id=payload["credential_id"],+ datasource_providers=ANY,)assert status == 200assert response == preview_resultdiff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.pyindex 53445379c5c..36032f3650c 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.py@@ -375,7 +375,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=True),- patch.object(Pipeline, "retrieve_dataset", return_value=dataset),+ patch.object(module, "get_pipeline_dataset", return_value=dataset),patch.object(module.DatasetService, "check_dataset_permission") as legacy_acl,patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,):@@ -408,7 +408,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=True),- patch.object(Pipeline, "retrieve_dataset", return_value=dataset),+ patch.object(module, "get_pipeline_dataset", return_value=dataset),patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,):response, status = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)@@ -424,7 +424,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=True),- patch.object(Pipeline, "retrieve_dataset", return_value=object()),+ patch.object(module, "get_pipeline_dataset", return_value=object()),patch.object(module.RagPipelineService,"publish_customized_pipeline_template",@@ -445,7 +445,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=False),- patch.object(Pipeline, "retrieve_dataset", return_value=dataset),+ patch.object(module, "get_pipeline_dataset", return_value=dataset),patch.object(module.DatasetService, "check_dataset_permission") as check_permission,patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,):@@ -463,7 +463,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=False),- patch.object(Pipeline, "retrieve_dataset", return_value=object()),+ patch.object(module, "get_pipeline_dataset", return_value=object()),patch.object(module.DatasetService, "check_dataset_permission") as check_permission,patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,pytest.raises(Forbidden),@@ -481,7 +481,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=False),- patch.object(Pipeline, "retrieve_dataset", return_value=object()),+ patch.object(module, "get_pipeline_dataset", return_value=object()),patch.object(module.DatasetService,"check_dataset_permission",@@ -502,7 +502,7 @@ class TestPublishCustomizedPipelineTemplateApi:with (config_overrides_context(RBAC_ENABLED=False),- patch.object(Pipeline, "retrieve_dataset", return_value=None),+ patch.object(module, "get_pipeline_dataset", return_value=None),patch.object(module.DatasetService, "check_dataset_permission") as check_permission,patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,pytest.raises(NotFound, match="Dataset not found"),diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.pyindex 294e0c04f35..8794c7d7b6a 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py@@ -6,7 +6,7 @@ import jsonfrom collections.abc import Iteratorfrom datetime import datetimefrom inspect import unwrap as unwrap_all-from unittest.mock import MagicMock, patch+from unittest.mock import patchfrom uuid import UUIDimport pytest@@ -25,6 +25,7 @@ from controllers.console.datasets.rag_pipeline.rag_pipeline_workflow import (WorkflowUpdatePayload,)from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom models.account import Account, TenantAccountRolefrom models.dataset import Dataset, Pipelinefrom models.engine import db@@ -108,9 +109,9 @@ def database_app() -> Iterator[Flask]:db.init_app(app)with app.app_context():- Account.__table__.create(db.engine)- WorkflowToolProvider.__table__.create(db.engine)- Workflow.__table__.create(db.engine)+ db.metadata.tables[Account.__tablename__].create(db.engine)+ db.metadata.tables[WorkflowToolProvider.__tablename__].create(db.engine)+ db.metadata.tables[Workflow.__tablename__].create(db.engine)db.session.add(_account())db.session.commit()@@ -387,6 +388,7 @@ def test_rag_pipeline_transform_skips_legacy_acl_when_rbac_is_enabled(sqlite_engdef test_rag_pipeline_run_uses_sqlite_session(app: Flask,sqlite_engine: Engine,+ pipeline_application: PipelineGenerator,api_type: type,payload: dict[str, object],) -> None:@@ -398,7 +400,7 @@ def test_rag_pipeline_run_uses_sqlite_session(Session(sqlite_engine) as session,app.test_request_context("/", json=payload),patch.object(module, "load_rag_pipeline", return_value=pipeline) as load_pipeline,- patch.object(module.PipelineGenerateService, "generate", return_value=MagicMock()) as generate,+ patch.object(module.PipelineGenerateService, "generate", return_value={"event": "done"}) as generate,patch.object(module.helper, "compact_generate_response", return_value={"ok": True}),):req_data = (@@ -411,6 +413,7 @@ def test_rag_pipeline_run_uses_sqlite_session(assert response == {"ok": True}load_pipeline.assert_called_once_with(session, pipeline.id)assert generate.call_args.kwargs["session"] is session+ assert generate.call_args.kwargs["generator"] is pipeline_applicationassert session.get_bind() is sqlite_engine@@ -418,9 +421,10 @@ def test_rag_pipeline_run_uses_sqlite_session(def test_rag_pipeline_run_translates_rate_limit(app: Flask,sqlite_engine: Engine,+ pipeline_application: PipelineGenerator,api_type: type,) -> None:- payload = {+ payload: dict[str, object] = {"inputs": {},"datasource_type": "x","datasource_info_list": [],@@ -439,7 +443,10 @@ def test_rag_pipeline_run_translates_rate_limit(Session(sqlite_engine) as session,app.test_request_context("/", json=payload),patch.object(module, "load_rag_pipeline", return_value=pipeline),- patch.object(module.PipelineGenerateService, "generate", side_effect=InvokeRateLimitError("limit")),+ patch.object(module.PipelineGenerateService, "generate", side_effect=InvokeRateLimitError("limit")) as generate,pytest.raises(InvokeRateLimitHttpError),):handler(api, req_data, session, _account(), pipeline.id)++ assert generate.call_args.kwargs["session"] is session+ assert generate.call_args.kwargs["generator"] is pipeline_applicationdiff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.pyindex fd26d0ec33f..9226d33433b 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py@@ -54,6 +54,9 @@ from models.enums import CreatorUserRolefrom models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFromfrom services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError+pytestmark = pytest.mark.usefixtures("pipeline_application")++DEFAULT_WORKFLOW_TENANT_ID = "00000000-0000-0000-0000-000000000001"DEFAULT_WORKFLOW_APP_ID = "00000000-0000-0000-0000-000000000002"DEFAULT_WORKFLOW_CREATED_BY = "00000000-0000-0000-0000-000000000003"diff --git a/api/tests/unit_tests/controllers/console/datasets/test_application_boundaries.py b/api/tests/unit_tests/controllers/console/datasets/test_application_boundaries.pynew file mode 100644index 00000000000..9a459f032c8--- /dev/null+++ b/api/tests/unit_tests/controllers/console/datasets/test_application_boundaries.py@@ -0,0 +1,62 @@+"""Guard all migrated dataset endpoints against transport/persistence regressions."""++import ast+from pathlib import Path++import pytest+++ ("filename", "count"),+ [("data_source.py", 7), ("datasets.py", 20), ("datasets_document.py", 24), ("external.py", 8)],+)+def test_every_endpoint_declares_admission_context_and_application_service(filename: str, count: int) -> None:+ root = Path(__file__).resolve().parents[5]+ tree = ast.parse((root / "controllers/console/datasets" / filename).read_text())+ # This namespace package is not traversed by import-linter's package graph.+ forbidden = (+ "sqlalchemy",+ "repositories",+ "controllers.common.session",+ "libs.login",+ "services.knowledge.dataset_service",+ "services.knowledge.external.service",+ "services.hit_testing_service",+ "services.knowledge.dataset_read_service",+ "services.knowledge.datasets.adapters",+ "services.knowledge.documents.adapters",+ "services.knowledge.external.adapters",+ )+ for node in ast.walk(tree):+ modules = (+ [node.module or ""]+ if isinstance(node, ast.ImportFrom)+ else [item.name for item in node.names]+ if isinstance(node, ast.Import)+ else []+ )+ for module in modules:+ assert not any(module == name or module.startswith(name + ".") for name in forbidden), module+ endpoints = [+ method+ for resource in tree.body+ if isinstance(resource, ast.ClassDef)+ for method in resource.body+ if isinstance(method, ast.FunctionDef) and method.name in {"get", "post", "patch", "put", "delete"}+ ]+ assert len(endpoints) == count+ for endpoint in endpoints:+ assert any(+ isinstance(d, ast.Call) and ast.unparse(d.func) == "console_account_admission"+ for d in endpoint.decorator_list+ ), endpoint.name+ assert any(+ arg.arg == "request_context"+ and arg.annotation is not None+ and ast.unparse(arg.annotation) == "RequestContext"+ for arg in endpoint.args.args+ ), endpoint.name+ assert any(+ isinstance(n, ast.Call) and ast.unparse(n.func) == "application_services" for n in ast.walk(endpoint)+ ), endpoint.name+ assert not {"session", "current_user", "current_tenant_id"}.intersection(arg.arg for arg in endpoint.args.args)diff --git a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py b/api/tests/unit_tests/controllers/console/datasets/test_data_source.pyindex 009c48f1ad5..b5463712cbf 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py@@ -1,302 +1,597 @@from __future__ import annotations-import inspect-from collections.abc import Callable, Iterator+from collections.abc import Callable+from dataclasses import dataclass, fieldfrom datetime import UTC, datetime-from typing import Literal, cast-from unittest.mock import MagicMock, PropertyMock, patch-from uuid import UUID+from inspect import unwrap+from typing import TYPE_CHECKING, castimport pytestfrom flask import Flask-from sqlalchemy import select-from sqlalchemy.orm import Session-from werkzeug.exceptions import NotFound+from flask.testing import FlaskClient+from flask_restx import Api++from controllers.common.errors import NotFoundError+from controllers.common.rbac import DatasetId+from controllers.console import console_ns+from controllers.console.app.error import ProviderNotInitializeError++# Importing the module registers only the resources under test on console_ns.+from controllers.console.datasets import data_source as data_source_controller+from controllers.console.datasets.error import IndexingEstimateError+from core.entities.knowledge_entities import IndexingEstimate+from core.rbac import RBACPermission+from enums import DeploymentEdition+from libs.login import AccountWithTenant+from machinery.context import RequestContext+from services.data_source.binding_application_service import (+ BindingMutationResult,+ DataSourceBindingApplicationService,+)+from services.data_source.entities.notion_import import (+ AuthorizedNotionPage,+ NotionPageIcon,+ NotionPageType,+ NotionWorkspace,+)+from services.data_source.entities.oauth import DataSourceBindingSummary+from services.data_source.notion_import_application_service import NotionImportApplicationService+from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessService, DatasetAccessSnapshot+from services.knowledge.document_sync import DocumentSyncApplicationService, SyncDocumentRecord+from services.knowledge.entities.indexing_estimate import (+ NewSourcesEstimateCommand,+ NotionEstimateSource,+)+from services.knowledge.indexing.estimate import (+ DatasetEstimateRecord,+ EstimateDocumentRecord,+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+)+from services.knowledge.resource_scope import DatasetRef, DocumentRef+from tests.unit_tests.controllers.rbac_introspection import rbac_checks-from controllers.console.datasets import data_source as module-from controllers.console.datasets.data_source import DataSourceApi, DataSourceNotionListApi, DataSourceNotionListQuery-from models import Account, DataSourceOauthBinding-from models.dataset import Dataset-from models.engine import db-from models.enums import DataSourceType+if TYPE_CHECKING:+ from models import Account-ControllerMethod = Callable[..., tuple[dict[str, object], int]]+TENANT_ID = "11111111-1111-1111-1111-111111111111"+ACCOUNT_ID = "22222222-2222-2222-2222-222222222222"+BINDING_ID = "33333333-3333-3333-3333-333333333333"+DATASET_ID = "44444444-4444-4444-4444-444444444444"+DOCUMENT_ID = "55555555-5555-5555-5555-555555555555"+PAGE_ID = "66666666-6666-6666-6666-666666666666"+FOREIGN_DATASET_ID = "77777777-7777-7777-7777-777777777777"+++@dataclass+class RecordingBindingStore:+ bindings: tuple[DataSourceBindingSummary, ...] = ()+ mutations: list[tuple[str, str, bool]] = field(default_factory=list)+ mutation_result: BindingMutationResult = "updated"++ def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:+ assert workspace_id == TENANT_ID+ return self.bindings++ def change_disabled_state(+ self,+ *,+ workspace_id: str,+ binding_id: str,+ disabled: bool,+ ) -> BindingMutationResult:+ self.mutations.append((workspace_id, binding_id, disabled))+ return self.mutation_result+++@dataclass+class DatasetCatalog:+ record: DatasetAccessRecord++ def get_access_snapshot(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ actor_id: str,+ ) -> DatasetAccessSnapshot | None:+ assert actor_id == ACCOUNT_ID+ if workspace_id != self.record.workspace_id or dataset_id != self.record.id:+ return None+ return DatasetAccessSnapshot(dataset=self.record, actor_has_partial_access=False)++ def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:+ if dataset_ref != DatasetRef(self.record.workspace_id, self.record.id):+ return None+ return DatasetEstimateRecord(+ id=self.record.id,+ workspace_id=self.record.workspace_id,+ indexing_technique="economy",+ )+ def is_notion_dataset(self, dataset_ref: DatasetRef) -> bool:+ return dataset_ref == DatasetRef(self.record.workspace_id, self.record.id)+++class OwnerWorkspaceRoles:+ def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None:+ assert workspace_id == TENANT_ID+ assert account_id == ACCOUNT_ID+ return "owner"+++@dataclass+class KnowledgeDocuments:+ bound_page_ids: frozenset[str] = frozenset()+ active_refs: tuple[DocumentRef, ...] = ()++ def list_bound_notion_page_ids(self, dataset_ref: DatasetRef) -> frozenset[str]:+ assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)+ return self.bound_page_ids++ def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]:+ assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)+ return self.active_refs++ def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None:+ if document_ref.document_id != DOCUMENT_ID:+ return None+ return EstimateDocumentRecord(+ id=DOCUMENT_ID,+ workspace_id=TENANT_ID,+ dataset_id=DATASET_ID,+ data_source_type="notion_import",+ data_source_info=None,+ indexing_status="waiting",+ doc_form="text_model",+ doc_language="English",+ dataset_process_rule_id=None,+ )-def unwrap(func: object) -> ControllerMethod:- return cast(ControllerMethod, inspect.unwrap(cast(Callable[..., object], func)))+ def list_estimate_documents_by_batch(+ self, dataset_ref: DatasetRef, batch: str+ ) -> tuple[EstimateDocumentRecord, ...]:+ document = self.get_estimate_document(dataset_ref.document(DOCUMENT_ID))+ if document is None or batch != "batch":+ return ()+ return (document,)+ def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None:+ if document_ref.document_id != DOCUMENT_ID:+ return None+ return SyncDocumentRecord(id=DOCUMENT_ID, data_source_type="notion_import")-def flask_app() -> Iterator[Flask]:- app = Flask(__name__)- app.config["TESTING"] = True- app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"- db.init_app(app)- with app.app_context():- DataSourceOauthBinding.__table__.create(db.engine)- yield app+@dataclass+class RecordingNotionSource:+ workspaces: tuple[NotionWorkspace, ...]+ list_calls: list[tuple[str, str, str]] = field(default_factory=list)+ preview_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)+ def list_authorized_pages(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ ) -> tuple[NotionWorkspace, ...]:+ self.list_calls.append((workspace_id, actor_id, credential_id))+ return self.workspaces-def current_user() -> Account:- account = Account(name="Test User", email="[email protected]")- account.id = "user-1"- return account+ def preview_page(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ page_id: str,+ page_type: str,+ ) -> str:+ self.preview_calls.append((workspace_id, actor_id, credential_id, page_id, page_type))+ return "preview text"-TENANT_ID = "11111111-1111-1111-1111-111111111111"-BINDING_ID = "22222222-2222-2222-2222-222222222222"+@dataclass+class RecordingEstimateService:+ calls: list[tuple[RequestContext, NewSourcesEstimateCommand]] = field(default_factory=list)+ error: Exception | None = None+ def estimate_new_sources(self, context: RequestContext, command: NewSourcesEstimateCommand) -> IndexingEstimate:+ self.calls.append((context, command))+ if self.error is not None:+ raise self.error+ return IndexingEstimate(total_segments=3, preview=[])-def _add_binding(session: Session, *, disabled: bool) -> DataSourceOauthBinding:- binding = DataSourceOauthBinding(- tenant_id=TENANT_ID,- access_token="token",- provider="notion",- source_info={- "workspace_name": "Workspace",- "workspace_id": "workspace-1",- "workspace_icon": None,- "total": 1,- "pages": [- {- "page_id": "page-1",- "page_name": "Page",- "page_icon": {"type": "emoji", "emoji": "P", "url": None},- "parent_id": "parent-1",- "type": "page",- }- ],- },- disabled=disabled,- )- binding.id = BINDING_ID- binding.created_at = datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC)- session.add(binding)- session.commit()- return binding+@dataclass+class RecordingDispatcher:+ calls: list[tuple[str, str]] = field(default_factory=list)-def test_get_data_source_integrates_serializes_orm_binding(- flask_app: Flask,-) -> None:- binding = _add_binding(db.session, disabled=False)- expected_created_at = int(binding.created_at.timestamp())-- with flask_app.test_request_context("/"):- response, status = unwrap(DataSourceApi().get)(DataSourceApi(), TENANT_ID)-- assert status == 200- assert response == {- "data": [- {- "id": BINDING_ID,- "provider": "notion",- "created_at": expected_created_at,- "is_bound": True,- "disabled": False,- "source_info": {- "workspace_name": "Workspace",- "workspace_id": "workspace-1",- "workspace_icon": None,- "pages": [- {- "page_name": "Page",- "page_id": "page-1",- "page_icon": {"type": "emoji", "url": None, "emoji": "P"},- "parent_id": "parent-1",- "type": "page",- }- ],- "total": 1,- },- "link": "http://localhost/console/api/oauth/data-source/notion",- }- ]- }+ def dispatch(self, *, dataset_id: str, document_id: str) -> None:+ self.calls.append((dataset_id, document_id))-def test_get_data_source_integrates_preserves_empty_list_when_no_binding(- flask_app: Flask,-) -> None:- with flask_app.test_request_context("/"):- response, status = unwrap(DataSourceApi().get)(DataSourceApi(), TENANT_ID)+@dataclass(frozen=True)+class DataSourceApplicationServices:+ bindings: DataSourceBindingApplicationService+ notion_imports: NotionImportApplicationService+++@dataclass(frozen=True)+class KnowledgeApplicationServices:+ document_sync: DocumentSyncApplicationService+ indexing_estimates: RecordingEstimateService+++@dataclass(frozen=True)+class ApplicationServiceRegistry:+ data_sources: DataSourceApplicationServices+ knowledge: KnowledgeApplicationServices- assert status == 200- assert response == {"data": []}++@dataclass(frozen=True)+class ControllerCollaborators:+ client: FlaskClient+ account: AdmittedAccount+ bindings: RecordingBindingStore+ notion_source: RecordingNotionSource+ estimates: RecordingEstimateService+ dispatcher: RecordingDispatcher+++@dataclass+class AdmittedAccount:+ id: str+ role: str+ status: str+++def collaborators(+ config_overrides: Callable[..., None],+ monkeypatch: pytest.MonkeyPatch,+) -> ControllerCollaborators:+ config_overrides(+ DEPLOYMENT_EDITION=DeploymentEdition.CLOUD,+ LOGIN_DISABLED=True,+ RBAC_ENABLED=False,+ CONSOLE_API_URL="https://console-api.example.com",+ )+ app = Flask(__name__)+ app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)+ Api(app).add_namespace(console_ns, path="/console/api")++ account = AdmittedAccount(id=ACCOUNT_ID, role="owner", status="active")+ account_with_tenant = AccountWithTenant(account=cast("Account", account), tenant_id=TENANT_ID)+ monkeypatch.setattr(+ "controllers.console.flask_admission.current_account_with_tenant",+ lambda: account_with_tenant,+ )+ monkeypatch.setattr(+ "controllers.console.wraps.current_account_with_tenant",+ lambda: account_with_tenant,+ )++ dataset = DatasetAccessRecord(+ id=DATASET_ID,+ workspace_id=TENANT_ID,+ maintainer_id=ACCOUNT_ID,+ permission="only_me",+ )+ dataset_catalog = DatasetCatalog(dataset)+ dataset_access = DatasetAccessService(+ datasets=dataset_catalog,+ workspace_roles=OwnerWorkspaceRoles(),+ legacy_permissions_enabled=True,+ )+ documents = KnowledgeDocuments(+ bound_page_ids=frozenset({PAGE_ID}),+ active_refs=(DatasetRef(TENANT_ID, DATASET_ID).document(DOCUMENT_ID),),+ )+ bindings = RecordingBindingStore()+ notion_source = RecordingNotionSource(+ workspaces=(+ NotionWorkspace(+ workspace_id="notion-workspace-1",+ workspace_name="One",+ workspace_icon=None,+ pages=(+ AuthorizedNotionPage(+ page_id=PAGE_ID,+ page_name="Page one",+ page_icon=NotionPageIcon(type="emoji", emoji="📄"),+ parent_id=None,+ page_type=NotionPageType.PAGE,+ ),+ ),+ ),+ NotionWorkspace(+ workspace_id="notion-workspace-2",+ workspace_name="Two",+ workspace_icon=None,+ pages=(),+ ),+ )+ )+ estimates = RecordingEstimateService()+ dispatcher = RecordingDispatcher()+ app.extensions["application_services"] = ApplicationServiceRegistry(+ data_sources=DataSourceApplicationServices(+ bindings=DataSourceBindingApplicationService(bindings=bindings),+ notion_imports=NotionImportApplicationService(+ dataset_access=dataset_access,+ datasets=dataset_catalog,+ documents=documents,+ source=notion_source,+ ),+ ),+ knowledge=KnowledgeApplicationServices(+ document_sync=DocumentSyncApplicationService(+ dataset_access=dataset_access,+ documents=documents,+ dispatcher=dispatcher,+ ),+ indexing_estimates=estimates,+ ),+ )+ return ControllerCollaborators(+ client=app.test_client(),+ account=account,+ bindings=bindings,+ notion_source=notion_source,+ estimates=estimates,+ dispatcher=dispatcher,+ )+++def test_integration_collection_only_exposes_get(collaborators: ControllerCollaborators) -> None:+ client = collaborators.client+ assert client.get("/console/api/data-source/integrates").status_code == 200+ assert client.patch("/console/api/data-source/integrates").status_code == 405@pytest.mark.parametrize(- ("disabled", "action", "expected_disabled"),- [(True, "enable", False), (False, "disable", True)],+ ("method", "path"),+ [+ ("get", "/console/api/data-source/integrates"),+ ("patch", f"/console/api/data-source/integrates/{BINDING_ID}/enable"),+ ],)[email protected]("sqlite_session", [(DataSourceOauthBinding,)], indirect=True)-def test_patch_data_source_binding_updates_state(- flask_app: Flask,- sqlite_session: Session,- disabled: bool,- action: Literal["enable", "disable"],- expected_disabled: bool,+def test_integration_management_rejects_non_admin_role(+ collaborators: ControllerCollaborators,+ method: str,+ path: str,) -> None:- _add_binding(sqlite_session, disabled=disabled)- sqlite_session.expunge_all()+ collaborators.account.role = "editor"- with flask_app.test_request_context("/"):- response, status = unwrap(DataSourceApi().patch)(- DataSourceApi(), sqlite_session, TENANT_ID, UUID(BINDING_ID), action- )-- sqlite_session.flush()- sqlite_session.expire_all()- binding = sqlite_session.scalar(select(DataSourceOauthBinding).where(DataSourceOauthBinding.id == BINDING_ID))- assert status == 200- assert response == {"result": "success"}- assert binding is not None- assert binding.disabled is expected_disabled+ assert collaborators.client.open(path, method=method.upper()).status_code == 403[email protected]("sqlite_session", [(DataSourceOauthBinding,)], indirect=True)-def test_patch_data_source_binding_rejects_unknown_binding(- flask_app: Flask,- sqlite_session: Session,+def test_integration_management_enforces_workspace_credential_permission(+ collaborators: ControllerCollaborators,+ config_overrides: Callable[..., None],+ monkeypatch: pytest.MonkeyPatch,) -> None:- with flask_app.test_request_context("/"), pytest.raises(NotFound, match="Data source binding not found"):- unwrap(DataSourceApi().patch)(DataSourceApi(), sqlite_session, TENANT_ID, UUID(BINDING_ID), "enable")+ config_overrides(RBAC_ENABLED=True)+ calls: list[dict[str, object]] = []++ def record_access(tenant_id: str, account_id: str, **requirement: object) -> bool:+ calls.append({"tenant_id": tenant_id, "account_id": account_id, **requirement})+ return True++ monkeypatch.setattr("controllers.common.rbac.checks.RBACService.CheckAccess.check", record_access)++ list_response = collaborators.client.get("/console/api/data-source/integrates")+ mutation_response = collaborators.client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/enable")++ assert list_response.status_code == 200+ assert mutation_response.status_code == 200+ assert calls == [+ {+ "tenant_id": TENANT_ID,+ "account_id": ACCOUNT_ID,+ "scene": RBACPermission.CREDENTIAL_MANAGE,+ "resource_type": None,+ "resource_id": None,+ },+ {+ "tenant_id": TENANT_ID,+ "account_id": ACCOUNT_ID,+ "scene": RBACPermission.CREDENTIAL_MANAGE,+ "resource_type": None,+ "resource_id": None,+ },+ ][email protected](("disabled", "action"), [(False, "enable"), (True, "disable")])[email protected]("sqlite_session", [(DataSourceOauthBinding,)], indirect=True)-def test_patch_data_source_binding_rejects_current_state(- flask_app: Flask,- sqlite_session: Session,[email protected](("action", "disabled"), [("enable", False), ("disable", True)])+def test_integration_item_only_exposes_patch_and_rejects_unknown_action(+ collaborators: ControllerCollaborators,+ action: str,disabled: bool,- action: Literal["enable", "disable"],) -> None:- _add_binding(sqlite_session, disabled=disabled)- sqlite_session.expunge_all()+ client = collaborators.client+ item_path = f"/console/api/data-source/integrates/{BINDING_ID}/{action}"+ assert client.get(item_path).status_code == 405+ response = client.patch(item_path)+ assert response.status_code == 200+ assert response.get_json() == {"result": "success"}+ assert collaborators.bindings.mutations == [(TENANT_ID, BINDING_ID, disabled)]+ invalid = client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/remove")+ assert invalid.status_code == 400+ assert invalid.get_json()["code"] == "invalid_action"+ assert len(collaborators.bindings.mutations) == 1+++def test_integration_list_serializes_application_result_and_http_link(+ collaborators: ControllerCollaborators,+) -> None:+ collaborators.bindings.bindings = (+ DataSourceBindingSummary(+ id=BINDING_ID,+ provider="notion",+ created_at=datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC),+ disabled=False,+ source_info={+ "workspace_name": "Workspace",+ "workspace_id": "workspace-1",+ "workspace_icon": None,+ "pages": [],+ "total": 0,+ },+ ),+ )- with flask_app.test_request_context("/"), pytest.raises(ValueError):- unwrap(DataSourceApi().patch)(DataSourceApi(), sqlite_session, TENANT_ID, UUID(BINDING_ID), action)+ response = collaborators.client.get("/console/api/data-source/integrates")++ assert response.status_code == 200+ assert response.get_json()["data"][0] == {+ "id": BINDING_ID,+ "provider": "notion",+ "created_at": 1779670923,+ "is_bound": True,+ "disabled": False,+ "source_info": {+ "workspace_name": "Workspace",+ "workspace_id": "workspace-1",+ "workspace_icon": None,+ "pages": [],+ "total": 0,+ },+ "link": "https://console-api.example.com/console/api/oauth/data-source/notion",+ }[email protected]("sqlite_session", [()], indirect=True)-def test_notion_pre_import_pages_serializes_frontend_list_shape(- flask_app: Flask,- current_user: Account,- sqlite_session: Session,+def test_notion_list_preserves_workspace_groups_and_bound_state(collaborators: ControllerCollaborators) -> None:+ response = collaborators.client.get(+ "/console/api/notion/pre-import/pages",+ query_string={"credential_id": "credential-1", "dataset_id": DATASET_ID},+ )++ assert response.status_code == 200+ payload = response.get_json()["notion_info"]+ assert [workspace["workspace_id"] for workspace in payload] == [+ "notion-workspace-1",+ "notion-workspace-2",+ ]+ assert payload[0]["pages"][0]["is_bound"] is True+ assert payload[0]["pages"][0]["type"] == "page"+ assert "page_type" not in payload[0]["pages"][0]+ assert collaborators.notion_source.list_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1")]++[email protected]("page_type", ["page", "database"])+def test_notion_preview_validates_page_type_and_delegates(+ collaborators: ControllerCollaborators, page_type: str) -> None:- page = MagicMock(- page_id="page-1",- page_name="Page",- type="page",- parent_id="parent-1",- page_icon={"type": "emoji", "emoji": "P", "url": None},+ response = collaborators.client.get(+ f"/console/api/notion/pages/{PAGE_ID}/{page_type}/preview",+ query_string={"credential_id": "credential-1"},)- online_document_message = MagicMock(- result=[- MagicMock(- workspace_id="workspace-1",- workspace_name="Workspace",- workspace_icon=None,- pages=[page],- )- ]+ invalid = collaborators.client.get(+ f"/console/api/notion/pages/{PAGE_ID}/collection/preview",+ query_string={"credential_id": "credential-1"},)- runtime = MagicMock(- get_online_document_pages=MagicMock(return_value=iter([online_document_message])),- datasource_provider_type=MagicMock(return_value="online_document"),++ assert response.status_code == 200+ assert response.get_json() == {"content": "preview text"}+ assert invalid.status_code == 400+ assert invalid.get_json()["code"] == "invalid_param"+ assert collaborators.notion_source.preview_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1", PAGE_ID, page_type)]+++def test_notion_estimate_maps_nested_payload_to_shared_command(collaborators: ControllerCollaborators) -> None:+ response = collaborators.client.post(+ "/console/api/datasets/notion-indexing-estimate",+ json={+ "notion_info_list": [+ {+ "workspace_id": "notion-workspace-1",+ "credential_id": "credential-1",+ "pages": [{"page_id": PAGE_ID, "type": "page"}],+ }+ ],+ "process_rule": {"mode": "automatic"},+ "doc_form": "text_model",+ "doc_language": "English",+ },)- with (- flask_app.test_request_context("/?credential_id=credential-1"),- patch.object(- module.DatasourceProviderService,- "get_datasource_credentials",- return_value={"token": "token"},- ),- patch.object(type(module.db), "engine", new_callable=PropertyMock, return_value=MagicMock()),- patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime),- ):- response, status = unwrap(DataSourceNotionListApi().get)(- DataSourceNotionListApi(),- DataSourceNotionListQuery(credential_id="credential-1"),- sqlite_session,- "tenant-1",- current_user,- )- assert status == 200- assert response == {- "notion_info": [- {- "workspace_name": "Workspace",- "workspace_id": "workspace-1",- "workspace_icon": None,- "pages": [- {- "page_name": "Page",- "page_id": "page-1",- "page_icon": {"type": "emoji", "url": None, "emoji": "P"},- "parent_id": "parent-1",- "type": "page",- "is_bound": False,- }- ],- }- ]- }- runtime.get_online_document_pages.assert_called_once()- assert runtime.get_online_document_pages.call_args.kwargs["datasource_parameters"] == {}+ assert response.status_code == 200+ assert response.get_json()["total_segments"] == 3+ context, command = collaborators.estimates.calls[0]+ assert context.active_workspace_id == TENANT_ID+ assert context.account_id == ACCOUNT_ID+ assert command.process_rule == {"mode": "automatic"}+ source = command.sources[0]+ assert isinstance(source, NotionEstimateSource)+ assert source.credential_id == "credential-1"+ assert source.page_id == PAGE_ID[email protected]("sqlite_session", [()], indirect=True)-def test_notion_pre_import_pages_rejects_missing_credential(- flask_app: Flask,- current_user: Account,- sqlite_session: Session,+ ("error", "expected_http_error"),+ [+ (IndexingEstimateCredentialUnavailableError(), NotFoundError),+ (EstimateSourceNotFoundError("page-1"), NotFoundError),+ (IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),+ (IndexingEstimateExecutionError(), IndexingEstimateError),+ ],+)+def test_notion_estimate_maps_application_errors(+ error: Exception,+ expected_http_error: type[Exception],+ collaborators: ControllerCollaborators,) -> None:- with (- flask_app.test_request_context("/?credential_id=credential-1"),- patch.object(module.DatasourceProviderService, "get_datasource_credentials", return_value=None),- pytest.raises(NotFound, match="Credential not found"),- ):- unwrap(DataSourceNotionListApi().get)(- DataSourceNotionListApi(),- DataSourceNotionListQuery(credential_id="credential-1"),- sqlite_session,- TENANT_ID,- current_user,- )+ collaborators.estimates.error = error+ api = data_source_controller.DataSourceNotionIndexingEstimateApi()+ method = unwrap(api.post)+ payload = data_source_controller.NotionEstimatePayload.model_validate(+ {+ "notion_info_list": [+ {+ "workspace_id": "notion-workspace",+ "credential_id": "credential-1",+ "pages": [{"page_id": "page-1", "type": "page"}],+ }+ ],+ "process_rule": {"mode": "automatic"},+ }+ )+ context = RequestContext("request-1", None, "account-1", "workspace-1")+ with collaborators.client.application.app_context(), pytest.raises(expected_http_error):+ method(api, payload, context)[email protected]("sqlite_session", [()], indirect=True)-def test_notion_pre_import_pages_rejects_non_notion_dataset(- flask_app: Flask,- current_user: Account,- sqlite_session: Session,-) -> None:- dataset = Dataset(- tenant_id=TENANT_ID,- name="Non-Notion dataset",- created_by=current_user.id,- data_source_type=DataSourceType.WEBSITE_CRAWL,++def test_sync_routes_dispatch_only_application_selected_documents(collaborators: ControllerCollaborators) -> None:+ dataset_response = collaborators.client.get(f"/console/api/datasets/{DATASET_ID}/notion/sync")+ document_response = collaborators.client.get(+ f"/console/api/datasets/{DATASET_ID}/documents/{DOCUMENT_ID}/notion/sync")- with (- flask_app.test_request_context("/?credential_id=credential-1&dataset_id=dataset-1"),- patch.object(- module.DatasourceProviderService,- "get_datasource_credentials",- return_value={"token": "token"},- ),- patch.object(module.DatasetService, "get_dataset", return_value=dataset),- pytest.raises(ValueError, match="Dataset is not notion type"),- ):- unwrap(DataSourceNotionListApi().get)(- DataSourceNotionListApi(),- DataSourceNotionListQuery(credential_id="credential-1", dataset_id="dataset-1"),- sqlite_session,- TENANT_ID,- current_user,- )+ assert dataset_response.status_code == 200+ assert dataset_response.get_json() == {"result": "success"}+ assert document_response.status_code == 200+ assert document_response.get_json() == {"result": "success"}+ assert collaborators.dispatcher.calls == [(DATASET_ID, DOCUMENT_ID), (DATASET_ID, DOCUMENT_ID)]+++def test_sync_route_does_not_dispatch_cross_workspace_dataset(collaborators: ControllerCollaborators) -> None:+ response = collaborators.client.get(f"/console/api/datasets/{FOREIGN_DATASET_ID}/notion/sync")++ assert response.status_code == 404+ assert response.get_json()["code"] == "not_found"+ assert collaborators.dispatcher.calls == []+++ "resource",+ [data_source_controller.DataSourceNotionDatasetSyncApi, data_source_controller.DataSourceNotionDocumentSyncApi],+)+def test_notion_sync_requires_dataset_edit_permission(resource):+ [check] = rbac_checks(resource.get)+ assert check.scene is RBACPermission.DATASET_EDIT+ assert isinstance(check.locator, DatasetId)diff --git a/api/tests/unit_tests/controllers/console/datasets/test_data_source_notion_apis.py b/api/tests/unit_tests/controllers/console/datasets/test_data_source_notion_apis.pydeleted file mode 100644index 8bbea58ba8e..00000000000--- a/api/tests/unit_tests/controllers/console/datasets/test_data_source_notion_apis.py+++ /dev/null@@ -1,204 +0,0 @@-"""Unit tests for controllers.console.datasets.data_source Notion endpoints."""--from __future__ import annotations--import inspect-from unittest.mock import MagicMock, patch--import pytest-from flask import Flask-from sqlalchemy.orm import Session-from werkzeug.exceptions import NotFound--from controllers.common.rbac import DatasetId-from controllers.console.datasets.data_source import (- DataSourceNotionDatasetSyncApi,- DataSourceNotionDocumentSyncApi,- DataSourceNotionIndexingEstimateApi,- DataSourceNotionPreviewApi,- DataSourceNotionPreviewQuery,- NotionEstimatePayload,-)-from controllers.console.wraps import RBACPermission-from core.rag.index_processor.constant.index_type import IndexStructureType-from models import Account, Dataset, Document-from models.dataset import DataSourceType-from tests.unit_tests.controllers.rbac_introspection import rbac_checks-from tests.unit_tests.model_factories import make_dataset, make_document---def _dataset() -> Dataset:- return make_dataset(dataset_id="ds-1", created_by="u1")---def _document() -> Document:- return make_document(- document_id="d1",- dataset_id="ds-1",- data_source_type=DataSourceType.NOTION_IMPORT,- name="Notion page",- created_by="u1",- )---def current_user() -> Account:- account = Account(name="Test User", email="[email protected]")- account.id = "u1"- return account---class TestDataSourceNotionPreviewApi:- def test_get_preview_success(self, app: Flask) -> None:- api = DataSourceNotionPreviewApi()- method = inspect.unwrap(api.get)-- extractor = MagicMock(extract=lambda: [MagicMock(page_content="hello")])-- with (- app.test_request_context("/?credential_id=c1"),- patch(- "controllers.console.datasets.data_source.DatasourceProviderService.get_datasource_credentials",- return_value={"integration_secret": "t"},- ),- patch(- "controllers.console.datasets.data_source.NotionExtractor",- return_value=extractor,- ),- ):- response, status = method(api, DataSourceNotionPreviewQuery(credential_id="c1"), "tenant-1", "p1", "page")-- assert status == 200---class TestDataSourceNotionIndexingEstimateApi:- @pytest.mark.parametrize("sqlite_session", [()], indirect=True)- def test_post_indexing_estimate_success(self, app: Flask, sqlite_session: Session) -> None:- api = DataSourceNotionIndexingEstimateApi()- method = inspect.unwrap(api.post)-- empty_rules: dict[str, object] = {}- payload: dict[str, object] = {- "notion_info_list": [- {- "workspace_id": "w1",- "credential_id": "c1",- "pages": [{"page_id": "p1", "type": "page"}],- }- ],- "process_rule": {"rules": empty_rules},- "doc_form": IndexStructureType.PARAGRAPH_INDEX,- "doc_language": "English",- }-- with (- app.test_request_context("/", method="POST", json=payload, headers={"Content-Type": "application/json"}),- patch(- "controllers.console.datasets.data_source.DocumentService.estimate_args_validate",- ),- patch(- "controllers.console.datasets.data_source.IndexingRunner.indexing_estimate",- return_value=MagicMock(model_dump=lambda: {"total_pages": 1}),- ),- ):- response, status = method(api, NotionEstimatePayload.model_validate(payload), sqlite_session, "tenant-1")-- assert status == 200---class TestDataSourceNotionDatasetSyncApi:- def test_get_requires_dataset_edit_permission(self) -> None:- [check] = rbac_checks(DataSourceNotionDatasetSyncApi.get)-- assert check.scene is RBACPermission.DATASET_EDIT- assert isinstance(check.locator, DatasetId)-- @pytest.mark.parametrize("sqlite_session", [()], indirect=True)- def test_get_success(self, app: Flask, sqlite_session: Session) -> None:- api = DataSourceNotionDatasetSyncApi()- method = inspect.unwrap(api.get)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.data_source.DatasetService.get_dataset",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.data_source.DocumentService.get_document_by_dataset_id",- return_value=[_document()],- ),- patch(- "controllers.console.datasets.data_source.document_indexing_sync_task.delay",- return_value=None,- ),- ):- response, status = method(api, sqlite_session, "ds-1")-- assert status == 200-- @pytest.mark.parametrize("sqlite_session", [()], indirect=True)- def test_get_dataset_not_found(self, app: Flask, sqlite_session: Session) -> None:- api = DataSourceNotionDatasetSyncApi()- method = inspect.unwrap(api.get)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.data_source.DatasetService.get_dataset",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, sqlite_session, "ds-1")---class TestDataSourceNotionDocumentSyncApi:- def test_get_requires_dataset_edit_permission(self) -> None:- [check] = rbac_checks(DataSourceNotionDocumentSyncApi.get)-- assert check.scene is RBACPermission.DATASET_EDIT- assert isinstance(check.locator, DatasetId)-- @pytest.mark.parametrize("sqlite_session", [()], indirect=True)- def test_get_success(self, app: Flask, sqlite_session: Session) -> None:- api = DataSourceNotionDocumentSyncApi()- method = inspect.unwrap(api.get)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.data_source.DatasetService.get_dataset",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.data_source.DocumentService.get_document",- return_value=_document(),- ),- patch(- "controllers.console.datasets.data_source.document_indexing_sync_task.delay",- return_value=None,- ),- ):- response, status = method(api, sqlite_session, "ds-1", "doc-1")-- assert status == 200-- @pytest.mark.parametrize("sqlite_session", [()], indirect=True)- def test_get_document_not_found(self, app: Flask, sqlite_session: Session) -> None:- api = DataSourceNotionDocumentSyncApi()- method = inspect.unwrap(api.get)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.data_source.DatasetService.get_dataset",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.data_source.DocumentService.get_document",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, sqlite_session, "ds-1", "doc-1")diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.pyindex 81ea5da31d4..cf4a06e98e5 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py@@ -1,29 +1,20 @@-import datetime-import json-from collections.abc import Callable-from contextlib import ExitStackfrom inspect import unwrapfrom types import SimpleNamespace-from unittest.mock import ANY, MagicMock, PropertyMock, patch+from unittest.mock import MagicMock, create_autospec, patch+from uuid import UUIDimport pytest-from flask import Flask-from sqlalchemy import event, select-from sqlalchemy.orm import Session, sessionmaker-from werkzeug.exceptions import BadRequest, Forbidden, NotFound+from werkzeug.exceptions import Forbidden, NotFoundimport services-from controllers.common.rbac import DatasetId, Workspace-from controllers.console import console_ns+from controllers.common.errors import InvalidArgumentError, NotFoundError+from controllers.common.rbac import DatasetId, RBACPermission, Workspacefrom controllers.console.app.error import ProviderNotInitializeError-from controllers.console.datasets import datasets as datasets_controller+from controllers.console.datasets import datasets as controllerfrom controllers.console.datasets.datasets import (DatasetApi,DatasetApiBaseUrlApi,- DatasetApiDeleteApi,- DatasetApiKeyApi,DatasetAutoDisableLogApi,- DatasetCreatePayload,DatasetEnableApiApi,DatasetErrorDocs,DatasetIndexingEstimateApi,@@ -37,119 +28,34 @@ from controllers.console.datasets.datasets import (DatasetUpdatePayload,DatasetUseCheckApi,IndexingEstimatePayload,- _get_retrieval_methods_by_vector_type,+ _new_estimate_sources,+)+from controllers.console.datasets.error import (+ DatasetAccessDeniedRequestError,+ DatasetInUseError,+ DatasetNameDuplicateError,+ IndexingEstimateError,)-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError-from controllers.console.wraps import RBACPermission-from core.entities.knowledge_entities import IndexingEstimate-from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError-from core.provider_manager import ProviderManager-from core.rag.datasource.vdb.vector_type import VectorType-from core.rag.index_processor.constant.index_type import IndexStructureType-from core.rag.retrieval.retrieval_methods import RetrievalMethod-from extensions.application_services.knowledge import build_dataset_api_key_service-from extensions.storage.storage_type import StorageType-from fields.dataset_fields import build_dataset_detail_prefetchfrom machinery.context import RequestContext-from models.account import Account, Tenant, TenantAccountRole-from models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegment-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus-from models.model import ApiToken, AppMode, DatasetApiTokenBinding, IconType, UploadFile-from services.dataset_ref_service import DatasetRef-from services.dataset_service import DatasetPermissionService, DatasetService-from services.enterprise import rbac_service as enterprise_rbac_service-from services.entities.app_entities import AppRecord+from services.data_source.entities.notion_import import NotionPageType+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.datasets.application import DatasetApplicationService, DatasetListFilter+from services.knowledge.entities.indexing_estimate import (+ NotionEstimateSource,+ UploadFileEstimateSource,+ WebsiteEstimateSource,+)+from services.knowledge.indexing.estimate import (+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+ UnsupportedEstimateSourceError,+)from tests.unit_tests.controllers.rbac_introspection import rbac_checks-pytestmark = pytest.mark.usefixtures("app_query_services")--[email protected](autouse=True)-def dataset_model_property_defaults():- getter_values: dict[str, object] = {- "get_app_count": 0,- "get_document_count": 0,- "get_word_count": 0,- "get_author_name": None,- "get_tags": [],- "get_doc_form": None,- "get_external_knowledge_info": None,- "get_doc_metadata": [],- "get_is_published": False,- "get_total_documents": 0,- "get_total_available_documents": 0,- }- getters = {}- with ExitStack() as stack:- for name, value in getter_values.items():- getters[name] = stack.enter_context(patch.object(Dataset, name, autospec=True, return_value=value))- stack.enter_context(- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=enterprise_rbac_service.MyPermissionsResponse(),- )- )- stack.enter_context(- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetPermissions.batch_get",- return_value={},- )- )- yield getters---def make_dataset(**overrides) -> Dataset:- base = {- "id": "ds-1",- "tenant_id": "tenant-1",- "name": "Dataset",- "description": "desc",- "provider": "vendor",- "permission": "only_me",- "data_source_type": None,- "indexing_technique": "economy",- "created_by": "account-1",- "created_at": datetime.datetime(2024, 1, 1, 12, 0, 0, tzinfo=datetime.UTC),- "updated_by": None,- "updated_at": datetime.datetime(2024, 1, 1, 12, 0, 0, tzinfo=datetime.UTC),- "embedding_model": None,- "embedding_model_provider": None,- "retrieval_model": None,- "summary_index_setting": None,- "built_in_field_enabled": False,- "pipeline_id": None,- "runtime_mode": "general",- "chunk_structure": None,- "icon_info": None,- "enable_api": False,- "is_multimodal": False,- }- base.update(overrides)- return Dataset(**base)---def make_account(role: TenantAccountRole = TenantAccountRole.EDITOR) -> Account:- account = Account(name="Test User", email="[email protected]")- account.id = "account-1"- account.role = role- return account---def make_related_app(**overrides) -> AppRecord:- base = {- "id": "app-1",- "name": "App",- "description": "desc",- "mode_compatible_with_agent": AppMode.CHAT,- "icon_type": IconType.EMOJI,- "icon": "🤖",- "icon_background": "#fff",- "enable_site": False,- "enable_api": False,- "created_by": "account-1",- }- base.update(overrides)- return AppRecord(**base)+CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")+DATASET_ID = UUID(int=1)def test_dataset_delete_requires_dataset_delete_permission() -> None:@@ -159,1714 +65,262 @@ def test_dataset_delete_requires_dataset_delete_permission() -> None:assert isinstance(check.locator, DatasetId)-def make_document_status(**overrides) -> Document:- base = {- "id": "doc-1",- "tenant_id": "tenant-1",- "dataset_id": "dataset-1",- "position": 1,- "data_source_type": DataSourceType.UPLOAD_FILE,- "batch": "batch-1",- "name": "doc.txt",- "created_from": DocumentCreatedFrom.WEB,- "created_by": "account-1",- "indexing_status": IndexingStatus.COMPLETED,- "enabled": True,- "archived": False,- "processing_started_at": None,- "parsing_completed_at": None,- "cleaning_completed_at": None,- "splitting_completed_at": None,- "completed_at": None,- "paused_at": None,- "error": None,- "stopped_at": None,- }- base.update(overrides)- return Document(**base)---def make_document_segment(- *,- position: int,- completed: bool,- document_id: str = "doc-1",- tenant_id: str = "tenant-1",- dataset_id: str = "dataset-1",- status: SegmentStatus = SegmentStatus.WAITING,-) -> DocumentSegment:- return DocumentSegment(- tenant_id=tenant_id,- dataset_id=dataset_id,- document_id=document_id,- position=position,- content=f"segment {position}",- word_count=2,- tokens=2,- created_by="account-1",- status=status,- completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) if completed else None,+def datasets(monkeypatch):+ service = create_autospec(DatasetApplicationService, instance=True, spec_set=True)+ monkeypatch.setattr(+ controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(datasets=service)))---class _UsesSQLiteSession:- session: Session-- @pytest.fixture(autouse=True)- def _inject_sqlite_session(self, sqlite_session: Session) -> None:- self.session = sqlite_session---class TestDatasetList(_UsesSQLiteSession):- def _mock_user(self):- user = make_account()- return user-- def test_get_success_basic(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [make_dataset(icon_info={"icon": "📙", "icon_type": "emoji"})]- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- assert status == 200- assert resp["total"] == 1- assert resp["data"][0]["embedding_available"] is True- assert resp["data"][0]["icon_info"] == {- "icon": "📙",- "icon_background": None,- "icon_type": "emoji",- "icon_url": None,- }-- def test_get_batch_loads_database_fields_with_caller_session(self, app: Flask, dataset_model_property_defaults):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- dataset = make_dataset()- session = self.session- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([dataset], 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- patch(- "controllers.console.datasets.datasets.build_dataset_detail_prefetch",- wraps=build_dataset_detail_prefetch,- ) as prefetch_mock,- ):- resp, status = method(api, session, "tenant-1", current_user)-- assert status == 200- prefetch_mock.assert_called_once_with([dataset], session=session)- # The page is served from that single batch load, so no field may fall back- # to a per-dataset query.- for getter in dataset_model_property_defaults.values():- getter.assert_not_called()- assert resp["data"][0]["document_count"] == 0- assert resp["data"][0]["tags"] == []-- def test_get_with_ids_filter(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [make_dataset()]- with app.test_request_context("/datasets?ids=1&ids=2"):- with (- patch.object(DatasetService, "get_datasets_by_ids", return_value=(datasets, 2)) as by_ids_mock,- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- by_ids_mock.assert_called_once()- assert status == 200- assert resp["total"] == 2-- def test_get_attaches_current_user_permission_keys(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- dataset = make_dataset(id="dataset-1")- permissions = enterprise_rbac_service.MyPermissionsResponse(- dataset=enterprise_rbac_service.ResourcePermissionSnapshot(- default_permission_keys=["dataset.acl.readonly"],- overrides=[- enterprise_rbac_service.ResourcePermissionKeys(- resource_id="dataset-1", permission_keys=["dataset.acl.readonly", "dataset.acl.edit"]- )- ],- )- )- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([dataset], 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ) as get_permissions,- ):- resp, status = method(api, self.session, "tenant-1", current_user)- get_permissions.assert_called_once_with("tenant-1", current_user.id, session=ANY)- assert status == 200- assert resp["data"][0]["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"]-- def test_get_restricted_whitelist_blocks_own_dataset_fallback(- self, app: Flask, config_overrides: Callable[..., None]- ):- config_overrides(RBAC_ENABLED=True)- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- permissions = enterprise_rbac_service.MyPermissionsResponse(- workspace=enterprise_rbac_service.WorkspacePermissionSnapshot(- permission_keys=["dataset.create_and_management"]- )- )- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([], 0)) as get_datasets,- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources",- return_value=SimpleNamespace(resource_ids=[]),- ),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- method(api, self.session, "tenant-1", current_user)- assert get_datasets.call_args.kwargs["accessible_dataset_ids"] == []- assert get_datasets.call_args.kwargs["include_own_datasets"] is False-- def test_get_default_read_is_unrestricted_when_whitelist_unrestricted(- self, app: Flask, config_overrides: Callable[..., None]- ):- config_overrides(RBAC_ENABLED=True)- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- permissions = enterprise_rbac_service.MyPermissionsResponse(- dataset=enterprise_rbac_service.ResourcePermissionSnapshot(default_permission_keys=["dataset.preview"])- )- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([], 0)) as get_datasets,- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources",- return_value=SimpleNamespace(unrestricted=True, resource_ids=[]),- ),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- method(api, self.session, "tenant-1", current_user)- assert get_datasets.call_args.kwargs["accessible_dataset_ids"] is None-- def test_get_restricted_whitelist_overrides_default_read_permission(- self, app: Flask, config_overrides: Callable[..., None]- ):- config_overrides(RBAC_ENABLED=True)- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- permissions = enterprise_rbac_service.MyPermissionsResponse(- dataset=enterprise_rbac_service.ResourcePermissionSnapshot(default_permission_keys=["dataset.preview"])- )- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([], 0)) as get_datasets,- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources",- return_value=SimpleNamespace(resource_ids=["dataset-whitelist-only"]),- ),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- method(api, MagicMock(), "tenant-1", current_user)- assert get_datasets.call_args.kwargs["accessible_dataset_ids"] == ["dataset-whitelist-only"]- assert get_datasets.call_args.kwargs["include_own_datasets"] is False-- def test_get_restricted_whitelist_ignores_dataset_read_overrides(- self, app: Flask, config_overrides: Callable[..., None]- ):- config_overrides(RBAC_ENABLED=True)- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- permissions = enterprise_rbac_service.MyPermissionsResponse(- dataset=enterprise_rbac_service.ResourcePermissionSnapshot(- overrides=[- enterprise_rbac_service.ResourcePermissionKeys(- resource_id="dataset-acl-shared", permission_keys=["dataset.acl.preview"]- ),- enterprise_rbac_service.ResourcePermissionKeys(- resource_id="dataset-full", permission_keys=["dataset.full_access"]- ),- enterprise_rbac_service.ResourcePermissionKeys(- resource_id="dataset-shared", permission_keys=["dataset.preview"]- ),- enterprise_rbac_service.ResourcePermissionKeys(resource_id="dataset-hidden", permission_keys=[]),- ]- )- )- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=([], 0)) as get_datasets,- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources",- return_value=SimpleNamespace(resource_ids=["dataset-whitelist-only"]),- ),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- method(api, self.session, "tenant-1", current_user)- assert get_datasets.call_args.kwargs["accessible_dataset_ids"] == [- "dataset-whitelist-only",- ]- assert get_datasets.call_args.kwargs["include_own_datasets"] is False-- def test_get_with_ids_applies_dataset_visibility(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(RBAC_ENABLED=True)- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- permissions = enterprise_rbac_service.MyPermissionsResponse()- with app.test_request_context("/datasets?ids=dataset-1"):- with (- patch.object(DatasetService, "get_datasets_by_ids", return_value=([], 0)) as get_datasets_by_ids,- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=permissions,- ),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.DatasetAccess.whitelist_resources",- return_value=SimpleNamespace(resource_ids=[]),- ),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- method(api, self.session, "tenant-1", current_user)- session = get_datasets_by_ids.call_args.kwargs["session"]- assert session is self.session- assert get_datasets_by_ids.call_args.args == (["dataset-1"], "tenant-1")- assert get_datasets_by_ids.call_args.kwargs == {- "user": current_user,- "accessible_dataset_ids": [],- "include_own_datasets": False,- "session": session,- }-- def test_get_with_tag_ids(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [make_dataset()]- with app.test_request_context("/datasets?tag_ids=tag1"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- assert status == 200-- def test_get_allows_legacy_weighted_score_without_weight_type(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [- make_dataset(- retrieval_model={- "search_method": "hybrid_search",- "reranking_enable": True,- "reranking_mode": "weighted_score",- "reranking_model": None,- "weights": {- "vector_setting": {- "vector_weight": 0.7,- "embedding_model_name": "text-embedding",- "embedding_provider_name": "openai",- },- "keyword_setting": {"keyword_weight": 0.3},- },- "top_k": 3,- "score_threshold_enabled": False,- "score_threshold": 0.0,- }- )- ]- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- assert status == 200- assert resp["data"][0]["retrieval_model_dict"]["weights"]["weight_type"] is None-- def test_get_merges_partial_retrieval_model_defaults(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [make_dataset(retrieval_model={"top_k": 4, "score_threshold_enabled": False})]- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- assert status == 200- retrieval_model = resp["data"][0]["retrieval_model_dict"]- assert retrieval_model["search_method"] == "semantic_search"- assert retrieval_model["reranking_enable"] is False- assert retrieval_model["top_k"] == 4- assert retrieval_model["score_threshold_enabled"] is False-- def test_embedding_available_false(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [- make_dataset(- indexing_technique="high_quality", embedding_model="text-embed", embedding_model_provider="openai"- )- ]- config = MagicMock()- config.get_models.return_value = []- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=config),- ):- resp, status = method(api, self.session, "tenant-1", current_user)- assert resp["data"][0]["embedding_available"] is False-- def test_partial_members_permission(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.get)- current_user = self._mock_user()- datasets = [make_dataset(permission="partial_members")]- session = self.session- session.add(DatasetPermission(dataset_id="ds-1", account_id="u1", tenant_id="tenant-1"))- session.flush()- with app.test_request_context("/datasets"):- with (- patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),- patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),- ):- resp, status = method(api, session, "tenant-1", current_user)- assert resp["data"][0]["partial_member_list"] == ["u1"]---class TestDatasetListApiPost(_UsesSQLiteSession):- def test_post_success(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- payload = {"name": "My Dataset", "description": "desc", "indexing_technique": "economy", "provider": "vendor"}- user = make_account()- dataset = make_dataset(name=payload["name"], description=payload["description"])- with (- app.test_request_context("/datasets", json=payload),- patch.object(type(console_ns), "payload", payload),- patch.object(DatasetService, "create_empty_dataset", return_value=dataset),- ):- _, status = method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)- assert status == 201-- def test_post_forbidden(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- payload = {"name": "test"}- user = make_account(TenantAccountRole.NORMAL)- with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):- with pytest.raises(Forbidden):- method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)-- def test_post_duplicate_name(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- payload = {"name": "duplicate"}- user = make_account()- with (- app.test_request_context("/datasets", json=payload),- patch.object(type(console_ns), "payload", payload),- patch.object(- DatasetService, "create_empty_dataset", side_effect=services.errors.dataset.DatasetNameDuplicateError()- ),- ):- with pytest.raises(DatasetNameDuplicateError):- method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)-- def test_post_invalid_payload_missing_name(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- with app.test_request_context("/datasets", json={}), patch.object(type(console_ns), "payload", {}):- with pytest.raises(ValueError):- method(api, DatasetCreatePayload(), self.session, "tenant-1", make_account())-- def test_post_invalid_indexing_technique(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- payload = {"name": "bad", "indexing_technique": "invalid-tech"}- with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):- with pytest.raises(ValueError, match="Invalid indexing technique"):- method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", make_account())-- def test_post_invalid_provider(self, app: Flask):- api = DatasetListApi()- method = unwrap(api.post)- payload = {"name": "bad", "provider": "unknown"}- with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):- with pytest.raises(ValueError, match="Invalid provider"):- method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", make_account())---class TestDatasetApiGet(_UsesSQLiteSession):- def test_get_success_basic(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "123e4567-e89b-12d3-a456-426614174000"- user = make_account()- tenant_id = "tenant-1"- dataset = make_dataset(id=dataset_id)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,- ):- provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, self.session, tenant_id, user, dataset_id)- assert status == 200- assert data["embedding_available"] is True-- def test_get_attaches_permission_keys_when_rbac_enabled(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(RBAC_ENABLED=True)- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "123e4567-e89b-12d3-a456-426614174000"- user = make_account()- tenant_id = "tenant-1"- dataset = make_dataset(id=dataset_id)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch(- "controllers.console.datasets.datasets.enterprise_rbac_service.RBACService.MyPermissions.get",- return_value=enterprise_rbac_service.MyPermissionsResponse(- dataset=enterprise_rbac_service.ResourcePermissionSnapshot(- overrides=[- enterprise_rbac_service.ResourcePermissionKeys(- resource_id=dataset_id, permission_keys=["dataset.acl.readonly", "dataset.acl.edit"]- )- ]- )- ),- ) as get_permissions,- patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,- ):- provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, self.session, tenant_id, user, dataset_id)- get_permissions.assert_called_once_with(tenant_id, user.id, dataset_id=dataset_id, session=ANY)- assert status == 200- assert data["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"]-- def test_get_uses_default_external_retrieval_model(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- dataset = make_dataset(id=dataset_id, retrieval_model=None)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,- ):- provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, self.session, "tenant", make_account(), dataset_id)- assert status == 200- assert data["external_retrieval_model"] == {"top_k": 2, "score_threshold": 0.0, "score_threshold_enabled": None}-- def test_get_dataset_not_found(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "missing-id"- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=None),- ):- with pytest.raises(NotFound, match="Dataset not found"):- method(api, self.session, "tenant", make_account(), dataset_id)-- def test_get_permission_denied(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- dataset = make_dataset(id=dataset_id)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(- DatasetService,- "check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no access"),- ),- ):- with pytest.raises(Forbidden, match="no access"):- method(api, self.session, "tenant", make_account(), dataset_id)-- def test_get_high_quality_embedding_unavailable(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- user = make_account()- tenant_id = "tenant-1"- dataset = make_dataset(- id=dataset_id,- indexing_technique="high_quality",- embedding_model="text-embedding",- embedding_model_provider="openai",- )- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,- ):- provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, _ = method(api, self.session, tenant_id, user, dataset_id)- assert data["embedding_available"] is False-- def test_get_partial_members_permission(self, app: Flask):- api = DatasetApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- dataset = make_dataset(id=dataset_id, permission="partial_members")- partial_members = ["u1", "u2"]- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=partial_members),- patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,- ):- provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, _ = method(api, self.session, "tenant", make_account(), dataset_id)- assert data["partial_member_list"] == partial_members---class TestDatasetApiPatch(_UsesSQLiteSession):- def test_patch_success_basic(self, app: Flask):- api = DatasetApi()- method = unwrap(api.patch)- dataset_id = "dataset-id"- payload = {"name": "updated-name", "description": "updated description"}- user = make_account()- tenant_id = "tenant-1"- dataset = make_dataset(id=dataset_id, tenant_id=tenant_id)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(type(console_ns), "payload", payload),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "check_permission", return_value=None),- patch.object(DatasetService, "update_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=[]),- ):- result, status = method(api, DatasetUpdatePayload(), self.session, tenant_id, user, dataset_id)- assert status == 200- assert result["partial_member_list"] == []-- def test_patch_dataset_not_found(self, app: Flask):- api = DatasetApi()- method = unwrap(api.patch)- with (- app.test_request_context("/datasets/missing"),- patch.object(DatasetService, "get_dataset", return_value=None),- ):- with pytest.raises(NotFound, match="Dataset not found"):- method(api, DatasetUpdatePayload(), self.session, "tenant-1", make_account(), "missing")-- def test_patch_permission_denied(self, app: Flask):- api = DatasetApi()- method = unwrap(api.patch)- dataset_id = "dataset-id"- dataset = make_dataset(id=dataset_id)- payload = {"name": "x"}- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(type(console_ns), "payload", payload),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "check_permission", side_effect=Forbidden("no permission")),- ):- with pytest.raises(Forbidden):- method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)-- def test_patch_partial_members_update(self, app: Flask):- api = DatasetApi()- method = unwrap(api.patch)- dataset_id = "dataset-id"- payload = {"permission": "partial_members", "partial_member_list": [{"user_id": "u1"}, {"user_id": "u2"}]}- dataset = make_dataset(id=dataset_id, permission="partial_members")- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(type(console_ns), "payload", payload),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "check_permission", return_value=None),- patch.object(DatasetService, "update_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "update_partial_member_list", return_value=None),- patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=["u1", "u2"]),- ):- result, _ = method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)- assert result["partial_member_list"] == ["u1", "u2"]-- def test_patch_clear_partial_members(self, app: Flask):- api = DatasetApi()- method = unwrap(api.patch)- dataset_id = "dataset-id"- payload = {"permission": "only_me"}- dataset = make_dataset(id=dataset_id)- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(type(console_ns), "payload", payload),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "check_permission", return_value=None),- patch.object(DatasetService, "update_dataset", return_value=dataset),- patch.object(DatasetPermissionService, "clear_partial_member_list", return_value=None),- patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=[]),- ):- result, _ = method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)- assert result["partial_member_list"] == []---class TestDatasetApiDelete(_UsesSQLiteSession):- def test_delete_success(self, app: Flask):- api = DatasetApi()- method = unwrap(api.delete)- dataset_id = "dataset-id"- user = make_account()- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "delete_dataset", return_value=True),- patch.object(DatasetPermissionService, "clear_partial_member_list", return_value=None),- ):- result, status = method(api, self.session, user, dataset_id)- assert status == 204- assert result == ""-- def test_delete_forbidden_no_permission(self, app: Flask):- api = DatasetApi()- method = unwrap(api.delete)- dataset_id = "dataset-id"- user = make_account(TenantAccountRole.NORMAL)- with app.test_request_context(f"/datasets/{dataset_id}"):- with pytest.raises(Forbidden):- method(api, self.session, user, dataset_id)-- def test_delete_dataset_not_found(self, app: Flask):- api = DatasetApi()- method = unwrap(api.delete)- dataset_id = "missing-dataset"- user = make_account()- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "delete_dataset", return_value=False),- ):- with pytest.raises(NotFound, match="Dataset not found"):- method(api, self.session, user, dataset_id)-- def test_delete_dataset_in_use(self, app: Flask):- api = DatasetApi()- method = unwrap(api.delete)- dataset_id = "dataset-id"- user = make_account()- with (- app.test_request_context(f"/datasets/{dataset_id}"),- patch.object(DatasetService, "delete_dataset", side_effect=services.errors.dataset.DatasetInUseError()),- ):- with pytest.raises(DatasetInUseError):- method(api, self.session, user, dataset_id)---class TestDatasetUseCheckApi(_UsesSQLiteSession):- @pytest.mark.parametrize("is_using", [True, False])- def test_get_use_check(self, app: Flask, is_using: bool):- api = DatasetUseCheckApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- dataset = make_dataset(id=dataset_id)- current_user = make_account()- session = self.session- with (- app.test_request_context(f"/datasets/{dataset_id}/use-check"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(DatasetService, "dataset_use_check", return_value=is_using) as dataset_use_check,- ):- result, status = method(api, session, "tenant-1", current_user, dataset_id)- assert status == 200- assert result == {"is_using": is_using}- get_dataset.assert_called_once_with(dataset_id, "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, current_user, session)- dataset_use_check.assert_called_once_with(DatasetRef("tenant-1", dataset_id), session)-- def test_get_use_check_delegates_permission_check_in_rbac_mode(- self, app: Flask, config_overrides: Callable[..., None]- ):- config_overrides(RBAC_ENABLED=True)- api = DatasetUseCheckApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-id")- current_user = make_account()- session = self.session- with (- app.test_request_context("/datasets/dataset-id/use-check"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(DatasetService, "dataset_use_check", return_value=False),- ):- _, status = method(api, session, "tenant-1", current_user, "dataset-id")-- assert status == 200- check_permission.assert_called_once_with(dataset, current_user, session)+ return service@pytest.mark.parametrize(- "api_cls",- [DatasetUseCheckApi, DatasetIndexingStatusApi, DatasetErrorDocs, DatasetAutoDisableLogApi],+ ("resource", "operation"),+ [+ (DatasetApi, lambda service: service.get_dataset),+ (DatasetUseCheckApi, lambda service: service.is_in_use),+ (DatasetQueryApi, lambda service: service.queries),+ (DatasetRelatedAppListApi, lambda service: service.related_apps),+ (DatasetIndexingStatusApi, lambda service: service.indexing_status),+ (DatasetErrorDocs, lambda service: service.error_documents),+ (DatasetPermissionUserListApi, lambda service: service.partial_members),+ (DatasetAutoDisableLogApi, lambda service: service.auto_disable_logs),+ ],)-def test_dataset_scoped_read_permission_denied(app: Flask, api_cls, sqlite_session: Session):- api = api_cls()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- session = sqlite_session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(- DatasetService,- "check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no permission"),- ),- ):- with pytest.raises(Forbidden, match="no permission"):- method(api, session, "tenant-1", make_account(), "dataset-1")---class TestDatasetQueryApi(_UsesSQLiteSession):- def _query_record(self, index: int = 1) -> DatasetQuery:- query = DatasetQuery(- dataset_id="dataset-id",- content=json.dumps([{"content_type": "text_query", "content": f"question {index}", "file_info": None}]),- source="hit_testing",- source_app_id=None,- created_by_role=CreatorUserRole.ACCOUNT,- created_by=f"account-{index}",- )- query.id = f"query-{index}"- query.created_at = datetime.datetime(2024, 1, index, 12, 0, 0, tzinfo=datetime.UTC)- return query+ ("error", "http_error"), [(DatasetNotFoundError(), NotFound), (DatasetAccessDeniedError(), Forbidden)]+)+def test_scoped_reads_pass_context_and_map_access_errors(app, datasets, resource, operation, error, http_error):+ method = operation(datasets)+ method.side_effect = error+ with app.test_request_context("/"), pytest.raises(http_error):+ unwrap(resource.get)(resource(), CONTEXT, DATASET_ID)+ assert method.call_args.args == (CONTEXT,)+ assert method.call_args.kwargs["dataset_id"] == str(DATASET_ID)+++def test_list_parses_repeated_filters_and_serializes_page(app, datasets):+ datasets.list_datasets.return_value = {"data": [], "page": 2, "limit": 3, "total": 4, "has_more": False}+ with app.test_request_context("/?page=2&limit=3&ids=a&ids=b&tag_ids=x&tag_ids=y&include_all=true&keyword=term"):+ result, status = unwrap(DatasetListApi.get)(DatasetListApi(), CONTEXT)+ datasets.list_datasets.assert_called_once_with(+ CONTEXT,+ DatasetListFilter(page=2, limit=3, ids=["a", "b"], tag_ids=["x", "y"], include_all=True, keyword="term"),+ )+ assert status == 200+ assert result == datasets.list_datasets.return_value- def test_get_queries_success(self, app: Flask):- api = DatasetQueryApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- current_user = make_account()- dataset = make_dataset(id=dataset_id)- queries = [self._query_record(1), self._query_record(2)]- with (- app.test_request_context("/datasets/queries?page=1&limit=20"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch.object(DatasetService, "get_dataset_queries", return_value=(queries, 2)),- ):- response, status = method(api, self.session, current_user, dataset_id)- assert status == 200- assert response["total"] == 2- assert response["page"] == 1- assert response["limit"] == 20- assert response["has_more"] is False- assert len(response["data"]) == 2- assert response["data"][0] == {- "id": "query-1",- "queries": [{"content_type": "text_query", "content": "question 1", "file_info": None}],- "source": "hit_testing",- "source_app_id": None,- "created_by_role": "account",- "created_by": "account-1",- "created_at": 1704110400,- }- def test_get_image_query_uses_caller_session(self, app: Flask):- api = DatasetQueryApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-id")- query = self._query_record()- query.content = json.dumps([{"content_type": "image_query", "content": "file-1"}])- upload_file = UploadFile(- tenant_id="tenant-1",- storage_type=StorageType.LOCAL,- key="image.png",- name="image.png",- size=10,- extension="png",- mime_type="image/png",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="account-1",- created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),- used=False,- )- upload_file.id = "file-1"- session = self.session- session.add(upload_file)- session.flush()- with (- app.test_request_context("/datasets/queries"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch.object(DatasetService, "get_dataset_queries", return_value=([query], 1)),- patch("models.dataset.sign_upload_file_preview_url", return_value="signed-url"),- ):- response, status = method(api, session, make_account(), "dataset-id")+def test_patch_does_not_turn_omitted_fields_into_updates(datasets):+ datasets.update_dataset.side_effect = DatasetNotFoundError()+ with pytest.raises(NotFound):+ unwrap(DatasetApi.patch)(DatasetApi(), DatasetUpdatePayload(name="Changed"), CONTEXT, DATASET_ID)+ datasets.update_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), values={"name": "Changed"})- assert status == 200- assert response["data"][0]["queries"] == [- {- "content_type": "image_query",- "content": "file-1",- "file_info": {- "id": "file-1",- "name": "image.png",- "size": 10,- "extension": "png",- "mime_type": "image/png",- "source_url": "signed-url",- },- }- ]- assert session.get(UploadFile, "file-1") is upload_file-- def test_get_queries_dataset_not_found(self, app: Flask):- api = DatasetQueryApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- current_user = make_account()- with (- app.test_request_context("/datasets/queries"),- patch.object(DatasetService, "get_dataset", return_value=None),- ):- with pytest.raises(NotFound, match="Dataset not found"):- method(api, self.session, current_user, dataset_id)- def test_get_queries_permission_denied(self, app: Flask):- api = DatasetQueryApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- current_user = make_account()- dataset = make_dataset(id=dataset_id)- with (- app.test_request_context("/datasets/queries"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(- DatasetService,- "check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no access"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, current_user, dataset_id)+ ("error", "http_error"),+ [+ (services.errors.dataset.DatasetNameDuplicateError(), DatasetNameDuplicateError),+ (services.errors.dataset.DatasetInUseError(), DatasetInUseError),+ ],+)+def test_domain_errors_are_mapped_at_transport(datasets, error, http_error):+ datasets.delete_dataset.side_effect = error+ with pytest.raises(http_error):+ unwrap(DatasetApi.delete)(DatasetApi(), CONTEXT, DATASET_ID)- def test_get_queries_pagination_has_more(self, app: Flask):- api = DatasetQueryApi()- method = unwrap(api.get)- dataset_id = "dataset-id"- current_user = make_account()- dataset = make_dataset(id=dataset_id)- queries = [self._query_record(index) for index in range(1, 21)]- with (- app.test_request_context("/datasets/queries?page=1&limit=20"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", return_value=None),- patch.object(DatasetService, "get_dataset_queries", return_value=(queries, 40)),- ):- response, status = method(api, self.session, current_user, dataset_id)- assert status == 200- assert response["has_more"] is True- assert len(response["data"]) == 20+def test_delete_success_returns_empty_204(datasets):+ assert unwrap(DatasetApi.delete)(DatasetApi(), CONTEXT, DATASET_ID) == ("", 204)+ datasets.delete_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID))-class TestDatasetIndexingEstimateApi(_UsesSQLiteSession):- def _upload_file(self, *, tenant_id: str = "tenant-1", file_id: str = "file-1") -> UploadFile:- upload_file = UploadFile(- tenant_id=tenant_id,- storage_type=StorageType.LOCAL,- key="key",- name="name.txt",- size=1,- extension="txt",- mime_type="text/plain",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="user-1",- created_at=datetime.datetime.now(tz=datetime.UTC),- used=False,- )- upload_file.id = file_id- return upload_file- def _base_payload(self):- return {- "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},- "process_rule": {"chunk_size": 100},- "indexing_technique": "high_quality",- "doc_form": IndexStructureType.PARAGRAPH_INDEX,- "doc_language": "English",- "dataset_id": None,+def test_request_base_url_and_explicit_status_are_forwarded(app, datasets):+ datasets.api_base_url.return_value = "https://api.example/v1"+ with app.test_request_context("/", base_url="https://console.example/"):+ assert unwrap(DatasetApiBaseUrlApi.get)(DatasetApiBaseUrlApi(), CONTEXT) == {+ "api_base_url": "https://api.example/v1"}-- @pytest.mark.parametrize(- ("dataset_id", "expected_scene", "expected_locator_type", "expected_path_args"),- [- ("dataset-1", RBACPermission.DATASET_USE, DatasetId, {"dataset_id": "dataset-1"}),- (None, RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace, None),- ],+ datasets.api_base_url.assert_called_once_with(CONTEXT, request_base_url="https://console.example")+ assert unwrap(DatasetEnableApiApi.post)(DatasetEnableApiApi(), CONTEXT, DATASET_ID, "disable") == (+ {"result": "success"},+ 200,)- def test_post_authorizes_existing_and_new_dataset_workflows(- self,- app: Flask,- dataset_id: str | None,- expected_scene: RBACPermission,- expected_locator_type: type[DatasetId] | type[Workspace],- expected_path_args: dict[str, str] | None,- ) -> None:- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- payload = {**self._base_payload(), "dataset_id": dataset_id}- session = self.session- session.add(self._upload_file())- session.flush()- current_user = make_account()-- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- patch(- "controllers.console.datasets.datasets.IndexingRunner.indexing_estimate",- return_value=IndexingEstimate(total_segments=1, preview=[]),- ),- patch("controllers.console.datasets.datasets.enforce_rbac_checks", create=True) as enforce_checks,- ):- method(api, IndexingEstimatePayload(**payload), session, "tenant-1", current_user)+ datasets.set_api_enabled.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), status="disable")- enforce_checks.assert_called_once()- kwargs = enforce_checks.call_args.kwargs- assert kwargs["tenant_id"] == "tenant-1"- assert kwargs["account_id"] == current_user.id- assert kwargs.get("path_args") == expected_path_args- [check] = kwargs["checks"]- assert check.scene is expected_scene- assert isinstance(check.locator, expected_locator_type)- def test_post_success_upload_file(self, app: Flask):- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- payload = self._base_payload()- mock_file = self._upload_file()- session = self.session- session.add(mock_file)- session.flush()+def test_retrieval_settings_forward_mock_flag(datasets):+ datasets.retrieval_settings.return_value = {"retrieval_method": ["semantic_search"]}+ assert (+ unwrap(DatasetRetrievalSettingApi.get)(DatasetRetrievalSettingApi(), CONTEXT)+ == datasets.retrieval_settings.return_value+ )+ unwrap(DatasetRetrievalSettingMockApi.get)(DatasetRetrievalSettingMockApi(), CONTEXT, "milvus")+ datasets.retrieval_settings.assert_called_with(CONTEXT, vector_type="milvus", is_mock=True)- mock_response = IndexingEstimate(total_segments=100, preview=[])- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- patch("controllers.console.datasets.datasets.IndexingRunner.indexing_estimate", return_value=mock_response),- ):- response, status = method(- api,- IndexingEstimatePayload(**payload),- session,- "tenant-1",- make_account(),- )- assert status == 200- assert response == {- "tokens": 0,- "total_price": 0,- "currency": "USD",- "total_segments": 100,- "preview": [],+def test_new_estimate_sources_maps_each_supported_transport_shape() -> None:+ upload_sources = _new_estimate_sources(+ {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1", "file-2"]}}+ )+ notion_sources = _new_estimate_sources(+ {+ "data_source_type": "notion_import",+ "notion_info_list": [+ {+ "workspace_id": "notion-workspace",+ "credential_id": "credential-1",+ "pages": [{"page_id": "page-1", "type": "page"}],+ }+ ],}+ )+ website_sources = _new_estimate_sources(+ {+ "data_source_type": "website_crawl",+ "website_info_list": {+ "provider": "firecrawl",+ "job_id": "job-1",+ "urls": ["https://example.com/a", "https://example.com/b"],+ "only_main_content": True,+ },+ }+ )- def test_post_file_not_found(self, app: Flask):- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- payload = self._base_payload()- session = self.session- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- ):- with pytest.raises(NotFound):- method(- api,- IndexingEstimatePayload(**payload),- session,- "tenant-1",- make_account(),- )-- def test_post_llm_bad_request_error(self, app: Flask):- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- mock_file = self._upload_file()- payload = self._base_payload()- session = self.session- session.add(mock_file)- session.flush()- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- patch(- "controllers.console.datasets.datasets.IndexingRunner.indexing_estimate",- side_effect=LLMBadRequestError(),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(- api,- IndexingEstimatePayload(**payload),- session,- "tenant-1",- make_account(),- )-- def test_post_provider_token_not_init(self, app: Flask):- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- mock_file = self._upload_file()- payload = self._base_payload()- session = self.session- session.add(mock_file)- session.flush()- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- patch(- "controllers.console.datasets.datasets.IndexingRunner.indexing_estimate",- side_effect=ProviderTokenNotInitError("token missing"),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(- api,- IndexingEstimatePayload(**payload),- session,- "tenant-1",- make_account(),- )-- def test_post_generic_exception(self, app: Flask):- api = DatasetIndexingEstimateApi()- method = unwrap(api.post)- mock_file = self._upload_file()- payload = self._base_payload()- session = self.session- session.add(mock_file)- session.flush()- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch("controllers.console.datasets.datasets.DocumentService.estimate_args_validate", return_value=None),- patch(- "controllers.console.datasets.datasets.IndexingRunner.indexing_estimate", side_effect=Exception("boom")- ),- ):- with pytest.raises(IndexingEstimateError):- method(- api,- IndexingEstimatePayload(**payload),- session,- "tenant-1",- make_account(),- )+ assert upload_sources == (UploadFileEstimateSource("file-1"), UploadFileEstimateSource("file-2"))+ assert notion_sources == (NotionEstimateSource("notion-workspace", "page-1", NotionPageType.PAGE, "credential-1"),)+ assert website_sources == (+ WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/a", only_main_content=True),+ WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/b", only_main_content=True),+ )-class TestDatasetRelatedAppListApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = DatasetRelatedAppListApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- app1 = make_related_app(id="app-1", name="App 1")- app2 = make_related_app(id="app-2", name="App 2")- join1 = AppDatasetJoin(app_id="app-1", dataset_id="dataset-1")- join2 = AppDatasetJoin(app_id="app-2", dataset_id="dataset-1")- session = self.session- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets.DatasetService.check_dataset_permission", return_value=None),- patch("controllers.console.datasets.datasets.DatasetService.get_related_apps", return_value=[join1, join2]),- patch(- "services.app.query_service.AppQueryService.related_apps",- return_value=[app1, app2],- ) as related_apps,- ):- response, status = method(api, session, make_account(), "dataset-1")- assert status == 200- assert response["total"] == 2- assert response["data"] == [- {- "id": "app-1",- "name": "App 1",- "description": "desc",- "mode": "chat",- "icon_type": "emoji",- "icon": "🤖",- "icon_background": "#fff",- "icon_url": None,- },- {- "id": "app-2",- "name": "App 2",- "description": "desc",- "mode": "chat",- "icon_type": "emoji",- "icon": "🤖",- "icon_background": "#fff",- "icon_url": None,[email protected](("value", "expected"), [("false", False), ("true", True), (False, False), (True, True)])+def test_website_estimate_parses_boolean_values(value: str | bool, expected: bool) -> None:+ sources = _new_estimate_sources(+ {+ "data_source_type": "website_crawl",+ "website_info_list": {+ "provider": "firecrawl",+ "job_id": "job-1",+ "urls": ["https://example.com"],+ "only_main_content": value,},- ]- related_apps.assert_called_once_with(dataset.tenant_id, ["app-1", "app-2"])-- def test_get_dataset_not_found(self, app: Flask):- api = DatasetRelatedAppListApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=None),- ):- with pytest.raises(NotFound):- method(api, self.session, make_account(), "dataset-1")+ }+ )+ assert sources == (WebsiteEstimateSource("firecrawl", "job-1", "https://example.com", only_main_content=expected),)- def test_get_permission_denied(self, app: Flask):- api = DatasetRelatedAppListApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no permission"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, make_account(), "dataset-1")- def test_get_filters_none_apps(self, app: Flask):- api = DatasetRelatedAppListApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- app1 = make_related_app()- join1 = AppDatasetJoin(app_id="app-1", dataset_id="dataset-1")- join2 = AppDatasetJoin(app_id="app-2", dataset_id="dataset-1")- session = self.session- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets.DatasetService.check_dataset_permission", return_value=None),- patch("controllers.console.datasets.datasets.DatasetService.get_related_apps", return_value=[join1, join2]),- patch(- "services.app.query_service.AppQueryService.related_apps",- return_value=[app1],- ),- ):- response, status = method(api, session, make_account(), "dataset-1")- assert status == 200- assert response["total"] == 1- assert response["data"] == [[email protected]("values", [{"only_main_content": "invalid"}, {"urls": [42]}])+def test_website_estimate_rejects_invalid_field_types(values: dict[str, object]) -> None:+ with pytest.raises(ValueError):+ _new_estimate_sources({- "id": "app-1",- "name": "App",- "description": "desc",- "mode": "chat",- "icon_type": "emoji",- "icon": "🤖",- "icon_background": "#fff",- "icon_url": None,+ "data_source_type": "website_crawl",+ "website_info_list": {+ "provider": "firecrawl",+ "job_id": "job-1",+ "urls": ["https://example.com"],+ **values,+ },}- ]---class TestDatasetIndexingStatusApi(_UsesSQLiteSession):- def test_get_success_with_documents(self, app: Flask):- api = DatasetIndexingStatusApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- current_user = make_account()- document = make_document_status()- session = self.session- session.add(document)- session.add_all([make_document_segment(position=position, completed=True) for position in range(1, 4)])- session.flush()- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- ):- response, status = method(api, session, "tenant-1", current_user, "dataset-1")- assert status == 200- assert "data" in response- assert len(response["data"]) == 1- item = response["data"][0]- assert item["completed_segments"] == 3- assert item["total_segments"] == 3- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, current_user, session)- assert session.get(Document, "doc-1") is document-- def test_get_success_no_documents(self, app: Flask):- api = DatasetIndexingStatusApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- ):- response, status = method(api, session, "tenant-1", make_account(), "dataset-1")- assert status == 200- assert response == {"data": []}-- def test_segment_counts_different_values(self, app: Flask):- api = DatasetIndexingStatusApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- document = make_document_status(indexing_status=IndexingStatus.INDEXING)- session = self.session- session.add(document)- session.add_all([make_document_segment(position=position, completed=position <= 2) for position in range(1, 6)])- session.flush()- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- ):- response, status = method(api, session, "tenant-1", make_account(), "dataset-1")- assert status == 200- item = response["data"][0]- assert item["completed_segments"] == 2- assert item["total_segments"] == 5-- def test_get_batch_segment_counts_uses_one_aggregate_query(self, app: Flask, sqlite_engine):- api = DatasetIndexingStatusApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- documents = [make_document_status(id=f"doc-{index}", position=index) for index in range(1, 6)]- session = self.session- session.add_all(documents)- session.add_all(- [- make_document_segment(- position=position,- completed=position <= 2,- document_id=document.id,- )- for document in documents- for position in range(1, 4)- ]- )- session.add_all(- [- make_document_segment(- position=4,- completed=True,- document_id=documents[0].id,- status=SegmentStatus.RE_SEGMENT,- ),- make_document_segment(- position=5,- completed=True,- document_id=documents[0].id,- tenant_id="other-tenant",- ),- ])- session.flush()-- select_statements: list[str] = []-- def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):- if statement.lstrip().upper().startswith("SELECT"):- select_statements.append(statement)-- event.listen(sqlite_engine, "before_cursor_execute", record_select)- try:- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- ):- response, status = method(api, session, "tenant-1", make_account(), "dataset-1")- finally:- event.remove(sqlite_engine, "before_cursor_execute", record_select)-- assert status == 200- assert len(response["data"]) == 5- assert all(item["completed_segments"] == 2 for item in response["data"])- assert all(item["total_segments"] == 3 for item in response["data"])- assert len(select_statements) == 2--class TestDatasetApiKeyApi(_UsesSQLiteSession):- context = RequestContext("request", None, "actor", "tenant-1")-- @pytest.fixture(autouse=True)- def key_services(self, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch):- tenant = Tenant(name="Workspace")- tenant.id = "tenant-1"- sqlite_session.add(tenant)- sqlite_session.commit()- service = build_dataset_api_key_service(database_client=sqlite_session_factory)- monkeypatch.setattr(- datasets_controller,- "application_services",- lambda: SimpleNamespace(dataset_api_keys=service),- )-- def test_get_api_keys_success(self, app: Flask):- api = DatasetApiKeyApi()- self.session.add_all(- [- ApiToken(id="key-1", tenant_id="tenant-1", type="dataset", token="dataset-aaaa1111bbbb"),- ApiToken(id="key-2", tenant_id="tenant-1", type="dataset", token="dataset-cccc2222dddd"),- ]- )- self.session.commit()- with app.test_request_context("/"):- response = unwrap(api.get)(api, self.context)- assert [key["token"] for key in response["data"]] == ["datas...bbbb", "datas...dddd"]-- def test_post_create_api_key_success(self, app: Flask):- api = DatasetApiKeyApi()- with app.test_request_context("/"):- response, status = unwrap(api.post)(api, self.context)- assert status == 200- assert response["token"].startswith("dataset-")- assert response["type"] == "dataset"- assert response["dataset_ids"] == []- assert response["created_at"] is not None- stored = self.session.get(ApiToken, response["id"])- assert stored is not None- assert stored.token == response["token"]-- def test_post_exceed_max_keys(self, app: Flask):- api = DatasetApiKeyApi()- self.session.add_all(- [ApiToken(id=f"key-{i}", tenant_id="tenant-1", type="dataset", token=f"ds-{i}") for i in range(10)]- )- self.session.commit()- with app.test_request_context("/"), pytest.raises(BadRequest) as exc_info:- unwrap(api.post)(api, self.context)- assert vars(exc_info.value)["data"] == {- "message": "Cannot create more than 10 API keys for this resource type.",- "custom": "max_keys_exceeded",- }-- def test_scoped_key_persists_bindings_and_lists_masked_scope(self, app: Flask):- api = DatasetApiKeyApi()- self.session.add_all(- [- make_dataset(id="ds-1", tenant_id="tenant-1"),- make_dataset(id="ds-2", tenant_id="tenant-1"),- ]- )- self.session.commit()- with app.test_request_context("/", json={"dataset_ids": ["ds-1", "ds-2", "ds-1"]}):- response, status = unwrap(api.post)(api, self.context)- listed = unwrap(api.get)(api, self.context)- assert status == 200- assert response["dataset_ids"] == ["ds-1", "ds-2"]- bindings = self.session.scalars(- select(DatasetApiTokenBinding).where(DatasetApiTokenBinding.api_token_id == response["id"])- ).all()- assert {binding.dataset_id for binding in bindings} == {"ds-1", "ds-2"}- assert set(listed["data"][0]["dataset_ids"]) == {"ds-1", "ds-2"}- assert listed["data"][0]["token"] != response["token"]-- def test_post_rejects_dataset_ids_from_another_tenant(self, app: Flask):- api = DatasetApiKeyApi()- self.session.add(make_dataset(id="foreign-ds", tenant_id="foreign"))- self.session.commit()- with app.test_request_context("/", json={"dataset_ids": ["foreign-ds"]}), pytest.raises(BadRequest) as exc_info:- unwrap(api.post)(api, self.context)- assert "Unknown knowledge base id(s)" in vars(exc_info.value)["data"]["message"]- assert self.session.scalar(select(ApiToken)) is None-- @pytest.mark.parametrize("dataset_ids", ["not-a-list", [1]])- def test_post_rejects_invalid_dataset_ids(self, app: Flask, dataset_ids):- api = DatasetApiKeyApi()- with app.test_request_context("/", json={"dataset_ids": dataset_ids}), pytest.raises(BadRequest) as exc_info:- unwrap(api.post)(api, self.context)- assert vars(exc_info.value)["data"]["message"] == "dataset_ids must be a list of strings."-- def test_delete_success(self, app: Flask):- api = DatasetApiDeleteApi()- self.session.add(ApiToken(id="api-key-id", tenant_id="tenant-1", type="dataset", token="dataset-secret"))- self.session.commit()- with app.test_request_context("/"), patch("services.api_token_service.ApiTokenCache.delete") as invalidate:- result = unwrap(api.delete)(api, self.context, "api-key-id")- assert result == ("", 204)- self.session.expire_all()- assert self.session.get(ApiToken, "api-key-id") is None- invalidate.assert_called_once_with("dataset-secret", "dataset")-- def test_delete_key_not_found(self, app: Flask):- api = DatasetApiDeleteApi()- with app.test_request_context("/"), pytest.raises(NotFound):- unwrap(api.delete)(api, self.context, "api-key-id")---class TestDatasetEnableApiApi(_UsesSQLiteSession):- @pytest.mark.parametrize(("status_value", "enabled"), [("enable", True), ("disable", False)])- def test_update_api_status(self, app: Flask, status_value: str, enabled: bool):- api = DatasetEnableApiApi()- method = unwrap(api.post)- dataset = make_dataset(id="dataset-1")- current_user = make_account()- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(DatasetService, "update_dataset_api_status") as update_status,- ):- response, status = method(api, session, "tenant-1", current_user, "dataset-1", status_value)- assert status == 200- assert response["result"] == "success"- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, current_user, session)- update_status.assert_called_once_with(dataset, enabled, current_user, session)-- def test_rejects_non_editor(self, app: Flask):- api = DatasetEnableApiApi()- method = unwrap(api.post)- dataset = make_dataset(id="dataset-1")- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- patch.object(DatasetService, "update_dataset_api_status") as update_status,- ):- with pytest.raises(Forbidden):- method(- api,- session,- "tenant-1",- make_account(TenantAccountRole.NORMAL),- "dataset-1",- "enable",- )- update_status.assert_not_called()---class TestDatasetApiBaseUrlApi:- def test_get_api_base_url_from_config(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(SERVICE_API_URL="https://example.com")- api = DatasetApiBaseUrlApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- ):- response = method(api)- assert response["api_base_url"] == "https://example.com/v1"-- def test_get_api_base_url_from_request(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(SERVICE_API_URL=None)- api = DatasetApiBaseUrlApi()- method = unwrap(api.get)- with (- app.test_request_context("http://localhost:5000/"),- ):- response = method(api)- assert response["api_base_url"] == "http://localhost:5000/v1"-- def test_get_api_base_url_no_double_v1(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(SERVICE_API_URL="https://example.com/v1")- api = DatasetApiBaseUrlApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- ):- response = method(api)- assert response["api_base_url"] == "https://example.com/v1"---class TestDatasetRetrievalSettingApi:- def test_get_success(self, app: Flask, config_overrides: Callable[..., None]):- config_overrides(VECTOR_STORE="qdrant")- api = DatasetRetrievalSettingApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets._get_retrieval_methods_by_vector_type",- return_value={"retrieval_method": ["semantic", "hybrid"]},- ),- ):- response = method(api)- assert "retrieval_method" in response-- def test_tidb_vector_returns_semantic_only_when_fulltext_disabled(self, config_overrides: Callable[..., None]):- config_overrides(TIDB_VECTOR_ENABLE_FULLTEXT_SEARCH=False)- response = _get_retrieval_methods_by_vector_type(VectorType.TIDB_VECTOR)-- assert response["retrieval_method"] == [RetrievalMethod.SEMANTIC_SEARCH.value]-- def test_tidb_vector_returns_full_methods_when_fulltext_enabled(self, config_overrides: Callable[..., None]):- config_overrides(TIDB_VECTOR_ENABLE_FULLTEXT_SEARCH=True)- response = _get_retrieval_methods_by_vector_type(VectorType.TIDB_VECTOR)-- assert response["retrieval_method"] == [- RetrievalMethod.SEMANTIC_SEARCH.value,- RetrievalMethod.FULL_TEXT_SEARCH.value,- RetrievalMethod.HYBRID_SEARCH.value,- ]---class TestDatasetRetrievalSettingMockApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = DatasetRetrievalSettingMockApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets._get_retrieval_methods_by_vector_type",- return_value={"retrieval_method": ["semantic"]},- ),- ):- response = method(api, "milvus")- assert response["retrieval_method"] == ["semantic"]+def test_new_estimate_sources_deduplicates_upload_ids_without_reordering() -> None:+ sources = _new_estimate_sources(+ {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-2", "file-1", "file-2"]}}+ )-class TestDatasetErrorDocs(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = DatasetErrorDocs()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- error_doc = make_document_status(id="error-doc", indexing_status=IndexingStatus.ERROR, error="failed")- current_user = make_account()- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch(- "controllers.console.datasets.datasets.DocumentService.get_error_documents_by_dataset_ref",- return_value=[error_doc],- ) as get_error_documents,- ):- response, status = method(api, session, "tenant-1", current_user, "dataset-1")- assert status == 200- assert response["total"] == 1- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, current_user, session)- get_error_documents.assert_called_once_with(DatasetRef("tenant-1", "dataset-1"), session)+ assert sources == (UploadFileEstimateSource("file-2"), UploadFileEstimateSource("file-1"))- def test_get_dataset_not_found(self, app: Flask):- api = DatasetErrorDocs()- method = unwrap(api.get)- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=None) as get_dataset,- ):- with pytest.raises(NotFound):- method(api, session, "tenant-1", make_account(), "dataset-1")- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)+ "info_list",+ [+ {"data_source_type": "upload_file", "file_info_list": {}},+ {"data_source_type": "notion_import", "notion_info_list": [{"workspace_id": "workspace"}]},+ {+ "data_source_type": "notion_import",+ "notion_info_list": [+ {+ "workspace_id": "workspace",+ "credential_id": "credential",+ "pages": [{"page_id": "page", "type": "unknown"}],+ }+ ],+ },+ {"data_source_type": "unsupported"},+ ],+)+def test_new_estimate_sources_rejects_malformed_transport_shapes(info_list: dict[str, object]) -> None:+ with pytest.raises(ValueError):+ _new_estimate_sources(info_list)-class TestDatasetPermissionUserListApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = DatasetPermissionUserListApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- users = ["u1", "u2"]- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets.DatasetService.check_dataset_permission", return_value=None),- patch(- "controllers.console.datasets.datasets.DatasetPermissionService.get_dataset_partial_member_list",- return_value=users,- ),- ):- response, status = method(api, self.session, make_account(), "dataset-1")- assert status == 200- assert response["data"] == users- def test_get_permission_denied(self, app: Flask):- api = DatasetPermissionUserListApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no permission"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, make_account(), "dataset-1")+ ("error", "expected_http_error"),+ [+ (IndexingEstimateCredentialUnavailableError(), NotFoundError),+ (EstimateSourceNotFoundError("source-1"), NotFoundError),+ (DatasetNotFoundError(), NotFoundError),+ (DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),+ (UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),+ (IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),+ (IndexingEstimateExecutionError(), IndexingEstimateError),+ ],+)+def test_new_source_estimate_maps_application_errors(+ error: Exception,+ expected_http_error: type[Exception],+) -> None:+ estimates = MagicMock()+ estimates.estimate_new_sources.side_effect = error+ registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))+ api = DatasetIndexingEstimateApi()+ method = unwrap(api.post)+ payload = IndexingEstimatePayload(+ info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},+ process_rule={"mode": "automatic"},+ indexing_technique="economy",+ )+ context = RequestContext("request-1", None, "account-1", "workspace-1")+ with patch("controllers.console.datasets.datasets.application_services", return_value=registry):+ with pytest.raises(expected_http_error):+ method(api, payload, context)-class TestDatasetAutoDisableLogApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = DatasetAutoDisableLogApi()- method = unwrap(api.get)- dataset = make_dataset(id="dataset-1")- logs = {"document_ids": ["doc-1"], "count": 1}- current_user = make_account()- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(DatasetService, "get_dataset_auto_disable_logs", return_value=logs) as get_logs,- ):- response, status = method(api, session, "tenant-1", current_user, "dataset-1")- assert status == 200- assert response == logs- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, current_user, session)- get_logs.assert_called_once_with(DatasetRef("tenant-1", "dataset-1"), session)- def test_get_dataset_not_found(self, app: Flask):- api = DatasetAutoDisableLogApi()- method = unwrap(api.get)- session = self.session- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=None) as get_dataset,- ):- with pytest.raises(NotFound):- method(api, session, "tenant-1", make_account(), "dataset-1")- get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)+ ("dataset_id", "scene", "locator_type"),+ [+ ("dataset-1", RBACPermission.DATASET_USE, DatasetId),+ (None, RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace),+ ],+)+def test_new_source_estimate_authorizes_before_execution(dataset_id, scene, locator_type):+ estimates = MagicMock()+ registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))+ payload = IndexingEstimatePayload(+ info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},+ process_rule={"mode": "automatic"},+ indexing_technique="economy",+ dataset_id=dataset_id,+ )+ with (+ patch.object(controller, "application_services", return_value=registry),+ patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,+ pytest.raises(Forbidden),+ ):+ unwrap(DatasetIndexingEstimateApi.post)(DatasetIndexingEstimateApi(), payload, CONTEXT)++ enforce_checks.assert_called_once()+ kwargs = enforce_checks.call_args.kwargs+ assert kwargs["tenant_id"] == CONTEXT.active_workspace_id+ assert kwargs["account_id"] == CONTEXT.account_id+ assert kwargs["path_args"] == ({"dataset_id": dataset_id} if dataset_id else None)+ [check] = kwargs["checks"]+ assert check.scene is scene+ assert isinstance(check.locator, locator_type)+ estimates.estimate_new_sources.assert_not_called()diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.pyindex b4a737b8171..1286f1a3f90 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py@@ -1,17 +1,16 @@import datetime-import jsonfrom inspect import unwrap-from unittest.mock import MagicMock, patch+from types import SimpleNamespace+from unittest.mock import MagicMock, create_autospec, patch+from uuid import UUIDimport pytest-from flask import Flask-from sqlalchemy import event-from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound-import services-from controllers.common.rbac import DatasetId-from controllers.console import console_ns+from controllers.common.errors import InvalidArgumentError, NotFoundError+from controllers.common.rbac import DatasetByDocument, DatasetId, RBACPermission+from controllers.console.app.error import ProviderNotInitializeError+from controllers.console.datasets import datasets_document as controllerfrom controllers.console.datasets.datasets_document import (DatasetDocumentListApi,DatasetInitApi,@@ -31,7 +30,6 @@ from controllers.console.datasets.datasets_document import (DocumentRecoverApi,DocumentRenameApi,DocumentRenamePayload,- DocumentResource,DocumentRetryApi,DocumentRetryPayload,DocumentStatusApi,@@ -41,190 +39,466 @@ from controllers.console.datasets.datasets_document import (WebsiteDocumentSyncApi,)from controllers.console.datasets.error import (+ DatasetAccessDeniedRequestError,DocumentAlreadyFinishedError,DocumentIndexingError,IndexingEstimateError,InvalidActionError,InvalidMetadataError,)-from controllers.console.wraps import RBACPermission-from core.entities.knowledge_entities import IndexingEstimate-from core.rag.index_processor.constant.index_type import IndexStructureType-from extensions.storage.storage_type import StorageType-from models.account import Account, TenantAccountRole-from models.dataset import (- Dataset,- DatasetPermissionEnum,- DatasetProcessRule,- DocumentPipelineExecutionLog,- DocumentSegment,+from machinery.context import RequestContext+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.documents.application import (+ DatasetDocumentApplicationService,+ DocumentIndexingStateError,+ DocumentInvalidActionError,+ DocumentListFilter,+ DocumentNotFoundError,+ DocumentProviderError,)-from models.dataset import Document as DatasetDocument-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus-from models.model import UploadFile-from services.dataset_ref_service import DatasetRef, DocumentRef-from services.vector_space_admission_service import (- VECTOR_SPACE_ADMISSION_ERROR_CODE,- format_vector_space_admission_error,+from services.knowledge.indexing.estimate import (+ EstimateDocumentAlreadyFinishedError,+ EstimateDocumentNotFoundError,+ EstimateSourceNotFoundError,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+ UnsupportedEstimateSourceError,)-from tests.unit_tests.config_override import config_overrides_contextfrom tests.unit_tests.controllers.rbac_introspection import rbac_checks+CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")+DS = UUID(int=1)+DOC = UUID(int=2)-def make_serializable_document(**overrides):- return make_document(data_source_info=json.dumps({"upload_file_id": "file-1"}), **overrides)--def make_document_detail(**overrides):- doc_metadata_details = overrides.pop("doc_metadata_details", [])- return make_document(- data_source_info=json.dumps({"upload_file_id": "file-1"}),- created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),- tokens=10,- doc_type="others",- doc_metadata=doc_metadata_details,- doc_language="English",- **overrides,+def documents(monkeypatch):+ documents = create_autospec(DatasetDocumentApplicationService, instance=True, spec_set=True)+ monkeypatch.setattr(+ controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents)))+ monkeypatch.setattr(controller, "check_knowledge_rate_limit", lambda: None)+ return documents-def make_dataset(**overrides):- attrs = {- "id": "ds-1",- "tenant_id": "tenant-1",- "name": "Dataset",- "indexing_technique": "economy",- "chunk_structure": IndexStructureType.PARAGRAPH_INDEX,- "created_by": "u1",- "summary_index_setting": {"enable": True},- }- attrs.update(overrides)- return Dataset(**attrs)---def make_document(**overrides):- attrs = {- "id": "doc-1",- "tenant_id": "tenant-1",- "dataset_id": "ds-1",- "position": 1,- "data_source_type": DataSourceType.UPLOAD_FILE,- "data_source_info": None,- "batch": "batch-1",- "name": "Document",- "created_from": DocumentCreatedFrom.WEB,- "created_by": "u1",- "indexing_status": IndexingStatus.COMPLETED,- "enabled": True,- "archived": False,- "doc_metadata": None,- "doc_form": IndexStructureType.PARAGRAPH_INDEX,- "need_summary": False,- }- attrs.update(overrides)- return DatasetDocument(**attrs)+def test_process_rule_authorizes_document_dataset_before_read(app, documents):+ with (+ app.test_request_context(f"/?document_id={DOC}"),+ patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,+ pytest.raises(Forbidden),+ ):+ unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), CONTEXT)++ enforce_checks.assert_called_once()+ kwargs = enforce_checks.call_args.kwargs+ assert kwargs["tenant_id"] == CONTEXT.active_workspace_id+ assert kwargs["account_id"] == CONTEXT.account_id+ assert kwargs["path_args"] == {"document_id": str(DOC)}+ [check] = kwargs["checks"]+ assert check.scene is RBACPermission.DATASET_READONLY+ assert isinstance(check.locator, DatasetByDocument)+ documents.get_process_rule.assert_not_called()-def make_account(role: TenantAccountRole = TenantAccountRole.EDITOR) -> Account:- account = Account(name="Test User", email="[email protected]")- account.id = "u1"- account.role = role- return account+def test_default_process_rule_does_not_require_dataset_permission(app, documents):+ documents.get_process_rule.return_value = {"mode": "automatic", "rules": {}, "limits": {}}+ with app.test_request_context("/"), patch.object(controller, "enforce_rbac_checks") as enforce_checks:+ unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), CONTEXT)+ enforce_checks.assert_not_called()+ documents.get_process_rule.assert_called_once_with(CONTEXT, document_id=None)-def make_segment(*, position: int, completed: bool = True, document_id: str = "doc-1") -> DocumentSegment:- return DocumentSegment(- tenant_id="tenant-1",- dataset_id="ds-1",- document_id=document_id,- position=position,- content=f"segment {position}",- word_count=2,- tokens=2,- created_by="u1",- completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) if completed else None,+ ("resource", "verb", "operation", "kwargs"),+ [+ (GetProcessRuleApi, GetProcessRuleApi.get, lambda service: service.get_process_rule, {}),+ (+ DatasetDocumentListApi,+ DatasetDocumentListApi.get,+ lambda service: service.list_documents,+ {"dataset_id": DS},+ ),+ (+ DatasetDocumentListApi,+ DatasetDocumentListApi.post,+ lambda service: service.create_documents,+ {"dataset_id": DS},+ ),+ (+ DatasetDocumentListApi,+ DatasetDocumentListApi.delete,+ lambda service: service.delete_documents,+ {"dataset_id": DS},+ ),+ (DatasetInitApi, DatasetInitApi.post, lambda service: service.initialize_dataset, {}),+ (+ DocumentBatchIndexingStatusApi,+ DocumentBatchIndexingStatusApi.get,+ lambda service: service.get_batch_indexing_status,+ {"dataset_id": DS, "batch": "batch-1"},+ ),+ (+ DocumentIndexingStatusApi,+ DocumentIndexingStatusApi.get,+ lambda service: service.get_indexing_status,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (DocumentApi, DocumentApi.get, lambda service: service.get_document, {"dataset_id": DS, "document_id": DOC}),+ (+ DocumentApi,+ DocumentApi.delete,+ lambda service: service.delete_document,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentDownloadApi,+ DocumentDownloadApi.get,+ lambda service: service.get_download_url,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentBatchDownloadZipApi,+ DocumentBatchDownloadZipApi.post,+ lambda service: service.build_download_zip,+ {"dataset_id": DS},+ ),+ (+ DocumentProcessingApi,+ DocumentProcessingApi.patch,+ lambda service: service.update_processing,+ {"dataset_id": DS, "document_id": DOC, "action": "pause"},+ ),+ (+ DocumentMetadataApi,+ DocumentMetadataApi.put,+ lambda service: service.update_metadata,+ {+ "dataset_id": DS,+ "document_id": DOC,+ "req_data": DocumentMetadataUpdatePayload(doc_type="book", doc_metadata={"title": "Book"}),+ },+ ),+ (+ DocumentStatusApi,+ DocumentStatusApi.patch,+ lambda service: service.change_status,+ {"dataset_id": DS, "action": "enable"},+ ),+ (+ DocumentPauseApi,+ DocumentPauseApi.patch,+ lambda service: service.pause_document,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentRecoverApi,+ DocumentRecoverApi.patch,+ lambda service: service.recover_document,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentRetryApi,+ DocumentRetryApi.post,+ lambda service: service.retry_documents,+ {"dataset_id": DS, "req_data": DocumentRetryPayload(document_ids=[str(DOC)])},+ ),+ (+ DocumentRenameApi,+ DocumentRenameApi.post,+ lambda service: service.rename_document,+ {"dataset_id": DS, "document_id": DOC, "req_data": DocumentRenamePayload(name="New name")},+ ),+ (+ WebsiteDocumentSyncApi,+ WebsiteDocumentSyncApi.get,+ lambda service: service.sync_website,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentPipelineExecutionLogApi,+ DocumentPipelineExecutionLogApi.get,+ lambda service: service.get_execution_log,+ {"dataset_id": DS, "document_id": DOC},+ ),+ (+ DocumentGenerateSummaryApi,+ DocumentGenerateSummaryApi.post,+ lambda service: service.generate_summary,+ {"dataset_id": DS, "req_data": GenerateSummaryPayload(document_list=[str(DOC)])},+ ),+ (+ DocumentSummaryStatusApi,+ DocumentSummaryStatusApi.get,+ lambda service: service.get_summary_status,+ {"dataset_id": DS, "document_id": DOC},+ ),+ ],+)+def test_endpoints_pass_request_context_and_map_application_errors(app, documents, resource, verb, operation, kwargs):+ operation(documents).side_effect = DocumentNotFoundError("Document not found.")+ with app.test_request_context(+ "/?document_id=" + str(DOC), json={"indexing_technique": "economy", "document_ids": [str(DOC)]}+ ):+ with pytest.raises(NotFound):+ unwrap(verb)(resource(), request_context=CONTEXT, **kwargs)+ called = operation(documents)+ assert called.call_count == 1+ assert called.call_args.args == (CONTEXT,)+ if "dataset_id" in kwargs:+ assert called.call_args.kwargs["dataset_id"] == str(DS)+ if "document_id" in kwargs:+ assert called.call_args.kwargs["document_id"] == str(DOC)++[email protected](("fetch", "expected"), [("true", True), ("YES", True), ("false", False), ("invalid", False)])+def test_list_preserves_filters_pagination_and_fetch_parsing(app, documents, fetch, expected):+ documents.list_documents.return_value = {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}+ with app.test_request_context(f"/?page=2&limit=3&keyword=needle&status=available&sort=hit_count&fetch={fetch}"):+ response = unwrap(DatasetDocumentListApi.get)(DatasetDocumentListApi(), CONTEXT, DS)+ documents.list_documents.assert_called_once_with(+ CONTEXT,+ dataset_id=str(DS),+ query=DocumentListFilter(+ page=2, limit=3, search="needle", status="available", sort="hit_count", fetch=expected+ ),)---def make_upload_file() -> UploadFile:- upload_file = UploadFile(- tenant_id="tenant-1",- storage_type=StorageType.LOCAL,- key="document.txt",- name="document.txt",- size=10,- extension="txt",- mime_type="text/plain",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="u1",- created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),- used=False,+ assert response == {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}++[email protected]("metadata", ["only", "without", "all"])+def test_document_detail_metadata_projection_and_raw_source(app, documents, metadata):+ documents.get_document.return_value = {+ "id": str(DOC),+ "doc_type": "others",+ "doc_metadata": [],+ "name": "Name",+ "data_source_info": {"file_path": "/path/to/local/file"},+ "data_source_detail_dict": {},+ }+ with app.test_request_context(f"/?metadata={metadata}"):+ response, status = unwrap(DocumentApi.get)(DocumentApi(), CONTEXT, DS, DOC)+ assert status == 200+ documents.get_document.assert_called_once_with(+ CONTEXT, dataset_id=str(DS), document_id=str(DOC), metadata_only=metadata == "only")- upload_file.id = "file-1"- return upload_file---def tenant_ctx():- account = Account(name="Test User", email="[email protected]")- account.id = "u1"- account.role = TenantAccountRole.EDITOR- return (account, "tenant-1")-+ if metadata == "only":+ assert response == {"id": str(DOC), "doc_type": "others", "doc_metadata": []}+ else:+ assert response["data_source_info"] == {"file_path": "/path/to/local/file"}+ assert ("doc_metadata" in response) is (metadata == "all")[email protected](autouse=True)-def bypass_knowledge_rate_limit():- with patch("controllers.console.datasets.datasets_document.check_knowledge_rate_limit") as check:- yield check+def test_invalid_metadata_fails_before_application_call(app, documents):+ with app.test_request_context("/?metadata=invalid"), pytest.raises(InvalidMetadataError):+ unwrap(DocumentApi.get)(DocumentApi(), CONTEXT, DS, DOC)+ documents.get_document.assert_not_called()-def patch_tenant(tenant_ctx):- return tenant_ctx+ ("resource", "verb", "kwargs"),+ [+ (DocumentApi, DocumentApi.delete, {"document_id": DOC}),+ (DatasetDocumentListApi, DatasetDocumentListApi.delete, {}),+ (DocumentPauseApi, DocumentPauseApi.patch, {"document_id": DOC}),+ (DocumentRecoverApi, DocumentRecoverApi.patch, {"document_id": DOC}),+ (DocumentRetryApi, DocumentRetryApi.post, {"req_data": DocumentRetryPayload(document_ids=[str(DOC)])}),+ ],+)[email protected]("documents")+def test_no_content_mutations_return_empty_204(app, resource, verb, kwargs):+ with app.test_request_context("/?document_id=" + str(DOC)):+ result = unwrap(verb)(resource(), request_context=CONTEXT, dataset_id=DS, **kwargs)+ assert result == ("", 204)-def dataset():- return make_dataset()+ ("resource", "verb", "operation", "kwargs"),+ [+ (+ DocumentProcessingApi,+ DocumentProcessingApi.patch,+ lambda service: service.update_processing,+ {"document_id": DOC, "action": "resume"},+ ),+ (DocumentStatusApi, DocumentStatusApi.patch, lambda service: service.change_status, {"action": "disable"}),+ (+ WebsiteDocumentSyncApi,+ WebsiteDocumentSyncApi.get,+ lambda service: service.sync_website,+ {"document_id": DOC},+ ),+ (+ DocumentGenerateSummaryApi,+ DocumentGenerateSummaryApi.post,+ lambda service: service.generate_summary,+ {"req_data": GenerateSummaryPayload(document_list=[str(DOC)])},+ ),+ ],+)+def test_success_mutations_return_success_response(app, documents, resource, verb, operation, kwargs):+ with app.test_request_context("/?document_id=" + str(DOC)):+ result = unwrap(verb)(resource(), request_context=CONTEXT, dataset_id=DS, **kwargs)+ assert result == ({"result": "success"}, 200)+ assert operation(documents).call_count == 1++[email protected]("resource", [DatasetInitApi, DatasetDocumentListApi])+def test_creation_serializes_materialized_dataset_and_documents(app, documents, resource):+ operation = documents.initialize_dataset if resource is DatasetInitApi else documents.create_documents+ operation.return_value = {+ "dataset": {"id": str(DS), "name": "Dataset", "created_at": datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC)},+ "documents": [+ {+ "id": str(DOC),+ "name": "Document",+ "data_source_detail_dict": {},+ "data_source_info_dict": {},+ "doc_metadata_details": [],+ }+ ],+ "batch": "batch-1",+ }+ with app.test_request_context("/", json={"indexing_technique": "economy"}):+ response = unwrap(resource.post)(+ resource(), request_context=CONTEXT, **({} if resource is DatasetInitApi else {"dataset_id": DS})+ )+ assert response["dataset"]["created_at"] == 1704067200+ assert response["documents"][0]["data_source_info"] == {}+ assert response["documents"][0]["doc_metadata"] == []+ assert response["batch"] == "batch-1"++[email protected]("resource", [DocumentIndexingStatusApi, DocumentBatchIndexingStatusApi])+def test_indexing_status_keeps_timestamps_counts_and_error_fields(app, documents, resource):+ value = {+ "id": str(DOC),+ "indexing_status": "paused",+ "completed_segments": 2,+ "total_segments": 3,+ **dict.fromkeys(+ (+ "processing_started_at",+ "parsing_completed_at",+ "cleaning_completed_at",+ "splitting_completed_at",+ "completed_at",+ "paused_at",+ "error",+ "stopped_at",+ )+ ),+ }+ value["processing_started_at"] = datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC)+ batch = resource is DocumentBatchIndexingStatusApi+ operation = documents.get_batch_indexing_status if batch else documents.get_indexing_status+ operation.return_value = {"data": [value]} if batch else value+ with app.test_request_context("/"):+ response = unwrap(resource.get)(+ resource(),+ request_context=CONTEXT,+ dataset_id=DS,+ **({"batch": "batch-1"} if batch else {"document_id": DOC}),+ )+ row = response["data"][0] if batch else response+ assert row["indexing_status"] == "paused"+ assert row["processing_started_at"] == 1704067200+ assert row["completed_segments"] == 2+ assert row["total_segments"] == 3-def document():- return make_document(- indexing_status=IndexingStatus.INDEXING,- is_paused=False,- data_source_info=json.dumps({"upload_file_id": "file-1"}),- )+ ("error", "http_error"),+ [+ (DatasetNotFoundError(), NotFound),+ (DocumentNotFoundError("missing"), NotFound),+ (DatasetAccessDeniedError(), Forbidden),+ (DocumentIndexingStateError("indexing"), DocumentIndexingError),+ (DocumentInvalidActionError("invalid"), InvalidActionError),+ (DocumentProviderError("provider", kind="uninitialized"), ProviderNotInitializeError),+ ],+)+def test_document_errors_are_mapped_at_transport_boundary(error, http_error):+ with pytest.raises(http_error):+ controller._raise_document_error(error)-def patch_dataset(dataset):- with patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset):- yield+def test_unexpected_error_is_not_hidden():+ error = RuntimeError("unexpected")+ with pytest.raises(RuntimeError) as raised:+ controller._raise_document_error(error)+ assert raised.value is error-def patch_scoped_dataset(dataset):- with patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant", return_value=dataset- ):- yield+class TestIndexingEstimateExceptionMapping:+ @staticmethod+ def _registry(method_name: str, error: Exception) -> SimpleNamespace:+ estimates = MagicMock()+ method = estimates.estimate_document if method_name == "estimate_document" else estimates.estimate_batch+ method.side_effect = error+ return SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))-def patch_permission():- with patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission", return_value=None- ):- yield+ @pytest.mark.parametrize(+ ("error", "expected_http_error"),+ [+ (DatasetNotFoundError(), NotFoundError),+ (EstimateDocumentNotFoundError(), NotFoundError),+ (EstimateSourceNotFoundError("source-1"), NotFoundError),+ (IndexingEstimateCredentialUnavailableError(), NotFoundError),+ (DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),+ (EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),+ (UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),+ (IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),+ (IndexingEstimateExecutionError(), IndexingEstimateError),+ ],+ )+ def test_document_estimate_maps_application_errors(+ self,+ error: Exception,+ expected_http_error: type[Exception],+ ) -> None:+ api = DocumentIndexingEstimateApi()+ method = unwrap(api.get)+ context = RequestContext("request-1", None, "account-1", "workspace-1")+ registry = self._registry("estimate_document", error)+ with patch(+ "controllers.console.datasets.datasets_document.application_services",+ return_value=registry,+ ):+ with pytest.raises(expected_http_error):+ method(api, context, UUID(int=1), UUID(int=2))-class _UsesSQLiteSession:- session: Session+ @pytest.mark.parametrize(+ ("error", "expected_http_error"),+ [+ (DatasetNotFoundError(), NotFoundError),+ (EstimateDocumentNotFoundError(), NotFoundError),+ (EstimateSourceNotFoundError("source-1"), NotFoundError),+ (IndexingEstimateCredentialUnavailableError(), NotFoundError),+ (DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),+ (EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),+ (UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),+ (IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),+ (IndexingEstimateExecutionError(), IndexingEstimateError),+ ],+ )+ def test_batch_estimate_maps_application_errors(+ self,+ error: Exception,+ expected_http_error: type[Exception],+ ) -> None:+ api = DocumentBatchIndexingEstimateApi()+ method = unwrap(api.get)+ context = RequestContext("request-1", None, "account-1", "workspace-1")+ registry = self._registry("estimate_batch", error)- @pytest.fixture(autouse=True)- def _inject_sqlite_session(self, sqlite_session: Session) -> None:- self.session = sqlite_session+ with patch(+ "controllers.console.datasets.datasets_document.application_services",+ return_value=registry,+ ):+ with pytest.raises(expected_http_error):+ method(api, context, UUID(int=1), "unknown-batch")@pytest.mark.parametrize(@@ -270,1508 +544,3 @@ def test_document_mutation_routes_require_operation_specific_permissions(method,assert check.scene is expected_sceneassert isinstance(check.locator, DatasetId)---class TestGetProcessRuleApi(_UsesSQLiteSession):- def test_get_default_success(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, _ = patch_tenant- with app.test_request_context("/"):- response = method(api, self.session, user)- assert "rules" in response-- def test_get_with_document_preserves_legacy_segmentation_delimiter(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, _ = patch_tenant-- document = make_document()- session = self.session- dataset = make_dataset()- process_rule = DatasetProcessRule(- dataset_id="ds-1",- mode="custom",- rules=json.dumps({"segmentation": {"delimiter": "---", "max_tokens": 123}}),- created_by="u1",- )- session.add(process_rule)- session.flush()-- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=document,- ) as mock_get_document,- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- response = method(api, session, user)-- mock_get_document.assert_called_once_with("doc-1", session)- assert response["rules"]["segmentation"]["separator"] == "---"- assert response["rules"]["segmentation"]["max_tokens"] == 123- assert "delimiter" not in response["rules"]["segmentation"]-- def test_get_with_document_preserves_null_rules(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, _ = patch_tenant- session = self.session- dataset = make_dataset()- process_rule = DatasetProcessRule(dataset_id="ds-1", mode="custom", rules=None, created_by="u1")- session.add(process_rule)- session.flush()- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=make_document(),- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- response = method(api, session, user)-- assert response["mode"] == "custom"- assert response["rules"] is None-- def test_get_with_document_requires_dataset_readonly_permission(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document()- dataset = make_dataset(tenant_id=tenant_id)-- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=document,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_document.enforce_rbac_checks", create=True) as enforce_checks,- ):- method(api, self.session, user)-- enforce_checks.assert_called_once()- kwargs = enforce_checks.call_args.kwargs- assert kwargs["tenant_id"] == tenant_id- assert kwargs["account_id"] == user.id- assert kwargs["path_args"] == {"dataset_id": dataset.id}- [check] = kwargs["checks"]- assert check.scene is RBACPermission.DATASET_READONLY- assert isinstance(check.locator, DatasetId)-- def test_get_with_document_dataset_not_found(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, _ = patch_tenant- document = make_document()- session = self.session- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=document,- ),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=None),- ):- with pytest.raises(NotFound):- method(api, session, user)---class TestDatasetDocumentListApi(_UsesSQLiteSession):- def test_get_with_fetch_true_counts_segments(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- doc = make_serializable_document()- pagination = MagicMock(items=[doc], total=1)- session = self.session- session.add_all([make_segment(position=1), make_segment(position=2)])- session.flush()- with (- app.test_request_context("/?fetch=true"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- resp = method(api, session, tenant_id, user, "ds-1")- assert resp["data"][0]["id"] == "doc-1"- assert resp["data"][0]["completed_segments"] == 2- assert resp["data"][0]["total_segments"] == 2-- def test_get_with_search_status_and_created_at_sort(- self, app: Flask, patch_tenant, patch_dataset, patch_permission- ):- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- pagination = MagicMock(items=[make_serializable_document()], total=1)- with (- app.test_request_context("/?keyword=test&status=enabled&sort=created_at"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.apply_display_status_filter",- side_effect=lambda q, s: q,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- resp = method(api, self.session, tenant_id, user, "ds-1")- assert resp["total"] == 1-- def test_get_success(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_serializable_document()- pagination = MagicMock(items=[document], total=1)- session = self.session- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- response = method(api, session, tenant_id, user, "ds-1")- assert response["total"] == 1- assert response["data"][0]["id"] == "doc-1"- assert "completed_segments" not in response["data"][0]- assert "total_segments" not in response["data"][0]- assert response["data"][0]["data_source_info"] == {"upload_file_id": "file-1"}- assert response["data"][0]["doc_metadata"] == []-- def test_post_success(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- api = DatasetDocumentListApi()- method = unwrap(api.post)- user, _ = patch_tenant- payload = {"indexing_technique": "economy"}- created_dataset = make_dataset()- created_document = make_document()- session = self.session- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=created_dataset,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.document_create_args_validate",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.save_document_with_dataset_id",- return_value=([created_document], "batch-1"),- ),- ):- response = method(api, session, user, "ds-1")- assert "documents" in response- assert response["dataset"]["id"] == "ds-1"- assert response["documents"][0]["id"] == "doc-1"- assert response["documents"][0]["data_source_info"] == {}- assert response["documents"][0]["doc_metadata"] == []- assert "data_source_info_dict" not in response["documents"][0]- assert "doc_metadata_details" not in response["documents"][0]-- def test_post_forbidden(self, app: Flask):- api = DatasetDocumentListApi()- method = unwrap(api.post)- user = make_account(TenantAccountRole.NORMAL)- with (- app.test_request_context("/", json={}),- patch.object(type(console_ns), "payload", {}),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- ):- with pytest.raises(Forbidden):- method(api, self.session, user, "ds-1")-- def test_get_with_fetch_true_and_invalid_fetch(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- pagination = MagicMock(items=[make_serializable_document()], total=1)- with (- app.test_request_context("/?fetch=maybe"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- response = method(api, self.session, tenant_id, user, "ds-1")- assert response["total"] == 1-- def test_get_sort_hit_count(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- pagination = MagicMock(items=[], total=0)- with (- app.test_request_context("/?sort=hit_count"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- response = method(api, self.session, tenant_id, user, "ds-1")- assert response["total"] == 0---class TestDatasetInitApi(_UsesSQLiteSession):- def test_post_success_serializes_created_dataset_and_documents(self, app: Flask, patch_tenant):- api = DatasetInitApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"indexing_technique": "economy"}- created_dataset = make_dataset()- created_document = make_document(id="doc-init")- session = self.session- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- config_overrides_context(RBAC_ENABLED=True),- patch(- "controllers.console.datasets.datasets_document.DocumentService.document_create_args_validate",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.save_document_without_dataset_id",- return_value=(created_dataset, [created_document], "batch-init"),- ),- patch(- "controllers.console.datasets.datasets_document.enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist"- ),- ):- response = method(api, session, tenant_id, user)- assert response["dataset"]["id"] == "ds-1"- assert response["documents"][0]["id"] == "doc-init"- assert response["documents"][0]["data_source_info"] == {}- assert response["documents"][0]["doc_metadata"] == []- assert response["batch"] == "batch-init"- assert created_dataset.permission == DatasetPermissionEnum.ALL_TEAM-- def test_post_success_syncs_creator_access_on_upload_create(self, app: Flask, patch_tenant):- """RBAC: the console upload create must keep the creator bound to their own dataset.-- Regression guard for the #42430 flip to automatic_include_workspace_members=False:- unlike POST /console/api/datasets (DatasetService.create_empty_dataset syncs the- creator internally), this path bypasses that helper, so the controller has to sync- the creator explicitly or the creator gets 403 on their own dataset.- """- api = DatasetInitApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"indexing_technique": "economy"}- created_dataset = make_dataset()- created_document = make_document(id="doc-init")- session = self.session- captured = {}- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- config_overrides_context(RBAC_ENABLED=True),- patch(- "controllers.console.datasets.datasets_document.DocumentService.document_create_args_validate",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.save_document_without_dataset_id",- return_value=(created_dataset, [created_document], "batch-init"),- ),- patch(- "controllers.console.datasets.datasets_document.enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist"- ),- patch(- "controllers.console.datasets.datasets_document.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"- ) as sync_creator,- ):- response = method(api, session, tenant_id, user)- captured["sync_calls"] = sync_creator.call_args_list- assert sync_creator.called, "upload create must sync the creator's own access byte-through"- assert response["dataset"]["id"] == "ds-1"---class TestDocumentResource(_UsesSQLiteSession):- def test_get_document_resolves_owner_chain(self, dataset):- api = DocumentResource()- session = self.session- user = make_account()- document = make_document()-- with (- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ) as get_dataset,- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission"- ) as check_permission,- patch(- "controllers.console.datasets.datasets_document.DatasetRefService.get_document_by_ref",- return_value=document,- ) as get_document,- ):- assert api.get_document(session, "ds-1", "doc-1", user, "tenant-1") is document-- get_dataset.assert_called_once_with("ds-1", "tenant-1", session=session)- check_permission.assert_called_once_with(dataset, user, session)- get_document.assert_called_once_with(- DocumentRef(dataset=DatasetRef(tenant_id="tenant-1", dataset_id="ds-1"), document_id="doc-1"),- session=session,- )-- def test_get_document_delegates_permission_check_in_rbac_mode(self, dataset):- api = DocumentResource()- session = self.session- user = make_account()- with (- config_overrides_context(RBAC_ENABLED=True),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission"- ) as check_permission,- patch(- "controllers.console.datasets.datasets_document.DatasetRefService.get_document_by_ref",- return_value=make_document(),- ),- ):- api.get_document(session, "ds-1", "doc-1", user, "tenant-1")-- check_permission.assert_called_once_with(dataset, user, session)---class TestDocumentApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask, patch_tenant):- api = DocumentApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document_detail()- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document),- patch("controllers.console.datasets.datasets_document.DatasetService.get_process_rules", return_value={}),- ):- response, status = method(api, self.session, tenant_id, user, "ds-1", "doc-1")- assert status == 200-- def test_get_invalid_metadata(self, app: Flask, patch_tenant):- api = DocumentApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- with (- app.test_request_context("/?metadata=wrong"),- patch.object(api, "get_document", return_value=make_document()),- ):- with pytest.raises(InvalidMetadataError):- method(api, self.session, tenant_id, user, "ds-1", "doc-1")-- def test_delete_success(self, app: Flask, patch_tenant, patch_dataset):- api = DocumentApi()- method = unwrap(api.delete)- user, tenant_id = patch_tenant- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch.object(api, "get_document", return_value=make_document()),- patch("controllers.console.datasets.datasets_document.DocumentService.delete_document", return_value=None),- ):- response, status = method(api, self.session, tenant_id, user, "ds-1", "doc-1")- assert status == 204-- def test_delete_indexing_error(self, app: Flask, patch_tenant, patch_dataset):- api = DocumentApi()- method = unwrap(api.delete)- user, tenant_id = patch_tenant- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch.object(api, "get_document", return_value=make_document()),- patch(- "controllers.console.datasets.datasets_document.DocumentService.delete_document",- side_effect=services.errors.document.DocumentIndexingError(),- ),- ):- with pytest.raises(DocumentIndexingError):- method(api, self.session, tenant_id, user, "ds-1", "doc-1")---class TestDocumentDownloadApi(_UsesSQLiteSession):- def test_download_success(self, app: Flask, patch_tenant):- api = DocumentDownloadApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document()- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_download_url",- return_value="url",- ),- ):- response = method(api, self.session, tenant_id, user, "ds-1", "doc-1")- assert response["url"] == "url"---class TestDocumentProcessingApi(_UsesSQLiteSession):- def test_processing_forbidden_when_not_editor(self, app: Flask):- api = DocumentProcessingApi()- method = unwrap(api.patch)- user = make_account(TenantAccountRole.NORMAL)- with app.test_request_context("/"), patch.object(api, "get_document", return_value=make_document()):- with pytest.raises(Forbidden):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "pause")-- def test_resume_from_error_state(self, app: Flask, patch_tenant):- api = DocumentProcessingApi()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- doc = make_document(indexing_status=IndexingStatus.ERROR, is_paused=True)- session = self.session- with app.test_request_context("/"), patch.object(api, "get_document", return_value=doc):- _, status = method(api, session, tenant_id, user, "ds-1", "doc-1", "resume")- assert status == 200-- def test_resume_success(self, app: Flask, patch_tenant):- api = DocumentProcessingApi()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- document = make_document(indexing_status=IndexingStatus.PAUSED, is_paused=True)- session = self.session- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1", "resume")- assert status == 200-- def test_pause_success(self, app: Flask, patch_tenant):- api = DocumentProcessingApi()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- document = make_document(indexing_status=IndexingStatus.INDEXING)- session = self.session- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1", "pause")- assert status == 200-- def test_pause_invalid(self, app: Flask, patch_tenant):- api = DocumentProcessingApi()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- document = make_document(indexing_status=IndexingStatus.COMPLETED)- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- with pytest.raises(InvalidActionError):- method(api, self.session, tenant_id, user, "ds-1", "doc-1", "pause")---class TestDocumentMetadataApi(_UsesSQLiteSession):- def test_put_metadata_schema_filtering(self, app: Flask, patch_tenant):- api = DocumentMetadataApi()- method = unwrap(api.put)- user, tenant_id = patch_tenant- doc = make_document()- payload = {"doc_type": "invoice", "doc_metadata": {"amount": 10, "invalid": "x"}}- schema = {"amount": int}- session = self.session- req_data = DocumentMetadataUpdatePayload.model_validate(payload)- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=doc),- patch(- "controllers.console.datasets.datasets_document.DocumentService.DOCUMENT_METADATA_SCHEMA",- {"invoice": schema},- ),- ):- method(api, req_data, session, tenant_id, user, "ds-1", "doc-1")- assert doc.doc_metadata == {"amount": 10}-- def test_put_success(self, app: Flask, patch_tenant):- api = DocumentMetadataApi()- method = unwrap(api.put)- user, tenant_id = patch_tenant- document = make_document()- payload = {"doc_type": "others", "doc_metadata": {"a": 1}}- session = self.session- req_data = DocumentMetadataUpdatePayload.model_validate(payload)- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_document.DocumentService.DOCUMENT_METADATA_SCHEMA",- {"others": {}},- ),- ):- response, status = method(api, req_data, session, tenant_id, user, "ds-1", "doc-1")- assert status == 200-- def test_put_invalid_payload(self, app: Flask, patch_tenant):- api = DocumentMetadataApi()- method = unwrap(api.put)- user, tenant_id = patch_tenant- req_data = DocumentMetadataUpdatePayload.model_validate({})- with app.test_request_context("/"), patch.object(api, "get_document", return_value=make_document()):- with pytest.raises(ValueError):- method(api, req_data, self.session, tenant_id, user, "ds-1", "doc-1")-- def test_put_invalid_doc_type(self, app: Flask, patch_tenant):- api = DocumentMetadataApi()- method = unwrap(api.put)- user, tenant_id = patch_tenant- payload = {"doc_type": "invalid", "doc_metadata": {}}- req_data = DocumentMetadataUpdatePayload.model_validate(payload)- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=make_document()),- patch(- "controllers.console.datasets.datasets_document.DocumentService.DOCUMENT_METADATA_SCHEMA",- {"others": {}},- ),- ):- with pytest.raises(ValueError):- method(api, req_data, self.session, tenant_id, user, "ds-1", "doc-1")---class TestDocumentStatusApi(_UsesSQLiteSession):- def test_patch_success(self, app: Flask, patch_tenant, patch_dataset):- api = DocumentStatusApi()- method = unwrap(api.patch)- user, _ = patch_tenant- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.batch_update_document_status",- return_value=None,- ),- ):- response, status = method(api, self.session, user, "ds-1", "enable")- assert status == 200-- def test_patch_invalid_action(self, app: Flask, patch_tenant, patch_dataset):- api = DocumentStatusApi()- method = unwrap(api.patch)- user, _ = patch_tenant- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.batch_update_document_status",- side_effect=ValueError("x"),- ),- ):- with pytest.raises(InvalidActionError):- method(api, self.session, user, "ds-1", "enable")---class TestDocumentRetryApi(_UsesSQLiteSession):- def test_retry_archived_document_skipped(self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission):- api = DocumentRetryApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"document_ids": ["doc-1"]}- req_data = DocumentRetryPayload.model_validate(payload)- doc = make_document(indexing_status=IndexingStatus.INDEXING)- session = self.session- session.add(doc)- session.flush()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DocumentService.check_archived", return_value=True),- patch("controllers.console.datasets.datasets_document.DocumentService.retry_document") as retry_mock,- ):- resp, status = method(api, req_data, session, tenant_id, user, "ds-1")- assert status == 204- retry_mock.assert_called_once_with("ds-1", [], session)-- def test_retry_success(self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission):- api = DocumentRetryApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"document_ids": ["doc-1"]}- req_data = DocumentRetryPayload.model_validate(payload)- document = make_document(indexing_status=IndexingStatus.INDEXING)- session = self.session- session.add(document)- session.flush()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DocumentService.check_archived", return_value=False),- patch(- "controllers.console.datasets.datasets_document.DocumentService.retry_document", return_value=None- ) as retry_mock,- ):- response, status = method(api, req_data, session, tenant_id, user, "ds-1")- assert status == 204- retry_mock.assert_called_once_with("ds-1", [document], session)-- def test_retry_loads_selected_documents_in_one_scoped_query(- self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission- ):- api = DocumentRetryApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"document_ids": ["doc-1", "doc-2"]}- req_data = DocumentRetryPayload.model_validate(payload)- first_document = make_document(id="doc-1", indexing_status=IndexingStatus.ERROR)- second_document = make_document(id="doc-2", position=2, indexing_status=IndexingStatus.ERROR)- decoy = make_document(id="doc-decoy", tenant_id="other-tenant", dataset_id="other-dataset")- session = self.session- session.add_all([first_document, second_document, decoy])- session.flush()-- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DocumentService.check_archived", return_value=False),- patch(- "controllers.console.datasets.datasets_document.DocumentService.retry_document", return_value=None- ) as retry_mock,- ):- response, status = method(api, req_data, session, tenant_id, user, "ds-1")-- assert status == 204- retry_mock.assert_called_once_with("ds-1", [first_document, second_document], session)-- def test_retry_skips_completed_document(self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission):- api = DocumentRetryApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload = {"document_ids": ["doc-1"]}- req_data = DocumentRetryPayload.model_validate(payload)- document = make_document(indexing_status=IndexingStatus.COMPLETED)- session = self.session- session.add(document)- session.flush()- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.retry_document", return_value=None- ) as retry_mock,- ):- response, status = method(api, req_data, session, tenant_id, user, "ds-1")- assert status == 204- retry_mock.assert_called_once_with("ds-1", [], session)-- def test_retry_foreign_dataset_has_no_side_effects(self, app: Flask, patch_tenant, bypass_knowledge_rate_limit):- api = DocumentRetryApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- session = self.session- payload = {"document_ids": ["doc-1"]}- req_data = DocumentRetryPayload.model_validate(payload)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=None,- ),- patch("controllers.console.datasets.datasets_document.DocumentService.retry_document") as retry_document,- ):- with pytest.raises(NotFound):- method(api, req_data, session, tenant_id, user, "foreign-dataset")-- bypass_knowledge_rate_limit.assert_not_called()- retry_document.assert_not_called()---class TestDocumentPauseRecoverApi(_UsesSQLiteSession):- @pytest.mark.parametrize(- ("api_type", "service_method"),- [(DocumentPauseApi, "pause_document"), (DocumentRecoverApi, "recover_document")],- )- def test_patch_uses_scoped_document(- self, app: Flask, patch_tenant, bypass_knowledge_rate_limit, api_type, service_method- ):- api = api_type()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- session = self.session- document = make_document()-- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document) as get_document,- patch("controllers.console.datasets.datasets_document.DocumentService.check_archived", return_value=False),- patch(- f"controllers.console.datasets.datasets_document.DocumentService.{service_method}"- ) as process_document,- ):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1")-- assert (response, status) == ("", 204)- get_document.assert_called_once_with(session, "ds-1", "doc-1", user, tenant_id)- bypass_knowledge_rate_limit.assert_called_once_with()- process_document.assert_called_once_with(document, session)---class TestWebsiteDocumentSyncApi(_UsesSQLiteSession):- def test_get_uses_scoped_dataset_and_document(self, app: Flask, patch_tenant, dataset):- api = WebsiteDocumentSyncApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- session = self.session- document = make_document(data_source_type=DataSourceType.WEBSITE_CRAWL)-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ) as get_dataset,- patch.object(api, "get_document", return_value=document) as get_document,- patch("controllers.console.datasets.datasets_document.DocumentService.check_archived", return_value=False),- patch(- "controllers.console.datasets.datasets_document.DocumentService.sync_website_document"- ) as sync_document,- ):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1")-- assert status == 200- assert response["result"] == "success"- get_dataset.assert_called_once_with("ds-1", tenant_id, session=session)- get_document.assert_called_once_with(session, dataset.id, "doc-1", user, tenant_id)- sync_document.assert_called_once_with(dataset, document, session)-- def test_get_rejects_non_editor_before_loading_document(self, app: Flask, dataset):- api = WebsiteDocumentSyncApi()- method = unwrap(api.get)- user = make_account(TenantAccountRole.NORMAL)- session = self.session-- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ),- patch.object(api, "get_document") as get_document,- patch(- "controllers.console.datasets.datasets_document.DocumentService.sync_website_document"- ) as sync_document,- ):- with pytest.raises(Forbidden):- method(api, session, "tenant-1", user, "ds-1", "doc-1")-- get_document.assert_not_called()- sync_document.assert_not_called()---class TestDocumentPipelineExecutionLogApi(_UsesSQLiteSession):- def test_get_log_success(self, app: Flask, patch_tenant):- api = DocumentPipelineExecutionLogApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- log = DocumentPipelineExecutionLog(- pipeline_id="pipeline-1",- document_id="trusted-doc",- datasource_type="file",- datasource_info="{}",- datasource_node_id="n1",- input_data={},- created_by="u1",- )- document = make_document(id="trusted-doc")- session = self.session- session.add(log)- session.flush()- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document) as get_document,- ):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1")- assert status == 200- get_document.assert_called_once_with(session, "ds-1", "doc-1", user, tenant_id)- assert response["datasource_node_id"] == "n1"---class TestDocumentGenerateSummaryApi(_UsesSQLiteSession):- def test_generate_summary_missing_documents(self, app: Flask, patch_tenant, patch_permission):- api = DocumentGenerateSummaryApi()- method = unwrap(api.post)- user, _ = patch_tenant- dataset = make_dataset(indexing_technique="high_quality", summary_index_setting={"enable": True})- payload = {"document_list": ["doc-1", "doc-2"]}- req_data = GenerateSummaryPayload.model_validate(payload)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_documents_by_ids",- return_value=[make_document(id="doc-1")],- ),- ):- with pytest.raises(NotFound):- method(api, req_data, self.session, user, "ds-1")-- def test_generate_not_enabled(self, app: Flask, patch_tenant, patch_permission):- api = DocumentGenerateSummaryApi()- method = unwrap(api.post)- user, _ = patch_tenant- dataset = make_dataset(indexing_technique="high_quality", summary_index_setting={"enable": False})- payload = {"document_list": ["doc-1"]}- req_data = GenerateSummaryPayload.model_validate(payload)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- ):- with pytest.raises(ValueError):- method(api, req_data, self.session, user, "ds-1")-- def test_generate_summary_success_with_qa_skip(self, app: Flask, patch_tenant, patch_permission):- api = DocumentGenerateSummaryApi()- method = unwrap(api.post)- user, _ = patch_tenant- dataset = make_dataset(indexing_technique="high_quality", summary_index_setting={"enable": True})- doc1 = make_document(id="doc-1", doc_form=IndexStructureType.QA_INDEX)- doc2 = make_document(id="doc-2", position=2, doc_form=IndexStructureType.PARAGRAPH_INDEX)- payload = {"document_list": ["doc-1", "doc-2"]}- req_data = GenerateSummaryPayload.model_validate(payload)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_documents_by_ids",- return_value=[doc1, doc2],- ),- patch(- "controllers.console.datasets.datasets_document.generate_summary_index_task.delay", return_value=None- ),- ):- response, status = method(api, req_data, self.session, user, "ds-1")- assert status == 200---class TestDocumentSummaryStatusApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask, patch_tenant, patch_permission):- api = DocumentSummaryStatusApi()- method = unwrap(api.get)- user, _ = patch_tenant- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=make_dataset(),- ),- patch(- "services.summary_index_service.SummaryIndexService.get_document_summary_status_detail",- return_value={- "total_segments": 1,- "summary_status": {"timeout": 1},- "summaries": [- {- "segment_id": "segment-1",- "segment_position": 1,- "status": "timeout",- }- ],- },- ),- ):- response, status = method(api, self.session, user, "ds-1", "doc-1")- assert status == 200- assert response["summary_status"]["timeout"] == 1- assert response["summaries"][0]["status"] == "timeout"---class TestDocumentIndexingEstimateApi(_UsesSQLiteSession):- def test_indexing_estimate_file_not_found(self, app: Flask, patch_tenant):- api = DocumentIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_info=json.dumps({"upload_file_id": "file-1"}),- )- session = self.session- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- with pytest.raises(NotFound):- method(api, session, tenant_id, user, "ds-1", "doc-1")-- def test_indexing_estimate_generic_exception(self, app: Flask, patch_tenant):- api = DocumentIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_info=json.dumps({"upload_file_id": "file-1"}),- )- upload_file = make_upload_file()- mock_indexing_runner = MagicMock()- mock_indexing_runner.indexing_estimate.side_effect = RuntimeError("Some indexing error")- session = self.session- session.add(upload_file)- session.flush()- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document),- patch("controllers.console.datasets.datasets_document.ExtractSetting", return_value=MagicMock()),- patch("controllers.console.datasets.datasets_document.IndexingRunner", return_value=mock_indexing_runner),- ):- with pytest.raises(IndexingEstimateError):- method(api, session, tenant_id, user, "ds-1", "doc-1")-- def test_get_finished(self, app: Flask, patch_tenant):- api = DocumentIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(indexing_status=IndexingStatus.COMPLETED)- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- with pytest.raises(DocumentAlreadyFinishedError):- method(api, self.session, tenant_id, user, "ds-1", "doc-1")---class TestDocumentBatchDownloadZipApi(_UsesSQLiteSession):- def test_post_no_documents(self, app: Flask, patch_tenant):- api = DocumentBatchDownloadZipApi()- method = unwrap(api.post)- user, tenant_id = patch_tenant- payload: dict[str, list[str]] = {"document_ids": []}- with app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload):- with pytest.raises(ValueError):- method(api, self.session, tenant_id, user, "ds-1")---class TestDatasetDocumentListApiDelete(_UsesSQLiteSession):- def test_delete_success(self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission):- """Test successful deletion of documents"""- api = DatasetDocumentListApi()- method = unwrap(api.delete)- user, tenant_id = patch_tenant- session = self.session- with (- app.test_request_context("/?document_id=doc-1&document_id=doc-2"),- patch("controllers.console.datasets.datasets_document.DocumentService.delete_documents", return_value=None),- ):- response, status = method(api, session, tenant_id, user, "ds-1")- assert status == 204-- def test_delete_indexing_error(self, app: Flask, patch_tenant, patch_scoped_dataset, patch_permission):- """Test deletion with indexing error"""- api = DatasetDocumentListApi()- method = unwrap(api.delete)- user, tenant_id = patch_tenant- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.delete_documents",- side_effect=services.errors.document.DocumentIndexingError(),- ),- ):- with pytest.raises(DocumentIndexingError):- method(api, self.session, tenant_id, user, "ds-1")-- def test_delete_dataset_not_found(self, app: Flask, patch_tenant, bypass_knowledge_rate_limit):- """Test deletion when dataset not found"""- api = DatasetDocumentListApi()- method = unwrap(api.delete)- user, tenant_id = patch_tenant- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset_for_tenant",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.delete_documents"- ) as delete_documents,- ):- with pytest.raises(NotFound):- method(api, self.session, tenant_id, user, "foreign-dataset")-- bypass_knowledge_rate_limit.assert_not_called()- delete_documents.assert_not_called()---class TestDocumentBatchIndexingEstimateApi(_UsesSQLiteSession):- def test_batch_indexing_estimate_website(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- doc = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_type=DataSourceType.WEBSITE_CRAWL,- data_source_info=json.dumps(- {- "provider": "firecrawl",- "job_id": "j1",- "url": "https://x.com",- "mode": "single",- "only_main_content": True,- }- ),- )- with (- app.test_request_context("/"),- patch.object(api, "get_batch_documents", return_value=[doc]),- patch(- "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",- return_value=IndexingEstimate(total_segments=2, preview=[]),- ),- ):- resp, status = method(api, self.session, tenant_id, user, "ds-1", "batch-1")- assert status == 200-- def test_batch_indexing_estimate_notion(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- doc = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_type=DataSourceType.NOTION_IMPORT,- data_source_info=json.dumps(- {- "credential_id": "c1",- "notion_workspace_id": "w1",- "notion_page_id": "p1",- "type": "page",- }- ),- )- with (- app.test_request_context("/"),- patch.object(api, "get_batch_documents", return_value=[doc]),- patch(- "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",- return_value=IndexingEstimate(total_segments=1, preview=[]),- ),- ):- resp, status = method(api, self.session, tenant_id, user, "ds-1", "batch-1")- assert status == 200-- def test_batch_estimate_unsupported_datasource(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_type="unknown",- data_source_info="{}",- )- with app.test_request_context("/"), patch.object(api, "get_batch_documents", return_value=[document]):- with pytest.raises(ValueError):- method(api, self.session, tenant_id, user, "ds-1", "batch-1")-- def test_get_batch_estimate_invalid_batch(self, app: Flask, patch_tenant):- """Test batch estimation with invalid batch"""- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- with app.test_request_context("/"), patch.object(api, "get_batch_documents", side_effect=NotFound()):- with pytest.raises(NotFound):- method(api, self.session, tenant_id, user, "ds-1", "invalid-batch")---class TestDocumentBatchIndexingStatusApi(_UsesSQLiteSession):- def test_get_batch_status_success_serializes_status_shape(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingStatusApi()- method = unwrap(api.get)- user, _ = patch_tenant- error = format_vector_space_admission_error(61, 50)- document = make_document(- indexing_status=IndexingStatus.ERROR,- is_paused=False,- error=error,- )- session = self.session- session.add_all([make_segment(position=1), make_segment(position=2), make_segment(position=3, completed=False)])- session.flush()- with app.test_request_context("/"), patch.object(api, "get_batch_documents", return_value=[document]):- response = method(api, session, user, "ds-1", "batch-1")- assert response == {- "data": [- {- "id": "doc-1",- "indexing_status": "error",- "processing_started_at": None,- "parsing_completed_at": None,- "cleaning_completed_at": None,- "splitting_completed_at": None,- "completed_at": None,- "paused_at": None,- "error": error,- "error_code": VECTOR_SPACE_ADMISSION_ERROR_CODE,- "estimated_vector_space_mb": 61,- "vector_space_limit_mb": 50,- "stopped_at": None,- "completed_segments": 2,- "total_segments": 3,- }- ]- }-- def test_get_batch_status_invalid_batch(self, app: Flask, patch_tenant):- """Test batch status with invalid batch"""- api = DocumentBatchIndexingStatusApi()- method = unwrap(api.get)- user, _ = patch_tenant- with app.test_request_context("/"), patch.object(api, "get_batch_documents", side_effect=NotFound()):- with pytest.raises(NotFound):- method(api, self.session, user, "ds-1", "invalid-batch")-- def test_get_batch_status_uses_one_aggregate_query(self, app: Flask, patch_tenant, sqlite_engine):- api = DocumentBatchIndexingStatusApi()- method = unwrap(api.get)- user, _ = patch_tenant- documents = [make_document(id=f"doc-{index}", position=index) for index in range(1, 6)]- session = self.session- session.add_all(- [- make_segment(- position=position,- completed=position <= 2,- document_id=document.id,- )- for document in documents- for position in range(1, 4)- ]- )- session.flush()-- select_statements: list[str] = []-- def record_select(_connection, _cursor, statement, _parameters, _context, _executemany):- if statement.lstrip().upper().startswith("SELECT"):- select_statements.append(statement)-- event.listen(sqlite_engine, "before_cursor_execute", record_select)- try:- with app.test_request_context("/"), patch.object(api, "get_batch_documents", return_value=documents):- response = method(api, session, user, "ds-1", "batch-1")- finally:- event.remove(sqlite_engine, "before_cursor_execute", record_select)-- assert len(response["data"]) == 5- assert all(item["completed_segments"] == 2 for item in response["data"])- assert all(item["total_segments"] == 3 for item in response["data"])- assert len(select_statements) == 1---class TestDocumentIndexingStatusApi(_UsesSQLiteSession):- def test_get_status_success_serializes_status_shape(self, app: Flask, patch_tenant):- api = DocumentIndexingStatusApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(- indexing_status=IndexingStatus.INDEXING,- is_paused=False,- )- session = self.session- session.add_all([make_segment(position=position, completed=position == 1) for position in range(1, 5)])- session.flush()- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- response = method(api, session, tenant_id, user, "ds-1", "doc-1")- assert response["id"] == "doc-1"- assert response["indexing_status"] == "indexing"- assert response["completed_segments"] == 1- assert response["total_segments"] == 4-- def test_get_status_document_not_found(self, app: Flask, patch_tenant):- """Test getting status for non-existent document"""- api = DocumentIndexingStatusApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- with app.test_request_context("/"), patch.object(api, "get_document", side_effect=NotFound()):- with pytest.raises(NotFound):- method(api, self.session, tenant_id, user, "ds-1", "invalid-doc")---class TestDocumentRenameApi(_UsesSQLiteSession):- def test_post_success_serializes_document_shape(self, app: Flask, patch_tenant):- api = DocumentRenameApi()- method = unwrap(api.post)- user, _ = patch_tenant- payload = {"name": "Renamed Document"}- req_data = DocumentRenamePayload.model_validate(payload)- renamed_document = make_document(id="doc-renamed", name="Renamed Document")- session = self.session- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=make_dataset()- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_operator_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_document.DocumentService.rename_document",- return_value=renamed_document,- ),- ):- response = method(api, req_data, session, user, "ds-1", "doc-1")- assert response["id"] == "doc-renamed"- assert response["name"] == "Renamed Document"- assert response["data_source_info"] == {}- assert response["doc_metadata"] == []- assert "data_source_info_dict" not in response---class TestDocumentApiMetadata(_UsesSQLiteSession):- def test_get_with_only_option(self, app: Flask, patch_tenant):- """Test get with 'only' metadata option"""- api = DocumentApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document_detail(doc_metadata_details=[])- with (- app.test_request_context("/?metadata=only"),- patch.object(api, "get_document", return_value=document),- patch("controllers.console.datasets.datasets_document.DatasetService.get_process_rules", return_value={}),- ):- response, status = method(api, self.session, tenant_id, user, "ds-1", "doc-1")- assert status == 200-- def test_get_with_without_option(self, app: Flask, patch_tenant):- """Test get with 'without' metadata option"""- api = DocumentApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document_detail()- with (- app.test_request_context("/?metadata=without"),- patch.object(api, "get_document", return_value=document),- patch("controllers.console.datasets.datasets_document.DatasetService.get_process_rules", return_value={}),- ):- response, status = method(api, self.session, tenant_id, user, "ds-1", "doc-1")- assert status == 200---class TestDocumentGenerateSummaryApiSuccess(_UsesSQLiteSession):- def test_generate_not_enabled_high_quality(self, app: Flask, patch_tenant, patch_permission):- """Test summary generation on non-high-quality dataset"""- api = DocumentGenerateSummaryApi()- method = unwrap(api.post)- user, _ = patch_tenant- dataset = make_dataset(indexing_technique="economy", summary_index_setting={"enable": True})- payload = {"document_list": ["doc-1"]}- req_data = GenerateSummaryPayload.model_validate(payload)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- ):- with pytest.raises(ValueError):- method(api, req_data, self.session, user, "ds-1")---class TestDocumentProcessingApiResume(_UsesSQLiteSession):- def test_resume_invalid_status(self, app: Flask, patch_tenant):- """Test resume on non-paused document"""- api = DocumentProcessingApi()- method = unwrap(api.patch)- user, tenant_id = patch_tenant- document = make_document(indexing_status=IndexingStatus.COMPLETED, is_paused=False)- with app.test_request_context("/"), patch.object(api, "get_document", return_value=document):- with pytest.raises(InvalidActionError):- method(api, self.session, tenant_id, user, "ds-1", "doc-1", "resume")---class TestDocumentPermissionCases(_UsesSQLiteSession):- def test_document_batch_get_permission_denied(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=make_dataset(),- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("No permission"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, tenant_id, user, "ds-1", "batch-1")-- def test_document_batch_get_documents_not_found(self, app: Flask, patch_tenant):- api = DocumentBatchIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- with (- app.test_request_context("/"),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=make_dataset(),- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- patch.object(api, "get_batch_documents", return_value=None),- ):- response, status = method(api, self.session, tenant_id, user, "ds-1", "batch-1")- assert status == 200- assert response == {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}-- def test_process_rule_get_by_document_success(self, app: Flask, patch_tenant):- api = GetProcessRuleApi()- method = unwrap(api.get)- user, _ = patch_tenant- document = make_document()- session = self.session- dataset = make_dataset()- process_rule = DatasetProcessRule(dataset_id="ds-1", mode="custom", rules=json.dumps({"a": 1}), created_by="u1")- session.add(process_rule)- session.flush()- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=document,- ),- patch("controllers.console.datasets.datasets_document.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- result = method(api, session, user)- if isinstance(result, tuple):- response, status = result- else:- response, status = (result, 200)- assert status == 200- assert response["mode"] == "custom"-- def test_process_rule_permission_denied(self, app: Flask):- api = GetProcessRuleApi()- method = unwrap(api.get)- user = make_account()- document = make_document()- session = self.session- with (- app.test_request_context("/?document_id=doc-1"),- patch(- "controllers.console.datasets.datasets_document.DocumentService.get_document_by_id",- return_value=document,- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.get_dataset",- return_value=make_dataset(),- ),- patch(- "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("No permission"),- ),- ):- with pytest.raises(Forbidden):- method(api, session, user)---class TestDocumentListAdvancedCases(_UsesSQLiteSession):- def test_document_list_with_multiple_sort_options(self, app: Flask, patch_tenant, patch_dataset, patch_permission):- """Test document list with different sort options"""- api = DatasetDocumentListApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- pagination = MagicMock(items=[make_serializable_document()], total=1)- with (- app.test_request_context("/?sort=updated_at"),- patch("controllers.console.datasets.datasets_document.paginate_query", return_value=pagination),- patch(- "controllers.console.datasets.datasets_document.DocumentService.enrich_documents_with_summary_index_status",- return_value=None,- ),- ):- response = method(api, self.session, tenant_id, user, "ds-1")- assert response["total"] == 1-- def test_document_metadata_with_schema_validation(self, app: Flask, patch_tenant):- """Test document metadata update with schema validation"""- api = DocumentMetadataApi()- method = unwrap(api.put)- user, tenant_id = patch_tenant- doc = make_document()- payload = {"doc_type": "contract", "doc_metadata": {"amount": 5000, "currency": "USD", "invalid_field": "x"}}- schema = {"amount": int, "currency": str}- session = self.session- req_data = DocumentMetadataUpdatePayload.model_validate(payload)- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=doc),- patch(- "controllers.console.datasets.datasets_document.DocumentService.DOCUMENT_METADATA_SCHEMA",- {"contract": schema},- ),- ):- response, status = method(api, req_data, session, tenant_id, user, "ds-1", "doc-1")- assert status == 200- assert doc.doc_metadata == {"amount": 5000, "currency": "USD"}---class TestDocumentIndexingEdgeCases(_UsesSQLiteSession):- def test_document_indexing_with_extraction_setting(self, app: Flask, patch_tenant):- api = DocumentIndexingEstimateApi()- method = unwrap(api.get)- user, tenant_id = patch_tenant- document = make_document(- indexing_status=IndexingStatus.INDEXING,- data_source_info=json.dumps({"upload_file_id": "file-1"}),- )- upload_file = make_upload_file()- session = self.session- session.add(upload_file)- session.flush()- with (- app.test_request_context("/"),- patch.object(api, "get_document", return_value=document),- patch("controllers.console.datasets.datasets_document.ExtractSetting", return_value=MagicMock()),- patch(- "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",- return_value=IndexingEstimate(total_segments=5, preview=[]),- ),- ):- response, status = method(api, session, tenant_id, user, "ds-1", "doc-1")- assert status == 200diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.pyindex 88dab19cd99..92b64d4875f 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py@@ -1,461 +1,196 @@-"""-Unit tests for the dataset document download endpoint.+"""Document downloads through the controller, application service, and SQLite adapter."""-These tests validate that the controller returns a signed download URL for-upload-file documents, and rejects unsupported or missing file cases.-"""--from __future__ import annotations--import importlibimport json-import sys-from collections import UserDict-from datetime import UTC, datetime+from datetime import datetimefrom inspect import unwrapfrom io import BytesIO+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import create_autospec+from uuid import UUIDfrom zipfile import ZipFileimport pytest-from flask import Flask-from sqlalchemy.orm import Session+from flask import send_file+from sqlalchemy import eventfrom werkzeug.exceptions import NotFound-from extensions.storage.storage_type import StorageType-from models.account import Account, TenantAccountRole+from controllers.console.datasets import datasets_document as controller+from machinery.context import RequestContextfrom models.dataset import Dataset, Document-from models.enums import CreatorUserRolefrom models.model import UploadFile---def app() -> Flask:- """Create a minimal Flask app for request-context based controller tests."""- app = Flask(__name__)- app.config["TESTING"] = True- return app---def datasets_document_module(monkeypatch: pytest.MonkeyPatch):- """- Reload `controllers.console.datasets.datasets_document` with lightweight decorators.-- We patch auth / setup / rate-limit decorators to no-ops so we can unit test the- controller logic without requiring the full console stack.- """-- from controllers.console import console_ns, wraps- from libs import login-- def _noop(func): # type: ignore[no-untyped-def]- return func-- # Bypass login/setup/account checks in unit tests.- monkeypatch.setattr(login, "login_required", _noop)- monkeypatch.setattr(wraps, "setup_required", _noop)- monkeypatch.setattr(wraps, "account_initialization_required", _noop)-- # Bypass billing-related decorators used by other endpoints in this module.- monkeypatch.setattr(wraps, "cloud_edition_billing_resource_check", lambda *_args, **_kwargs: lambda f: f)- monkeypatch.setattr(wraps, "cloud_edition_billing_rate_limit_check", lambda *_args, **_kwargs: lambda f: f)-- # Avoid Flask-RESTX route registration side effects during import.- def _noop_route(*_args, **_kwargs): # type: ignore[override]- def _decorator(cls):- return cls-- return _decorator-- monkeypatch.setattr(console_ns, "route", _noop_route)-- module_name = "controllers.console.datasets.datasets_document"- sys.modules.pop(module_name, None)- return importlib.import_module(module_name)---def _account(*, is_dataset_editor: bool = True) -> Account:- """Build a real account with the role used by dataset permission checks."""- account = Account(name="Dataset User", email="[email protected]")- account.id = "user-123"- account.role = TenantAccountRole.EDITOR if is_dataset_editor else TenantAccountRole.NORMAL- return account---def _document(- *,- document_id: str,- tenant_id: str,- data_source_type: str,- upload_file_id: str | None,-) -> Document:- """Build a real document entity used by the controller."""- document = Document(- id=document_id,- tenant_id=tenant_id,- dataset_id="ds-1",- position=1,- data_source_type=data_source_type,- data_source_info=json.dumps({"upload_file_id": upload_file_id}) if upload_file_id is not None else "{}",- batch="batch-1",- name="document.txt",- created_from="web",- created_by="user-123",- )- return document---def _dataset() -> Dataset:- return Dataset(id="ds-1", tenant_id="tenant-123", name="Dataset", created_by="user-123")---def _upload_file(*, file_id: str, name: str = "document.txt", key: str = "key") -> UploadFile:- upload_file = UploadFile(- tenant_id="tenant-123",- storage_type=StorageType.LOCAL,- key=key,- name=name,- size=1,- extension="txt",- mime_type="text/plain",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="user-123",- created_at=datetime.now(UTC),- used=False,- )- upload_file.id = file_id- return upload_file---def _wire_common_success_mocks(- *,- module,- monkeypatch: pytest.MonkeyPatch,- document_tenant_id: str,- data_source_type: str,- upload_file_id: str | None,- upload_file_exists: bool,- signed_url: str,-) -> None:- """Patch controller dependencies to create a deterministic test environment."""- import services.dataset_service as dataset_service_module-- # Return a dataset object and allow permission checks to pass.- monkeypatch.setattr(- module.DatasetService,- "get_dataset_for_tenant",- lambda *_args, **_kwargs: _dataset(),- )- monkeypatch.setattr(module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None)-- # Return a document that will be validated inside DocumentResource.get_document.- document = _document(- document_id="doc-1",- tenant_id=document_tenant_id,- data_source_type=data_source_type,- upload_file_id=upload_file_id,- )- monkeypatch.setattr(- module.DatasetRefService,- "get_document_by_ref",- lambda *_args, **_kwargs: document if document.tenant_id == "tenant-123" else None,- )-- # Mock UploadFile lookup via FileService batch helper.- upload_files_by_id: dict[str, object] = {}- if upload_file_exists and upload_file_id is not None:- upload_files_by_id[upload_file_id] = _upload_file(file_id=upload_file_id)- monkeypatch.setattr(module.FileService, "get_upload_files_by_ids", lambda *_args, **_kwargs: upload_files_by_id)-- # Mock signing helper so the returned URL is deterministic.- monkeypatch.setattr(dataset_service_module.file_helpers, "get_signed_file_url", lambda **_kwargs: signed_url)---def _mock_send_file(obj, **kwargs): # type: ignore[no-untyped-def]- """Return a lightweight representation of `send_file(...)` for unit tests."""-- class _ResponseMock(UserDict):- def __init__(self, sent_file: object, send_file_kwargs: dict[str, object]) -> None:- super().__init__({"_sent_file": sent_file, "_send_file_kwargs": send_file_kwargs})- self._on_close: object | None = None-- def call_on_close(self, func): # type: ignore[no-untyped-def]- self._on_close = func- return func-- return _ResponseMock(obj, kwargs)---def test_batch_download_zip_returns_send_file(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure batch ZIP download returns a zip attachment via `send_file`."""-- monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())- monkeypatch.setattr(- datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None- )-- # Two upload-file documents, each referencing an UploadFile.- doc1 = _document(- document_id="11111111-1111-1111-1111-111111111111",- tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-1",- )- doc2 = _document(- document_id="22222222-2222-2222-2222-222222222222",- tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-2",- )- monkeypatch.setattr(- datasets_document_module.DocumentService,- "get_documents_by_ids",- lambda *_args, **_kwargs: [doc1, doc2],- )- monkeypatch.setattr(- datasets_document_module.FileService,- "get_upload_files_by_ids",- lambda *_args, **_kwargs: {- "file-1": _upload_file(file_id="file-1", name="a.txt", key="k1"),- "file-2": _upload_file(file_id="file-2", name="b.txt", key="k2"),- },- )-- # Mock storage streaming content.- import services.file_service as file_service_module-- monkeypatch.setattr(file_service_module.storage, "load", lambda _key, stream=True: [b"hello"])-- # Replace send_file used by the controller to avoid a real Flask response object.- monkeypatch.setattr(datasets_document_module, "send_file", _mock_send_file)-- # Act- with app.test_request_context(- "/datasets/ds-1/documents/download-zip",- method="POST",- json={"document_ids": ["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"]},- ):- api = datasets_document_module.DocumentBatchDownloadZipApi()- method = unwrap(api.post)- result = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")-- # Assert: we returned via send_file with correct mime type and attachment.- assert result["_send_file_kwargs"]["mimetype"] == "application/zip"- assert result["_send_file_kwargs"]["as_attachment"] is True- assert isinstance(result["_send_file_kwargs"]["download_name"], str)- assert result["_send_file_kwargs"]["download_name"].endswith(".zip")- # Ensure our cleanup hook is registered and execute it to avoid temp file leaks in unit tests.- assert getattr(result, "_on_close", None) is not None- result._on_close() # type: ignore[attr-defined]---def test_batch_download_zip_response_is_openable_zip(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure the real Flask `send_file` response body is a valid ZIP that can be opened."""-- # Arrange: same controller mocks as the lightweight send_file test, but we keep the real `send_file`.- monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())- monkeypatch.setattr(- datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None- )-- doc1 = _document(- document_id="33333333-3333-3333-3333-333333333333",- tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-1",- )- doc2 = _document(- document_id="44444444-4444-4444-4444-444444444444",- tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-2",- )- monkeypatch.setattr(- datasets_document_module.DocumentService,- "get_documents_by_ids",- lambda *_args, **_kwargs: [doc1, doc2],- )- monkeypatch.setattr(- datasets_document_module.FileService,- "get_upload_files_by_ids",- lambda *_args, **_kwargs: {- "file-1": _upload_file(file_id="file-1", name="a.txt", key="k1"),- "file-2": _upload_file(file_id="file-2", name="b.txt", key="k2"),- },+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations+from services.knowledge.documents.application import DatasetDocumentApplicationService++CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")+DOC_ID = str(UUID(int=1))+FILE_ID = str(UUID(int=2))+++def _document(**values):+ return Document(+ **{+ "id": DOC_ID,+ "tenant_id": "tenant-1",+ "dataset_id": "dataset-1",+ "position": 1,+ "data_source_type": "upload_file",+ "data_source_info": json.dumps({"upload_file_id": FILE_ID}),+ "batch": "batch-1",+ "name": "document.txt",+ "created_from": "web",+ "created_by": "actor-1",+ **values,+ }+ )++[email protected](autouse=True)+def downloads(monkeypatch, sqlite_session_factory):+ with sqlite_session_factory.begin() as session:+ session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="actor-1"))+ session.add(_document())+ upload = UploadFile(+ tenant_id="tenant-1",+ storage_type="local",+ key="key-1",+ name="document.txt",+ size=5,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="actor-1",+ created_at=datetime(2024, 1, 1),+ used=False,+ )+ upload.id = FILE_ID+ session.add(upload)+ access = create_autospec(DatasetAccess, instance=True, spec_set=True)+ access.require_accessible.return_value = AccessibleDataset("dataset-1", "tenant-1")+ documents = DatasetDocumentApplicationService(+ dataset_access=access,+ operations=SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory),+ metadata_schema={},)-- # Stream distinct bytes per key so we can verify both ZIP entries.- import services.file_service as file_service_module-monkeypatch.setattr(- file_service_module.storage, "load", lambda key, stream=True: [b"one"] if key == "k1" else [b"two"]- )-- # Act- with app.test_request_context(- "/datasets/ds-1/documents/download-zip",- method="POST",- json={"document_ids": ["33333333-3333-3333-3333-333333333333", "44444444-4444-4444-4444-444444444444"]},+ controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents))+ )+ return documents+++def test_document_download_returns_signed_url(app, monkeypatch):+ from services.knowledge import dataset_service++ signed = []++ def sign(**kwargs):+ signed.append(kwargs)+ return "https://files.example/download"++ monkeypatch.setattr(dataset_service.file_helpers, "get_signed_file_url", sign)+ with app.test_request_context("/"):+ result = unwrap(controller.DocumentDownloadApi.get)(+ controller.DocumentDownloadApi(), CONTEXT, "dataset-1", DOC_ID+ )+ assert result == {"url": "https://files.example/download"}+ assert signed == [{"upload_file_id": FILE_ID, "as_attachment": True}]+++ "values",+ [+ {"data_source_type": "website_crawl"},+ {"data_source_info": "{}"},+ {"data_source_info": json.dumps({"upload_file_id": "missing"})},+ {"tenant_id": "foreign"},+ {"dataset_id": "foreign"},+ ],+)+def test_invalid_or_foreign_document_is_not_downloadable(app, sqlite_session_factory, values):+ with sqlite_session_factory.begin() as session:+ document = session.get(Document, DOC_ID)+ for key, value in values.items():+ setattr(document, key, value)+ with app.test_request_context("/"), pytest.raises(NotFound):+ unwrap(controller.DocumentDownloadApi.get)(controller.DocumentDownloadApi(), CONTEXT, "dataset-1", DOC_ID)+++def test_cross_tenant_upload_file_is_not_signed(app, sqlite_session_factory):+ with sqlite_session_factory.begin() as session:+ session.get(UploadFile, FILE_ID).tenant_id = "foreign"+ with app.test_request_context("/"), pytest.raises(NotFound):+ unwrap(controller.DocumentDownloadApi.get)(controller.DocumentDownloadApi(), CONTEXT, "dataset-1", DOC_ID)+++def test_zip_is_readable_and_cleanup_waits_for_response_close(app, monkeypatch, sqlite_engine):+ from services import file_service++ paths = []+ checked_out = set()++ def checkout(_connection, record, _proxy):+ checked_out.add(record)++ def checkin(_connection, record):+ checked_out.discard(record)++ def load(key, *, stream):+ assert not checked_out, "storage I/O must run after the read transaction closes"+ assert key == "key-1"+ assert stream is True+ return iter([b"hello"])++ def send(path, **kwargs):+ paths.append(Path(path))+ return send_file(path, **kwargs)++ monkeypatch.setattr(file_service.storage, "load", load)+ monkeypatch.setattr(controller, "send_file", send)+ event.listen(sqlite_engine, "checkout", checkout)+ event.listen(sqlite_engine, "checkin", checkin)+ try:+ with app.test_request_context("/", json={"document_ids": [DOC_ID]}):+ response = unwrap(controller.DocumentBatchDownloadZipApi.post)(+ controller.DocumentBatchDownloadZipApi(), CONTEXT, "dataset-1"+ )+ assert paths[0].exists()+ response.direct_passthrough = False+ with ZipFile(BytesIO(response.get_data())) as archive:+ assert archive.read("document.txt") == b"hello"+ response.close()+ assert not paths[0].exists()+ finally:+ event.remove(sqlite_engine, "checkout", checkout)+ event.remove(sqlite_engine, "checkin", checkin)+++def test_zip_cleanup_when_response_creation_fails(app, monkeypatch):+ from services import file_service++ paths = []+ monkeypatch.setattr(file_service.storage, "load", lambda *_args, **_kwargs: iter([b"hello"]))++ def fail_send(path, **_kwargs):+ paths.append(Path(path))+ raise RuntimeError("response failed")++ monkeypatch.setattr(controller, "send_file", fail_send)+ with (+ app.test_request_context("/", json={"document_ids": [DOC_ID]}),+ pytest.raises(RuntimeError, match="response failed"),):- api = datasets_document_module.DocumentBatchDownloadZipApi()- method = unwrap(api.post)- response = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")-- # Assert: response body is a valid ZIP and contains the expected entries.- response.direct_passthrough = False- data = response.get_data()- response.close()-- with ZipFile(BytesIO(data), mode="r") as zf:- assert zf.namelist() == ["a.txt", "b.txt"]- assert zf.read("a.txt") == b"one"- assert zf.read("b.txt") == b"two"---def test_batch_download_zip_rejects_non_upload_file_document(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure batch ZIP download rejects non upload-file documents."""-- monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())- monkeypatch.setattr(- datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None- )-- doc = _document(- document_id="55555555-5555-5555-5555-555555555555",- tenant_id="tenant-123",- data_source_type="website_crawl",- upload_file_id="file-1",- )- monkeypatch.setattr(- datasets_document_module.DocumentService,- "get_documents_by_ids",- lambda *_args, **_kwargs: [doc],- )-- with app.test_request_context(- "/datasets/ds-1/documents/download-zip",- method="POST",- json={"document_ids": ["55555555-5555-5555-5555-555555555555"]},- ):- api = datasets_document_module.DocumentBatchDownloadZipApi()- method = unwrap(api.post)- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")---def test_document_download_returns_url_for_upload_file_document(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure upload-file documents return a `{url}` JSON payload."""-- _wire_common_success_mocks(- module=datasets_document_module,- monkeypatch=monkeypatch,- document_tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-123",- upload_file_exists=True,- signed_url="https://example.com/signed",- )-- # Build a request context then call the resource method directly.- with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):- api = datasets_document_module.DocumentDownloadApi()- method = unwrap(api.get)- result = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")-- assert result == {"url": "https://example.com/signed"}---def test_document_download_rejects_non_upload_file_document(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure non-upload documents raise 404 (no file to download)."""-- _wire_common_success_mocks(- module=datasets_document_module,- monkeypatch=monkeypatch,- document_tenant_id="tenant-123",- data_source_type="website_crawl",- upload_file_id="file-123",- upload_file_exists=True,- signed_url="https://example.com/signed",- )-- with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):- api = datasets_document_module.DocumentDownloadApi()- method = unwrap(api.get)- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")---def test_document_download_rejects_missing_upload_file_id(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure missing `upload_file_id` raises 404."""-- _wire_common_success_mocks(- module=datasets_document_module,- monkeypatch=monkeypatch,- document_tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id=None,- upload_file_exists=False,- signed_url="https://example.com/signed",- )-- with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):- api = datasets_document_module.DocumentDownloadApi()- method = unwrap(api.get)- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")---def test_document_download_rejects_when_upload_file_record_missing(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure missing UploadFile row raises 404."""-- _wire_common_success_mocks(- module=datasets_document_module,- monkeypatch=monkeypatch,- document_tenant_id="tenant-123",- data_source_type="upload_file",- upload_file_id="file-123",- upload_file_exists=False,- signed_url="https://example.com/signed",- )-- with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):- api = datasets_document_module.DocumentDownloadApi()- method = unwrap(api.get)- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")---def test_document_download_rejects_document_owner_mismatch(- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- """Ensure an owner mismatch is rejected by the shared document resolver."""-- _wire_common_success_mocks(- module=datasets_document_module,- monkeypatch=monkeypatch,- document_tenant_id="tenant-999",- data_source_type="upload_file",- upload_file_id="file-123",- upload_file_exists=True,- signed_url="https://example.com/signed",- )-- with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):- api = datasets_document_module.DocumentDownloadApi()- method = unwrap(api.get)- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")+ unwrap(controller.DocumentBatchDownloadZipApi.post)(+ controller.DocumentBatchDownloadZipApi(), CONTEXT, "dataset-1"+ )+ assert paths+ assert not paths[0].exists()+++def test_zip_rejects_non_upload_document(app, sqlite_session_factory):+ with sqlite_session_factory.begin() as session:+ session.get(Document, DOC_ID).data_source_type = "website_crawl"+ with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(NotFound):+ unwrap(controller.DocumentBatchDownloadZipApi.post)(+ controller.DocumentBatchDownloadZipApi(), CONTEXT, "dataset-1"+ )diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.pyindex 00e4737228d..6b5b92b74a5 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py@@ -1,17 +1,14 @@-from datetime import datetime+from contextlib import AbstractContextManager+from dataclasses import dataclass+from datetime import UTC, datetimefrom inspect import unwrap-from types import SimpleNamespace-from typing import Any, cast-from unittest.mock import MagicMock, call, patch+from unittest.mock import create_autospec, patchimport pytestfrom flask import Flask-from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import Forbidden, NotFound-import servicesfrom controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayload-from controllers.console import console_nsfrom controllers.console.app.error import ProviderNotInitializeErrorfrom controllers.console.datasets.datasets_segments import (BatchImportPayload,@@ -21,1303 +18,360 @@ from controllers.console.datasets.datasets_segments import (DatasetDocumentSegmentAddApi,DatasetDocumentSegmentApi,DatasetDocumentSegmentBatchImportApi,+ DatasetDocumentSegmentBatchImportStatusApi,DatasetDocumentSegmentListApi,DatasetDocumentSegmentUpdateApi,SegmentCreatePayload,SegmentUpdatePayload,+ _raise_segment_error,)from controllers.console.datasets.error import ChildChunkDeleteIndexError, ChildChunkIndexingError, InvalidActionError-from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError-from core.rag.index_processor.constant.index_type import IndexStructureType-from fields.segment_fields import segment_response_with_summary, segment_responses_with_summaries-from libs.datetime_utils import naive_utc_now-from models.account import Account, TenantAccountRole-from models.dataset import ChildChunk, Dataset, Document, DocumentSegment-from models.enums import PermissionEnum, SegmentStatus, SegmentType-from models.model import UploadFile-from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError-from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError-from tests.unit_tests.model_factories import make_account, make_dataset, make_document, make_upload_file+from machinery.context import RequestContext+from services.knowledge.entities.segments import ChildChunkRecord, ChildChunkUpdateArgs, SegmentRecord+from services.knowledge.segments.application import (+ ChildChunkDeleteIndexApplicationError,+ ChildChunkIndexingApplicationError,+ ChildChunkNotFoundError,+ ChildChunkPage,+ DatasetSegmentApplicationService,+ SegmentBatchImport,+ SegmentBatchImportDispatchError,+ SegmentBatchImportNotFoundError,+ SegmentDatasetModelUnavailableError,+ SegmentDatasetNotFoundError,+ SegmentDetail,+ SegmentDocumentIndexingError,+ SegmentDocumentNotFoundError,+ SegmentEmbeddingModelUnavailableError,+ SegmentInvalidFileTypeError,+ SegmentListFilter,+ SegmentNotFoundError,+ SegmentPage,+ SegmentPermissionDeniedError,+ SegmentStatusUpdateError,+ SegmentUploadFileNotFoundError,+)-def _segment():- segment = DocumentSegment(- tenant_id="tenant-1",- dataset_id="ds-1",- document_id="doc-1",- position=1,- content="c",- word_count=1,- tokens=1,- created_by="u1",- answer="a",- keywords=["test"],- index_node_id="n1",- index_node_hash="h",- status=SegmentStatus.COMPLETED,- updated_by="u1",+def _context() -> RequestContext:+ return RequestContext(+ request_id="request-1",+ trace_id=None,+ account_id="account-1",+ active_workspace_id="tenant-1",)- segment.id = "seg-1"- segment.created_at = naive_utc_now()- segment.updated_at = naive_utc_now()- return segment--def _child_chunk():- child_chunk = ChildChunk(- tenant_id="tenant-1",- dataset_id="ds-1",- document_id="doc-1",- segment_id="seg-1",- position=1,- content="child",- word_count=1,- created_by="u1",- type=SegmentType.CUSTOMIZED,+def _child_chunk_data() -> ChildChunkRecord:+ now = datetime(2026, 1, 1, tzinfo=UTC)+ return ChildChunkRecord.model_validate(+ {+ "id": "child-1",+ "segment_id": "segment-1",+ "content": "child",+ "position": 1,+ "word_count": 5,+ "type": "customized",+ "created_at": now,+ "updated_at": now,+ })- child_chunk.id = "cc-1"- child_chunk.created_at = naive_utc_now()- child_chunk.updated_at = naive_utc_now()- return child_chunk-def _account() -> Account:- return make_account(- account_id="u1", name="Dataset Editor", email="[email protected]", role=TenantAccountRole.OWNER+def _segment_data() -> SegmentRecord:+ now = datetime(2026, 1, 1, tzinfo=UTC)+ return SegmentRecord.model_validate(+ {+ "id": "segment-1",+ "position": 1,+ "document_id": "document-1",+ "content": "content",+ "sign_content": "content",+ "answer": None,+ "word_count": 7,+ "tokens": 2,+ "keywords": [],+ "index_node_id": "node-1",+ "index_node_hash": "hash",+ "hit_count": 0,+ "enabled": True,+ "disabled_at": None,+ "disabled_by": None,+ "status": "completed",+ "created_by": "account-1",+ "created_at": now,+ "updated_at": now,+ "updated_by": None,+ "indexing_at": None,+ "completed_at": now,+ "error": None,+ "stopped_at": None,+ "child_chunks": [],+ "attachments": [],+ "summary": None,+ })-def _dataset(- *,- dataset_id: str = "ds-1",- tenant_id: str = "tenant-1",- indexing_technique: str = "economy",- embedding_model_provider: str | None = None,- embedding_model: str | None = None,-) -> Dataset:- return make_dataset(- dataset_id=dataset_id,- tenant_id=tenant_id,- description="",- provider="vendor",- permission=PermissionEnum.ONLY_ME,- indexing_technique=indexing_technique,- embedding_model_provider=embedding_model_provider,- embedding_model=embedding_model,- created_by="u1",- )+@dataclass(frozen=True)+class KnowledgeServiceStub:+ segments: DatasetSegmentApplicationService-def _document(- *,- document_id: str = "doc-1",- dataset_id: str = "ds-1",- tenant_id: str = "tenant-1",- doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,-) -> Document:- return make_document(- document_id=document_id,- tenant_id=tenant_id,- dataset_id=dataset_id,- created_from="api",- created_by="u1",- doc_form=doc_form,- )+@dataclass(frozen=True)+class ApplicationServiceStub:+ knowledge: KnowledgeServiceStub-def _upload_file(*, name: str = "test.csv", file_id: str = "test-file-id") -> UploadFile:- return make_upload_file(- file_id=file_id,- storage_type="opendal",- key="test-key",- name=name,- size=0,- extension=name.rsplit(".", maxsplit=1)[-1],- mime_type="text/csv",- created_by="u1",- created_at=datetime.now(),+def _patch_services(segments: DatasetSegmentApplicationService) -> AbstractContextManager[object]:+ return patch(+ "controllers.console.datasets.datasets_segments.application_services",+ return_value=ApplicationServiceStub(knowledge=KnowledgeServiceStub(segments=segments)),)-class SQLiteControllerTest:- session: Session-- @pytest.fixture(autouse=True)- def _use_sqlite_session(self, sqlite_session: Session) -> None:- self.session = sqlite_session---def _segment_response_dict():- return {- "id": "seg-1",- "position": 1,- "document_id": "doc-1",- "content": "c",- "sign_content": "c",- "answer": "a",- "word_count": 1,- "tokens": 1,- "keywords": ["test"],- "index_node_id": "n1",- "index_node_hash": "h",- "hit_count": 0,- "enabled": True,- "disabled_at": None,- "disabled_by": None,- "status": "completed",- "created_by": "u1",- "created_at": 1779678000,- "updated_at": 1779678000,- "updated_by": "u1",- "indexing_at": None,- "completed_at": None,- "error": None,- "stopped_at": None,- "child_chunks": [],- "attachments": [],- "summary": None,- }-+def test_list_segments_delegates_typed_query_and_serializes_page(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.list_segments.return_value = SegmentPage(+ items=(_segment_data(),), total=1, total_pages=1, page=2, limit=10+ )+ method = unwrap(DatasetDocumentSegmentListApi.get)-def test_segment_response_with_summary(sqlite_session: Session):- segment = _segment()with (- patch.object(DocumentSegment, "get_child_chunks", autospec=True, return_value=[]) as get_child_chunks,- patch.object(DocumentSegment, "get_attachments", autospec=True, return_value=[]) as get_attachments,+ app.test_request_context("/?status=completed&limit=10&page=2&enabled=true&keyword=needle"),+ _patch_services(segments),):- result = segment_response_with_summary(segment, "summary", session=sqlite_session)- assert result.summary == "summary"- assert result.id == segment.id- get_child_chunks.assert_called_once_with(segment, session=sqlite_session, include_full_doc=False)- get_attachments.assert_called_once_with(segment, session=sqlite_session)---def test_segment_responses_with_summaries_reuses_caller_session(sqlite_session: Session):- segments = [_segment(), _segment()]- segments[1].id = "seg-2"- expected_responses = [MagicMock(), MagicMock()]+ response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")++ assert status == 200+ assert response["data"][0]["id"] == "segment-1"+ assert response["data"][0]["created_at"] == 1767225600+ assert response["data"][0]["keywords"] == []+ assert response["data"][0]["disabled_at"] is None+ query = segments.list_segments.call_args.kwargs["query"]+ assert query == SegmentListFilter(+ page=2,+ limit=10,+ statuses=("completed",),+ enabled="true",+ keyword="needle",+ )- with patch(- "fields.segment_fields.segment_response_with_summary", side_effect=expected_responses- ) as serialize_segment:- responses = segment_responses_with_summaries(- segments,- {"seg-1": "summary-1", "seg-2": None},- session=sqlite_session,- )- assert responses == expected_responses- assert serialize_segment.call_args_list == [- call(segments[0], "summary-1", session=sqlite_session),- call(segments[1], None, session=sqlite_session),- ]+def test_delete_segments_delegates_all_selected_ids(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ method = unwrap(DatasetDocumentSegmentListApi.delete)+ with app.test_request_context("/?segment_id=one&segment_id=two"), _patch_services(segments):+ response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")-class TestDatasetDocumentSegmentListApi(SQLiteControllerTest):- def test_get_success(self, app: Flask):- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- dataset = _dataset()- document = _document()- user = _account()- segment = _segment()- pagination = MagicMock()- pagination.items = [segment]- pagination.total = 1- pagination.pages = 1- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch("controllers.console.datasets.datasets_segments.paginate_query", return_value=pagination),- patch("services.summary_index_service.SummaryIndexService.get_segments_summaries", return_value={}),- ):- response, status = method(api, self.session, "tenant-1", user, "ds-1", "doc-1")- assert status == 200+ assert (response, status) == ("", 204)+ assert segments.delete_segments.call_args.kwargs["segment_ids"] == ["one", "two"]- def test_get_dataset_not_found(self, app: Flask):- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- user = _account()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=None),- ):- with pytest.raises(NotFound):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1")- def test_get_permission_denied(self, app: Flask):- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- dataset = _dataset()- user = _account()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no access"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1")[email protected]("action", ["enable", "disable"])+def test_change_segment_status_delegates_action(app: Flask, action: str) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ method = unwrap(DatasetDocumentSegmentApi.patch)+ with app.test_request_context("/?segment_id=one"), _patch_services(segments):+ response, status = method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)-class TestDatasetDocumentSegmentApi(SQLiteControllerTest):- def test_patch_success(self, app: Flask):- api = DatasetDocumentSegmentApi()- method = unwrap(api.patch)- user = _account()- dataset = _dataset()- document = _document()- with (- app.test_request_context("/?segment_id=s1&segment_id=s2"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.redis_client.get", return_value=None),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.update_segments_status",- return_value=None,- ),- ):- response, status = method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "enable")- assert status == 200- assert response["result"] == "success"+ assert status == 200+ assert response == {"result": "success"}+ assert segments.change_segment_status.call_args.kwargs["action"] == action- def test_patch_document_indexing_in_progress(self, app: Flask):- api = DatasetDocumentSegmentApi()- method = unwrap(api.patch)- user = _account()- dataset = _dataset()- document = _document()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.redis_client.get", return_value=b"running"),- ):- with pytest.raises(InvalidActionError):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "disable")- def test_patch_llm_bad_request(self, app: Flask):- api = DatasetDocumentSegmentApi()- method = unwrap(api.patch)- user = _account()- dataset = _dataset(- indexing_technique="high_quality", embedding_model_provider="openai", embedding_model="text-embed"- )- document = _document()- with (- app.test_request_context("/?segment_id=s1"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.ModelManager.get_model_instance",- side_effect=LLMBadRequestError(),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "enable")[email protected]("action", ["typo", "ENABLE", "enable "])+def test_change_segment_status_rejects_unknown_action_before_service_call(app: Flask, action: str) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ method = unwrap(DatasetDocumentSegmentApi.patch)- def test_patch_provider_token_not_init(self, app: Flask):- api = DatasetDocumentSegmentApi()- method = unwrap(api.patch)- user = _account()- dataset = _dataset(- indexing_technique="high_quality", embedding_model_provider="openai", embedding_model="text-embed"- )- document = _document()- with (- app.test_request_context("/?segment_id=s1"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.ModelManager.get_model_instance",- side_effect=ProviderTokenNotInitError("token missing"),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "enable")+ with (+ app.test_request_context("/?segment_id=one"),+ _patch_services(segments),+ pytest.raises(InvalidActionError) as error,+ ):+ method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)+ assert error.value.code == 400+ segments.change_segment_status.assert_not_called()-class TestDatasetDocumentSegmentAddApi(SQLiteControllerTest):- def test_post_success(self, app: Flask):- api = DatasetDocumentSegmentAddApi()- method = unwrap(api.post)- payload = {"content": "hello"}- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.segment_create_args_validate",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.SegmentService.create_segment", return_value=segment),- patch(- "controllers.console.datasets.datasets_segments.SummaryIndexService.get_segment_summary",- return_value=None,- ),- ):- response, status = method(- api, SegmentCreatePayload(content="test content"), self.session, "tenant-1", user, "ds-1", "doc-1"- )- assert status == 200- assert response["data"]["id"] == "seg-1"- def test_post_llm_bad_request(self, app: Flask):- api = DatasetDocumentSegmentAddApi()- method = unwrap(api.post)- payload = {"content": "x"}- user = _account()- dataset = _dataset(- indexing_technique="high_quality", embedding_model_provider="openai", embedding_model="text-embed"- )- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.ModelManager.get_model_instance",- side_effect=LLMBadRequestError(),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(- api, SegmentCreatePayload(content="test content"), self.session, "tenant-1", user, "ds-1", "doc-1"- )+def test_create_segment_delegates_payload_and_serializes_detail(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.create_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")+ method = unwrap(DatasetDocumentSegmentAddApi.post)- def test_post_provider_token_not_init(self, app: Flask):- api = DatasetDocumentSegmentAddApi()- method = unwrap(api.post)- payload = {"content": "x"}- user = _account()- dataset = _dataset(- indexing_technique="high_quality", embedding_model_provider="openai", embedding_model="text-embed"+ with app.test_request_context("/"), _patch_services(segments):+ response, status = method(+ DatasetDocumentSegmentAddApi(),+ SegmentCreatePayload(content="content"),+ _context(),+ "dataset-1",+ "document-1",)- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.ModelManager.get_model_instance",- side_effect=ProviderTokenNotInitError("token missing"),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(- api, SegmentCreatePayload(content="test content"), self.session, "tenant-1", user, "ds-1", "doc-1"- )+ assert status == 200+ assert response["data"]["id"] == "segment-1"+ assert segments.create_segment.call_args.kwargs["values"] == {"content": "content"}-class TestDatasetDocumentSegmentUpdateApi(SQLiteControllerTest):- def test_patch_success(self, app: Flask):- api = DatasetDocumentSegmentUpdateApi()- method = unwrap(api.patch)- payload = {"content": "updated"}- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.segment_create_args_validate",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.SegmentService.update_segment", return_value=segment),- patch(- "controllers.console.datasets.datasets_segments.SummaryIndexService.get_segment_summary",- return_value=None,- ),- ):- response, status = method(- api,- SegmentUpdatePayload(content="test content"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )- assert status == 200- assert "data" in response- def test_patch_document_outside_dataset_is_not_found(self, app: Flask):- api = DatasetDocumentSegmentUpdateApi()- method = unwrap(api.patch)- payload = {"content": "updated"}- user = _account()- dataset = _dataset()- document = _document(dataset_id="other-dataset")- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(- api,- SegmentUpdatePayload(content="test content"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )+def test_update_and_delete_segment_delegate_owned_ids(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.update_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")- def test_patch_segment_not_found(self, app: Flask):- api = DatasetDocumentSegmentUpdateApi()- method = unwrap(api.patch)- payload = {"content": "updated"}- user = _account()- dataset = _dataset()- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=None- ),- ):- with pytest.raises(NotFound):- method(- api,- SegmentUpdatePayload(content="test content"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )-- def test_patch_llm_bad_request(self, app: Flask):- api = DatasetDocumentSegmentUpdateApi()- method = unwrap(api.patch)- payload = {"content": "x"}- user = _account()- dataset = _dataset(- indexing_technique="high_quality", embedding_model_provider="openai", embedding_model="text-embed"+ with app.test_request_context("/"), _patch_services(segments):+ update_response, update_status = unwrap(DatasetDocumentSegmentUpdateApi.patch)(+ DatasetDocumentSegmentUpdateApi(),+ SegmentUpdatePayload(content="updated"),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",+ )+ delete_response, delete_status = unwrap(DatasetDocumentSegmentUpdateApi.delete)(+ DatasetDocumentSegmentUpdateApi(),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",)- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.ModelManager.get_model_instance",- side_effect=LLMBadRequestError(),- ),- ):- with pytest.raises(ProviderNotInitializeError):- method(- api,- SegmentUpdatePayload(content="test content"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )---class TestDatasetDocumentSegmentBatchImportApi(SQLiteControllerTest):- def test_post_success(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- upload_file = _upload_file()- self.session.add(upload_file)- self.session.commit()- user = _account()- dataset = _dataset()- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ) as get_dataset_for_tenant,- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=document,- ) as get_document_by_ref,- patch("controllers.console.datasets.datasets_segments.redis_client.setnx", return_value=True),- patch(- "controllers.console.datasets.datasets_segments.batch_create_segment_to_index_task.delay",- return_value=None,- ),- ):- response, status = method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )- assert status == 200- assert response["job_status"] == "waiting"- get_dataset_for_tenant.assert_called_once_with("ds-1", "tenant-1", session=self.session)- document_ref = get_document_by_ref.call_args.args[0]- assert document_ref.dataset.tenant_id == "tenant-1"- assert document_ref.dataset.dataset_id == "ds-1"- assert document_ref.document_id == "doc-1"-- def test_post_dataset_not_found(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- user = _account()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_post_document_not_found(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- user = _account()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_post_upload_file_not_found(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- user = _account()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=_document(),- ),- ):- with pytest.raises(NotFound):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_post_invalid_file_type(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- upload_file = _upload_file(name="test.txt")- self.session.add(upload_file)- self.session.commit()- user = _account()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=_document(),- ),- ):- with pytest.raises(ValueError):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_post_async_task_failure(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- payload = {"upload_file_id": "file-1"}- upload_file = _upload_file()- self.session.add(upload_file)- self.session.commit()- user = _account()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=_dataset(),- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=_document(),- ),- patch(- "controllers.console.datasets.datasets_segments.redis_client.setnx", side_effect=Exception("redis down")- ),- ):- response, status = method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )- assert status == 500- assert "error" in response-- def test_get_job_not_found_in_redis(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.redis_client.get", return_value=None),- ):- with pytest.raises(ValueError):- method(api, job_id="job-1")---class TestChildChunkAddApi(SQLiteControllerTest):- def test_patch_documents_batch_update_payload(self):- patch_method = cast(Any, ChildChunkAddApi.patch)- api_doc = cast(dict[str, Any], patch_method.__apidoc__)- expected_model = ChildChunkBatchUpdatePayload.__name__- assert [model.name for model in api_doc["expect"]] == [expected_model]-- def test_get_uses_default_pagination_for_malformed_ints(self, app: Flask):- api = ChildChunkAddApi()- method = unwrap(api.get)- dataset = _dataset()- document = _document()- pagination = MagicMock(items=[], total=0, pages=0)- with (- app.test_request_context("/?page=bad&limit="),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_model_setting",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref",- return_value=_segment(),- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_child_chunks",- return_value=pagination,- ) as get_child_chunks,- ):- response, status = method(api, self.session, "tenant-1", "ds-1", "doc-1", "seg-1")- assert status == 200- assert response["page"] == 1- assert response["limit"] == 20- session = get_child_chunks.call_args.kwargs["session"]- assert session is self.session- assert get_child_chunks.call_args.args == ("seg-1", "doc-1", "ds-1", 1, 20, None)-- def test_post_success(self, app: Flask):- api = ChildChunkAddApi()- method = unwrap(api.post)- payload = {"content": "child"}- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- child_chunk = _child_chunk()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.create_child_chunk",- return_value=child_chunk,- ),- ):- response, status = method(- api,- ChildChunkCreatePayload(content="child"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )- assert status == 200- assert response["data"]["id"] == "cc-1"-- def test_post_child_chunk_indexing_error(self, app: Flask):- api = ChildChunkAddApi()- method = unwrap(api.post)- payload = {"content": "child"}- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.create_child_chunk",- side_effect=ChildChunkIndexingServiceError("fail"),- ),- ):- with pytest.raises(ChildChunkIndexingError):- method(- api,- ChildChunkCreatePayload(content="child"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )-- def test_post_permission_denied(self, app: Flask):- api = ChildChunkAddApi()- method = unwrap(api.post)- payload = {"content": "child"}- user = _account()- dataset = _dataset()- document = _document()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("no access"),- ),- ):- with pytest.raises(Forbidden):- method(- api,- ChildChunkCreatePayload(content="child"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- )+ assert update_status == 200+ assert update_response["data"]["content"] == "content"+ assert (delete_response, delete_status) == ("", 204)+ assert segments.update_segment.call_args.kwargs["segment_id"] == "segment-1"+ assert segments.delete_segment.call_args.kwargs["segment_id"] == "segment-1"+++def test_batch_import_delegates_and_preserves_dispatch_failure_response(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.start_batch_import.return_value = SegmentBatchImport(job_id="job-1", job_status="waiting")+ method = unwrap(DatasetDocumentSegmentBatchImportApi.post)++ with app.test_request_context("/"), _patch_services(segments):+ response, status = method(+ DatasetDocumentSegmentBatchImportApi(),+ BatchImportPayload(upload_file_id="file-1"),+ _context(),+ "dataset-1",+ "document-1",+ )+ assert status == 200+ assert response == {"job_id": "job-1", "job_status": "waiting"}++ segments.start_batch_import.side_effect = SegmentBatchImportDispatchError("redis down")+ with app.test_request_context("/"), _patch_services(segments):+ response, status = method(+ DatasetDocumentSegmentBatchImportApi(),+ BatchImportPayload(upload_file_id="file-1"),+ _context(),+ "dataset-1",+ "document-1",+ )+ assert (response, status) == ({"error": "redis down"}, 500)-class TestChildChunkUpdateApi(SQLiteControllerTest):- def test_delete_success(self, app: Flask):- api = ChildChunkUpdateApi()- method = unwrap(api.delete)- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- child_chunk = _child_chunk()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_child_chunk_by_segment_ref",- return_value=child_chunk,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.delete_child_chunk", return_value=None- ),- ):- response, status = method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "seg-1", "cc-1")- assert status == 204- assert response == ""- def test_delete_child_chunk_index_error(self, app: Flask):- api = ChildChunkUpdateApi()- method = unwrap(api.delete)- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- child_chunk = _child_chunk()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_child_chunk_by_segment_ref",- return_value=child_chunk,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.delete_child_chunk",- side_effect=ChildChunkDeleteIndexServiceError("fail"),- ),- ):- with pytest.raises(ChildChunkDeleteIndexError):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "seg-1", "cc-1")+def test_batch_import_status_preserves_missing_job_contract(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ method = unwrap(DatasetDocumentSegmentBatchImportStatusApi.get)- def test_delete_child_chunk_not_found(self, app: Flask):- api = ChildChunkUpdateApi()- method = unwrap(api.delete)- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_child_chunk_by_segment_ref",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1", "seg-1", "cc-1")+ segments.get_batch_import_status.side_effect = SegmentBatchImportNotFoundError("The job does not exist.")+ with app.test_request_context("/"), _patch_services(segments), pytest.raises(ValueError):+ method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")- def test_patch_child_chunk_not_found(self, app: Flask):- api = ChildChunkUpdateApi()- method = unwrap(api.patch)- payload = {"content": "updated child"}- user = _account()- dataset = _dataset()- document = _document()- segment = _segment()- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_segment_by_ref", return_value=segment- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.get_child_chunk_by_segment_ref",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(- api,- ChildChunkUpdatePayload(content="updated child"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- "seg-1",- "cc-1",- )+ segments.get_batch_import_status.side_effect = None+ segments.get_batch_import_status.return_value = SegmentBatchImport(job_id="job-1", job_status="completed")+ with app.test_request_context("/"), _patch_services(segments):+ response, status = method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")+ assert status == 200+ assert response == {"job_id": "job-1", "job_status": "completed"}-class TestSegmentListAdvancedCases(SQLiteControllerTest):- def test_segment_list_with_keyword_filter(self, app: Flask):- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- dataset = _dataset()- document = _document()- user = _account()- segment = _segment()- pagination = MagicMock(items=[segment], total=1, pages=1)- with (- app.test_request_context("/?keyword=test"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch("controllers.console.datasets.datasets_segments.paginate_query", return_value=pagination),- patch("services.summary_index_service.SummaryIndexService.get_segments_summaries", return_value={}),- ):- result = method(api, self.session, "tenant-1", user, "ds-1", "doc-1")- if isinstance(result, tuple):- response, status = result- else:- response, status = (result, 200)- assert status == 200- assert response["total"] == 1+def test_child_chunk_create_and_list_delegate(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.create_child_chunk.return_value = _child_chunk_data()+ segments.list_child_chunks.return_value = ChildChunkPage(+ items=(_child_chunk_data(),), total=1, total_pages=1, page=1, limit=20+ )- def test_segment_list_postgres_keyword_filter_handles_scalar_keywords(self, app: Flask):- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- dataset = _dataset(dataset_id="22222222-2222-2222-2222-222222222222")- document = _document(- document_id="33333333-3333-3333-3333-333333333333",- dataset_id=dataset.id,+ with app.test_request_context("/"), _patch_services(segments):+ create_response, create_status = unwrap(ChildChunkAddApi.post)(+ ChildChunkAddApi(),+ ChildChunkCreatePayload(content="child"),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",+ )+ list_response, list_status = unwrap(ChildChunkAddApi.get)(+ ChildChunkAddApi(), _context(), "dataset-1", "document-1", "segment-1")- user = _account()- pagination = MagicMock(items=[], total=0, pages=0)- with (- app.test_request_context("/?keyword=test"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.dify_config",- SimpleNamespace(SQLALCHEMY_DATABASE_URI_SCHEME="postgresql"),- ),- patch(- "controllers.console.datasets.datasets_segments.paginate_query", return_value=pagination- ) as paginate_mock,- ):- method(- api,- self.session,- "11111111-1111-1111-1111-111111111111",- user,- "22222222-2222-2222-2222-222222222222",- "33333333-3333-3333-3333-333333333333",- )- query = paginate_mock.call_args.args[0]- sql = str(query.compile(compile_kwargs={"literal_binds": True}))- assert "jsonb_array_elements_text(CASE" in sql- assert "ELSE CAST('[]' AS JSONB)" in sql-- def test_segment_list_permission_denied(self, app: Flask):- """Test segment list with permission denied"""- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- user = _account()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=_dataset()),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- side_effect=services.errors.account.NoPermissionError("No permission"),- ),- ):- with pytest.raises(Forbidden):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1")-- def test_segment_list_dataset_not_found(self, app: Flask):- """Test segment list with dataset not found"""- api = DatasetDocumentSegmentListApi()- method = unwrap(api.get)- user = _account()- with (- app.test_request_context("/"),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=None),- ):- with pytest.raises(NotFound):- method(api, self.session, "tenant-1", user, "ds-1", "doc-1")---class TestSegmentOperationCases(SQLiteControllerTest):- def test_segment_add_with_provider_token_error(self, app: Flask):- """Test segment add with provider token not initialized"""- api = DatasetDocumentSegmentAddApi()- method = unwrap(api.post)- user = _account()- dataset = _dataset()- document = _document()- payload = {"content": "new content", "answer": None}- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.datasets_segments.DatasetService.get_dataset", return_value=dataset),- patch("controllers.console.datasets.datasets_segments.DocumentService.get_document", return_value=document),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",- return_value=None,- ),- patch(- "controllers.console.datasets.datasets_segments.SegmentService.create_segment",- side_effect=ProviderTokenNotInitError("Token not init"),- ),- ):- with pytest.raises(ProviderTokenNotInitError):- method(- api,- SegmentCreatePayload(content="test content"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_batch_import_with_document_not_found(self, app: Flask):- """Test batch import with document not found"""- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- user = _account()- dataset = _dataset()- payload = {"upload_file_id": "file-1"}- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )-- def test_batch_import_with_invalid_file(self, app: Flask):- """Test batch import with invalid file type"""- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- user = _account()- dataset = _dataset()- document = _document()- payload = {"upload_file_id": "file-1"}- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=document,- ),- ):- with pytest.raises(NotFound):- method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )- def test_batch_import_with_async_task_failure(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.post)- user = _account()- dataset = _dataset()- document = _document()- upload_file = _upload_file()- self.session.add(upload_file)- self.session.commit()- payload = {"upload_file_id": "file-1"}- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.datasets_segments.DatasetService.get_dataset_for_tenant",- return_value=dataset,- ),- patch(- "controllers.console.datasets.datasets_segments.DatasetRefService.get_document_by_ref",- return_value=document,- ),- patch(- "controllers.console.datasets.datasets_segments.batch_create_segment_to_index_task.delay",- side_effect=Exception("Task failed"),- ),- ):- response, status = method(- api,- BatchImportPayload(upload_file_id="test-file-id"),- self.session,- "tenant-1",- user,- "ds-1",- "doc-1",- )- assert status == 500- assert "error" in response+ assert create_status == list_status == 200+ assert create_response["data"]["id"] == "child-1"+ assert create_response["data"]["created_at"] == 1767225600+ assert list_response["data"][0]["id"] == "child-1"+++def test_child_chunk_mutations_delegate(app: Flask) -> None:+ segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)+ segments.update_child_chunks.return_value = (_child_chunk_data(),)+ segments.update_child_chunk.return_value = _child_chunk_data()+ chunks = [ChildChunkUpdateArgs(id="child-1", content="updated")]++ with app.test_request_context("/"), _patch_services(segments):+ batch_response, batch_status = unwrap(ChildChunkAddApi.patch)(+ ChildChunkAddApi(),+ ChildChunkBatchUpdatePayload(chunks=chunks),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",+ )+ update_response, update_status = unwrap(ChildChunkUpdateApi.patch)(+ ChildChunkUpdateApi(),+ ChildChunkUpdatePayload(content="updated"),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",+ "child-1",+ )+ delete_response, delete_status = unwrap(ChildChunkUpdateApi.delete)(+ ChildChunkUpdateApi(),+ _context(),+ "dataset-1",+ "document-1",+ "segment-1",+ "child-1",+ )- def test_batch_import_get_job_not_found(self, app: Flask):- api = DatasetDocumentSegmentBatchImportApi()- method = unwrap(api.get)- with (- app.test_request_context("/?job_id=invalid-job"),- patch("controllers.console.datasets.datasets_segments.redis_client.get", return_value=None),- ):- with pytest.raises(ValueError):- method(api, "invalid-job")+ assert batch_status == update_status == 200+ assert batch_response["data"][0]["id"] == "child-1"+ assert update_response["data"]["id"] == "child-1"+ assert (delete_response, delete_status) == ("", 204)+++ ("error", "expected"),+ [+ (SegmentDatasetNotFoundError(), NotFound),+ (SegmentDocumentNotFoundError(), NotFound),+ (SegmentNotFoundError(), NotFound),+ (ChildChunkNotFoundError(), NotFound),+ (SegmentUploadFileNotFoundError(), NotFound),+ (SegmentPermissionDeniedError(), Forbidden),+ (SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),+ (SegmentDatasetModelUnavailableError(), ValueError),+ (SegmentDocumentIndexingError(), InvalidActionError),+ (SegmentStatusUpdateError(), InvalidActionError),+ (SegmentInvalidFileTypeError(), ValueError),+ (ChildChunkIndexingApplicationError(), ChildChunkIndexingError),+ (ChildChunkDeleteIndexApplicationError(), ChildChunkDeleteIndexError),+ ],+)+def test_segment_application_errors_map_to_existing_http_contract(error: Exception, expected: type[Exception]) -> None:+ with pytest.raises(expected):+ _raise_segment_error(error)diff --git a/api/tests/unit_tests/controllers/console/datasets/test_external.py b/api/tests/unit_tests/controllers/console/datasets/test_external.pyindex 2ebc0b2e9f8..6d90902ee51 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_external.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_external.py@@ -1,710 +1,164 @@-import inspect-import json-from datetime import datetime-from typing import Any-from unittest.mock import ANY, PropertyMock, patch+from inspect import unwrap+from types import SimpleNamespace+from unittest.mock import create_autospec+from uuid import UUIDimport pytest-from flask import Flask-from sqlalchemy.orm import Session-from werkzeug.exceptions import Forbidden, NotFound+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound-import services-from controllers.console import console_ns+from controllers.console.datasets import external as controllerfrom controllers.console.datasets.error import DatasetNameDuplicateError-from controllers.console.datasets.external import (- ExternalApiTemplateApi,- ExternalApiTemplateListApi,- ExternalApiTemplateListQuery,- ExternalApiUseCheckApi,- ExternalDatasetCreateApi,- ExternalHitTestingPayload,- ExternalKnowledgeApiPayload,- ExternalKnowledgeHitTestingApi,+from machinery.context import RequestContext+from services.errors.dataset import DatasetNameDuplicateError as DuplicateName+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.external.application import (+ ExternalHitTestingError,+ ExternalKnowledgeApplicationService,+ ExternalTemplateNotFoundError,)-from fields.dataset_fields import DatasetDetailResponse-from models.account import Account, TenantAccountRole-from models.dataset import Dataset, ExternalKnowledgeApis, ExternalKnowledgeBindings-from services.dataset_service import DatasetService-from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload-from services.external_knowledge_service import ExternalDatasetService-from services.hit_testing_service import HitTestingService--def app() -> Flask:- app = Flask("test_external_dataset")- app.config["TESTING"] = True- return app+CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")+ID = UUID(int=1)@pytest.fixture-def current_user() -> Account:- user = Account(name="Test User", email="[email protected]")- user.id = "user-1"- user.role = TenantAccountRole.EDITOR- return user+def external(monkeypatch):+ service = create_autospec(ExternalKnowledgeApplicationService, instance=True, spec_set=True)+ monkeypatch.setattr(+ controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(external=service))+ )+ return service-def _external_api_dict(api_id: str = "api-1") -> dict:- return {- "id": api_id,+def test_templates_preserve_pagination(external):+ page = {"data": [], "total": 0, "has_more": False, "page": 1, "limit": 100}+ external.list_templates.return_value = page+ result = unwrap(controller.ExternalApiTemplateListApi.get)(+ controller.ExternalApiTemplateListApi(),+ controller.ExternalApiTemplateListQuery(page=0, limit=1000, keyword="test"),+ CONTEXT,+ )+ assert result == (page, 200)+ external.list_templates.assert_called_once_with(CONTEXT, page=0, limit=1000, keyword="test")+++ ("resource", "verb", "operation", "payload"),+ [+ (+ controller.ExternalApiTemplateListApi,+ controller.ExternalApiTemplateListApi.post,+ lambda service: service.create_template,+ True,+ ),+ (+ controller.ExternalApiTemplateApi,+ controller.ExternalApiTemplateApi.get,+ lambda service: service.get_template,+ False,+ ),+ (+ controller.ExternalApiTemplateApi,+ controller.ExternalApiTemplateApi.patch,+ lambda service: service.update_template,+ True,+ ),+ (+ controller.ExternalApiTemplateApi,+ controller.ExternalApiTemplateApi.delete,+ lambda service: service.delete_template,+ False,+ ),+ ],+)+def test_template_missing_is_mapped_and_context_forwarded(external, resource, verb, operation, payload):+ method = operation(external)+ method.side_effect = ExternalTemplateNotFoundError("API template not found.")+ args = (+ [+ controller.ExternalKnowledgeApiPayload(+ name="API", settings={"endpoint": "https://example", "api_key": "secret"}+ )+ ]+ if payload+ else []+ )+ args.append(CONTEXT)+ if resource is controller.ExternalApiTemplateApi:+ args.append(ID)+ with pytest.raises(NotFound):+ unwrap(verb)(resource(), *args)+ assert method.call_args.args == (CONTEXT,)+ if resource is controller.ExternalApiTemplateApi:+ assert method.call_args.kwargs["template_id"] == str(ID)+++def test_template_serializes_settings_and_timestamp(external):+ external.get_template.return_value = {+ "id": str(ID),+ "name": "API","tenant_id": "tenant-1",- "name": f"External API {api_id}",- "description": f"Description for {api_id}",- "settings": {- "endpoint": f"https://external.example.com/{api_id}",- "api_key": "secret",- "headers": {"X-Source": "unit-test"},- "timeout": 30,- },- "dataset_bindings": [- {"id": f"dataset-{api_id}", "name": f"Dataset {api_id}"},- ],- "created_by": "user-1",+ "settings": {"endpoint": "https://example", "api_key": "[__HIDDEN__]"},+ "description": "",+ "created_by": "actor-1","created_at": "2024-01-01T00:00:00",+ "dataset_bindings": [],}+ result, status = unwrap(controller.ExternalApiTemplateApi.get)(controller.ExternalApiTemplateApi(), CONTEXT, ID)+ assert status == 200+ assert result == external.get_template.return_value-def _external_api_object(session: Session, api_id: str = "api-1") -> ExternalKnowledgeApis:- payload = _external_api_dict(api_id)- api = ExternalKnowledgeApis(- name=payload["name"],- description=payload["description"],- tenant_id=payload["tenant_id"],- settings=json.dumps(payload["settings"]),- created_by=payload["created_by"],- updated_by=None,+def test_delete_and_usage_response_contracts(external):+ assert unwrap(controller.ExternalApiTemplateApi.delete)(controller.ExternalApiTemplateApi(), CONTEXT, ID) == (+ "",+ 204,)- api.id = api_id- api.created_at = datetime.fromisoformat(payload["created_at"])- bound_dataset = Dataset(- id=payload["dataset_bindings"][0]["id"],- tenant_id=payload["tenant_id"],- name=payload["dataset_bindings"][0]["name"],- created_by=payload["created_by"],+ external.template_usage.return_value = (True, 3)+ assert unwrap(controller.ExternalApiUseCheckApi.get)(controller.ExternalApiUseCheckApi(), CONTEXT, ID) == (+ {"is_using": True, "count": 3},+ 200,)- binding = ExternalKnowledgeBindings(- tenant_id=payload["tenant_id"],- external_knowledge_api_id=api_id,- dataset_id=bound_dataset.id,- external_knowledge_id=f"knowledge-{api_id}",- created_by=payload["created_by"],- )- session.add_all([api, bound_dataset, binding])- session.flush()- return api---def _expected_dataset_detail_payload() -> dict[str, Any]:- return {- "id": "dataset-1",- "name": "Support knowledge",- "description": "External support articles",- "provider": "external",- "permission": "only_me",- "data_source_type": "external",- "indexing_technique": "economy",- "app_count": 2,- "document_count": 7,- "word_count": 2048,- "created_by": "user-1",- "author_name": "Test User",- "created_at": 1710000000,- "updated_by": "user-2",- "updated_at": 1710003600,- "embedding_model": None,- "embedding_model_provider": None,- "embedding_available": False,- "retrieval_model_dict": {- "search_method": "semantic_search",- "reranking_enable": False,- "reranking_mode": None,- "reranking_model": {"reranking_provider_name": None, "reranking_model_name": None},- "weights": None,- "top_k": 4,- "score_threshold_enabled": True,- "score_threshold": 0.5,- },- "summary_index_setting": {- "enable": True,- "model_name": "summary-model",- "model_provider_name": "provider-a",- "summary_prompt": "Summarize this.",- },- "tags": [{"id": "tag-1", "name": "Support", "type": "knowledge"}],- "doc_form": "text_model",- "external_knowledge_info": {- "external_knowledge_id": "knowledge-1",- "external_knowledge_api_id": "api-1",- "external_knowledge_api_name": "External API api-1",- "external_knowledge_api_endpoint": "https://external.example.com/api-1",- },- "external_retrieval_model": {- "top_k": 4,- "score_threshold": 0.5,- "score_threshold_enabled": True,- },- "doc_metadata": [{"id": "metadata-1", "name": "source", "type": "string"}],- "built_in_field_enabled": True,- "pipeline_id": None,- "runtime_mode": "external",- "chunk_structure": "general",- "icon_info": {- "icon_type": "emoji",- "icon": "book",- "icon_background": "#FFF4ED",- "icon_url": None,- },- "is_published": True,- "total_documents": 7,- "total_available_documents": 6,- "enable_api": True,- "is_multimodal": False,- "maintainer": None,- "permission_keys": [],- }-def _dataset() -> Dataset:- payload = _expected_dataset_detail_payload()- return Dataset(- id=payload["id"],- tenant_id="tenant-1",- name=payload["name"],- description=payload["description"],- provider=payload["provider"],- permission=payload["permission"],- data_source_type=payload["data_source_type"],- indexing_technique=payload["indexing_technique"],- created_by=payload["created_by"],+def test_external_dataset_duplicate_name_maps_to_409(external):+ payload = controller.ExternalDatasetCreatePayload(+ name="Dataset", external_knowledge_api_id="api", external_knowledge_id="knowledge")---def _dataset_detail_response() -> DatasetDetailResponse:- return DatasetDetailResponse.model_validate(_expected_dataset_detail_payload())---class _UsesSQLiteSession:- session: Session-- @pytest.fixture(autouse=True)- def _inject_sqlite_session(self, sqlite_session: Session) -> None:- self.session = sqlite_session---class TestExternalApiTemplateListApi(_UsesSQLiteSession):- def test_get_success(self, app: Flask):- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.get)-- session = self.session- api_item = _external_api_object(session, "api-1")-- with (- app.test_request_context("/?page=2&limit=1&keyword=vector"),- patch.object(- ExternalDatasetService,- "get_external_knowledge_apis",- return_value=([api_item], 3),- ) as get_external_knowledge_apis,- ):- resp, status = method(- api, ExternalApiTemplateListQuery(page=2, limit=1, keyword="vector"), session, "tenant-1"- )-- assert status == 200- assert resp == {- "data": [_external_api_dict("api-1")],- "has_more": True,- "limit": 1,- "total": 3,- "page": 2,- }- get_external_knowledge_apis.assert_called_once_with(2, 1, "tenant-1", "vector", session=ANY)-- def test_post_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.post)-- payload = {- "name": "Vendor Search",- "settings": {- "endpoint": "https://external.example.com/search",- "api_key": "secret",- "headers": {"X-Source": "unit-test"},- "timeout": 30,- },- }- session = self.session- created = _external_api_object(session, "api-created")-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,- patch.object(- ExternalDatasetService,- "create_external_knowledge_api",- return_value=created,- ) as create_external_knowledge_api,- ):- resp, status = method(- api, ExternalKnowledgeApiPayload.model_validate(payload), session, "tenant-1", current_user- )-- assert status == 201- assert resp == _external_api_dict("api-created")- validate_api_list.assert_called_once_with(payload["settings"])- create_external_knowledge_api.assert_called_once_with(- tenant_id="tenant-1",- user_id="user-1",- args=payload,- session=session,- )-- def test_post_forbidden(self, app: Flask, current_user: Account):- current_user.role = TenantAccountRole.NORMAL- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.post)-- payload = {"name": "x", "settings": {"k": "v"}}-- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(ExternalDatasetService, "validate_api_list"),- ):- with pytest.raises(Forbidden):- method(api, ExternalKnowledgeApiPayload.model_validate(payload), self.session, "tenant-1", current_user)-- def test_post_duplicate_name(self, app: Flask, current_user: Account):- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.post)-- payload = {"name": "x", "settings": {"k": "v"}}-- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(ExternalDatasetService, "validate_api_list"),- patch.object(- ExternalDatasetService,- "create_external_knowledge_api",- side_effect=services.errors.dataset.DatasetNameDuplicateError(),- ),- ):- with pytest.raises(DatasetNameDuplicateError):- method(api, ExternalKnowledgeApiPayload.model_validate(payload), self.session, "tenant-1", current_user)---class TestExternalApiTemplateApi(_UsesSQLiteSession):- def test_get_success_returns_template_contract(self, app: Flask):- api = ExternalApiTemplateApi()- method = inspect.unwrap(api.get)- session = self.session- template = _external_api_object(session, "api-detail")-- with (- app.test_request_context("/"),- patch.object(- ExternalDatasetService,- "get_external_knowledge_api",- return_value=template,- ) as get_external_knowledge_api,- ):- resp, status = method(api, session, "tenant-1", "api-detail")-- assert status == 200- assert resp == _external_api_dict("api-detail")- get_external_knowledge_api.assert_called_once_with(- external_knowledge_api_id="api-detail", tenant_id="tenant-1", session=session- )-- def test_get_not_found(self, app: Flask):- api = ExternalApiTemplateApi()- method = inspect.unwrap(api.get)-- with (- app.test_request_context("/"),- patch.object(- ExternalDatasetService,- "get_external_knowledge_api",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, self.session, "tenant-1", "api-id")-- def test_patch_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):- api = ExternalApiTemplateApi()- method = inspect.unwrap(api.patch)-- payload = {- "name": "Updated API",- "settings": {- "endpoint": "https://external.example.com/updated",- "api_key": "new-secret",- "headers": {"X-Version": "2"},- },- }- session = self.session- updated = _external_api_object(session, "api-updated")-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,- patch.object(- ExternalDatasetService,- "update_external_knowledge_api",- return_value=updated,- ) as update_external_knowledge_api,- ):- resp, status = method(- api,- ExternalKnowledgeApiPayload.model_validate(payload),- session,- "tenant-1",- current_user,- "api-updated",- )-- assert status == 200- assert resp == _external_api_dict("api-updated")- validate_api_list.assert_called_once_with(payload["settings"])- update_external_knowledge_api.assert_called_once_with(- tenant_id="tenant-1",- user_id="user-1",- external_knowledge_api_id="api-updated",- args=payload,- session=session,- )-- def test_delete_forbidden(self, app: Flask, current_user: Account):- current_user.role = TenantAccountRole.NORMAL-- api = ExternalApiTemplateApi()- method = inspect.unwrap(api.delete)-- with app.test_request_context("/"):- with pytest.raises(Forbidden):- method(api, self.session, "tenant-1", current_user, "api-id")---class TestExternalApiUseCheckApi(_UsesSQLiteSession):- def test_get_scopes_usage_check_to_current_tenant(self, app: Flask):- api = ExternalApiUseCheckApi()- method = inspect.unwrap(api.get)-- session = self.session-- with (- app.test_request_context("/"),- patch.object(- ExternalDatasetService,- "external_knowledge_api_use_check",- return_value=(True, 2),- ) as mock_use_check,- ):- response, status = method(api, session, "tenant-1", "api-id")-- assert status == 200- assert response == {"is_using": True, "count": 2}- mock_use_check.assert_called_once_with("api-id", "tenant-1", session=ANY)---class TestExternalDatasetCreateApi(_UsesSQLiteSession):- def test_create_success(self, app: Flask, current_user: Account):- api = ExternalDatasetCreateApi()- method = inspect.unwrap(api.post)-- payload = {- "external_knowledge_api_id": "api-1",- "external_knowledge_id": "knowledge-1",- "name": "Support knowledge",- "description": "External support articles",- "external_retrieval_model": {- "top_k": 4,- "score_threshold": 0.5,- "score_threshold_enabled": True,- },- }-- dataset = _dataset()-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(- ExternalDatasetService,- "create_external_dataset",- return_value=dataset,- ) as create_external_dataset,- patch(- "controllers.console.datasets.external.dataset_detail_response_source",- return_value=_dataset_detail_response(),- ) as dataset_response_source,- ):- resp, status = method(- api,- ExternalDatasetCreatePayload.model_validate(payload),- self.session,- "tenant-1",- current_user,- )-- assert status == 201- assert resp == _expected_dataset_detail_payload()- create_external_dataset.assert_called_once_with(- tenant_id="tenant-1",- user_id="user-1",- args=ExternalDatasetCreatePayload.model_validate(payload),- session=self.session,- )- dataset_response_source.assert_called_once_with(dataset, session=self.session)-- def test_create_forbidden(self, app: Flask, current_user: Account):- current_user.role = TenantAccountRole.NORMAL- api = ExternalDatasetCreateApi()- method = inspect.unwrap(api.post)-- payload = {- "external_knowledge_api_id": "api",- "external_knowledge_id": "kid",- "name": "dataset",- }-- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- ):- with pytest.raises(Forbidden):- method(- api, ExternalDatasetCreatePayload.model_validate(payload), self.session, "tenant-1", current_user- )---class TestExternalKnowledgeHitTestingApi(_UsesSQLiteSession):- def test_hit_testing_dataset_not_found(self, app: Flask, current_user: Account):- api = ExternalKnowledgeHitTestingApi()- method = inspect.unwrap(api.post)-- with (- app.test_request_context("/"),- patch.object(- DatasetService,- "get_dataset",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, ExternalHitTestingPayload(query="test"), self.session, current_user, "dataset-id")-- def test_hit_testing_success(self, app: Flask, current_user: Account):- api = ExternalKnowledgeHitTestingApi()- method = inspect.unwrap(api.post)-- payload = {- "query": "hello",- "external_retrieval_model": {- "top_k": 3,- "score_threshold": 0.25,- "score_threshold_enabled": True,- },- "metadata_filtering_conditions": {- "logical_operator": "and",- "conditions": [{"name": "source", "comparison_operator": "contains", "value": "external"}],- },- }-- dataset = _dataset()- retrieve_response = {- "query": {"content": "hello"},- "records": [- {- "content": "answer",- "title": "doc",- "score": 0.9,- "metadata": {"source": "external", "page": 2},- }- ],- }- session = self.session-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission") as check_dataset_permission,- patch.object(HitTestingService, "hit_testing_args_check") as hit_testing_args_check,- patch.object(- HitTestingService,- "external_retrieve",- return_value=retrieve_response,- ) as external_retrieve,- patch("controllers.console.datasets.external.dump_response", side_effect=lambda _model, value: value),- ):- resp = method(api, ExternalHitTestingPayload.model_validate(payload), session, current_user, "dataset-id")-- assert resp == retrieve_response- check_dataset_permission.assert_called_once_with(dataset, current_user, session)- hit_testing_args_check.assert_called_once_with(payload)- external_retrieve.assert_called_once_with(- session=session,- dataset=dataset,- query="hello",- account=current_user,- external_retrieval_model=payload["external_retrieval_model"],- metadata_filtering_conditions=payload["metadata_filtering_conditions"],+ external.create_dataset.side_effect = DuplicateName()+ with pytest.raises(DatasetNameDuplicateError):+ unwrap(controller.ExternalDatasetCreateApi.post)(controller.ExternalDatasetCreateApi(), payload, CONTEXT)+ external.create_dataset.assert_called_once_with(CONTEXT, payload=payload)+++ ("error", "http_error"),+ [+ (DatasetNotFoundError(), NotFound),+ (DatasetAccessDeniedError(), Forbidden),+ (ExternalHitTestingError("failed"), InternalServerError),+ ],+)+def test_retrieval_maps_application_errors(external, error, http_error):+ external.hit_testing.side_effect = error+ with pytest.raises(http_error):+ unwrap(controller.ExternalKnowledgeHitTestingApi.post)(+ controller.ExternalKnowledgeHitTestingApi(),+ controller.ExternalHitTestingPayload(query="query"),+ CONTEXT,+ ID,)+ external.hit_testing.assert_called_once_with(+ CONTEXT, dataset_id=str(ID), query="query", retrieval_model=None, metadata_filters=None+ )-class TestExternalApiTemplateListApiAdvanced(_UsesSQLiteSession):- def test_post_duplicate_name_error(self, app: Flask, current_user: Account):- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.post)-- payload = {"name": "duplicate_api", "settings": {"key": "value"}}-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch("controllers.console.datasets.external.ExternalDatasetService.validate_api_list"),- patch(- "controllers.console.datasets.external.ExternalDatasetService.create_external_knowledge_api",- side_effect=services.errors.dataset.DatasetNameDuplicateError("Duplicate"),- ),- ):- with pytest.raises(DatasetNameDuplicateError):- method(api, ExternalKnowledgeApiPayload.model_validate(payload), self.session, "tenant-1", current_user)-- def test_get_with_pagination(self, app: Flask):- api = ExternalApiTemplateListApi()- method = inspect.unwrap(api.get)-- templates = [_external_api_object(self.session, f"api-{i}") for i in range(3)]-- with (- app.test_request_context("/?page=2&limit=3"),- patch(- "controllers.console.datasets.external.ExternalDatasetService.get_external_knowledge_apis",- return_value=(templates, 25),- ) as get_external_knowledge_apis,- ):- resp, status = method(api, ExternalApiTemplateListQuery(page=2, limit=3), self.session, "tenant-1")-- assert status == 200- assert resp == {- "data": [_external_api_dict(f"api-{i}") for i in range(3)],- "has_more": True,- "limit": 3,- "total": 25,- "page": 2,- }- get_external_knowledge_apis.assert_called_once_with(2, 3, "tenant-1", None, session=ANY)---class TestExternalDatasetCreateApiAdvanced(_UsesSQLiteSession):- def test_create_forbidden(self, app: Flask, current_user: Account):- """Test creating external dataset without permission"""- api = ExternalDatasetCreateApi()- method = inspect.unwrap(api.post)-- current_user.role = TenantAccountRole.NORMAL-- payload = {- "external_knowledge_api_id": "api-1",- "external_knowledge_id": "ek-1",- "name": "new_dataset",- "description": "A dataset",- }-- with app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload):- with pytest.raises(Forbidden):- method(- api, ExternalDatasetCreatePayload.model_validate(payload), self.session, "tenant-1", current_user- )---class TestExternalKnowledgeHitTestingApiAdvanced(_UsesSQLiteSession):- def test_hit_testing_dataset_not_found(self, app: Flask, current_user: Account):- """Test hit testing on non-existent dataset"""- api = ExternalKnowledgeHitTestingApi()- method = inspect.unwrap(api.post)-- payload = {- "query": "test query",- "external_retrieval_model": None,- }-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.external.DatasetService.get_dataset",- return_value=None,- ),- ):- with pytest.raises(NotFound):- method(api, ExternalHitTestingPayload.model_validate(payload), self.session, current_user, "ds-1")-- def test_hit_testing_with_custom_retrieval_model(self, app: Flask, current_user: Account):- api = ExternalKnowledgeHitTestingApi()- method = inspect.unwrap(api.post)-- dataset = _dataset()- payload = {- "query": "test query",- "external_retrieval_model": {"type": "bm25"},- "metadata_filtering_conditions": {"status": "active"},- }- session = self.session-- with (- app.test_request_context("/", json=payload),- patch.object(type(console_ns), "payload", payload),- patch(- "controllers.console.datasets.external.DatasetService.get_dataset",- return_value=dataset,- ),- patch("controllers.console.datasets.external.DatasetService.check_dataset_permission") as check_permission,- patch("controllers.console.datasets.external.HitTestingService.hit_testing_args_check") as args_check,- patch(- "controllers.console.datasets.external.HitTestingService.external_retrieve",- return_value={- "query": {"content": "test query"},- "records": [- {- "content": None,- "title": "metadata-only",- "score": None,- "metadata": {"status": "active"},- }- ],- },- ) as external_retrieve,- ):- resp = method(api, ExternalHitTestingPayload.model_validate(payload), session, current_user, "ds-1")-- assert resp == {- "query": {"content": "test query"},- "records": [- {- "content": None,- "title": "metadata-only",- "score": None,- "metadata": {"status": "active"},- }- ],- }- check_permission.assert_called_once_with(dataset, current_user, session)- args_check.assert_called_once_with(payload)- external_retrieve.assert_called_once_with(- session=session,- dataset=dataset,- query="test query",- account=current_user,- external_retrieval_model={"type": "bm25"},- metadata_filtering_conditions={"status": "active"},- )+def test_retrieval_response_is_serialized(external):+ external.hit_testing.return_value = {+ "query": {"content": "query"},+ "records": [{"content": "answer", "title": "title", "score": 0.8, "metadata": {"source": "doc"}}],+ }+ result = unwrap(controller.ExternalKnowledgeHitTestingApi.post)(+ controller.ExternalKnowledgeHitTestingApi(), controller.ExternalHitTestingPayload(query="query"), CONTEXT, ID+ )+ assert result == external.hit_testing.return_valuediff --git a/api/tests/unit_tests/controllers/console/datasets/test_hit_testing_base.py b/api/tests/unit_tests/controllers/console/datasets/test_hit_testing_base.pyindex 39ddd2ec787..bc7d18b8fc0 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_hit_testing_base.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_hit_testing_base.py@@ -23,8 +23,8 @@ from core.errors.error import (from graphon.model_runtime.errors.invoke import InvokeErrorfrom models.account import Account, Tenant, TenantAccountRolefrom models.dataset import Dataset-from services.dataset_service import DatasetServicefrom services.hit_testing_service import HitTestingService+from services.knowledge.dataset_service import DatasetService@pytest.fixturediff --git a/api/tests/unit_tests/controllers/console/datasets/test_metadata.py b/api/tests/unit_tests/controllers/console/datasets/test_metadata.pyindex dc528761f74..63c28b60a33 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_metadata.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_metadata.py@@ -1,340 +1,122 @@-import uuid-from inspect import unwrap-from unittest.mock import PropertyMock, patch+"""Console metadata routes use admission, explicit context, and response models."""++from types import SimpleNamespace+from unittest.mock import create_autospec+from uuid import uuid4import pytestfrom flask import Flask-from pytest_mock import MockerFixture-from sqlalchemy.orm import Session-from werkzeug.exceptions import Forbidden, NotFound+from flask_restx import Api-from controllers.common.controller_schemas import MetadataUpdatePayload-from controllers.common.rbac import DatasetId+from controllers.common.rbac import DatasetId, RBACPermissionfrom controllers.console import console_ns-from controllers.console.datasets.metadata import (- DatasetMetadataApi,- DatasetMetadataBuiltInFieldActionApi,- DatasetMetadataBuiltInFieldApi,- DatasetMetadataCreateApi,- DocumentMetadataEditApi,-)-from controllers.console.wraps import RBACPermission-from models.account import Account-from models.dataset import Dataset-from services.dataset_service import DatasetService-from services.entities.knowledge_entities.knowledge_entities import MetadataArgs, MetadataOperationData-from services.errors.account import NoPermissionError+from controllers.console.datasets.metadata import DatasetMetadataCreateApi+from enums import DeploymentEdition+from libs.login import AccountWithTenant+from models.account import Account, TenantAccountRolefrom services.errors.metadata import MetadataResourceNotFoundError-from services.metadata_service import MetadataService-from tests.unit_tests.config_override import config_overrides_context+from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetReffrom tests.unit_tests.controllers.rbac_introspection import rbac_checks@pytest.fixture-def app():- app = Flask("test_dataset_metadata")-- app.config["TESTING"] = True- return app---def current_user() -> Account:- user = Account(name="Test User", email="[email protected]")- user.id = "user-1"- return user---def dataset() -> Dataset:- return Dataset(id="dataset-1", tenant_id="tenant-1", name="Test Dataset", created_by="user-1")---def dataset_id():- return uuid.uuid4()---def metadata_id():- return uuid.uuid4()--[email protected](autouse=True)-def bypass_decorators(mocker: MockerFixture):- """Bypass setup/login/license decorators."""- mocker.patch("controllers.console.datasets.metadata.setup_required", lambda f: f)- mocker.patch("controllers.console.datasets.metadata.login_required", lambda f: f)- mocker.patch("controllers.console.datasets.metadata.account_initialization_required", lambda f: f)- mocker.patch("controllers.console.datasets.metadata.enterprise_license_required", lambda f: f)---class TestDatasetMetadataCreateApi:- def test_get_requires_dataset_readonly_permission(self):- [check] = rbac_checks(DatasetMetadataCreateApi.get)-- assert check.scene is RBACPermission.DATASET_READONLY- assert isinstance(check.locator, DatasetId)-- def test_create_metadata_success(self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session):- api = DatasetMetadataCreateApi()- method = unwrap(api.post)- payload = {"name": "author"}- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(- MetadataArgs,- "model_validate",- return_value=MetadataArgs(type="string", name="author"),- ),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- patch.object(- MetadataService, "create_metadata", return_value={"id": "m1", "type": "string", "name": "author"}- ),- ):- result, status = method(- api, MetadataArgs(type="string", name="author"), sqlite_session, "tenant-1", current_user, dataset_id- )- assert status == 201- assert result["type"] == "string"- assert result["name"] == "author"-- def test_create_metadata_dataset_not_found(self, app: Flask, current_user, dataset_id, sqlite_session: Session):- api = DatasetMetadataCreateApi()- method = unwrap(api.post)- valid_payload = {"type": "string", "name": "author"}- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=valid_payload),- patch.object(- MetadataArgs,- "model_validate",- return_value=MetadataArgs(type="string", name="author"),- ),- patch.object(DatasetService, "get_dataset", return_value=None),- ):- with pytest.raises(NotFound, match="Dataset not found"):- method(- api,- MetadataArgs(type="string", name="author"),- sqlite_session,- "tenant-1",- current_user,- dataset_id,- )---class TestDatasetMetadataGetApi:- def test_get_metadata_success(self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session):- api = DatasetMetadataCreateApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(- MetadataService,- "get_dataset_metadatas",- return_value={- "doc_metadata": [{"id": "m1", "name": "author", "type": "string", "count": 0}],- "built_in_field_enabled": False,- },- ),- ):- result, status = method(api, sqlite_session, "tenant-1", current_user, dataset_id)- assert status == 200- assert result["doc_metadata"] == [{"id": "m1", "name": "author", "type": "string", "count": 0}]- assert result["built_in_field_enabled"] is False- get_dataset.assert_called_once_with(str(dataset_id), "tenant-1", session=sqlite_session)- check_permission.assert_called_once_with(dataset, current_user, sqlite_session)-- def test_get_metadata_rejects_foreign_tenant_before_read(- self, app: Flask, current_user, dataset_id, sqlite_session: Session- ):- api = DatasetMetadataCreateApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=None) as get_dataset,- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(MetadataService, "get_dataset_metadatas") as get_metadata,- ):- with pytest.raises(NotFound):- method(api, sqlite_session, "tenant-1", current_user, dataset_id)-- get_dataset.assert_called_once_with(str(dataset_id), "tenant-1", session=sqlite_session)- check_permission.assert_not_called()- get_metadata.assert_not_called()-- def test_get_metadata_delegates_permission_check_in_rbac_mode(- self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session- ):- api = DatasetMetadataCreateApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- config_overrides_context(RBAC_ENABLED=True),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission") as check_permission,- patch.object(- MetadataService,- "get_dataset_metadatas",- return_value={"doc_metadata": [], "built_in_field_enabled": False},- ),- ):- _, status = method(api, sqlite_session, "tenant-1", current_user, dataset_id)-- assert status == 200- check_permission.assert_called_once_with(dataset, current_user, sqlite_session)-- def test_get_metadata_rejects_inaccessible_dataset(- self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session- ):- api = DatasetMetadataCreateApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission", side_effect=NoPermissionError),- patch.object(MetadataService, "get_dataset_metadatas") as get_metadata,- ):- with pytest.raises(Forbidden):- method(api, sqlite_session, "tenant-1", current_user, dataset_id)-- get_metadata.assert_not_called()---class TestDatasetMetadataApi:- def test_update_metadata_success(- self, app: Flask, current_user, dataset, dataset_id, metadata_id, sqlite_session: Session- ):- api = DatasetMetadataApi()- method = unwrap(api.patch)- payload = {"name": "updated-name"}- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission"),- patch.object(- MetadataService,- "update_metadata_name",- return_value={"id": "m1", "type": "string", "name": "updated-name"},- ) as update_metadata,- ):- result, status = method(- api,- MetadataUpdatePayload(name="updated-name"),- sqlite_session,- "tenant-1",- current_user,- dataset_id,- metadata_id,- )- assert status == 200- assert result["type"] == "string"- assert result["name"] == "updated-name"- get_dataset.assert_called_once_with(str(dataset_id), "tenant-1", session=sqlite_session)- update_metadata.assert_called_once_with(- dataset, str(metadata_id), "updated-name", current_user, session=sqlite_session- )-- def test_delete_metadata_success(- self, app: Flask, current_user, dataset, dataset_id, metadata_id, sqlite_session: Session- ):- api = DatasetMetadataApi()- method = unwrap(api.delete)- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,- patch.object(DatasetService, "check_dataset_permission"),- patch.object(MetadataService, "delete_metadata") as delete_metadata,- ):- result, status = method(api, sqlite_session, "tenant-1", current_user, dataset_id, metadata_id)- assert status == 204- assert result == ""- get_dataset.assert_called_once_with(str(dataset_id), "tenant-1", session=sqlite_session)- delete_metadata.assert_called_once_with(dataset, str(metadata_id), sqlite_session)---class TestDatasetMetadataBuiltInFieldApi:- def test_get_built_in_fields(self, app: Flask):- api = DatasetMetadataBuiltInFieldApi()- method = unwrap(api.get)- with (- app.test_request_context("/"),- patch.object(- MetadataService,- "get_built_in_fields",- return_value=[{"name": "document_name", "type": "string"}, {"name": "source", "type": "string"}],- ),- ):- result, status = method(api)- assert status == 200- assert result["fields"] == [{"name": "document_name", "type": "string"}, {"name": "source", "type": "string"}]---class TestDatasetMetadataBuiltInFieldActionApi:- def test_enable_built_in_field(self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session):- api = DatasetMetadataBuiltInFieldActionApi()- method = unwrap(api.post)- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- patch.object(MetadataService, "enable_built_in_field"),- ):- result, status = method(api, sqlite_session, current_user, dataset_id, "enable")- assert status == 204- assert result == ""---class TestDocumentMetadataEditApi:- def test_update_document_metadata_success(- self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session- ):- api = DocumentMetadataEditApi()- method = unwrap(api.post)- payload = {"operation": "add", "metadata": {}}- with (- app.test_request_context("/"),- patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- patch.object(MetadataService, "update_documents_metadata"),- ):- result, status = method(- api,- MetadataOperationData(- operation_data=[{"document_id": "00000000-0000-0000-0000-000000000001", "metadata_list": []}]- ),- sqlite_session,- dataset.tenant_id,- current_user,- dataset_id,- )- assert status == 204- assert result == ""-- def test_update_document_metadata_translates_missing_resource(- self, app: Flask, current_user, dataset, dataset_id, sqlite_session: Session- ):- api = DocumentMetadataEditApi()- method = unwrap(api.post)- request = MetadataOperationData(operation_data=[])- with (- app.test_request_context("/"),- patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),- patch.object(DatasetService, "check_dataset_permission"),- patch.object(- MetadataService,- "update_documents_metadata",- side_effect=MetadataResourceNotFoundError("Metadata not found."),- ),- pytest.raises(NotFound) as exc_info,- ):- method(api, request, sqlite_session, dataset.tenant_id, current_user, dataset_id)-- assert exc_info.value.description == "Metadata not found."+def routes(monkeypatch: pytest.MonkeyPatch, config_overrides):+ config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, LOGIN_DISABLED=True, RBAC_ENABLED=False)+ app = Flask(__name__)+ app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)+ Api(app).add_namespace(console_ns, path="/console/api")+ account = Account(name="Owner", email="[email protected]")+ account.id = "account-1"+ account.role = TenantAccountRole.OWNER+ identity = AccountWithTenant(account=account, tenant_id="tenant-1")+ monkeypatch.setattr("controllers.console.flask_admission.current_account_with_tenant", lambda: identity)+ monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: identity)+ service = create_autospec(MetadataService, instance=True, spec_set=True)+ service.require_dataset.side_effect = lambda context, dataset_id: DatasetRef(+ context.active_workspace_id, dataset_id+ )+ app.extensions["application_services"] = SimpleNamespace(knowledge=SimpleNamespace(metadata=service))+ return app.test_client(), service+++def test_create_serializes_result_and_passes_actor(routes):+ client, service = routes+ dataset_id = str(uuid4())+ service.create_metadata.return_value = {"id": str(uuid4()), "name": "author", "type": "string"}+ response = client.post(f"/console/api/datasets/{dataset_id}/metadata", json={"type": "string", "name": "author"})+ assert response.status_code == 201+ assert response.get_json()["name"] == "author"+ assert service.create_metadata.call_args.args[0] == DatasetRef("tenant-1", dataset_id)+ assert service.create_metadata.call_args.kwargs == {"actor_id": "account-1"}++[email protected]("payload", [{"name": None, "type": "string"}, {"name": "author", "type": None}])+def test_create_rejects_invalid_payload_before_application(routes, payload):+ client, service = routes+ response = client.post(f"/console/api/datasets/{uuid4()}/metadata", json=payload)+ assert response.status_code == 422+ service.create_metadata.assert_not_called()++[email protected](("error", "status"), [(DatasetNotFoundError(), 404), (DatasetAccessDeniedError(), 403)])+def test_get_maps_access_errors(routes, error, status):+ client, service = routes+ service.require_dataset.side_effect = error+ assert client.get(f"/console/api/datasets/{uuid4()}/metadata").status_code == status+ service.get_dataset_metadatas.assert_not_called()+++def test_get_serializes_field_counts(routes):+ client, service = routes+ service.get_dataset_metadatas.return_value = {"doc_metadata": [], "built_in_field_enabled": False}+ response = client.get(f"/console/api/datasets/{uuid4()}/metadata")+ assert response.status_code == 200+ assert response.get_json() == {"doc_metadata": [], "built_in_field_enabled": False}+++def test_rename_and_delete_use_explicit_metadata_id(routes):+ client, service = routes+ dataset_id, metadata_id = str(uuid4()), str(uuid4())+ path = f"/console/api/datasets/{dataset_id}/metadata/{metadata_id}"+ service.update_metadata_name.return_value = {"id": metadata_id, "name": "writer", "type": "string"}+ assert client.patch(path, json={"name": "writer"}).status_code == 200+ service.update_metadata_name.assert_called_once_with(+ DatasetRef("tenant-1", dataset_id), metadata_id, "writer", actor_id="account-1"+ )+ response = client.delete(path)+ assert response.status_code == 204+ assert response.data == b""+ service.delete_metadata.assert_called_once_with(DatasetRef("tenant-1", dataset_id), metadata_id)++[email protected]("action", ["enable", "disable"])+def test_builtin_toggle_is_dispatched(routes, action):+ client, service = routes+ dataset_id = str(uuid4())+ response = client.post(f"/console/api/datasets/{dataset_id}/metadata/built-in/{action}")+ assert response.status_code == 204+ operation = service.enable_built_in_field if action == "enable" else service.disable_built_in_field+ operation.assert_called_once_with(DatasetRef("tenant-1", dataset_id))+++def test_builtin_listing_uses_response_model(routes):+ client, service = routes+ service.get_built_in_fields.return_value = [{"name": "uploader", "type": "string"}]+ response = client.get("/console/api/datasets/metadata/built-in")+ assert response.status_code == 200+ assert response.get_json() == {"fields": [{"name": "uploader", "type": "string"}]}+++def test_document_metadata_missing_resource_is_not_found(routes):+ client, service = routes+ service.update_documents_metadata.side_effect = MetadataResourceNotFoundError("Document not found.")+ response = client.post(f"/console/api/datasets/{uuid4()}/documents/metadata", json={"operation_data": []})+ assert response.status_code == 404+++def test_get_requires_dataset_readonly_permission():+ [check] = rbac_checks(DatasetMetadataCreateApi.get)+ assert check.scene is RBACPermission.DATASET_READONLY+ assert isinstance(check.locator, DatasetId)diff --git a/api/tests/unit_tests/controllers/console/datasets/test_website.py b/api/tests/unit_tests/controllers/console/datasets/test_website.pyindex b790059f0c7..900b0aa8642 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_website.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_website.py@@ -11,7 +11,7 @@ from controllers.console.datasets.website import (WebsiteCrawlStatusApi,WebsiteCrawlStatusQuery,)-from services.website_service import (+from services.data_source.website_service import (WebsiteCrawlApiRequest,WebsiteCrawlStatusApiRequest,WebsiteService,diff --git a/api/tests/unit_tests/controllers/console/test_apikey.py b/api/tests/unit_tests/controllers/console/test_apikey.pyindex 882bd9bdaeb..a6bea948603 100644--- a/api/tests/unit_tests/controllers/console/test_apikey.py+++ b/api/tests/unit_tests/controllers/console/test_apikey.py@@ -18,6 +18,7 @@ from controllers.console.agent import rosterfrom controllers.console.datasets import datasetsfrom core.rbac import RBACPermissionfrom enums import DeploymentEdition+from extensions.application_services.datasets import build_dataset_dependenciesfrom extensions.application_services.knowledge import build_dataset_api_key_servicefrom extensions.ext_application_services import ApplicationServicesfrom libs.login import AccountWithTenant@@ -25,6 +26,7 @@ from machinery.context import RequestContextfrom models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Datasetfrom models.model import ApiToken+from repositories.workspace_member_query_repository import WorkspaceMemberQueryRepositoryfrom services.app.api_key_service import AppApiKeyNotReadyErrorfrom services.auth.api_key_contracts import (ApiKeyLimitExceededError,@@ -282,7 +284,14 @@ def persisted_keys_app(])sqlite_session.commit()- service = build_dataset_api_key_service(database_client=sqlite_session_factory)+ dataset_access = build_dataset_dependencies(+ database_client=sqlite_session_factory,+ workspace_roles=WorkspaceMemberQueryRepository(session_factory=sqlite_session_factory),+ ).access+ service = build_dataset_api_key_service(+ database_client=sqlite_session_factory,+ dataset_access=dataset_access,+ )services = Mock(dataset_api_keys=service)monkeypatch.setattr(apikey, "application_services", lambda: services)monkeypatch.setattr(datasets, "application_services", lambda: services)diff --git a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.pyindex fac5aa79ae7..83a840b0e67 100644--- a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py+++ b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py@@ -10,6 +10,7 @@ from typing import Literal, NotRequired, TypedDictfrom unittest.mock import MagicMockfrom models.dataset import Document+from services.knowledge.dataset_read_service import get_document_source_detailclass LocalFileInfo(TypedDict):@@ -117,7 +118,7 @@ class TestDocumentDetailDataSourceInfo:)# data_source_detail_dict should return raw data for notion_import- detail_result = document.get_data_source_detail_dict(session=MagicMock())+ detail_result = get_document_source_detail(document, session=MagicMock())assert detail_result == notion_data# Test website_crawl@@ -128,7 +129,7 @@ class TestDocumentDetailDataSourceInfo:)# data_source_detail_dict should return raw data for website_crawl- detail_result = document.get_data_source_detail_dict(session=MagicMock())+ detail_result = get_document_source_detail(document, session=MagicMock())assert detail_result == website_datadef test_local_file_data_source_detail_dict_without_db(self):@@ -140,5 +141,5 @@ class TestDocumentDetailDataSourceInfo:)# Should return empty dict for local_file type (handled in the model)- detail_result = document.get_data_source_detail_dict(session=MagicMock())+ detail_result = get_document_source_detail(document, session=MagicMock())assert detail_result == {}diff --git a/api/tests/unit_tests/controllers/console/test_workspace_credential_mutation_permissions.py b/api/tests/unit_tests/controllers/console/test_workspace_credential_mutation_permissions.pyindex f89ccb3e765..7e4888e185b 100644--- a/api/tests/unit_tests/controllers/console/test_workspace_credential_mutation_permissions.py+++ b/api/tests/unit_tests/controllers/console/test_workspace_credential_mutation_permissions.py@@ -6,7 +6,6 @@ import pytestfrom controllers.common.rbac import AgentId, RBACPermission, Workspacefrom controllers.console.agent.composer import AgentComposerApifrom controllers.console.agent.roster import AgentAppApi-from controllers.console.datasets.data_source import DataSourceApifrom controllers.console.datasets.rag_pipeline.datasource_auth import DatasourceAuthfrom controllers.console.workspace.model_providers import ModelProviderCredentialApifrom controllers.console.workspace.models import ModelProviderModelCredentialApi@@ -22,8 +21,6 @@ from tests.unit_tests.controllers.rbac_introspection import rbac_checks("method", "permission"),[(ToolBuiltinProviderAddApi.post, RBACPermission.CREDENTIAL_CREATE),- (DataSourceApi.patch, RBACPermission.CREDENTIAL_MANAGE),- (DataSourceApi.get, RBACPermission.CREDENTIAL_MANAGE),],)def test_workspace_credential_mutations_require_management_permission(diff --git a/api/tests/unit_tests/controllers/console/workspace/test_models.py b/api/tests/unit_tests/controllers/console/workspace/test_models.pyindex d67f7122bef..2afd26c4644 100644--- a/api/tests/unit_tests/controllers/console/workspace/test_models.py+++ b/api/tests/unit_tests/controllers/console/workspace/test_models.py@@ -137,7 +137,7 @@ class TestDefaultModelApi:with (app.test_request_context("/"),- patch("services.model_provider_service.create_plugin_provider_manager", return_value=manager),+ patch("services.model_provider.service.create_plugin_provider_manager", return_value=manager),patch.object(manager, "get_configurations", return_value=configurations),):result = unwrap(api.post)(api, payload, "tenant1")@@ -164,7 +164,7 @@ class TestDefaultModelApi:with (app.test_request_context("/"),- patch("services.model_provider_service.create_plugin_provider_manager", return_value=manager),+ patch("services.model_provider.service.create_plugin_provider_manager", return_value=manager),):result = unwrap(api.post)(api, ParserPostDefault(model_settings=[]), "tenant1")diff --git a/api/tests/unit_tests/controllers/console/workspace/test_tool_provider_apis.py b/api/tests/unit_tests/controllers/console/workspace/test_tool_provider_apis.pyindex 1f9f4d863b4..1d078671a53 100644--- a/api/tests/unit_tests/controllers/console/workspace/test_tool_provider_apis.py+++ b/api/tests/unit_tests/controllers/console/workspace/test_tool_provider_apis.py@@ -2,8 +2,10 @@from __future__ import annotations+from collections.abc import Iteratorfrom inspect import unwrap-from unittest.mock import patch+from types import SimpleNamespace+from unittest.mock import MagicMock, create_autospec, patchimport pytestfrom flask import Flask@@ -54,6 +56,17 @@ from controllers.console.workspace.tool_providers import ()from core.tools.entities.api_entities import ToolProviderApiEntity as CoreToolProviderApiEntityfrom models.account import Account, TenantAccountRole+from services.credentials.query import CredentialQuery+++def credential_query() -> Iterator[MagicMock]:+ query = create_autospec(CredentialQuery, instance=True, spec_set=True)+ with patch(+ "controllers.console.workspace.tool_providers.application_services",+ return_value=SimpleNamespace(credential_queries=query),+ ):+ yield querydef empty_mapping() -> dict[str, object]:@@ -337,7 +350,7 @@ class TestBuiltinProviderApis:req = BuiltinToolUpdatePayload(**payload)assert method(api, req, "t", make_account(), "provider")["result"] == "success"- def test_get_credentials(self, app: Flask) -> None:+ def test_get_credentials(self, app: Flask, credential_query: MagicMock) -> None:api = ToolBuiltinProviderGetCredentialsApi()method = unwrap(api.get)@@ -346,9 +359,10 @@ class TestBuiltinProviderApis:patch("controllers.console.workspace.tool_providers.BuiltinToolManageService.get_builtin_tool_provider_credentials",return_value=[credential_payload()],- ),+ ) as get_credentials,):assert method(api, "t", make_account(id="user-1"), "provider")[0]["id"] == "credential-1"+ assert get_credentials.call_args.kwargs["credential_query"] is credential_querydef test_icon(self, app: Flask) -> None:api = ToolBuiltinProviderIconApi()@@ -391,7 +405,7 @@ class TestBuiltinProviderApis:req = BuiltinProviderDefaultCredentialPayload(id="c1")assert method(api, req, "t", "provider")["result"] == "success"- def test_get_credential_info(self, app: Flask) -> None:+ def test_get_credential_info(self, app: Flask, credential_query: MagicMock) -> None:api = ToolBuiltinProviderGetCredentialInfoApi()method = unwrap(api.get)@@ -400,9 +414,10 @@ class TestBuiltinProviderApis:patch("controllers.console.workspace.tool_providers.BuiltinToolManageService.get_builtin_tool_provider_credential_info",return_value=credential_info_payload(),- ),+ ) as get_info,):assert method(api, "t", make_account(), "provider")["credentials"][0]["id"] == "credential-1"+ assert get_info.call_args.kwargs["credential_query"] is credential_querydef test_get_oauth_client_schema(self, app: Flask) -> None:api = ToolBuiltinProviderGetOauthClientSchemaApi()diff --git a/api/tests/unit_tests/controllers/console/workspace/test_tool_providers.py b/api/tests/unit_tests/controllers/console/workspace/test_tool_providers.pyindex c205f931cc6..cb61284cd35 100644--- a/api/tests/unit_tests/controllers/console/workspace/test_tool_providers.py+++ b/api/tests/unit_tests/controllers/console/workspace/test_tool_providers.py@@ -7,7 +7,7 @@ import importlibfrom collections.abc import Callablefrom contextlib import ExitStack, contextmanagerfrom inspect import unwrap-from types import ModuleType+from types import ModuleType, SimpleNamespacefrom unittest.mock import MagicMock, patchfrom uuid import NAMESPACE_URL, uuid5@@ -24,6 +24,8 @@ from models import Account, BuiltinToolProvider, Tenant, TenantAccountJoinfrom models.account import TenantAccountRolefrom models.credential_permission import CredentialPermissionfrom models.enums import PermissionEnum+from repositories.credentials.query_repository import CredentialQueryRepository+from services.credentials.query import CredentialQueryif not hasattr(builtins, "MethodView"):builtins.MethodView = MethodView # type: ignore[attr-defined]@@ -79,6 +81,9 @@ def controller_module(monkeypatch: pytest.MonkeyPatch, config_overrides: Callabllogin_module = importlib.import_module("libs.login")monkeypatch.setattr(login_module, "check_csrf_token", lambda *args, **kwargs: None)+ monkeypatch.setattr(+ module, "application_services", lambda: SimpleNamespace(credential_queries=MagicMock(spec=CredentialQuery))+ )return module@@ -128,11 +133,18 @@ def _provider_credential(@contextmanagerdef _bind_database_session(session: Session):- database_session = scoped_session(- sessionmaker(bind=session.get_bind(), expire_on_commit=False),- )+ session_factory = sessionmaker(bind=session.get_bind(), expire_on_commit=False)+ database_session = scoped_session(session_factory)try:- with patch("extensions.ext_database.db.session", database_session):+ with (+ patch("extensions.ext_database.db.session", database_session),+ patch(+ "controllers.console.workspace.tool_providers.application_services",+ return_value=SimpleNamespace(+ credential_queries=CredentialQueryRepository(session_factory=session_factory)+ ),+ ),+ ):yieldfinally:database_session.remove()diff --git a/api/tests/unit_tests/controllers/console/workspace/test_trigger_provider_apis.py b/api/tests/unit_tests/controllers/console/workspace/test_trigger_provider_apis.pyindex 074f8d527c2..bf5ab7485af 100644--- a/api/tests/unit_tests/controllers/console/workspace/test_trigger_provider_apis.py+++ b/api/tests/unit_tests/controllers/console/workspace/test_trigger_provider_apis.py@@ -2,9 +2,11 @@from __future__ import annotations+from collections.abc import Iteratorfrom datetime import datetimefrom inspect import unwrap-from unittest.mock import ANY, MagicMock, patch+from types import SimpleNamespace+from unittest.mock import ANY, MagicMock, create_autospec, patchimport pytestfrom flask import Flask@@ -36,6 +38,17 @@ from core.plugin.entities.plugin_daemon import CredentialTypefrom core.trigger.entities.api_entities import SubscriptionBuilderApiEntity, TriggerProviderApiEntityfrom core.trigger.entities.entities import RequestLogfrom models.account import Account+from services.credentials.query import CredentialQuery+++def credential_query() -> Iterator[MagicMock]:+ query = create_autospec(CredentialQuery, instance=True, spec_set=True)+ with patch(+ "controllers.console.workspace.trigger_providers.application_services",+ return_value=SimpleNamespace(credential_queries=query),+ ):+ yield querydef mock_user() -> Account:@@ -127,7 +140,7 @@ class TestTriggerProviderApis:class TestTriggerSubscriptionListApi:- def test_list_success(self, app: Flask) -> None:+ def test_list_success(self, app: Flask, credential_query: MagicMock) -> None:api = TriggerSubscriptionListApi()method = unwrap(api.get)@@ -136,11 +149,12 @@ class TestTriggerSubscriptionListApi:patch("controllers.console.workspace.trigger_providers.TriggerProviderService.list_trigger_provider_subscriptions",return_value=[],- ),+ ) as list_subscriptions,):assert method(api, "t1", mock_user(), "github") == []+ assert list_subscriptions.call_args.kwargs["credential_query"] is credential_query- def test_list_invalid_provider(self, app: Flask) -> None:+ def test_list_invalid_provider(self, app: Flask, credential_query: MagicMock) -> None:api = TriggerSubscriptionListApi()method = unwrap(api.get)@@ -149,10 +163,11 @@ class TestTriggerSubscriptionListApi:patch("controllers.console.workspace.trigger_providers.TriggerProviderService.list_trigger_provider_subscriptions",side_effect=ValueError("bad"),- ),+ ) as list_subscriptions,):result, status = method(api, "t1", mock_user(), "bad")assert status == 404+ assert list_subscriptions.call_args.kwargs["credential_query"] is credential_queryclass TestTriggerSubscriptionBuilderApis:diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.pyindex 4a87b4a8ff6..4cb3d5ec648 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py@@ -46,12 +46,16 @@ from controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow import (KnowledgebasePipelineFileUploadApi,PipelineRunApi,)+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom extensions.storage.storage_type import StorageTypefrom models.account import Accountfrom models.dataset import Dataset, Pipelinefrom models.enums import CreatorUserRolefrom models.model import UploadFile+from repositories.credentials.query_repository import CredentialQueryRepository+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from services.data_source.provider_service import DatasourceProviderServicefrom services.errors.file import FileTooLargeError as FileTooLargeServiceErrorfrom services.errors.file import UnsupportedFileTypeErrorfrom services.rag_pipeline.entity.pipeline_service_api_entities import (@@ -86,6 +90,15 @@ def _persist_pipeline(session: Session, *, tenant_id: str) -> Pipeline:def _bind_database(sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch):session_proxy = scoped_session(sqlite_session_factory)monkeypatch.setattr(workflow_module.db, "session", session_proxy)+ query = CredentialQueryRepository(session_factory=sqlite_session_factory)+ providers = DatasourceProviderService(+ credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ )+ monkeypatch.setattr(+ workflow_module,+ "application_services",+ lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)),+ )yieldsession_proxy.remove()@@ -448,7 +461,11 @@ class TestDatasourcePluginsApiGet:assert status == 200assert response == datasource_pluginsmock_svc_instance.get_datasource_plugins.assert_called_once_with(- tenant_id=tenant_id, dataset_id=dataset_id, is_published=True+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ is_published=True,+ credential_query=workflow_module.application_services().credential_queries,+ datasource_providers=workflow_module.application_services().data_sources.providers,)@patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.RagPipelineService")@@ -469,7 +486,11 @@ class TestDatasourcePluginsApiGet:assert status == 200assert response == []mock_svc_instance.get_datasource_plugins.assert_called_once_with(- tenant_id=tenant_id, dataset_id=dataset_id, is_published=False+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ is_published=False,+ credential_query=workflow_module.application_services().credential_queries,+ datasource_providers=workflow_module.application_services().data_sources.providers,)def test_get_plugins_not_found(self, app: Flask, sqlite_session: Session):@@ -607,7 +628,15 @@ class TestPipelineRunApiPost:@patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.RagPipelineService")@patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.service_api_ns")def test_post_success_streaming(- self, mock_ns, mock_svc_cls, mock_current_user, mock_gen_svc, mock_helper, app, sqlite_session: Session+ self,+ mock_ns,+ mock_svc_cls,+ mock_current_user,+ mock_gen_svc,+ mock_helper,+ app: Flask,+ sqlite_session: Session,+ pipeline_application: PipelineGenerator,):"""Test successful pipeline run with streaming response."""tenant_id = str(uuid.uuid4())@@ -639,6 +668,8 @@ class TestPipelineRunApiPost:assert response == {"result": "ok"}mock_svc_cls.assert_called_once_with(sqlite_session)mock_gen_svc.generate.assert_called_once()+ assert mock_gen_svc.generate.call_args.kwargs["generator"] is pipeline_application+ assert mock_gen_svc.generate.call_args.kwargs["session"] is sqlite_sessiondef test_post_not_found(self, app: Flask, sqlite_session: Session):"""Test NotFound when dataset check fails."""diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.pyindex 56aefeda1c7..96eb5c48afa 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py@@ -17,7 +17,8 @@ Focus on:import inspectimport uuidfrom collections.abc import Callable-from unittest.mock import ANY, Mock, patch+from types import SimpleNamespace+from unittest.mock import ANY, Mock, create_autospec, patchimport pytestfrom flask import Flask@@ -42,10 +43,19 @@ from models.account import Account, Tenantfrom models.dataset import ChildChunk, Dataset, Document, DocumentSegment, DocumentSegmentSummaryfrom models.enums import IndexingStatus, SegmentTypefrom services.api_token_service import CachedApiToken-from services.dataset_service import DocumentService, SegmentService+from services.knowledge.dataset_service import DocumentService, SegmentService+from services.knowledge.segments.application import SegmentMutationServicefrom tests.unit_tests.model_factories import make_account[email protected](autouse=True)+def segment_mutations():+ mutations = create_autospec(SegmentMutationService, instance=True, spec_set=True)+ services = SimpleNamespace(knowledge=SimpleNamespace(segments=SimpleNamespace(mutations=mutations)))+ with patch("controllers.service_api.dataset.segment.application_services", return_value=services):+ yield mutations++def _segment_response_dict(summary: str | None = None):return {"id": "seg-1",@@ -555,6 +565,7 @@ class TestChildChunkServiceMockedBehavior:document=Document(),dataset=Dataset(),session=unbound_session,+ actor_id="account-id",)assert result == mock_child_chunk@@ -627,6 +638,7 @@ class TestChildChunkServiceMockedBehavior:document=Document(),dataset=Dataset(),session=unbound_session,+ actor_id="account-id",)assert result.content == "Updated content"@@ -722,7 +734,7 @@ class TestSegmentUpdatePayload:def test_payload_with_segment_args(self):"""Test payload with SegmentUpdateArgs."""from controllers.service_api.dataset.segment import SegmentUpdatePayload- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgssegment_args = SegmentUpdateArgs(content="Updated content")payload = SegmentUpdatePayload(segment=segment_args)@@ -731,7 +743,7 @@ class TestSegmentUpdatePayload:def test_payload_with_answer_update(self):"""Test payload with answer update."""from controllers.service_api.dataset.segment import SegmentUpdatePayload- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgssegment_args = SegmentUpdateArgs(answer="Updated answer")payload = SegmentUpdatePayload(segment=segment_args)@@ -740,7 +752,7 @@ class TestSegmentUpdatePayload:def test_payload_with_keywords_update(self):"""Test payload with keywords update."""from controllers.service_api.dataset.segment import SegmentUpdatePayload- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgssegment_args = SegmentUpdateArgs(keywords=["new", "keywords"])payload = SegmentUpdatePayload(segment=segment_args)@@ -749,7 +761,7 @@ class TestSegmentUpdatePayload:def test_payload_with_enabled_toggle(self):"""Test payload with enabled toggle."""from controllers.service_api.dataset.segment import SegmentUpdatePayload- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgssegment_args = SegmentUpdateArgs(enabled=True)payload = SegmentUpdatePayload(segment=segment_args)@@ -758,7 +770,7 @@ class TestSegmentUpdatePayload:def test_payload_with_regenerate_child_chunks(self):"""Test payload with regenerate_child_chunks flag."""from controllers.service_api.dataset.segment import SegmentUpdatePayload- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgssegment_args = SegmentUpdateArgs(regenerate_child_chunks=True)payload = SegmentUpdatePayload(segment=segment_args)@@ -770,7 +782,7 @@ class TestSegmentUpdateArgs:def test_args_with_defaults(self):"""Test args with default values."""- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgsargs = SegmentUpdateArgs()assert args.content is None@@ -781,14 +793,14 @@ class TestSegmentUpdateArgs:def test_args_with_content(self):"""Test args with content update."""- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgsargs = SegmentUpdateArgs(content="New content here")assert args.content == "New content here"def test_args_with_all_fields(self):"""Test args with all fields populated."""- from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs+ from services.knowledge.entities.segments import SegmentUpdateArgsargs = SegmentUpdateArgs(content="Full content",@@ -813,7 +825,7 @@ class TestSegmentCreateArgs:def test_args_with_defaults(self):"""Test args with default values."""- from services.entities.knowledge_entities.knowledge_entities import SegmentCreateArgs+ from services.knowledge.entities.knowledge_entities import SegmentCreateArgsargs = SegmentCreateArgs()assert args.content is None@@ -823,7 +835,7 @@ class TestSegmentCreateArgs:def test_args_with_content_and_answer(self):"""Test args with content and answer for Q&A mode."""- from services.entities.knowledge_entities.knowledge_entities import SegmentCreateArgs+ from services.knowledge.entities.knowledge_entities import SegmentCreateArgsargs = SegmentCreateArgs(content="Question?", answer="Answer!")assert args.content == "Question?"@@ -831,7 +843,7 @@ class TestSegmentCreateArgs:def test_args_with_keywords(self):"""Test args with keywords for search indexing."""- from services.entities.knowledge_entities.knowledge_entities import SegmentCreateArgs+ from services.knowledge.entities.knowledge_entities import SegmentCreateArgsargs = SegmentCreateArgs(content="Test content", keywords=["machine learning", "AI", "neural networks"])assert args.keywords is not None@@ -843,7 +855,7 @@ class TestChildChunkUpdateArgs:def test_args_with_content_only(self):"""Test args with content only."""- from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs+ from services.knowledge.entities.segments import ChildChunkUpdateArgsargs = ChildChunkUpdateArgs(content="Updated chunk content")assert args.content == "Updated chunk content"@@ -851,7 +863,7 @@ class TestChildChunkUpdateArgs:def test_args_with_id_and_content(self):"""Test args with both id and content."""- from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs+ from services.knowledge.entities.segments import ChildChunkUpdateArgschunk_id = str(uuid.uuid4())args = ChildChunkUpdateArgs(id=chunk_id, content="Updated content")@@ -1033,8 +1045,8 @@ class TestSegmentApiGet(SQLiteEndpointTest):``current_account_with_tenant()`` and response serialization."""- @patch("controllers.service_api.dataset.segment.segment_responses_with_summaries")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segments_summaries")+ @patch("controllers.service_api.dataset.segment.load_segment_details")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segments_summaries")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.current_account_with_tenant")@@ -1077,8 +1089,8 @@ class TestSegmentApiGet(SQLiteEndpointTest):mock_dump_segments.assert_called_once_with([mock_segment], {}, session=ANY)assert isinstance(mock_dump_segments.call_args.kwargs["session"], Session)- @patch("controllers.service_api.dataset.segment.segment_responses_with_summaries")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segments_summaries")+ @patch("controllers.service_api.dataset.segment.load_segment_details")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segments_summaries")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.current_account_with_tenant")@@ -1119,8 +1131,8 @@ class TestSegmentApiGet(SQLiteEndpointTest):assert response["total"] == page_sizeassert response["page"] == 1- @patch("controllers.service_api.dataset.segment.segment_responses_with_summaries")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segments_summaries")+ @patch("controllers.service_api.dataset.segment.load_segment_details")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segments_summaries")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.current_account_with_tenant")@@ -1224,8 +1236,8 @@ class TestSegmentApiPost(SQLiteEndpointTest):mock_rate_limit.enabled = Falsemock_feature_svc.get_knowledge_rate_limit.return_value = mock_rate_limit- @patch("controllers.service_api.dataset.segment.segment_responses_with_summaries")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segments_summaries")+ @patch("controllers.service_api.dataset.segment.load_segment_details")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segments_summaries")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.current_account_with_tenant")@@ -1426,7 +1438,9 @@ class TestDatasetSegmentApiDelete(SQLiteEndpointTest):# Assertassert response == ("", 204)- mock_seg_svc.delete_segment.assert_called_once_with(mock_segment, mock_doc, mock_dataset, self.session)+ mock_seg_svc.delete_segment.assert_called_once_with(+ mock_segment, mock_doc, mock_dataset, self.session, mutations=ANY+ )@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@@ -1564,8 +1578,8 @@ class TestDatasetSegmentApiUpdate(SQLiteEndpointTest):mock_rate_limit.enabled = Falsemock_feature_svc.get_knowledge_rate_limit.return_value = mock_rate_limit- @patch("controllers.service_api.dataset.segment.segment_response_with_summary")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segment_summary")+ @patch("controllers.service_api.dataset.segment.load_segment_detail")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segment_summary")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.DatasetService")@@ -1717,8 +1731,8 @@ class TestDatasetSegmentApiGetSingle(SQLiteEndpointTest):``current_account_with_tenant()`` and response serialization."""- @patch("controllers.service_api.dataset.segment.segment_response_with_summary")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segment_summary")+ @patch("controllers.service_api.dataset.segment.load_segment_detail")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segment_summary")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.DatasetService")@@ -1764,8 +1778,8 @@ class TestDatasetSegmentApiGetSingle(SQLiteEndpointTest):mock_dump_segment.assert_called_once_with(mock_segment, None, session=ANY)assert isinstance(mock_dump_segment.call_args.kwargs["session"], Session)- @patch("controllers.service_api.dataset.segment.segment_response_with_summary")- @patch("controllers.service_api.dataset.segment.SummaryIndexService.get_segment_summary")+ @patch("controllers.service_api.dataset.segment.load_segment_detail")+ @patch("controllers.service_api.dataset.segment.SummaryIndexAdapter.get_segment_summary")@patch("controllers.service_api.dataset.segment.SegmentService")@patch("controllers.service_api.dataset.segment.DocumentService")@patch("controllers.service_api.dataset.segment.DatasetService")@@ -1783,7 +1797,7 @@ class TestDatasetSegmentApiGetSingle(SQLiteEndpointTest):mock_dataset,mock_segment,):- """Test that single segment response includes summary content from SummaryIndexService."""+ """Test that single segment response includes summary content from SummaryIndexAdapter."""mock_account_fn.return_value = (_account(), mock_tenant.id)self._persist_dataset(mock_dataset, mock_tenant.id)mock_dataset_svc.check_dataset_model_setting.return_value = Nonediff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.pyindex 665d4cb0607..159dddfbbc2 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py@@ -60,10 +60,10 @@ from models.enums import (SegmentStatus,)from models.model import ApiToken, UploadFile-from services.dataset_ref_service import DatasetRef-from services.dataset_service import DocumentService-from services.entities.knowledge_entities.knowledge_entities import ProcessRule, RetrievalModelfrom services.errors.file import FileTooLargeError as FileTooLargeServiceError+from services.knowledge.dataset_service import DocumentService+from services.knowledge.entities.knowledge_entities import ProcessRule, RetrievalModel+from services.knowledge.resource_scope import DatasetReffrom tests.unit_tests.config_override import config_overrides_contextfrom tests.unit_tests.model_factories import make_account, make_upload_file@@ -639,8 +639,8 @@ class TestDocumentServiceBatchMethods:class TestDocumentServiceFileOperations:"""Test DocumentService file related operations."""- @patch("services.dataset_service.file_helpers.get_signed_file_url")- @patch("services.dataset_service.DocumentService._get_upload_file_for_upload_file_document")+ @patch("services.knowledge.dataset_service.file_helpers.get_signed_file_url")+ @patch("services.knowledge.dataset_service.DocumentService._get_upload_file_for_upload_file_document")def test_get_document_download_url(self, mock_get_file, mock_signed_url, sqlite_session: Session):"""Test generation of download URL."""mock_doc = make_serializable_document()@@ -661,8 +661,8 @@ class TestDocumentServiceSaveValidation:"""Test validations during document saving."""@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)- @patch("services.dataset_service.DatasetService.check_doc_form")- @patch("services.dataset_service.FeatureService.get_features")+ @patch("services.knowledge.dataset_service.DatasetService.check_doc_form")+ @patch("services.knowledge.dataset_service.FeatureService.get_features")def test_save_document_validates_doc_form(self, mock_features, mock_check_form, sqlite_session: Session):"""Test that doc_form is validated during save."""dataset = make_dataset(tenant_id="tenant_id")diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_hit_testing.py b/api/tests/unit_tests/controllers/service_api/dataset/test_hit_testing.pyindex f47de0c8d50..b1c5a2a6250 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_hit_testing.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_hit_testing.py@@ -25,7 +25,7 @@ import servicesfrom controllers.service_api.dataset.hit_testing import HitTestingApi, HitTestingPayloadfrom models.account import Account, Tenant, TenantAccountRolefrom models.dataset import Dataset-from services.entities.knowledge_entities.knowledge_entities import RetrievalModel+from services.knowledge.entities.knowledge_entities import RetrievalModel# ---------------------------------------------------------------------------# HitTestingPayload Model Testsdiff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py b/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.pyindex e138a5b3c77..55b42a4e355 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py@@ -38,8 +38,9 @@ from controllers.service_api.dataset.metadata import (from models.account import Account, Tenantfrom models.dataset import Datasetfrom models.enums import PermissionEnum-from services.entities.knowledge_entities.knowledge_entities import MetadataArgs, MetadataOperationDatafrom services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.entities.knowledge_entities import MetadataArgs, MetadataOperationData+from services.knowledge.resource_scope import DatasetRef@pytest.fixture@@ -93,6 +94,30 @@ class _UsesSQLiteSession:self.session = sqlite_session[email protected]("method", ["patch", "delete"])+def test_metadata_mutation_translates_missing_resource(method, sqlite_session, mock_tenant, mock_dataset):+ with (+ patch.object(metadata_module, "DatasetService") as datasets,+ patch.object(metadata_module, "application_services") as services,+ ):+ datasets.get_dataset_for_tenant.return_value = mock_dataset+ metadata = services.return_value.knowledge.metadata+ operation = metadata.update_metadata_name if method == "patch" else metadata.delete_metadata+ operation.side_effect = MetadataResourceNotFoundError("Metadata not found.")+ resource = DatasetMetadataServiceApi()+ arguments = {+ "session": sqlite_session,+ "tenant_id": mock_tenant.id,+ "dataset_id": mock_dataset.id,+ "metadata_id": uuid.uuid4(),+ }+ if method == "patch":+ arguments["payload"] = MetadataUpdatePayload(name="renamed")+ endpoint = DatasetMetadataServiceApi.patch if method == "patch" else DatasetMetadataServiceApi.delete+ with pytest.raises(NotFound, match="Metadata not found"):+ unwrap(endpoint)(resource, **arguments)++class TestDatasetMetadataCreatePost(_UsesSQLiteSession):"""Tests for DatasetMetadataCreateServiceApi.post().@@ -107,7 +132,7 @@ class TestDatasetMetadataCreatePost(_UsesSQLiteSession):metadata_args = MetadataArgs.model_validate(request.get_json() or {})return unwrap(api.post)(api, metadata_args, session, **kwargs)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_create_metadata_success(self,@@ -118,7 +143,8 @@ class TestDatasetMetadataCreatePost(_UsesSQLiteSession):mock_dataset,):"""Test successful metadata creation."""- mock_dataset_svc.get_dataset.return_value = mock_dataset+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadata+ mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = Nonemock_metadata = {"id": "meta-1", "type": "string", "name": "Author"}mock_meta_svc.create_metadata.return_value = mock_metadata@@ -150,7 +176,7 @@ class TestDatasetMetadataCreatePost(_UsesSQLiteSession):mock_dataset,):"""Test 404 when dataset not found."""- mock_dataset_svc.get_dataset.return_value = None+ mock_dataset_svc.get_dataset_for_tenant.return_value = Nonewith app.test_request_context(f"/datasets/{mock_dataset.id}/metadata",@@ -171,7 +197,7 @@ class TestDatasetMetadataCreatePost(_UsesSQLiteSession):class TestDatasetMetadataCreateGet(_UsesSQLiteSession):"""Tests for DatasetMetadataCreateServiceApi.get()."""- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_get_metadata_success(self,@@ -182,7 +208,8 @@ class TestDatasetMetadataCreateGet(_UsesSQLiteSession):mock_dataset,):"""Test successful metadata list retrieval."""- mock_dataset_svc.get_dataset.return_value = mock_dataset+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadata+ mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_meta_svc.get_dataset_metadatas.return_value = {"doc_metadata": [{"id": "m1", "name": "Author", "type": "string", "count": 0}],"built_in_field_enabled": False,@@ -213,7 +240,7 @@ class TestDatasetMetadataCreateGet(_UsesSQLiteSession):mock_dataset,):"""Test 404 when dataset not found."""- mock_dataset_svc.get_dataset.return_value = None+ mock_dataset_svc.get_dataset_for_tenant.return_value = Nonewith app.test_request_context(f"/datasets/{mock_dataset.id}/metadata",@@ -240,7 +267,7 @@ class TestDatasetMetadataServiceApiPatch(_UsesSQLiteSession):payload = MetadataUpdatePayload.model_validate(request.get_json() or {})return unwrap(api.patch)(api, payload, session, **kwargs)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_update_metadata_name_success(self,@@ -252,6 +279,7 @@ class TestDatasetMetadataServiceApiPatch(_UsesSQLiteSession):account: Account,):"""Test successful metadata name update."""+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadatametadata_id = str(uuid.uuid4())mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = None@@ -278,7 +306,7 @@ class TestDatasetMetadataServiceApiPatch(_UsesSQLiteSession):str(mock_dataset.id), mock_tenant.id, session=session)mock_meta_svc.update_metadata_name.assert_called_once_with(- mock_dataset, metadata_id, "New Name", account, session=session+ DatasetRef(mock_tenant.id, mock_dataset.id), metadata_id, "New Name", actor_id=account.id)@patch("controllers.service_api.dataset.metadata.DatasetService")@@ -320,7 +348,7 @@ class TestDatasetMetadataServiceApiDelete(_UsesSQLiteSession):def _call_delete(api, session: Session, **kwargs):return unwrap(api.delete)(api, session, **kwargs)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_delete_metadata_success(self,@@ -331,6 +359,7 @@ class TestDatasetMetadataServiceApiDelete(_UsesSQLiteSession):mock_dataset,):"""Test successful metadata deletion."""+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadatametadata_id = str(uuid.uuid4())mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = None@@ -354,7 +383,7 @@ class TestDatasetMetadataServiceApiDelete(_UsesSQLiteSession):mock_dataset_svc.get_dataset_for_tenant.assert_called_once_with(str(mock_dataset.id), mock_tenant.id, session=session)- mock_meta_svc.delete_metadata.assert_called_once_with(mock_dataset, metadata_id, session)+ mock_meta_svc.delete_metadata.assert_called_once_with(DatasetRef(mock_tenant.id, mock_dataset.id), metadata_id)@patch("controllers.service_api.dataset.metadata.DatasetService")def test_delete_metadata_dataset_not_found(@@ -392,7 +421,7 @@ class TestDatasetMetadataServiceApiDelete(_UsesSQLiteSession):class TestDatasetMetadataBuiltInFieldGet(_UsesSQLiteSession):"""Tests for DatasetMetadataBuiltInFieldServiceApi.get()."""- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")def test_get_built_in_fields_success(self,mock_meta_svc,@@ -401,6 +430,7 @@ class TestDatasetMetadataBuiltInFieldGet(_UsesSQLiteSession):mock_dataset,):"""Test successful built-in fields retrieval."""+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadatamock_meta_svc.get_built_in_fields.return_value = [{"name": "source", "type": "string"},]@@ -434,7 +464,7 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):def _call_post(api, session: Session, **kwargs):return unwrap(api.post)(api, session, **kwargs)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_enable_built_in_field(self,@@ -445,7 +475,8 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):mock_dataset,):"""Test enabling built-in metadata field."""- mock_dataset_svc.get_dataset.return_value = mock_dataset+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadata+ mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = Nonewith app.test_request_context(@@ -464,9 +495,9 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):assert status == 200assert response["result"] == "success"- mock_meta_svc.enable_built_in_field.assert_called_once_with(mock_dataset, session)+ mock_meta_svc.enable_built_in_field.assert_called_once_with(DatasetRef(mock_tenant.id, mock_dataset.id))- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_disable_built_in_field(self,@@ -477,7 +508,8 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):mock_dataset,):"""Test disabling built-in metadata field."""- mock_dataset_svc.get_dataset.return_value = mock_dataset+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadata+ mock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = Nonewith app.test_request_context(@@ -495,7 +527,7 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):)assert status == 200- mock_meta_svc.disable_built_in_field.assert_called_once_with(mock_dataset, session)+ mock_meta_svc.disable_built_in_field.assert_called_once_with(DatasetRef(mock_tenant.id, mock_dataset.id))@patch("controllers.service_api.dataset.metadata.DatasetService")def test_action_dataset_not_found(@@ -506,7 +538,7 @@ class TestDatasetMetadataBuiltInFieldAction(_UsesSQLiteSession):mock_dataset,):"""Test 404 when dataset not found."""- mock_dataset_svc.get_dataset.return_value = None+ mock_dataset_svc.get_dataset_for_tenant.return_value = Nonewith app.test_request_context(f"/datasets/{mock_dataset.id}/metadata/built-in/enable",@@ -540,7 +572,7 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):metadata_args = MetadataOperationData.model_validate(request.get_json() or {})return unwrap(api.post)(api, metadata_args, session, **kwargs)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_update_documents_metadata_success(self,@@ -552,6 +584,7 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):account: Account,):"""Test successful documents metadata update."""+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadatamock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_dataset_svc.check_dataset_permission.return_value = Nonemock_meta_svc.update_documents_metadata.return_value = None@@ -573,10 +606,9 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):assert status == 200assert response["result"] == "success"mock_meta_svc.update_documents_metadata.assert_called_once_with(- mock_dataset,+ DatasetRef(mock_tenant.id, mock_dataset.id),ANY,- account,- session=session,+ actor_id=account.id,)@patch("controllers.service_api.dataset.metadata.DatasetService")@@ -605,7 +637,7 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):dataset_id=mock_dataset.id,)- @patch("controllers.service_api.dataset.metadata.MetadataService")+ @patch("controllers.service_api.dataset.metadata.application_services")@patch("controllers.service_api.dataset.metadata.DatasetService")def test_update_documents_metadata_translates_missing_resource(self,@@ -615,6 +647,7 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):mock_tenant,mock_dataset,):+ mock_meta_svc = mock_meta_svc.return_value.knowledge.metadatamock_dataset_svc.get_dataset_for_tenant.return_value = mock_datasetmock_meta_svc.update_documents_metadata.side_effect = MetadataResourceNotFoundError("Document not found.")diff --git a/api/tests/unit_tests/controllers/test_rbac_route_resource_contract.py b/api/tests/unit_tests/controllers/test_rbac_route_resource_contract.pyindex 0fd2a205b2f..60a21436242 100644--- a/api/tests/unit_tests/controllers/test_rbac_route_resource_contract.py+++ b/api/tests/unit_tests/controllers/test_rbac_route_resource_contract.py@@ -27,15 +27,7 @@ NAMESPACES = (console_ns, openapi_ns)HTTP_METHODS = ("delete", "get", "head", "options", "patch", "post", "put")-# Known violations tracked separately: DatasetDocumentSegmentBatchImportApi binds one class-# to both the dataset-scoped import route and the job-scoped status route, so every method-# on it is reachable at a path carrying only a job id. Its permission points are genuinely-# per-dataset, so it needs the route split rather than a Workspace() locator. Remove these-# entries with that fix.-KNOWN_VIOLATIONS = {- ("DatasetDocumentSegmentBatchImportApi", "post"),- ("DatasetDocumentSegmentBatchImportApi", "get"),-}+KNOWN_VIOLATIONS: set[tuple[str, str]] = set()# The generated access-permission endpoints, spelled out here so this guard fails loudly if# the factory stops registering them (or renames a URL) rather than silently checking less.diff --git a/api/tests/unit_tests/controllers/test_swagger.py b/api/tests/unit_tests/controllers/test_swagger.pyindex 63933381138..33857795895 100644--- a/api/tests/unit_tests/controllers/test_swagger.py+++ b/api/tests/unit_tests/controllers/test_swagger.py@@ -262,6 +262,38 @@ def test_agent_tts_routes_document_voice_queries_and_binary_audio(openapi_json:assert _response_content_types(preview, status) == {"application/json"}+def test_console_segment_mutation_contracts_survive_admission(openapi_json: OpenAPIJSONLoader) -> None:+ from controllers.console import bp as console_bp++ payload = json.loads(openapi_json(console_bp, "/console/api/openapi.json", False))+ paths = payload["paths"]+ document_path = "/datasets/{dataset_id}/documents/{document_id}"+ mutations = (+ (f"{document_path}/segment", "post", "SegmentCreatePayload", "SegmentDetailResponse"),+ (f"{document_path}/segments/{{segment_id}}", "patch", "SegmentUpdatePayload", "SegmentDetailResponse"),+ (+ f"{document_path}/segments/{{segment_id}}/child_chunks",+ "patch",+ "ChildChunkBatchUpdatePayload",+ "ChildChunkBatchUpdateResponse",+ ),+ )+ for path, method, request_schema, response_schema in mutations:+ operation = paths[path][method]+ assert operation["requestBody"]["content"]["application/json"]["schema"] == {+ "$ref": f"#/components/schemas/{request_schema}"+ }+ assert operation["responses"]["200"]["content"]["application/json"]["schema"] == {+ "$ref": f"#/components/schemas/{response_schema}"+ }++ delete_operation = paths[f"{document_path}/segments"]["delete"]+ assert set(delete_operation["responses"]) == {"204"}+ parameters = _parameters_by_name(delete_operation)+ assert parameters["segment_id"]["in"] == "query"+ assert parameters["segment_id"]["schema"]["type"] == "array"++def test_service_document_file_routes_document_multipart_form_data(openapi_json: OpenAPIJSONLoader):from controllers.service_api import bp as service_api_bpdiff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.pyindex 841a003b84a..de16f2dc540 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py@@ -1,13 +1,13 @@import contextlibimport jsonfrom types import SimpleNamespace-from unittest.mock import MagicMock, PropertyMock+from unittest.mock import MagicMock, PropertyMock, create_autospecimport pytestfrom pytest_mock import MockerFixturefrom sqlalchemy import selectfrom sqlalchemy.engine import Engine-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerimport core.app.apps.pipeline.pipeline_generator as modulefrom core.app.apps.exc import GenerateTaskStoppedError@@ -17,6 +17,9 @@ from models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipefrom models.enums import DataSourceType, EndUserTypefrom models.model import EndUserfrom models.workflow import Workflow, WorkflowType+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from services.data_source.credential_gateway import DatasourceProviderCredentialStore+from services.data_source.provider_service import DatasourceProviderServiceTENANT_ID = "00000000-0000-0000-0000-000000000001"PIPELINE_ID = "00000000-0000-0000-0000-000000000002"@@ -38,7 +41,12 @@ class FakeRagPipelineGenerateEntity(SimpleNamespace):@pytest.fixturedef generator(mocker: MockerFixture, sqlite_engine: Engine):- gen = module.PipelineGenerator()+ gen = module.PipelineGenerator(+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),+ datasource_providers=DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )_patch_sqlite_engine(mocker, sqlite_engine)mocker.patch.object(module, "RagPipelineGenerateEntity", FakeRagPipelineGenerateEntity)@@ -228,10 +236,10 @@ def test_generate_published_pipeline_creates_documents_and_delay()mocker.patch.object(generator, "_prepare_user_inputs", return_value={"k": "v"})- mocker.patch("services.dataset_service.DocumentService.get_documents_position", return_value=1)+ mocker.patch("services.knowledge.dataset_service.DocumentService.get_documents_position", return_value=1)features = SimpleNamespace()get_features = mocker.patch("services.feature_service.FeatureService.get_features", return_value=features)- check_limits = mocker.patch("services.dataset_service.DocumentService.check_document_creation_limits")+ check_limits = mocker.patch("services.knowledge.dataset_service.DocumentService.check_document_creation_limits")mocker.patch.object(module.DifyCoreRepositoryFactory,@@ -292,7 +300,7 @@ def test_generate_published_pipeline_rejects_when_document_creation_limits_exceefeatures = SimpleNamespace()mocker.patch("services.feature_service.FeatureService.get_features", return_value=features)check_limits = mocker.patch(- "services.dataset_service.DocumentService.check_document_creation_limits",+ "services.knowledge.dataset_service.DocumentService.check_document_creation_limits",side_effect=ValueError("document limit exceeded"),)diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.pyindex 37e3912ba68..e2117219e00 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py@@ -19,7 +19,7 @@ import pytestfrom pytest_mock import MockerFixturefrom sqlalchemy import eventfrom sqlalchemy.engine import Engine-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerimport core.app.apps.pipeline.pipeline_runner as modulefrom core.app.apps.pipeline.pipeline_runner import PipelineRunner@@ -29,6 +29,7 @@ from models.dataset import Dataset, Document, Pipelinefrom models.enums import DocumentCreatedFromfrom models.model import EndUserfrom models.workflow import Workflow, WorkflowType+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom tests.unit_tests.model_factories import make_dataset, make_document, make_end_user, make_workflow@@ -117,7 +118,7 @@ def _build_app_generate_entity() -> SimpleNamespace:@pytest.fixture-def runner():+def runner(sqlite_engine: Engine):app_generate_entity = _build_app_generate_entity()queue_manager = MagicMock()variable_loader = MagicMock()@@ -133,6 +134,7 @@ def runner():system_user_id="sys",workflow_execution_repository=workflow_execution_repository,workflow_node_execution_repository=workflow_node_execution_repository,+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)@@ -174,12 +176,15 @@ def test_init_rag_pipeline_graph_not_found(mocker, runner):def test_update_document_status_on_failure(runner, sqlite_session: Session):document = _document()_, dataset, _ = _persist_scope(sqlite_session, documents=(document,))- dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)- document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, document.id)event = GraphRunFailedEvent(error="boom")- runner._update_document_status(event, document_ref)+ runner._update_document_status(+ event,+ workspace_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ )sqlite_session.expire_all()updated = sqlite_session.get(Document, document.id)@@ -190,10 +195,13 @@ def test_update_document_status_on_failure(runner, sqlite_session: Session):def test_update_document_status_skips_when_document_not_found(runner, sqlite_session: Session):_, dataset, _ = _persist_scope(sqlite_session)- dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)- document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, "missing")- runner._update_document_status(GraphRunFailedEvent(error="boom"), document_ref)+ runner._update_document_status(+ GraphRunFailedEvent(error="boom"),+ workspace_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id="missing",+ )assert sqlite_session.get(Document, "missing") is None@@ -207,14 +215,19 @@ def test_update_document_status_skips_without_document_ref(runner, sqlite_engineevent.listen(sqlite_engine, "checkout", record_checkout)try:- runner._update_document_status(GraphRunFailedEvent(error="boom"), None)+ runner._update_document_status(+ GraphRunFailedEvent(error="boom"),+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id=None,+ )finally:event.remove(sqlite_engine, "checkout", record_checkout)assert checkouts == 0-def test_run_pipeline_not_found():+def test_run_pipeline_not_found(sqlite_engine: Engine):app_generate_entity = _build_app_generate_entity()app_generate_entity.invoke_from = InvokeFrom.WEB_APPapp_generate_entity.single_iteration_run = None@@ -228,6 +241,7 @@ def test_run_pipeline_not_found():system_user_id="sys",workflow_execution_repository=MagicMock(),workflow_node_execution_repository=MagicMock(),+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)with pytest.raises(ValueError):@@ -299,7 +313,7 @@ def test_run_rejects_original_document_outside_pipeline_dataset_after_async_bounrunner.get_workflow.assert_not_called()-def test_run_workflow_not_initialized(sqlite_session: Session):+def test_run_workflow_not_initialized(sqlite_session: Session, sqlite_engine: Engine):app_generate_entity = _build_app_generate_entity()pipeline = _pipeline()@@ -316,18 +330,17 @@ def test_run_workflow_not_initialized(sqlite_session: Session):system_user_id="sys",workflow_execution_repository=MagicMock(),workflow_node_execution_repository=MagicMock(),+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)with pytest.raises(ValueError):runner.run()-def test_run_single_iteration_path(mocker: MockerFixture, sqlite_session: Session):+def test_run_single_iteration_path(mocker: MockerFixture, sqlite_session: Session, sqlite_engine: Engine):app_generate_entity = _build_app_generate_entity()app_generate_entity.single_iteration_run = MagicMock()_, dataset, _ = _persist_scope(sqlite_session, documents=(_document(),))- dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)- document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, "doc")runner = PipelineRunner(application_generate_entity=app_generate_entity,@@ -337,6 +350,7 @@ def test_run_single_iteration_path(mocker: MockerFixture, sqlite_session: Sessiosystem_user_id="sys",workflow_execution_repository=MagicMock(),workflow_node_execution_repository=MagicMock(),+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)runner._resolve_user_from = MagicMock(return_value=UserFrom.ACCOUNT)@@ -355,7 +369,12 @@ def test_run_single_iteration_path(mocker: MockerFixture, sqlite_session: Sessiorunner.run()runner._prepare_single_node_execution.assert_called_once()- runner._update_document_status.assert_called_once_with(event, document_ref)+ runner._update_document_status.assert_called_once_with(+ event,+ workspace_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id="doc",+ )runner._handle_event.assert_called()@@ -388,6 +407,7 @@ def test_run_normal_path_builds_graph(mocker: MockerFixture, sqlite_session: Sessystem_user_id="sys",workflow_execution_repository=MagicMock(),workflow_node_execution_repository=MagicMock(),+ documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)runner._resolve_user_from = MagicMock(return_value=UserFrom.ACCOUNT)diff --git a/api/tests/unit_tests/core/datasource/test_datasource_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_manager.pyindex 5e38166a5cd..da2c1591b65 100644--- a/api/tests/unit_tests/core/datasource/test_datasource_manager.py+++ b/api/tests/unit_tests/core/datasource/test_datasource_manager.py@@ -236,10 +236,6 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFixfake_runtime = mocker.Mock()fake_runtime.get_online_document_page_content.side_effect = _doc_messagesmocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)- mocker.patch(- "core.datasource.datasource_manager.DatasourceProviderService.get_datasource_credentials",- return_value=None,- )gen = DatasourceManager.stream_online_results(user_id="u1",@@ -247,11 +243,9 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFixdatasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,+ credentials={},)msgs = list(gen)assert len(msgs) == 1@@ -268,10 +262,6 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_doruntime = _Runtime()mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)- mocker.patch(- "core.datasource.datasource_manager.DatasourceProviderService.get_datasource_credentials",- return_value={"token": "t"},- )gen = DatasourceManager.stream_online_results(user_id="u1",@@ -279,11 +269,9 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_dodatasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="cred",datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,+ credentials={"token": "t"},)messages, final_value = _drain_generator(gen)@@ -304,10 +292,6 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture)yield from _gen_messages_text_only("never")mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=_Runtime())- mocker.patch(- "core.datasource.datasource_manager.DatasourceProviderService.get_datasource_credentials",- return_value={},- )with pytest.raises(ValueError, match="datasource_param is required for ONLINE_DOCUMENT streaming"):list(@@ -317,11 +301,9 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture)datasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",datasource_param=None,online_drive_request=None,+ credentials={},))@@ -333,11 +315,9 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture)datasource_type="online_drive",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",datasource_param=None,online_drive_request=None,+ credentials={},))@@ -352,10 +332,6 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_driruntime = _Runtime()mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)- mocker.patch(- "core.datasource.datasource_manager.DatasourceProviderService.get_datasource_credentials",- return_value={"token": "t"},- )gen = DatasourceManager.stream_online_results(user_id="u1",@@ -363,11 +339,9 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dridatasource_type="online_drive",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="cred",datasource_param=None,online_drive_request=types.SimpleNamespace(id="fid", bucket="b"),+ credentials={"token": "t"},)messages, final_value = _drain_generator(gen)@@ -378,10 +352,6 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dridef test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture):mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=mocker.Mock())- mocker.patch(- "core.datasource.datasource_manager.DatasourceProviderService.get_datasource_credentials",- return_value={},- )with pytest.raises(ValueError, match="Unsupported datasource type for streaming"):list(@@ -391,11 +361,9 @@ def test_stream_online_results_raises_for_unsupported_stream_type(mocker: Mockerdatasource_type="website_crawl",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",datasource_param=None,online_drive_request=None,+ credentials={},))@@ -416,14 +384,12 @@ def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):datasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={"k": "v"},datasource_info={"user_id": "u1"},variable_pool=mocker.Mock(),datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,+ credentials={},))# should contain one StreamChunkEvent then a final chunk (empty) and a completed event@@ -503,14 +469,12 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mockdatasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={"k": "v"},datasource_info={"info": "x"},variable_pool=variable_pool,datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,+ credentials={},))@@ -554,14 +518,12 @@ def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture,datasource_type="online_document",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={},datasource_info={},variable_pool=mocker.Mock(),datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,+ credentials={},))@@ -599,14 +561,12 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(mocdatasource_type="online_drive",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={},datasource_info={"k": "v"},variable_pool=variable_pool,datasource_param=None,online_drive_request=types.SimpleNamespace(id="id", bucket="b"),+ credentials={},))@@ -644,14 +604,12 @@ def test_stream_node_events_skips_file_build_for_non_online_types(mocker: Mockerdatasource_type="website_crawl",provider_id="p/x",tenant_id="t1",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={},datasource_info={},variable_pool=mocker.Mock(),datasource_param=None,online_drive_request=None,+ credentials={},))diff --git a/api/tests/unit_tests/core/datasource/test_notion_provider.py b/api/tests/unit_tests/core/datasource/test_notion_provider.pyindex 1e4c5bbb355..d1e232f8f57 100644--- a/api/tests/unit_tests/core/datasource/test_notion_provider.py+++ b/api/tests/unit_tests/core/datasource/test_notion_provider.py@@ -35,6 +35,8 @@ from core.rag.models.document import Documentfrom models.base import TypeBasefrom models.dataset import Document as DocumentModelfrom models.enums import DataSourceType, DocumentCreatedFrom+from services.data_source.credential_gateway import DatasourceProviderCredentialStore+from services.data_source.provider_service import DatasourceProviderService@dataclass(frozen=True)@@ -102,6 +104,7 @@ class TestNotionExtractorAuthentication:tenant_id="tenant-789",notion_access_token="explicit-token-abc",document_model=mock_document_model,+ notion_token_loader=lambda: "token",)# Assert@@ -110,13 +113,11 @@ class TestNotionExtractorAuthentication:assert extractor._notion_obj_id == "page-456"assert extractor._notion_page_type == "page"- @patch("core.rag.extractor.notion_extractor.DatasourceProviderService")- def test_init_with_credential_id(self, mock_service_class, mock_document_model):+ def test_init_with_credential_id(self, mock_document_model):"""Test NotionExtractor initialization with credential ID retrieval."""# Arrangemock_service = Mock()- mock_service.get_datasource_credentials.return_value = {"integration_secret": "credential-token-xyz"}- mock_service_class.return_value = mock_service+ mock_service.get_stored_notion_access_token.return_value = "credential-token-xyz"# Actextractor = NotionExtractor(@@ -126,23 +127,25 @@ class TestNotionExtractorAuthentication:tenant_id="tenant-789",credential_id="cred-123",document_model=mock_document_model,+ notion_token_loader=lambda: mock_service.get_stored_notion_access_token(+ tenant_id="tenant-789", credential_id="cred-123"+ ),)# Assertassert extractor._notion_access_token == "credential-token-xyz"- mock_service.get_datasource_credentials.assert_called_once_with(+ mock_service.get_stored_notion_access_token.assert_called_once_with(tenant_id="tenant-789",credential_id="cred-123",- provider="notion_datasource",- plugin_id="langgenius/notion_datasource",)- @patch("core.rag.extractor.notion_extractor.NotionExtractor._get_access_token")+ @patch("services.data_source.provider_service.DatasourceProviderService.get_datasource_credentials")def test_init_with_integration_token_fallback(self, mock_get_token, mock_document_model, config_overrides):"""Test NotionExtractor falls back to integration token when credential not found."""# Arrangemock_get_token.side_effect = Exception("No credential id found")config_overrides(NOTION_INTEGRATION_TOKEN="integration-token-fallback")+ providers = DatasourceProviderService(credentials=Mock(spec=DatasourceProviderCredentialStore))# Actextractor = NotionExtractor(@@ -152,17 +155,21 @@ class TestNotionExtractorAuthentication:tenant_id="tenant-789",credential_id=None,document_model=mock_document_model,+ notion_token_loader=lambda: providers.get_stored_notion_access_token(+ tenant_id="tenant-789", credential_id=None+ ),)# Assertassert extractor._notion_access_token == "integration-token-fallback"- @patch("core.rag.extractor.notion_extractor.NotionExtractor._get_access_token")+ @patch("services.data_source.provider_service.DatasourceProviderService.get_datasource_credentials")def test_init_missing_credentials_raises_error(self, mock_get_token, mock_document_model, config_overrides):"""Test NotionExtractor raises error when no credentials available."""# Arrangemock_get_token.side_effect = Exception("No credential id found")config_overrides(NOTION_INTEGRATION_TOKEN=None)+ providers = DatasourceProviderService(credentials=Mock(spec=DatasourceProviderCredentialStore))# Act & Assertwith pytest.raises(ValueError) as exc_info:@@ -173,6 +180,9 @@ class TestNotionExtractorAuthentication:tenant_id="tenant-789",credential_id=None,document_model=mock_document_model,+ notion_token_loader=lambda: providers.get_stored_notion_access_token(+ tenant_id="tenant-789", credential_id=None+ ),)assert "Must specify `integration_token`" in str(exc_info.value)@@ -196,6 +206,7 @@ class TestNotionExtractorPageRetrieval:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)def _create_mock_response(self, data: dict[str, Any], status_code: int = 200) -> Mock:@@ -396,6 +407,7 @@ class TestNotionExtractorDatabaseRetrieval:notion_page_type="database",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)def _create_database_page(self, page_id: str, properties: dict[str, Any]) -> dict[str, Any]:@@ -600,6 +612,7 @@ class TestNotionExtractorTableParsing:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)@patch("httpx.request")@@ -751,6 +764,7 @@ class TestNotionExtractorLastEditedTime:tenant_id="tenant-789",notion_access_token="test-token",document_model=mock_document_model,+ notion_token_loader=lambda: "token",)@pytest.fixture@@ -763,6 +777,7 @@ class TestNotionExtractorLastEditedTime:tenant_id="tenant-789",notion_access_token="test-token",document_model=mock_document_model,+ notion_token_loader=lambda: "token",)@patch("httpx.request")@@ -869,6 +884,7 @@ class TestNotionExtractorIntegration:tenant_id="tenant-789",notion_access_token="test-token",document_model=persisted_document,+ notion_token_loader=lambda: "token",)# Mock last edited time request@@ -940,6 +956,7 @@ class TestNotionExtractorIntegration:tenant_id="tenant-789",notion_access_token="test-token",document_model=persisted_document,+ notion_token_loader=lambda: "token",)# Mock last edited time request@@ -990,6 +1007,7 @@ class TestNotionExtractorIntegration:notion_page_type="invalid_type",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)# Act & Assert@@ -1016,6 +1034,7 @@ class TestNotionExtractorReadBlock:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)@patch("httpx.request")@@ -1172,6 +1191,7 @@ class TestNotionExtractorAdvancedBlockTypes:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)def _create_block_with_rich_text(@@ -1343,6 +1363,7 @@ class TestNotionExtractorDatabaseAdvanced:notion_page_type="database",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)def _create_database_page_with_properties(self, page_id: str, properties: dict[str, Any]) -> dict[str, Any]:@@ -1535,6 +1556,7 @@ class TestNotionExtractorErrorScenarios:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)@pytest.mark.parametrize(@@ -1680,6 +1702,7 @@ class TestNotionExtractorTableAdvanced:notion_page_type="page",tenant_id="tenant-789",notion_access_token="test-token",+ notion_token_loader=lambda: "token",)@patch("httpx.request")diff --git a/api/tests/unit_tests/core/datasource/test_stream_node_events.py b/api/tests/unit_tests/core/datasource/test_stream_node_events.pyindex 888c1a8d9ee..c05f9092a25 100644--- a/api/tests/unit_tests/core/datasource/test_stream_node_events.py+++ b/api/tests/unit_tests/core/datasource/test_stream_node_events.py@@ -33,14 +33,12 @@ def test_stream_node_events_accumulates_variables(mocker: MockerFixture) -> Nonedatasource_type="online_document",provider_id="p/x",tenant_id="t",- provider="prov",- plugin_id="plug",- credential_id="",parameters_for_log={},datasource_info={"user_id": "u"},variable_pool=mocker.Mock(),datasource_param=type("P", (), {"workspace_id": "w", "page_id": "pg", "type": "t"})(),online_drive_request=None,+ credentials={},))assert isinstance(events[-1], StreamCompletedEvent)diff --git a/api/tests/unit_tests/core/datasource/test_website_crawl.py b/api/tests/unit_tests/core/datasource/test_website_crawl.pyindex 53000881ddc..d589acb747b 100644--- a/api/tests/unit_tests/core/datasource/test_website_crawl.py+++ b/api/tests/unit_tests/core/datasource/test_website_crawl.py@@ -12,11 +12,12 @@ The tests cover multiple crawl providers (Firecrawl, WaterCrawl, JinaReader)and ensure proper handling of crawl options, status checking, and data retrieval."""-from unittest.mock import Mock, patch+from unittest.mock import Mock, create_autospec, patchimport pytestfrom pytest_mock import MockerFixture+import services.data_source.website_service as website_service_modulefrom core.datasource.entities.datasource_entities import (DatasourceEntity,DatasourceIdentity,@@ -27,7 +28,8 @@ from core.datasource.entities.datasource_entities import (from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePluginfrom core.datasource.website_crawl.website_crawl_provider import WebsiteCrawlDatasourcePluginProviderControllerfrom core.rag.extractor.watercrawl.provider import WaterCrawlProvider-from services.website_service import CrawlOptions, CrawlRequest, WebsiteService+from services.data_source.credential_gateway import DatasourceProviderCredentialStore+from services.data_source.website_service import CrawlOptions, CrawlRequest, WebsiteService# ============================================================================# Fixtures@@ -489,8 +491,8 @@ class TestCrawlStatus:class TestWebsiteService:"""Test suite for WebsiteService with multiple providers."""- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_crawl_url_firecrawl(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):"""Test crawling with Firecrawl provider."""# Setup mocks@@ -500,14 +502,14 @@ class TestWebsiteService:"base_url": "https://api.firecrawl.dev",}- mock_firecrawl = mocker.patch("services.website_service.FirecrawlApp")+ mock_firecrawl = mocker.patch("services.data_source.website_service.FirecrawlApp")mock_firecrawl_instance = mock_firecrawl.return_valuemock_firecrawl_instance.crawl_url.return_value = "job-123"# Mock redis- mocker.patch("services.website_service.redis_client")+ mocker.patch("services.data_source.website_service.redis_client")- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequestapi_request = WebsiteCrawlApiRequest(provider="firecrawl",@@ -515,13 +517,17 @@ class TestWebsiteService:options={"limit": 10, "crawl_sub_pages": True, "only_main_content": True},)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "job-123"- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_crawl_url_watercrawl(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):"""Test crawling with WaterCrawl provider."""# Setup mocks@@ -531,11 +537,11 @@ class TestWebsiteService:"base_url": "https://app.watercrawl.dev",}- mock_watercrawl = mocker.patch("services.website_service.WaterCrawlProvider")+ mock_watercrawl = mocker.patch("services.data_source.website_service.WaterCrawlProvider")mock_watercrawl_instance = mock_watercrawl.return_valuemock_watercrawl_instance.crawl_url.return_value = {"status": "active", "job_id": "job-456"}- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequestapi_request = WebsiteCrawlApiRequest(provider="watercrawl",@@ -543,13 +549,17 @@ class TestWebsiteService:options={"limit": 20, "crawl_sub_pages": True, "max_depth": 2},)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "job-456"- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_crawl_url_jinareader(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):"""Test crawling with JinaReader provider."""# Setup mocks@@ -561,11 +571,11 @@ class TestWebsiteService:mock_response = Mock()mock_response.json.return_value = {"code": 200, "data": {"taskId": "task-789"}}mock_httpx_post = mocker.patch(- "services.website_service._adaptive_http_client.post",+ "services.data_source.website_service._adaptive_http_client.post",return_value=mock_response,)- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequestapi_request = WebsiteCrawlApiRequest(provider="jinareader",@@ -573,7 +583,11 @@ class TestWebsiteService:options={"limit": 15, "crawl_sub_pages": True, "use_sitemap": True},)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "task-789"@@ -855,11 +869,11 @@ class TestContentExtraction:class TestErrorHandling:"""Test suite for error handling in crawl operations."""- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_invalid_provider_error(self, mock_provider_service: Mock, mock_current_user: Mock):"""Test that invalid provider raises ValueError."""- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Setup mocksmock_current_user.current_tenant_id = "test_tenant"@@ -873,7 +887,11 @@ class TestErrorHandling:# The error should be raised when trying to crawl with invalid providerwith pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.crawl_url(api_request)+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)def test_missing_api_key_error(self, mocker: MockerFixture):"""Test that missing API key is handled properly at the httpx client level."""@@ -1267,8 +1285,8 @@ class TestProviderSpecificFeatures:unique features and API behaviors that require specific testing."""- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_firecrawl_with_prompt_parameter(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):@@ -1285,14 +1303,14 @@ class TestProviderSpecificFeatures:"base_url": "https://api.firecrawl.dev",}- mock_firecrawl = mocker.patch("services.website_service.FirecrawlApp")+ mock_firecrawl = mocker.patch("services.data_source.website_service.FirecrawlApp")mock_firecrawl_instance = mock_firecrawl.return_valuemock_firecrawl_instance.crawl_url.return_value = "prompt-job-123"# Mock redis- mocker.patch("services.website_service.redis_client")+ mocker.patch("services.data_source.website_service.redis_client")- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Include a prompt for AI-guided extractionapi_request = WebsiteCrawlApiRequest(@@ -1306,7 +1324,11 @@ class TestProviderSpecificFeatures:},)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "prompt-job-123"@@ -1317,8 +1339,8 @@ class TestProviderSpecificFeatures:assert "prompt" in paramsassert params["prompt"] == "Extract only technical documentation and API references"- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_jinareader_single_page_mode(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):@@ -1343,23 +1365,27 @@ class TestProviderSpecificFeatures:"url": "https://example.com/page",},}- mocker.patch("services.website_service._jina_http_client.get", return_value=mock_response)+ mocker.patch("services.data_source.website_service._jina_http_client.get", return_value=mock_response)- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Single page mode (crawl_sub_pages = False)api_request = WebsiteCrawlApiRequest(provider="jinareader", url="https://example.com/page", options={"crawl_sub_pages": False, "limit": 1})- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)# In single-page mode, JinaReader returns data immediatelyassert result["status"] == "active"assert "data" in result- @patch("services.website_service.current_user")- @patch("services.website_service.DatasourceProviderService")+ @patch("services.data_source.website_service.current_user")+ @patch("services.data_source.website_service.DatasourceProviderService")def test_watercrawl_with_tag_filtering(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):@@ -1376,11 +1402,11 @@ class TestProviderSpecificFeatures:"base_url": "https://app.watercrawl.dev",}- mock_watercrawl = mocker.patch("services.website_service.WaterCrawlProvider")+ mock_watercrawl = mocker.patch("services.data_source.website_service.WaterCrawlProvider")mock_watercrawl_instance = mock_watercrawl.return_valuemock_watercrawl_instance.crawl_url.return_value = {"status": "active", "job_id": "tag-filter-job"}- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Configure with tag filteringapi_request = WebsiteCrawlApiRequest(@@ -1394,7 +1420,11 @@ class TestProviderSpecificFeatures:},)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "tag-filter-job"@@ -1451,7 +1481,7 @@ class TestDataStructureValidation:This conversion ensures that external API parameters are properlymapped to internal data structures."""- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Create API request with all optionsapi_request = WebsiteCrawlApiRequest(@@ -1634,7 +1664,7 @@ class TestEdgeCasesAndBoundaries:Empty or invalid URLs should fail validation before attemptingto crawl."""- from services.website_service import WebsiteCrawlApiRequest+ from services.data_source.website_service import WebsiteCrawlApiRequest# Empty URL should raise ValueError during validationwith pytest.raises(ValueError, match="URL is required"):diff --git a/api/tests/unit_tests/core/rag/datasource/keyword/jieba/test_jieba.py b/api/tests/unit_tests/core/rag/datasource/keyword/jieba/test_jieba.pyindex 2cfbfd5db68..41edd48db92 100644--- a/api/tests/unit_tests/core/rag/datasource/keyword/jieba/test_jieba.py+++ b/api/tests/unit_tests/core/rag/datasource/keyword/jieba/test_jieba.py@@ -1,407 +1,275 @@-import json-from types import SimpleNamespace-from typing import Any-from unittest.mock import MagicMock+from collections.abc import Generator+from contextlib import contextmanager+from dataclasses import dataclass, fieldimport pytest-from sqlalchemy import event, select+from sqlalchemy import selectfrom sqlalchemy.orm import Session-import core.rag.datasource.keyword.jieba.jieba as jieba_module-from core.rag.datasource.keyword.jieba.jieba import Jieba, dumps_with_sets, set_orjson_default+from core.rag.datasource.keyword.jieba import jieba as jieba_module+from core.rag.datasource.keyword.jieba.jieba import Jiebafrom core.rag.models.document import Documentfrom models.dataset import Dataset, DatasetKeywordTable, DocumentSegment+from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table+from repositories.knowledge.keyword_table_repository import load_keyword_table+from tests.unit_tests.config_override import apply_config_overridesfrom tests.unit_tests.model_factories import make_dataset-class _DummyLock:- def __enter__(self):- return self-- def __exit__(self, exc_type, exc, tb):- return False---def _dataset_keyword_table(- data_source_type: str = "database", keyword_table_dict: dict[str, Any] | None = None-) -> DatasetKeywordTable:- keyword_table = DatasetKeywordTable(- dataset_id="dataset-1",- data_source_type=data_source_type,- keyword_table="",- )- keyword_table.get_keyword_table_dict = MagicMock(return_value=keyword_table_dict)- return keyword_table-+@dataclass+class KeywordStorage:+ files: dict[str, bytes] = field(default_factory=dict)+ reads: list[str] = field(default_factory=list)+ writes: list[str] = field(default_factory=list)++ def load_once(self, key: str) -> bytes:+ self.reads.append(key)+ if key not in self.files:+ raise FileNotFoundError(key)+ return self.files[key]++ def exists(self, key: str) -> bool:+ return key in self.files++ def save(self, key: str, data: bytes) -> None:+ self.files[key] = data+ self.writes.append(key)++ def delete(self, key: str) -> None:+ del self.files[key]+++@dataclass+class KeywordLocks:+ acquired: list[str] = field(default_factory=list)+ held: bool = False++ @contextmanager+ def lock(self, name: str, *, timeout: int) -> Generator[None]:+ assert timeout == 600+ assert not self.held+ self.acquired.append(name)+ self.held = True+ try:+ yield+ finally:+ self.held = False+++@dataclass+class KeywordRuntime:+ session: Session+ dataset: Dataset+ storage: KeywordStorage+ locks: KeywordLocks+ extracted: list[tuple[str, int]]++ def table(self) -> dict[str, set[str]]:+ row = get_dataset_keyword_table(self.dataset, session=self.session)+ assert row is not None+ payload = load_keyword_table(+ storage=self.storage,+ tenant_id=self.dataset.tenant_id,+ dataset_id=self.dataset.id,+ storage_type=row.data_source_type,+ data=row.keyword_table,+ )+ assert payload is not None+ return payload["__data__"]["table"]-def _dataset(dataset_keyword_table: DatasetKeywordTable | None = None, keyword_number: int | None = None) -> Dataset:- dataset = make_dataset(keyword_number=keyword_number)- dataset.get_dataset_keyword_table = MagicMock(return_value=dataset_keyword_table)- return dataset[email protected](params=["database", "file"])+def runtime(request: pytest.FixtureRequest, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> KeywordRuntime:+ storage = KeywordStorage()+ locks = KeywordLocks()+ extracted: list[tuple[str, int]] = []-def patched_runtime(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session):- storage = MagicMock()- lock = MagicMock(return_value=_DummyLock())- redis_client = SimpleNamespace(lock=lock)+ class KeywordExtractor:+ def extract_keywords(self, text: str, keyword_number: int = 10) -> set[str]:+ extracted.append((text, keyword_number))+ return set(text.split()[:keyword_number])+ monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", KeywordExtractor)+ monkeypatch.setattr(jieba_module, "redis_client", locks)monkeypatch.setattr(jieba_module, "storage", storage)- monkeypatch.setattr(jieba_module, "redis_client", redis_client)-- return SimpleNamespace(session=sqlite_session, storage=storage, lock=lock)+ apply_config_overrides(monkeypatch, KEYWORD_DATA_SOURCE_TYPE=request.param)+ dataset = make_dataset(name="Test", created_by="author", keyword_number=2)+ sqlite_session.add(dataset)+ sqlite_session.commit()+ return KeywordRuntime(sqlite_session, dataset, storage, locks, extracted)-def _segment(*, index_node_id: str = "node-2") -> DocumentSegment:- segment = DocumentSegment(- tenant_id="tenant-1",+def segment(node_id: str, *, tenant_id: str = "tenant-1") -> DocumentSegment:+ return DocumentSegment(+ tenant_id=tenant_id,dataset_id="dataset-1",- document_id="doc-2",+ document_id="document-1",position=1,- content="segment-content",- word_count=1,- tokens=1,- created_by="user-1",- enabled=True,- keywords=[],- answer=None,- index_node_id=index_node_id,- index_node_hash="hash-2",- status="completed",+ content=node_id,+ word_count=len(node_id),+ tokens=0,+ created_by="author",+ index_node_id=node_id,)- segment.id = "segment-1"- return segment--def test_create_indexes_documents_and_returns_self(monkeypatch: pytest.MonkeyPatch, patched_runtime):- dataset = _dataset(_dataset_keyword_table(), keyword_number=2)- keyword = Jieba(dataset)- handler = MagicMock()- handler.extract_keywords.return_value = {"kw1", "kw2"}-- monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", lambda: handler)- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={}))- monkeypatch.setattr(keyword, "_update_segment_keywords", MagicMock())- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())+def test_create_extracts_keywords_and_persists_the_shared_format(runtime: KeywordRuntime) -> None:+ chunk = segment("node-1")+ foreign = segment("node-1", tenant_id="tenant-2")+ runtime.session.add_all([chunk, foreign])+ keyword = Jieba(runtime.dataset)result = keyword.create(- [- Document(page_content="alpha", metadata={"doc_id": "node-1"}),- SimpleNamespace(page_content="ignored", metadata=None),- ],- patched_runtime.session,+ [Document(page_content="中文 alpha ignored", metadata={"doc_id": "node-1"})], runtime.session)+ runtime.session.commit()assert result is keyword- keyword._update_segment_keywords.assert_called_once()- call_args = keyword._update_segment_keywords.call_args.args- assert call_args[0] == "dataset-1"- assert call_args[1] == "node-1"- assert set(call_args[2]) == {"kw1", "kw2"}- assert call_args[3] is patched_runtime.session- saved_table = keyword._save_dataset_keyword_table.call_args.args[0]- assert saved_table["kw1"] == {"node-1"}- assert saved_table["kw2"] == {"node-1"}- patched_runtime.lock.assert_called_once_with("keyword_indexing_lock_dataset-1", timeout=600)---def test_add_texts_supports_keywords_list_and_extract_fallback(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table(), keyword_number=3))- handler = MagicMock()- handler.extract_keywords.return_value = {"auto"}-- monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", lambda: handler)- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={}))- monkeypatch.setattr(keyword, "_update_segment_keywords", MagicMock())- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- texts = [- Document(page_content="extract-this", metadata={"doc_id": "node-1"}),- Document(page_content="use-manual", metadata={"doc_id": "node-2"}),- ]- keyword.add_texts(texts, patched_runtime.session, keywords_list=[[], ["manual"]])-- assert keyword._update_segment_keywords.call_count == 2- first_call = keyword._update_segment_keywords.call_args_list[0].args- second_call = keyword._update_segment_keywords.call_args_list[1].args- assert set(first_call[2]) == {"auto"}- assert second_call[2] == ["manual"]- assert first_call[3] is patched_runtime.session- assert second_call[3] is patched_runtime.session- keyword._save_dataset_keyword_table.assert_called_once_with(- {"auto": {"node-1"}, "manual": {"node-2"}}, patched_runtime.session- )---def test_add_texts_without_keywords_list_always_uses_extractor(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table(), keyword_number=1))- handler = MagicMock()- handler.extract_keywords.return_value = {"from-extractor"}-- monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", lambda: handler)- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={}))- monkeypatch.setattr(keyword, "_update_segment_keywords", MagicMock())- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- keyword.add_texts([Document(page_content="content", metadata={"doc_id": "node-1"})], patched_runtime.session)-- handler.extract_keywords.assert_called_once_with("content", 1)- assert set(keyword._update_segment_keywords.call_args.args[2]) == {"from-extractor"}- assert keyword._update_segment_keywords.call_args.args[3] is patched_runtime.session---def test_text_exists_handles_missing_and_existing_keyword_table(- monkeypatch: pytest.MonkeyPatch, unbound_session: Session-):- keyword = Jieba(_dataset(_dataset_keyword_table(keyword_table_dict=None)))- session = unbound_session- assert keyword.text_exists("node-1", session=session) is False-- keyword = Jieba(- _dataset(- _dataset_keyword_table(- keyword_table_dict={"__type__": "keyword_table", "__data__": {"table": {"k": {"node-1", "node-2"}}}}- )- )+ assert set(chunk.keywords) == {"中文", "alpha"}+ assert foreign.keywords is None+ assert runtime.table() == {"中文": {"node-1"}, "alpha": {"node-1"}}+ assert runtime.extracted == [("中文 alpha ignored", 2)]+ assert runtime.locks.acquired == ["keyword_indexing_lock_dataset-1"]+ assert runtime.locks.held is False+ row = get_dataset_keyword_table(runtime.dataset, session=runtime.session)+ assert row is not None+ payload = load_keyword_table(+ storage=runtime.storage,+ tenant_id=runtime.dataset.tenant_id,+ dataset_id=runtime.dataset.id,+ storage_type=row.data_source_type,+ data=row.keyword_table,)- assert keyword.text_exists("node-2", session=session) is True- assert keyword.text_exists("node-x", session=session) is False---def test_delete_by_ids_updates_table_when_present(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(- _dataset(- _dataset_keyword_table(- keyword_table_dict={"__type__": "keyword_table", "__data__": {"table": {"k": {"node-1", "node-2"}}}}- )- )- )- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={"k": {"node-1", "node-2"}}))- monkeypatch.setattr(keyword, "_delete_ids_from_keyword_table", MagicMock(return_value={"k": {"node-2"}}))- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- keyword.delete_by_ids(["node-1"], patched_runtime.session)-- keyword._get_dataset_keyword_table.assert_called_once_with(patched_runtime.session)- keyword._delete_ids_from_keyword_table.assert_called_once_with({"k": {"node-1", "node-2"}}, ["node-1"])- keyword._save_dataset_keyword_table.assert_called_once_with({"k": {"node-2"}}, patched_runtime.session)---def test_delete_by_ids_saves_none_when_keyword_table_is_missing(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table()))- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value=None))- monkeypatch.setattr(keyword, "_delete_ids_from_keyword_table", MagicMock())- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- keyword.delete_by_ids(["node-1"], patched_runtime.session)-- keyword._get_dataset_keyword_table.assert_called_once_with(patched_runtime.session)- keyword._delete_ids_from_keyword_table.assert_not_called()- keyword._save_dataset_keyword_table.assert_called_once_with(None, patched_runtime.session)---def test_search_returns_documents_in_rank_order_and_applies_filter(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table()))- patched_runtime.session.add(_segment())- patched_runtime.session.flush()- monkeypatch.setattr(keyword, "_retrieve_ids_by_query", MagicMock(return_value=["node-1", "node-2"]))-- documents = keyword.search("query", session=patched_runtime.session, top_k=2, document_ids_filter=["doc-2"])-- assert len(documents) == 1- assert documents[0].page_content == "segment-content"- assert documents[0].metadata["doc_id"] == "node-2"- assert documents[0].metadata["doc_hash"] == "hash-2"---def test_delete_removes_keyword_table_and_optional_file(patched_runtime):- db_keyword = DatasetKeywordTable(dataset_id="dataset-1", keyword_table="", data_source_type="database")- patched_runtime.session.add(db_keyword)- patched_runtime.session.commit()- commits: list[str] = []- event.listen(patched_runtime.session, "after_commit", lambda _session: commits.append("commit"))-- keyword_db = Jieba(_dataset(db_keyword))- keyword_db.delete(session=patched_runtime.session)- patched_runtime.storage.delete.assert_not_called()- assert patched_runtime.session.get(DatasetKeywordTable, db_keyword.id) is None-- file_keyword = DatasetKeywordTable(dataset_id="dataset-1", keyword_table="", data_source_type="object_storage")- patched_runtime.session.add(file_keyword)- patched_runtime.session.commit()- keyword_file = Jieba(_dataset(file_keyword))- keyword_file.delete(session=patched_runtime.session)-- patched_runtime.storage.delete.assert_called_once_with("keyword_files/tenant-1/dataset-1.txt")- assert patched_runtime.session.get(DatasetKeywordTable, file_keyword.id) is None- assert commits == ["commit", "commit", "commit"]---def test_save_dataset_keyword_table_to_database(patched_runtime):- dataset_keyword_table = DatasetKeywordTable(dataset_id="dataset-1", keyword_table="", data_source_type="database")- patched_runtime.session.add(dataset_keyword_table)- patched_runtime.session.flush()- keyword = Jieba(_dataset(dataset_keyword_table))-- keyword._save_dataset_keyword_table({"kw": {"node-1"}}, patched_runtime.session)-- assert '"__type__":"keyword_table"' in dataset_keyword_table.keyword_table- assert '"index_id":"dataset-1"' in dataset_keyword_table.keyword_table---def test_save_dataset_keyword_table_to_file_storage(patched_runtime):- dataset_keyword_table = DatasetKeywordTable(dataset_id="dataset-1", keyword_table="", data_source_type="file")- patched_runtime.session.add(dataset_keyword_table)- patched_runtime.session.flush()- keyword = Jieba(_dataset(dataset_keyword_table))- patched_runtime.storage.exists.return_value = True-- keyword._save_dataset_keyword_table({"kw": {"node-1"}}, patched_runtime.session)-- patched_runtime.storage.delete.assert_called_once_with("keyword_files/tenant-1/dataset-1.txt")- patched_runtime.storage.save.assert_called_once()- save_args = patched_runtime.storage.save.call_args.args- assert save_args[0] == "keyword_files/tenant-1/dataset-1.txt"- assert isinstance(save_args[1], bytes)---def test_get_dataset_keyword_table_returns_existing_table_data(patched_runtime):- existing = DatasetKeywordTable(- dataset_id="dataset-1",- keyword_table="",- data_source_type="database",- )- existing.get_keyword_table_dict = MagicMock(- return_value={"__type__": "keyword_table", "__data__": {"table": {"kw": ["node-1"]}}}- )- patched_runtime.session.add(existing)- patched_runtime.session.flush()- keyword = Jieba(_dataset(existing))- assert keyword._get_dataset_keyword_table(patched_runtime.session) == {"kw": ["node-1"]}-- existing.get_keyword_table_dict = MagicMock(return_value=None)- keyword_with_missing_payload = Jieba(_dataset(existing))- assert keyword_with_missing_payload._get_dataset_keyword_table(patched_runtime.session) == {}---def test_get_dataset_keyword_table_creates_table_when_missing(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(dataset_keyword_table=None))- from tests.unit_tests.config_override import apply_config_overrides-- apply_config_overrides(monkeypatch, KEYWORD_DATA_SOURCE_TYPE="database")- result = keyword._get_dataset_keyword_table(patched_runtime.session)-- assert result == {}- created_table = patched_runtime.session.scalar(- select(DatasetKeywordTable).where(DatasetKeywordTable.dataset_id == "dataset-1")+ assert payload is not None+ assert payload["__type__"] == "keyword_table"+ assert payload["__data__"]["index_id"] == "dataset-1"+ if row.data_source_type == "file":+ assert row.keyword_table == ""+ assert runtime.storage.writes == ["keyword_files/tenant-1/dataset-1.txt"]+ else:+ assert runtime.storage.reads == []+ assert runtime.storage.writes == []+++def test_add_uses_manual_keywords_and_extracts_empty_selections(runtime: KeywordRuntime) -> None:+ chunks = [segment("node-1"), segment("node-2")]+ runtime.session.add_all(chunks)+ keyword = Jieba(runtime.dataset)+ keyword.add_texts(+ [+ Document(page_content="automatic", metadata={"doc_id": "node-1"}),+ Document(page_content="ignored", metadata={"doc_id": "node-2"}),+ ],+ runtime.session,+ keywords_list=[[], ["manual"]],)- assert created_table is not None- assert created_table.dataset_id == "dataset-1"- assert created_table.data_source_type == "database"- assert '"index_id":"dataset-1"' in created_table.keyword_table---def test_add_and_delete_ids_from_keyword_table_helpers():- keyword = Jieba(_dataset(_dataset_keyword_table()))- keyword_table = {"kw1": {"node-1"}, "kw2": {"node-1", "node-2"}}-- updated = keyword._add_text_to_keyword_table(keyword_table, "node-3", ["kw1", "kw3"])- assert updated["kw1"] == {"node-1", "node-3"}- assert updated["kw3"] == {"node-3"}-- deleted = keyword._delete_ids_from_keyword_table(updated, ["node-1", "node-3"])- assert "kw3" not in deleted- assert "kw1" not in deleted- assert deleted["kw2"] == {"node-2"}-+ runtime.session.commit()+ assert runtime.table() == {"automatic": {"node-1"}, "manual": {"node-2"}}+ assert runtime.extracted == [("automatic", 2)]+ assert chunks[0].keywords == ["automatic"]+ assert chunks[1].keywords == ["manual"]-def test_retrieve_ids_by_query_ranks_by_keyword_frequency(monkeypatch: pytest.MonkeyPatch):- keyword = Jieba(_dataset(_dataset_keyword_table()))- handler = MagicMock()- handler.extract_keywords.return_value = ["kw-a", "kw-b"]- monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", lambda: handler)- ranked_ids = keyword._retrieve_ids_by_query(- {"kw-a": {"node-1", "node-2"}, "kw-b": {"node-2"}, "kw-c": {"node-3"}},- "query",- k=1,+def test_delete_ids_preserves_unrelated_entries(runtime: KeywordRuntime) -> None:+ keyword = Jieba(runtime.dataset)+ keyword.add_texts(+ [+ Document(page_content="shared first", metadata={"doc_id": "node-1"}),+ Document(page_content="shared second", metadata={"doc_id": "node-2"}),+ ],+ runtime.session,)-- assert ranked_ids == ["node-2"]---def test_update_segment_keywords_updates_when_segment_exists(patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table()))- segment = _segment(index_node_id="node-1")- patched_runtime.session.add(segment)- patched_runtime.session.flush()-- keyword._update_segment_keywords("dataset-1", "node-1", ["kw1", "kw2"], patched_runtime.session)-- assert segment.keywords == ["kw1", "kw2"]-- keyword._update_segment_keywords("dataset-1", "node-missing", ["kw3"], patched_runtime.session)- assert segment.keywords == ["kw1", "kw2"]---def test_create_segment_keywords_and_update_segment_keywords_index(monkeypatch: pytest.MonkeyPatch, patched_runtime):- keyword = Jieba(_dataset(_dataset_keyword_table()))- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={}))- monkeypatch.setattr(keyword, "_update_segment_keywords", MagicMock())- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- keyword.create_segment_keywords("node-1", ["kw"], patched_runtime.session)- keyword._get_dataset_keyword_table.assert_called_once_with(patched_runtime.session)- keyword._update_segment_keywords.assert_called_once_with("dataset-1", "node-1", ["kw"], patched_runtime.session)- keyword._save_dataset_keyword_table.assert_called_once_with({"kw": {"node-1"}}, patched_runtime.session)-- keyword._get_dataset_keyword_table.reset_mock()- keyword._save_dataset_keyword_table.reset_mock()- keyword.update_segment_keywords_index("node-2", ["kw2"], patched_runtime.session)- keyword._get_dataset_keyword_table.assert_called_once_with(patched_runtime.session)- keyword._save_dataset_keyword_table.assert_called_once_with({"kw2": {"node-2"}}, patched_runtime.session)---def test_multi_create_segment_keywords_uses_provided_and_extracted_keywords(- monkeypatch: pytest.MonkeyPatch, patched_runtime-):- keyword = Jieba(_dataset(_dataset_keyword_table(), keyword_number=2))- handler = MagicMock()- handler.extract_keywords.return_value = {"auto"}- monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", lambda: handler)- monkeypatch.setattr(keyword, "_get_dataset_keyword_table", MagicMock(return_value={}))- monkeypatch.setattr(keyword, "_save_dataset_keyword_table", MagicMock())-- first_segment = _segment(index_node_id="node-1")- first_segment.content = "first content"- first_segment.keywords = None- second_segment = _segment(index_node_id="node-2")- second_segment.content = "second content"- second_segment.keywords = None-- keyword.multi_create_segment_keywords(+ runtime.session.commit()+ keyword.delete_by_ids(["node-1"], runtime.session)+ runtime.session.commit()+ assert runtime.table() == {"shared": {"node-2"}, "second": {"node-2"}}+ assert keyword.text_exists("node-1", session=runtime.session) is False+ assert keyword.text_exists("node-2", session=runtime.session) is True+ keyword.delete_by_ids(["node-2"], runtime.session)+ runtime.session.commit()+ assert runtime.table() == {}+ assert keyword.text_exists("node-2", session=runtime.session) is False+++def test_delete_without_an_existing_table_is_idempotent(runtime: KeywordRuntime) -> None:+ keyword = Jieba(runtime.dataset)+ assert keyword.text_exists("missing", session=runtime.session) is False+ keyword.delete_by_ids(["missing"], runtime.session)+ runtime.session.commit()+ keyword.delete_by_ids(["missing"], runtime.session)+ runtime.session.commit()+ assert runtime.table() == {}+++def test_search_preserves_ranking_and_document_filters(runtime: KeywordRuntime) -> None:+ first, second = segment("node-1"), segment("node-2")+ second.document_id = "document-2"+ runtime.session.add_all([first, second])+ keyword = Jieba(runtime.dataset)+ keyword.add_texts([- {"segment": first_segment, "keywords": ["manual"]},- {"segment": second_segment, "keywords": []},+ Document(page_content="alpha", metadata={"doc_id": "node-1"}),+ Document(page_content="alpha beta", metadata={"doc_id": "node-2"}),],- patched_runtime.session,+ runtime.session,)-- assert first_segment.keywords == ["manual"]- assert second_segment.keywords == ["auto"]- saved_table = keyword._save_dataset_keyword_table.call_args.args[0]- assert saved_table["manual"] == {"node-1"}- assert saved_table["auto"] == {"node-2"}- assert keyword._save_dataset_keyword_table.call_args.args[1] is patched_runtime.session---def test_set_orjson_default_and_dumps_with_sets():- assert set(set_orjson_default({"a", "b"})) == {"a", "b"}-- with pytest.raises(TypeError, match="is not JSON serializable"):- set_orjson_default(("not", "a", "set"))-- payload = {"items": {"a", "b"}}- json_payload = dumps_with_sets(payload)- decoded = json.loads(json_payload)- assert set(decoded["items"]) == {"a", "b"}+ runtime.session.commit()+ documents = keyword.search("alpha beta", session=runtime.session, top_k=2)+ assert [doc.metadata["doc_id"] for doc in documents] == ["node-2", "node-1"]+ documents = keyword.search("alpha beta", session=runtime.session, top_k=2, document_ids_filter=["document-1"])+ assert [doc.metadata["doc_id"] for doc in documents] == ["node-1"]+++def test_delete_removes_the_database_record_and_file(runtime: KeywordRuntime) -> None:+ keyword = Jieba(runtime.dataset)+ keyword.create([Document(page_content="alpha", metadata={"doc_id": "node-1"})], runtime.session)+ runtime.session.commit()+ keyword.delete(session=runtime.session)+ assert runtime.session.scalar(select(DatasetKeywordTable)) is None+ assert runtime.storage.files == {}+++def test_mutation_does_not_replace_an_unreadable_table(+ runtime: KeywordRuntime, monkeypatch: pytest.MonkeyPatch+) -> None:+ keyword = Jieba(runtime.dataset)+ keyword.create([Document(page_content="original", metadata={"doc_id": "node-1"})], runtime.session)+ runtime.session.commit()+ row = get_dataset_keyword_table(runtime.dataset, session=runtime.session)+ assert row is not None+ stored_files = dict(runtime.storage.files)+ writes = list(runtime.storage.writes)+ if row.data_source_type == "database":+ row.keyword_table = "invalid json"+ runtime.session.commit()+ expected_error = ValueError+ else:++ def fail_read(_key: str) -> bytes:+ raise OSError("storage unavailable")++ monkeypatch.setattr(runtime.storage, "load_once", fail_read)+ expected_error = OSError+ with pytest.raises(expected_error):+ keyword.add_texts([Document(page_content="replacement", metadata={"doc_id": "node-1"})], runtime.session)+ runtime.session.rollback()+ assert runtime.storage.files == stored_files+ assert runtime.storage.writes == writes+ assert runtime.locks.held is False+ if row.data_source_type == "database":+ assert row.keyword_table == "invalid json"+++def test_deletion_does_not_initialize_the_keyword_extractor(+ runtime: KeywordRuntime, monkeypatch: pytest.MonkeyPatch+) -> None:+ keyword = Jieba(runtime.dataset)+ keyword.create([Document(page_content="original", metadata={"doc_id": "node-1"})], runtime.session)+ runtime.session.commit()++ def unexpected_extractor() -> None:+ pytest.fail("Index cleanup must not initialize the text extractor")++ monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", unexpected_extractor)+ keyword.delete_by_ids(["node-1"], runtime.session)+ runtime.session.commit()+ assert runtime.table() == {}diff --git a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.pyindex 4c78edf6615..4d81e4424ed 100644--- a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py+++ b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py@@ -899,7 +899,7 @@ class TestRetrievalServiceInternals:monkeypatch.setattr(RetrievalService,"get_segment_attachment_infos",- lambda attachment_ids, session: [+ lambda attachment_ids, *, dataset_refs, session: [{"attachment_id": "attach-node-1","attachment_info": {@@ -1172,7 +1172,9 @@ class TestRetrievalServiceInternals:assert result is Nonedef test_get_segment_attachment_infos_returns_empty_when_upload_files_missing(self, sqlite_session: Session):- result = RetrievalService.get_segment_attachment_infos(["upload-1"], sqlite_session)+ result = RetrievalService.get_segment_attachment_infos(+ ["upload-1"], dataset_refs=(("tenant-id", "dataset-id"),), session=sqlite_session+ )assert result == []@@ -1180,7 +1182,9 @@ class TestRetrievalServiceInternals:sqlite_session.add(_upload_file(file_id="upload-1", name="file-name"))sqlite_session.commit()- result = RetrievalService.get_segment_attachment_infos(["upload-1"], sqlite_session)+ result = RetrievalService.get_segment_attachment_infos(+ ["upload-1"], dataset_refs=(("tenant-id", "dataset-id"),), session=sqlite_session+ )assert result == []@@ -1200,7 +1204,9 @@ class TestRetrievalServiceInternals:sqlite_session.add_all([upload_file_1, upload_file_2, binding])sqlite_session.commit()- result = RetrievalService.get_segment_attachment_infos(["upload-1", "upload-2"], sqlite_session)+ result = RetrievalService.get_segment_attachment_infos(+ ["upload-1", "upload-2"], dataset_refs=(("tenant-id", "dataset-id"),), session=sqlite_session+ )assert result == [{diff --git a/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py b/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.pyindex 6ebfbf4c053..8f89aca59d0 100644--- a/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py+++ b/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py@@ -55,6 +55,7 @@ def test_file_access_grants_ignore_empty_inputs_and_missing_scope() -> None:@pytest.mark.parametrize("sqlite_session", [(UploadFile, SegmentAttachmentBinding)], indirect=True)def test_segment_attachment_lookup_grants_returned_upload_files_to_current_scope(sqlite_session: Session) -> None:tenant_id = str(uuid4())+ dataset_id = str(uuid4())upload_file_id = str(uuid4())segment_id = str(uuid4())user_id = str(uuid4())@@ -74,7 +75,7 @@ def test_segment_attachment_lookup_grants_returned_upload_files_to_current_scopeupload_file.id = upload_file_idbinding = SegmentAttachmentBinding(tenant_id=tenant_id,- dataset_id=str(uuid4()),+ dataset_id=dataset_id,document_id=str(uuid4()),segment_id=segment_id,attachment_id=upload_file_id,@@ -89,7 +90,11 @@ def test_segment_attachment_lookup_grants_returned_upload_files_to_current_scope)with bind_file_access_scope(scope):- result = RetrievalService.get_segment_attachment_infos([upload_file_id], sqlite_session)+ result = RetrievalService.get_segment_attachment_infos(+ [upload_file_id],+ dataset_refs=((tenant_id, dataset_id),),+ session=sqlite_session,+ )scoped_stmt = DatabaseFileAccessController().apply_upload_file_filters(select(UploadFile).where(UploadFile.id == upload_file_id))@@ -100,6 +105,59 @@ def test_segment_attachment_lookup_grants_returned_upload_files_to_current_scopeassert "upload_files.id IN" in whereclause[email protected]("sqlite_session", [(UploadFile, SegmentAttachmentBinding)], indirect=True)+def test_segment_attachment_lookup_rejects_another_tenants_upload(sqlite_session: Session) -> None:+ tenant_id = str(uuid4())+ foreign_tenant_id = str(uuid4())+ upload_file_id = str(uuid4())+ foreign_dataset_id = str(uuid4())+ upload_file = UploadFile(+ tenant_id=foreign_tenant_id,+ storage_type=StorageType.LOCAL,+ key="uploads/foreign.png",+ name="foreign.png",+ size=1024,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.END_USER,+ created_by=str(uuid4()),+ created_at=datetime.now(UTC),+ used=False,+ )+ upload_file.id = upload_file_id+ sqlite_session.add_all(+ [+ upload_file,+ SegmentAttachmentBinding(+ tenant_id=foreign_tenant_id,+ dataset_id=foreign_dataset_id,+ document_id=str(uuid4()),+ segment_id=str(uuid4()),+ attachment_id=upload_file_id,+ ),+ ]+ )+ sqlite_session.commit()+ scope = FileAccessScope(+ tenant_id=tenant_id,+ user_id=str(uuid4()),+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ )++ with bind_file_access_scope(scope):+ result = RetrievalService.get_segment_attachment_infos(+ [upload_file_id],+ dataset_refs=((tenant_id, str(uuid4())),),+ session=sqlite_session,+ )+ current_scope = get_current_file_access_scope()++ assert result == []+ assert current_scope is not None+ assert current_scope.granted_upload_file_ids == frozenset()++@pytest.mark.parametrize("sqlite_session", [(Dataset, DatasetDocument, DocumentSegment)], indirect=True)def test_knowledge_retrieval_grants_returned_segments_to_current_scope(monkeypatch: pytest.MonkeyPatch,diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.pyindex 5ed354d15ac..9c4dc266ad1 100644--- a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py+++ b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py@@ -431,6 +431,7 @@ def test_create_multimodal_filters_missing_uploads(vector._embeddings.embed_multimodal_documents.return_value = [[0.1, 0.2]]vector._vector_processor = MagicMock()vector._session = sqlite_session+ vector._dataset = Dataset(tenant_id=upload_file.tenant_id, name="dataset", created_by=upload_file.created_by)monkeypatch.setattr(vector_factory_module.storage, "load_once", MagicMock(return_value=b"abc"))docs = [diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.pyindex 3bf1441c0af..cce929d3780 100644--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py@@ -3,7 +3,7 @@import jsonfrom collections.abc import Mappingfrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import MagicMock, create_autospecimport pytestfrom pytest_mock import MockerFixture@@ -11,6 +11,8 @@ from pytest_mock import MockerFixtureimport core.rag.extractor.firecrawl.firecrawl_app as firecrawl_modulefrom core.rag.extractor.firecrawl.firecrawl_app import FirecrawlAppfrom core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor+from services.data_source.provider_service import DatasourceProviderService+from services.data_source.website_service import WebsiteServicedef _response(status_code: int, json_data: Mapping[str, Any] | None = None, text: str = "") -> MagicMock:@@ -389,7 +391,13 @@ class TestFirecrawlWebExtractor:},)- extractor = FirecrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")+ extractor = FirecrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )docs = extractor.extract()assert len(docs) == 1@@ -402,7 +410,13 @@ class TestFirecrawlWebExtractor:return_value=None,)- extractor = FirecrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")+ extractor = FirecrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []def test_extract_scrape_mode_returns_document(self, mocker: MockerFixture):@@ -417,7 +431,12 @@ class TestFirecrawlWebExtractor:)extractor = FirecrawlWebExtractor(- "https://example.com", "job-1", "tenant-1", mode="scrape", only_main_content=False+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="scrape",+ only_main_content=False,+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),)docs = extractor.extract()@@ -426,5 +445,11 @@ class TestFirecrawlWebExtractor:mock_scrape.assert_called_once_with("firecrawl", "https://example.com", "tenant-1", False)def test_extract_unknown_mode_returns_empty(self):- extractor = FirecrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="unknown")+ extractor = FirecrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="unknown",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []diff --git a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.pyindex abfc01d671b..344ee8d9d55 100644--- a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py@@ -2,11 +2,13 @@ from datetime import datetimefrom pathlib import Pathfrom types import SimpleNamespace+import httpximport pytestfrom sqlalchemy.orm import Sessionimport core.rag.extractor.extract_processor as processor_modulefrom core.rag.extractor.entity.datasource_type import DatasourceType+from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfofrom core.rag.extractor.extract_processor import ExtractProcessorfrom core.rag.models.document import Documentfrom models.model import UploadFile@@ -76,8 +78,6 @@ def _patch_all_extractors(monkeypatch) -> _ExtractorFactory:class TestExtractProcessorLoaders:def test_load_from_upload_file_return_docs_and_text(self, monkeypatch: pytest.MonkeyPatch):- monkeypatch.setattr(processor_module, "ExtractSetting", lambda **kwargs: SimpleNamespace(**kwargs))-monkeypatch.setattr(ExtractProcessor,"extract",@@ -126,9 +126,8 @@ class TestExtractProcessorLoaders:def test_load_from_url_builds_temp_file_with_correct_suffix(self, monkeypatch: pytest.MonkeyPatch, url, headers, expected_suffix):- response = SimpleNamespace(headers=headers, content=b"body")+ response = httpx.Response(200, headers=headers, content=b"body")monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)- monkeypatch.setattr(processor_module, "ExtractSetting", lambda **kwargs: SimpleNamespace(**kwargs))captured = {}@@ -150,7 +149,7 @@ class TestExtractProcessorLoaders:def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch):content = "a" * 100_000- response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode())+ response = httpx.Response(200, headers={"Content-Type": "text/plain"}, content=content.encode())monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)apply_config_overrides(monkeypatch, ETL_TYPE="SelfHosted")@@ -215,7 +214,7 @@ class TestExtractProcessorFileRouting:monkeypatch.setattr(processor_module.storage, "download", fake_download)monkeypatch.setattr(processor_module.tempfile, "_get_candidate_names", lambda: iter(["candidate-name"]))- setting = SimpleNamespace(+ setting = ExtractSetting(datasource_type=DatasourceType.FILE,upload_file=_upload_file(key=f"uploaded{extension}"),)@@ -296,7 +295,7 @@ class TestExtractProcessorFileRouting:assert kwargs["session"] is unbound_sessiondef test_extract_requires_upload_file_when_file_path_not_provided(self):- setting = SimpleNamespace(datasource_type=DatasourceType.FILE, upload_file=None)+ setting = ExtractSetting(datasource_type=DatasourceType.FILE, upload_file=None)with pytest.raises(AssertionError, match="upload_file is required"):ExtractProcessor.extract(setting)@@ -306,20 +305,22 @@ class TestExtractProcessorDatasourceRouting:def test_extract_routes_notion_datasource(self, monkeypatch: pytest.MonkeyPatch):factory = _patch_all_extractors(monkeypatch)- notion_info = SimpleNamespace(+ notion_info = NotionInfo(notion_workspace_id="ws",notion_obj_id="obj",notion_page_type="page",- document="doc",+ document=None,tenant_id="tenant",credential_id="cred",+ notion_access_token="secret",)- setting = SimpleNamespace(datasource_type=DatasourceType.NOTION, notion_info=notion_info)+ setting = ExtractSetting(datasource_type=DatasourceType.NOTION, notion_info=notion_info)docs = ExtractProcessor.extract(setting)assert docs[0].page_content == "extracted-by-NotionExtractor"assert factory.calls[-1][0] == "NotionExtractor"+ assert factory.calls[-1][2]["notion_access_token"] == "secret"@pytest.mark.parametrize(("provider", "expected"),@@ -334,7 +335,7 @@ class TestExtractProcessorDatasourceRouting:):factory = _patch_all_extractors(monkeypatch)- website_info = SimpleNamespace(+ website_info = WebsiteInfo(provider=provider,url="https://example.com",job_id="job",@@ -342,14 +343,14 @@ class TestExtractProcessorDatasourceRouting:mode="crawl",only_main_content=True,)- setting = SimpleNamespace(datasource_type=DatasourceType.WEBSITE, website_info=website_info)+ setting = ExtractSetting(datasource_type=DatasourceType.WEBSITE, website_info=website_info)docs = ExtractProcessor.extract(setting)assert docs[0].page_content == f"extracted-by-{expected}"assert factory.calls[-1][0] == expecteddef test_extract_unsupported_website_provider(self):- bad_provider = SimpleNamespace(+ bad_provider = WebsiteInfo(provider="unknown",url="https://example.com",job_id="job",@@ -357,19 +358,19 @@ class TestExtractProcessorDatasourceRouting:mode="crawl",only_main_content=True,)- setting = SimpleNamespace(datasource_type=DatasourceType.WEBSITE, website_info=bad_provider)+ setting = ExtractSetting(datasource_type=DatasourceType.WEBSITE, website_info=bad_provider)with pytest.raises(ValueError, match="Unsupported website provider"):ExtractProcessor.extract(setting)def test_extract_unsupported_datasource_type(self):with pytest.raises(ValueError, match="Unsupported datasource type"):- ExtractProcessor.extract(SimpleNamespace(datasource_type="unknown"))+ ExtractProcessor.extract(ExtractSetting(datasource_type="unknown"))def test_extract_requires_notion_info(self):with pytest.raises(AssertionError, match="notion_info is required"):- ExtractProcessor.extract(SimpleNamespace(datasource_type=DatasourceType.NOTION, notion_info=None))+ ExtractProcessor.extract(ExtractSetting(datasource_type=DatasourceType.NOTION, notion_info=None))def test_extract_requires_website_info(self):with pytest.raises(AssertionError, match="website_info is required"):- ExtractProcessor.extract(SimpleNamespace(datasource_type=DatasourceType.WEBSITE, website_info=None))+ ExtractProcessor.extract(ExtractSetting(datasource_type=DatasourceType.WEBSITE, website_info=None))diff --git a/api/tests/unit_tests/core/rag/extractor/test_jina_reader_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_jina_reader_extractor.pyindex 0b4c9bd809e..79fcac2f88b 100644--- a/api/tests/unit_tests/core/rag/extractor/test_jina_reader_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_jina_reader_extractor.py@@ -1,6 +1,10 @@+from unittest.mock import create_autospec+from pytest_mock import MockerFixturefrom core.rag.extractor.jina_reader_extractor import JinaReaderWebExtractor+from services.data_source.provider_service import DatasourceProviderService+from services.data_source.website_service import WebsiteServiceclass TestJinaReaderWebExtractor:@@ -15,7 +19,13 @@ class TestJinaReaderWebExtractor:},)- extractor = JinaReaderWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")+ extractor = JinaReaderWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )docs = extractor.extract()assert len(docs) == 1@@ -32,7 +42,13 @@ class TestJinaReaderWebExtractor:return_value=None,)- extractor = JinaReaderWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")+ extractor = JinaReaderWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []@@ -41,7 +57,13 @@ class TestJinaReaderWebExtractor:"core.rag.extractor.jina_reader_extractor.WebsiteService.get_crawl_url_data",return_value={"content": "unused"},)- extractor = JinaReaderWebExtractor("https://example.com", "job-1", "tenant-1", mode="scrape")+ extractor = JinaReaderWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="scrape",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []mock_get_crawl.assert_not_called()diff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.pyindex 56c9e254e38..2eac243de8c 100644--- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py@@ -13,11 +13,11 @@ from sqlalchemy import Enginefrom sqlalchemy.orm import Session, sessionmakerfrom core.rag.extractor import notion_extractor+from core.rag.extractor.entity.extract_setting import StoredDocumentExtractionInputfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom models.base import TypeBasefrom models.dataset import Document as DocumentModelfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus-from tests.unit_tests.config_override import apply_config_overrides@pytest.fixture@@ -65,36 +65,27 @@ class TestNotionExtractorInitAndPublicMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)assert extractor._notion_access_token == "token"- def test_init_falls_back_to_env_token_when_credential_lookup_fails(self, monkeypatch: pytest.MonkeyPatch):- monkeypatch.setattr(- notion_extractor.NotionExtractor,- "_get_access_token",- classmethod(lambda cls, tenant_id, credential_id: (_ for _ in ()).throw(Exception("credential error"))),- )- apply_config_overrides(monkeypatch, NOTION_INTEGRATION_TOKEN="env-token")-+ def test_init_uses_injected_token_loader(self):+ loader = mock.Mock(return_value="resolved-token")extractor = notion_extractor.NotionExtractor(notion_workspace_id="ws",notion_obj_id="obj",notion_page_type="page",tenant_id="tenant",credential_id="cred",+ notion_token_loader=loader,)- assert extractor._notion_access_token == "env-token"-- def test_init_raises_if_no_credential_and_no_env_token(self, monkeypatch: pytest.MonkeyPatch):- monkeypatch.setattr(- notion_extractor.NotionExtractor,- "_get_access_token",- classmethod(lambda cls, tenant_id, credential_id: (_ for _ in ()).throw(Exception("credential error"))),- )- apply_config_overrides(monkeypatch, NOTION_INTEGRATION_TOKEN=None)+ assert extractor._notion_access_token == "resolved-token"+ loader.assert_called_once_with()+ def test_init_propagates_missing_token_from_loader(self):+ loader = mock.Mock(side_effect=ValueError("Must specify `integration_token`"))with pytest.raises(ValueError, match="Must specify `integration_token`"):notion_extractor.NotionExtractor(notion_workspace_id="ws",@@ -102,6 +93,7 @@ class TestNotionExtractorInitAndPublicMethods:notion_page_type="page",tenant_id="tenant",credential_id="cred",+ notion_token_loader=loader,)def test_extract_updates_last_edited_and_loads_documents(self, monkeypatch: pytest.MonkeyPatch):@@ -111,6 +103,7 @@ class TestNotionExtractorInitAndPublicMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)update_mock = mock.Mock()@@ -131,6 +124,7 @@ class TestNotionExtractorInitAndPublicMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)monkeypatch.setattr(extractor, "_get_notion_block_data", lambda _: ["line1", "line2"])@@ -153,6 +147,7 @@ class TestNotionDatabase:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)first_page = {@@ -204,6 +199,7 @@ class TestNotionDatabase:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)page = {@@ -246,6 +242,7 @@ class TestNotionDatabase:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)mocker.patch("httpx.post", return_value=_mock_response({"results": None}))@@ -259,6 +256,7 @@ class TestNotionDatabase:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)mock_post = mocker.patch("httpx.post", return_value=_mock_response({"results": None}))@@ -278,6 +276,7 @@ class TestNotionDatabase:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)extractor._notion_access_token = None@@ -293,6 +292,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)first_response = {@@ -349,6 +349,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)mocker.patch("httpx.request", side_effect=httpx.HTTPError("network"))@@ -370,6 +371,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)root_payload = {@@ -419,6 +421,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)mocker.patch("httpx.request", return_value=_mock_response({"results": None, "next_cursor": None}))@@ -431,6 +434,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)page_one = {@@ -492,6 +496,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)# A cell with mixed formatting arrives as multiple rich text segments.@@ -534,6 +539,7 @@ class TestNotionBlocks:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)page = {@@ -577,6 +583,7 @@ class TestNotionMetadataAndCredentialMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)assert extractor.update_last_edited_time(None) is None@@ -592,11 +599,13 @@ class TestNotionMetadataAndCredentialMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)monkeypatch.setattr(extractor, "get_notion_last_edited_time", lambda: "2026-01-01T00:00:00.000Z")session_maker, document = persisted_document- extractor.update_last_edited_time(document)+ snapshot = StoredDocumentExtractionInput.model_validate(document)+ extractor.update_last_edited_time(snapshot)# Closing the writer session rolls back an uncommitted update before the independent read below.notion_extractor.db.session.close()@@ -608,6 +617,28 @@ class TestNotionMetadataAndCredentialMethods:"last_edited_time": "2026-01-01T00:00:00.000Z",}+ def test_last_edited_time_update_rejects_a_foreign_owner_snapshot(+ self,+ monkeypatch: pytest.MonkeyPatch,+ persisted_document: tuple[sessionmaker[Session], DocumentModel],+ ):+ session_maker, document = persisted_document+ extractor = notion_extractor.NotionExtractor(+ notion_workspace_id="ws",+ notion_obj_id="obj",+ notion_page_type="page",+ tenant_id=document.tenant_id,+ notion_access_token="token",+ notion_token_loader=lambda: "token",+ )+ monkeypatch.setattr(extractor, "get_notion_last_edited_time", lambda: "2026-01-01T00:00:00.000Z")+ snapshot = StoredDocumentExtractionInput.model_validate(document).model_copy(update={"tenant_id": "foreign"})+ extractor.update_last_edited_time(snapshot)+ with session_maker() as session:+ stored = session.get(DocumentModel, document.id)+ assert stored is not None+ assert stored.data_source_info_dict["last_edited_time"] == "2025-01-01T00:00:00.000Z"+def test_get_notion_last_edited_time_uses_page_and_database_urls(self, mocker: MockerFixture):extractor_page = notion_extractor.NotionExtractor(notion_workspace_id="ws",@@ -615,6 +646,7 @@ class TestNotionMetadataAndCredentialMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)request_mock = mocker.patch("httpx.request", return_value=_mock_response({"last_edited_time": "2025-05-01T00:00:00.000Z"})@@ -629,6 +661,7 @@ class TestNotionMetadataAndCredentialMethods:notion_page_type="database",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)request_mock = mocker.patch("httpx.request", return_value=_mock_response({"last_edited_time": "2025-06-01T00:00:00.000Z"})@@ -644,32 +677,13 @@ class TestNotionMetadataAndCredentialMethods:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)extractor._notion_access_token = Nonewith pytest.raises(AssertionError, match="Notion access token is required"):extractor.get_notion_last_edited_time()- def test_get_access_token_success_and_errors(self, monkeypatch: pytest.MonkeyPatch):- with pytest.raises(Exception, match="No credential id found"):- notion_extractor.NotionExtractor._get_access_token("tenant", None)-- class FakeProviderServiceMissing:- def get_datasource_credentials(self, **kwargs):- return None-- monkeypatch.setattr(notion_extractor, "DatasourceProviderService", FakeProviderServiceMissing)- with pytest.raises(Exception, match="No notion credential found"):- notion_extractor.NotionExtractor._get_access_token("tenant", "cred")-- class FakeProviderServiceFound:- def get_datasource_credentials(self, **kwargs):- return {"integration_secret": "token-from-credential"}-- monkeypatch.setattr(notion_extractor, "DatasourceProviderService", FakeProviderServiceFound)-- assert notion_extractor.NotionExtractor._get_access_token("tenant", "cred") == "token-from-credential"-def test_get_cell_text_uses_plain_text_for_mention_and_equation_segments():# Notion rich text segments of type mention or equation carry plain_textdiff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_page_rich_text.py b/api/tests/unit_tests/core/rag/extractor/test_notion_page_rich_text.pyindex 4e403e311a7..15cbdc124f2 100644--- a/api/tests/unit_tests/core/rag/extractor/test_notion_page_rich_text.py+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_page_rich_text.py@@ -18,6 +18,7 @@ def _extractor() -> notion_extractor.NotionExtractor:notion_page_type="page",tenant_id="tenant",notion_access_token="token",+ notion_token_loader=lambda: "token",)diff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.pyindex 701198e968e..0b13ebc84c5 100644--- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py@@ -2,7 +2,7 @@import jsonfrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import MagicMock, create_autospecimport httpximport pytest@@ -16,6 +16,8 @@ from core.rag.extractor.watercrawl.exceptions import ()from core.rag.extractor.watercrawl.extractor import WaterCrawlWebExtractorfrom core.rag.extractor.watercrawl.provider import WaterCrawlProvider+from services.data_source.provider_service import DatasourceProviderService+from services.data_source.website_service import WebsiteServicedef _response(@@ -454,7 +456,7 @@ class TestWaterCrawlWebExtractor:def test_extract_crawl_and_scrape_modes(self, monkeypatch: pytest.MonkeyPatch):monkeypatch.setattr("core.rag.extractor.watercrawl.extractor.WebsiteService.get_crawl_url_data",- lambda job_id, provider, url, tenant_id: {+ lambda _self, job_id, provider, url, tenant_id: {"markdown": "crawl","source_url": url,"description": "d",@@ -463,7 +465,7 @@ class TestWaterCrawlWebExtractor:)monkeypatch.setattr("core.rag.extractor.watercrawl.extractor.WebsiteService.get_scrape_url_data",- lambda provider, url, tenant_id, only_main_content: {+ lambda _self, provider, url, tenant_id, only_main_content: {"markdown": "scrape","source_url": url,"description": "d",@@ -471,8 +473,20 @@ class TestWaterCrawlWebExtractor:},)- crawl_extractor = WaterCrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")- scrape_extractor = WaterCrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="scrape")+ crawl_extractor = WaterCrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )+ scrape_extractor = WaterCrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="scrape",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert crawl_extractor.extract()[0].page_content == "crawl"assert scrape_extractor.extract()[0].page_content == "scrape"@@ -480,14 +494,26 @@ class TestWaterCrawlWebExtractor:def test_extract_crawl_returns_empty_when_service_returns_none(self, monkeypatch: pytest.MonkeyPatch):monkeypatch.setattr("core.rag.extractor.watercrawl.extractor.WebsiteService.get_crawl_url_data",- lambda job_id, provider, url, tenant_id: None,+ lambda _self, job_id, provider, url, tenant_id: None,)- extractor = WaterCrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="crawl")+ extractor = WaterCrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="crawl",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []def test_extract_unknown_mode_returns_empty(self):- extractor = WaterCrawlWebExtractor("https://example.com", "job-1", "tenant-1", mode="other")+ extractor = WaterCrawlWebExtractor(+ "https://example.com",+ "job-1",+ "tenant-1",+ mode="other",+ website_service=WebsiteService(providers=create_autospec(DatasourceProviderService, instance=True)),+ )assert extractor.extract() == []diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.pyindex f9f8740cbf1..2f330e00159 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py@@ -184,7 +184,12 @@ class TestParagraphIndexProcessor:processor, "_get_content_files", return_value=[AttachmentDocument(page_content="image", metadata={})]),):- documents = processor.transform([source_document], process_rule=process_rule, session=session)+ documents = processor.transform(+ [source_document],+ process_rule=process_rule,+ tenant_id="tenant-1",+ session=session,+ )assert len(documents) == 1assert documents[0].page_content == "first"@@ -219,6 +224,7 @@ class TestParagraphIndexProcessor:processor.transform([Document(page_content="text", metadata={})],process_rule={"mode": "automatic"},+ tenant_id="tenant-1",session=session,)@@ -285,7 +291,7 @@ class TestParagraphIndexProcessor:with (patch(- "core.rag.index_processor.processor.paragraph_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.paragraph_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.paragraph_index_processor.Vector") as mock_vector_cls,):@@ -303,7 +309,7 @@ class TestParagraphIndexProcessor:with (patch(- "core.rag.index_processor.processor.paragraph_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.paragraph_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.paragraph_index_processor.Keyword") as mock_keyword_cls,):diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.pyindex 2919c71e0a0..32b09e69877 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py@@ -139,12 +139,13 @@ class TestParentChildIndexProcessor:patch.object(processor, "_get_content_files", return_value=[AttachmentDocument(page_content="image", metadata={})]),- patch.object(processor, "_split_child_nodes", return_value=child_docs),+ patch.object(processor, "split_child_nodes", return_value=child_docs),):result = processor.transform([parent_document],process_rule={"mode": "custom", "rules": {"enabled": True}},preview=False,+ tenant_id="tenant-1",session=self.session,)@@ -176,12 +177,13 @@ class TestParentChildIndexProcessor:return_value="hash",),patch.object(processor, "_get_content_files", return_value=[]),- patch.object(processor, "_split_child_nodes", return_value=[]),+ patch.object(processor, "split_child_nodes", return_value=[]),):result = processor.transform(documents,process_rule={"mode": "custom", "rules": {"enabled": True}},preview=True,+ tenant_id="tenant-1",session=self.session,)@@ -202,7 +204,7 @@ class TestParentChildIndexProcessor:patch.object(processor, "_get_content_files", return_value=[AttachmentDocument(page_content="image", metadata={})]),- patch.object(processor, "_split_child_nodes", return_value=child_docs),+ patch.object(processor, "split_child_nodes", return_value=child_docs),patch("core.rag.index_processor.processor.parent_child_index_processor.helper.generate_text_hash",return_value="hash",@@ -213,6 +215,7 @@ class TestParentChildIndexProcessor:docs,process_rule={"mode": "hierarchical", "rules": {"enabled": True}},preview=True,+ tenant_id="tenant-1",session=self.session,)@@ -336,7 +339,7 @@ class TestParentChildIndexProcessor:with (patch(- "core.rag.index_processor.processor.parent_child_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.parent_child_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.parent_child_index_processor.Vector"),):@@ -349,7 +352,7 @@ class TestParentChildIndexProcessor:) -> None:with (patch(- "core.rag.index_processor.processor.parent_child_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.parent_child_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.parent_child_index_processor.Vector"),):@@ -362,7 +365,7 @@ class TestParentChildIndexProcessor:rules = Rule(subchunk_segmentation=None)with pytest.raises(ValueError, match="No subchunk segmentation found"):- processor._split_child_nodes(Document(page_content="parent", metadata={}), rules, "custom", None)+ processor.split_child_nodes(Document(page_content="parent", metadata={}), rules, "custom", None)def test_split_child_nodes_generates_child_documents(self, processor: ParentChildIndexProcessor) -> None:rules = Rule(subchunk_segmentation=Segmentation(max_tokens=200, chunk_overlap=10, separator="\n"))@@ -379,7 +382,7 @@ class TestParentChildIndexProcessor:return_value="hash",),):- child_docs = processor._split_child_nodes(+ child_docs = processor.split_child_nodes(Document(page_content="parent", metadata={}), rules, "custom", None)diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.pyindex d95a201c1fc..57cbc443b59 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py@@ -289,7 +289,7 @@ class TestQAIndexProcessor:with (patch(- "core.rag.index_processor.processor.qa_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.qa_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.qa_index_processor.Vector") as mock_vector_cls,):@@ -305,7 +305,7 @@ class TestQAIndexProcessor:session = sqlite_sessionwith (patch(- "core.rag.index_processor.processor.qa_index_processor.SummaryIndexService.delete_summaries_for_segments"+ "core.rag.index_processor.processor.qa_index_processor.SummaryIndexAdapter.delete_summaries_for_segments") as mock_summary,patch("core.rag.index_processor.processor.qa_index_processor.Vector") as mock_vector_cls,):diff --git a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.pyindex faa6417900c..98cd0479635 100644--- a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py+++ b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py@@ -24,9 +24,14 @@ from models.tools import ToolFilefrom tests.unit_tests.config_override import config_overrides_context-def _upload(*, upload_id: str, name: str) -> UploadFile:+def _upload(+ *,+ upload_id: str,+ name: str,+ tenant_id: str = "tenant-1",+) -> UploadFile:upload = UploadFile(- tenant_id=str(uuid4()),+ tenant_id=tenant_id,storage_type=StorageType.LOCAL,key=f"uploads/{name}",name=name,@@ -42,8 +47,8 @@ def _upload(*, upload_id: str, name: str) -> UploadFile:return upload-def _persist_upload(session: Session, *, upload_id: str, name: str) -> UploadFile:- upload = _upload(upload_id=upload_id, name=name)+def _persist_upload(session: Session, *, upload_id: str, name: str, tenant_id: str = "tenant-1") -> UploadFile:+ upload = _upload(upload_id=upload_id, name=name, tenant_id=tenant_id)session.add(upload)return upload@@ -170,7 +175,7 @@ class TestBaseIndexProcessor:self, processor: _ForwardingBaseIndexProcessor, unbound_session: Session) -> None:document = Document(page_content="no image markdown", metadata={"document_id": "doc-1", "dataset_id": "ds-1"})- assert processor._get_content_files(document, session=unbound_session) == []+ assert processor._get_content_files(document, tenant_id="tenant-1", session=unbound_session) == []def test_get_content_files_handles_all_sources_and_duplicates(self, processor: _ForwardingBaseIndexProcessor, sqlite_session: Session@@ -195,7 +200,12 @@ class TestBaseIndexProcessor:patch.object(processor, "_download_tool_file", return_value=tool_upload.id) as mock_tool_download,patch.object(processor, "_download_image", return_value=remote_upload.id) as mock_image_download,):- files = processor._get_content_files(document, current_user=current_user, session=sqlite_session)+ files = processor._get_content_files(+ document,+ current_user=current_user,+ tenant_id="tenant-1",+ session=sqlite_session,+ )assert len(files) == 5assert all(isinstance(file, AttachmentDocument) for file in files)@@ -207,6 +217,7 @@ class TestBaseIndexProcessor:mock_tool_download.assert_called_once_with("cccccccc-cccc-cccc-cccc-cccccccccccc",current_user,+ tenant_id="tenant-1",session=sqlite_session,)mock_image_download.assert_called_once()@@ -218,7 +229,12 @@ class TestBaseIndexProcessor:images = ["/files/tools/cccccccc-cccc-cccc-cccc-cccccccccccc.png", "https://example.com/remote.png"]with patch.object(processor, "_extract_markdown_images", return_value=images):- files = processor._get_content_files(document, current_user=None, session=unbound_session)+ files = processor._get_content_files(+ document,+ current_user=None,+ tenant_id="tenant-1",+ session=unbound_session,+ )assert files == []@@ -238,7 +254,9 @@ class TestBaseIndexProcessor:patch.object(processor, "_extract_markdown_images", return_value=images),patch.object(processor, "_download_image") as mock_image_download,):- files = processor._get_content_files(document, current_user=Mock(), session=unbound_session)+ files = processor._get_content_files(+ document, current_user=_account(), tenant_id="tenant-1", session=unbound_session+ )assert files == []mock_image_download.assert_not_called()@@ -251,7 +269,22 @@ class TestBaseIndexProcessor:with (patch.object(processor, "_extract_markdown_images", return_value=images),):- files = processor._get_content_files(document, session=sqlite_session)+ files = processor._get_content_files(document, tenant_id="tenant-1", session=sqlite_session)++ assert files == []++ def test_get_content_files_ignores_upload_from_another_tenant(+ self, processor: _ForwardingBaseIndexProcessor, sqlite_session: Session+ ) -> None:+ upload_id = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"+ _persist_upload(sqlite_session, upload_id=upload_id, name="foreign.png", tenant_id="tenant-2")+ sqlite_session.commit()+ document = Document(+ page_content=f"",+ metadata={"document_id": "doc-1", "dataset_id": "ds-1"},+ )++ files = processor._get_content_files(document, tenant_id="tenant-1", session=sqlite_session)assert files == []@@ -333,7 +366,15 @@ class TestBaseIndexProcessor:def test_download_tool_file_returns_none_when_not_found(self, processor: _ForwardingBaseIndexProcessor, sqlite_session: Session) -> None:- assert processor._download_tool_file(str(uuid4()), current_user=_account(), session=sqlite_session) is None+ assert (+ processor._download_tool_file(+ str(uuid4()),+ current_user=_account(),+ tenant_id="tenant-1",+ session=sqlite_session,+ )+ is None+ )def test_download_tool_file_uploads_file_when_found(self, processor: _ForwardingBaseIndexProcessor, sqlite_session: Session, sqlite_engine: Engine@@ -360,8 +401,39 @@ class TestBaseIndexProcessor:patch("services.file_service.FileService") as mock_file_service,):mock_file_service.return_value.upload_file.return_value = upload_result- result = processor._download_tool_file(tool_file.id, current_user=_account(), session=sqlite_session)+ result = processor._download_tool_file(+ tool_file.id,+ current_user=_account(),+ tenant_id=tool_file.tenant_id,+ session=sqlite_session,+ )assert result == upload_result.idmock_load.assert_called_once_with("k1")mock_file_service.return_value.upload_file.assert_called_once()++ def test_download_tool_file_rejects_file_from_another_tenant(+ self, processor: _ForwardingBaseIndexProcessor, sqlite_session: Session+ ) -> None:+ tool_file = ToolFile(+ user_id=str(uuid4()),+ tenant_id="tenant-2",+ conversation_id=None,+ file_key="foreign-key",+ mimetype="image/png",+ name="foreign.png",+ size=4,+ )+ sqlite_session.add(tool_file)+ sqlite_session.commit()++ with patch("core.rag.index_processor.index_processor_base.storage.load_once") as mock_load:+ result = processor._download_tool_file(+ tool_file.id,+ current_user=_account(),+ tenant_id="tenant-1",+ session=sqlite_session,+ )++ assert result is None+ mock_load.assert_not_called()diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.pydeleted file mode 100644index 7bea0a9fe34..00000000000--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py+++ /dev/null@@ -1,2018 +0,0 @@-"""Comprehensive unit tests for IndexingRunner.--This test module provides complete coverage of the IndexingRunner class, which is responsible-for orchestrating the document indexing pipeline in the Dify RAG system.--Test Coverage Areas:-==================-1. **Document Parsing Pipeline (Extract Phase)**- - Tests extraction from various data sources (upload files, Notion, websites)- - Validates metadata preservation and document status updates- - Ensures proper error handling for missing or invalid sources--2. **Chunk Creation Logic (Transform Phase)**- - Tests document splitting with different segmentation strategies- - Validates embedding model integration for high-quality indexing- - Tests text cleaning and preprocessing rules--3. **Embedding Generation Orchestration**- - Tests parallel processing of document chunks- - Validates token counting and embedding generation- - Tests integration with various embedding model providers--4. **Vector Storage Integration (Load Phase)**- - Tests vector index creation and updates- - Validates keyword index generation for economy mode- - Tests parent-child index structures--5. **Retry Logic & Error Handling**- - Tests pause/resume functionality- - Validates error recovery and status updates- - Tests handling of provider token errors and deleted documents--6. **Document Status Management**- - Tests status transitions (parsing → splitting → indexing → completed)- - Validates timestamp updates and error state persistence- - Tests concurrent document processing--Testing Approach:-================-- All tests use mocking to avoid external dependencies (database, storage, Redis)-- Tests follow the Arrange-Act-Assert (AAA) pattern for clarity-- Each test is isolated and can run independently-- Fixtures provide reusable test data and mock objects-- Comprehensive docstrings explain the purpose and assertions of each test--Note: These tests focus on unit testing the IndexingRunner logic. Integration tests-for the full indexing pipeline are handled separately in the integration test suite.-"""--import json-import uuid-from datetime import UTC, datetime-from typing import Any-from unittest.mock import MagicMock, Mock, patch--import pytest-from sqlalchemy import event-from sqlalchemy.orm import Session, sessionmaker-from sqlalchemy.orm.exc import ObjectDeletedError--from core.entities.knowledge_entities import PreviewDetail-from core.errors.error import ProviderTokenNotInitError-from core.indexing_runner import (- DocumentIsDeletedPausedError,- DocumentIsPausedError,- IndexingRunner,-)-from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from core.rag.models.document import ChildDocument, Document-from enums import DeploymentEdition-from extensions.storage.storage_type import StorageType-from graphon.model_runtime.entities.model_entities import ModelType-from libs.datetime_utils import naive_utc_now-from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment-from models.dataset import Document as DatasetDocument-from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus-from models.model import Account, UploadFile-from services.vector_space_admission_service import VectorSpaceAdmissionError--# ============================================================================-# Helper Functions-# ============================================================================---def create_mock_dataset(- dataset_id: str | None = None,- tenant_id: str | None = None,- indexing_technique: str = IndexTechniqueType.HIGH_QUALITY,- embedding_provider: str = "openai",- embedding_model: str = "text-embedding-ada-002",-) -> Dataset:- """Create a Dataset object with configurable parameters.-- This helper function creates a properly configured Dataset object that can be- used across multiple tests, ensuring consistency in test data.-- Args:- dataset_id: Optional dataset ID. If None, generates a new UUID.- tenant_id: Optional tenant ID. If None, generates a new UUID.- indexing_technique: The indexing technique ("high_quality" or "economy").- embedding_provider: The embedding model provider name.- embedding_model: The embedding model name.-- Returns:- Dataset: A configured Dataset object with all required attributes.-- Example:- >>> dataset = create_mock_dataset(indexing_technique="economy")- >>> assert dataset.indexing_technique == "economy"- """- dataset = Dataset(- id=dataset_id or str(uuid.uuid4()),- tenant_id=tenant_id or str(uuid.uuid4()),- indexing_technique=indexing_technique,- embedding_model_provider=embedding_provider,- embedding_model=embedding_model,- )- return dataset---def create_mock_dataset_document(- document_id: str | None = None,- dataset_id: str | None = None,- tenant_id: str | None = None,- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,- data_source_type: str = "upload_file",- doc_language: str = "English",-) -> DatasetDocument:- """Create a DatasetDocument object with configurable parameters.-- This helper function creates a properly configured DatasetDocument object,- reducing boilerplate code in individual tests.-- Args:- document_id: Optional document ID. If None, generates a new UUID.- dataset_id: Optional dataset ID. If None, generates a new UUID.- tenant_id: Optional tenant ID. If None, generates a new UUID.- doc_form: The document form/index type (e.g., PARAGRAPH_INDEX, QA_INDEX).- data_source_type: The data source type ("upload_file", "notion_import", etc.).- doc_language: The document language.-- Returns:- DatasetDocument: A configured DatasetDocument object with all required attributes.-- Example:- >>> doc = create_mock_dataset_document(doc_form=IndexStructureType.QA_INDEX)- >>> assert doc.doc_form == IndexStructureType.QA_INDEX- """- return DatasetDocument(- id=document_id or str(uuid.uuid4()),- dataset_id=dataset_id or str(uuid.uuid4()),- tenant_id=tenant_id or str(uuid.uuid4()),- doc_form=doc_form,- doc_language=doc_language,- data_source_type=data_source_type,- data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),- dataset_process_rule_id=str(uuid.uuid4()),- created_by=str(uuid.uuid4()),- )---def persist_indexing_scope(session: Session) -> tuple[Dataset, DatasetDocument]:- tenant_id = str(uuid.uuid4())- created_by = str(uuid.uuid4())- dataset = Dataset(- id=str(uuid.uuid4()),- tenant_id=tenant_id,- name="Dataset",- description="",- created_by=created_by,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- document = DatasetDocument(- id=str(uuid.uuid4()),- tenant_id=tenant_id,- dataset_id=dataset.id,- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- batch="batch",- name="Document",- created_from=DocumentCreatedFrom.API,- created_by=created_by,- )- session.add_all([dataset, document])- session.commit()- return dataset, document---def create_sample_documents(- count: int = 3,- include_children: bool = False,- base_content: str = "Sample chunk content",-) -> list[Document]:- """Create a list of sample Document objects for testing.-- This helper function generates test documents with proper metadata,- optionally including child documents for hierarchical indexing tests.-- Args:- count: Number of documents to create.- include_children: Whether to add child documents to each parent.- base_content: Base content string for documents.-- Returns:- list[Document]: A list of Document objects with metadata.-- Example:- >>> docs = create_sample_documents(count=2, include_children=True)- >>> assert len(docs) == 2- >>> assert docs[0].children is not None- """- documents = []- for i in range(count):- doc = Document(- page_content=f"{base_content} {i + 1}",- metadata={- "doc_id": f"chunk{i + 1}",- "doc_hash": f"hash{i + 1}",- "document_id": "doc1",- "dataset_id": "dataset1",- },- )-- # Add child documents if requested (for parent-child indexing)- if include_children:- doc.children = [- ChildDocument(- page_content=f"Child of {base_content} {i + 1}",- metadata={- "doc_id": f"child_chunk{i + 1}",- "doc_hash": f"child_hash{i + 1}",- },- )- ]-- documents.append(doc)-- return documents---def create_mock_process_rule(- mode: str = "automatic",- max_tokens: int = 500,- chunk_overlap: int = 50,- separator: str = "\\n\\n",-) -> dict[str, Any]:- """Create a mock processing rule dictionary.-- This helper function creates a processing rule configuration that matches- the structure expected by the IndexingRunner.-- Args:- mode: Processing mode ("automatic", "custom", or "hierarchical").- max_tokens: Maximum tokens per chunk.- chunk_overlap: Number of overlapping tokens between chunks.- separator: Separator string for splitting.-- Returns:- dict: A processing rule configuration dictionary.-- Example:- >>> rule = create_mock_process_rule(mode="custom", max_tokens=1000)- >>> assert rule["mode"] == "custom"- >>> assert rule["rules"]["segmentation"]["max_tokens"] == 1000- """- return {- "mode": mode,- "rules": {- "segmentation": {- "max_tokens": max_tokens,- "chunk_overlap": chunk_overlap,- "separator": separator,- },- "pre_processing_rules": [{"id": "remove_extra_spaces", "enabled": True}],- },- }---# ============================================================================-# Test Classes-# ============================================================================---class TestIndexingRunnerExtract:- """Unit tests for IndexingRunner._extract method.-- Tests cover:- - Upload file extraction- - Notion import extraction- - Website crawl extraction- - Document status updates during extraction- - Error handling for missing data sources- """-- @pytest.fixture- def mock_dependencies(self, sqlite_session: Session):- """Mock all external dependencies for extract tests."""- with (- patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,- patch("core.indexing_runner.storage") as mock_storage,- ):- yield {- "session": sqlite_session,- "factory": mock_factory,- "storage": mock_storage,- }-- @pytest.fixture- def sample_dataset_document(self):- """Create a sample dataset document for testing."""- return DatasetDocument(- id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- tenant_id=str(uuid.uuid4()),- doc_form=IndexStructureType.PARAGRAPH_INDEX,- data_source_type="upload_file",- data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),- )-- @pytest.fixture- def sample_process_rule(self):- """Create a sample processing rule."""- return {- "mode": "automatic",- "rules": {- "segmentation": {"max_tokens": 500, "chunk_overlap": 50, "separator": "\\n\\n"},- "pre_processing_rules": [{"id": "remove_extra_spaces", "enabled": True}],- },- }-- def test_extract_upload_file_success(self, mock_dependencies, sample_dataset_document, sample_process_rule):- """Test successful extraction from uploaded file.-- This test verifies that the IndexingRunner can successfully extract content- from an uploaded file and properly update document metadata. It ensures:- - The processor's extract method is called with correct parameters- - Document and dataset IDs are properly added to metadata- - The document status is updated during extraction-- Expected behavior:- - Extract should return documents with updated metadata- - Each document should have document_id and dataset_id in metadata- - The processor's extract method should be called exactly once- """- # Arrange: Set up the test environment with mocked dependencies- runner = IndexingRunner()- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Create mock extracted documents that simulate PDF page extraction- extracted_docs = [- Document(- page_content="Test content 1",- metadata={"doc_id": "doc1", "source": "test.pdf", "page": 1},- ),- Document(- page_content="Test content 2",- metadata={"doc_id": "doc2", "source": "test.pdf", "page": 2},- ),- ]- mock_processor.extract.return_value = extracted_docs- file_id = json.loads(sample_dataset_document.data_source_info)["upload_file_id"]- upload_file = UploadFile(- tenant_id=sample_dataset_document.tenant_id,- storage_type=StorageType.LOCAL,- key="uploads/test.pdf",- name="test.pdf",- size=10,- extension="pdf",- mime_type="application/pdf",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="user-id",- created_at=datetime.now(UTC),- used=True,- )- upload_file.id = file_id- mock_dependencies["session"].add(upload_file)- mock_dependencies["session"].commit()-- with patch.object(runner, "_update_document_index_status"), patch("core.indexing_runner.ExtractSetting"):- result = runner._extract(- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]- )-- # Assert: Verify the extraction results- assert len(result) == 2, "Should extract 2 documents from the PDF"- assert result[0].page_content == "Test content 1", "First document content should match"- # Verify metadata was properly updated with document and dataset IDs- assert result[0].metadata["document_id"] == sample_dataset_document.id- assert result[0].metadata["dataset_id"] == sample_dataset_document.dataset_id- assert result[1].page_content == "Test content 2", "Second document content should match"- # Verify the processor was called exactly once (not multiple times)- mock_processor.extract.assert_called_once()- assert mock_processor.extract.call_args.kwargs["session"] is mock_dependencies["session"]-- def test_extract_notion_import_success(self, mock_dependencies, sample_dataset_document, sample_process_rule):- """Test successful extraction from Notion import."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.data_source_type = "notion_import"- sample_dataset_document.data_source_info = json.dumps(- {- "credential_id": str(uuid.uuid4()),- "notion_workspace_id": "workspace123",- "notion_page_id": "page123",- "type": "page",- }- )-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- extracted_docs = [Document(page_content="Notion content", metadata={"doc_id": "notion1", "source": "notion"})]- mock_processor.extract.return_value = extracted_docs-- # Mock update_document_index_status to avoid database calls- with patch.object(runner, "_update_document_index_status"):- # Act- result = runner._extract(- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]- )-- # Assert- assert len(result) == 1- assert result[0].page_content == "Notion content"- assert result[0].metadata["document_id"] == sample_dataset_document.id-- def test_extract_website_crawl_success(self, mock_dependencies, sample_dataset_document, sample_process_rule):- """Test successful extraction from website crawl."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.data_source_type = "website_crawl"- sample_dataset_document.data_source_info = json.dumps(- {- "provider": "firecrawl",- "url": "https://example.com",- "job_id": "job123",- "mode": "crawl",- "only_main_content": True,- }- )-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- extracted_docs = [- Document(page_content="Website content", metadata={"doc_id": "web1", "url": "https://example.com"})- ]- mock_processor.extract.return_value = extracted_docs-- # Mock update_document_index_status to avoid database calls- with patch.object(runner, "_update_document_index_status"):- # Act- result = runner._extract(- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]- )-- # Assert- assert len(result) == 1- assert result[0].page_content == "Website content"- assert result[0].metadata["document_id"] == sample_dataset_document.id-- def test_extract_missing_upload_file(self, mock_dependencies, sample_dataset_document, sample_process_rule):- """Test extraction fails when upload file is missing."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.data_source_info = "{}"-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Act & Assert- with pytest.raises(ValueError, match="no upload file found"):- runner._extract(mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"])-- def test_extract_unsupported_data_source(self, mock_dependencies, sample_dataset_document, sample_process_rule):- """Test extraction returns empty list for unsupported data sources."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.data_source_type = "unsupported_type"-- mock_processor = MagicMock()-- # Act- result = runner._extract(- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]- )-- # Assert- assert result == []---class TestIndexingRunnerTransform:- """Unit tests for IndexingRunner._transform method.-- Tests cover:- - Document chunking with different splitters- - Embedding model instance retrieval- - Text cleaning and preprocessing- - Metadata preservation- - Child chunk generation for hierarchical indexing- """-- @pytest.fixture- def mock_dependencies(self, unbound_session: Session):- """Mock all external dependencies for transform tests."""- with (- patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,- ):- yield {- "session": unbound_session,- "model_manager": mock_model_manager,- }-- @pytest.fixture- def sample_dataset(self):- """Create a sample dataset for testing."""- dataset = Dataset(- id=str(uuid.uuid4()),- tenant_id=str(uuid.uuid4()),- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- embedding_model_provider="openai",- embedding_model="text-embedding-ada-002",- )- return dataset-- @pytest.fixture- def sample_text_docs(self):- """Create sample text documents for transformation."""- return [- Document(- page_content="This is a long document that needs to be split into multiple chunks. " * 10,- metadata={"doc_id": "doc1", "source": "test.pdf"},- ),- Document(- page_content="Another document with different content. " * 5,- metadata={"doc_id": "doc2", "source": "test.pdf"},- ),- ]-- def test_transform_with_high_quality_indexing(self, mock_dependencies, sample_dataset, sample_text_docs):- """Test transformation with high quality indexing (embeddings)."""- # Arrange- runner = IndexingRunner()- mock_embedding_instance = MagicMock()- model_manager = mock_dependencies["model_manager"].return_value- model_manager.get_model_instance.return_value = mock_embedding_instance-- mock_processor = MagicMock()- transformed_docs = [- Document(- page_content="Chunk 1",- metadata={"doc_id": "chunk1", "doc_hash": "hash1", "document_id": "doc1"},- ),- Document(- page_content="Chunk 2",- metadata={"doc_id": "chunk2", "doc_hash": "hash2", "document_id": "doc1"},- ),- ]- mock_processor.transform.return_value = transformed_docs-- process_rule = {- "mode": "automatic",- "rules": {"segmentation": {"max_tokens": 500, "chunk_overlap": 50}},- }-- # Act- result = runner._transform(- mock_processor,- sample_dataset,- sample_text_docs,- "English",- process_rule,- session=mock_dependencies["session"],- )-- # Assert- assert len(result) == 2- assert result[0].page_content == "Chunk 1"- assert result[1].page_content == "Chunk 2"- model_manager.get_model_instance.assert_called_once_with(- tenant_id=sample_dataset.tenant_id,- provider=sample_dataset.embedding_model_provider,- model_type=ModelType.TEXT_EMBEDDING,- model=sample_dataset.embedding_model,- )- mock_processor.transform.assert_called_once()-- def test_transform_with_economy_indexing(self, mock_dependencies, sample_dataset, sample_text_docs):- """Test transformation with economy indexing (no embeddings)."""- # Arrange- runner = IndexingRunner()- model_manager = mock_dependencies["model_manager"].return_value- sample_dataset.indexing_technique = IndexTechniqueType.ECONOMY-- mock_processor = MagicMock()- transformed_docs = [- Document(- page_content="Chunk 1",- metadata={"doc_id": "chunk1", "doc_hash": "hash1"},- )- ]- mock_processor.transform.return_value = transformed_docs-- process_rule = {"mode": "automatic", "rules": {}}-- # Act- result = runner._transform(- mock_processor,- sample_dataset,- sample_text_docs,- "English",- process_rule,- session=mock_dependencies["session"],- )-- # Assert- assert len(result) == 1- model_manager.get_model_instance.assert_not_called()-- def test_transform_with_custom_segmentation(self, mock_dependencies, sample_dataset, sample_text_docs):- """Test transformation with custom segmentation rules."""- # Arrange- runner = IndexingRunner()- mock_embedding_instance = MagicMock()- model_manager = mock_dependencies["model_manager"].return_value- model_manager.get_model_instance.return_value = mock_embedding_instance-- mock_processor = MagicMock()- transformed_docs = [Document(page_content="Custom chunk", metadata={"doc_id": "custom1", "doc_hash": "hash1"})]- mock_processor.transform.return_value = transformed_docs-- process_rule = {- "mode": "custom",- "rules": {"segmentation": {"max_tokens": 1000, "chunk_overlap": 100, "separator": "\\n"}},- }-- # Act- result = runner._transform(- mock_processor,- sample_dataset,- sample_text_docs,- "Chinese",- process_rule,- session=mock_dependencies["session"],- )-- # Assert- assert len(result) == 1- assert result[0].page_content == "Custom chunk"- # Verify transform was called with correct parameters- call_args = mock_processor.transform.call_args- assert call_args[1]["doc_language"] == "Chinese"- assert call_args[1]["process_rule"] == process_rule---class TestIndexingRunnerLoad:- """Unit tests for IndexingRunner._load method.-- Tests cover:- - Vector index creation- - Keyword index creation- - Multi-threaded processing- - Document segment status updates- - Precomputed token totals- - Error handling during loading- """-- @pytest.fixture- def mock_dependencies(self, unbound_session: Session):- """Mock all external dependencies for load tests."""- with (- patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,- patch("core.indexing_runner.current_app") as mock_app,- patch("core.indexing_runner.threading.Thread") as mock_thread,- patch("core.indexing_runner.concurrent.futures.ThreadPoolExecutor") as mock_executor,- ):- mock_app._get_current_object = Mock(return_value=Mock())- yield {- "session": unbound_session,- "model_manager": mock_model_manager,- "app": mock_app,- "thread": mock_thread,- "executor": mock_executor,- }-- @pytest.fixture- def sample_dataset(self):- """Create a sample dataset for testing."""- dataset = Dataset(- id=str(uuid.uuid4()),- tenant_id=str(uuid.uuid4()),- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- embedding_model_provider="openai",- embedding_model="text-embedding-ada-002",- )- return dataset-- @pytest.fixture- def sample_dataset_document(self):- """Create a sample dataset document for testing."""- return DatasetDocument(- id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- doc_form=IndexStructureType.PARAGRAPH_INDEX,- )-- @pytest.fixture- def sample_documents(self):- """Create sample documents for loading."""- return [- Document(- page_content="Chunk 1 content",- metadata={"doc_id": "chunk1", "doc_hash": "hash1", "document_id": "doc1"},- ),- Document(- page_content="Chunk 2 content",- metadata={"doc_id": "chunk2", "doc_hash": "hash2", "document_id": "doc1"},- ),- Document(- page_content="Chunk 3 content",- metadata={"doc_id": "chunk3", "doc_hash": "hash3", "document_id": "doc1"},- ),- ]-- def test_load_with_high_quality_indexing(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test loading with high quality indexing (vector embeddings)."""- # Arrange- runner = IndexingRunner()-- # Mock ThreadPoolExecutor- mock_future = MagicMock()- mock_future.result.return_value = None- mock_executor_instance = MagicMock()- mock_executor_instance.__enter__.return_value = mock_executor_instance- mock_executor_instance.__exit__.return_value = None- mock_executor_instance.submit.return_value = mock_future- mock_dependencies["executor"].return_value = mock_executor_instance-- # Mock update_document_index_status to avoid database calls- with patch.object(runner, "_update_document_index_status") as mock_update_status:- # Act- runner._load(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- total_tokens=300,- )-- # Assert- mock_dependencies["model_manager"].assert_not_called()- # Verify executor was used for parallel processing- assert mock_executor_instance.submit.called- for submit_call in mock_executor_instance.submit.call_args_list:- assert submit_call.args[0].__name__ == runner._process_chunk.__name__- assert len(submit_call.args) == 6- mock_future.result.assert_called()- assert mock_update_status.call_args.kwargs["extra_update_params"][DatasetDocument.tokens] == 300-- def test_load_propagates_worker_errors(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- runner = IndexingRunner()- mock_future = MagicMock()- mock_future.result.side_effect = RuntimeError("index failed")- mock_executor_instance = MagicMock()- mock_executor_instance.__enter__.return_value = mock_executor_instance- mock_executor_instance.__exit__.return_value = None- mock_executor_instance.submit.return_value = mock_future- mock_dependencies["executor"].return_value = mock_executor_instance-- with (- patch.object(runner, "_update_document_index_status") as mock_update_status,- pytest.raises(RuntimeError, match="index failed"),- ):- runner._load(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- total_tokens=300,- )-- mock_update_status.assert_not_called()-- def test_load_with_economy_indexing(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test loading with economy indexing (keyword only)."""- # Arrange- runner = IndexingRunner()- sample_dataset.indexing_technique = IndexTechniqueType.ECONOMY-- # Mock thread for keyword indexing- mock_thread_instance = MagicMock()- mock_thread_instance.join = MagicMock()- mock_dependencies["thread"].return_value = mock_thread_instance-- # Mock update_document_index_status to avoid database calls- with patch.object(runner, "_update_document_index_status"):- # Act- runner._load(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- total_tokens=0,- )-- # Assert- # Verify keyword thread was created and joined- mock_dependencies["thread"].assert_called_once()- mock_thread_instance.start.assert_called_once()- mock_thread_instance.join.assert_called_once()-- def test_load_with_parent_child_index(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test loading with parent-child index structure."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX- sample_dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY-- # Add child documents- for doc in sample_documents:- doc.children = [- ChildDocument(- page_content=f"Child of {doc.page_content}",- metadata={"doc_id": f"child_{doc.metadata['doc_id']}", "doc_hash": "child_hash"},- )- ]-- # Mock ThreadPoolExecutor- mock_future = MagicMock()- mock_future.result.return_value = None- mock_executor_instance = MagicMock()- mock_executor_instance.__enter__.return_value = mock_executor_instance- mock_executor_instance.__exit__.return_value = None- mock_executor_instance.submit.return_value = mock_future- mock_dependencies["executor"].return_value = mock_executor_instance-- # Mock update_document_index_status to avoid database calls- with patch.object(runner, "_update_document_index_status"):- # Act- runner._load(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- total_tokens=150,- )-- # Assert- mock_dependencies["model_manager"].assert_not_called()- # Verify no keyword thread for parent-child index- mock_dependencies["thread"].assert_not_called()---class TestIndexingRunnerRun:- """Unit tests for IndexingRunner.run method.-- Tests cover:- - Complete end-to-end indexing flow- - Error handling and recovery- - Document status transitions- - Pause detection- - Multiple document processing- """-- @pytest.fixture- def mock_dependencies(self, sqlite_session: Session):- """Mock all external dependencies for run tests."""- with (- patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,- patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,- patch("core.indexing_runner.storage") as mock_storage,- patch("core.indexing_runner.threading.Thread") as mock_thread,- ):- yield {- "session": sqlite_session,- "factory": mock_factory,- "model_manager": mock_model_manager,- "storage": mock_storage,- "thread": mock_thread,- }-- @pytest.fixture- def sample_dataset_documents(self, sqlite_session: Session):- """Create sample dataset documents for testing."""- docs = []- for i in range(2):- tenant_id = str(uuid.uuid4())- dataset_id = str(uuid.uuid4())- account = Account(name=f"Account {i}", email=f"account-{i}@example.com")- account.id = str(uuid.uuid4())- dataset = Dataset(- id=dataset_id,- tenant_id=tenant_id,- name=f"Dataset {i}",- description="",- created_by=account.id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- process_rule = DatasetProcessRule(- dataset_id=dataset_id,- mode="automatic",- rules="{}",- created_by=account.id,- )- document = DatasetDocument(- id=str(uuid.uuid4()),- dataset_id=dataset_id,- tenant_id=tenant_id,- position=1,- doc_form=IndexStructureType.PARAGRAPH_INDEX,- doc_language="English",- data_source_type=DataSourceType.UPLOAD_FILE,- data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),- dataset_process_rule_id=process_rule.id,- batch="batch",- name=f"Document {i}",- created_from=DocumentCreatedFrom.API,- created_by=account.id,- )- sqlite_session.add_all([account, dataset, process_rule, document])- docs.append(document)- sqlite_session.commit()- return docs-- def test_run_in_indexing_status_loads_child_chunks_with_caller_session(- self, mock_dependencies, sample_dataset_documents- ):- runner = IndexingRunner()- dataset_document = sample_dataset_documents[0]- dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX- session = mock_dependencies["session"]- dataset = session.get(Dataset, dataset_document.dataset_id)- assert dataset is not None- process_rule = session.get(DatasetProcessRule, dataset_document.dataset_process_rule_id)- assert process_rule is not None- process_rule.mode = "hierarchical"- process_rule.rules = json.dumps({"parent_mode": "paragraph"})- segment = DocumentSegment(- tenant_id=dataset_document.tenant_id,- dataset_id=dataset_document.dataset_id,- document_id=dataset_document.id,- position=1,- content="parent",- word_count=0,- tokens=12,- created_by="account-id",- status="waiting",- index_node_id="parent-node",- index_node_hash="parent-hash",- )- child_chunk = ChildChunk(- tenant_id=dataset_document.tenant_id,- dataset_id=dataset_document.dataset_id,- document_id=dataset_document.id,- segment_id=segment.id,- position=1,- content="child",- word_count=1,- created_by=dataset_document.created_by,- index_node_id="child-node",- index_node_hash="child-hash",- )- session.add_all([segment, child_chunk])- session.commit()-- with (- patch.object(runner, "_load") as load,- ):- runner.run_in_indexing_status(dataset_document, session)-- assert load.call_args.kwargs["documents"][0].children[0].page_content == "child"- assert load.call_args.kwargs["total_tokens"] == 12-- def test_run_in_indexing_status_uses_tokens_from_all_segments(self, mock_dependencies, sample_dataset_documents):- runner = IndexingRunner()- dataset_document = sample_dataset_documents[0]- session = mock_dependencies["session"]- dataset = session.get(Dataset, dataset_document.dataset_id)- assert dataset is not None- completed_segment = DocumentSegment(- tenant_id=dataset_document.tenant_id,- dataset_id=dataset_document.dataset_id,- document_id=dataset_document.id,- position=1,- content="",- word_count=0,- tokens=10,- created_by="account-id",- status=SegmentStatus.COMPLETED,- )- incomplete_segment = DocumentSegment(- tenant_id="tenant-id",- dataset_id=dataset_document.dataset_id,- document_id=dataset_document.id,- position=1,- content="pending",- word_count=0,- tokens=20,- created_by="account-id",- status=SegmentStatus.WAITING,- index_node_id="pending-node",- index_node_hash="pending-hash",- )- session.add_all([completed_segment, incomplete_segment])- session.commit()-- with patch.object(runner, "_load") as load:- runner.run_in_indexing_status(dataset_document, session)-- assert load.call_args.kwargs["documents"][0].page_content == "pending"- assert load.call_args.kwargs["total_tokens"] == 30-- @patch.object(Account, "set_tenant_id_with_session", autospec=True)- def test_run_success_single_document(self, set_tenant_id, mock_dependencies, sample_dataset_documents):- """Test successful run with single document."""- # Arrange- runner = IndexingRunner()- doc = sample_dataset_documents[0]- session = mock_dependencies["session"]- mock_dataset = session.get(Dataset, doc.dataset_id)- mock_current_user = session.get(Account, doc.created_by)- assert mock_dataset is not None- assert mock_current_user is not None- mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY- session.commit()-- # Mock processor- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Mock extract, transform, load- mock_processor.extract.return_value = [Document(page_content="Test content", metadata={"doc_id": "doc1"})]- mock_processor.transform.return_value = [- Document(- page_content="Chunk 1",- metadata={"doc_id": "chunk1", "doc_hash": "hash1"},- )- ]-- # Mock thread for keyword indexing- mock_thread_instance = MagicMock()- mock_dependencies["thread"].return_value = mock_thread_instance-- commits = 0-- def record_commit(_session) -> None:- nonlocal commits- commits += 1-- event.listen(session, "after_commit", record_commit)- with (- patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]) as mock_extract,- patch.object(- runner,- "_transform",- return_value=[Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})],- ) as mock_transform,- patch.object(runner, "_load_segments") as mock_load_segments,- patch.object(runner, "_load") as mock_load,- ):- # Act- runner.run([doc], session)- event.remove(session, "after_commit", record_commit)-- # Assert - verify the methods were called- mock_extract.assert_called_once()- mock_transform.assert_called_once()- mock_load_segments.assert_called_once()- mock_load.assert_called_once()- assert commits == 2- set_tenant_id.assert_called_once_with(mock_current_user, mock_dataset.tenant_id, session=session)-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Mock _extract to raise DocumentIsPausedError- with patch.object(runner, "_extract", side_effect=DocumentIsPausedError("Document paused")):- # Act & Assert- with pytest.raises(DocumentIsPausedError):- runner.run([doc], mock_dependencies["session"])-- @patch.object(Account, "set_tenant_id_with_session", autospec=True)- def test_run_counts_each_transformed_document_once(- self, set_tenant_id, mock_dependencies, sample_dataset_documents- ):- runner = IndexingRunner()- dataset_document = sample_dataset_documents[0]- session = mock_dependencies["session"]- dataset = session.get(Dataset, dataset_document.dataset_id)- current_user = session.get(Account, dataset_document.created_by)- assert dataset is not None- assert current_user is not None- transformed_documents = [- Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),- Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),- ]-- with (- patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),- patch.object(runner, "_transform", return_value=transformed_documents),- patch.object(runner, "_load_segments") as load_segments,- patch.object(runner, "_load") as load,- patch(- "core.indexing_runner.calculate_segment_token_counts",- return_value=[11, 22],- ) as calculate_token_counts,- ):- runner.run([dataset_document], mock_dependencies["session"])-- calculate_token_counts.assert_called_once_with(dataset=dataset, documents=transformed_documents)- load_segments.assert_called_once_with(- session=mock_dependencies["session"],- dataset=dataset,- dataset_document=dataset_document,- documents=transformed_documents,- token_counts=[11, 22],- )- assert load.call_args.kwargs["total_tokens"] == 33- set_tenant_id.assert_called_once_with(- current_user,- dataset.tenant_id,- session=mock_dependencies["session"],- )-- @patch.object(Account, "set_tenant_id_with_session", autospec=True)- def test_run_rejects_before_segment_or_vector_writes(- self, set_tenant_id, mock_dependencies, sample_dataset_documents- ):- runner = IndexingRunner(enforce_vector_space_admission=True)- dataset_document = sample_dataset_documents[0]- dataset_document.need_summary = False- session = mock_dependencies["session"]- dataset = session.get(Dataset, dataset_document.dataset_id)- current_user = session.get(Account, dataset_document.created_by)- assert dataset is not None- assert current_user is not None- transformed_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})]- admission_error = VectorSpaceAdmissionError("estimated storage exceeds capacity")- admission_service = Mock()- admission_service.ensure_document_can_be_indexed.side_effect = admission_error-- with (- patch("core.indexing_runner.VectorSpaceAdmissionService", return_value=admission_service),- patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),- patch.object(- runner,- "_transform",- return_value=transformed_documents,- ),- patch.object(runner, "_load_segments") as load_segments,- patch.object(runner, "_load") as load,- patch.object(runner, "_handle_indexing_error") as handle_error,- ):- runner.run([dataset_document], mock_dependencies["session"])-- load_segments.assert_not_called()- load.assert_not_called()- admission_service.ensure_document_can_be_indexed.assert_called_once_with(- dataset=dataset,- document_id=dataset_document.id,- doc_form=dataset_document.doc_form,- documents=transformed_documents,- include_summaries=False,- session=mock_dependencies["session"],- )- handle_error.assert_called_once_with(dataset_document.id, admission_error, mock_dependencies["session"])- set_tenant_id.assert_called_once_with(- current_user,- dataset.tenant_id,- session=mock_dependencies["session"],- )-- @patch.object(Account, "set_tenant_id_with_session", autospec=True)- def test_run_in_splitting_status_counts_each_transformed_document_once(- self, set_tenant_id, mock_dependencies, sample_dataset_documents- ):- runner = IndexingRunner()- dataset_document = sample_dataset_documents[0]- session = mock_dependencies["session"]- dataset = session.get(Dataset, dataset_document.dataset_id)- current_user = session.get(Account, dataset_document.created_by)- assert dataset is not None- assert current_user is not None- transformed_documents = [- Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),- Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),- ]-- with (- patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),- patch.object(runner, "_transform", return_value=transformed_documents),- patch.object(runner, "_load_segments") as load_segments,- patch.object(runner, "_load") as load,- patch(- "core.indexing_runner.calculate_segment_token_counts",- return_value=[11, 22],- ) as calculate_token_counts,- ):- runner.run_in_splitting_status(dataset_document, mock_dependencies["session"])-- calculate_token_counts.assert_called_once_with(dataset=dataset, documents=transformed_documents)- load_segments.assert_called_once_with(- session=mock_dependencies["session"],- dataset=dataset,- dataset_document=dataset_document,- documents=transformed_documents,- token_counts=[11, 22],- )- assert load.call_args.kwargs["total_tokens"] == 33- set_tenant_id.assert_called_once_with(- current_user,- dataset.tenant_id,- session=mock_dependencies["session"],- )-- def test_run_handles_provider_token_error(self, mock_dependencies, sample_dataset_documents):- """Test run handles ProviderTokenNotInitError and updates document status."""- # Arrange- runner = IndexingRunner()- doc = sample_dataset_documents[0]-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor- mock_processor.extract.side_effect = ProviderTokenNotInitError("Token not initialized")-- # Act- with patch.object(runner, "_extract", side_effect=ProviderTokenNotInitError("Token not initialized")):- runner.run([doc], mock_dependencies["session"])-- mock_dependencies["session"].refresh(doc)- assert doc.indexing_status == "error"-- def test_run_handles_object_deleted_error(self, mock_dependencies, sample_dataset_documents):- """Test run handles ObjectDeletedError gracefully."""- # Arrange- runner = IndexingRunner()- doc = sample_dataset_documents[0]-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Mock _extract to raise ObjectDeletedError- with patch.object(runner, "_extract", side_effect=ObjectDeletedError(state=None, msg="Object deleted")):- # Act- runner.run([doc], mock_dependencies["session"])-- # Assert - should not raise, just log warning- # No exception should be raised-- @patch.object(Account, "set_tenant_id_with_session", autospec=True)- def test_run_processes_multiple_documents(self, set_tenant_id, mock_dependencies, sample_dataset_documents):- """Test run processes multiple documents sequentially."""- # Arrange- runner = IndexingRunner()- docs = sample_dataset_documents- for doc in docs:- dataset = mock_dependencies["session"].get(Dataset, doc.dataset_id)- assert dataset is not None- dataset.indexing_technique = IndexTechniqueType.ECONOMY- mock_dependencies["session"].commit()-- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- # Mock thread- mock_thread_instance = MagicMock()- mock_dependencies["thread"].return_value = mock_thread_instance-- # Mock all internal methods- with (- patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]) as mock_extract,- patch.object(- runner,- "_transform",- return_value=[Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})],- ),- patch.object(runner, "_load_segments"),- patch.object(runner, "_load"),- ):- # Act- runner.run(docs, mock_dependencies["session"])-- # Assert- # Verify extract was called for each document- assert mock_extract.call_count == len(docs)- assert set_tenant_id.call_count == len(docs)---class TestIndexingRunnerRetryLogic:- """Unit tests for retry logic and error handling.-- Tests cover:- - Document pause status checking- - Document status updates- - Error state persistence- - Deleted document handling- """-- @pytest.fixture- def mock_dependencies(self, sqlite_session: Session):- """Mock all external dependencies."""- with (- patch("core.indexing_runner.redis_client") as mock_redis,- ):- yield {- "session": sqlite_session,- "redis": mock_redis,- }-- def test_check_document_paused_status_not_paused(self, mock_dependencies):- """Test document pause check when document is not paused."""- # Arrange- mock_dependencies["redis"].get.return_value = None- document_id = str(uuid.uuid4())-- # Act & Assert - should not raise- IndexingRunner._check_document_paused_status(document_id)-- def test_check_document_paused_status_is_paused(self, mock_dependencies):- """Test document pause check when document is paused."""- # Arrange- mock_dependencies["redis"].get.return_value = "1"- document_id = str(uuid.uuid4())-- # Act & Assert- with pytest.raises(DocumentIsPausedError):- IndexingRunner._check_document_paused_status(document_id)-- def test_update_document_index_status_success(self, mock_dependencies):- """Test successful document status update."""- document_id = str(uuid.uuid4())- session = mock_dependencies["session"]- document = DatasetDocument(- id=document_id,- tenant_id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- batch="batch",- name="Document",- created_from=DocumentCreatedFrom.API,- created_by=str(uuid.uuid4()),- is_paused=False,- )- session.add(document)- session.commit()-- # Act- IndexingRunner._update_document_index_status(- document_id,- "completed",- {"tokens": 100, "completed_at": naive_utc_now()},- session=session,- )-- session.refresh(document)- assert document.indexing_status == "completed"- assert document.tokens == 100-- def test_update_document_index_status_paused(self, mock_dependencies):- """Test document status update when document is paused."""- document_id = str(uuid.uuid4())- session = mock_dependencies["session"]- document = DatasetDocument(- id=document_id,- tenant_id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- batch="batch",- name="Paused document",- created_from=DocumentCreatedFrom.API,- created_by=str(uuid.uuid4()),- is_paused=True,- )- session.add(document)- session.commit()-- # Act & Assert- with pytest.raises(DocumentIsPausedError):- IndexingRunner._update_document_index_status(document_id, "completed", session=session)-- def test_update_document_index_status_deleted(self, mock_dependencies):- """Test document status update when document is deleted."""- # Arrange- document_id = str(uuid.uuid4())- # Act & Assert- with pytest.raises(DocumentIsDeletedPausedError):- IndexingRunner._update_document_index_status(document_id, "completed", session=mock_dependencies["session"])---class TestIndexingRunnerDocumentCleaning:- """Unit tests for document cleaning and preprocessing.-- Tests cover:- - Text cleaning rules- - Whitespace normalization- - Special character handling- - Custom preprocessing rules- """-- @pytest.fixture- def sample_process_rule_automatic(self):- """Create automatic processing rule."""- rule = DatasetProcessRule(- dataset_id="dataset-id",- mode="automatic",- rules=None,- created_by="account-id",- )- return rule-- @pytest.fixture- def sample_process_rule_custom(self):- """Create custom processing rule."""- rule = DatasetProcessRule(- dataset_id="dataset-id",- mode="custom",- rules=json.dumps(- {- "pre_processing_rules": [- {"id": "remove_extra_spaces", "enabled": True},- {"id": "remove_urls_emails", "enabled": True},- ]- }- ),- created_by="account-id",- )- return rule-- def test_document_clean_automatic_mode(self, sample_process_rule_automatic):- """Test document cleaning with automatic mode."""- # Arrange- text = "This is a test document with extra spaces."-- # Act- with patch("core.indexing_runner.CleanProcessor.clean") as mock_clean:- mock_clean.return_value = "This is a test document with extra spaces."- result = IndexingRunner._document_clean(text, sample_process_rule_automatic)-- # Assert- assert "extra spaces" in result- mock_clean.assert_called_once()-- def test_document_clean_custom_mode(self, sample_process_rule_custom):- """Test document cleaning with custom rules."""- # Arrange- text = "Visit https://example.com or email [email protected] for more info."-- # Act- with patch("core.indexing_runner.CleanProcessor.clean") as mock_clean:- mock_clean.return_value = "Visit or email for more info."- result = IndexingRunner._document_clean(text, sample_process_rule_custom)-- # Assert- assert "https://" not in result- assert "@" not in result- mock_clean.assert_called_once()-- def test_filter_string_removes_special_characters(self):- """Test filter_string removes special control characters."""- # Arrange- text = "Normal text\x00with\x08control\x1fcharacters\x7f"-- # Act- result = IndexingRunner.filter_string(text)-- # Assert- assert "\x00" not in result- assert "\x08" not in result- assert "\x1f" not in result- assert "\x7f" not in result- assert "Normal text" in result-- def test_filter_string_handles_unicode_fffe(self):- """Test filter_string removes Unicode U+FFFE."""- # Arrange- text = "Text with \ufffe unicode issue"-- # Act- result = IndexingRunner.filter_string(text)-- # Assert- assert "\ufffe" not in result- assert "Text with" in result-- def test_filter_string_preserves_valid_extended_characters(self):- """filter_string must keep valid printable characters like 'ï', '¿', '¾'."""- # Arrange- text = "naïve ¿Cómo? ¾ done"-- # Act- result = IndexingRunner.filter_string(text)-- # Assert- assert result == text- # The U+FFFE noncharacter is still stripped.- assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop"---class TestIndexingRunnerSplitter:- """Unit tests for text splitter configuration.-- Tests cover:- - Custom segmentation rules- - Automatic segmentation- - Chunk size validation- - Separator handling- """-- @pytest.fixture- def mock_embedding_instance(self):- """Create mock embedding model instance."""- instance = MagicMock()- instance.get_text_embedding_num_tokens.return_value = 100- return instance-- def test_get_splitter_custom_mode(self, mock_embedding_instance):- """Test splitter creation with custom mode."""- # Arrange- with patch("core.indexing_runner.FixedRecursiveCharacterTextSplitter") as mock_splitter_class:- mock_splitter = MagicMock()- mock_splitter_class.from_encoder.return_value = mock_splitter-- # Act- result = IndexingRunner._get_splitter(- processing_rule_mode="custom",- max_tokens=500,- chunk_overlap=50,- separator="\\n\\n",- embedding_model_instance=mock_embedding_instance,- )-- # Assert- assert result == mock_splitter- mock_splitter_class.from_encoder.assert_called_once()- call_kwargs = mock_splitter_class.from_encoder.call_args[1]- assert call_kwargs["chunk_size"] == 500- assert call_kwargs["chunk_overlap"] == 50- assert call_kwargs["fixed_separator"] == "\n\n"-- def test_get_splitter_automatic_mode(self, mock_embedding_instance):- """Test splitter creation with automatic mode."""- # Arrange- with patch("core.indexing_runner.EnhanceRecursiveCharacterTextSplitter") as mock_splitter_class:- mock_splitter = MagicMock()- mock_splitter_class.from_encoder.return_value = mock_splitter-- # Act- result = IndexingRunner._get_splitter(- processing_rule_mode="automatic",- max_tokens=500,- chunk_overlap=50,- separator="",- embedding_model_instance=mock_embedding_instance,- )-- # Assert- assert result == mock_splitter- mock_splitter_class.from_encoder.assert_called_once()-- def test_get_splitter_validates_max_tokens_too_small(self, mock_embedding_instance):- """Test splitter validation rejects max_tokens below minimum."""- # Act & Assert- with pytest.raises(ValueError, match="Custom segment length should be between"):- IndexingRunner._get_splitter(- processing_rule_mode="custom",- max_tokens=30, # Below minimum of 50- chunk_overlap=10,- separator="\\n",- embedding_model_instance=mock_embedding_instance,- )-- def test_get_splitter_validates_max_tokens_too_large(self, mock_embedding_instance, config_overrides):- """Test splitter validation rejects max_tokens above maximum."""- config_overrides(INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=5000)- with pytest.raises(ValueError, match="Custom segment length should be between"):- IndexingRunner._get_splitter(- processing_rule_mode="custom",- max_tokens=10000, # Above maximum- chunk_overlap=100,- separator="\\n",- embedding_model_instance=mock_embedding_instance,- )---class TestIndexingRunnerLoadSegments:- """Unit tests for segment loading and storage.-- Tests cover:- - Segment creation in database- - Child chunk handling- - Document status updates- - Word count calculation- """-- @pytest.fixture- def mock_dependencies(self, unbound_session: Session):- """Mock all external dependencies."""- with (- patch("core.indexing_runner.DatasetDocumentStore") as mock_docstore,- ):- yield {- "session": unbound_session,- "docstore": mock_docstore,- }-- @pytest.fixture- def sample_dataset(self):- """Create sample dataset."""- dataset = Dataset(- id=str(uuid.uuid4()),- tenant_id=str(uuid.uuid4()),- )- return dataset-- @pytest.fixture- def sample_dataset_document(self):- """Create sample dataset document."""- return DatasetDocument(- id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- created_by=str(uuid.uuid4()),- doc_form=IndexStructureType.PARAGRAPH_INDEX,- )-- @pytest.fixture- def sample_documents(self):- """Create sample documents."""- return [- Document(- page_content="This is chunk 1 with some content.",- metadata={"doc_id": "chunk1", "doc_hash": "hash1"},- ),- Document(- page_content="This is chunk 2 with different content.",- metadata={"doc_id": "chunk2", "doc_hash": "hash2"},- ),- ]-- def test_load_segments_paragraph_index(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test loading segments for paragraph index."""- # Arrange- runner = IndexingRunner()- mock_docstore_instance = MagicMock()- mock_dependencies["docstore"].return_value = mock_docstore_instance-- # Mock update methods to avoid database calls- with (- patch.object(runner, "_update_document_index_status"),- patch.object(runner, "_update_segments_by_document"),- ):- # Act- runner._load_segments(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- token_counts=[10, 20],- )-- # Assert- mock_dependencies["docstore"].assert_called_once_with(- dataset=sample_dataset,- user_id=sample_dataset_document.created_by,- document_id=sample_dataset_document.id,- )- mock_docstore_instance.add_documents.assert_called_once_with(- session=mock_dependencies["session"],- docs=sample_documents,- save_child=False,- token_counts=[10, 20],- )-- def test_load_segments_parent_child_index(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test loading segments for parent-child index."""- # Arrange- runner = IndexingRunner()- sample_dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX-- # Add child documents- for doc in sample_documents:- doc.children = [- ChildDocument(- page_content=f"Child of {doc.page_content}",- metadata={"doc_id": f"child_{doc.metadata['doc_id']}", "doc_hash": "child_hash"},- )- ]-- mock_docstore_instance = MagicMock()- mock_dependencies["docstore"].return_value = mock_docstore_instance-- # Mock update methods to avoid database calls- with (- patch.object(runner, "_update_document_index_status"),- patch.object(runner, "_update_segments_by_document"),- ):- # Act- runner._load_segments(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- token_counts=[10, 20],- )-- # Assert- mock_docstore_instance.add_documents.assert_called_once_with(- session=mock_dependencies["session"],- docs=sample_documents,- save_child=True,- token_counts=[10, 20],- )-- def test_load_segments_updates_word_count(- self, mock_dependencies, sample_dataset, sample_dataset_document, sample_documents- ):- """Test load segments calculates and updates word count."""- # Arrange- runner = IndexingRunner()- mock_docstore_instance = MagicMock()- mock_dependencies["docstore"].return_value = mock_docstore_instance-- # Calculate expected word count- expected_word_count = sum(len(doc.page_content.split()) for doc in sample_documents)-- # Mock update methods to avoid database calls- with (- patch.object(runner, "_update_document_index_status") as mock_update_status,- patch.object(runner, "_update_segments_by_document"),- ):- # Act- runner._load_segments(- session=mock_dependencies["session"],- dataset=sample_dataset,- dataset_document=sample_dataset_document,- documents=sample_documents,- token_counts=[10, 20],- )-- # Assert- # Verify word count was calculated correctly and passed to status update- mock_update_status.assert_called_once()- call_kwargs = mock_update_status.call_args.kwargs- assert "extra_update_params" in call_kwargs---class TestIndexingRunnerEstimate:- """Unit tests for indexing estimation.-- Tests cover:- - Token estimation- - Segment count estimation- - Batch upload limit enforcement- """-- @pytest.fixture- def mock_dependencies(self, sqlite_session: Session):- """Mock all external dependencies."""- with (- patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,- ):- yield {- "session": sqlite_session,- "factory": mock_factory,- }-- def test_indexing_estimate_respects_batch_limit(self, mock_dependencies, config_overrides):- """Test indexing estimate enforces batch upload limit."""- config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, BATCH_UPLOAD_LIMIT=10)- # Arrange- runner = IndexingRunner()- tenant_id = str(uuid.uuid4())-- extract_settings = [MagicMock() for _ in range(15)]-- with pytest.raises(ValueError, match="batch upload limit"):- runner.indexing_estimate(- tenant_id=tenant_id,- extract_settings=extract_settings,- tmp_processing_rule={"mode": "automatic", "rules": {}},- doc_form=IndexStructureType.PARAGRAPH_INDEX,- session=mock_dependencies["session"],- )-- def test_indexing_estimate_limits_qa_preview(self, mock_dependencies, config_overrides):- """Test indexing estimate returns no more than ten QA preview items."""- config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)- runner = IndexingRunner()- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor-- qa_documents = [- Document(- page_content=f"Question {index}",- metadata={"answer": f"Answer {index}"},- )- for index in range(11)- ]- mock_processor.extract.return_value = [Document(page_content="Source content")]- mock_processor.transform.return_value = qa_documents-- result = runner.indexing_estimate(- tenant_id=str(uuid.uuid4()),- extract_settings=[MagicMock()],- tmp_processing_rule=create_mock_process_rule(),- doc_form=IndexStructureType.QA_INDEX,- session=mock_dependencies["session"],- )-- assert result.total_segments == 220- assert result.qa_preview is not None- assert len(result.qa_preview) == 10- assert result.qa_preview[-1].question == "Question 9"-- def test_indexing_estimate_commits_preview_cleanup_before_summary_workers(- self, mock_dependencies, config_overrides- ):- """Test preview cleanup is visible before summary workers use independent sessions."""- config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)- runner = IndexingRunner()- tenant_id = str(uuid.uuid4())- mock_processor = MagicMock()- mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor- phase_events: list[str] = []- session = mock_dependencies["session"]- event.listen(session, "after_commit", lambda _session: phase_events.append("commit"))-- preview_doc = Document(- page_content="",- metadata={},- )- mock_processor.extract.return_value = [preview_doc]- mock_processor.transform.return_value = [preview_doc]- mock_processor.generate_summary_preview.side_effect = lambda *_args, **_kwargs: (- phase_events.append("summary") or [PreviewDetail(content=preview_doc.page_content)]- )-- image_file = UploadFile(- tenant_id=tenant_id,- storage_type=StorageType.LOCAL,- key="image_files/tenant-1/source-file-1/image.png",- name="image.png",- size=10,- extension="png",- mime_type="image/png",- created_by_role=CreatorUserRole.ACCOUNT,- created_by="user-id",- created_at=datetime.now(UTC),- used=True,- )- image_file.id = "image-1"- session.add(image_file)- session.commit()- phase_events.clear()-- with (- patch("core.indexing_runner.get_image_upload_file_ids", return_value=["image-1"]),- patch("core.indexing_runner.storage") as mock_storage,- ):- result = runner.indexing_estimate(- tenant_id=tenant_id,- extract_settings=[MagicMock()],- tmp_processing_rule={- "mode": "automatic",- "rules": {},- "summary_index_setting": {"enable": True},- },- doc_form=IndexStructureType.PARAGRAPH_INDEX,- session=session,- )-- assert result.total_segments == 1- mock_storage.delete.assert_called_once_with(image_file.key)- assert session.get(UploadFile, image_file.id) is None- assert phase_events == ["commit", "summary"]---class TestIndexingRunnerProcessChunk:- """Unit tests for chunk processing in parallel.-- Tests cover:- - Vector index creation- - Segment status updates- - Pause detection during processing- """-- @pytest.fixture- def mock_dependencies(self, sqlite_session: Session):- """Mock all external dependencies."""- with (- patch("core.indexing_runner.redis_client") as mock_redis,- ):- yield {- "session": sqlite_session,- "redis": mock_redis,- }-- @pytest.fixture- def mock_flask_app(self):- """Create mock Flask app context."""- app = MagicMock()- app.app_context.return_value.__enter__ = MagicMock()- app.app_context.return_value.__exit__ = MagicMock()- return app-- def test_process_chunk_loads_index_and_completes_segments(- self,- mock_dependencies,- mock_flask_app,- sqlite_session_factory: sessionmaker[Session],- ):- """Test process chunk loads the index and completes segments without counting tokens."""- # Arrange- from core.indexing_runner import IndexingRunner-- runner = IndexingRunner()- mock_processor = MagicMock()- chunk_documents = [- Document(page_content="Chunk 1", metadata={"doc_id": "c1"}),- Document(page_content="Chunk 2", metadata={"doc_id": "c2"}),- ]-- session = mock_dependencies["session"]- mock_dataset, mock_dataset_document = persist_indexing_scope(session)- segments = [- DocumentSegment(- tenant_id=mock_dataset.tenant_id,- dataset_id=mock_dataset.id,- document_id=mock_dataset_document.id,- position=position,- content=f"Chunk {position}",- word_count=1,- tokens=1,- created_by=mock_dataset.created_by,- index_node_id=doc.metadata["doc_id"],- status=SegmentStatus.INDEXING,- )- for position, doc in enumerate(chunk_documents, start=1)- ]- session.add_all(segments)- session.commit()-- mock_dependencies["redis"].get.return_value = None-- # Create a proper context manager mock- mock_context = MagicMock()- mock_context.__enter__ = MagicMock(return_value=None)- mock_context.__exit__ = MagicMock(return_value=None)- mock_flask_app.app_context.return_value = mock_context-- with (- patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory),- patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,- ):- mock_factory.return_value.init_index_processor.return_value = mock_processor-- # Act - the method creates its own app_context and session- result = runner._process_chunk(- mock_flask_app,- IndexStructureType.PARAGRAPH_INDEX,- chunk_documents,- mock_dataset.id,- mock_dataset_document.id,- )-- # Assert- assert result is None- mock_processor.load.assert_called_once()- session.expire_all()- assert all(session.get(DocumentSegment, segment.id).status == SegmentStatus.COMPLETED for segment in segments)-- def test_process_chunk_detects_pause(- self,- mock_dependencies,- mock_flask_app,- sqlite_session_factory: sessionmaker[Session],- ):- """Test process chunk detects document pause."""- # Arrange- from core.indexing_runner import IndexingRunner-- runner = IndexingRunner()- chunk_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1"})]-- mock_dataset, mock_dataset_document = persist_indexing_scope(mock_dependencies["session"])-- # Mock Redis to return paused status- mock_dependencies["redis"].get.return_value = "1"- # Create a proper context manager mock- mock_context = MagicMock()- mock_context.__enter__ = MagicMock(return_value=None)- mock_context.__exit__ = MagicMock(return_value=None)- mock_flask_app.app_context.return_value = mock_context-- with patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory):- # Act & Assert - the method creates its own app_context and session- with pytest.raises(DocumentIsPausedError):- runner._process_chunk(- mock_flask_app,- IndexStructureType.PARAGRAPH_INDEX,- chunk_documents,- mock_dataset.id,- mock_dataset_document.id,- )diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.pyindex 4bf883cba6d..01bc89c2831 100644--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py@@ -8,6 +8,8 @@ import pytestfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker+from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.app.file_access import FileAccessScope, bind_file_access_scope, get_current_file_access_scopefrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.models.document import Documentfrom core.rag.retrieval import dataset_retrieval as retrieval_module@@ -322,17 +324,22 @@ class TestDatasetRetrievalKnowledgeRetrieval:"format_retrieval_documents",MagicMock(return_value=[record]),)- grant_access = MagicMock()- monkeypatch.setattr(retrieval_module, "grant_retriever_segment_access", grant_access)-- with retrieval_database.session_maker() as caller_session:+ scope = FileAccessScope(+ tenant_id=dataset.tenant_id,+ user_id="visitor",+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ )+ with bind_file_access_scope(scope), retrieval_database.session_maker() as caller_session:result = retrieval.knowledge_retrieval(caller_session, _request(dataset_ids=[dataset.id]))+ granted_scope = get_current_file_access_scope()assert len(result) == 1assert result[0].metadata.dataset_id == dataset.idassert result[0].metadata.document_id == document.idassert result[0].title == document.name- grant_access.assert_called_once_with([segment.id])+ assert granted_scope is not None+ assert granted_scope.granted_retriever_segment_ids == frozenset([segment.id])def test_metadata_filtering_disabled_skips_filter_builder(self,diff --git a/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py b/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.pyindex 1c04b710806..f19950b5c12 100644--- a/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py+++ b/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py@@ -363,7 +363,11 @@ def test_single_dataset_retriever_non_economy_run_sorts_context_and_resources(sq"format_retrieval_documents",return_value=records,),- patch.object(DocumentSegment, "get_sign_content", lambda segment: segment.content.replace("raw", "signed")),+ patch.object(+ single_retriever_module,+ "sign_segment_content",+ lambda segment, **_kwargs: segment.content.replace("raw", "signed"),+ ),):result = tool.run(session=sqlite_session, query="hello")@@ -560,7 +564,11 @@ def test_multi_dataset_retriever_run_orders_segments_and_returns_resources(sqlitfake_current_app = _FakeCurrentApp()with (- patch.object(DocumentSegment, "get_sign_content", lambda segment: segment.content.replace("raw", "signed")),+ patch.object(+ multi_retriever_module,+ "sign_segment_content",+ lambda segment, **_kwargs: segment.content.replace("raw", "signed"),+ ),patch.object(tool, "_retriever", side_effect=fake_retriever) as retriever_mock,patch.object(multi_retriever_module, "current_app", fake_current_app),patch.object(multi_retriever_module.threading, "Thread", _ImmediateThread),diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_validators.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_validators.pyindex 3024db1440c..328832171e8 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_validators.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_validators.py@@ -687,7 +687,7 @@ def test_publish_validation_rejects_missing_or_out_of_scope_knowledge_datasets(captured["tenant_id"] = tenant_idreturn [], 0- import services.dataset_service as dataset_service_module+ import services.knowledge.dataset_service as dataset_service_modulemonkeypatch.setattr(dataset_service_module.DatasetService, "get_datasets_by_ids", fake_get_datasets_by_ids)diff --git a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.pyindex 235d56e9897..d6987f0d49a 100644--- a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py@@ -92,6 +92,7 @@ def test_datasource_node_delegates_to_manager_stream(mocker: MockerFixture):),graph_init_params=gp,graph_runtime_state=gs,+ datasource_credentials=lambda _tenant, _provider, _plugin, _credential: {},)evts = list(node._run())diff --git a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.pyindex a5ca12e851a..4a26d3d29e1 100644--- a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py+++ b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py@@ -2,11 +2,16 @@import timefrom collections.abc import Generator+from types import SimpleNamespacefrom typing import Never+import pytestfrom pytest_mock import MockerFixture+from sqlalchemy import Engine+from sqlalchemy.orm import sessionmakerfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.workflow.node_factory import DifyNodeFactoryfrom core.workflow.nodes.datasource.datasource_node import DatasourceNodefrom core.workflow.nodes.datasource.entities import DatasourceNodeDatafrom core.workflow.system_variables import build_system_variables@@ -16,14 +21,15 @@ from graphon.runtime import GraphRuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_params-def test_node_integration_minimal_stream(mocker: MockerFixture) -> None:[email protected]("use_factory", [False, True])+def test_node_integration_minimal_stream(mocker: MockerFixture, sqlite_engine: Engine, use_factory: bool) -> None:variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(datasource_type="online_document",datasource_info={"workspace_id": "w","page": {"page_id": "pg", "type": "t"},- "credential_id": "",+ "credential_id": "credential-1",},))@@ -43,7 +49,9 @@ def test_node_integration_minimal_stream(mocker: MockerFixture) -> None:return "icon"@classmethod- def stream_node_events(cls, **_: object) -> Generator[StreamCompletedEvent, None, None]:+ def stream_node_events(cls, **kwargs: object) -> Generator[StreamCompletedEvent, None, None]:+ assert kwargs["tenant_id"] == "t1"+ assert kwargs["credentials"] == {"integration_secret": "token"}yield from ()yield StreamCompletedEvent(node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED))@@ -53,20 +61,36 @@ def test_node_integration_minimal_stream(mocker: MockerFixture) -> None:mocker.patch("core.workflow.nodes.datasource.datasource_node.DatasourceManager", new=_Mgr)- node = DatasourceNode(- node_id="n",- data=DatasourceNodeData(- type="datasource",- version="1",- title="Datasource",- provider_type="plugin",- provider_name="p",- plugin_id="plug",- datasource_name="ds",- ),- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ load_credentials = mocker.Mock(return_value={"integration_secret": "token"})+ node_data = DatasourceNodeData(+ type="datasource",+ version="1",+ title="Datasource",+ provider_type="plugin",+ provider_name="p",+ plugin_id="plug",+ datasource_name="ds",)+ if use_factory:+ database_client = sessionmaker(sqlite_engine)+ mocker.patch("core.workflow.node_factory.get_session_maker", return_value=database_client)+ build_credentials = mocker.patch(+ "core.workflow.node_factory.build_data_source_credentials",+ return_value=SimpleNamespace(providers=SimpleNamespace(get_datasource_credentials=load_credentials)),+ )+ factory = DifyNodeFactory(graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state)+ node = factory.create_node({"id": "n", "data": node_data.model_dump()})+ build_credentials.assert_called_once_with(database_client=database_client)+ assert isinstance(node, DatasourceNode)+ else:+ node = DatasourceNode(+ node_id="n",+ data=node_data,+ graph_init_params=graph_init_params,+ graph_runtime_state=graph_runtime_state,+ datasource_credentials=load_credentials,+ )out = list(node._run())assert isinstance(out[-1], StreamCompletedEvent)+ load_credentials.assert_called_once_with("t1", "p", "plug", "credential-1")diff --git a/api/tests/unit_tests/events/event_handlers/test_create_document_index.py b/api/tests/unit_tests/events/event_handlers/test_create_document_index.pyindex fad8932f9e2..c381796ff6b 100644--- a/api/tests/unit_tests/events/event_handlers/test_create_document_index.py+++ b/api/tests/unit_tests/events/event_handlers/test_create_document_index.py@@ -1,13 +1,15 @@import logging+from collections.abc import Sequencefrom unittest.mock import MagicMock, patchimport pytestfrom sqlalchemy.orm import Session-from core.indexing_runner import DocumentIsPausedErrorfrom events.event_handlers import create_document_index as handler_modulefrom models.dataset import Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DocumentRef@pytest.fixture@@ -45,7 +47,7 @@ def test_handle_logs_document_pause() -> None:mock_indexing_runner.run.side_effect = DocumentIsPausedError("Document is paused")- with patch.object(handler_module, "IndexingRunner", return_value=mock_indexing_runner):+ with patch.object(handler_module, "build_document_indexing_service", return_value=mock_indexing_runner):with caplog.at_level(logging.INFO, logger=handler_module.logger.name):handler_module.handle("dataset-1", document_ids=["doc-1"])@@ -64,7 +66,7 @@ def test_handle_logs_unexpected_indexing_errors() -> None:mock_indexing_runner.run.side_effect = RuntimeError("Indexing failed")- with patch.object(handler_module, "IndexingRunner", return_value=mock_indexing_runner):+ with patch.object(handler_module, "build_document_indexing_service", return_value=mock_indexing_runner):with caplog.at_level(logging.ERROR, logger=handler_module.logger.name):handler_module.handle("dataset-1", document_ids=["doc-1"])@@ -83,12 +85,16 @@ def test_handle_runs_indexing_on_success(mock_indexing_runner: MagicMock,caplog: pytest.LogCaptureFixture,) -> None:- def assert_status_committed(_documents: list[Document], session: Session) -> None:- assert not session.in_transaction()+ def assert_status_committed(refs: Sequence[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as observer:+ row = observer.get(Document, refs[0].document_id)+ assert row is not None+ assert row.indexing_status == IndexingStatus.PARSING+ assert row.processing_started_at is not Nonemock_indexing_runner.run.side_effect = assert_status_committed- with patch.object(handler_module, "IndexingRunner", return_value=mock_indexing_runner):+ with patch.object(handler_module, "build_document_indexing_service", return_value=mock_indexing_runner):with caplog.at_level(logging.INFO, logger=handler_module.logger.name):handler_module.handle("dataset-1", document_ids=["doc-1"])diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.pyindex 4ad63dcc7f5..1362d844e47 100644--- a/api/tests/unit_tests/extensions/test_ext_application_services.py+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py@@ -41,6 +41,7 @@ from models.model import (Site,TrialApp,)+from models.provider import ProviderCredentialfrom repositories.account_activation_repository import SQLAlchemyAccountActivationRepositoryfrom repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepositoryfrom repositories.account_oauth_repository import (@@ -96,10 +97,14 @@ from services.app_site_service import AppSiteServicefrom services.app_tracing_config_gateway import OpsTraceManagerGatewayfrom services.app_tracing_config_service import AppTracingConfigServicefrom services.audio_types import AudioAppRef, AudioOutput, AudioUpload-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthServicefrom services.billing_portal_service import BillingPortalServicefrom services.billing_service import BillingServicefrom services.compliance_download_service import ComplianceDownloadService+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService+from services.data_source.binding_application_service import DataSourceBindingApplicationService+from services.data_source.credential_gateway import ActorAwareDatasourceCredentialGateway+from services.data_source.notion_import_adapters import PluginNotionSourceGateway+from services.data_source.notion_import_application_service import NotionImportApplicationServicefrom services.errors.enterprise import EnterpriseAPIError, EnterpriseAPINotFoundError, EnterpriseServiceErrorfrom services.file_service import FileServicefrom services.human_input_file_upload_service import HumanInputFileUploadService@@ -108,6 +113,15 @@ from services.installed_app_access_service import InstalledAppAccessDeniedError,from services.installed_app_generation_adapters import AppGenerateServiceRuntime as InstalledAppGenerateServiceRuntimefrom services.installed_app_generation_service import InstalledAppGenerationServicefrom services.knowledge.api_key_service import DatasetApiKeyService+from services.knowledge.dataset_access import DatasetAccessService+from services.knowledge.datasets.application import DatasetApplicationService+from services.knowledge.document_sync import DocumentSyncApplicationService+from services.knowledge.documents.application import DatasetDocumentApplicationService+from services.knowledge.external.application import ExternalKnowledgeApplicationService+from services.knowledge.indexing.adapters.estimate import IndexingEstimateAdapter, SQLAlchemyProcessRuleReader+from services.knowledge.indexing.adapters.sources import NotionSourceResolver+from services.knowledge.indexing.estimate import IndexingEstimateApplicationService+from services.knowledge.segments.application import DatasetSegmentApplicationServicefrom services.message_file_preview_service import MessageFilePreviewServicefrom services.oauth_device_application_service import OAuthDeviceApplicationServicefrom services.partner_tenant_binding_service import PartnerTenantBindingService@@ -239,6 +253,7 @@ def test_build_application_services_preserves_composed_boundaries(assert isinstance(services.app_api_keys, AppApiKeyService)assert isinstance(services.dataset_api_keys, DatasetApiKeyService)+ assert services.dataset_api_keys._access is services.knowledge.datasets._dataset_accessassert isinstance(services.oauth_device, OAuthDeviceApplicationService)assert register_script.call_count == 3@@ -768,7 +783,73 @@ def test_build_application_services_wires_data_source_api_key_auth(redis=_redis(),)- assert isinstance(services.data_source_api_key_auth, DataSourceApiKeyAuthService)+ assert isinstance(services.data_sources.api_key_auth, DataSourceApiKeyAuthService)+++def test_build_application_services_groups_dataset_services_and_reuses_repositories(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ services = ext_application_services.build_application_services(+ database_client=sqlite_session_factory,+ deployment_edition=DeploymentEdition.COMMUNITY,+ initialization_password="",+ redis=MagicMock(spec=RedisClientWrapper),+ )++ assert isinstance(services.data_sources.bindings, DataSourceBindingApplicationService)+ assert isinstance(services.data_sources.notion_imports, NotionImportApplicationService)+ assert isinstance(services.knowledge.document_sync, DocumentSyncApplicationService)+ assert isinstance(services.knowledge.documents, DatasetDocumentApplicationService)+ assert isinstance(services.knowledge.datasets, DatasetApplicationService)+ assert isinstance(services.knowledge.external, ExternalKnowledgeApplicationService)+ assert services.knowledge.datasets._dataset_access is services.knowledge.document_sync._dataset_access+ assert services.knowledge.external._dataset_access is services.knowledge.document_sync._dataset_access+ assert services.knowledge.documents._dataset_access is services.knowledge.document_sync._dataset_access+ assert isinstance(services.knowledge.indexing_estimates, IndexingEstimateApplicationService)+ assert isinstance(services.knowledge.segments, DatasetSegmentApplicationService)+ assert services.data_sources.bindings._bindings is services.data_sources.oauth["notion"]._bindings+ assert services.data_sources.notion_imports._dataset_access is services.knowledge.document_sync._dataset_access+ assert services.data_sources.notion_imports._dataset_access is services.knowledge.indexing_estimates._dataset_access+ assert services.data_sources.notion_imports._documents is services.knowledge.document_sync._documents+ assert services.data_sources.notion_imports._documents is services.knowledge.indexing_estimates._documents+ notion_source = services.data_sources.notion_imports._source+ estimate_service = services.knowledge.indexing_estimates+ dataset_access = services.data_sources.notion_imports._dataset_access+ assert isinstance(notion_source, PluginNotionSourceGateway)+ assert isinstance(estimate_service._runner, IndexingEstimateAdapter)+ assert isinstance(estimate_service._process_rules, SQLAlchemyProcessRuleReader)+ assert isinstance(dataset_access, DatasetAccessService)+ notion_resolver = estimate_service._notion+ assert isinstance(notion_resolver, NotionSourceResolver)+ actor_credentials = notion_source._credentials+ assert isinstance(actor_credentials, ActorAwareDatasourceCredentialGateway)+ assert actor_credentials is notion_resolver._actor_credentials+ assert services.data_sources.providers._credentials is actor_credentials._credentials+ assert services.knowledge.pipeline_generator._datasource_providers is services.data_sources.providers+ assert notion_resolver._stored_credentials is not notion_resolver._actor_credentials+ assert services.workspace_member_queries._members is dataset_access._workspace_roles+++def test_build_application_services_wires_credential_query(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ services = ext_application_services.build_application_services(+ database_client=sqlite_session_factory,+ deployment_edition=DeploymentEdition.COMMUNITY,+ initialization_password="",+ redis=MagicMock(spec=RedisClientWrapper),+ )+ tenant_id, actor_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ credential = ProviderCredential(+ tenant_id=tenant_id, provider_name="openai", credential_name="Team", encrypted_config="encrypted"+ )+ session.add(credential)+ credential_id = credential.id++ records = services.credential_queries.list_models(workspace_id=tenant_id, provider="openai", actor_id=actor_id)++ assert [(record.id, record.name) for record in records] == [(credential_id, "Team")]def test_build_application_services_uses_supplied_redis_for_both_workflow_stop_signals(@@ -851,6 +932,19 @@ def installed_app_ref(sqlite_session_factory: sessionmaker[Session]) -> Installereturn result+def test_build_application_services_reuses_installed_app_generation_dependencies(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ services = ext_application_services.build_application_services(+ database_client=sqlite_session_factory,+ deployment_edition=DeploymentEdition.COMMUNITY,+ initialization_password="",+ redis=MagicMock(spec=RedisClientWrapper),+ )++ assert services.installed_apps.access._installed_apps is services.installed_apps.generation._usage++@pytest.mark.parametrize(("deployment_edition", "permission_result"),[diff --git a/api/tests/unit_tests/fields/test_dataset_fields.py b/api/tests/unit_tests/fields/test_dataset_fields.pyindex ecde29d57fe..02f0692079b 100644--- a/api/tests/unit_tests/fields/test_dataset_fields.py+++ b/api/tests/unit_tests/fields/test_dataset_fields.py@@ -2,16 +2,12 @@ import jsonfrom collections.abc import Iteratorfrom contextlib import contextmanager-import pytestfrom sqlalchemy import eventfrom sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom fields.dataset_fields import (- DatasetDetailPrefetch,DatasetDetailResponse,- build_dataset_detail_prefetch,- dataset_detail_response_source,)from models.account import Accountfrom models.dataset import (@@ -25,6 +21,7 @@ from models.dataset import ()from models.enums import DatasetMetadataType, DataSourceType, DocumentCreatedFrom, TagTypefrom models.model import App, AppMode, IconType, Tag, TagBinding+from services.knowledge.dataset_read_service import load_dataset_detail, load_dataset_detailsdef _dataset_detail_payload(**overrides):@@ -207,8 +204,7 @@ def test_dataset_detail_expands_missing_weighted_score_nested_fields():}[email protected]("sqlite_session", [(Dataset, Account, App, AppDatasetJoin)], indirect=True)-def test_dataset_detail_response_source_uses_caller_session_for_database_fields(sqlite_session: Session):+def test_dataset_detail_serializes_after_session_closes(sqlite_session: Session) -> None:account = Account(name="Ada", email="[email protected]")account.id = "account-1"dataset = Dataset(@@ -247,10 +243,10 @@ def test_dataset_detail_response_source_uses_caller_session_for_database_fields(sqlite_session.add_all([account, dataset, decoy_app, decoy_join])sqlite_session.flush()- response = DatasetDetailResponse.model_validate(- dataset_detail_response_source(dataset, session=sqlite_session),- from_attributes=True,- )+ detail = load_dataset_detail(dataset, session=sqlite_session)+ sqlite_session.expire_all()+ sqlite_session.close()+ response = DatasetDetailResponse.model_validate(detail)assert response.app_count == 0assert response.document_count == 0@@ -390,11 +386,8 @@ def _seed_dataset_page(session: Session) -> list[Dataset]:return datasets-def _dump(dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None) -> dict[str, object]:- return DatasetDetailResponse.model_validate(- dataset_detail_response_source(dataset, session=session, prefetch=prefetch),- from_attributes=True,- ).model_dump(mode="json")+def _dump(dataset: Dataset, *, session: Session) -> dict[str, object]:+ return DatasetDetailResponse.model_validate(load_dataset_detail(dataset, session=session)).model_dump(mode="json")@contextmanager@@ -419,8 +412,10 @@ def test_dataset_detail_prefetch_matches_per_dataset_lookups(sqlite_session: Sesexpected = [_dump(dataset, session=sqlite_session) for dataset in datasets]- prefetch = build_dataset_detail_prefetch(datasets, session=sqlite_session)- prefetched = [_dump(dataset, session=sqlite_session, prefetch=prefetch) for dataset in datasets]+ details = load_dataset_details(datasets, session=sqlite_session)+ sqlite_session.expire_all()+ sqlite_session.close()+ prefetched = [DatasetDetailResponse.model_validate(detail).model_dump(mode="json") for detail in details]assert prefetched == expected@@ -452,9 +447,8 @@ def test_dataset_detail_prefetch_keeps_query_count_independent_of_page_size(sqlidef select_count(page: list[Dataset]) -> int:with _count_selects(sqlite_session) as statements:- prefetch = build_dataset_detail_prefetch(page, session=sqlite_session)- for dataset in page:- _dump(dataset, session=sqlite_session, prefetch=prefetch)+ for detail in load_dataset_details(page, session=sqlite_session):+ DatasetDetailResponse.model_validate(detail).model_dump(mode="json")return len(statements)small_page = select_count(datasets[:2])diff --git a/api/tests/unit_tests/fields/test_document_fields.py b/api/tests/unit_tests/fields/test_document_fields.pyindex c097de67b65..7f4e82a7583 100644--- a/api/tests/unit_tests/fields/test_document_fields.py+++ b/api/tests/unit_tests/fields/test_document_fields.py@@ -1,18 +1,17 @@import jsonfrom datetime import datetime-import pytestfrom sqlalchemy.orm import Sessionfrom extensions.storage.storage_type import StorageType-from fields.document_fields import DocumentWithSession+from fields.document_fields import DocumentResponsefrom models.dataset import Document, DocumentSegmentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFromfrom models.model import UploadFile+from services.knowledge.dataset_read_service import load_document_details[email protected]("sqlite_session", [(Document, DocumentSegment, UploadFile)], indirect=True)-def test_document_with_session_uses_explicit_getters(sqlite_session: Session) -> None:+def test_document_detail_serializes_after_session_closes(sqlite_session: Session) -> None:upload = UploadFile(tenant_id="tenant-1",storage_type=StorageType.LOCAL,@@ -52,8 +51,11 @@ def test_document_with_session_uses_explicit_getters(sqlite_session: Session) ->)sqlite_session.add_all([upload, document, segment])sqlite_session.flush()- source = DocumentWithSession(document=document, session=sqlite_session)+ source = load_document_details([document], session=sqlite_session)[0]+ sqlite_session.expire_all()+ sqlite_session.close()+ response = DocumentResponse.model_validate(source).model_dump(mode="json")- assert source.data_source_detail_dict["upload_file"]["name"] == "source.txt"- assert source.hit_count == 3- assert source.doc_metadata_details is None+ assert response["data_source_detail_dict"]["upload_file"]["name"] == "source.txt"+ assert response["hit_count"] == 3+ assert response["doc_metadata"] == []diff --git a/api/tests/unit_tests/fields/test_segment_fields.py b/api/tests/unit_tests/fields/test_segment_fields.pynew file mode 100644index 00000000000..71beaa42fc0--- /dev/null+++ b/api/tests/unit_tests/fields/test_segment_fields.py@@ -0,0 +1,87 @@+import json+from datetime import datetime++from sqlalchemy.orm import Session++from extensions.storage.storage_type import StorageType+from fields.segment_fields import SegmentResponse+from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment, SegmentAttachmentBinding+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, ProcessRuleMode+from models.model import UploadFile+from services.knowledge.dataset_read_service import load_segment_details+++def test_segment_detail_serializes_nested_relations_after_session_closes(sqlite_session: Session) -> None:+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.HIERARCHICAL,+ rules=json.dumps({"parent_mode": "paragraph"}),+ created_by="account-1",+ )+ document = Document(+ id="document-1",+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ dataset_process_rule_id=rule.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name="source.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by="account-1",+ )+ segment = DocumentSegment(+ tenant_id=document.tenant_id,+ dataset_id=document.dataset_id,+ document_id=document.id,+ position=1,+ content="hello world",+ word_count=2,+ tokens=2,+ created_by="account-1",+ )+ chunk = ChildChunk(+ tenant_id=document.tenant_id,+ dataset_id=document.dataset_id,+ document_id=document.id,+ segment_id=segment.id,+ position=1,+ content="hello",+ word_count=1,+ created_by="account-1",+ )+ attachment = UploadFile(+ tenant_id=document.tenant_id,+ storage_type=StorageType.LOCAL,+ key="documents/image.png",+ name="image.png",+ size=12,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime(2024, 1, 1),+ used=True,+ )+ binding = SegmentAttachmentBinding(+ tenant_id=document.tenant_id,+ dataset_id=document.dataset_id,+ document_id=document.id,+ segment_id=segment.id,+ attachment_id=attachment.id,+ )+ sqlite_session.add_all([rule, document, segment, chunk, attachment, binding])+ sqlite_session.flush()+ details = load_segment_details([segment], {segment.id: "A summary"}, session=sqlite_session)++ # Expire even nested ORM objects so a retained proxy cannot pass by using cached attributes.+ sqlite_session.expire_all()+ sqlite_session.close()+ response = SegmentResponse.model_validate(details[0]).model_dump(mode="json")++ assert response["summary"] == "A summary"+ assert response["sign_content"] == "hello world"+ assert response["child_chunks"][0]["content"] == "hello"+ assert isinstance(response["child_chunks"][0]["created_at"], int)+ assert response["attachments"][0]["name"] == "image.png"+ assert "&sign=" in response["attachments"][0]["source_url"]diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.pyindex 77e36f10f63..e2fa9ead2d4 100644--- a/api/tests/unit_tests/models/test_dataset_models.py+++ b/api/tests/unit_tests/models/test_dataset_models.py@@ -1,3 +1,5 @@+from repositories.knowledge.keyword_table_repository import load_keyword_table+"""Comprehensive unit tests for Dataset models.@@ -12,7 +14,7 @@ This test suite covers:import jsonimport picklefrom datetime import UTC, datetime-from unittest.mock import patch+from unittest.mock import Mock, patchfrom urllib.parse import parse_qs, urlparsefrom uuid import uuid4@@ -46,6 +48,37 @@ from models.enums import (SegmentStatus,)from models.model import App, AppMode, IconType, UploadFile+from repositories.knowledge.dataset_read_repository import (+ get_child_dataset,+ get_child_document,+ get_child_segment,+ get_dataset_app_count,+ get_dataset_author_name,+ get_dataset_available_document_count,+ get_dataset_creator,+ get_dataset_doc_form,+ get_dataset_doc_metadata,+ get_dataset_document_count,+ get_dataset_is_published,+ get_dataset_keyword_table,+ get_dataset_tags,+ get_dataset_word_count,+ get_document_dataset,+ get_document_hit_count,+ get_document_process_rule,+ get_document_segment_count,+ get_external_api_dataset_bindings,+ get_external_knowledge_info,+ get_joined_app,+ get_latest_dataset_process_rule,+ get_next_segment,+ get_previous_segment,+ get_segment_child_chunks,+ get_segment_dataset,+ get_segment_document,+)+from repositories.knowledge.segment_read_adapter import get_segment_attachments+from services.knowledge.dataset_read_service import get_dataset_queriesfrom tests.unit_tests.config_override import apply_config_overrides@@ -186,9 +219,9 @@ class TestDatasetModelValidation:sqlite_session.add_all([dataset, account, process_rule, document])sqlite_session.flush()- assert dataset.get_created_by_account(session=sqlite_session) is account- assert dataset.get_latest_process_rule(session=sqlite_session) is process_rule- assert dataset.get_doc_form(session=sqlite_session) == IndexStructureType.PARAGRAPH_INDEX+ assert get_dataset_creator(dataset, session=sqlite_session) is account+ assert get_latest_dataset_process_rule(dataset, session=sqlite_session) is process_rule+ assert get_dataset_doc_form(dataset, session=sqlite_session) == IndexStructureType.PARAGRAPH_INDEX@pytest.mark.parametrize("sqlite_session", [(Dataset, Document)], indirect=True)def test_get_doc_form_ignores_foreign_tenant_document(self, sqlite_session: Session) -> None:@@ -201,7 +234,7 @@ class TestDatasetModelValidation:sqlite_session.add_all([dataset, foreign_document])sqlite_session.flush()- assert dataset.get_doc_form(session=sqlite_session) is None+ assert get_dataset_doc_form(dataset, session=sqlite_session) is None@pytest.mark.parametrize("sqlite_session", [(Dataset, DatasetKeywordTable)], indirect=True)def test_get_dataset_keyword_table_uses_caller_session(self, sqlite_session: Session):@@ -213,7 +246,7 @@ class TestDatasetModelValidation:sqlite_session.add_all([dataset, keyword_table])sqlite_session.flush()- result = dataset.get_dataset_keyword_table(session=sqlite_session)+ result = get_dataset_keyword_table(dataset, session=sqlite_session)assert result is keyword_table@@ -250,17 +283,17 @@ class TestDatasetModelValidation:)sqlite_session.flush()- assert dataset.get_total_documents(session=sqlite_session) == 2- assert dataset.get_total_available_documents(session=sqlite_session) == 1- assert dataset.get_app_count(session=sqlite_session) == 1- assert dataset.get_document_count(session=sqlite_session) == 2- assert dataset.get_word_count(session=sqlite_session) == 500- assert dataset.get_author_name(session=sqlite_session) == "Ada"- assert dataset.get_tags(session=sqlite_session) == []- assert dataset.get_doc_form(session=sqlite_session) == IndexStructureType.PARAGRAPH_INDEX- assert dataset.get_external_knowledge_info(session=sqlite_session) is None- assert dataset.get_doc_metadata(session=sqlite_session) == []- assert dataset.get_is_published(session=sqlite_session) is False+ assert get_dataset_document_count(dataset, session=sqlite_session) == 2+ assert get_dataset_available_document_count(dataset, session=sqlite_session) == 1+ assert get_dataset_app_count(dataset, session=sqlite_session) == 1+ assert get_dataset_document_count(dataset, session=sqlite_session) == 2+ assert get_dataset_word_count(dataset, session=sqlite_session) == 500+ assert get_dataset_author_name(dataset, session=sqlite_session) == "Ada"+ assert get_dataset_tags(dataset, session=sqlite_session) == []+ assert get_dataset_doc_form(dataset, session=sqlite_session) == IndexStructureType.PARAGRAPH_INDEX+ assert get_external_knowledge_info(dataset, session=sqlite_session) is None+ assert get_dataset_doc_metadata(dataset, session=sqlite_session) == []+ assert get_dataset_is_published(dataset, session=sqlite_session) is Falsedef test_dataset_indexing_technique_validation(self):"""Test dataset indexing technique values."""@@ -387,7 +420,7 @@ class TestDatasetModelValidation:sqlite_session.add_all([dataset, external_api, binding])sqlite_session.flush()- assert dataset.get_external_knowledge_info(session=sqlite_session) is None+ assert get_external_knowledge_info(dataset, session=sqlite_session) is None@pytest.mark.parametrize("sqlite_session", [(ExternalKnowledgeApis, ExternalKnowledgeBindings, Dataset)], indirect=True@@ -433,7 +466,7 @@ class TestDatasetModelValidation:)sqlite_session.flush()- result = external_api.get_dataset_bindings(session=sqlite_session)+ result = get_external_api_dataset_bindings(external_api, session=sqlite_session)assert result == [{"id": dataset.id, "name": dataset.name}]@@ -461,11 +494,11 @@ class TestDatasetModelValidation:used=False,)upload_file.id = "file-1"- sqlite_session.add_all([dataset_query, upload_file])+ sqlite_session.add_all([_make_dataset(dataset_id=dataset_query.dataset_id), dataset_query, upload_file])sqlite_session.flush()- with patch("models.dataset.sign_upload_file_preview_url", return_value="signed-url"):- queries = dataset_query.get_queries(session=sqlite_session)+ with patch("services.knowledge.dataset_read_service.sign_upload_file_preview_url", return_value="signed-url"):+ queries = get_dataset_queries(dataset_query, session=sqlite_session)assert queries == [{@@ -604,9 +637,9 @@ class TestDocumentModelRelationships:sqlite_session.add_all([process_rule, document, *segments])sqlite_session.flush()- assert document.get_dataset_process_rule(session=sqlite_session) is process_rule- assert document.get_segment_count(session=sqlite_session) == 3- assert document.get_hit_count(session=sqlite_session) == 7+ assert get_document_process_rule(document, session=sqlite_session) is process_rule+ assert get_document_segment_count(document, session=sqlite_session) == 3+ assert get_document_hit_count(document, session=sqlite_session) == 7def test_document_display_status_queuing(self):"""Test document display_status property for queuing state."""@@ -818,7 +851,59 @@ class TestDocumentModelRelationships:sqlite_session.add_all([dataset, document])sqlite_session.flush()- assert document.get_dataset(session=sqlite_session) is dataset+ assert get_document_dataset(document, session=sqlite_session) is dataset++ @pytest.mark.parametrize("sqlite_session", [(Document, DocumentSegment)], indirect=True)+ def test_document_average_segment_length(+ self,+ sqlite_session: Session,+ ):+ """Test average_segment_length property calculation."""+ # Arrange+ document = Document(+ tenant_id=str(uuid4()),+ dataset_id=str(uuid4()),+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch_001",+ name="test.pdf",+ created_from=DocumentCreatedFrom.WEB,+ created_by=str(uuid4()),+ word_count=1000,+ )+ sqlite_session.add(document)+ sqlite_session.flush()+ sqlite_session.add_all(_make_segments(document, [0] * 10))+ sqlite_session.commit()++ # Act+ result = document.to_dict(+ segment_count=get_document_segment_count(document, session=sqlite_session), hit_count=0, process_rule=None+ )["average_segment_length"]++ # Assert+ assert result == 100++ def test_document_average_segment_length_zero(self):+ """Test average_segment_length property when word_count is zero."""+ # Arrange+ document = Document(+ tenant_id=str(uuid4()),+ dataset_id=str(uuid4()),+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch_001",+ name="test.pdf",+ created_from=DocumentCreatedFrom.WEB,+ created_by=str(uuid4()),+ word_count=0,+ )++ # Act+ result = document.to_dict(segment_count=0, hit_count=0, process_rule=None)["average_segment_length"]++ # Assert+ assert result == 0class TestDocumentSegmentIndexing:@@ -858,7 +943,7 @@ class TestDocumentSegmentIndexing:sqlite_session.add_all([process_rule, document, segment, child_chunk])sqlite_session.flush()- result = segment.get_child_chunks(session=sqlite_session)+ result = get_segment_child_chunks(segment, session=sqlite_session)assert result == [child_chunk]@pytest.mark.parametrize(@@ -895,8 +980,8 @@ class TestDocumentSegmentIndexing:sqlite_session.add_all([process_rule, document, segment, child_chunk])sqlite_session.flush()- result = segment.get_child_chunks(session=sqlite_session)- response_result = segment.get_child_chunks(session=sqlite_session, include_full_doc=False)+ result = get_segment_child_chunks(segment, session=sqlite_session)+ response_result = get_segment_child_chunks(segment, session=sqlite_session, include_full_doc=False)assert result == [child_chunk]assert response_result == []@@ -917,8 +1002,8 @@ class TestDocumentSegmentIndexing:sqlite_session.add_all([dataset, document, segment])sqlite_session.flush()- assert segment.get_dataset(session=sqlite_session) is dataset- assert segment.get_document(session=sqlite_session) is document+ assert get_segment_dataset(segment, session=sqlite_session) is dataset+ assert get_segment_document(segment, session=sqlite_session) is documentdef test_document_segment_creation_with_required_fields(self):"""Test creating a document segment with all required fields."""@@ -1116,8 +1201,8 @@ class TestDocumentSegmentIndexing:sqlite_session.add_all([segment, attachment, binding])sqlite_session.flush()- monkeypatch.setattr("models.dataset.time.time", lambda: 1700000000)- monkeypatch.setattr("models.dataset.os.urandom", lambda _: b"\x01" * 16)+ monkeypatch.setattr("repositories.knowledge.segment_read_adapter.time.time", lambda: 1700000000)+ monkeypatch.setattr("repositories.knowledge.segment_read_adapter.os.urandom", lambda _: b"\x01" * 16)apply_config_overrides(monkeypatch,SECRET_KEY="unit-secret",@@ -1126,7 +1211,7 @@ class TestDocumentSegmentIndexing:)# Act- attachments = segment.get_attachments(session=sqlite_session)+ attachments = get_segment_attachments(segment, session=sqlite_session)# Assertassert len(attachments) == 1@@ -1376,7 +1461,13 @@ class TestDatasetKeywordTable:sqlite_session.add_all([dataset, keyword_table])sqlite_session.flush()- result = keyword_table.get_keyword_table_dict(session=sqlite_session)+ result = load_keyword_table(+ storage=Mock(),+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ storage_type=keyword_table.data_source_type,+ data=keyword_table.keyword_table,+ )assert result == {"__data__": {"table": {"keyword": {"node-1"}}}}@@ -1519,6 +1610,50 @@ class TestModelIntegration:assert document.word_count == 100assert segment.status == SegmentStatus.COMPLETED+ @pytest.mark.parametrize("sqlite_session", [(Document, DocumentSegment)], indirect=True)+ def test_document_to_dict_serialization(+ self,+ sqlite_session: Session,+ ):+ """Test document to_dict method for serialization."""+ # Arrange+ tenant_id = str(uuid4())+ dataset_id = str(uuid4())+ created_by = str(uuid4())++ document = Document(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch_001",+ name="test.pdf",+ created_from=DocumentCreatedFrom.WEB,+ created_by=created_by,+ word_count=100,+ indexing_status=IndexingStatus.COMPLETED,+ )+ sqlite_session.add(document)+ sqlite_session.flush()+ sqlite_session.add_all(_make_segments(document, [2, 2, 2, 2, 2]))+ sqlite_session.commit()++ # Act+ result = document.to_dict(+ segment_count=get_document_segment_count(document, session=sqlite_session),+ hit_count=get_document_hit_count(document, session=sqlite_session),+ process_rule=get_document_process_rule(document, session=sqlite_session),+ )++ # Assert+ assert result["tenant_id"] == tenant_id+ assert result["dataset_id"] == dataset_id+ assert result["name"] == "test.pdf"+ assert result["word_count"] == 100+ assert result["indexing_status"] == IndexingStatus.COMPLETED+ assert result["segment_count"] == 5+ assert result["hit_count"] == 10+class TestChildChunkSessionAccessors:"""Regression coverage for the ``ChildChunk`` accessors refactored to take a caller-provided session.@@ -1544,9 +1679,9 @@ class TestChildChunkSessionAccessors:sqlite_session.add_all([dataset, document, segment, child_chunk])sqlite_session.flush()- assert child_chunk.dataset(session=sqlite_session) is dataset- assert child_chunk.document(session=sqlite_session) is document- assert child_chunk.segment(session=sqlite_session) is segment+ assert get_child_dataset(child_chunk, session=sqlite_session) is dataset+ assert get_child_document(child_chunk, session=sqlite_session) is document+ assert get_child_segment(child_chunk, session=sqlite_session) is segmentdef test_accessors_return_none_when_related_rows_are_absent(self, sqlite_session: Session):child_chunk = ChildChunk(@@ -1560,9 +1695,9 @@ class TestChildChunkSessionAccessors:created_by="account-1",)- assert child_chunk.dataset(session=sqlite_session) is None- assert child_chunk.document(session=sqlite_session) is None- assert child_chunk.segment(session=sqlite_session) is None+ assert get_child_dataset(child_chunk, session=sqlite_session) is None+ assert get_child_document(child_chunk, session=sqlite_session) is None+ assert get_child_segment(child_chunk, session=sqlite_session) is Noneclass TestDatasetAvailableDocumentCount:@@ -1590,7 +1725,7 @@ class TestDatasetAvailableDocumentCount:sqlite_session.add_all([dataset, completed, waiting])sqlite_session.flush()- assert dataset.get_available_document_count(session=sqlite_session) == 1+ assert get_dataset_available_document_count(dataset, session=sqlite_session) == 1class TestDocumentSegmentNeighborAccessors:@@ -1609,8 +1744,8 @@ class TestDocumentSegmentNeighborAccessors:sqlite_session.flush()middle_segment = segments[1]- assert middle_segment.previous_segment(session=sqlite_session) is segments[0]- assert middle_segment.next_segment(session=sqlite_session) is segments[2]+ assert get_previous_segment(middle_segment, session=sqlite_session) is segments[0]+ assert get_next_segment(middle_segment, session=sqlite_session) is segments[2]def test_accessors_return_none_when_neighbors_are_absent(self, sqlite_session: Session):dataset = _make_dataset(dataset_id=str(uuid4()), tenant_id=str(uuid4()))@@ -1619,8 +1754,8 @@ class TestDocumentSegmentNeighborAccessors:sqlite_session.add_all([dataset, document, segment])sqlite_session.flush()- assert segment.previous_segment(session=sqlite_session) is None- assert segment.next_segment(session=sqlite_session) is None+ assert get_previous_segment(segment, session=sqlite_session) is None+ assert get_next_segment(segment, session=sqlite_session) is Noneclass TestAppDatasetJoinSessionAccessors:@@ -1639,7 +1774,7 @@ class TestAppDatasetJoinSessionAccessors:sqlite_session.add_all([app, join])sqlite_session.flush()- assert join.app(session=sqlite_session) is app+ assert get_joined_app(join, session=sqlite_session) is appdef test_app_accessor_returns_none_when_app_absent(self, sqlite_session: Session):join = AppDatasetJoin(@@ -1647,4 +1782,4 @@ class TestAppDatasetJoinSessionAccessors:dataset_id=str(uuid4()),)- assert join.app(session=sqlite_session) is None+ assert get_joined_app(join, session=sqlite_session) is Nonediff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.tomlindex 01f5847d4db..21274d824f5 100644--- a/api/tests/unit_tests/pyrefly.toml+++ b/api/tests/unit_tests/pyrefly.toml@@ -477,7 +477,6 @@ project-excludes = ["core/rag/indexing/processor/test_qa_index_processor.py","core/rag/indexing/test_index_processor.py","core/rag/indexing/test_index_processor_base.py",- "core/rag/indexing/test_indexing_runner.py","core/rag/pipeline/test_queue.py","core/rag/rerank/test_reranker.py","core/rag/retrieval/test_dataset_retrieval.py",@@ -843,7 +842,6 @@ project-excludes = ["services/test_telemetry_service.py","services/test_trigger_provider_service.py","services/test_variable_truncator.py",- "services/test_vector_service.py","services/test_webhook_service.py","services/test_webhook_service_additional.py","services/test_website_service.py",diff --git a/api/tests/unit_tests/repositories/credentials/__init__.py b/api/tests/unit_tests/repositories/credentials/__init__.pynew file mode 100644index 00000000000..6c42b60d5a7--- /dev/null+++ b/api/tests/unit_tests/repositories/credentials/__init__.py@@ -0,0 +1 @@+"""Tests for credentials persistence adapters."""diff --git a/api/tests/unit_tests/repositories/credentials/test_query_repository.py b/api/tests/unit_tests/repositories/credentials/test_query_repository.pynew file mode 100644index 00000000000..9bb0ffab543--- /dev/null+++ b/api/tests/unit_tests/repositories/credentials/test_query_repository.py@@ -0,0 +1,451 @@+"""Unit tests for persisted credential permission policy.++Tests visibility filtering and complete domain reads against real SQLite rows.+"""++from datetime import datetime+from uuid import uuid4++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.plugin.entities.plugin_daemon import CredentialType as TriggerCredentialType+from models.account import Account, TenantAccountRole+from models.credential_permission import CredentialPermission, CredentialType+from models.enums import PermissionEnum+from models.oauth import DatasourceProvider+from models.provider import ProviderCredential+from models.tools import BuiltinToolProvider+from models.trigger import TriggerSubscription, WorkflowPluginTrigger+from repositories.credentials.query_repository import (+ CredentialQueryRepository,+ apply_credential_visibility_filter_for_actor,+)+++def tenant_id() -> str:+ return str(uuid4())+++def user_id() -> str:+ return str(uuid4())+++def other_user_id() -> str:+ return str(uuid4())+++def repository(sqlite_session_factory: sessionmaker[Session]) -> CredentialQueryRepository:+ return CredentialQueryRepository(session_factory=sqlite_session_factory)++[email protected]("provider", ["openai", "langgenius/openai/openai"])+def test_model_aliases_preserve_visibility_and_tenant_scope(+ sqlite_session: Session, repository: CredentialQueryRepository, provider: str+) -> None:+ tenant, actor, teammate = (str(uuid4()) for _ in range(3))+ rows = [+ ProviderCredential(+ tenant_id=tenant,+ provider_name="openai" if index % 2 else "langgenius/openai/openai",+ credential_name=name,+ encrypted_config="encrypted",+ user_id=owner,+ visibility=visibility,+ )+ for index, (name, owner, visibility) in enumerate(+ [+ ("team", teammate, PermissionEnum.ALL_TEAM),+ ("mine", actor, PermissionEnum.ONLY_ME),+ ("legacy", None, PermissionEnum.ONLY_ME),+ ("shared", teammate, PermissionEnum.PARTIAL_TEAM),+ ("disabled", teammate, PermissionEnum.PARTIAL_TEAM),+ ("wrong-tenant-grant", teammate, PermissionEnum.PARTIAL_TEAM),+ ("wrong-type-grant", teammate, PermissionEnum.PARTIAL_TEAM),+ ("private", teammate, PermissionEnum.ONLY_ME),+ ]+ )+ ]+ for index, row in enumerate(rows):+ row.created_at = datetime(2026, 1, index + 1)+ sqlite_session.add_all(rows)+ sqlite_session.add_all(+ [+ CredentialPermission(+ credential_id=rows[index].id,+ credential_type=CredentialType.BUILTIN_TOOL_PROVIDER+ if index == 6+ else CredentialType.PROVIDER_CREDENTIAL,+ account_id=actor,+ tenant_id=str(uuid4()) if index == 5 else tenant,+ has_permission=index != 4,+ )+ for index in (3, 4, 5, 6)+ ]+ )+ sqlite_session.add_all(+ [+ ProviderCredential(+ tenant_id=str(uuid4()), provider_name="openai", credential_name="foreign", encrypted_config="secret"+ ),+ ProviderCredential(+ tenant_id=tenant,+ provider_name="anthropic",+ credential_name="different-provider",+ encrypted_config="secret",+ ),+ ]+ )+ sqlite_session.commit()++ records = repository.list_models(workspace_id=tenant, provider=provider, actor_id=actor)++ assert [record.name for record in records] == ["shared", "legacy", "mine", "team"]+ internal = repository.list_models(workspace_id=tenant, provider=provider, actor_id=None)+ assert {record.id for record in internal} == {row.id for row in rows}+++def test_datasource_hidden_default_and_complete_owner_scope(+ sqlite_session: Session, repository: CredentialQueryRepository+) -> None:+ tenant, actor, teammate = (str(uuid4()) for _ in range(3))+ visible = DatasourceProvider(+ tenant_id=tenant,+ name="visible",+ provider="notion",+ plugin_id="plugin",+ auth_type="oauth2",+ encrypted_credentials={"access_token": "encrypted"},+ user_id=actor,+ visibility=PermissionEnum.ONLY_ME,+ )+ hidden_default = DatasourceProvider(+ tenant_id=tenant,+ name="hidden",+ provider="notion",+ plugin_id="plugin",+ auth_type="oauth2",+ encrypted_credentials={},+ user_id=teammate,+ visibility=PermissionEnum.ONLY_ME,+ is_default=True,+ )+ sqlite_session.add_all([visible, hidden_default])+ for owner, provider, plugin in [+ (str(uuid4()), "notion", "plugin"),+ (tenant, "other", "plugin"),+ (tenant, "notion", "other"),+ ]:+ sqlite_session.add(+ DatasourceProvider(+ tenant_id=owner,+ name="unrelated",+ provider=provider,+ plugin_id=plugin,+ auth_type="oauth2",+ encrypted_credentials={},+ is_default=True,+ )+ )+ sqlite_session.commit()++ records = repository.list_datasources(workspace_id=tenant, provider="notion", plugin_id="plugin", actor_id=actor)++ assert len(records) == 1+ assert records[0].id == visible.id+ assert records[0].encrypted_credentials == {"access_token": "encrypted"}+ assert records[0].is_default is False+ internal = repository.list_datasources(workspace_id=tenant, provider="notion", plugin_id="plugin", actor_id=None)+ assert {record.id for record in internal if record.is_default} == {hidden_default.id}+++def test_tool_borrowing_members_and_default_do_not_escape_scope_or_write_state(+ sqlite_session: Session, repository: CredentialQueryRepository+) -> None:+ tenant, actor, teammate = (str(uuid4()) for _ in range(3))+ visible = BuiltinToolProvider(+ name="visible",+ tenant_id=tenant,+ user_id=actor,+ provider="tool",+ visibility=PermissionEnum.PARTIAL_TEAM,+ encrypted_credentials='{"api_key": "encrypted"}',+ )+ borrowed = BuiltinToolProvider(+ name="borrowed",+ tenant_id=tenant,+ user_id=teammate,+ provider="tool",+ visibility=PermissionEnum.ONLY_ME,+ is_default=True,+ )+ foreign = BuiltinToolProvider(name="foreign", tenant_id=str(uuid4()), user_id=teammate, provider="tool")+ other_provider = BuiltinToolProvider(name="other", tenant_id=tenant, user_id=teammate, provider="other")+ sqlite_session.add_all([visible, borrowed, foreign, other_provider])+ sqlite_session.add_all(+ [+ CredentialPermission(+ credential_id=visible.id,+ credential_type=kind,+ account_id=member,+ tenant_id=owner,+ has_permission=enabled,+ )+ for member, owner, kind, enabled in [+ (teammate, tenant, CredentialType.BUILTIN_TOOL_PROVIDER, True),+ (str(uuid4()), tenant, CredentialType.BUILTIN_TOOL_PROVIDER, False),+ (str(uuid4()), str(uuid4()), CredentialType.BUILTIN_TOOL_PROVIDER, True),+ (str(uuid4()), tenant, CredentialType.PROVIDER_CREDENTIAL, True),+ ]+ ]+ )+ sqlite_session.commit()++ records = repository.list_tools(+ workspace_id=tenant,+ provider="tool",+ actor_id=actor,+ include_credential_ids=[visible.id, borrowed.id, borrowed.id, foreign.id, other_provider.id, ""],+ )++ assert [record.id for record in records] == [visible.id, borrowed.id]+ assert records[0].partial_member_ids == (teammate,)+ assert records[0].credentials == {"api_key": "encrypted"}+ assert records[0].is_default is True+ assert records[0].from_other_member is False+ assert records[1].from_other_member is True+ sqlite_session.refresh(visible)+ assert visible.is_default is False+++def test_trigger_visibility_and_usage_counts_are_tenant_scoped(+ sqlite_session: Session, repository: CredentialQueryRepository+) -> None:+ tenant, actor, teammate, app_id = (str(uuid4()) for _ in range(4))+ rows = [+ TriggerSubscription(+ name=name,+ tenant_id=owner,+ user_id=teammate,+ provider_id=provider,+ endpoint_id=f"{name}-endpoint",+ parameters={"event": "created"},+ properties={"label": name},+ credentials={"api_key": "encrypted"},+ credential_type=TriggerCredentialType.API_KEY,+ visibility=visibility,+ )+ for name, owner, provider, visibility in [+ ("shared", tenant, "trigger", PermissionEnum.PARTIAL_TEAM),+ ("private", tenant, "trigger", PermissionEnum.ONLY_ME),+ ("foreign", str(uuid4()), "trigger", PermissionEnum.ALL_TEAM),+ ("other-provider", tenant, "other", PermissionEnum.ALL_TEAM),+ ]+ ]+ sqlite_session.add_all(rows)+ sqlite_session.add(+ CredentialPermission(+ credential_id=rows[0].id,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ tenant_id=tenant,+ account_id=actor,+ )+ )+ sqlite_session.add_all(+ [+ WorkflowPluginTrigger(+ app_id=app,+ node_id=node,+ tenant_id=owner,+ provider_id="trigger",+ event_name="created",+ subscription_id=rows[0].id,+ )+ for owner, app, node in [+ (tenant, app_id, "node-1"),+ (tenant, app_id, "node-2"),+ (tenant, str(uuid4()), "node-3"),+ (str(uuid4()), str(uuid4()), "foreign-node"),+ ]+ ]+ )+ sqlite_session.commit()++ records = repository.list_trigger_subscriptions(workspace_id=tenant, provider="trigger", actor_id=actor)++ assert len(records) == 1+ assert records[0].id == rows[0].id+ assert records[0].workflows_in_use == 2+ assert records[0].parameters == {"event": "created"}+ assert records[0].endpoint_id == "shared-endpoint"+ assert records[0].credentials == {"api_key": "encrypted"}+++def test_empty_lists_do_not_require_related_rows(repository: CredentialQueryRepository) -> None:+ assert repository.list_models(workspace_id="empty", provider="openai", actor_id="actor") == []+ assert (+ repository.list_datasources(workspace_id="empty", provider="notion", plugin_id="plugin", actor_id="actor") == []+ )+ assert repository.list_tools(workspace_id="empty", provider="tool", actor_id="actor") == []+ assert repository.list_trigger_subscriptions(workspace_id="empty", provider="trigger", actor_id="actor") == []+++def _subscription(+ *,+ tenant_id: str,+ owner_id: str,+ name: str,+ visibility: PermissionEnum,+) -> TriggerSubscription:+ return TriggerSubscription(+ name=name,+ tenant_id=tenant_id,+ user_id=owner_id,+ provider_id="test/provider",+ endpoint_id=f"{name}-endpoint",+ parameters={},+ properties={},+ credentials={},+ credential_type=TriggerCredentialType.API_KEY,+ visibility=visibility,+ )+++def _user(user_id: str, *, is_admin: bool) -> Account:+ user = Account(name="Credential User", email=f"{user_id}@example.com")+ user.id = user_id+ user.role = TenantAccountRole.ADMIN if is_admin else TenantAccountRole.NORMAL+ return user+++class TestApplyVisibilityFilter:+ def test_admin_does_not_bypass_personal_visibility(+ self,+ sqlite_session: Session,+ tenant_id: str,+ user_id: str,+ other_user_id: str,+ ) -> None:+ private_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=other_user_id,+ name="private",+ visibility=PermissionEnum.ONLY_ME,+ )+ sqlite_session.add(private_subscription)+ sqlite_session.commit()++ query = apply_credential_visibility_filter_for_actor(+ select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id),+ tenant_id=tenant_id,+ model_id_column=TriggerSubscription.id,+ model_user_id_column=TriggerSubscription.user_id,+ model_visibility_column=TriggerSubscription.visibility,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ actor_id=_user(user_id, is_admin=True).id,+ )++ assert sqlite_session.scalars(query).all() == []++ def test_non_admin_sees_team_owned_and_partial_member_subscriptions(+ self,+ sqlite_session: Session,+ tenant_id: str,+ user_id: str,+ other_user_id: str,+ ) -> None:+ team_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=other_user_id,+ name="team",+ visibility=PermissionEnum.ALL_TEAM,+ )+ owned_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=user_id,+ name="owned",+ visibility=PermissionEnum.ONLY_ME,+ )+ shared_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=other_user_id,+ name="shared",+ visibility=PermissionEnum.PARTIAL_TEAM,+ )+ private_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=other_user_id,+ name="private",+ visibility=PermissionEnum.ONLY_ME,+ )+ sqlite_session.add_all(+ [+ team_subscription,+ owned_subscription,+ shared_subscription,+ private_subscription,+ CredentialPermission(+ credential_id=shared_subscription.id,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ account_id=user_id,+ tenant_id=tenant_id,+ ),+ ]+ )+ sqlite_session.commit()++ query = apply_credential_visibility_filter_for_actor(+ select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id),+ tenant_id=tenant_id,+ model_id_column=TriggerSubscription.id,+ model_user_id_column=TriggerSubscription.user_id,+ model_visibility_column=TriggerSubscription.visibility,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ actor_id=_user(user_id, is_admin=False).id,+ )++ visible_ids = {subscription.id for subscription in sqlite_session.scalars(query)}+ assert visible_ids == {team_subscription.id, owned_subscription.id, shared_subscription.id}++ def test_partial_permissions_are_scoped_by_tenant(+ self,+ sqlite_session: Session,+ tenant_id: str,+ user_id: str,+ other_user_id: str,+ ) -> None:+ other_tenant_id = str(uuid4())+ shared_subscription = _subscription(+ tenant_id=tenant_id,+ owner_id=other_user_id,+ name="shared",+ visibility=PermissionEnum.PARTIAL_TEAM,+ )+ sqlite_session.add(shared_subscription)+ sqlite_session.flush()+ sqlite_session.add(+ CredentialPermission(+ credential_id=shared_subscription.id,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ account_id=user_id,+ tenant_id=other_tenant_id,+ )+ )+ sqlite_session.commit()++ query = apply_credential_visibility_filter_for_actor(+ select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id),+ tenant_id=tenant_id,+ model_id_column=TriggerSubscription.id,+ model_user_id_column=TriggerSubscription.user_id,+ model_visibility_column=TriggerSubscription.visibility,+ credential_type=CredentialType.TRIGGER_SUBSCRIPTION,+ actor_id=_user(user_id, is_admin=False).id,+ )++ assert sqlite_session.scalars(query).all() == []diff --git a/api/tests/unit_tests/repositories/data_source/__init__.py b/api/tests/unit_tests/repositories/data_source/__init__.pynew file mode 100644index 00000000000..a32a4aee0cd--- /dev/null+++ b/api/tests/unit_tests/repositories/data_source/__init__.py@@ -0,0 +1 @@+"""Tests for data-source persistence adapters."""diff --git a/api/tests/unit_tests/repositories/test_data_source_api_key_auth_repository.py b/api/tests/unit_tests/repositories/data_source/test_api_key_auth_repository.pysimilarity index 94%rename from api/tests/unit_tests/repositories/test_data_source_api_key_auth_repository.pyrename to api/tests/unit_tests/repositories/data_source/test_api_key_auth_repository.pyindex d2a4742aa3e..802fd19f400 100644--- a/api/tests/unit_tests/repositories/test_data_source_api_key_auth_repository.py+++ b/api/tests/unit_tests/repositories/data_source/test_api_key_auth_repository.py@@ -4,8 +4,8 @@ from sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmakerfrom models.source import DataSourceApiKeyAuthBinding-from repositories.data_source_api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsdef _binding(diff --git a/api/tests/unit_tests/repositories/data_source/test_credential_repository.py b/api/tests/unit_tests/repositories/data_source/test_credential_repository.pynew file mode 100644index 00000000000..23ab5055a5a--- /dev/null+++ b/api/tests/unit_tests/repositories/data_source/test_credential_repository.py@@ -0,0 +1,496 @@+import json++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from models.credential_permission import CredentialPermission, CredentialType+from models.dataset import Dataset, Document+from models.enums import DataSourceType, DocumentCreatedFrom, PermissionEnum+from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProvider+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from services.data_source.entities.credential import DatasourceCredentialRecord+++def test_provider_reads_return_scoped_snapshots_and_preserve_default_selection(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ selected = _provider(is_default=True)+ other = _provider()+ other.name = "Other credential"+ foreign = _provider(workspace_id="workspace-2", is_default=True)+ wrong_provider = _provider(provider="other-provider", is_default=True)+ wrong_plugin = _provider(plugin_id="other-plugin", is_default=True)+ session.add_all([selected, other, foreign, wrong_provider, wrong_plugin])+ selected_id, other_id = selected.id, other.id+ forbidden_ids = [row.id for row in (foreign, wrong_provider, wrong_plugin)]++ records = repository.list_provider_credentials(+ workspace_id="workspace-1", provider="notion_datasource", plugin_id="langgenius/notion_datasource"+ )+ assert [record.id for record in records] == [selected_id, other_id]+ assert all(isinstance(record, DatasourceCredentialRecord) for record in records)+ assert records[0].encrypted_credentials == {"integration_secret": "encrypted"}+ default = repository.get_provider_credential(+ workspace_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ credential_id=None,+ )+ assert default is not None+ assert default.id == selected_id+ for credential_id in forbidden_ids:+ assert (+ repository.get_provider_credential(+ workspace_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ credential_id=credential_id,+ )+ is None+ )+++def _provider(+ *,+ workspace_id: str = "workspace-1",+ owner_id: str = "owner-1",+ provider: str = "notion_datasource",+ plugin_id: str = "langgenius/notion_datasource",+ visibility: PermissionEnum = PermissionEnum.ONLY_ME,+ is_default: bool = False,+) -> DatasourceProvider:+ return DatasourceProvider(+ tenant_id=workspace_id,+ name="Notion",+ provider=provider,+ plugin_id=plugin_id,+ auth_type="oauth2",+ encrypted_credentials={"integration_secret": "encrypted"},+ user_id=owner_id,+ visibility=visibility,+ is_default=is_default,+ expires_at=-1,+ )+++def _notion_source(*, workspace_id: str, credential_id: str | None) -> tuple[Dataset, Document]:+ dataset = Dataset(+ id="dataset-1",+ tenant_id=workspace_id,+ name="Notion dataset",+ data_source_type=DataSourceType.NOTION_IMPORT,+ created_by="owner-1",+ )+ document = Document(+ id="document-1",+ tenant_id=workspace_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.NOTION_IMPORT,+ data_source_info=json.dumps(+ {+ **({"credential_id": credential_id} if credential_id is not None else {}),+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "page",+ }+ ),+ batch="batch-1",+ name="Page",+ created_from=DocumentCreatedFrom.API,+ created_by="owner-1",+ )+ return dataset, document+++def test_get_visible_scopes_workspace_provider_plugin_and_actor(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider(visibility=PermissionEnum.PARTIAL_TEAM)+ session.add(credential)+ session.flush()+ session.add(+ CredentialPermission(+ credential_id=credential.id,+ credential_type=CredentialType.DATASOURCE_PROVIDER,+ account_id="member-1",+ tenant_id="workspace-1",+ )+ )+ credential_id = credential.id++ record = repository.get_visible(+ workspace_id="workspace-1",+ actor_id="member-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert record is not None+ assert record.encrypted_credentials == {"integration_secret": "encrypted"}+ assert (+ repository.get_visible(+ workspace_id="workspace-2",+ actor_id="member-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )+ assert (+ repository.get_visible(+ workspace_id="workspace-1",+ actor_id="member-1",+ credential_id=credential_id,+ provider="other",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )+ assert (+ repository.get_visible(+ workspace_id="workspace-1",+ actor_id="member-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="other/plugin",+ )+ is None+ )+ assert (+ repository.get_visible(+ workspace_id="workspace-1",+ actor_id="outsider-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )+++def test_get_visible_ignores_revoked_partial_permission(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider(visibility=PermissionEnum.PARTIAL_TEAM)+ session.add(credential)+ session.flush()+ session.add(+ CredentialPermission(+ credential_id=credential.id,+ credential_type=CredentialType.DATASOURCE_PROVIDER,+ account_id="member-1",+ tenant_id="workspace-1",+ has_permission=False,+ )+ )+ credential_id = credential.id++ assert (+ repository.get_visible(+ workspace_id="workspace-1",+ actor_id="member-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )+++def test_get_visible_does_not_apply_stale_share_to_private_credential(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider(visibility=PermissionEnum.ONLY_ME)+ session.add(credential)+ session.flush()+ session.add(+ CredentialPermission(+ credential_id=credential.id,+ credential_type=CredentialType.DATASOURCE_PROVIDER,+ account_id="member-1",+ tenant_id="workspace-1",+ )+ )+ credential_id = credential.id++ assert (+ repository.get_visible(+ workspace_id="workspace-1",+ actor_id="member-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )+++def test_get_oauth_client_config_returns_detached_tenant_and_system_rows(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ DatasourceOauthTenantParamConfig(+ tenant_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ client_params={"client_secret": "tenant-cipher"},+ enabled=True,+ ),+ DatasourceOauthTenantParamConfig(+ tenant_id="workspace-2",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ client_params={"client_secret": "other-cipher"},+ enabled=True,+ ),+ DatasourceOauthParamConfig(+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ system_credentials={"client_secret": "system-secret"},+ ),+ ]+ )++ record = repository.get_oauth_client_config(+ workspace_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert record.encrypted_tenant_params == {"client_secret": "tenant-cipher"}+ assert record.system_credentials == {"client_secret": "system-secret"}+++def test_update_if_unchanged_detects_concurrent_change(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider()+ session.add(credential)+ session.flush()+ credential_id = credential.id++ record = repository.get_visible(+ workspace_id="workspace-1",+ actor_id="owner-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ assert record is not None++ with sqlite_session_factory.begin() as session:+ stored = session.scalar(select(DatasourceProvider).where(DatasourceProvider.id == credential_id))+ assert stored is not None+ stored.encrypted_credentials = {"integration_secret": "concurrent"}+ stored.expires_at = 200++ assert not repository.update_if_unchanged(+ record=record,+ encrypted_credentials={"integration_secret": "new"},+ expires_at=100,+ )+++def test_update_if_unchanged_persists_refreshed_snapshot(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider()+ session.add(credential)+ session.flush()+ credential_id = credential.id++ record = repository.get_visible(+ workspace_id="workspace-1",+ actor_id="owner-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ assert record is not None+ assert repository.update_if_unchanged(+ record=record,+ encrypted_credentials={"integration_secret": "new"},+ expires_at=100,+ )++ with sqlite_session_factory() as session:+ stored = session.get(DatasourceProvider, credential_id)+ assert stored is not None+ assert stored.encrypted_credentials == {"integration_secret": "new"}+ assert stored.expires_at == 100+ assert stored.updated_at >= record.updated_at+++def test_get_for_stored_document_reconstructs_complete_owner_chain(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider(visibility=PermissionEnum.ONLY_ME)+ session.add(credential)+ session.flush()+ dataset, document = _notion_source(workspace_id="workspace-1", credential_id=credential.id)+ session.add_all([dataset, document])+ credential_id = credential.id++ record = repository.get_for_stored_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert record is not None+ assert record.id == credential_id+++def test_get_for_legacy_stored_document_uses_tenant_default_credential(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ session.add(_provider(is_default=False))+ default_credential = _provider(is_default=True)+ default_credential.name = "Default Notion"+ session.add(default_credential)+ session.flush()+ dataset, document = _notion_source(workspace_id="workspace-1", credential_id=None)+ session.add_all([dataset, document])+ default_credential_id = default_credential.id++ record = repository.get_for_stored_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=None,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert record is not None+ assert record.id == default_credential_id+++def test_get_for_legacy_stored_document_does_not_use_non_default_credential(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ session.add(_provider(is_default=False))+ dataset, document = _notion_source(workspace_id="workspace-1", credential_id=None)+ session.add_all([dataset, document])++ record = repository.get_for_stored_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=None,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert record is None+++ ("workspace_id", "provider", "plugin_id"),+ [+ ("workspace-2", "notion_datasource", "langgenius/notion_datasource"),+ ("workspace-1", "other", "langgenius/notion_datasource"),+ ("workspace-1", "notion_datasource", "other/plugin"),+ ],+)+def test_get_for_stored_document_rejects_owner_provider_and_plugin_mismatch(+ sqlite_session_factory: sessionmaker[Session],+ workspace_id: str,+ provider: str,+ plugin_id: str,+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider()+ session.add(credential)+ session.flush()+ dataset, document = _notion_source(workspace_id="workspace-1", credential_id=credential.id)+ session.add_all([dataset, document])+ credential_id = credential.id++ assert (+ repository.get_for_stored_document(+ workspace_id=workspace_id,+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=credential_id,+ provider=provider,+ plugin_id=plugin_id,+ )+ is None+ )+++def test_get_for_stored_document_rejects_deleted_or_different_stored_credential(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ credential = _provider()+ session.add(credential)+ session.flush()+ dataset, document = _notion_source(workspace_id="workspace-1", credential_id="different-credential")+ session.add_all([dataset, document])+ credential_id = credential.id++ assert (+ repository.get_for_stored_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )++ with sqlite_session_factory.begin() as session:+ stored_document = session.get(Document, "document-1")+ assert stored_document is not None+ source_info = stored_document.data_source_info_dict+ source_info["credential_id"] = credential_id+ stored_document.data_source_info = json.dumps(source_info)+ stored_credential = session.get(DatasourceProvider, credential_id)+ assert stored_credential is not None+ session.delete(stored_credential)++ assert (+ repository.get_for_stored_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ is None+ )diff --git a/api/tests/unit_tests/repositories/data_source/test_oauth_binding_repository.py b/api/tests/unit_tests/repositories/data_source/test_oauth_binding_repository.pynew file mode 100644index 00000000000..9e5b1dced33--- /dev/null+++ b/api/tests/unit_tests/repositories/data_source/test_oauth_binding_repository.py@@ -0,0 +1,247 @@+from sqlalchemy import null, select, update+from sqlalchemy.orm import Session, sessionmaker++from models.source import DataSourceOauthBinding+from repositories.data_source.oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository+from services.data_source.entities.oauth import DataSourceOAuthAuthorization+++def test_upsert_authorization_inserts_and_reenables_existing_binding(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)+ authorization = DataSourceOAuthAuthorization(access_token="token", source_info={"pages": []})++ repository.upsert_authorization(+ workspace_id="workspace-1",+ provider="notion",+ authorization=authorization,+ )+ with sqlite_session_factory.begin() as session:+ binding = session.scalar(select(DataSourceOauthBinding))+ assert binding is not None+ binding.disabled = True++ updated_authorization = DataSourceOAuthAuthorization(+ access_token="token",+ source_info={"pages": [{"page_id": "page-1"}]},+ )+ repository.upsert_authorization(+ workspace_id="workspace-1",+ provider="notion",+ authorization=updated_authorization,+ )++ with sqlite_session_factory() as session:+ bindings = session.scalars(select(DataSourceOauthBinding)).all()+ assert len(bindings) == 1+ assert bindings[0].disabled is False+ assert bindings[0].source_info == updated_authorization.source_info+++def test_get_enabled_scopes_by_workspace_provider_and_state(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ enabled = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="enabled-token",+ source_info={"pages": []},+ disabled=False,+ )+ disabled = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="disabled-token",+ source_info={"pages": []},+ disabled=True,+ )+ session.add_all([enabled, disabled])++ record = repository.get_enabled(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=enabled.id,+ )++ assert record is not None+ assert record.id == enabled.id+ assert record.access_token == "enabled-token"+ assert (+ repository.get_enabled(+ workspace_id="workspace-2",+ provider="notion",+ binding_id=enabled.id,+ )+ is None+ )+ assert (+ repository.get_enabled(+ workspace_id="workspace-1",+ provider="other",+ binding_id=enabled.id,+ )+ is None+ )+ assert (+ repository.get_enabled(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=disabled.id,+ )+ is None+ )+++def test_update_source_info_only_updates_matching_enabled_binding(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ binding = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="token",+ source_info={"pages": []},+ disabled=False,+ )+ session.add(binding)++ assert repository.update_source_info(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=binding.id,+ source_info={"pages": [{"page_id": "page-1"}]},+ )+ assert not repository.update_source_info(+ workspace_id="workspace-2",+ provider="notion",+ binding_id=binding.id,+ source_info={"pages": []},+ )+ assert not repository.update_source_info(+ workspace_id="workspace-1",+ provider="other",+ binding_id=binding.id,+ source_info={"pages": []},+ )++ with sqlite_session_factory() as session:+ updated = session.get(DataSourceOauthBinding, binding.id)+ assert updated is not None+ assert updated.source_info == {"pages": [{"page_id": "page-1"}]}++ with sqlite_session_factory.begin() as session:+ stored = session.get(DataSourceOauthBinding, binding.id)+ assert stored is not None+ stored.disabled = True++ assert not repository.update_source_info(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=binding.id,+ source_info={"pages": []},+ )+++def test_binding_management_lists_enabled_and_changes_state_atomically(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ enabled = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="enabled-token",+ source_info={"workspace_id": "notion-workspace"},+ disabled=False,+ )+ disabled = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="disabled-token",+ source_info={},+ disabled=True,+ )+ session.add_all([enabled, disabled])++ summaries = repository.list_enabled_bindings(workspace_id="workspace-1")++ assert [summary.id for summary in summaries] == [enabled.id]+ assert (+ repository.change_disabled_state(workspace_id="workspace-1", binding_id=enabled.id, disabled=True) == "updated"+ )+ assert (+ repository.change_disabled_state(workspace_id="workspace-1", binding_id=enabled.id, disabled=True)+ == "already_disabled"+ )+ assert (+ repository.change_disabled_state(workspace_id="workspace-1", binding_id=disabled.id, disabled=False)+ == "updated"+ )+ assert (+ repository.change_disabled_state(workspace_id="workspace-2", binding_id=disabled.id, disabled=False)+ == "not_found"+ )+++def test_null_disabled_state_is_treated_as_enabled(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ binding = DataSourceOauthBinding(+ tenant_id="workspace-1",+ provider="notion",+ access_token="token",+ source_info={"workspace_id": "notion-workspace"},+ disabled=False,+ )+ session.add(binding)+ session.flush()+ binding_id = binding.id+ session.execute(+ update(DataSourceOauthBinding).where(DataSourceOauthBinding.id == binding_id).values(disabled=null())+ )++ assert (+ repository.get_enabled(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=binding_id,+ )+ is not None+ )+ assert [summary.id for summary in repository.list_enabled_bindings(workspace_id="workspace-1")] == [binding_id]+ assert (+ repository.change_disabled_state(+ workspace_id="workspace-1",+ binding_id=binding_id,+ disabled=False,+ )+ == "already_enabled"+ )+ assert repository.update_source_info(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=binding_id,+ source_info={"workspace_id": "updated-workspace"},+ )+ assert (+ repository.change_disabled_state(+ workspace_id="workspace-1",+ binding_id=binding_id,+ disabled=True,+ )+ == "updated"+ )+ assert (+ repository.get_enabled(+ workspace_id="workspace-1",+ provider="notion",+ binding_id=binding_id,+ )+ is None+ )diff --git a/api/tests/unit_tests/repositories/knowledge/__init__.py b/api/tests/unit_tests/repositories/knowledge/__init__.pyindex e69de29bb2d..3151bb4b71c 100644--- a/api/tests/unit_tests/repositories/knowledge/__init__.py+++ b/api/tests/unit_tests/repositories/knowledge/__init__.py@@ -0,0 +1 @@+"""Tests for knowledge persistence adapters."""diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.pynew file mode 100644index 00000000000..b2e67607f93--- /dev/null+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py@@ -0,0 +1,303 @@+"""Behavioral and architecture regressions for explicit knowledge reads."""++import ast+from collections.abc import Generator+from contextlib import contextmanager+from datetime import datetime+from pathlib import Path+from urllib.parse import parse_qs, urlsplit++import pytest+from sqlalchemy import event+from sqlalchemy.orm import Session++from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.app.file_access.scope import FileAccessScope, bind_file_access_scope, grant_upload_file_access+from core.tools.signature import verify_tool_file_signature+from extensions.storage.storage_type import StorageType+from fields.document_fields import DocumentResponse+from graphon.file import helpers as file_helpers+from models import dataset as dataset_module+from models.dataset import (+ Dataset,+ DatasetMetadata,+ DatasetMetadataBinding,+ DatasetQuery,+ Document,+ DocumentSegment,+ ExternalKnowledgeApis,+ ExternalKnowledgeBindings,+)+from models.enums import CreatorUserRole+from models.model import UploadFile+from models.tools import ToolFile+from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch+from repositories.knowledge.segment_read_adapter import sign_segment_content+from services.knowledge.dataset_read_service import (+ get_dataset_queries,+ get_document_metadata_details,+ load_document_details,+)+++def _dataset() -> Dataset:+ return Dataset(+ id="dataset-1", tenant_id="tenant-1", name="Knowledge", created_by="account-1", built_in_field_enabled=False+ )+++def _document(index: int = 1) -> Document:+ return Document(+ id=f"document-{index}",+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ position=index,+ data_source_type="upload_file",+ batch="batch-1",+ name="source.txt",+ created_from="web",+ created_by="account-1",+ )+++def _segment(content: str) -> DocumentSegment:+ return DocumentSegment(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ position=1,+ content=content,+ word_count=1,+ tokens=1,+ created_by="account-1",+ )+++def _upload(tenant_id: str = "tenant-1") -> UploadFile:+ return UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key="image.png",+ name="image.png",+ size=10,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime(2024, 1, 1),+ used=True,+ )+++@contextmanager+def _queries(session: Session) -> Generator[list[str]]:+ statements: list[str] = []++ def count(+ _conn: object, _cursor: object, statement: str, _parameters: object, _context: object, _executemany: bool+ ) -> None:+ statements.append(statement)++ engine = session.get_bind()+ event.listen(engine, "before_cursor_execute", count)+ try:+ yield statements+ finally:+ event.remove(engine, "before_cursor_execute", count)+++def test_mixed_links_are_signed_in_place_and_old_parameters_are_replaced(sqlite_session: Session) -> None:+ upload = _upload()+ tool = ToolFile(+ user_id="account-1", tenant_id="tenant-1", conversation_id=None, file_key="tool.png", mimetype="image/png"+ )+ sqlite_session.add_all([upload, tool])+ sqlite_session.flush()+ paths = [f"/files/tools/{tool.id}.png", f"/files/{upload.id}/image-preview", f"/files/{upload.id}/file-preview"]+ segment = _segment(+ " BEFORE " + " MIDDLE ".join(path + "?timestamp=1&nonce=old&sign=old" for path in paths) + " AFTER "+ )+ signed = sign_segment_content(segment, session=sqlite_session)+ assert signed.startswith(" BEFORE ")+ assert signed.endswith(" AFTER ")+ urls = signed.removeprefix(" BEFORE ").removesuffix(" AFTER ").split(" MIDDLE ")+ assert [urlsplit(url).path for url in urls] == paths+ for index, url in enumerate(urls):+ params = parse_qs(urlsplit(url).query)+ assert all(len(value) == 1 for value in params.values())+ values = {key: value[0] for key, value in params.items()}+ assert values["nonce"] != "old"+ if index == 0:+ assert verify_tool_file_signature(tool.id, **values)+ elif index == 1:+ assert file_helpers.verify_image_signature(upload_file_id=upload.id, **values)+ else:+ assert file_helpers.verify_file_signature(upload_file_id=upload.id, **values)+ # A second pass must also yield valid URLs rather than appending stale parameters.+ segment.content = signed+ resigned = sign_segment_content(segment, session=sqlite_session)+ assert resigned.count("?timestamp=") == 3+ for url in resigned.removeprefix(" BEFORE ").removesuffix(" AFTER ").split(" MIDDLE "):+ params = parse_qs(urlsplit(url).query)+ assert all(len(values) == 1 for values in params.values())+ assert params["nonce"] != ["old"]+ assert params["sign"] != ["old"]+++def test_foreign_file_reference_never_receives_a_signature(sqlite_session: Session) -> None:+ foreign = _upload("tenant-2")+ foreign_tool = ToolFile(+ user_id="account-2", tenant_id="tenant-2", conversation_id=None, file_key="tool.png", mimetype="image/png"+ )+ sqlite_session.add_all([foreign, foreign_tool])+ sqlite_session.flush()+ path = f"/files/{foreign.id}/file-preview"+ segment = _segment(path + "?timestamp=1&nonce=old&sign=old")+ assert sign_segment_content(segment, session=sqlite_session) == path+ tool_path = f"/files/tools/{foreign_tool.id}.png"+ assert sign_segment_content(_segment(tool_path + "?nonce=old&sign=old"), session=sqlite_session) == tool_path+++def test_signing_respects_end_user_ownership_in_same_tenant(sqlite_session: Session) -> None:+ upload = _upload()+ sqlite_session.add(upload)+ sqlite_session.flush()+ path = f"/files/{upload.id}/file-preview"+ scope = FileAccessScope(+ tenant_id="tenant-1", user_id="end-user-1", user_from=UserFrom.END_USER, invoke_from=InvokeFrom.WEB_APP+ )+ with bind_file_access_scope(scope):+ assert sign_segment_content(_segment(path), session=sqlite_session) == path+ grant_upload_file_access([upload.id])+ assert "?timestamp=" in sign_segment_content(_segment(path), session=sqlite_session)+++def test_signing_handles_uppercase_ids_and_ignores_malformed_links(sqlite_session: Session) -> None:+ upload = _upload()+ sqlite_session.add(upload)+ sqlite_session.flush()+ path = f"/files/{upload.id.upper()}/file-preview"+ signed = sign_segment_content(_segment(path), session=sqlite_session)+ params = {key: values[0] for key, values in parse_qs(urlsplit(signed).query).items()}+ assert file_helpers.verify_file_signature(upload_file_id=upload.id, **params)++ malformed = "/files/not-a-uuid/file-preview"+ with _queries(sqlite_session) as queries:+ assert sign_segment_content(_segment(malformed), session=sqlite_session) == malformed+ assert queries == []++[email protected]("content", ["123", "null", "[1]", '{"a":1}', '"text"', "[]", "plain text"])+def test_plain_text_query_is_not_confused_with_json_records(sqlite_session: Session, content: str) -> None:+ query = DatasetQuery(+ dataset_id="dataset-1",+ content=content,+ source="hit_testing",+ source_app_id=None,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ )+ assert get_dataset_queries(query, session=sqlite_session) == [+ {"content_type": "text_query", "content": content, "file_info": None}+ ]++[email protected]("enabled", [False, True])+def test_document_metadata_respects_dataset_builtin_flag(sqlite_session: Session, enabled: bool) -> None:+ dataset = _dataset()+ dataset.built_in_field_enabled = enabled+ document = _document()+ document.doc_metadata = {"custom": "value"}+ metadata = DatasetMetadata(+ tenant_id=dataset.tenant_id, dataset_id=dataset.id, type="string", name="custom", created_by="account-1"+ )+ binding = DatasetMetadataBinding(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ metadata_id=metadata.id,+ created_by="account-1",+ )+ sqlite_session.add_all([dataset, document, metadata, binding])+ sqlite_session.flush()+ details = get_document_metadata_details(document, session=sqlite_session)+ assert details is not None+ assert details[0]["value"] == "value"+ assert len([item for item in details if item["id"] == "built-in"]) == (5 if enabled else 0)+++def test_waiting_document_reports_paused_until_resumed() -> None:+ document = _document()+ document.indexing_status = "waiting"+ document.is_paused = True+ assert document.display_status == "paused"+ document.is_paused = False+ assert document.display_status == "queuing"+++def test_document_page_load_is_bounded_and_serialization_performs_no_queries(sqlite_session: Session) -> None:+ documents = [_document(i) for i in range(1, 21)]+ sqlite_session.add_all([_dataset(), *documents])+ sqlite_session.flush()+ with _queries(sqlite_session) as single_queries:+ load_document_details(documents[:1], session=sqlite_session)+ with _queries(sqlite_session) as page_queries:+ sources = load_document_details(documents, session=sqlite_session)+ assert len(page_queries) == len(single_queries)+ with _queries(sqlite_session) as serialization_queries:+ responses = [DocumentResponse.model_validate(source, from_attributes=True).model_dump() for source in sources]+ assert len(responses) == 20+ assert serialization_queries == []+++def test_external_api_bindings_are_batched_and_tenant_scoped(sqlite_session: Session) -> None:+ dataset = _dataset()+ apis = [+ ExternalKnowledgeApis(+ tenant_id="tenant-1",+ name=f"API {i}",+ description="",+ settings="{}",+ created_by="account-1",+ updated_by=None,+ )+ for i in range(20)+ ]+ foreign_dataset = Dataset(id="dataset-foreign", tenant_id="tenant-2", name="Private", created_by="account-2")+ bindings = [+ ExternalKnowledgeBindings(+ tenant_id="tenant-1",+ external_knowledge_api_id=api.id,+ dataset_id=dataset.id,+ external_knowledge_id="knowledge",+ created_by="account-1",+ )+ for api in apis+ ]+ bindings.append(+ ExternalKnowledgeBindings(+ tenant_id="tenant-2",+ external_knowledge_api_id=apis[0].id,+ dataset_id=foreign_dataset.id,+ external_knowledge_id="foreign",+ created_by="account-2",+ )+ )+ sqlite_session.add_all([dataset, foreign_dataset, *apis, *bindings])+ sqlite_session.flush()+ with _queries(sqlite_session) as queries:+ result = get_external_api_bindings_batch(apis, session=sqlite_session)+ assert len(queries) == 1+ assert result == {api.id: [{"id": dataset.id, "name": dataset.name}] for api in apis}+++def test_dataset_models_have_no_infrastructure_dependencies_or_relative_imports() -> None:+ tree = ast.parse(Path(dataset_module.__file__).read_text())+ forbidden = ("repositories", "services", "extensions", "core.tools", "models.engine")+ for node in ast.walk(tree):+ if isinstance(node, ast.ImportFrom):+ assert node.level == 0+ assert not (node.module or "").startswith(forbidden)+ assert not any(alias.name in {"Session", "scoped_session"} for alias in node.names)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):+ assert ast.unparse(node.func.value) not in {"session", "db.session", "storage"}diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_repository.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_repository.pynew file mode 100644index 00000000000..42564cad1ad--- /dev/null+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_repository.py@@ -0,0 +1,111 @@+from sqlalchemy.orm import Session, sessionmaker++from models.dataset import Dataset, DatasetPermission+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from services.knowledge.resource_scope import DatasetRef+++def _dataset(+ dataset_id: str,+ workspace_id: str,+ *,+ permission: str = "only_me",+ data_source_type: str = "notion_import",+) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=workspace_id,+ name=f"Dataset {dataset_id}",+ description="",+ provider="vendor",+ permission=permission,+ created_by="account-1",+ maintainer="account-1",+ data_source_type=data_source_type,+ indexing_technique="economy",+ )+++def test_estimate_record_and_access_snapshot_are_separate_and_tenant_scoped(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1", permission="partial_members"))+ session.add(+ DatasetPermission(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ account_id="account-2",+ )+ )++ repository = SQLAlchemyDatasetRepository(session_factory=sqlite_session_factory)++ estimate_record = repository.get_estimate_record(DatasetRef("workspace-1", "dataset-1"))+ snapshot = repository.get_access_snapshot(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ actor_id="account-2",+ )++ assert estimate_record is not None+ assert estimate_record.workspace_id == "workspace-1"+ assert estimate_record.indexing_technique == "economy"+ assert snapshot is not None+ assert snapshot.dataset.workspace_id == "workspace-1"+ assert snapshot.dataset.permission == "partial_members"+ assert snapshot.actor_has_partial_access is True+ assert repository.is_notion_dataset(DatasetRef("workspace-1", "dataset-1")) is True+ assert repository.is_notion_dataset(DatasetRef("workspace-2", "dataset-1")) is False+ assert repository.get_estimate_record(DatasetRef("workspace-2", "dataset-1")) is None+ assert (+ repository.get_access_snapshot(+ workspace_id="workspace-2",+ dataset_id="dataset-1",+ actor_id="account-2",+ )+ is None+ )+++def test_access_snapshot_ignores_revoked_permissions(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1", permission="partial_members"))+ session.add_all(+ [+ DatasetPermission(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ account_id="granted-account",+ has_permission=True,+ ),+ DatasetPermission(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ account_id="revoked-account",+ has_permission=False,+ ),+ ]+ )++ repository = SQLAlchemyDatasetRepository(session_factory=sqlite_session_factory)+ revoked_snapshot = repository.get_access_snapshot(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ actor_id="revoked-account",+ )+ assert revoked_snapshot is not None+ assert revoked_snapshot.actor_has_partial_access is False+++def test_notion_dataset_query_rejects_other_source_types(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1", data_source_type="upload_file"))++ repository = SQLAlchemyDatasetRepository(session_factory=sqlite_session_factory)++ assert repository.is_notion_dataset(DatasetRef("workspace-1", "dataset-1")) is Falsediff --git a/api/tests/unit_tests/repositories/knowledge/test_document_repository.py b/api/tests/unit_tests/repositories/knowledge/test_document_repository.pynew file mode 100644index 00000000000..5820f173e93--- /dev/null+++ b/api/tests/unit_tests/repositories/knowledge/test_document_repository.py@@ -0,0 +1,273 @@+import json+from datetime import datetime++import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.rag.index_processor.constant.index_type import IndexStructureType+from models.dataset import Dataset, DatasetProcessRule, Document+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError+from services.knowledge.resource_scope import DatasetRef+++def _dataset(dataset_id: str, workspace_id: str) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=workspace_id,+ name=f"Dataset {dataset_id}",+ description="",+ provider="vendor",+ created_by="account-1",+ maintainer="account-1",+ data_source_type="notion_import",+ indexing_technique="economy",+ )+++def _document(+ document_id: str,+ *,+ workspace_id: str = "workspace-1",+ dataset_id: str = "dataset-1",+ source_type: str = DataSourceType.NOTION_IMPORT,+ enabled: bool = True,+ archived: bool = False,+ status: str = IndexingStatus.COMPLETED,+ batch: str = "batch-1",+) -> Document:+ source_info = (+ {"notion_page_id": f"page-{document_id}"}+ if source_type == DataSourceType.NOTION_IMPORT+ else {"upload_file_id": f"file-{document_id}"}+ )+ return Document(+ id=document_id,+ tenant_id=workspace_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type=source_type,+ data_source_info=json.dumps(source_info),+ batch=batch,+ name=f"Document {document_id}",+ created_from=DocumentCreatedFrom.API,+ created_by="account-1",+ created_at=datetime(2026, 1, 1),+ updated_at=datetime(2026, 1, 1),+ indexing_status=status,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ word_count=1,+ enabled=enabled,+ archived=archived,+ is_paused=False,+ )+++def test_document_queries_enforce_the_complete_owner_chain(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _dataset("dataset-1", "workspace-1"),+ _dataset("dataset-2", "workspace-2"),+ _document("document-1"),+ _document("document-2", workspace_id="workspace-2", dataset_id="dataset-2"),+ ]+ )++ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ document_ref = DatasetRef("workspace-1", "dataset-1").document("document-1")++ record = repository.get_estimate_document(document_ref)++ assert record is not None+ assert record.workspace_id == "workspace-1"+ assert record.data_source_info == {"notion_page_id": "page-document-1"}+ assert repository.get_estimate_document(DatasetRef("workspace-2", "dataset-1").document("document-1")) is None+++def test_pipeline_document_store_owns_sessions_and_scopes_updates(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1"))++ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ assert repository.exists(workspace_id="workspace-1", dataset_id="dataset-1", document_id="document-1")+ assert not repository.exists(workspace_id="workspace-2", dataset_id="dataset-1", document_id="document-1")+ repository.mark_failed(+ workspace_id="workspace-2", dataset_id="dataset-1", document_id="document-1", error="foreign"+ )+ with sqlite_session_factory() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.indexing_status == IndexingStatus.COMPLETED+ repository.mark_failed(workspace_id="workspace-1", dataset_id="dataset-1", document_id="document-1", error="failed")+ with sqlite_session_factory() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.indexing_status == IndexingStatus.ERROR+ assert document.error == "failed"+++def test_active_notion_sync_excludes_archived_while_binding_includes_it(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add_all(+ [+ _document("notion"),+ _document("upload", source_type=DataSourceType.UPLOAD_FILE),+ _document("disabled", enabled=False),+ _document("archived", archived=True),+ ]+ )++ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ dataset_ref = DatasetRef("workspace-1", "dataset-1")++ assert repository.list_active_notion_refs(dataset_ref) == (dataset_ref.document("notion"),)+ assert repository.list_bound_notion_page_ids(dataset_ref) == frozenset({"page-notion", "page-archived"})+++def test_document_records_normalize_invalid_mapping_data_and_ignore_invalid_notion_page_ids(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ invalid_json = _document("invalid-json")+ invalid_json.data_source_info = "{invalid"+ missing_info = _document("missing-info")+ missing_info.data_source_info = None+ invalid_page_id = _document("invalid-page-id")+ invalid_page_id.data_source_info = json.dumps({"notion_page_id": 42})+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add_all([invalid_json, missing_info, invalid_page_id])++ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ dataset_ref = DatasetRef("workspace-1", "dataset-1")++ invalid_record = repository.get_estimate_document(dataset_ref.document("invalid-json"))+ missing_record = repository.get_estimate_document(dataset_ref.document("missing-info"))+ assert invalid_record is not None+ assert invalid_record.data_source_info == {}+ assert missing_record is not None+ assert missing_record.data_source_info is None+ assert repository.list_bound_notion_page_ids(dataset_ref) == frozenset()+++def test_batch_query_is_scoped_and_keeps_explicit_state_semantics(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _dataset("dataset-1", "workspace-1"),+ _dataset("dataset-2", "workspace-1"),+ _dataset("dataset-3", "workspace-2"),+ ]+ )+ session.add_all(+ [+ _document("available"),+ _document("waiting", status=IndexingStatus.WAITING),+ _document("disabled", enabled=False),+ _document("archived", archived=True),+ _document("other-batch", batch="batch-2"),+ _document("other-dataset", dataset_id="dataset-2"),+ _document("other-workspace", workspace_id="workspace-2", dataset_id="dataset-3"),+ ]+ )++ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ dataset_ref = DatasetRef("workspace-1", "dataset-1")++ assert {record.id for record in repository.list_estimate_documents_by_batch(dataset_ref, "batch-1")} == {+ "available",+ "waiting",+ "disabled",+ "archived",+ }+++ ("tenant", "dataset", "document"),+ [+ ("wrong", "dataset-1", "document-1"),+ ("workspace-1", "wrong", "document-1"),+ ("workspace-1", "dataset-1", "wrong"),+ ],+)+def test_indexing_state_writes_reject_wrong_owner_chain(+ sqlite_session_factory: sessionmaker[Session], tenant: str, dataset: str, document: str+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all([_dataset("dataset-1", "workspace-1"), _document("document-1")])+ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ ref = DatasetRef(tenant, dataset).document(document)+ for operation in [+ lambda: repository.get_indexing_document(ref),+ lambda: repository.mark_splitting(ref),+ lambda: repository.complete_indexing(ref, tokens=9, latency=1),+ ]:+ with pytest.raises(DocumentIsDeletedPausedError):+ operation()+ repository.fail_indexing(ref, "wrong owner")+ with sqlite_session_factory() as session:+ persisted_document = session.get(Document, "document-1")+ assert persisted_document is not None+ assert persisted_document.error is None+++def test_indexing_load_validates_dataset_owner_and_process_rule(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ document = _document("document-1")+ document.dataset_process_rule_id = "other-rule"+ rule = DatasetProcessRule(+ dataset_id="dataset-2", mode=ProcessRuleMode.AUTOMATIC, rules="{}", created_by="account-1"+ )+ rule.id = "other-rule"+ session.add_all(+ [+ _dataset("dataset-1", "workspace-1"),+ _dataset("dataset-2", "workspace-2"),+ document,+ rule,+ ]+ )+ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1")+ assert repository.get_indexing_document(ref).processing_rule == {}+ with sqlite_session_factory.begin() as session:+ persisted_dataset = session.get(Dataset, "dataset-1")+ assert persisted_dataset is not None+ persisted_dataset.tenant_id = "workspace-2"+ with pytest.raises(DocumentIsDeletedPausedError):+ repository.get_indexing_document(ref)+++def test_indexing_transitions_commit_and_preserve_pause(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all([_dataset("dataset-1", "workspace-1"), _document("document-1", status=IndexingStatus.PARSING)])+ repository = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1")+ repository.mark_splitting(ref)+ with sqlite_session_factory() as session:+ persisted_document = session.get(Document, "document-1")+ assert persisted_document is not None+ assert persisted_document.parsing_completed_at is not None+ repository.complete_indexing(ref, tokens=37, latency=2)+ with sqlite_session_factory.begin() as session:+ row = session.get(Document, "document-1")+ assert row is not None+ assert (row.indexing_status, row.tokens, row.indexing_latency) == (IndexingStatus.COMPLETED, 37, 2)+ row.is_paused = True+ with pytest.raises(DocumentIsPausedError):+ repository.fail_indexing(ref, "late worker error")+ with sqlite_session_factory() as session:+ persisted_document = session.get(Document, "document-1")+ assert persisted_document is not None+ assert persisted_document.error is Nonediff --git a/api/tests/unit_tests/repositories/knowledge/test_indexing_segments.py b/api/tests/unit_tests/repositories/knowledge/test_indexing_segments.pynew file mode 100644index 00000000000..d4b68d33346--- /dev/null+++ b/api/tests/unit_tests/repositories/knowledge/test_indexing_segments.py@@ -0,0 +1,119 @@+from dataclasses import replace++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.rag.index_processor.constant.index_type import IndexStructureType+from core.rag.models.document import ChildDocument+from core.rag.models.document import Document as IndexDocument+from models.dataset import ChildChunk, DatasetProcessRule, Document, DocumentSegment+from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from services.knowledge.indexing.errors import DocumentIsDeletedPausedError+from services.knowledge.indexing.execution import IndexingDocument+from services.knowledge.resource_scope import DatasetRef+from tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _document++IndexingRows = tuple[SQLAlchemySegmentRepository, IndexingDocument, list[IndexDocument]]+++def indexing_rows(sqlite_session_factory: sessionmaker[Session]) -> IndexingRows:+ with sqlite_session_factory.begin() as session:+ document = _document("document-1", status=IndexingStatus.SPLITTING)+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.HIERARCHICAL,+ rules='{"parent_mode": "paragraph"}',+ created_by="account-1",+ )+ rule.id = "rule-1"+ document.dataset_process_rule_id = rule.id+ session.add_all([_dataset("dataset-1", "workspace-1"), document, _document("document-2"), rule])+ documents = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ segments = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ job = documents.get_indexing_document(DatasetRef("workspace-1", "dataset-1").document("document-1"))+ chunks = [+ IndexDocument(+ page_content=f"parent-{i}",+ metadata={"doc_id": f"node-{i}", "doc_hash": f"hash-{i}"},+ children=[ChildDocument(page_content="child", metadata={"doc_id": f"child-{i}", "doc_hash": "child-hash"})],+ )+ for i in range(2)+ ]+ return segments, job, chunks+++def test_saved_segments_are_visible_before_workers_and_resume_preserves_total(+ sqlite_session_factory: sessionmaker[Session], indexing_rows: IndexingRows+) -> None:+ segments, job, chunks = indexing_rows+ segments.save_for_indexing(job, chunks, [13, 17])+ segments.complete_indexing_segments(job.ref, ["node-0"])+ resumed, tokens = segments.resume_indexing(job)+ assert tokens == 30+ assert [chunk.page_content for chunk in resumed] == ["parent-1"]+ assert resumed[0].children is not None+ assert resumed[0].children[0].page_content == "child"+ with sqlite_session_factory() as session:+ persisted_document = session.get(Document, "document-1")+ assert persisted_document is not None+ assert persisted_document.indexing_status == IndexingStatus.INDEXING+ rows = session.scalars(select(DocumentSegment).order_by(DocumentSegment.position)).all()+ assert [row.status for row in rows] == [SegmentStatus.COMPLETED, SegmentStatus.INDEXING]+ segments.clear_for_indexing(job)+ with sqlite_session_factory() as session:+ assert session.scalars(select(DocumentSegment)).all() == []+ assert session.scalars(select(ChildChunk)).all() == []+++def test_repeated_save_does_not_duplicate_segments_or_children(+ sqlite_session_factory: sessionmaker[Session], indexing_rows: IndexingRows+) -> None:+ segments, job, chunks = indexing_rows+ segments.save_for_indexing(job, chunks, [13, 17])+ segments.save_for_indexing(job, chunks, [19, 23])+ with sqlite_session_factory() as session:+ assert len(session.scalars(select(ChildChunk)).all()) == 2+ assert segments.resume_indexing(job)[1] == 42+++def test_token_count_mismatch_rolls_back_segments_and_state(+ sqlite_session_factory: sessionmaker[Session], indexing_rows: IndexingRows+) -> None:+ segments, job, chunks = indexing_rows+ with pytest.raises(ValueError):+ segments.save_for_indexing(job, chunks, [13])+ with sqlite_session_factory() as session:+ assert session.scalars(select(DocumentSegment)).all() == []+ persisted_document = session.get(Document, "document-1")+ assert persisted_document is not None+ assert persisted_document.indexing_status == IndexingStatus.SPLITTING+++def test_node_id_collision_cannot_overwrite_another_document(indexing_rows: IndexingRows) -> None:+ segments, job, chunks = indexing_rows+ other = replace(job, source=job.source._replace(document_ref=job.ref.dataset.document("document-2")))+ segments.save_for_indexing(other, chunks, [3, 5])+ segments.save_for_indexing(job, chunks, [13, 17])+ segments.complete_indexing_segments(job.ref, ["node-0", "node-1"])+ segments.clear_for_indexing(job)+ assert segments.resume_indexing(other)[1] == 8+ assert len(segments.resume_indexing(other)[0]) == 2+++def test_segment_mutations_reject_foreign_tenant(indexing_rows: IndexingRows) -> None:+ segments, job, chunks = indexing_rows+ foreign = replace(+ job, source=job.source._replace(document_ref=DatasetRef("foreign", "dataset-1").document("document-1"))+ )+ for operation in [+ lambda: segments.save_for_indexing(foreign, chunks, [13, 17]),+ lambda: segments.clear_for_indexing(foreign),+ lambda: segments.complete_indexing_segments(foreign.ref, ["node-0"]),+ ]:+ with pytest.raises(DocumentIsDeletedPausedError):+ operation()diff --git a/api/tests/unit_tests/repositories/knowledge/test_keyword_table_repository.py b/api/tests/unit_tests/repositories/knowledge/test_keyword_table_repository.pynew file mode 100644index 00000000000..e8f85b93eaf--- /dev/null+++ b/api/tests/unit_tests/repositories/knowledge/test_keyword_table_repository.py@@ -0,0 +1,66 @@+import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session++from models.dataset import Dataset, DatasetKeywordTable, DocumentSegment+from repositories.knowledge.keyword_table_repository import persist_keyword_table++[email protected]("existing_table", [False, True])+def test_keyword_writes_roll_back_with_the_callers_transaction(sqlite_session: Session, existing_table: bool) -> None:+ sqlite_session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Test", created_by="author"))+ segment = DocumentSegment(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="document-1",+ position=1,+ content="original",+ word_count=8,+ tokens=1,+ created_by="author",+ index_node_id="node-1",+ keywords=["original"],+ )+ sqlite_session.add(segment)+ if existing_table:+ sqlite_session.add(DatasetKeywordTable(dataset_id="dataset-1", keyword_table="original"))+ sqlite_session.commit()++ persist_keyword_table(+ sqlite_session,+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ storage_type="database",+ data="replacement",+ keywords={"node-1": ["replacement"]},+ )+ assert segment.keywords == ["replacement"]+ sqlite_session.rollback()++ assert segment.keywords == ["original"]+ row = sqlite_session.scalar(select(DatasetKeywordTable))+ if existing_table:+ assert row is not None+ assert row.keyword_table == "original"+ else:+ assert row is None++[email protected](("tenant_id", "dataset_id"), [("tenant-2", "dataset-1"), ("tenant-1", "missing")])+def test_keyword_writes_require_a_tenant_owned_dataset(+ sqlite_session: Session, tenant_id: str, dataset_id: str+) -> None:+ sqlite_session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Test", created_by="author"))+ sqlite_session.commit()++ with pytest.raises(LookupError, match="Dataset no longer exists"):+ persist_keyword_table(+ sqlite_session,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ storage_type="database",+ data="replacement",+ keywords={},+ )++ assert sqlite_session.scalar(select(DatasetKeywordTable)) is Nonediff --git a/api/tests/unit_tests/repositories/test_data_source_oauth_binding_repository.py b/api/tests/unit_tests/repositories/test_data_source_oauth_binding_repository.pydeleted file mode 100644index 468c41f46fd..00000000000--- a/api/tests/unit_tests/repositories/test_data_source_oauth_binding_repository.py+++ /dev/null@@ -1,120 +0,0 @@-from sqlalchemy import select-from sqlalchemy.orm import Session, sessionmaker--from models.source import DataSourceOauthBinding-from repositories.data_source_oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository-from services.entities.data_source_oauth_entities import DataSourceOAuthAuthorization---def test_upsert_authorization_inserts_and_reenables_existing_binding(- sqlite_session_factory: sessionmaker[Session],-) -> None:- repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)- authorization = DataSourceOAuthAuthorization(access_token="token", source_info={"pages": []})-- repository.upsert_authorization(- workspace_id="workspace-1",- provider="notion",- authorization=authorization,- )- with sqlite_session_factory.begin() as session:- binding = session.scalar(select(DataSourceOauthBinding))- assert binding is not None- binding.disabled = True-- updated_authorization = DataSourceOAuthAuthorization(- access_token="token",- source_info={"pages": [{"page_id": "page-1"}]},- )- repository.upsert_authorization(- workspace_id="workspace-1",- provider="notion",- authorization=updated_authorization,- )-- with sqlite_session_factory() as session:- bindings = session.scalars(select(DataSourceOauthBinding)).all()- assert len(bindings) == 1- assert bindings[0].disabled is False- assert bindings[0].source_info == updated_authorization.source_info---def test_get_enabled_scopes_by_workspace_provider_and_state(- sqlite_session_factory: sessionmaker[Session],-) -> None:- repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)- with sqlite_session_factory.begin() as session:- enabled = DataSourceOauthBinding(- tenant_id="workspace-1",- provider="notion",- access_token="enabled-token",- source_info={"pages": []},- disabled=False,- )- disabled = DataSourceOauthBinding(- tenant_id="workspace-1",- provider="notion",- access_token="disabled-token",- source_info={"pages": []},- disabled=True,- )- session.add_all([enabled, disabled])-- record = repository.get_enabled(- workspace_id="workspace-1",- provider="notion",- binding_id=enabled.id,- )-- assert record is not None- assert record.id == enabled.id- assert record.access_token == "enabled-token"- assert (- repository.get_enabled(- workspace_id="workspace-2",- provider="notion",- binding_id=enabled.id,- )- is None- )- assert (- repository.get_enabled(- workspace_id="workspace-1",- provider="notion",- binding_id=disabled.id,- )- is None- )---def test_update_source_info_only_updates_matching_enabled_binding(- sqlite_session_factory: sessionmaker[Session],-) -> None:- repository = SQLAlchemyDataSourceOAuthBindingRepository(sqlite_session_factory)- with sqlite_session_factory.begin() as session:- binding = DataSourceOauthBinding(- tenant_id="workspace-1",- provider="notion",- access_token="token",- source_info={"pages": []},- disabled=False,- )- session.add(binding)-- assert repository.update_source_info(- workspace_id="workspace-1",- provider="notion",- binding_id=binding.id,- source_info={"pages": [{"page_id": "page-1"}]},- )- assert not repository.update_source_info(- workspace_id="workspace-2",- provider="notion",- binding_id=binding.id,- source_info={"pages": []},- )-- with sqlite_session_factory() as session:- updated = session.get(DataSourceOauthBinding, binding.id)- assert updated is not None- assert updated.source_info == {"pages": [{"page_id": "page-1"}]}diff --git a/api/tests/unit_tests/repositories/test_workspace_member_query_repository.py b/api/tests/unit_tests/repositories/test_workspace_member_query_repository.pyindex 852dc543e67..8aacf2a03b1 100644--- a/api/tests/unit_tests/repositories/test_workspace_member_query_repository.py+++ b/api/tests/unit_tests/repositories/test_workspace_member_query_repository.py@@ -124,3 +124,34 @@ def test_list_for_workspace_returns_empty_tuple_without_membership(result = WorkspaceMemberQueryRepository(sqlite_session_factory).list_for_workspace("workspace-1")assert result == ()+++def test_get_legacy_role_is_scoped_to_workspace_and_account(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ created_at = datetime(2026, 1, 1)+ account = make_account("member", status=AccountStatus.ACTIVE, created_at=created_at)+ workspace = make_tenant("workspace-1")+ other_workspace = make_tenant("workspace-2")+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ account,+ workspace,+ other_workspace,+ TenantAccountJoin(+ tenant_id=workspace.id,+ account_id=account.id,+ role=TenantAccountRole.DATASET_OPERATOR,+ ),+ ]+ )++ repository = WorkspaceMemberQueryRepository(sqlite_session_factory)++ assert (+ repository.get_legacy_role(workspace_id=workspace.id, account_id=account.id)+ == TenantAccountRole.DATASET_OPERATOR.value+ )+ assert repository.get_legacy_role(workspace_id=other_workspace.id, account_id=account.id) is None+ assert repository.get_legacy_role(workspace_id=workspace.id, account_id="other-account") is Nonediff --git a/api/tests/unit_tests/services/agent/test_agent_services.py b/api/tests/unit_tests/services/agent/test_agent_services.pyindex dcf3a6e5a3e..e4266eeed99 100644--- a/api/tests/unit_tests/services/agent/test_agent_services.py+++ b/api/tests/unit_tests/services/agent/test_agent_services.py@@ -6750,7 +6750,7 @@ def test_dataset_rows_filters_malformed_ids(monkeypatch: pytest.MonkeyPatch, sqlcaptured["ids"] = idsreturn [], 0- import services.dataset_service as dataset_service_module+ import services.knowledge.dataset_service as dataset_service_modulefrom services.agent.knowledge_datasets import get_tenant_knowledge_dataset_rowsmonkeypatch.setattr(dataset_service_module.DatasetService, "get_datasets_by_ids", fake_get_datasets_by_ids)@@ -6779,7 +6779,7 @@ def test_composer_save_rejects_malformed_knowledge_dataset_ids(captured["tenant_id"] = tenant_idreturn [], 0- import services.dataset_service as dataset_service_module+ import services.knowledge.dataset_service as dataset_service_modulemonkeypatch.setattr(dataset_service_module.DatasetService, "get_datasets_by_ids", fake_get_datasets_by_ids)@@ -6818,7 +6818,7 @@ def test_composer_save_rejects_missing_or_out_of_scope_knowledge_datasets(captured["tenant_id"] = tenant_idreturn [], 0- import services.dataset_service as dataset_service_module+ import services.knowledge.dataset_service as dataset_service_modulemonkeypatch.setattr(dataset_service_module.DatasetService, "get_datasets_by_ids", fake_get_datasets_by_ids)@@ -6877,7 +6877,7 @@ def test_save_agent_composer_allows_incomplete_knowledge_draft(event.listen(session, "after_flush", count_flush)- import services.dataset_service as dataset_service_module+ import services.knowledge.dataset_service as dataset_service_modulemonkeypatch.setattr(dataset_service_module.DatasetService,diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.pyindex 48158240e60..75eb694c9e8 100644--- a/api/tests/unit_tests/services/app/test_console_service.py+++ b/api/tests/unit_tests/services/app/test_console_service.py@@ -62,7 +62,7 @@ from services.entities.dsl_entities import (Import,ImportStatus,)-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceCONTEXT = RequestContext("request", "trace", "actor", "workspace")RECORD = AppRecord(id="app", name="Example", mode_compatible_with_agent="chat")diff --git a/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_gateways.py b/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_gateways.pyindex 8de8e952484..f29296890ff 100644--- a/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_gateways.py+++ b/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_gateways.py@@ -3,13 +3,13 @@ from unittest.mock import MagicMock, patchimport httpximport pytest-from services.auth.data_source_api_key_auth_gateways import ProviderApiKeyAuthCredentialValidatorfrom services.auth.errors import (DataSourceApiKeyAuthCredentialValidationError,DataSourceApiKeyAuthProviderUnavailableError,UnsupportedDataSourceApiKeyAuthProviderError,)-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.auth.api_key_gateways import ProviderApiKeyAuthCredentialValidator+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsclass TestProviderApiKeyAuthCredentialValidator:@@ -18,17 +18,17 @@ class TestProviderApiKeyAuthCredentialValidator:[("firecrawl",- "services.auth.firecrawl.firecrawl.FirecrawlAuth",+ "services.data_source.auth.firecrawl.firecrawl.FirecrawlAuth",DataSourceApiKeyAuthCredentials("bearer", "test_key", {}),),("watercrawl",- "services.auth.watercrawl.watercrawl.WatercrawlAuth",+ "services.data_source.auth.watercrawl.watercrawl.WatercrawlAuth",DataSourceApiKeyAuthCredentials("x-api-key", "test_key", {}),),("jinareader",- "services.auth.jina.jina.JinaAuth",+ "services.data_source.auth.jina.jina.JinaAuth",DataSourceApiKeyAuthCredentials("bearer", "test_key", {}),),],@@ -65,7 +65,7 @@ class TestProviderApiKeyAuthCredentialValidator:auth_instance.validate_credentials.return_value = validation_resultwith patch(- "services.auth.firecrawl.firecrawl.FirecrawlAuth",+ "services.data_source.auth.firecrawl.firecrawl.FirecrawlAuth",autospec=True,return_value=auth_instance,):@@ -83,7 +83,7 @@ class TestProviderApiKeyAuthCredentialValidator:)with patch(- "services.auth.firecrawl.firecrawl.FirecrawlAuth",+ "services.data_source.auth.firecrawl.firecrawl.FirecrawlAuth",autospec=True,return_value=auth_instance,):@@ -98,7 +98,7 @@ class TestProviderApiKeyAuthCredentialValidator:auth_instance.validate_credentials.side_effect = httpx.ConnectError("Authentication endpoint unavailable")with patch(- "services.auth.firecrawl.firecrawl.FirecrawlAuth",+ "services.data_source.auth.firecrawl.firecrawl.FirecrawlAuth",autospec=True,return_value=auth_instance,):@@ -119,7 +119,7 @@ class TestProviderApiKeyAuthCredentialValidator:auth_instance.validate_credentials.side_effect = status_errorwith patch(- "services.auth.firecrawl.firecrawl.FirecrawlAuth",+ "services.data_source.auth.firecrawl.firecrawl.FirecrawlAuth",autospec=True,return_value=auth_instance,):diff --git a/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_service.py b/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_service.pyindex ee3a31b9004..4ab25292f30 100644--- a/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_service.py+++ b/api/tests/unit_tests/services/auth/test_data_source_api_key_auth_service.py@@ -4,8 +4,8 @@ from datetime import datetimefrom unittest.mock import MagicMockfrom machinery.context import RequestContext-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthService-from services.entities.data_source_api_key_auth_entities import (+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService+from services.data_source.entities.api_key_auth import (DataSourceApiKeyAuthBindingCreate,DataSourceApiKeyAuthBindingRecord,DataSourceApiKeyAuthCredentials,diff --git a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py b/api/tests/unit_tests/services/auth/test_firecrawl_auth.pyindex 476a685f15b..9514d70ed48 100644--- a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py+++ b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py@@ -9,8 +9,8 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.auth.firecrawl.firecrawl import FirecrawlAuth-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.auth.firecrawl.firecrawl import FirecrawlAuth+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsdef _credentials(@@ -64,7 +64,7 @@ class TestFirecrawlAuth:with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):FirecrawlAuth(_credentials(api_key=""))- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_validate_valid_credentials_successfully(self, mock_post: MagicMock, auth_instance: FirecrawlAuth):"""Test successful credential validation"""mock_response = MagicMock()@@ -95,7 +95,7 @@ class TestFirecrawlAuth:(409, "Conflict error"),],)- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_handle_http_errors(self, mock_post: MagicMock, status_code, error_message, auth_instance: FirecrawlAuth):@@ -110,7 +110,7 @@ class TestFirecrawlAuth:assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"@pytest.mark.parametrize("status_code", [429, 500, 502, 503])- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_map_upstream_failure_to_provider_unavailable(self,mock_post: MagicMock,@@ -136,7 +136,7 @@ class TestFirecrawlAuth:(401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"),],)- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_handle_unexpected_errors(self,mock_post: MagicMock,@@ -169,7 +169,7 @@ class TestFirecrawlAuth:(httpx.ConnectTimeout, "Connection timeout"),],)- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_handle_network_errors(self, mock_post: MagicMock, exception_type, exception_message, auth_instance: FirecrawlAuth):@@ -193,7 +193,7 @@ class TestFirecrawlAuth:FirecrawlAuth(_credentials(auth_type="basic", api_key="super_secret_key_12345"))assert "super_secret_key_12345" not in str(exc_info.value)- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_use_custom_base_url_in_validation(self, mock_post):"""Test that custom base URL is used in validation and normalized"""mock_response = MagicMock()@@ -208,7 +208,7 @@ class TestFirecrawlAuth:assert result is Trueassert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl"- @patch("services.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)def test_should_handle_timeout_with_retry_suggestion(self, mock_post: MagicMock, auth_instance: FirecrawlAuth):"""Test that timeout errors are handled gracefully with appropriate error message"""mock_post.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")diff --git a/api/tests/unit_tests/services/auth/test_jina_auth.py b/api/tests/unit_tests/services/auth/test_jina_auth.pyindex a1e41938e9d..507edc466b0 100644--- a/api/tests/unit_tests/services/auth/test_jina_auth.py+++ b/api/tests/unit_tests/services/auth/test_jina_auth.py@@ -9,8 +9,8 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.auth.jina.jina import JinaAuth-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.auth.jina.jina import JinaAuth+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsdef _credentials(@@ -41,7 +41,7 @@ class TestJinaAuth:JinaAuth(credentials)assert str(exc_info.value) == "No API key provided"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_validate_valid_credentials_successfully(self, mock_post: MagicMock):"""Test successful credential validation"""mock_response = MagicMock()@@ -59,7 +59,7 @@ class TestJinaAuth:json={"url": "https://example.com"},)- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_http_402_error(self, mock_post: MagicMock):"""Test handling of 402 Payment Required error"""mock_response = MagicMock()@@ -74,7 +74,7 @@ class TestJinaAuth:auth.validate_credentials()assert str(exc_info.value) == "Failed to authorize. Status code: 402. Error: Payment required"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_http_error_with_non_json_text_response(self, mock_post):"""Test handling of known HTTP errors with non-JSON text response."""mock_response = MagicMock()@@ -90,7 +90,7 @@ class TestJinaAuth:auth.validate_credentials()assert str(exc_info.value) == "Failed to authorize. Status code: 402. Error: Payment required"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_http_409_error(self, mock_post):"""Test handling of 409 Conflict error"""mock_response = MagicMock()@@ -106,7 +106,7 @@ class TestJinaAuth:assert str(exc_info.value) == "Failed to authorize. Status code: 409. Error: Conflict error"@pytest.mark.parametrize("status_code", [429, 500, 502, 503])- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_map_upstream_failure_to_provider_unavailable(self,mock_post: MagicMock,@@ -123,7 +123,7 @@ class TestJinaAuth:assert exc_info.value.provider == "jinareader"assert exc_info.value.status_code == status_code- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_unexpected_error_with_text_response(self, mock_post: MagicMock):"""Test handling of unexpected errors with text response"""mock_response = MagicMock()@@ -139,7 +139,7 @@ class TestJinaAuth:auth.validate_credentials()assert str(exc_info.value) == "Failed to authorize. Status code: 403. Error: Forbidden"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_unexpected_error_with_non_json_text_response(self, mock_post):"""Test handling of unexpected errors with non-JSON text response."""mock_response = MagicMock()@@ -155,7 +155,7 @@ class TestJinaAuth:auth.validate_credentials()assert str(exc_info.value) == "Failed to authorize. Status code: 403. Error: Forbidden"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_unexpected_error_without_text(self, mock_post):"""Test handling of unexpected errors without text response"""mock_response = MagicMock()@@ -171,7 +171,7 @@ class TestJinaAuth:auth.validate_credentials()assert str(exc_info.value) == "Unexpected error occurred while trying to authorize. Status code: 404"- @patch("services.auth.jina.jina._http_client.post", autospec=True)+ @patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)def test_should_handle_network_errors(self, mock_post: MagicMock):"""Test handling of network connection errors"""mock_post.side_effect = httpx.ConnectError("Network error")diff --git a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py b/api/tests/unit_tests/services/auth/test_watercrawl_auth.pyindex 09fd6e86a80..e54df0dee9e 100644--- a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py+++ b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py@@ -9,8 +9,8 @@ from services.auth.errors import (DataSourceApiKeyAuthProviderUnavailableError,InvalidDataSourceApiKeyAuthCredentialsError,)-from services.auth.watercrawl.watercrawl import WatercrawlAuth-from services.entities.data_source_api_key_auth_entities import DataSourceApiKeyAuthCredentials+from services.data_source.auth.watercrawl.watercrawl import WatercrawlAuth+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentialsdef _credentials(@@ -64,7 +64,7 @@ class TestWatercrawlAuth:with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):WatercrawlAuth(_credentials(api_key=""))- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_validate_valid_credentials_successfully(self, mock_get, auth_instance):"""Test successful credential validation"""mock_response = MagicMock()@@ -87,7 +87,7 @@ class TestWatercrawlAuth:(409, "Conflict error"),],)- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_handle_http_errors(self, mock_get, status_code, error_message, auth_instance):"""Test handling of various HTTP error codes"""mock_response = MagicMock()@@ -100,7 +100,7 @@ class TestWatercrawlAuth:assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"@pytest.mark.parametrize("status_code", [429, 500, 502, 503])- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_map_upstream_failure_to_provider_unavailable(self,mock_get: MagicMock,@@ -116,7 +116,7 @@ class TestWatercrawlAuth:assert exc_info.value.provider == "watercrawl"assert exc_info.value.status_code == status_code- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_handle_http_error_with_non_json_text_response(self, mock_get, auth_instance):"""Test handling of known HTTP errors with non-JSON text response."""mock_response = MagicMock()@@ -137,7 +137,7 @@ class TestWatercrawlAuth:(401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"),],)- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_handle_unexpected_errors(self, mock_get, status_code, response_text, has_json_error, expected_error_contains, auth_instance):@@ -162,7 +162,7 @@ class TestWatercrawlAuth:(httpx.ConnectTimeout, "Connection timeout"),],)- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_handle_network_errors(self, mock_get, exception_type, exception_message, auth_instance):"""Test handling of various network-related errors including timeouts"""mock_get.side_effect = exception_type(exception_message)@@ -184,7 +184,7 @@ class TestWatercrawlAuth:WatercrawlAuth(_credentials(auth_type="bearer", api_key="super_secret_key_12345"))assert "super_secret_key_12345" not in str(exc_info.value)- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_use_custom_base_url_in_validation(self, mock_get):"""Test that custom base URL is used in validation"""mock_response = MagicMock()@@ -206,7 +206,7 @@ class TestWatercrawlAuth:("https://app.watercrawl.dev//", "https://app.watercrawl.dev/api/v1/core/crawl-requests/"),],)- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_use_urljoin_for_url_construction(self, mock_get, base_url, expected_url):"""Test that urljoin is used correctly for URL construction with various base URLs"""mock_response = MagicMock()@@ -220,7 +220,7 @@ class TestWatercrawlAuth:# Verify the correct URL was calledassert mock_get.call_args[0][0] == expected_url- @patch("services.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)def test_should_handle_timeout_with_retry_suggestion(self, mock_get, auth_instance):"""Test that timeout errors are handled gracefully with appropriate error message"""mock_get.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")diff --git a/api/tests/unit_tests/services/data_source/__init__.py b/api/tests/unit_tests/services/data_source/__init__.pynew file mode 100644index 00000000000..2f04e29e200--- /dev/null+++ b/api/tests/unit_tests/services/data_source/__init__.py@@ -0,0 +1 @@+"""Tests for data-source application services and adapters."""diff --git a/api/tests/unit_tests/services/data_source/test_binding_application_service.py b/api/tests/unit_tests/services/data_source/test_binding_application_service.pynew file mode 100644index 00000000000..f8bb0c64bd5--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_binding_application_service.py@@ -0,0 +1,89 @@+from dataclasses import dataclass, field+from datetime import datetime++import pytest++from machinery.context import RequestContext+from services.data_source.binding_application_service import (+ BindingAction,+ BindingMutationResult,+ DataSourceBindingApplicationService,+ DataSourceBindingNotFoundError,+ DataSourceBindingStateError,+)+from services.data_source.entities.oauth import DataSourceBindingSummary+++def _context() -> RequestContext:+ return RequestContext("request-1", None, "account-1", "workspace-1")+++@dataclass+class InMemoryBindingStore:+ bindings: tuple[DataSourceBindingSummary, ...] = ()+ mutation_result: BindingMutationResult = "updated"+ changes: list[tuple[str, str, bool]] = field(default_factory=list)++ def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:+ assert workspace_id == "workspace-1"+ return self.bindings++ def change_disabled_state(+ self,+ *,+ workspace_id: str,+ binding_id: str,+ disabled: bool,+ ) -> BindingMutationResult:+ self.changes.append((workspace_id, binding_id, disabled))+ return self.mutation_result+++def test_list_integrations_returns_secret_free_records() -> None:+ binding = DataSourceBindingSummary(+ id="binding-1",+ provider="notion",+ created_at=datetime(2026, 1, 1),+ disabled=False,+ source_info={"workspace_id": "notion-workspace"},+ )+ unsupported = DataSourceBindingSummary(+ id="binding-2",+ provider="unsupported",+ created_at=datetime(2026, 1, 1),+ disabled=False,+ source_info={},+ )+ service = DataSourceBindingApplicationService(bindings=InMemoryBindingStore(bindings=(binding, unsupported)))++ assert service.list_integrations(_context()) == (binding,)++[email protected](("action", "disabled"), [(BindingAction.ENABLE, False), (BindingAction.DISABLE, True)])+def test_change_state_passes_stable_context(action: BindingAction, disabled: bool) -> None:+ store = InMemoryBindingStore()+ service = DataSourceBindingApplicationService(bindings=store)++ service.change_state(_context(), "binding-1", action)++ assert store.changes == [("workspace-1", "binding-1", disabled)]+++ ("action", "result", "error", "message"),+ [+ (BindingAction.ENABLE, "not_found", DataSourceBindingNotFoundError, "Data source binding not found"),+ (BindingAction.ENABLE, "already_enabled", DataSourceBindingStateError, "Data source is already enabled"),+ (BindingAction.DISABLE, "already_disabled", DataSourceBindingStateError, "Data source is already disabled"),+ ],+)+def test_change_state_translates_repository_outcomes(+ action: BindingAction,+ result: BindingMutationResult,+ error: type[Exception],+ message: str,+) -> None:+ service = DataSourceBindingApplicationService(bindings=InMemoryBindingStore(mutation_result=result))++ with pytest.raises(error, match=message):+ service.change_state(_context(), "binding-1", action)diff --git a/api/tests/unit_tests/services/data_source/test_credential_gateway.py b/api/tests/unit_tests/services/data_source/test_credential_gateway.pynew file mode 100644index 00000000000..0bb453f4de1--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_credential_gateway.py@@ -0,0 +1,631 @@+from collections.abc import Mapping+from dataclasses import dataclass, field, replace+from datetime import datetime, timedelta+from typing import cast++import pytest+from sqlalchemy.orm import Session, sessionmaker++import services.data_source.credential_adapters as credential_adapters_module+from core.datasource.entities.datasource_entities import (+ DatasourceProviderEntityWithPlugin,+ DatasourceProviderIdentity,+ DatasourceProviderType,+)+from core.entities.provider_entities import ProviderConfig, ProviderConfigType+from core.plugin.entities.oauth import OAuthSchema+from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity, PluginOAuthCredentialsResponse+from core.plugin.impl.datasource import PluginDatasourceManager+from core.plugin.impl.oauth import OAuthHandler+from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig+from models.provider_ids import DatasourceProviderID+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from services.data_source.credential_adapters import (+ OAuthDatasourceCredentialRefresher,+ PluginDatasourceCredentialCodec,+ PluginDatasourceOAuthClientResolver,+)+from services.data_source.credential_gateway import (+ ActorAwareDatasourceCredentialGateway,+ DatasourceCredentialConcurrentUpdateError,+ DatasourceCredentialError,+ DatasourceCredentialNotFoundError,+ DatasourceCredentialRefreshError,+ RefreshedDatasourceCredential,+ TrustedStoredDatasourceCredentialGateway,+)+from services.data_source.entities.credential import DatasourceCredentialRecord+++def _record(+ *,+ auth_type: str = "oauth2",+ expires_at: int = -1,+ updated_at: datetime | None = None,+) -> DatasourceCredentialRecord:+ return DatasourceCredentialRecord(+ id="credential-1",+ workspace_id="workspace-1",+ owner_id="credential-owner-1",+ name="Notion",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ auth_type=auth_type,+ encrypted_credentials={"integration_secret": "encrypted"},+ expires_at=expires_at,+ updated_at=updated_at or datetime(2026, 1, 1),+ )+++@dataclass+class ScriptedCredentialStore:+ records: list[DatasourceCredentialRecord | None]+ update_result: bool = True+ get_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)+ updates: list[tuple[DatasourceCredentialRecord, Mapping[str, object], int]] = field(default_factory=list)+ events: list[str] = field(default_factory=list)++ def get_visible(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ credential_id: str,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None:+ self.events.append("read")+ self.get_calls.append((workspace_id, actor_id, credential_id, provider, plugin_id))+ return self.records.pop(0)++ def update_if_unchanged(+ self,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+ ) -> bool:+ self.events.append("update")+ self.updates.append((record, encrypted_credentials, expires_at))+ return self.update_result+++@dataclass+class ReversibleCredentialCodec:+ def decrypt(self, record: DatasourceCredentialRecord) -> dict[str, object]:+ return {"integration_secret": f"plain:{record.encrypted_credentials['integration_secret']}"}++ def encrypt(self, record: DatasourceCredentialRecord, credentials: Mapping[str, object]) -> dict[str, object]:+ del record+ return {"integration_secret": f"cipher:{credentials['integration_secret']}"}+++@dataclass+class StoredCredentialCatalog:+ record: DatasourceCredentialRecord | None+ calls: list[dict[str, str | None]] = field(default_factory=list)+ updates: list[tuple[DatasourceCredentialRecord, Mapping[str, object], int]] = field(default_factory=list)++ def get_for_stored_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord | None:+ self.calls.append(+ {+ "workspace_id": workspace_id,+ "dataset_id": dataset_id,+ "document_id": document_id,+ "credential_id": credential_id,+ "provider": provider,+ "plugin_id": plugin_id,+ }+ )+ return self.record++ def update_if_unchanged(+ self,+ *,+ record: DatasourceCredentialRecord,+ encrypted_credentials: Mapping[str, object],+ expires_at: int,+ ) -> bool:+ self.updates.append((record, encrypted_credentials, expires_at))+ return True+++@dataclass+class RecordingCredentialRefresher:+ calls: list[tuple[str, str, DatasourceCredentialRecord, Mapping[str, object]]] = field(default_factory=list)+ events: list[str] = field(default_factory=list)++ def refresh(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> RefreshedDatasourceCredential:+ self.events.append("refresh")+ self.calls.append((workspace_id, actor_id, record, credentials))+ return RefreshedDatasourceCredential({"integration_secret": "refreshed"}, 500)+++@dataclass+class StaticProviderManager:+ provider: PluginDatasourceProviderEntity+ calls: list[tuple[str, str]] = field(default_factory=list)++ def fetch_datasource_provider(self, *, tenant_id: str, provider_id: str) -> PluginDatasourceProviderEntity:+ self.calls.append((tenant_id, provider_id))+ return self.provider+++@dataclass+class RecordingOAuthHandler:+ response: PluginOAuthCredentialsResponse | None = None+ error: Exception | None = None+ calls: list[dict[str, object]] = field(default_factory=list)++ def refresh_credentials(+ self,+ tenant_id: str,+ user_id: str,+ plugin_id: str,+ provider: str,+ redirect_uri: str,+ system_credentials: Mapping[str, object],+ credentials: Mapping[str, object],+ ) -> PluginOAuthCredentialsResponse:+ self.calls.append(+ {+ "tenant_id": tenant_id,+ "user_id": user_id,+ "plugin_id": plugin_id,+ "provider": provider,+ "redirect_uri": redirect_uri,+ "system_credentials": system_credentials,+ "credentials": credentials,+ }+ )+ if self.error is not None:+ raise self.error+ assert self.response is not None+ return self.response+++@dataclass+class RecordingCredentialEncrypter:+ decrypted: Mapping[str, object]+ calls: list[Mapping[str, object]] = field(default_factory=list)++ def decrypt(self, credentials: Mapping[str, object]) -> Mapping[str, object]:+ self.calls.append(credentials)+ return self.decrypted+++class StaticOAuthClientResolver:+ def resolve(self, *, workspace_id: str, provider_id: DatasourceProviderID) -> dict[str, object]:+ del workspace_id, provider_id+ return {}+++def _oauth_refresher(+ session_factory: sessionmaker[Session],+ *,+ manager: StaticProviderManager,+ handler: RecordingOAuthHandler,+) -> OAuthDatasourceCredentialRefresher:+ repository = SQLAlchemyDatasourceCredentialRepository(session_factory=session_factory)+ oauth_clients = PluginDatasourceOAuthClientResolver(+ configs=repository,+ provider_manager=cast(PluginDatasourceManager, manager),+ )+ return OAuthDatasourceCredentialRefresher(+ oauth_clients=oauth_clients,+ oauth_handler=cast(OAuthHandler, handler),+ )+++def _provider(*, with_oauth_schema: bool = True) -> PluginDatasourceProviderEntity:+ oauth_schema = (+ OAuthSchema(+ client_schema=[ProviderConfig(type=ProviderConfigType.SECRET_INPUT, name="client_secret")],+ credentials_schema=[+ ProviderConfig(type=ProviderConfigType.SECRET_INPUT, name="integration_secret"),+ ProviderConfig(type=ProviderConfigType.TEXT_INPUT, name="workspace_name"),+ ],+ )+ if with_oauth_schema+ else None+ )+ return PluginDatasourceProviderEntity(+ provider="notion_datasource",+ plugin_unique_identifier="langgenius/notion_datasource:1.0.0",+ plugin_id="langgenius/notion_datasource",+ declaration=DatasourceProviderEntityWithPlugin(+ identity=DatasourceProviderIdentity(+ author="langgenius",+ name="notion_datasource",+ description={"en_US": "Notion", "zh_Hans": "Notion"},+ icon="notion.svg",+ label={"en_US": "Notion", "zh_Hans": "Notion"},+ ),+ credentials_schema=[+ ProviderConfig(type=ProviderConfigType.SECRET_INPUT, name="api_secret"),+ ProviderConfig(type=ProviderConfigType.TEXT_INPUT, name="label"),+ ],+ oauth_schema=oauth_schema,+ provider_type=DatasourceProviderType.ONLINE_DOCUMENT,+ ),+ )+++def _gateway(+ store: ScriptedCredentialStore, refresher: RecordingCredentialRefresher | None = None+) -> tuple[ActorAwareDatasourceCredentialGateway, RecordingCredentialRefresher]:+ refresh = refresher or RecordingCredentialRefresher()+ return (+ ActorAwareDatasourceCredentialGateway(+ credentials=store,+ codec=ReversibleCredentialCodec(),+ refresher=refresh,+ now=lambda: 100,+ ),+ refresh,+ )+++def _resolve(gateway: ActorAwareDatasourceCredentialGateway) -> dict[str, object]:+ return gateway.resolve(+ workspace_id="workspace-1",+ actor_id="actor-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+++def test_resolve_fails_closed_when_credential_is_not_visible() -> None:+ gateway, _ = _gateway(ScriptedCredentialStore(records=[None]))++ with pytest.raises(DatasourceCredentialNotFoundError):+ _resolve(gateway)+++def test_trusted_stored_source_resolver_uses_distinct_owner_chain_port() -> None:+ catalog = StoredCredentialCatalog(_record())+ gateway = TrustedStoredDatasourceCredentialGateway(+ credentials=catalog,+ codec=ReversibleCredentialCodec(),+ refresher=RecordingCredentialRefresher(),+ now=lambda: 100,+ )++ result = gateway.resolve_for_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert result == {"integration_secret": "plain:encrypted"}+ assert catalog.calls == [+ {+ "workspace_id": "workspace-1",+ "dataset_id": "dataset-1",+ "document_id": "document-1",+ "credential_id": "credential-1",+ "provider": "notion_datasource",+ "plugin_id": "langgenius/notion_datasource",+ }+ ]+++def test_trusted_stored_source_resolver_fails_closed_for_missing_credential() -> None:+ gateway = TrustedStoredDatasourceCredentialGateway(+ credentials=StoredCredentialCatalog(None),+ codec=ReversibleCredentialCodec(),+ refresher=RecordingCredentialRefresher(),+ now=lambda: 100,+ )++ with pytest.raises(DatasourceCredentialNotFoundError):+ gateway.resolve_for_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+++def test_trusted_stored_source_refreshes_with_persisted_credential_owner() -> None:+ record = _record(expires_at=120)+ catalog = StoredCredentialCatalog(record)+ refresher = RecordingCredentialRefresher()+ gateway = TrustedStoredDatasourceCredentialGateway(+ credentials=catalog,+ codec=ReversibleCredentialCodec(),+ refresher=refresher,+ now=lambda: 100,+ )++ result = gateway.resolve_for_document(+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )++ assert result == {"integration_secret": "refreshed"}+ assert refresher.calls == [("workspace-1", "credential-owner-1", record, {"integration_secret": "plain:encrypted"})]+ assert catalog.updates == [(record, {"integration_secret": "cipher:refreshed"}, 500)]+++def test_resolve_decrypts_without_refreshing_non_expiring_credential() -> None:+ store = ScriptedCredentialStore(records=[_record()])+ gateway, refresher = _gateway(store)++ assert _resolve(gateway) == {"integration_secret": "plain:encrypted"}+ assert refresher.calls == []+ assert store.updates == []+++def test_refresh_runs_between_read_and_conditional_update() -> None:+ record = _record(expires_at=120)+ events: list[str] = []+ store = ScriptedCredentialStore(records=[record], events=events)+ gateway, refresher = _gateway(store, RecordingCredentialRefresher(events=events))++ assert _resolve(gateway) == {"integration_secret": "refreshed"}+ assert refresher.calls == [("workspace-1", "actor-1", record, {"integration_secret": "plain:encrypted"})]+ assert store.updates == [(record, {"integration_secret": "cipher:refreshed"}, 500)]+ assert events == ["read", "refresh", "update"]+++def test_concurrent_refresh_uses_newer_non_expiring_snapshot() -> None:+ stale = _record(expires_at=120)+ latest = replace(+ stale,+ encrypted_credentials={"integration_secret": "latest"},+ expires_at=500,+ updated_at=stale.updated_at + timedelta(seconds=1),+ )+ store = ScriptedCredentialStore(records=[stale, latest], update_result=False)+ gateway, _ = _gateway(store)++ assert _resolve(gateway) == {"integration_secret": "plain:latest"}+ assert store.get_calls == [+ ("workspace-1", "actor-1", "credential-1", "notion_datasource", "langgenius/notion_datasource"),+ ("workspace-1", "actor-1", "credential-1", "notion_datasource", "langgenius/notion_datasource"),+ ]+++def test_concurrent_refresh_fails_closed_if_latest_snapshot_is_still_expired() -> None:+ stale = _record(expires_at=120)+ latest = replace(stale, updated_at=stale.updated_at + timedelta(seconds=1))+ gateway, _ = _gateway(ScriptedCredentialStore(records=[stale, latest], update_result=False))++ with pytest.raises(DatasourceCredentialConcurrentUpdateError):+ _resolve(gateway)+++ ("auth_type", "secret_name"),+ [("api-key", "api_secret"), ("oauth2", "integration_secret")],+)+def test_plugin_codec_encrypts_and_decrypts_only_declared_secret_fields(+ monkeypatch: pytest.MonkeyPatch,+ auth_type: str,+ secret_name: str,+) -> None:+ manager = StaticProviderManager(_provider())+ codec = PluginDatasourceCredentialCodec(provider_manager=cast(PluginDatasourceManager, manager))+ record = replace(+ _record(auth_type=auth_type),+ encrypted_credentials={secret_name: "cipher", "label": "plain"},+ )+ decrypt_calls: list[tuple[str, str]] = []+ encrypt_calls: list[tuple[str, str]] = []++ def decrypt_token(workspace_id: str, value: str) -> str:+ decrypt_calls.append((workspace_id, value))+ return f"decrypted:{value}"++ def encrypt_token(workspace_id: str, value: str) -> str:+ encrypt_calls.append((workspace_id, value))+ return f"encrypted:{value}"++ monkeypatch.setattr(credential_adapters_module.encrypter, "decrypt_token", decrypt_token)+ monkeypatch.setattr(credential_adapters_module.encrypter, "encrypt_token", encrypt_token)++ assert codec.decrypt(record) == {secret_name: "decrypted:cipher", "label": "plain"}+ assert codec.encrypt(record, {secret_name: "secret", "label": "plain"}) == {+ secret_name: "encrypted:secret",+ "label": "plain",+ }+ assert decrypt_calls == [("workspace-1", "cipher")]+ assert encrypt_calls == [("workspace-1", "secret")]++ non_string_record = replace(record, encrypted_credentials={secret_name: 42})+ assert codec.decrypt(non_string_record) == {secret_name: 42}+ assert codec.encrypt(record, {secret_name: None}) == {secret_name: None}+ assert decrypt_calls == [("workspace-1", "cipher")]+ assert encrypt_calls == [("workspace-1", "secret")]+++def test_plugin_codec_rejects_unsupported_credential_type() -> None:+ codec = PluginDatasourceCredentialCodec(+ provider_manager=cast(PluginDatasourceManager, StaticProviderManager(_provider()))+ )++ with pytest.raises(DatasourceCredentialError, match="Unsupported credential type"):+ codec.decrypt(_record(auth_type="unauthorized"))+++def test_oauth_refresher_prefers_enabled_tenant_client_and_returns_refreshed_snapshot(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ DatasourceOauthTenantParamConfig(+ tenant_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ client_params={"client_secret": "tenant-cipher"},+ enabled=True,+ ),+ DatasourceOauthParamConfig(+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ system_credentials={"client_secret": "system-secret"},+ ),+ ]+ )+ manager = StaticProviderManager(_provider())+ handler = RecordingOAuthHandler(+ response=PluginOAuthCredentialsResponse(+ credentials={"integration_secret": "refreshed-secret"},+ expires_at=500,+ )+ )+ encrypter = RecordingCredentialEncrypter({"client_secret": "tenant-secret"})++ def create_encrypter(**kwargs: object) -> tuple[RecordingCredentialEncrypter, None]:+ assert kwargs["tenant_id"] == "workspace-1"+ return encrypter, None++ monkeypatch.setattr(credential_adapters_module, "create_provider_encrypter", create_encrypter)+ refresher = _oauth_refresher(sqlite_session_factory, manager=manager, handler=handler)++ result = refresher.refresh(+ workspace_id="workspace-1",+ actor_id="actor-1",+ record=_record(),+ credentials={"integration_secret": "old-secret"},+ )++ assert result == RefreshedDatasourceCredential({"integration_secret": "refreshed-secret"}, 500)+ assert encrypter.calls == [{"client_secret": "tenant-cipher"}]+ assert handler.calls[0]["system_credentials"] == {"client_secret": "tenant-secret"}+ assert handler.calls[0]["credentials"] == {"integration_secret": "old-secret"}+ assert handler.calls[0]["user_id"] == "actor-1"+++def test_oauth_refresher_rejects_tenant_client_when_provider_has_no_oauth_schema(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(+ DatasourceOauthTenantParamConfig(+ tenant_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ client_params={"client_secret": "tenant-cipher"},+ enabled=True,+ )+ )+ provider = _provider(with_oauth_schema=False)+ handler = RecordingOAuthHandler()+ refresher = _oauth_refresher(+ sqlite_session_factory,+ manager=StaticProviderManager(provider),+ handler=handler,+ )++ with pytest.raises(DatasourceCredentialError, match="oauth schema not found"):+ refresher.refresh(+ workspace_id="workspace-1",+ actor_id="actor-1",+ record=_record(),+ credentials={"integration_secret": "old"},+ )++ assert handler.calls == []++[email protected]("verified", [True, False])+def test_oauth_refresher_uses_system_client_only_for_verified_plugin(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_session_factory: sessionmaker[Session],+ verified: bool,+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ DatasourceOauthTenantParamConfig(+ tenant_id="workspace-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ client_params={"client_secret": "disabled-tenant-secret"},+ enabled=False,+ ),+ DatasourceOauthParamConfig(+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ system_credentials={"client_secret": "system-secret"},+ ),+ ]+ )+ manager = StaticProviderManager(_provider())+ handler = RecordingOAuthHandler(+ response=PluginOAuthCredentialsResponse(credentials={"integration_secret": "refreshed"}, expires_at=500)+ )+ verification_calls: list[tuple[str, str]] = []++ def is_plugin_verified(workspace_id: str, plugin_unique_identifier: str) -> bool:+ verification_calls.append((workspace_id, plugin_unique_identifier))+ return verified++ monkeypatch.setattr(credential_adapters_module.PluginService, "is_plugin_verified", is_plugin_verified)+ refresher = _oauth_refresher(sqlite_session_factory, manager=manager, handler=handler)++ if not verified:+ with pytest.raises(DatasourceCredentialError, match="OAuth client is not configured"):+ refresher.refresh(+ workspace_id="workspace-1",+ actor_id="actor-1",+ record=_record(),+ credentials={"integration_secret": "old"},+ )+ assert handler.calls == []+ else:+ refresher.refresh(+ workspace_id="workspace-1",+ actor_id="actor-1",+ record=_record(),+ credentials={"integration_secret": "old"},+ )+ assert handler.calls[0]["system_credentials"] == {"client_secret": "system-secret"}++ assert verification_calls == [("workspace-1", "langgenius/notion_datasource:1.0.0")]+++def test_oauth_refresher_translates_plugin_refresh_failure() -> None:+ handler = RecordingOAuthHandler(error=RuntimeError("plugin unavailable"))+ refresher = OAuthDatasourceCredentialRefresher(+ oauth_clients=StaticOAuthClientResolver(),+ oauth_handler=cast(OAuthHandler, handler),+ )++ with pytest.raises(DatasourceCredentialRefreshError, match="credential-1"):+ refresher.refresh(+ workspace_id="workspace-1",+ actor_id="actor-1",+ record=_record(),+ credentials={"integration_secret": "old"},+ )diff --git a/api/tests/unit_tests/services/data_source/test_notion_import_adapters.py b/api/tests/unit_tests/services/data_source/test_notion_import_adapters.pynew file mode 100644index 00000000000..da149afe07f--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_notion_import_adapters.py@@ -0,0 +1,202 @@+from collections.abc import Sequence+from unittest.mock import Mock, create_autospec++import pytest++from core.datasource.__base.datasource_runtime import DatasourceRuntime+from core.datasource.entities.datasource_entities import (+ DatasourceProviderType,+ OnlineDocumentInfo,+ OnlineDocumentPage,+ OnlineDocumentPagesMessage,+)+from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin+from core.rag.extractor.notion_extractor import NotionExtractor+from core.rag.models.document import Document+from services.data_source.credential_gateway import (+ ActorDatasourceCredentialResolver,+ DatasourceCredentialError,+ DatasourceCredentialNotFoundError,+)+from services.data_source.entities.notion_import import NotionPageType+from services.data_source.notion_import_adapters import PluginNotionSourceGateway+from services.data_source.notion_import_application_service import NotionImportCredentialUnavailableError+++def _credentials(credentials: dict[str, object], *, error: DatasourceCredentialError | None = None) -> Mock:+ resolver = create_autospec(ActorDatasourceCredentialResolver, instance=True, spec_set=True)+ resolver.resolve.return_value = credentials+ resolver.resolve.side_effect = error+ return resolver+++def _runtime(messages: Sequence[OnlineDocumentPagesMessage] = ()) -> Mock:+ runtime = create_autospec(OnlineDocumentDatasourcePlugin, instance=True)+ runtime.runtime = DatasourceRuntime(tenant_id="workspace-1")+ runtime.get_online_document_pages.return_value = iter(messages)+ runtime.datasource_provider_type.return_value = DatasourceProviderType.ONLINE_DOCUMENT+ return runtime+++def _page(page_id: str, *, with_icon: bool = True, page_type: str = "page") -> OnlineDocumentPage:+ return OnlineDocumentPage(+ page_id=page_id,+ page_name=f"Page {page_id}",+ page_icon={"type": "emoji", "emoji": "📄"} if with_icon else None,+ type=page_type,+ last_edited_time="2026-01-01T00:00:00Z",+ parent_id=None,+ )+++def test_list_authorized_pages_groups_paginated_results_by_workspace() -> None:+ runtime = _runtime(+ messages=(+ OnlineDocumentPagesMessage(+ result=[+ OnlineDocumentInfo(+ workspace_id="w1",+ workspace_name="One",+ workspace_icon=None,+ total=1,+ pages=[_page("p1")],+ )+ ]+ ),+ OnlineDocumentPagesMessage(+ result=[+ OnlineDocumentInfo(+ workspace_id="w2",+ workspace_name="Two",+ workspace_icon=None,+ total=1,+ pages=[_page("p2", with_icon=False)],+ ),+ OnlineDocumentInfo(+ workspace_id="w1",+ workspace_name="One",+ workspace_icon=None,+ total=1,+ pages=[_page("p3")],+ ),+ ]+ ),+ )+ )+ resolver = _credentials({"integration_secret": "secret"})+ loader = Mock(return_value=runtime)+ gateway = PluginNotionSourceGateway(credentials=resolver, runtime_loader=loader)++ workspaces = gateway.list_authorized_pages(+ workspace_id="workspace-1", actor_id="actor-1", credential_id="credential-1"+ )++ assert [workspace.workspace_id for workspace in workspaces] == ["w1", "w2"]+ assert [page.page_id for page in workspaces[0].pages] == ["p1", "p3"]+ assert [page.page_id for page in workspaces[1].pages] == ["p2"]+ assert workspaces[0].pages[0].page_icon is not None+ assert workspaces[0].pages[0].page_icon.emoji == "📄"+ assert workspaces[1].pages[0].page_icon is None+ assert runtime.runtime.credentials == {"integration_secret": "secret"}+ runtime.get_online_document_pages.assert_called_once_with(+ user_id="actor-1",+ datasource_parameters={},+ provider_type=DatasourceProviderType.ONLINE_DOCUMENT,+ )+ resolver.resolve.assert_called_once_with(+ workspace_id="workspace-1",+ actor_id="actor-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+ loader.assert_called_once_with(+ provider_id="langgenius/notion_datasource/notion_datasource",+ datasource_name="notion_datasource",+ tenant_id="workspace-1",+ datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,+ )+++def test_list_authorized_pages_skips_unknown_page_types() -> None:+ runtime = _runtime(+ messages=(+ OnlineDocumentPagesMessage(+ result=[+ OnlineDocumentInfo(+ workspace_id="w1",+ workspace_name="One",+ workspace_icon=None,+ total=2,+ pages=[_page("known"), _page("unknown", page_type="collection")],+ )+ ]+ ),+ )+ )+ gateway = PluginNotionSourceGateway(+ credentials=_credentials({"integration_secret": "secret"}),+ runtime_loader=Mock(return_value=runtime),+ )++ workspaces = gateway.list_authorized_pages(+ workspace_id="workspace-1", actor_id="actor-1", credential_id="credential-1"+ )++ assert [page.page_id for page in workspaces[0].pages] == ["known"]+++def test_preview_requires_secret_and_passes_it_only_to_extractor() -> None:+ factory = create_autospec(NotionExtractor, spec_set=True)+ factory.return_value.extract.return_value = [Document(page_content="one"), Document(page_content="two")]+ gateway = PluginNotionSourceGateway(+ credentials=_credentials({"integration_secret": "secret"}),+ runtime_loader=Mock(),+ extractor_factory=factory,+ )++ content = gateway.preview_page(+ workspace_id="workspace-1",+ actor_id="actor-1",+ credential_id="credential-1",+ page_id="page-1",+ page_type=NotionPageType.PAGE,+ )++ assert content == "one\ntwo"+ extractor_args = factory.call_args.kwargs+ assert extractor_args["notion_obj_id"] == "page-1"+ assert extractor_args["notion_page_type"] == "page"+ assert extractor_args["notion_access_token"] == "secret"+ assert extractor_args["notion_token_loader"]() == "secret"+ assert extractor_args["tenant_id"] == "workspace-1"+++def test_preview_fails_closed_when_secret_is_missing() -> None:+ gateway = PluginNotionSourceGateway(+ credentials=_credentials({}),+ runtime_loader=Mock(),+ )++ with pytest.raises(NotionImportCredentialUnavailableError):+ gateway.preview_page(+ workspace_id="workspace-1",+ actor_id="actor-1",+ credential_id="credential-1",+ page_id="page-1",+ page_type=NotionPageType.PAGE,+ )+++def test_gateway_translates_credential_infrastructure_failure() -> None:+ gateway = PluginNotionSourceGateway(+ credentials=_credentials({}, error=DatasourceCredentialNotFoundError()),+ runtime_loader=Mock(),+ )++ with pytest.raises(NotionImportCredentialUnavailableError):+ gateway.list_authorized_pages(+ workspace_id="workspace-1",+ actor_id="actor-1",+ credential_id="credential-1",+ )diff --git a/api/tests/unit_tests/services/data_source/test_notion_import_application_service.py b/api/tests/unit_tests/services/data_source/test_notion_import_application_service.pynew file mode 100644index 00000000000..e54fb40242a--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_notion_import_application_service.py@@ -0,0 +1,96 @@+from unittest.mock import Mock, create_autospec++import pytest++from machinery.context import RequestContext+from services.data_source.entities.notion_import import AuthorizedNotionPage, NotionPageType, NotionWorkspace+from services.data_source.notion_import_application_service import (+ DatasetIsNotNotionSourceError,+ NotionDatasetReader,+ NotionDocumentBindingReader,+ NotionImportApplicationService,+ NotionSourceGateway,+)+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess+from services.knowledge.resource_scope import DatasetRef++CONTEXT = RequestContext("request-1", None, "account-1", "workspace-1")+DATASET = DatasetRef("workspace-1", "dataset-1")+type ServiceFixture = tuple[NotionImportApplicationService, Mock, Mock, Mock, Mock]+++def fixture() -> ServiceFixture:+ access = create_autospec(DatasetAccess, instance=True, spec_set=True)+ access.require_accessible.return_value = AccessibleDataset(id="dataset-1", workspace_id="workspace-1")+ datasets = create_autospec(NotionDatasetReader, instance=True, spec_set=True)+ datasets.is_notion_dataset.return_value = True+ documents = create_autospec(NotionDocumentBindingReader, instance=True, spec_set=True)+ documents.list_bound_notion_page_ids.return_value = frozenset()+ source = create_autospec(NotionSourceGateway, instance=True, spec_set=True)+ source.list_authorized_pages.return_value = ()+ source.preview_page.return_value = "preview"+ service = NotionImportApplicationService(+ dataset_access=access, datasets=datasets, documents=documents, source=source+ )+ return service, access, datasets, documents, source+++def test_list_pages_preserves_workspaces_and_marks_bound_pages(fixture: ServiceFixture) -> None:+ service, access, datasets, documents, source = fixture+ page_1 = AuthorizedNotionPage("page-1", "One", None, None, NotionPageType.PAGE)+ page_2 = AuthorizedNotionPage("page-2", "Two", None, "page-1", NotionPageType.PAGE)+ source.list_authorized_pages.return_value = (+ NotionWorkspace("notion-workspace-1", "First", None, (page_1,)),+ NotionWorkspace("notion-workspace-2", "Second", None, (page_2,)),+ )+ documents.list_bound_notion_page_ids.return_value = frozenset({"page-2"})++ result = service.list_pages(CONTEXT, credential_id="credential-1", dataset_id="dataset-1")++ assert [workspace.workspace_id for workspace in result.workspaces] == ["notion-workspace-1", "notion-workspace-2"]+ assert result.workspaces[0].pages[0].is_bound is False+ assert result.workspaces[1].pages[0].is_bound is True+ access.require_accessible.assert_called_once_with(CONTEXT, "dataset-1")+ datasets.is_notion_dataset.assert_called_once_with(DATASET)+ documents.list_bound_notion_page_ids.assert_called_once_with(DATASET)+ source.list_authorized_pages.assert_called_once_with(+ workspace_id="workspace-1", actor_id="account-1", credential_id="credential-1"+ )+++def test_list_pages_without_dataset_does_not_read_document_bindings(fixture: ServiceFixture) -> None:+ service, access, _, documents, _ = fixture++ service.list_pages(CONTEXT, credential_id="credential-1")++ access.require_accessible.assert_not_called()+ documents.list_bound_notion_page_ids.assert_not_called()+++def test_list_pages_rejects_non_notion_dataset_before_external_call(fixture: ServiceFixture) -> None:+ service, _, datasets, _, source = fixture+ datasets.is_notion_dataset.return_value = False++ with pytest.raises(DatasetIsNotNotionSourceError):+ service.list_pages(CONTEXT, credential_id="credential-1", dataset_id="dataset-1")++ datasets.is_notion_dataset.assert_called_once_with(DATASET)+ source.list_authorized_pages.assert_not_called()+++def test_preview_passes_explicit_actor_and_workspace(fixture: ServiceFixture) -> None:+ service, _, _, _, source = fixture++ content = service.preview_page(+ CONTEXT, credential_id="credential-1", page_id="page-1", page_type=NotionPageType.PAGE+ )++ assert content == "preview"+ source.preview_page.assert_called_once_with(+ workspace_id="workspace-1",+ actor_id="account-1",+ credential_id="credential-1",+ page_id="page-1",+ page_type=NotionPageType.PAGE,+ )diff --git a/api/tests/unit_tests/services/data_source/test_notion_oauth_gateway.py b/api/tests/unit_tests/services/data_source/test_notion_oauth_gateway.pynew file mode 100644index 00000000000..fd96f1d9bb7--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_notion_oauth_gateway.py@@ -0,0 +1,401 @@+import json+from dataclasses import dataclass+from typing import Never, cast++import httpx+import pytest++from core.helper.ssrf_proxy import SSRFProxy+from core.tools.errors import ToolSSRFError+from services.data_source.notion_oauth_gateway import NotionDataSourceGateway+from services.data_source.oauth_service import DataSourceOAuthError+++@dataclass(frozen=True)+class RejectingSSRFClient:+ error: Exception++ def post(self, *_args: object, **_kwargs: object) -> Never:+ raise self.error+++def _gateway(handler: httpx.MockTransport) -> tuple[NotionDataSourceGateway, httpx.Client]:+ client = httpx.Client(transport=handler)+ return (+ NotionDataSourceGateway(+ client_id="client-id",+ client_secret="client-secret",+ redirect_uri="https://api.example/oauth/data-source/callback/notion",+ http_client=client,+ ),+ client,+ )+++def test_get_authorization_url_encodes_configured_values() -> None:+ gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(500)))+ try:+ result = gateway.get_authorization_url()+ finally:+ client.close()++ assert result == (+ "https://api.notion.com/v1/oauth/authorize?client_id=client-id&response_type=code&"+ "redirect_uri=https%3A%2F%2Fapi.example%2Foauth%2Fdata-source%2Fcallback%2Fnotion&owner=user"+ )+++def test_authorize_exchanges_code_and_builds_source_info_without_persistence() -> None:+ search_types: list[str] = []++ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path == "/v1/oauth/token":+ assert request.headers["authorization"].startswith("Basic ")+ return httpx.Response(+ 200,+ json={+ "access_token": "notion-token",+ "workspace_name": "Notion Workspace",+ "workspace_icon": "https://example/icon.png",+ "workspace_id": "notion-workspace",+ },+ )+ if request.url.path == "/v1/search":+ payload = json.loads(request.content)+ search_types.append(payload["filter"]["value"])+ return httpx.Response(200, json={"results": [], "has_more": False})+ raise AssertionError(f"Unexpected Notion request: {request.url}")++ gateway, client = _gateway(httpx.MockTransport(handler))+ try:+ authorization = gateway.authorize("authorization-code")+ finally:+ client.close()++ assert authorization.access_token == "notion-token"+ assert authorization.source_info == {+ "workspace_name": "Notion Workspace",+ "workspace_icon": "https://example/icon.png",+ "workspace_id": "notion-workspace",+ "pages": [],+ "total": 0,+ }+ assert search_types == ["page", "database"]++[email protected]("token_payload", [{}, {"access_token": ""}, {"access_token": 42}])+def test_authorize_rejects_missing_or_invalid_access_token(token_payload: dict[str, object]) -> None:+ gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(200, json=token_payload)))+ try:+ with pytest.raises(DataSourceOAuthError, match="did not include an access token"):+ gateway.authorize("authorization-code")+ finally:+ client.close()+++def test_authorize_internal_reads_workspace_name_and_builds_source_info() -> None:+ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path == "/v1/users/me":+ assert request.headers["authorization"] == "Bearer notion-token"+ return httpx.Response(+ 200,+ json={"object": "user", "type": "bot", "bot": {"workspace_name": "Internal Workspace"}},+ )+ if request.url.path == "/v1/search":+ return httpx.Response(200, json={"results": [], "has_more": False})+ raise AssertionError(f"Unexpected Notion request: {request.url}")++ gateway, client = _gateway(httpx.MockTransport(handler))+ try:+ authorization = gateway.authorize_internal("notion-token", "notion-workspace")+ finally:+ client.close()++ assert authorization.access_token == "notion-token"+ assert authorization.source_info == {+ "workspace_name": "Internal Workspace",+ "workspace_icon": None,+ "workspace_id": "notion-workspace",+ "pages": [],+ "total": 0,+ }+++ ("operation", "expected_message"),+ [+ ("authorize", "Notion OAuth response was invalid"),+ ("authorize_internal", "Notion integration response was invalid"),+ ("refresh", "Notion integration response was invalid"),+ ],+)+def test_public_operations_translate_malformed_page_payload(+ operation: str,+ expected_message: str,+) -> None:+ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path == "/v1/oauth/token":+ return httpx.Response(200, json={"access_token": "notion-token"})+ if request.url.path == "/v1/users/me":+ return httpx.Response(200, json={"object": "user", "type": "bot", "bot": {}})+ if request.url.path == "/v1/search":+ payload = json.loads(request.content)+ results: list[dict[str, object]] = [{}] if payload["filter"]["value"] == "page" else []+ return httpx.Response(200, json={"results": results, "has_more": False})+ raise AssertionError(f"Unexpected Notion request: {request.url}")++ gateway, client = _gateway(httpx.MockTransport(handler))++ def invoke() -> None:+ if operation == "authorize":+ gateway.authorize("authorization-code")+ elif operation == "authorize_internal":+ gateway.authorize_internal("notion-token", "notion-workspace")+ else:+ gateway.refresh("notion-token", {})++ try:+ with pytest.raises(DataSourceOAuthError, match=expected_message):+ invoke()+ finally:+ client.close()+++def test_authorized_pages_map_pagination_icons_and_parent_chains() -> None:+ page_requests: list[dict[str, object]] = []++ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path == "/v1/search":+ payload = json.loads(request.content)+ object_type = payload["filter"]["value"]+ if object_type == "page":+ page_requests.append(payload)+ if "start_cursor" not in payload:+ return httpx.Response(+ 200,+ json={+ "results": [+ {+ "id": "page-1",+ "properties": {"Name": {"title": [{"plain_text": "First page"}]}},+ "icon": {"type": "external", "external": {"url": "/page-icon"}},+ "parent": {"type": "block_id", "block_id": "child-block"},+ }+ ],+ "has_more": True,+ "next_cursor": "next-page",+ },+ )+ return httpx.Response(+ 200,+ json={+ "results": [+ {+ "id": "page-2",+ "properties": {},+ "icon": {"type": "emoji", "emoji": "📄"},+ "parent": {"type": "workspace", "workspace": True},+ }+ ],+ "has_more": False,+ },+ )+ return httpx.Response(+ 200,+ json={+ "results": [+ {+ "id": "database-1",+ "title": [{"plain_text": "Database"}],+ "icon": {"type": "emoji", "emoji": "🗃️"},+ "parent": {"type": "page_id", "page_id": "page-1"},+ },+ {+ "id": "database-2",+ "title": [],+ "icon": None,+ "parent": {"type": "workspace", "workspace": True},+ },+ ],+ "has_more": False,+ },+ )+ if request.url.path == "/v1/blocks/child-block":+ return httpx.Response(200, json={"parent": {"type": "block_id", "block_id": "root-block"}})+ if request.url.path == "/v1/blocks/root-block":+ return httpx.Response(200, json={"parent": {"type": "page_id", "page_id": "parent-page"}})+ raise AssertionError(f"Unexpected Notion request: {request.url}")++ gateway, client = _gateway(httpx.MockTransport(handler))+ try:+ pages = gateway.get_authorized_pages("notion-token")+ finally:+ client.close()++ assert pages == [+ {+ "page_id": "page-1",+ "page_name": "First page",+ "page_icon": {"type": "url", "url": "https://www.notion.so/page-icon"},+ "parent_id": "parent-page",+ "type": "page",+ },+ {+ "page_id": "page-2",+ "page_name": "Untitled",+ "page_icon": {"type": "emoji", "emoji": "📄"},+ "parent_id": "root",+ "type": "page",+ },+ {+ "page_id": "database-1",+ "page_name": "Database",+ "page_icon": {"type": "emoji", "emoji": "🗃️"},+ "parent_id": "page-1",+ "type": "database",+ },+ {+ "page_id": "database-2",+ "page_name": "Untitled",+ "page_icon": None,+ "parent_id": "root",+ "type": "database",+ },+ ]+ assert page_requests == [+ {"filter": {"value": "page", "property": "object"}},+ {"filter": {"value": "page", "property": "object"}, "start_cursor": "next-page"},+ ]+++def test_refresh_preserves_workspace_metadata() -> None:+ gateway, client = _gateway(+ httpx.MockTransport(+ lambda request: (+ httpx.Response(200, json={"results": [], "has_more": False})+ if request.url.path == "/v1/search"+ else httpx.Response(404)+ )+ )+ )+ try:+ source_info = gateway.refresh(+ "notion-token",+ {+ "workspace_name": "Workspace",+ "workspace_icon": None,+ "workspace_id": "notion-workspace",+ "pages": [+ {+ "page_id": "old",+ "page_name": "Old page",+ "page_icon": None,+ "parent_id": "root",+ "type": "page",+ }+ ],+ "total": 1,+ },+ )+ finally:+ client.close()++ assert source_info == {+ "workspace_name": "Workspace",+ "workspace_icon": None,+ "workspace_id": "notion-workspace",+ "pages": [],+ "total": 0,+ }+++def test_search_rejects_missing_cursor_when_more_results_are_declared() -> None:+ requests = 0++ def handler(_request: httpx.Request) -> httpx.Response:+ nonlocal requests+ requests += 1+ return httpx.Response(200, json={"results": [], "has_more": True, "next_cursor": None})++ gateway, client = _gateway(httpx.MockTransport(handler))+ try:+ with pytest.raises(DataSourceOAuthError, match="valid next_cursor"):+ gateway.notion_page_search("notion-token")+ finally:+ client.close()++ assert requests == 1+++def test_search_rejects_non_list_results() -> None:+ gateway, client = _gateway(+ httpx.MockTransport(lambda _request: httpx.Response(200, json={"results": {}, "has_more": False}))+ )+ try:+ with pytest.raises(DataSourceOAuthError, match="search response was invalid"):+ gateway.notion_page_search("notion-token")+ finally:+ client.close()+++ ("payload", "expected_message"),+ [+ ({"parent": []}, "block response was invalid"),+ ({"parent": {"type": "page_id"}}, "block parent was invalid"),+ ({"parent": {"type": "page_id", "page_id": 42}}, "parent identifier was invalid"),+ ],+)+def test_block_parent_rejects_invalid_provider_payload(+ payload: dict[str, object],+ expected_message: str,+) -> None:+ gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(200, json=payload)))+ try:+ with pytest.raises(DataSourceOAuthError, match=expected_message):+ gateway.notion_block_parent_page_id("notion-token", "block-1")+ finally:+ client.close()+++def test_provider_http_failure_is_translated_to_application_boundary_error() -> None:+ gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(503, text="unavailable")))+ try:+ with pytest.raises(DataSourceOAuthError, match="unsuccessful response"):+ gateway.authorize("authorization-code")+ finally:+ client.close()+++def test_provider_non_object_json_is_rejected() -> None:+ gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(200, json=[])))+ try:+ with pytest.raises(DataSourceOAuthError, match="response was invalid"):+ gateway.authorize("authorization-code")+ finally:+ client.close()+++def test_provider_get_failure_is_translated_to_application_boundary_error() -> None:+ def handler(request: httpx.Request) -> httpx.Response:+ raise httpx.ConnectError("offline", request=request)++ gateway, client = _gateway(httpx.MockTransport(handler))+ try:+ with pytest.raises(DataSourceOAuthError, match="Notion request failed"):+ gateway.authorize_internal("notion-token", "notion-workspace")+ finally:+ client.close()+++def test_ssrf_rejection_is_translated_to_application_boundary_error() -> None:+ client = RejectingSSRFClient(ToolSSRFError("blocked"))+ gateway = NotionDataSourceGateway(+ client_id="client-id",+ client_secret="client-secret",+ redirect_uri="https://api.example/oauth/data-source/callback/notion",+ http_client=cast(SSRFProxy, client),+ )++ with pytest.raises(DataSourceOAuthError, match="Notion request failed"):+ gateway.authorize("authorization-code")diff --git a/api/tests/unit_tests/services/data_source/test_oauth_service.py b/api/tests/unit_tests/services/data_source/test_oauth_service.pynew file mode 100644index 00000000000..66ebb3dfc67--- /dev/null+++ b/api/tests/unit_tests/services/data_source/test_oauth_service.py@@ -0,0 +1,236 @@+from collections.abc import Mapping+from dataclasses import dataclass, field++import pytest++from machinery.context import RequestContext+from services.data_source.entities.oauth import (+ DataSourceOAuthAuthorization,+ DataSourceOAuthBindingRecord,+ DataSourceOAuthCallback,+)+from services.data_source.oauth_service import (+ DataSourceOAuthConfigurationError,+ DataSourceOAuthError,+ DataSourceOAuthService,+ InvalidDataSourceOAuthCodeError,+)+++def _authorization() -> DataSourceOAuthAuthorization:+ return DataSourceOAuthAuthorization(access_token="token", source_info={"pages": []})+++@dataclass+class RecordingProviderGateway:+ authorization_url: str = "https://notion.example/authorize"+ authorization: DataSourceOAuthAuthorization = field(default_factory=_authorization)+ refreshed_source_info: Mapping[str, object] = field(default_factory=lambda: {"pages": [{"page_id": "page-1"}]})+ events: list[tuple[str, object]] = field(default_factory=list)++ def get_authorization_url(self) -> str:+ self.events.append(("get_authorization_url", ()))+ return self.authorization_url++ def authorize(self, code: str) -> DataSourceOAuthAuthorization:+ self.events.append(("authorize", code))+ return self.authorization++ def authorize_internal(self, access_token: str, workspace_id: str) -> DataSourceOAuthAuthorization:+ self.events.append(("authorize_internal", (access_token, workspace_id)))+ return self.authorization++ def refresh(self, access_token: str, source_info: Mapping[str, object]) -> Mapping[str, object]:+ self.events.append(("refresh", (access_token, source_info)))+ return self.refreshed_source_info+++@dataclass+class InMemoryBindingRepository:+ binding: DataSourceOAuthBindingRecord | None = None+ update_result: bool = True+ events: list[tuple[str, object]] = field(default_factory=list)++ def upsert_authorization(+ self,+ *,+ workspace_id: str,+ provider: str,+ authorization: DataSourceOAuthAuthorization,+ ) -> None:+ self.events.append(("persist", (workspace_id, provider, authorization)))++ def get_enabled(+ self,+ *,+ workspace_id: str,+ provider: str,+ binding_id: str,+ ) -> DataSourceOAuthBindingRecord | None:+ self.events.append(("read", (workspace_id, provider, binding_id)))+ return self.binding++ def update_source_info(+ self,+ *,+ workspace_id: str,+ provider: str,+ binding_id: str,+ source_info: Mapping[str, object],+ ) -> bool:+ self.events.append(("update", (workspace_id, provider, binding_id, source_info)))+ return self.update_result+++def request_context() -> RequestContext:+ return RequestContext(+ request_id="request-1",+ trace_id="trace-1",+ account_id="account-1",+ active_workspace_id="workspace-1",+ )+++def test_start_authorization_delegates_to_external_provider(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ provider = RecordingProviderGateway(events=events)+ bindings = InMemoryBindingRepository(events=events)+ service = DataSourceOAuthService(provider_name="notion", provider_gateway=provider, bindings=bindings)++ result = service.start_authorization(request_context)++ assert result == "https://notion.example/authorize"+ assert events == [("get_authorization_url", ())]+++def test_start_authorization_persists_internal_authorization(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ authorization = DataSourceOAuthAuthorization(access_token="secret", source_info={"pages": []})+ provider = RecordingProviderGateway(authorization=authorization, events=events)+ bindings = InMemoryBindingRepository(events=events)+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=provider,+ bindings=bindings,+ is_internal_provider=True,+ internal_access_token="secret",+ )++ result = service.start_authorization(request_context)++ assert result == "internal"+ assert events == [+ ("authorize_internal", ("secret", "workspace-1")),+ ("persist", ("workspace-1", "notion", authorization)),+ ]+++def test_start_authorization_rejects_missing_internal_secret(request_context: RequestContext) -> None:+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=RecordingProviderGateway(),+ bindings=InMemoryBindingRepository(),+ is_internal_provider=True,+ )++ with pytest.raises(DataSourceOAuthConfigurationError, match="Internal secret is not set"):+ service.start_authorization(request_context)+++ ("code", "error", "expected_error"),+ [+ ("auth/code", None, None),+ (None, "access denied", "access denied"),+ (None, None, "Access denied"),+ ],+)+def test_complete_callback_returns_framework_neutral_result(+ code: str | None,+ error: str | None,+ expected_error: str | None,+) -> None:+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=RecordingProviderGateway(),+ bindings=InMemoryBindingRepository(),+ )++ assert service.complete_callback(code=code, error=error) == DataSourceOAuthCallback(+ provider="notion",+ code=code,+ error=expected_error,+ )+++def test_bind_authorizes_before_persisting(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ authorization = _authorization()+ provider = RecordingProviderGateway(authorization=authorization, events=events)+ bindings = InMemoryBindingRepository(events=events)+ service = DataSourceOAuthService(provider_name="notion", provider_gateway=provider, bindings=bindings)++ service.bind(request_context, code="code-1")++ assert events == [+ ("authorize", "code-1"),+ ("persist", ("workspace-1", "notion", authorization)),+ ]+++def test_bind_rejects_empty_code(request_context: RequestContext) -> None:+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=RecordingProviderGateway(),+ bindings=InMemoryBindingRepository(),+ )++ with pytest.raises(InvalidDataSourceOAuthCodeError):+ service.bind(request_context, code="")+++def test_sync_reads_before_provider_io_and_persists_afterward(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ binding = DataSourceOAuthBindingRecord(id="binding-1", access_token="token", source_info={"pages": []})+ refreshed_source_info = {"pages": [{"page_id": "page-1"}]}+ bindings = InMemoryBindingRepository(binding=binding, events=events)+ provider = RecordingProviderGateway(refreshed_source_info=refreshed_source_info, events=events)+ service = DataSourceOAuthService(provider_name="notion", provider_gateway=provider, bindings=bindings)++ service.sync(request_context, binding_id="binding-1")++ assert events == [+ ("read", ("workspace-1", "notion", "binding-1")),+ ("refresh", ("token", {"pages": []})),+ ("update", ("workspace-1", "notion", "binding-1", refreshed_source_info)),+ ]+++def test_sync_rejects_missing_binding_without_provider_io(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=RecordingProviderGateway(events=events),+ bindings=InMemoryBindingRepository(events=events),+ )++ with pytest.raises(DataSourceOAuthError, match="Data source binding not found"):+ service.sync(request_context, binding_id="missing")++ assert events == [("read", ("workspace-1", "notion", "missing"))]+++def test_sync_rejects_binding_removed_before_update(request_context: RequestContext) -> None:+ events: list[tuple[str, object]] = []+ binding = DataSourceOAuthBindingRecord(id="binding-1", access_token="token", source_info={"pages": []})+ service = DataSourceOAuthService(+ provider_name="notion",+ provider_gateway=RecordingProviderGateway(events=events),+ bindings=InMemoryBindingRepository(binding=binding, update_result=False, events=events),+ )++ with pytest.raises(DataSourceOAuthError, match="Data source binding not found"):+ service.sync(request_context, binding_id="binding-1")++ assert [event for event, _ in events] == ["read", "refresh", "update"]diff --git a/api/tests/unit_tests/services/dataset_service/test_save_document_with_dataset_id_mysql_snapshot.py b/api/tests/unit_tests/services/dataset_service/test_save_document_with_dataset_id_mysql_snapshot.pyindex e8af9b89ca1..62b120a0d31 100644--- a/api/tests/unit_tests/services/dataset_service/test_save_document_with_dataset_id_mysql_snapshot.py+++ b/api/tests/unit_tests/services/dataset_service/test_save_document_with_dataset_id_mysql_snapshot.py@@ -20,7 +20,7 @@ from __future__ import annotationsimport inspect-from services.dataset_service import DocumentService+from services.knowledge.dataset_service import DocumentServicedef test_session_committed_before_upload_file_lookup() -> None:@@ -37,13 +37,13 @@ def test_session_committed_before_upload_file_lookup() -> None:invariant the reproduction in the issue depends on, without needinga live MySQL database."""- source_lines = inspect.getsource(DocumentService.save_document_with_dataset_id).splitlines()+ source_lines = inspect.getsource(DocumentService.save_prepared_documents).splitlines()upload_select_line = next((i for i, line in enumerate(source_lines) if "select(UploadFile)" in line),None,)- assert upload_select_line is not None, "save_document_with_dataset_id must contain a select(UploadFile) call"+ assert upload_select_line is not None, "save_prepared_documents must contain a select(UploadFile) call"# Find every ``session.commit()`` call, then keep the last one# whose line number is BEFORE the UploadFile SELECT. If the fix is@@ -56,7 +56,7 @@ def test_session_committed_before_upload_file_lookup() -> None:last_commit_before_upload = iassert last_commit_before_upload >= 0, (- "save_document_with_dataset_id must call session.commit() at least once before the UploadFile SELECT"+ "save_prepared_documents must call session.commit() at least once before the UploadFile SELECT")# The fix is a focused call: one extra ``session.commit()`` between@@ -66,5 +66,5 @@ def test_session_committed_before_upload_file_lookup() -> None:# doesn't accidentally delete the snapshot refresh without# understanding why it's there.assert "#41735" in "\n".join(source_lines), (- "The fix's invariant comment referencing issue #41735 must be present in save_document_with_dataset_id"+ "The fix's invariant comment referencing issue #41735 must be present in save_prepared_documents")diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.pyindex 6153afa3396..6b554f85bb8 100644--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py@@ -2,7 +2,7 @@These factories and lightweight builders are reused across the dataset,document, and segment service test modules that exercise-``api/services/dataset_service.py``.+``api/services/knowledge/dataset_service.py``."""import json@@ -31,28 +31,6 @@ from models.dataset import (DocumentSegment,)from models.model import UploadFile-from services.dataset_service import (- DatasetCollectionBindingService,- DatasetPermissionService,- DatasetService,- DocumentService,- SegmentService,-)-from services.entities.knowledge_entities.knowledge_entities import (- ChildChunkUpdateArgs,- DataSource,- FileInfo,- InfoList,- KnowledgeConfig,- NotionIcon,- NotionInfo,- NotionPage,- ProcessRule,- RerankingModel,- RetrievalModel,- SegmentUpdateArgs,- WebsiteInfo,-)from services.entities.knowledge_entities.rag_pipeline_entities import (IconInfo as PipelineIconInfo,)@@ -71,6 +49,27 @@ from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingfrom services.errors.dataset import DatasetNameDuplicateErrorfrom services.errors.document import DocumentIndexingErrorfrom services.errors.file import FileNotExistsError+from services.knowledge.dataset_service import (+ DatasetCollectionBindingService,+ DatasetPermissionService,+ DatasetService,+ DocumentService,+ SegmentService,+)+from services.knowledge.entities.knowledge_entities import (+ DataSource,+ FileInfo,+ InfoList,+ KnowledgeConfig,+ NotionIcon,+ NotionInfo,+ NotionPage,+ ProcessRule,+ RerankingModel,+ RetrievalModel,+ WebsiteInfo,+)+from services.knowledge.entities.segments import ChildChunkUpdateArgs, SegmentUpdateArgs__all__ = ["Account",diff --git a/api/tests/unit_tests/services/document_service_validation.py b/api/tests/unit_tests/services/document_service_validation.pyindex ff571eb856c..8ec2b3efc50 100644--- a/api/tests/unit_tests/services/document_service_validation.py+++ b/api/tests/unit_tests/services/document_service_validation.py@@ -116,8 +116,8 @@ from core.rag.entities import PreProcessingRule, Rule, Segmentationfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom graphon.model_runtime.entities.model_entities import ModelTypefrom models.dataset import Dataset, DatasetProcessRule, Document-from services.dataset_service import DatasetService, DocumentService-from services.entities.knowledge_entities.knowledge_entities import (+from services.knowledge.dataset_service import DatasetService, DocumentService+from services.knowledge.entities.knowledge_entities import (DataSource,FileInfo,InfoList,@@ -419,7 +419,7 @@ class TestDatasetServiceCheckDatasetModelSetting:Provides a mocked ModelManager that can be used to verifymodel instance retrieval and error handling."""- with patch("services.dataset_service.ModelManager.for_tenant") as mock_manager:+ with patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_manager:yield mock_managerdef test_check_dataset_model_setting_high_quality_success(self, mock_model_manager):@@ -568,7 +568,7 @@ class TestDatasetServiceCheckEmbeddingModelSetting:Provides a mocked ModelManager that can be used to verifymodel instance retrieval and error handling."""- with patch("services.dataset_service.ModelManager.for_tenant") as mock_manager:+ with patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_manager:yield mock_managerdef test_check_embedding_model_setting_success(self, mock_model_manager):@@ -690,7 +690,7 @@ class TestDatasetServiceCheckRerankingModelSetting:Provides a mocked ModelManager that can be used to verifymodel instance retrieval and error handling."""- with patch("services.dataset_service.ModelManager.for_tenant") as mock_manager:+ with patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_manager:yield mock_managerdef test_check_reranking_model_setting_success(self, mock_model_manager):diff --git a/api/tests/unit_tests/services/knowledge/__init__.py b/api/tests/unit_tests/services/knowledge/__init__.pyindex e69de29bb2d..71872abbaa3 100644--- a/api/tests/unit_tests/services/knowledge/__init__.py+++ b/api/tests/unit_tests/services/knowledge/__init__.py@@ -0,0 +1 @@+"""Tests for knowledge application services and adapters."""diff --git a/api/tests/unit_tests/services/knowledge/test_api_key_service.py b/api/tests/unit_tests/services/knowledge/test_api_key_service.pyindex f725f387de4..ce844001ffe 100644--- a/api/tests/unit_tests/services/knowledge/test_api_key_service.py+++ b/api/tests/unit_tests/services/knowledge/test_api_key_service.py@@ -7,12 +7,14 @@ import pytestfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker+from extensions.application_services.datasets import build_dataset_dependenciesfrom extensions.application_services.knowledge import build_dataset_api_key_servicefrom machinery.context import RequestContextfrom models.account import Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, DatasetPermissionfrom models.enums import PermissionEnumfrom models.model import ApiToken+from repositories.workspace_member_query_repository import WorkspaceMemberQueryRepositoryfrom services.api_token_service import ApiTokenCachefrom services.auth.api_key_contracts import ApiKeyNotFoundErrorfrom services.errors.account import NoPermissionError@@ -36,7 +38,14 @@ def service(])sqlite_session.commit()- return build_dataset_api_key_service(database_client=sqlite_session_factory)+ dataset_access = build_dataset_dependencies(+ database_client=sqlite_session_factory,+ workspace_roles=WorkspaceMemberQueryRepository(session_factory=sqlite_session_factory),+ ).access+ return build_dataset_api_key_service(+ database_client=sqlite_session_factory,+ dataset_access=dataset_access,+ )@pytest.mark.parametrize("workspace", [False, True])diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_access_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_access_service.pynew file mode 100644index 00000000000..40a6bf92622--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_dataset_access_service.py@@ -0,0 +1,171 @@+from dataclasses import dataclass++import pytest++from machinery.context import RequestContext+from services.knowledge.dataset_access import (+ DatasetAccessDeniedError,+ DatasetAccessRecord,+ DatasetAccessService,+ DatasetAccessSnapshot,+ DatasetNotFoundError,+)+++def _context(*, actor_id: str = "actor-1", workspace_id: str = "workspace-1") -> RequestContext:+ return RequestContext(+ request_id="request-1",+ trace_id=None,+ account_id=actor_id,+ active_workspace_id=workspace_id,+ )+++def _snapshot(+ *,+ permission: str,+ maintainer_id: str = "maintainer-1",+ partial_access: bool = False,+) -> DatasetAccessSnapshot:+ return DatasetAccessSnapshot(+ dataset=DatasetAccessRecord(+ id="dataset-1",+ workspace_id="workspace-1",+ maintainer_id=maintainer_id,+ permission=permission,+ ),+ actor_has_partial_access=partial_access,+ )+++@dataclass+class DatasetReaderStub:+ snapshot: DatasetAccessSnapshot | None++ def get_access_snapshot(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ actor_id: str,+ ) -> DatasetAccessSnapshot | None:+ assert workspace_id == "workspace-1"+ assert dataset_id == "dataset-1"+ assert actor_id == "actor-1"+ return self.snapshot+++@dataclass+class WorkspaceRoleReaderStub:+ role: str | None+ calls: int = 0++ def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None:+ assert workspace_id == "workspace-1"+ assert account_id == "actor-1"+ self.calls += 1+ return self.role+++ ("permission", "role", "maintainer_id", "partial_access"),+ [+ ("only_me", "owner", "maintainer-1", False),+ ("only_me", "normal", "actor-1", False),+ ("all_team_members", "normal", "maintainer-1", False),+ ("partial_members", "normal", "maintainer-1", True),+ ],+)+def test_require_accessible_allows_each_legacy_access_path(+ permission: str,+ role: str,+ maintainer_id: str,+ partial_access: bool,+) -> None:+ roles = WorkspaceRoleReaderStub(role)+ service = DatasetAccessService(+ datasets=DatasetReaderStub(+ _snapshot(+ permission=permission,+ maintainer_id=maintainer_id,+ partial_access=partial_access,+ )+ ),+ workspace_roles=roles,+ legacy_permissions_enabled=True,+ )++ result = service.require_accessible(_context(), "dataset-1")++ assert result.id == "dataset-1"+ assert roles.calls == 1+++ ("permission", "partial_access"),+ [+ ("only_me", False),+ ("partial_members", False),+ ("unknown", True),+ ],+)+def test_require_accessible_denies_unrecognized_or_ungranted_legacy_access(+ permission: str,+ partial_access: bool,+) -> None:+ service = DatasetAccessService(+ datasets=DatasetReaderStub(_snapshot(permission=permission, partial_access=partial_access)),+ workspace_roles=WorkspaceRoleReaderStub("normal"),+ legacy_permissions_enabled=True,+ )++ with pytest.raises(DatasetAccessDeniedError):+ service.require_accessible(_context(), "dataset-1")+++ ("permission", "maintainer_id", "partial_access"),+ [+ ("only_me", "actor-1", False),+ ("all_team_members", "maintainer-1", False),+ ("partial_members", "maintainer-1", True),+ ],+)+def test_require_accessible_denies_non_members_even_when_dataset_policy_would_allow(+ permission: str,+ maintainer_id: str,+ partial_access: bool,+) -> None:+ service = DatasetAccessService(+ datasets=DatasetReaderStub(+ _snapshot(permission=permission, maintainer_id=maintainer_id, partial_access=partial_access)+ ),+ workspace_roles=WorkspaceRoleReaderStub(None),+ legacy_permissions_enabled=True,+ )++ with pytest.raises(DatasetAccessDeniedError):+ service.require_accessible(_context(), "dataset-1")+++def test_require_accessible_only_enforces_owner_chain_when_legacy_permissions_are_disabled() -> None:+ roles = WorkspaceRoleReaderStub("normal")+ service = DatasetAccessService(+ datasets=DatasetReaderStub(_snapshot(permission="only_me")),+ workspace_roles=roles,+ legacy_permissions_enabled=False,+ )++ assert service.require_accessible(_context(), "dataset-1").id == "dataset-1"+ assert roles.calls == 0+++def test_require_accessible_hides_missing_or_cross_workspace_dataset() -> None:+ service = DatasetAccessService(+ datasets=DatasetReaderStub(None),+ workspace_roles=WorkspaceRoleReaderStub("owner"),+ legacy_permissions_enabled=True,+ )++ with pytest.raises(DatasetNotFoundError):+ service.require_accessible(_context(), "dataset-1")diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.pynew file mode 100644index 00000000000..06d1eb39524--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py@@ -0,0 +1,379 @@+from collections.abc import Callable, Iterator+from unittest.mock import patch++import pytest+from sqlalchemy import func, select+from sqlalchemy.orm import Session, sessionmaker++from controllers.console.datasets.datasets import (+ DatasetDetailResponse,+ DatasetListResponse,+ DatasetQueryListResponse,+ RelatedAppListResponse,+)+from extensions.application_services.app import AppServices+from machinery.context import RequestContext+from models import Account, App, Dataset, Document+from models.account import Tenant, TenantAccountJoin, TenantAccountRole+from models.dataset import (+ AppDatasetJoin,+ DatasetPermission,+ DatasetPermissionEnum,+ DatasetQuery,+ DocumentSegment,+)+from models.enums import CreatorUserRole, DatasetQuerySource, IndexingStatus, SegmentStatus+from services.enterprise import rbac_service+from services.knowledge.dataset_access import DatasetNotFoundError+from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations+from services.knowledge.datasets.application import DatasetListFilter+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations+from services.knowledge.entities.document_creation import DocumentIndexingJobs+from services.knowledge.entities.knowledge_entities import KnowledgeConfig+from services.knowledge.resource_scope import DatasetRef+from services.tag_application_service import CreateTagInput, TagApplicationService, TagBindingInput+from tests.unit_tests.config_override import apply_config_overrides++CONTEXT = RequestContext("request", None, "actor", "tenant")+REF = DatasetRef("tenant", "dataset")+++def dataset(**values: object) -> Dataset:+ return Dataset(+ **{+ "id": "dataset",+ "tenant_id": "tenant",+ "name": "Dataset",+ "created_by": "actor",+ "maintainer": "actor",+ "indexing_technique": "economy",+ "permission": "all_team_members",+ **values,+ }+ )+++def document(**values: object) -> Document:+ return Document(+ **{+ "id": "document",+ "tenant_id": "tenant",+ "dataset_id": "dataset",+ "position": 1,+ "data_source_type": "local_file",+ "batch": "batch",+ "name": "Document",+ "created_from": "web",+ "created_by": "actor",+ "indexing_status": IndexingStatus.ERROR,+ **values,+ }+ )+++def operations(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ application_tags: TagApplicationService,+ app_services: AppServices,+) -> Iterator[SQLAlchemyDatasetOperations]:+ apply_config_overrides(monkeypatch, RBAC_ENABLED=False)+ with sqlite_session_factory.begin() as session:+ tenant = Tenant(name="Workspace")+ tenant.id = "tenant"+ account = Account(name="Actor", email="[email protected]")+ account.id = "actor"+ session.add_all(+ [+ tenant,+ account,+ TenantAccountJoin(tenant_id="tenant", account_id="actor", role=TenantAccountRole.OWNER),+ dataset(),+ dataset(id="foreign", tenant_id="other", name="Foreign"),+ ]+ )+ with (+ patch.object(rbac_service.RBACService.MyPermissions, "get", return_value=rbac_service.MyPermissionsResponse()),+ patch.object(+ rbac_service.RBACService.DatasetPermissions, "batch_get", return_value={"dataset": ["dataset.preview"]}+ ),+ patch.object(rbac_service, "try_sync_creator_access_policy_member_bindings"),+ ):+ yield SQLAlchemyDatasetOperations(+ session_factory=sqlite_session_factory, tags=application_tags, app_queries=app_services.queries+ )+++def test_listing_materializes_page_and_owner_scoped_partial_members(+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ row = session.get(Dataset, "dataset")+ assert row is not None+ row.permission = DatasetPermissionEnum.PARTIAL_TEAM+ session.add_all(+ [+ DatasetPermission(tenant_id="tenant", dataset_id="dataset", account_id="member"),+ DatasetPermission(tenant_id="other", dataset_id="dataset", account_id="foreign"),+ ]+ )+ result = operations.list_datasets(+ CONTEXT, DatasetListFilter(page=0, limit=1000, ids=["dataset", "foreign"]), None, False+ )+ response = DatasetListResponse.model_validate(result).model_dump(mode="json")+ assert response["total"] == 1+ assert response["page"] == 1+ assert response["limit"] == 100+ assert response["has_more"] is False+ assert response["data"][0]["partial_member_list"] == ["member"]+ assert response["data"][0]["retrieval_model_dict"]["top_k"] == 2++[email protected]("foreign_tag", [False, True])+def test_listing_filters_tags_within_workspace(+ operations: SQLAlchemyDatasetOperations,+ application_tags: TagApplicationService,+ sqlite_session_factory: sessionmaker[Session],+ foreign_tag: bool,+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(dataset(id="untagged", name="Untagged"))+ context = RequestContext("request", None, "actor", "other") if foreign_tag else CONTEXT+ tag = application_tags.create_tag(context, CreateTagInput(name="Selected", type="knowledge"))+ application_tags.create_bindings(+ context,+ TagBindingInput(tag_ids=(tag.id,), target_id="foreign" if foreign_tag else "dataset", type="knowledge"),+ )++ result = operations.list_datasets(CONTEXT, DatasetListFilter(tag_ids=[tag.id]), None, False)+ response = DatasetListResponse.model_validate(result)++ assert [row.id for row in response.data] == ([] if foreign_tag else ["dataset"])+ assert response.total == (0 if foreign_tag else 1)++[email protected](("ids", "own"), [([], False), (["dataset"], False)])+def test_list_visibility_restricts_even_requested_ids(+ operations: SQLAlchemyDatasetOperations, ids: list[str], own: bool, monkeypatch: pytest.MonkeyPatch+) -> None:+ apply_config_overrides(monkeypatch, RBAC_ENABLED=True)+ result = operations.list_datasets(CONTEXT, DatasetListFilter(ids=["dataset"]), ids, own)+ assert result["total"] == len(ids)+++ "method",+ [+ "get_dataset",+ "is_in_use",+ "queries",+ "related_apps",+ "indexing_status",+ "error_documents",+ "partial_members",+ "update_dataset",+ "delete_dataset",+ "set_api_enabled",+ ],+)+def test_wrong_tenant_ref_cannot_read_or_write(operations: SQLAlchemyDatasetOperations, method: str) -> None:+ args: list[object] = (+ [CONTEXT] if method in {"get_dataset", "update_dataset", "delete_dataset", "set_api_enabled"} else []+ )+ args.append(DatasetRef("other", "dataset"))+ if method == "update_dataset":+ args.append({"name": "changed"})+ if method == "set_api_enabled":+ args.append(True)+ extra: dict[str, int] = {"page": 1, "limit": 20} if method == "queries" else {}+ methods: dict[str, Callable[..., object]] = {+ "get_dataset": operations.get_dataset,+ "is_in_use": operations.is_in_use,+ "queries": operations.queries,+ "related_apps": operations.related_apps,+ "indexing_status": operations.indexing_status,+ "error_documents": operations.error_documents,+ "partial_members": operations.partial_members,+ "update_dataset": operations.update_dataset,+ "delete_dataset": operations.delete_dataset,+ "set_api_enabled": operations.set_api_enabled,+ }+ with pytest.raises(DatasetNotFoundError):+ methods[method](*args, **extra)+++def test_create_update_and_api_status_commit_owned_changes(+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ created = operations.create_dataset(+ CONTEXT, {"name": "Created", "description": "desc", "indexing_technique": "economy", "permission": "only_me"}+ )+ assert DatasetDetailResponse.model_validate(created).name == "Created"+ result = operations.update_dataset(CONTEXT, REF, {"name": "Renamed", "permission": "all_team_members"})+ assert result["name"] == "Renamed"+ operations.set_api_enabled(CONTEXT, REF, True)+ with sqlite_session_factory() as session:+ row = session.get(Dataset, "dataset")+ assert row is not None+ assert row.name == "Renamed"+ assert row.enable_api is True+ assert session.get(Dataset, created["id"]) is not None++[email protected]("entry_point", ["empty", "documents"])[email protected]("rbac_enabled", [False, True])+def test_created_dataset_initializes_rbac_access(+ operations: SQLAlchemyDatasetOperations,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ entry_point: str,+ rbac_enabled: bool,+) -> None:+ apply_config_overrides(monkeypatch, RBAC_ENABLED=rbac_enabled)++ def save_documents(+ created_dataset: Dataset, _config: KnowledgeConfig, _account: Account, *, session: Session+ ) -> tuple[list[Document], str, DocumentIndexingJobs]:+ row = document(id="created-document", dataset_id=created_dataset.id, data_source_type="upload_file")+ session.add(row)+ session.flush()+ return [row], "batch", DocumentIndexingJobs(DatasetRef(created_dataset.tenant_id, created_dataset.id))++ with (+ patch.object(rbac_service.RBACService.DatasetAccess, "replace_whitelist") as replace_whitelist,+ patch.object(rbac_service, "try_sync_creator_access_policy_member_bindings") as sync_creator,+ patch(+ "tasks.initialize_created_app_rbac_access_task.initialize_created_app_rbac_access_task.delay"+ ) as initialize,+ patch(+ "services.knowledge.documents.adapters.DocumentService.save_prepared_documents",+ side_effect=save_documents,+ ),+ ):+ if entry_point == "empty":+ result = operations.create_dataset(+ CONTEXT,+ {"name": "Created", "description": "desc", "indexing_technique": "economy", "permission": "only_me"},+ )+ dataset_id = result["id"]+ else:+ result = SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory).initialize_dataset(+ CONTEXT,+ {+ "indexing_technique": "economy",+ "data_source": {+ "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file"]}}+ },+ "process_rule": {"mode": "automatic"},+ },+ )+ dataset_id = result["dataset"]["id"]+ if rbac_enabled:+ replace_whitelist.assert_called_once_with(+ CONTEXT.active_workspace_id,+ CONTEXT.account_id,+ dataset_id,+ rbac_service.ReplaceMemberBindings(automatic_include_workspace_members=False),+ )+ else:+ replace_whitelist.assert_not_called()+ if entry_point == "documents":+ if rbac_enabled:+ sync_creator.assert_called_once_with(+ CONTEXT.active_workspace_id,+ CONTEXT.account_id,+ rbac_service.RBACResourceType.DATASET,+ dataset_id,+ )+ else:+ sync_creator.assert_not_called()+ initialize.assert_not_called()++ with sqlite_session_factory() as session:+ assert session.get(Dataset, dataset_id) is not None+++def test_partial_members_update_and_clear_are_committed(+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ result = operations.update_dataset(+ CONTEXT, REF, {"permission": "partial_members", "partial_member_list": [{"user_id": "actor"}]}+ )+ assert result["partial_member_list"] == ["actor"]+ result = operations.update_dataset(CONTEXT, REF, {"permission": "all_team_members"})+ assert result["partial_member_list"] == []+ with sqlite_session_factory() as session:+ assert session.scalar(select(func.count()).select_from(DatasetPermission)) == 0+++def test_status_counts_and_errors_exclude_foreign_documents(+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ document(),+ document(id="foreign-doc", tenant_id="other"),+ DocumentSegment(+ tenant_id="tenant",+ dataset_id="dataset",+ document_id="document",+ position=1,+ content="content",+ word_count=1,+ tokens=1,+ created_by="actor",+ status=SegmentStatus.WAITING,+ ),+ DocumentSegment(+ tenant_id="other",+ dataset_id="dataset",+ document_id="document",+ position=1,+ content="content",+ word_count=1,+ tokens=1,+ created_by="actor",+ ),+ ]+ )+ result = operations.indexing_status(REF)+ assert len(result["data"]) == 1+ assert result["data"][0]["total_segments"] == 1+ assert result["data"][0]["completed_segments"] == 0+ assert operations.error_documents(REF)["total"] == 1+++def test_queries_and_related_apps_materialize_after_session_close(+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(+ DatasetQuery(+ dataset_id="dataset",+ content="query",+ source=DatasetQuerySource.HIT_TESTING,+ source_app_id=None,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="actor",+ )+ )+ session.add_all(+ [+ App(id="app", tenant_id="tenant", name="App", mode="chat", enable_api=False, enable_site=False),+ App(+ id="foreign-app", tenant_id="other", name="Other", mode="chat", enable_api=False, enable_site=False+ ),+ AppDatasetJoin(app_id="app", dataset_id="dataset"),+ AppDatasetJoin(app_id="foreign-app", dataset_id="dataset"),+ ]+ )+ response = DatasetQueryListResponse.model_validate(operations.queries(REF, page=0, limit=0)).model_dump(mode="json")+ assert response["limit"] == 1+ assert response["has_more"] is False+ assert response["data"][0]["queries"][0]["content"] == "query"+ related = RelatedAppListResponse.model_validate(operations.related_apps(REF)).model_dump(mode="json")+ assert related["total"] == 1+ assert related["data"][0]["id"] == "app"+ assert related["data"][0]["mode"] == "chat"diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_application_service.pynew file mode 100644index 00000000000..391c12abde7--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py@@ -0,0 +1,160 @@+from collections.abc import Callable+from unittest.mock import MagicMock, create_autospec++import pytest++from machinery.context import RequestContext+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError+from services.knowledge.datasets.application import (+ DatasetApplicationService,+ DatasetListFilter,+ DatasetOperations,+ DatasetVisibility,+)+from services.knowledge.datasets.retrieval import retrieval_methods++CONTEXT = RequestContext("request", None, "actor", "tenant")+++def operations() -> MagicMock:+ return create_autospec(DatasetOperations, instance=True, spec_set=True)+++def access() -> MagicMock:+ result = create_autospec(DatasetAccess, instance=True, spec_set=True)+ result.require_accessible.return_value = AccessibleDataset("dataset", "tenant")+ return result+++def service(operations: MagicMock, access: MagicMock) -> DatasetApplicationService:+ return DatasetApplicationService(+ dataset_access=access,+ operations=operations,+ rbac_enabled=True,+ service_api_url="",+ vector_store="milvus",+ tidb_fulltext=False,+ )+++ ("defaults", "workspace", "overrides"),+ [+ (["dataset.preview"], [], {}),+ ([], ["dataset.full_access", "dataset.create_and_management"], {}),+ ([], [], {"hidden": ["dataset.preview"]}),+ ],+)+def test_restricted_whitelist_wins_over_all_other_grants(+ defaults: list[str], workspace: list[str], overrides: dict[str, list[str]]+) -> None:+ visibility = DatasetVisibility(defaults, workspace, overrides, False, ["allowed"])+ assert visibility.list_scope(rbac_enabled=True) == (["allowed"], False)+++ ("visibility", "expected"),+ [+ (DatasetVisibility(default_permissions=["dataset.preview"]), (None, False)),+ (DatasetVisibility(workspace_permissions=["dataset.acl.preview"]), (None, False)),+ (DatasetVisibility(overrides={"a": ["dataset.preview"], "b": ["dataset.edit"]}), (["a"], False)),+ (DatasetVisibility(workspace_permissions=["dataset.create_and_management"]), ([], True)),+ (DatasetVisibility(unrestricted=False), ([], False)),+ ],+)+def test_visibility_scope(visibility: DatasetVisibility, expected: tuple[list[str] | None, bool]) -> None:+ assert visibility.list_scope(rbac_enabled=True) == expected+ assert visibility.list_scope(rbac_enabled=False) == (None, False)+++def test_list_applies_visibility_and_reports_model_availability(+ service: DatasetApplicationService, operations: MagicMock+) -> None:+ operations.visibility.return_value = DatasetVisibility(+ default_permissions=["dataset.preview"],+ overrides={"a": ["dataset.edit"]},+ unrestricted=False,+ whitelist_ids=["a"],+ )+ operations.list_datasets.return_value = {+ "data": [+ {+ "id": "a",+ "indexing_technique": "high_quality",+ "embedding_model": "missing",+ "embedding_model_provider": "provider",+ },+ {"id": "b", "indexing_technique": "economy", "embedding_model": None, "embedding_model_provider": None},+ ]+ }+ operations.embedding_models.return_value = set()+ result = service.list_datasets(CONTEXT, DatasetListFilter(ids=["a"]))+ operations.list_datasets.assert_called_once_with(CONTEXT, DatasetListFilter(ids=["a"]), ["a"], False)+ assert result["data"][0]["embedding_available"] is False+ assert result["data"][0]["permission_keys"] == ["dataset.edit"]+ assert result["data"][1]["embedding_available"] is True+++ ("method", "extra"),+ [+ (DatasetApplicationService.get_dataset, {}),+ (DatasetApplicationService.update_dataset, {"values": {}}),+ (DatasetApplicationService.delete_dataset, {}),+ (DatasetApplicationService.is_in_use, {}),+ (DatasetApplicationService.queries, {"page": 1, "limit": 20}),+ (DatasetApplicationService.related_apps, {}),+ (DatasetApplicationService.indexing_status, {}),+ (DatasetApplicationService.error_documents, {}),+ (DatasetApplicationService.partial_members, {}),+ (DatasetApplicationService.auto_disable_logs, {}),+ (DatasetApplicationService.set_api_enabled, {"status": "enable"}),+ ],+)+def test_access_denial_precedes_all_owned_operations(+ service: DatasetApplicationService,+ access: MagicMock,+ operations: MagicMock,+ method: Callable[..., object],+ extra: dict[str, object],+) -> None:+ access.require_accessible.side_effect = DatasetAccessDeniedError()+ with pytest.raises(DatasetAccessDeniedError):+ method(service, CONTEXT, dataset_id="dataset", **extra)+ assert operations.mock_calls == []+++def test_rbac_create_normalizes_permission_without_mutating_input(+ service: DatasetApplicationService, operations: MagicMock+) -> None:+ values = {"name": "name", "permission": "only_me"}+ service.create_dataset(CONTEXT, values=values)+ operations.create_dataset.assert_called_once_with(CONTEXT, {"name": "name", "permission": "all_team_members"})+ assert values["permission"] == "only_me"+++def test_settings_are_injected_and_url_is_normalized(service: DatasetApplicationService) -> None:+ assert service.api_base_url(CONTEXT, request_base_url="https://api.example/v1/") == "https://api.example/v1"+ assert service.retrieval_settings(CONTEXT)["retrieval_method"] == [+ "semantic_search",+ "full_text_search",+ "hybrid_search",+ ]+ assert service.retrieval_settings(CONTEXT, vector_type="milvus", is_mock=True) == {+ "retrieval_method": ["semantic_search"]+ }++[email protected]("fulltext", [True, False])+def test_tidb_retrieval_setting(fulltext: bool) -> None:+ result = retrieval_methods("tidb_vector", tidb_fulltext=fulltext)+ assert ("full_text_search" in result["retrieval_method"]) is fulltext++[email protected]("store", [None, "unsupported"])+def test_invalid_vector_store(store: str | None) -> None:+ with pytest.raises(ValueError):+ retrieval_methods(store)diff --git a/api/tests/unit_tests/services/knowledge/test_document_adapters.py b/api/tests/unit_tests/services/knowledge/test_document_adapters.pynew file mode 100644index 00000000000..eb85952f5b0--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_document_adapters.py@@ -0,0 +1,293 @@+import json+from datetime import datetime+from unittest.mock import patch++import pytest+from sqlalchemy import Engine, event+from sqlalchemy.orm import Session, sessionmaker++from controllers.console.datasets.datasets_document import DocumentDetailResponse, DocumentWithSegmentsListResponse+from core.rag.index_processor.constant.index_type import IndexTechniqueType+from models.dataset import Dataset, DatasetProcessRule, Document, DocumentPipelineExecutionLog, DocumentSegment+from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations+from services.knowledge.documents.application import DocumentListFilter, DocumentNotFoundError+from services.knowledge.resource_scope import DatasetRef++REF = DatasetRef("tenant-1", "dataset-1")+++def _dataset(**values: object) -> Dataset:+ return Dataset(+ **{+ "id": "dataset-1",+ "tenant_id": "tenant-1",+ "name": "Dataset",+ "created_by": "actor-1",+ "maintainer": "actor-1",+ "indexing_technique": "economy",+ "chunk_structure": "text_model",+ **values,+ }+ )+++def _document(**values: object) -> Document:+ return Document(+ **{+ "id": "document-1",+ "tenant_id": "tenant-1",+ "dataset_id": "dataset-1",+ "position": 1,+ "data_source_type": "local_file",+ "data_source_info": json.dumps({"file_path": "/path/to/file"}),+ "batch": "batch-1",+ "name": "Document",+ "created_from": "web",+ "created_by": "actor-1",+ "indexing_status": IndexingStatus.COMPLETED,+ "doc_form": "text_model",+ "enabled": True,+ "archived": False,+ "word_count": 10,+ "created_at": datetime(2024, 1, 1),+ **values,+ }+ )+++def _segment(+ segment_id: str,+ *,+ tenant_id: str = "tenant-1",+ dataset_id: str = "dataset-1",+ document_id: str = "document-1",+ status: SegmentStatus = SegmentStatus.COMPLETED,+ completed_at: datetime | None = datetime(2024, 1, 1),+ hit_count: int = 0,+) -> DocumentSegment:+ segment = DocumentSegment(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document_id,+ position=1,+ content="content",+ word_count=7,+ tokens=2,+ created_by="actor-1",+ status=status,+ completed_at=completed_at,+ hit_count=hit_count,+ )+ segment.id = segment_id+ return segment+++def operations(sqlite_session_factory: sessionmaker[Session]) -> SQLAlchemyDocumentOperations:+ with sqlite_session_factory.begin() as session:+ session.add_all([_dataset(), _document()])+ return SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory)++[email protected]("foreign", [{"tenant_id": "other"}, {"dataset_id": "other"}])+def test_document_reads_and_writes_reject_incorrect_owner(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session], foreign: dict[str, str]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_document(id="foreign", **foreign))+ ref = REF.document("foreign")+ assert operations.get_state(ref) is None+ with pytest.raises(DocumentNotFoundError):+ operations.get_detail(ref, metadata_only=False)+ with pytest.raises(DocumentNotFoundError):+ operations.update_document(ref, {"name": "changed"})+ with pytest.raises(DocumentNotFoundError):+ operations.rename_document(ref, "changed")+ with sqlite_session_factory() as session:+ document = session.get(Document, "foreign")+ assert document is not None+ assert document.name == "Document"+++def test_document_lookup_uses_active_workspace(operations: SQLAlchemyDocumentOperations) -> None:+ assert operations.find_document(workspace_id="other", document_id="document-1") is None+ assert operations.find_document(workspace_id="tenant-1", document_id="document-1") == REF.document("document-1")+++def test_listing_filters_scope_and_materializes_values(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _document(id="matching", name="Needle", position=2),+ _document(id="disabled", name="Needle", enabled=False),+ _document(id="foreign", name="Needle", tenant_id="other"),+ _document(id="other-dataset", name="Needle", dataset_id="other"),+ ]+ )+ result = operations.list_documents(REF, DocumentListFilter(search="Needle", status="available", fetch=True))+ response = DocumentWithSegmentsListResponse.model_validate(result).model_dump(mode="json")+ assert response["total"] == 1+ assert response["data"][0]["id"] == "matching"+ assert response["data"][0]["data_source_info"] == {"file_path": "/path/to/file"}+ assert response["data"][0]["completed_segments"] == 0+ assert response["data"][0]["total_segments"] == 0+++def test_listing_hit_count_sort_and_exact_last_page(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _document(id="popular", position=2),+ _segment("segment-1", hit_count=1),+ _segment("segment-2", document_id="popular", hit_count=10),+ _segment("foreign", tenant_id="other", hit_count=1000),+ ]+ )+ first = operations.list_documents(REF, DocumentListFilter(sort="-hit_count", limit=1))+ last = operations.list_documents(REF, DocumentListFilter(sort="-hit_count", limit=1, page=2))+ assert [row["id"] for row in first["data"]] == ["popular"]+ assert first["has_more"] is True+ assert last["has_more"] is False+ assert last["total"] == 2+++def test_status_counts_use_one_aggregate_and_complete_owner_chain(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session], sqlite_engine: Engine+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _segment("complete"),+ _segment("waiting", completed_at=None),+ _segment("excluded", status=SegmentStatus.RE_SEGMENT),+ _segment("foreign", tenant_id="other"),+ _segment("wrong-dataset", dataset_id="other"),+ _document(id="second", is_paused=True, indexing_status=IndexingStatus.INDEXING),+ ]+ )+ statements: list[str] = []++ def record(+ _conn: object, _cursor: object, statement: str, _parameters: object, _context: object, _many: bool+ ) -> None:+ statements.append(statement)++ event.listen(sqlite_engine, "before_cursor_execute", record)+ try:+ result = operations.get_batch_indexing_status(REF, "batch-1")+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", record)+ rows = {row["id"]: row for row in result["data"]}+ assert rows["document-1"]["completed_segments"] == 1+ assert rows["document-1"]["total_segments"] == 2+ assert rows["second"]["indexing_status"] == "paused"+ assert sum("FROM document_segments" in statement for statement in statements) == 1+++def test_unknown_batch_is_not_found(operations: SQLAlchemyDocumentOperations) -> None:+ with pytest.raises(DocumentNotFoundError):+ operations.get_batch_indexing_status(REF, "missing")++[email protected]("rules", [None, {"segmentation": {"delimiter": "\\n", "max_tokens": 500}}])+def test_process_rule_preserves_null_and_legacy_delimiter(+ operations: SQLAlchemyDocumentOperations,+ sqlite_session_factory: sessionmaker[Session],+ rules: dict[str, object] | None,+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(+ DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.CUSTOM,+ rules=json.dumps(rules) if rules else None,+ created_by="actor-1",+ )+ )+ result = operations.get_process_rule(REF)+ assert result["mode"] == "custom"+ assert result["rules"] == rules+++def test_detail_is_serializable_after_session_closes(operations: SQLAlchemyDocumentOperations) -> None:+ result = operations.get_detail(REF.document("document-1"), metadata_only=False)+ response = DocumentDetailResponse.model_validate(result).model_dump(mode="json")+ assert response["id"] == "document-1"+ assert response["data_source_info"] == {"file_path": "/path/to/file"}+ assert response["data_source_detail_dict"] == {}+ assert response["created_at"] == int(datetime(2024, 1, 1).timestamp())+++def test_metadata_only_avoids_loading_full_details(operations: SQLAlchemyDocumentOperations) -> None:+ with patch("services.knowledge.documents.adapters.get_document_source_detail") as source:+ result = operations.get_detail(REF.document("document-1"), metadata_only=True)+ assert set(result) == {"id", "doc_type", "doc_metadata"}+ source.assert_not_called()+++def test_update_is_committed_before_returning(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ operations.update_document(REF.document("document-1"), {"is_paused": True, "paused_by": "actor-1"})+ with sqlite_session_factory() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.is_paused is True+ assert document.paused_by == "actor-1"+++def test_latest_pipeline_log_and_empty_result(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ assert operations.get_execution_log(REF.document("document-1")) == {}+ with sqlite_session_factory.begin() as session:+ for year in [2024, 2025]:+ row = DocumentPipelineExecutionLog(+ pipeline_id="pipeline-1",+ document_id="document-1",+ datasource_type="local_file",+ datasource_info=json.dumps({"year": year}),+ datasource_node_id="node-1",+ input_data={"year": year},+ created_by="actor-1",+ )+ row.created_at = datetime(year, 1, 1)+ session.add(row)+ result = operations.get_execution_log(REF.document("document-1"))+ assert result["datasource_info"] == {"year": 2025}+ assert result["input_data"] == {"year": 2025}++[email protected]("settings", [None, {"enable": False}])+def test_summary_guard_rejects_disabled_summary(+ operations: SQLAlchemyDocumentOperations,+ sqlite_session_factory: sessionmaker[Session],+ settings: dict[str, object] | None,+) -> None:+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.summary_index_setting = settings+ with pytest.raises(ValueError, match="not enabled"):+ operations.require_summary_enabled(REF)+++def test_enable_summary_only_updates_requested_owner(+ operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_document(id="foreign", tenant_id="other", need_summary=False))+ operations.enable_summary(REF, ["document-1", "foreign"])+ with sqlite_session_factory() as session:+ document = session.get(Document, "document-1")+ foreign = session.get(Document, "foreign")+ assert document is not None+ assert foreign is not None+ assert document.need_summary is True+ assert foreign.need_summary is Falsediff --git a/api/tests/unit_tests/services/knowledge/test_document_application_service.py b/api/tests/unit_tests/services/knowledge/test_document_application_service.pynew file mode 100644index 00000000000..952d7024896--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_document_application_service.py@@ -0,0 +1,252 @@+from collections.abc import Callable+from dataclasses import replace+from typing import Literal+from unittest.mock import MagicMock, create_autospec++import pytest++from machinery.context import RequestContext+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError+from services.knowledge.documents.application import (+ DatasetDocumentApplicationService,+ DocumentArchivedError,+ DocumentIndexingStateError,+ DocumentInvalidActionError,+ DocumentNotFoundError,+ DocumentOperations,+ DocumentState,+)+from services.knowledge.resource_scope import DatasetRef++CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")+DATASET = DatasetRef("tenant-1", "dataset-1")+DOCUMENT = DATASET.document("document-1")+STATE = DocumentState("document-1", False, "indexing", False, "website_crawl", "text_model", False)+++def operations() -> MagicMock:+ result = create_autospec(DocumentOperations, instance=True, spec_set=True)+ result.get_state.return_value = STATE+ return result+++def access() -> MagicMock:+ result = create_autospec(DatasetAccess, instance=True, spec_set=True)+ result.require_accessible.return_value = AccessibleDataset("dataset-1", "tenant-1")+ return result+++def service(access: MagicMock, operations: MagicMock) -> DatasetDocumentApplicationService:+ return DatasetDocumentApplicationService(+ dataset_access=access, operations=operations, metadata_schema={"book": {"title": str}, "others": {}}+ )+++ "operation",+ [+ DatasetDocumentApplicationService.get_document,+ DatasetDocumentApplicationService.delete_document,+ DatasetDocumentApplicationService.pause_document,+ DatasetDocumentApplicationService.recover_document,+ DatasetDocumentApplicationService.sync_website,+ ],+)+def test_denied_dataset_never_loads_or_mutates_documents(+ service: DatasetDocumentApplicationService,+ access: MagicMock,+ operations: MagicMock,+ operation: Callable[..., object],+) -> None:+ access.require_accessible.side_effect = DatasetAccessDeniedError()+ with pytest.raises(DatasetAccessDeniedError):+ operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")+ assert operations.mock_calls == []+++ "operation",+ [+ DatasetDocumentApplicationService.get_document,+ DatasetDocumentApplicationService.delete_document,+ DatasetDocumentApplicationService.pause_document,+ DatasetDocumentApplicationService.recover_document,+ DatasetDocumentApplicationService.sync_website,+ ],+)+def test_missing_document_stops_before_any_side_effect(+ service: DatasetDocumentApplicationService,+ access: MagicMock,+ operations: MagicMock,+ operation: Callable[..., object],+) -> None:+ operations.get_state.return_value = None+ with pytest.raises(DocumentNotFoundError):+ operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")+ access.require_accessible.assert_called_once_with(CONTEXT, "dataset-1")+ operations.get_state.assert_called_once_with(DOCUMENT)+ assert len(operations.mock_calls) == 1+++ "operation",+ [+ DatasetDocumentApplicationService.pause_document,+ DatasetDocumentApplicationService.recover_document,+ DatasetDocumentApplicationService.sync_website,+ ],+)+def test_archived_documents_cannot_be_reprocessed(+ service: DatasetDocumentApplicationService, operations: MagicMock, operation: Callable[..., object]+) -> None:+ operations.get_state.return_value = replace(STATE, archived=True)+ with pytest.raises(DocumentArchivedError):+ operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")+ assert len(operations.mock_calls) == 1+++ ("action", "status", "paused"),+ [("pause", "indexing", True), ("resume", "paused", False), ("resume", "error", False)],+)+def test_processing_transition_records_explicit_actor(+ service: DatasetDocumentApplicationService,+ operations: MagicMock,+ action: Literal["pause", "resume"],+ status: str,+ paused: bool,+) -> None:+ operations.get_state.return_value = replace(STATE, indexing_status=status)+ service.update_processing(CONTEXT, dataset_id="dataset-1", document_id="document-1", action=action)+ ref, values = operations.update_document.call_args.args+ assert ref == DOCUMENT+ assert values["is_paused"] is paused+ assert values["paused_by"] == (CONTEXT.account_id if paused else None)+ assert (values["paused_at"] is not None) is paused+++ ("action", "status"), [("pause", "completed"), ("resume", "indexing"), ("invalid", "indexing")]+)+def test_invalid_processing_transition_does_not_write(+ service: DatasetDocumentApplicationService, operations: MagicMock, action: Literal["pause", "resume"], status: str+) -> None:+ operations.get_state.return_value = replace(STATE, indexing_status=status)+ with pytest.raises(DocumentInvalidActionError):+ service.update_processing(CONTEXT, dataset_id="dataset-1", document_id="document-1", action=action)+ operations.update_document.assert_not_called()+++ ("doc_type", "expected"), [("book", {"title": "Title"}), ("others", {"title": "Title", "unknown": 12})]+)+def test_metadata_policy_filters_schema_but_preserves_custom_fields(+ service: DatasetDocumentApplicationService, operations: MagicMock, doc_type: str | None, expected: dict[str, object]+) -> None:+ service.update_metadata(+ CONTEXT,+ dataset_id="dataset-1",+ document_id="document-1",+ doc_type=doc_type,+ doc_metadata={"title": "Title", "unknown": 12},+ )+ ref, values = operations.update_document.call_args.args+ assert ref == DOCUMENT+ assert values["doc_metadata"] == expected+ assert values["doc_type"] == doc_type++[email protected](("doc_type", "metadata"), [(None, {}), ("book", None), ("invalid", {})])+def test_invalid_metadata_never_writes(+ service: DatasetDocumentApplicationService,+ operations: MagicMock,+ doc_type: str | None,+ metadata: dict[str, object] | None,+) -> None:+ with pytest.raises(ValueError):+ service.update_metadata(+ CONTEXT, dataset_id="dataset-1", document_id="document-1", doc_type=doc_type, doc_metadata=metadata+ )+ operations.update_document.assert_not_called()+++def test_retry_filters_missing_archived_completed_and_duplicate_documents(+ service: DatasetDocumentApplicationService, operations: MagicMock+) -> None:+ operations.get_states.return_value = [+ replace(STATE, id="error", indexing_status="error"),+ replace(STATE, id="archived", archived=True),+ replace(STATE, id="completed", indexing_status="completed"),+ ]+ service.retry_documents(+ CONTEXT, dataset_id="dataset-1", document_ids=["archived", "missing", "error", "error", "completed"]+ )+ operations.retry_documents.assert_called_once_with(DATASET, ["error"], actor_id="actor-1")+++def test_summary_skips_qa_and_enables_summary_before_dispatch(+ service: DatasetDocumentApplicationService, operations: MagicMock+) -> None:+ operations.get_states.return_value = [+ STATE,+ replace(STATE, id="qa", doc_form="qa_model"),+ replace(STATE, id="ready", need_summary=True),+ ]+ service.generate_summary(CONTEXT, dataset_id="dataset-1", document_ids=["document-1", "qa", "ready"])+ operations.enable_summary.assert_called_once_with(DATASET, ["document-1"])+ assert [call.args[0] for call in operations.dispatch_summary.call_args_list] == [+ DOCUMENT,+ DATASET.document("ready"),+ ]+ names = [call[0] for call in operations.mock_calls]+ assert names.index("enable_summary") < names.index("dispatch_summary")+++def test_summary_missing_document_cannot_partially_schedule(+ service: DatasetDocumentApplicationService, operations: MagicMock+) -> None:+ operations.get_states.return_value = [STATE]+ with pytest.raises(DocumentNotFoundError, match="missing"):+ service.generate_summary(CONTEXT, dataset_id="dataset-1", document_ids=["document-1", "missing"])+ operations.enable_summary.assert_not_called()+ operations.dispatch_summary.assert_not_called()+++def test_process_rule_resolves_document_inside_active_workspace(+ service: DatasetDocumentApplicationService, operations: MagicMock, access: MagicMock+) -> None:+ operations.find_document.return_value = DOCUMENT+ service.get_process_rule(CONTEXT, document_id="document-1")+ operations.find_document.assert_called_once_with(workspace_id="tenant-1", document_id="document-1")+ access.require_accessible.assert_called_once_with(CONTEXT, "dataset-1")+ operations.get_process_rule.assert_called_once_with(DATASET)+++def test_default_process_rule_does_not_load_a_dataset(+ service: DatasetDocumentApplicationService, operations: MagicMock, access: MagicMock+) -> None:+ service.get_process_rule(CONTEXT, document_id=None)+ access.require_accessible.assert_not_called()+ operations.get_process_rule.assert_called_once_with(None)+++def test_pause_requires_indexing_state_and_recover_requires_pause(+ service: DatasetDocumentApplicationService, operations: MagicMock+) -> None:+ operations.get_state.return_value = replace(STATE, indexing_status="completed")+ with pytest.raises(DocumentIndexingStateError):+ service.pause_document(CONTEXT, dataset_id="dataset-1", document_id="document-1")+ with pytest.raises(DocumentIndexingStateError):+ service.recover_document(CONTEXT, dataset_id="dataset-1", document_id="document-1")+ operations.pause_document.assert_not_called()+ operations.recover_document.assert_not_called()+++def test_website_sync_rejects_other_sources(service: DatasetDocumentApplicationService, operations: MagicMock) -> None:+ operations.get_state.return_value = replace(STATE, data_source_type="upload_file")+ with pytest.raises(ValueError, match="not a website"):+ service.sync_website(CONTEXT, dataset_id="dataset-1", document_id="document-1")+ operations.sync_website.assert_not_called()diff --git a/api/tests/unit_tests/services/knowledge/test_document_creation.py b/api/tests/unit_tests/services/knowledge/test_document_creation.pynew file mode 100644index 00000000000..93ab1df3d0a--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_document_creation.py@@ -0,0 +1,356 @@+"""Exercise document creation with real SQL and assertions at external boundaries."""++from collections.abc import Callable, Generator, Iterator+from contextlib import contextmanager+from datetime import datetime+from unittest.mock import Mock++import pytest+from redis.exceptions import LockNotOwnedError+from sqlalchemy import Engine, event, select+from sqlalchemy.orm import Session, sessionmaker++from core.model_manager import ModelInstance+from core.rag.index_processor.constant.index_type import IndexTechniqueType+from enums import CloudPlan+from machinery.context import RequestContext+from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole+from models.dataset import Dataset, Document, DocumentSegment+from models.enums import IndexingStatus, SegmentStatus+from services.document_indexing_proxy.batch_indexing_base import BatchDocumentIndexingProxy+from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy+from services.document_indexing_proxy.duplicate_document_indexing_task_proxy import DuplicateDocumentIndexingTaskProxy+from services.entities.feature_entities import FeatureModel+from services.errors.file import FileNotExistsError+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations+from services.knowledge.resource_scope import DatasetRef+from tests.unit_tests.model_factories import make_upload_file++CONTEXT = RequestContext("request", None, "actor", "tenant")+REF = DatasetRef("tenant", "dataset")+++def settings(**values: object) -> dict[str, object]:+ return {+ "indexing_technique": "economy",+ "process_rule": {"mode": "automatic"},+ "data_source": {"info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file"]}}},+ **values,+ }+++def operations(+ sqlite_session_factory: sessionmaker[Session], config_overrides: Callable[..., None]+) -> SQLAlchemyDocumentOperations:+ config_overrides(DEPLOYMENT_EDITION="CLOUD", RBAC_ENABLED=False)+ with sqlite_session_factory.begin() as session:+ actor = Account(name="Actor", email="[email protected]")+ actor.id = "actor"+ tenant = Tenant(name="Tenant")+ tenant.id = "tenant"+ session.add_all(+ [+ actor,+ tenant,+ TenantAccountJoin(tenant_id="tenant", account_id="actor", role=TenantAccountRole.OWNER),+ Dataset(+ id="dataset", tenant_id="tenant", name="Dataset", created_by="actor", indexing_technique="economy"+ ),+ make_upload_file(file_id="file", tenant_id="tenant", name="example.txt", created_by="actor"),+ make_upload_file(file_id="foreign-file", tenant_id="foreign", name="foreign.txt"),+ ]+ )+ return SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory)+++def outside_transaction(sqlite_engine: Engine) -> Iterator[Callable[[], None]]:+ active: set[int] = set()++ def begin(connection: object) -> None:+ active.add(id(connection))++ def end(connection: object) -> None:+ active.discard(id(connection))++ def check() -> None:+ assert not active, "External I/O must not hold an open database transaction"++ listeners = [("begin", begin), ("commit", end), ("rollback", end)]+ for name, callback in listeners:+ event.listen(sqlite_engine, name, callback)+ yield check+ for name, callback in listeners:+ event.remove(sqlite_engine, name, callback)+++def external_calls(+ monkeypatch: pytest.MonkeyPatch,+ outside_transaction: Callable[[], None],+ sqlite_session_factory: sessionmaker[Session],+) -> list[tuple[str, tuple[str, ...]]]:+ calls: list[tuple[str, tuple[str, ...]]] = []++ def features(_tenant_id: str, *, exclude_vector_space: bool) -> FeatureModel:+ outside_transaction()+ assert exclude_vector_space is True+ result = FeatureModel()+ result.billing.subscription.plan = CloudPlan.PROFESSIONAL+ return result++ @contextmanager+ def lock(_name: str, *, timeout: int) -> Generator[None, None, None]:+ outside_transaction()+ assert timeout == 600+ yield+ outside_transaction()++ def dispatched(kind: str, dataset_id: str, document_ids: tuple[str, ...]) -> None:+ outside_transaction()+ with sqlite_session_factory() as session:+ for document_id in document_ids:+ document = session.get(Document, document_id)+ assert document is not None+ assert document.dataset_id == dataset_id+ if kind != "clean":+ assert document.indexing_status == IndexingStatus.WAITING+ calls.append((kind, document_ids))++ def create(proxy: BatchDocumentIndexingProxy) -> None:+ dispatched("create", proxy._dataset_id, tuple(proxy._document_ids))++ def duplicate(proxy: BatchDocumentIndexingProxy) -> None:+ dispatched("duplicate", proxy._dataset_id, tuple(proxy._document_ids))++ monkeypatch.setattr("services.knowledge.dataset_service.FeatureService.get_features", features)+ monkeypatch.setattr("services.knowledge.documents.adapters.redis_client.lock", lock)+ monkeypatch.setattr(DocumentIndexingTaskProxy, "delay", create)+ monkeypatch.setattr(DuplicateDocumentIndexingTaskProxy, "delay", duplicate)+ monkeypatch.setattr(+ "services.knowledge.dataset_service.document_indexing_update_task.delay",+ lambda dataset_id, document_id: dispatched("update", dataset_id, (document_id,)),+ )+ monkeypatch.setattr(+ "services.knowledge.dataset_service.clean_notion_document_task.delay",+ lambda document_ids, dataset_id: dispatched("clean", dataset_id, tuple(document_ids)),+ )+ return calls++[email protected]("initialize", [False, True])+def test_upload_preserves_response_and_commits_before_dispatch(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ sqlite_session_factory: sessionmaker[Session],+ initialize: bool,+) -> None:+ result = (+ operations.initialize_dataset(CONTEXT, settings())+ if initialize+ else operations.create_documents(CONTEXT, REF, settings())+ )+ (document,) = result["documents"]+ assert document["name"] == "example.txt"+ assert document["data_source_info_dict"] == {"upload_file_id": "file"}+ assert document["indexing_status"] == "waiting"+ assert result["batch"]+ assert external_calls == [("create", (document["id"],))]+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, result["dataset"]["id"])+ assert dataset is not None+ assert dataset.tenant_id == "tenant"+ assert dataset.name == ("example.txt..." if initialize else "Dataset")+ assert dataset.permission == "only_me"+++def test_duplicate_upload_reuses_document_and_dispatches_duplicate_job(+ operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]+) -> None:+ first = operations.create_documents(CONTEXT, REF, settings())+ second = operations.create_documents(CONTEXT, REF, settings())+ first_id = first["documents"][0]["id"]+ assert second["documents"][0]["id"] == first_id+ assert external_calls == [("create", (first_id,)), ("duplicate", (first_id,))]+++def test_reprocess_checks_model_outside_transaction_and_preserves_segment_reset(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ outside_transaction: Callable[[], None],+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ created = operations.create_documents(CONTEXT, REF, settings())+ document_id = created["documents"][0]["id"]+ with sqlite_session_factory.begin() as session:+ document = session.get(Document, document_id)+ assert document is not None+ document.indexing_status = IndexingStatus.COMPLETED+ document.completed_at = datetime(2026, 1, 1)+ dataset = session.get(Dataset, REF.dataset_id)+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.embedding_model = "embedding"+ dataset.embedding_model_provider = "provider"+ session.add(+ DocumentSegment(+ tenant_id="tenant",+ dataset_id="dataset",+ document_id=document_id,+ position=1,+ content="content",+ word_count=1,+ tokens=1,+ created_by="actor",+ status=SegmentStatus.COMPLETED,+ )+ )+ manager = Mock()+ manager.get_model_instance.side_effect = lambda **_kwargs: outside_transaction()+ monkeypatch.setattr("services.knowledge.dataset_service.ModelManager.for_tenant", lambda **_kwargs: manager)+ result = operations.create_documents(CONTEXT, REF, settings(original_document_id=document_id, name="Renamed"))+ assert result["documents"][0]["id"] == document_id+ assert result["documents"][0]["name"] == "Renamed"+ assert result["batch"] == created["batch"]+ assert external_calls[-1] == ("update", (document_id,))+ manager.get_model_instance.assert_called_once()+ with sqlite_session_factory() as session:+ segment = session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document_id))+ assert segment is not None+ assert segment.status == SegmentStatus.RE_SEGMENT+++def test_first_high_quality_upload_resolves_default_model_without_transaction(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ outside_transaction: Callable[[], None],+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, REF.dataset_id)+ assert dataset is not None+ dataset.indexing_technique = None++ def model(**_kwargs: object) -> ModelInstance:+ outside_transaction()+ instance = object.__new__(ModelInstance)+ instance.model_name = "default-embedding"+ instance.provider = "default-provider"+ return instance++ manager = Mock()+ manager.get_default_model_instance.side_effect = model+ monkeypatch.setattr("services.knowledge.dataset_service.ModelManager.for_tenant", lambda **_kwargs: manager)+ operations.create_documents(CONTEXT, REF, settings(indexing_technique="high_quality"))+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, REF.dataset_id)+ assert dataset is not None+ assert dataset.embedding_model == "default-embedding"+ assert dataset.embedding_model_provider == "default-provider"+ assert dataset.collection_binding_id is not None+ assert len(external_calls) == 1+++def test_foreign_upload_is_rejected_without_dispatch(+ operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]+) -> None:+ config = settings(+ data_source={"info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["foreign-file"]}}}+ )+ with pytest.raises(FileNotExistsError):+ operations.create_documents(CONTEXT, REF, config)+ assert external_calls == []+++def test_notion_removal_is_dispatched_after_new_documents_commit(+ operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]+) -> None:+ def notion(page_id: str) -> dict[str, object]:+ return settings(+ data_source={+ "info_list": {+ "data_source_type": "notion_import",+ "notion_info_list": [+ {+ "credential_id": "credential",+ "workspace_id": "workspace",+ "pages": [{"page_id": page_id, "page_name": page_id, "type": "page"}],+ }+ ],+ }+ }+ )++ first = operations.create_documents(CONTEXT, REF, notion("old-page"))+ old_id = first["documents"][0]["id"]+ operations.create_documents(CONTEXT, REF, notion("new-page"))+ assert external_calls[-2] == ("clean", (old_id,))+ assert external_calls[-1][0] == "create"+ assert old_id not in external_calls[-1][1]+++def test_cloud_quota_rejection_does_not_create_documents_or_dispatch(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ sqlite_session_factory: sessionmaker[Session],+ config_overrides: Callable[..., None],+) -> None:+ config_overrides(BATCH_UPLOAD_LIMIT=0)+ with pytest.raises(ValueError, match="batch upload limit"):+ operations.create_documents(CONTEXT, REF, settings())+ assert external_calls == []+ with sqlite_session_factory() as session:+ assert session.scalar(select(Document.id)) is None+++def test_response_read_failure_still_dispatches_already_committed_documents(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ response = Mock(side_effect=ValueError("response read failed"))+ monkeypatch.setattr("services.knowledge.documents.adapters._created_response", response)+ with pytest.raises(ValueError, match="response read failed"):+ operations.create_documents(CONTEXT, REF, settings())+ assert len(external_calls) == 1+ assert external_calls[0][0] == "create"+++def test_lost_creation_lock_keeps_empty_batch_response(+ operations: SQLAlchemyDocumentOperations,+ external_calls: list[tuple[str, tuple[str, ...]]],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ @contextmanager+ def lock(_name: str, **_kwargs: object) -> Generator[None, None, None]:+ raise LockNotOwnedError("lock lost")+ yield # pragma: no cover++ monkeypatch.setattr("services.knowledge.documents.adapters.redis_client.lock", lock)+ result = operations.create_documents(CONTEXT, REF, settings())+ assert result["documents"] == []+ assert result["batch"]+ assert result["dataset"]["id"] == REF.dataset_id+ assert external_calls == []+++def test_billing_http_boundary_does_not_hold_a_transaction(+ operations: SQLAlchemyDocumentOperations,+ outside_transaction: Callable[[], None],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ class StopBeforeNetworkError(Exception):+ pass++ def request(_method: str, url: str, **_kwargs: object) -> None:+ assert url.endswith("/subscription/info")+ outside_transaction()+ raise StopBeforeNetworkError()++ monkeypatch.setattr("services.billing_service._http_client.request", request)+ with pytest.raises(StopBeforeNetworkError):+ operations.create_documents(CONTEXT, REF, settings())+ outside_transaction()diff --git a/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.pynew file mode 100644index 00000000000..b0885001843--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py@@ -0,0 +1,279 @@+"""Document cleanup keeps SQL transactions closed during external index deletion."""++import json+from unittest.mock import MagicMock+from uuid import uuid4++import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from models.dataset import ChildChunk, Dataset, DatasetKeywordTable, Document, DocumentSegment, DocumentSegmentSummary+from models.enums import DataSourceType, DocumentCreatedFrom+from services.knowledge.indexing.adapters import cleanup+from services.knowledge.summaries import adapters as summaries+from tasks.batch_clean_document_task import batch_clean_document_task+from tasks.clean_document_task import clean_document_task+from tasks.clean_notion_document_task import clean_notion_document_task+++def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk]:+ dataset = Dataset(+ id=str(uuid4()),+ tenant_id=str(uuid4()),+ name="Dataset",+ created_by="admin",+ indexing_technique="high_quality",+ index_struct='{"type":"pgvector"}',+ )+ document = Document(+ id=str(uuid4()),+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.NOTION_IMPORT,+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom.WEB,+ created_by="admin",+ doc_form="text_model",+ )+ segment = DocumentSegment(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ position=1,+ content="Text",+ word_count=1,+ tokens=1,+ created_by="admin",+ index_node_id="body-node",+ )+ summary = DocumentSegmentSummary(+ dataset_id=dataset.id,+ document_id=document.id,+ chunk_id=segment.id,+ summary_content="Summary",+ summary_index_node_id="summary-node",+ )+ child = ChildChunk(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ segment_id=segment.id,+ position=1,+ content="Child",+ word_count=1,+ index_node_id="child-node",+ created_by="admin",+ index_node_hash="child-hash",+ )+ sqlite_session.add_all([dataset, document, segment, summary, child])+ sqlite_session.commit()+ return dataset, document, segment, summary, child+++def vector(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> MagicMock:+ sessions: list[Session] = []++ def new_session() -> Session:+ session = sqlite_session_factory()+ sessions.append(session)+ return session++ def assert_closed(_node_ids: list[str]) -> None:+ assert sessions+ assert all(not session.in_transaction() for session in sessions)++ backend = MagicMock()+ backend.delete_by_ids.side_effect = assert_closed+ monkeypatch.setattr(cleanup, "Vector", MagicMock(return_value=backend))+ monkeypatch.setattr("core.db.session_factory.session_factory.create_session", new_session)+ return backend++[email protected]("task_kind", ["single", "batch", "notion"])[email protected]("doc_form", [IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.PARENT_CHILD_INDEX])+def test_tasks_delete_summary_and_body_indexes_without_an_open_transaction(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ vector: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ task_kind: str,+ doc_form: IndexStructureType,+) -> None:+ dataset, document, segment, summary, child = graph+ document.doc_form = doc_form+ dataset.chunk_structure = doc_form+ ids = (dataset.id, document.id, segment.id, summary.id, child.id)+ if task_kind != "notion":+ sqlite_session.delete(document)+ sqlite_session.commit()+ monkeypatch.setattr("tasks.clean_document_task.schedule_billing_vector_space_refresh", lambda _tenant_id: None)+ monkeypatch.setattr(+ "tasks.batch_clean_document_task.schedule_billing_vector_space_refresh", lambda _tenant_id: None+ )++ if task_kind == "single":+ clean_document_task(ids[1], ids[0], doc_form, None)+ elif task_kind == "batch":+ batch_clean_document_task([ids[1]], ids[0], doc_form, [])+ else:+ clean_notion_document_task([ids[1]], ids[0])++ vector.delete_by_ids.assert_called_once_with(+ ["summary-node", "child-node" if doc_form == IndexStructureType.PARENT_CHILD_INDEX else "body-node"]+ )+ sqlite_session.expire_all()+ assert sqlite_session.get(DocumentSegmentSummary, ids[3]) is None+ assert sqlite_session.get(ChildChunk, ids[4]) is None+ assert sqlite_session.get(DocumentSegment, ids[2]) is None+++def test_failed_vector_cleanup_preserves_records_for_retry(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ vector: MagicMock,+) -> None:+ dataset, document, _, summary, child = graph+ vector.delete_by_ids.side_effect = RuntimeError("unavailable")+ with pytest.raises(RuntimeError, match="unavailable"):+ cleanup.clean_document_indexes(+ dataset_id=dataset.id, document_ids=[document.id], doc_form="text_model", new_session=sqlite_session_factory+ )+ assert sqlite_session.get(DocumentSegmentSummary, summary.id) is not None+ assert sqlite_session.get(ChildChunk, child.id) is not None++[email protected]("task_kind", ["single", "batch", "notion"])+def test_cleanup_retries_summaries_after_segments_have_been_deleted(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ vector: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+ task_kind: str,+) -> None:+ dataset, document, segment, summary, _ = graph+ summary_id = summary.id+ sqlite_session.delete(segment)+ sqlite_session.commit()+ monkeypatch.setattr("tasks.clean_document_task.schedule_billing_vector_space_refresh", lambda _tenant_id: None)+ monkeypatch.setattr(+ "tasks.batch_clean_document_task.schedule_billing_vector_space_refresh", lambda _tenant_id: None+ )+ if task_kind == "single":+ clean_document_task(document.id, dataset.id, "text_model", None)+ elif task_kind == "batch":+ batch_clean_document_task([document.id], dataset.id, "text_model", [])+ else:+ clean_notion_document_task([document.id], dataset.id)+ vector.delete_by_ids.assert_called_once_with(["summary-node"])+ sqlite_session.expire_all()+ assert sqlite_session.get(DocumentSegmentSummary, summary_id) is None+++def test_cleanup_keeps_other_documents_and_datasets(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ dataset, document, _, summary, _ = graph+ summary_id = summary.id+ retained = [+ DocumentSegmentSummary(+ dataset_id=owner_dataset,+ document_id=owner_document,+ chunk_id=str(uuid4()),+ summary_content="Retained",+ summary_index_node_id=f"retained-{index}",+ )+ for index, (owner_dataset, owner_document) in enumerate(+ [+ (dataset.id, str(uuid4())),+ (str(uuid4()), document.id),+ ]+ )+ ]+ sqlite_session.add_all(retained)+ sqlite_session.commit()+ backend = MagicMock()+ monkeypatch.setattr(cleanup, "Vector", MagicMock(return_value=backend))+ cleanup.clean_document_indexes(+ dataset_id=dataset.id, document_ids=[document.id], doc_form="text_model", new_session=sqlite_session_factory+ )+ backend.delete_by_ids.assert_called_once_with(["summary-node", "body-node"])+ sqlite_session.expire_all()+ assert sqlite_session.get(DocumentSegmentSummary, summary_id) is None+ for record in retained:+ assert sqlite_session.get(DocumentSegmentSummary, record.id) is not None+++def test_economy_cleanup_preserves_unrelated_keyword_entries(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ dataset, document, _, summary, _ = graph+ summary_id = summary.id+ dataset.indexing_technique = IndexTechniqueType.ECONOMY+ keyword_table = DatasetKeywordTable(+ dataset_id=dataset.id,+ keyword_table=json.dumps({"__data__": {"table": {"term": ["body-node", "unrelated-node"]}}}),+ )+ sqlite_session.add(keyword_table)+ sqlite_session.commit()+ cleanup.clean_document_indexes(+ dataset_id=dataset.id, document_ids=[document.id], doc_form="text_model", new_session=sqlite_session_factory+ )+ sqlite_session.refresh(keyword_table)+ assert json.loads(keyword_table.keyword_table)["__data__"]["table"] == {"term": ["unrelated-node"]}+ sqlite_session.expire_all()+ assert sqlite_session.get(DocumentSegmentSummary, summary_id) is None+++def test_legacy_summary_delete_does_not_commit_callers_transaction(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ dataset, _, segment, summary, _ = graph+ dataset_id, summary_id = dataset.id, summary.id+ monkeypatch.setattr(summaries, "Vector", MagicMock())++ def rollback_delete() -> None:+ with sqlite_session.begin():+ dataset.name = "Uncommitted name"+ summaries.SummaryIndexAdapter.delete_summaries_for_segments(dataset, [segment.id], session=sqlite_session)+ assert sqlite_session.in_transaction()+ assert sqlite_session.get(DocumentSegmentSummary, summary_id) is None+ raise RuntimeError("rollback caller")++ with pytest.raises(RuntimeError, match="rollback caller"):+ rollback_delete()+ assert sqlite_session.get(DocumentSegmentSummary, summary_id) is not None+ stored_dataset = sqlite_session.get(Dataset, dataset_id)+ assert stored_dataset is not None+ assert stored_dataset.name == "Dataset"+++def test_empty_document_selection_does_not_clear_the_dataset_index(+ graph: tuple[Dataset, Document, DocumentSegment, DocumentSegmentSummary, ChildChunk],+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ vector: MagicMock,+) -> None:+ dataset, _, segment, summary, child = graph+ assert (+ cleanup.clean_document_indexes(+ dataset_id=dataset.id, document_ids=[], doc_form="text_model", new_session=sqlite_session_factory+ )+ is None+ )+ vector.delete_by_ids.assert_not_called()+ assert sqlite_session.get(DocumentSegment, segment.id) is not None+ assert sqlite_session.get(DocumentSegmentSummary, summary.id) is not None+ assert sqlite_session.get(ChildChunk, child.id) is not Nonediff --git a/api/tests/unit_tests/services/knowledge/test_document_sync_application_service.py b/api/tests/unit_tests/services/knowledge/test_document_sync_application_service.pynew file mode 100644index 00000000000..e7e2fcd1d8f--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_document_sync_application_service.py@@ -0,0 +1,150 @@+from dataclasses import dataclass, field++import pytest++from machinery.context import RequestContext+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccessDeniedError, DatasetAccessSnapshot+from services.knowledge.document_sync import (+ DocumentSyncApplicationService,+ SyncDocumentNotFoundError,+ SyncDocumentRecord,+ SyncDocumentSourceError,+)+from services.knowledge.document_sync_adapters import CeleryDocumentSyncDispatcher+from services.knowledge.resource_scope import DatasetRef, DocumentRef+++def _context() -> RequestContext:+ return RequestContext("request-1", None, "account-1", "workspace-1")+++def _dataset() -> AccessibleDataset:+ return AccessibleDataset(+ id="dataset-1",+ workspace_id="workspace-1",+ )+++def _document(*, data_source_type: str = "notion_import") -> SyncDocumentRecord:+ return SyncDocumentRecord(+ id="document-1",+ data_source_type=data_source_type,+ )+++@dataclass+class DatasetAccessStub:+ error: Exception | None = None+ calls: list[str] = field(default_factory=list)++ def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset:+ assert context == _context()+ self.calls.append(dataset_id)+ if self.error is not None:+ raise self.error+ return _dataset()++ def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset:+ raise AssertionError(f"Unexpected snapshot access for dataset {snapshot.dataset.id} by {context.account_id}")+++@dataclass+class DocumentReaderStub:+ document: SyncDocumentRecord | None = field(default_factory=_document)+ active_refs: tuple[DocumentRef, ...] = ()+ list_calls: list[DatasetRef] = field(default_factory=list)+ get_calls: list[DocumentRef] = field(default_factory=list)++ def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]:+ self.list_calls.append(dataset_ref)+ return self.active_refs++ def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None:+ self.get_calls.append(document_ref)+ return self.document+++@dataclass+class DispatcherRecorder:+ calls: list[tuple[str, str]] = field(default_factory=list)++ def dispatch(self, *, dataset_id: str, document_id: str) -> None:+ self.calls.append((dataset_id, document_id))+++def test_celery_dispatcher_forwards_owned_identifiers_to_injected_delay() -> None:+ calls: list[tuple[str, str]] = []+ dispatcher = CeleryDocumentSyncDispatcher(+ delay=lambda dataset_id, document_id: calls.append((dataset_id, document_id))+ )++ dispatcher.dispatch(dataset_id="dataset-1", document_id="document-1")++ assert calls == [("dataset-1", "document-1")]+++def test_sync_dataset_dispatches_snapshot_after_tenant_scoped_read() -> None:+ dataset_ref = DatasetRef("workspace-1", "dataset-1")+ documents = DocumentReaderStub(active_refs=(dataset_ref.document("document-1"), dataset_ref.document("document-2")))+ dispatcher = DispatcherRecorder()+ service = DocumentSyncApplicationService(+ dataset_access=DatasetAccessStub(), documents=documents, dispatcher=dispatcher+ )++ count = service.sync_dataset(_context(), "dataset-1")++ assert count == 2+ assert documents.list_calls == [dataset_ref]+ assert dispatcher.calls == [("dataset-1", "document-1"), ("dataset-1", "document-2")]+++def test_sync_document_validates_owner_chain_and_source_before_dispatch() -> None:+ documents = DocumentReaderStub()+ dispatcher = DispatcherRecorder()+ service = DocumentSyncApplicationService(+ dataset_access=DatasetAccessStub(), documents=documents, dispatcher=dispatcher+ )++ service.sync_document(_context(), "dataset-1", "document-1")++ assert documents.get_calls == [DatasetRef("workspace-1", "dataset-1").document("document-1")]+ assert dispatcher.calls == [("dataset-1", "document-1")]+++ ("document", "error"),+ [+ (None, SyncDocumentNotFoundError),+ (_document(data_source_type="upload_file"), SyncDocumentSourceError),+ ],+)+def test_sync_document_rejects_invalid_document_before_dispatch(+ document: SyncDocumentRecord | None, error: type[Exception]+) -> None:+ dispatcher = DispatcherRecorder()+ service = DocumentSyncApplicationService(+ dataset_access=DatasetAccessStub(),+ documents=DocumentReaderStub(document=document),+ dispatcher=dispatcher,+ )++ with pytest.raises(error):+ service.sync_document(_context(), "dataset-1", "document-1")++ assert dispatcher.calls == []+++def test_sync_stops_before_document_read_and_dispatch_when_dataset_access_is_denied() -> None:+ documents = DocumentReaderStub()+ dispatcher = DispatcherRecorder()+ service = DocumentSyncApplicationService(+ dataset_access=DatasetAccessStub(error=DatasetAccessDeniedError()),+ documents=documents,+ dispatcher=dispatcher,+ )++ with pytest.raises(DatasetAccessDeniedError):+ service.sync_document(_context(), "dataset-1", "document-1")++ assert documents.get_calls == []+ assert dispatcher.calls == []diff --git a/api/tests/unit_tests/services/knowledge/test_external_adapters.py b/api/tests/unit_tests/services/knowledge/test_external_adapters.pynew file mode 100644index 00000000000..dc70a8baccd--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_external_adapters.py@@ -0,0 +1,203 @@+import json+from collections.abc import Iterator+from unittest.mock import patch++import httpx+import pytest+from sqlalchemy import Engine, event, func, select+from sqlalchemy.orm import Session, sessionmaker++from constants import HIDDEN_VALUE+from controllers.console.datasets.external import ExternalKnowledgeApiListResponse+from machinery.context import RequestContext+from models.dataset import Dataset, DatasetQuery, ExternalKnowledgeApis, ExternalKnowledgeBindings+from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload+from services.errors.knowledge_retrieval import ExternalKnowledgeRetrievalError+from services.knowledge.dataset_access import DatasetNotFoundError+from services.knowledge.external.adapters import SQLAlchemyExternalKnowledgeOperations+from services.knowledge.external.application import ExternalTemplateNotFoundError+from services.knowledge.resource_scope import DatasetRef++CONTEXT = RequestContext("request", None, "actor", "tenant")+REF = DatasetRef("tenant", "dataset")+++def operations(sqlite_session_factory: sessionmaker[Session]) -> SQLAlchemyExternalKnowledgeOperations:+ with sqlite_session_factory.begin() as session:+ for workspace, template_id, dataset_id in [+ ("tenant", "template", "dataset"),+ ("other", "foreign", "foreign-dataset"),+ ]:+ template = ExternalKnowledgeApis(+ tenant_id=workspace,+ created_by="actor",+ updated_by="actor",+ name="API",+ description="",+ settings=json.dumps({"endpoint": "https://example", "api_key": "secret"}),+ )+ template.id = template_id+ session.add_all(+ [+ template,+ Dataset(+ id=dataset_id, tenant_id=workspace, created_by="actor", name="Dataset", provider="external"+ ),+ ExternalKnowledgeBindings(+ tenant_id=workspace,+ dataset_id=dataset_id,+ external_knowledge_api_id=template_id,+ external_knowledge_id="knowledge",+ created_by="actor",+ ),+ ]+ )+ return SQLAlchemyExternalKnowledgeOperations(session_factory=sqlite_session_factory)+++def checked_out(sqlite_engine: Engine) -> Iterator[set[object]]:+ active: set[object] = set()++ def checkout(_connection: object, record: object, _proxy: object) -> None:+ active.add(record)++ def checkin(_connection: object, record: object) -> None:+ active.discard(record)++ event.listen(sqlite_engine, "checkout", checkout)+ event.listen(sqlite_engine, "checkin", checkin)+ try:+ yield active+ finally:+ event.remove(sqlite_engine, "checkout", checkout)+ event.remove(sqlite_engine, "checkin", checkin)+++def test_template_list_materializes_bindings_and_settings(+ operations: SQLAlchemyExternalKnowledgeOperations,+) -> None:+ result = ExternalKnowledgeApiListResponse.model_validate(+ operations.list_templates("tenant", page=0, limit=0, keyword=None)+ ).model_dump(mode="json")+ assert result["total"] == 1+ assert result["limit"] == 1+ assert result["has_more"] is False+ assert result["data"][0]["settings"] == {"endpoint": "https://example", "api_key": "secret"}+ assert result["data"][0]["dataset_bindings"] == [{"id": "dataset", "name": "Dataset"}]++[email protected]("method", ["get_template", "update_template", "delete_template"])+def test_template_read_write_scope(operations: SQLAlchemyExternalKnowledgeOperations, method: str) -> None:+ if method == "update_template":+ with pytest.raises(ExternalTemplateNotFoundError):+ operations.update_template(+ CONTEXT, "foreign", name="changed", settings={"api_key": "key", "endpoint": "https://example"}+ )+ else:+ operation = operations.get_template if method == "get_template" else operations.delete_template+ with pytest.raises(ExternalTemplateNotFoundError):+ operation("tenant", "foreign")+ assert operations.template_usage("tenant", "foreign") == (False, 0)+++def test_masked_update_preserves_secret_without_mutating_request(+ operations: SQLAlchemyExternalKnowledgeOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ settings = {"endpoint": "https://changed", "api_key": HIDDEN_VALUE}+ operations.update_template(CONTEXT, "template", name="Renamed", settings=settings)+ assert settings["api_key"] == HIDDEN_VALUE+ with sqlite_session_factory() as session:+ template = session.get(ExternalKnowledgeApis, "template")+ assert template is not None+ assert template.name == "Renamed"+ settings = template.settings_dict+ assert settings is not None+ assert settings["api_key"] == "secret"+++def test_creation_probes_outside_transaction_then_commits(+ operations: SQLAlchemyExternalKnowledgeOperations,+ checked_out: set[object],+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ def probe(_settings: dict[str, object]) -> None:+ assert not checked_out++ with patch(+ "services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key", side_effect=probe+ ):+ result = operations.create_template(+ CONTEXT, name="New", settings={"endpoint": "https://example", "api_key": "secret"}+ )+ with sqlite_session_factory() as session:+ assert session.get(ExternalKnowledgeApis, result["id"]) is not None+++def test_external_dataset_cannot_bind_foreign_template(operations: SQLAlchemyExternalKnowledgeOperations) -> None:+ with pytest.raises(ExternalTemplateNotFoundError):+ operations.create_dataset(+ CONTEXT,+ ExternalDatasetCreatePayload(+ name="New", external_knowledge_api_id="foreign", external_knowledge_id="knowledge"+ ),+ )+++def test_retrieval_releases_read_session_before_http_and_commits_query(+ operations: SQLAlchemyExternalKnowledgeOperations,+ checked_out: set[object],+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ def send(_request: object, _files: object) -> httpx.Response:+ assert not checked_out+ return httpx.Response(+ 200,+ json={"records": [{"content": "answer", "title": "title", "score": 0.9, "metadata": {"source": "doc"}}]},+ )++ with patch(+ "services.knowledge.external.service.ExternalDatasetService.process_external_api", side_effect=send+ ) as request:+ result = operations.retrieve(+ CONTEXT, REF, query='"question"', retrieval_model={"top_k": 3}, metadata_filters=None+ )+ assert result["query"] == {"content": '"question"'}+ assert result["records"][0]["content"] == "answer"+ prepared = request.call_args.args[0]+ assert prepared.params["query"] == '\\"question\\"'+ assert prepared.params["knowledge_id"] == "knowledge"+ assert prepared.params["retrieval_setting"] == {"top_k": 3, "score_threshold": 0.0}+ with sqlite_session_factory() as session:+ saved = session.scalar(select(DatasetQuery))+ assert saved is not None+ assert saved.created_by == "actor"+ assert saved.content == '"question"'+ assert saved.dataset_id == "dataset"+++def test_retrieval_failure_does_not_record_query(+ operations: SQLAlchemyExternalKnowledgeOperations, sqlite_session_factory: sessionmaker[Session]+) -> None:+ with patch(+ "services.knowledge.external.service.ExternalDatasetService.process_external_api",+ return_value=httpx.Response(502, text="unavailable"),+ ):+ with pytest.raises(ExternalKnowledgeRetrievalError):+ operations.retrieve(CONTEXT, REF, query="query", retrieval_model=None, metadata_filters=None)+ with sqlite_session_factory() as session:+ assert session.scalar(select(func.count()).select_from(DatasetQuery)) == 0+++def test_retrieval_rejects_foreign_dataset_before_network(operations: SQLAlchemyExternalKnowledgeOperations) -> None:+ with patch("services.knowledge.external.service.ExternalDatasetService.process_external_api") as request:+ with pytest.raises(DatasetNotFoundError):+ operations.retrieve(+ CONTEXT,+ DatasetRef("tenant", "foreign-dataset"),+ query="query",+ retrieval_model=None,+ metadata_filters=None,+ )+ request.assert_not_called()diff --git a/api/tests/unit_tests/services/knowledge/test_external_application_service.py b/api/tests/unit_tests/services/knowledge/test_external_application_service.pynew file mode 100644index 00000000000..5d98b8581b6--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_external_application_service.py@@ -0,0 +1,81 @@+from collections.abc import Callable+from unittest.mock import MagicMock, create_autospec++import pytest++from machinery.context import RequestContext+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError+from services.knowledge.external.application import (+ ExternalHitTestingError,+ ExternalKnowledgeApplicationService,+ ExternalKnowledgeOperations,+)+from services.knowledge.resource_scope import DatasetRef++CONTEXT = RequestContext("request", None, "actor", "tenant")+++def operations() -> MagicMock:+ return create_autospec(ExternalKnowledgeOperations, instance=True, spec_set=True)+++def access() -> MagicMock:+ result = create_autospec(DatasetAccess, instance=True, spec_set=True)+ result.require_accessible.return_value = AccessibleDataset("dataset", "tenant")+ return result+++def service(operations: MagicMock, access: MagicMock) -> ExternalKnowledgeApplicationService:+ return ExternalKnowledgeApplicationService(dataset_access=access, operations=operations)++[email protected]("settings", [{}, {"endpoint": "https://example"}, {"api_key": "secret"}])+ "method", [ExternalKnowledgeApplicationService.create_template, ExternalKnowledgeApplicationService.update_template]+)+def test_template_validation_prevents_io(+ service: ExternalKnowledgeApplicationService,+ operations: MagicMock,+ settings: dict[str, str],+ method: Callable[..., object],+) -> None:+ extra: dict[str, str] = (+ {"template_id": "template"} if method is ExternalKnowledgeApplicationService.update_template else {}+ )+ with pytest.raises(ValueError):+ method(service, CONTEXT, name="API", settings=settings, **extra)+ assert operations.mock_calls == []+++def test_retrieval_authorization_precedes_validation_and_network(+ service: ExternalKnowledgeApplicationService, access: MagicMock, operations: MagicMock+) -> None:+ access.require_accessible.side_effect = DatasetAccessDeniedError()+ with pytest.raises(DatasetAccessDeniedError):+ service.hit_testing(CONTEXT, dataset_id="dataset", query="", retrieval_model=None, metadata_filters=None)+ assert operations.mock_calls == []++[email protected]("query", ["", "a" * 251])+def test_invalid_query_prevents_retrieval(+ service: ExternalKnowledgeApplicationService, operations: MagicMock, query: str+) -> None:+ with pytest.raises(ValueError):+ service.hit_testing(CONTEXT, dataset_id="dataset", query=query, retrieval_model=None, metadata_filters=None)+ operations.retrieve.assert_not_called()+++def test_retrieval_failure_is_neutral_and_receives_owned_ref(+ service: ExternalKnowledgeApplicationService, operations: MagicMock+) -> None:+ operations.retrieve.side_effect = RuntimeError("unavailable")+ with pytest.raises(ExternalHitTestingError, match="unavailable"):+ service.hit_testing(+ CONTEXT, dataset_id="dataset", query="query", retrieval_model={"top_k": 2}, metadata_filters=None+ )+ operations.retrieve.assert_called_once_with(+ CONTEXT, DatasetRef("tenant", "dataset"), query="query", retrieval_model={"top_k": 2}, metadata_filters=None+ )diff --git a/api/tests/unit_tests/services/knowledge/test_indexing_estimate_adapters.py b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_adapters.pynew file mode 100644index 00000000000..16efd0c6953--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_adapters.py@@ -0,0 +1,221 @@+from collections.abc import Callable+from datetime import UTC, datetime+from unittest.mock import MagicMock, patch++import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.entities.knowledge_entities import PreviewDetail+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError+from core.plugin.impl.exc import PluginDaemonClientSideError+from core.rag.extractor.entity.datasource_type import DatasourceType+from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo+from core.rag.index_processor.constant.index_type import IndexTechniqueType+from core.rag.models.document import ChildDocument, Document+from enums import DeploymentEdition+from extensions.storage.storage_type import StorageType+from models.dataset import Dataset, DatasetProcessRule+from models.enums import CreatorUserRole, ProcessRuleMode+from models.model import UploadFile+from services.knowledge.indexing.adapters.estimate import (+ IndexingEstimateAdapter,+ SQLAlchemyProcessRuleReader,+)+from services.knowledge.indexing.estimate import (+ IndexingEstimateExecutionError,+ IndexingEstimateProviderUnavailableError,+)+from services.knowledge.resource_scope import DatasetRef+++def _dataset(dataset_id: str, workspace_id: str) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=workspace_id,+ name=f"Dataset {dataset_id}",+ description="",+ provider="vendor",+ permission="only_me",+ created_by="account-1",+ maintainer="account-1",+ data_source_type="upload_file",+ indexing_technique="economy",+ )+++def test_notion_access_token_is_excluded_from_model_serialization_and_repr() -> None:+ notion_info = NotionInfo(+ credential_id="credential-1",+ notion_workspace_id="workspace-1",+ notion_obj_id="page-1",+ notion_page_type="page",+ tenant_id="tenant-1",+ notion_access_token="secret",+ )++ assert "notion_access_token" not in notion_info.model_dump()+ assert "secret" not in repr(notion_info)+++def test_process_rule_reader_loads_tenant_scoped_rule(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.CUSTOM,+ rules='{"segmentation": {"separator": "\\n", "max_tokens": 100}}',+ created_by="account-1",+ )+ rule.id = "rule-1"+ session.add(rule)++ reader = SQLAlchemyProcessRuleReader(session_factory=sqlite_session_factory)++ assert reader.get_by_id(dataset_ref=DatasetRef("workspace-1", "dataset-1"), process_rule_id="rule-1") == {+ "mode": "custom",+ "rules": {"segmentation": {"separator": "\n", "max_tokens": 100}},+ }+ assert reader.get_by_id(dataset_ref=DatasetRef("workspace-2", "dataset-1"), process_rule_id="rule-1") is None++[email protected]("doc_form", ["text_model", "hierarchical_model", "qa_model"])+def test_estimates_segments_and_limits_previews(+ sqlite_session_factory: sessionmaker[Session], doc_form: str, config_overrides: Callable[..., None]+) -> None:+ config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)+ processor = MagicMock()+ processor.extract.return_value = [Document(page_content="source")]+ processor.transform.return_value = [+ Document(+ page_content=f"question-{i}",+ metadata={"answer": f"answer-{i}"},+ children=[ChildDocument(page_content="child")],+ )+ for i in range(11)+ ]+ with patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory") as factory:+ factory.return_value.init_index_processor.return_value = processor+ result = IndexingEstimateAdapter(session_factory=sqlite_session_factory).run(+ "workspace-1", [ExtractSetting(datasource_type=DatasourceType.WEBSITE)], {"mode": "automatic"}, doc_form+ )+ if doc_form == "qa_model":+ assert result.total_segments == 220+ assert len(result.qa_preview) == 10+ assert result.qa_preview[-1].answer == "answer-9"+ else:+ assert result.total_segments == 11+ assert len(result.preview) == 10+ assert result.preview[0].child_chunks == ["child"]+++def test_estimate_enforces_batch_limit(+ sqlite_session_factory: sessionmaker[Session], config_overrides: Callable[..., None]+) -> None:+ config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, BATCH_UPLOAD_LIMIT=1)+ with pytest.raises(IndexingEstimateExecutionError, match="batch upload limit"):+ IndexingEstimateAdapter(session_factory=sqlite_session_factory).run(+ "workspace-1", [ExtractSetting(datasource_type=DatasourceType.WEBSITE)] * 2, {"mode": "automatic"}+ )++[email protected]("storage_fails", [False, True])+def test_preview_cleanup_commits_before_summary_and_excludes_foreign_files(+ sqlite_session_factory: sessionmaker[Session], storage_fails: bool+) -> None:+ image_ids = []+ with sqlite_session_factory.begin() as session:+ for tenant in ["workspace-1", "foreign"]:+ upload = UploadFile(+ tenant_id=tenant,+ storage_type=StorageType.LOCAL,+ key=f"{tenant}/image.png",+ name="image.png",+ size=10,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user",+ created_at=datetime.now(UTC),+ used=True,+ )+ image_ids.append(upload.id)+ session.add(upload)+ processor = MagicMock()+ processor.transform.return_value = [Document(page_content="preview")]++ def summary(+ _tenant, previews: list[PreviewDetail], _settings, _language, *, session: Session+ ) -> list[PreviewDetail]:+ assert not session.in_transaction()+ with sqlite_session_factory() as observer:+ assert observer.get(UploadFile, image_ids[0]) is None+ assert observer.get(UploadFile, image_ids[1]) is not None+ return [PreviewDetail(content=previews[0].content, summary="summary")]++ processor.generate_summary_preview.side_effect = summary+ with (+ patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory") as factory,+ patch("services.knowledge.indexing.adapters.estimate.get_image_upload_file_ids", return_value=image_ids),+ patch("services.knowledge.indexing.adapters.estimate.storage") as storage,+ ):+ factory.return_value.init_index_processor.return_value = processor+ if storage_fails:+ storage.delete.side_effect = OSError("storage unavailable")+ result = IndexingEstimateAdapter(session_factory=sqlite_session_factory).run(+ "workspace-1",+ [ExtractSetting(datasource_type=DatasourceType.WEBSITE)],+ {"mode": "automatic", "summary_index_setting": {"enable": True}},+ "text_model",+ )+ storage.delete.assert_called_once_with("workspace-1/image.png")+ assert result.preview[0].summary == "summary"++[email protected]("provider", [None, "embedding-provider"])+def test_estimate_selects_dataset_embedding_and_rejects_foreign_tenant(+ sqlite_session_factory: sessionmaker[Session], provider: str | None+) -> None:+ with sqlite_session_factory.begin() as session:+ dataset = _dataset("dataset-1", "workspace-1")+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.embedding_model_provider = provider+ dataset.embedding_model = "embedding-model"+ session.add(dataset)+ adapter = IndexingEstimateAdapter(session_factory=sqlite_session_factory)+ with (+ patch("services.knowledge.indexing.adapters.estimate.ModelManager.for_tenant") as manager,+ patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory"),+ ):+ adapter.run("workspace-1", [], {"mode": "automatic"}, dataset_id="dataset-1")+ selected = (+ manager.return_value.get_model_instance if provider else manager.return_value.get_default_model_instance+ )+ selected.assert_called_once()+ with pytest.raises(IndexingEstimateExecutionError, match="Dataset not found"):+ adapter.run("foreign", [], {"mode": "automatic"}, dataset_id="dataset-1")+++ ("processor_error", "expected_error"),+ [+ (LLMBadRequestError("provider unavailable"), IndexingEstimateProviderUnavailableError),+ (ProviderTokenNotInitError("provider unavailable"), IndexingEstimateProviderUnavailableError),+ (PluginDaemonClientSideError("provider unavailable"), IndexingEstimateProviderUnavailableError),+ (RuntimeError("processor failed"), IndexingEstimateExecutionError),+ ],+)+def test_estimate_translates_infrastructure_errors(+ sqlite_session_factory: sessionmaker[Session],+ processor_error: Exception,+ expected_error: type[Exception],+) -> None:+ adapter = IndexingEstimateAdapter(session_factory=sqlite_session_factory)+ with (+ patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory", side_effect=processor_error),+ pytest.raises(expected_error, match="provider unavailable|processor failed"),+ ):+ adapter.run(+ "workspace-1",+ [ExtractSetting(datasource_type=DatasourceType.WEBSITE)],+ {"mode": "automatic", "rules": {}},+ )diff --git a/api/tests/unit_tests/services/knowledge/test_indexing_estimate_application_service.py b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_application_service.pynew file mode 100644index 00000000000..0ee5acbc6b5--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_application_service.py@@ -0,0 +1,376 @@+from collections.abc import Sequence+from dataclasses import dataclass+from unittest.mock import Mock, create_autospec++import pytest++from core.entities.knowledge_entities import IndexingEstimate+from core.rag.extractor.entity.datasource_type import DatasourceType+from core.rag.extractor.entity.extract_setting import ExtractSetting, UploadFileExtractionInput+from machinery.context import RequestContext+from services.data_source.credential_gateway import DatasourceCredentialNotFoundError+from services.data_source.entities.notion_import import NotionPageType+from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError+from services.knowledge.entities.indexing_estimate import (+ NewSourcesEstimateCommand,+ NotionEstimateSource,+ UploadFileEstimateSource,+ WebsiteEstimateSource,+)+from services.knowledge.indexing.errors import IndexingInputSourceError, UnsupportedStoredSourceError+from services.knowledge.indexing.estimate import (+ DatasetEstimateRecord,+ EstimateDatasetReader,+ EstimateDocumentAlreadyFinishedError,+ EstimateDocumentNotFoundError,+ EstimateDocumentReader,+ EstimateDocumentRecord,+ EstimateSourceNotFoundError,+ IndexingEstimateApplicationService,+ IndexingEstimateCredentialUnavailableError,+ IndexingEstimateExecutionError,+ IndexingEstimateRunner,+ NotionEstimateSourceResolver,+ ProcessRuleReader,+ StoredEstimateSourceResolver,+ StoredSource,+ UnsupportedEstimateSourceError,+ UploadEstimateSourceResolver,+ WebsiteEstimateSourceResolver,+)+from services.knowledge.resource_scope import DatasetRef+++def _context() -> RequestContext:+ return RequestContext("request-1", None, "account-1", "workspace-1")+++def _access_dataset() -> AccessibleDataset:+ return AccessibleDataset(+ id="dataset-1",+ workspace_id="workspace-1",+ )+++def _estimate_dataset() -> DatasetEstimateRecord:+ return DatasetEstimateRecord(+ id="dataset-1",+ workspace_id="workspace-1",+ indexing_technique="high_quality",+ )+++def _document(+ document_id: str = "document-1",+ *,+ status: str = "waiting",+ process_rule_id: str | None = "rule-1",+) -> EstimateDocumentRecord:+ return EstimateDocumentRecord(+ id=document_id,+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ data_source_type="upload_file",+ data_source_info={"upload_file_id": f"file-{document_id}"},+ indexing_status=status,+ doc_form="text_model",+ doc_language="English",+ dataset_process_rule_id=process_rule_id,+ )+++def _files(*, workspace_id: str, file_ids: Sequence[str], document_model: str) -> dict[str, ExtractSetting]:+ return {+ file_id: ExtractSetting(+ datasource_type=DatasourceType.FILE,+ document_model=document_model,+ upload_file=UploadFileExtractionInput(+ id=file_id, tenant_id=workspace_id, key=file_id, created_by="account-1"+ ),+ )+ for file_id in file_ids+ }+++@dataclass+class ServiceFixture:+ service: IndexingEstimateApplicationService+ access: Mock+ datasets: Mock+ documents: Mock+ files: Mock+ notion: Mock+ websites: Mock+ stored: Mock+ rules: Mock+ runner: Mock+++def fixture() -> ServiceFixture:+ access = create_autospec(DatasetAccess, instance=True, spec_set=True)+ access.require_accessible.return_value = _access_dataset()+ access.check_access.side_effect = AssertionError("Estimate access must use the selected dataset")+ datasets = create_autospec(EstimateDatasetReader, instance=True, spec_set=True)+ datasets.get_estimate_record.return_value = _estimate_dataset()+ documents = create_autospec(EstimateDocumentReader, instance=True, spec_set=True)+ documents.get_estimate_document.return_value = _document()+ documents.list_estimate_documents_by_batch.return_value = ()+ files = create_autospec(UploadEstimateSourceResolver, instance=True, spec_set=True)+ files.resolve_selections.side_effect = _files+ notion = create_autospec(NotionEstimateSourceResolver, instance=True, spec_set=True)+ notion.resolve_selection.return_value = ExtractSetting(datasource_type=DatasourceType.NOTION)+ websites = create_autospec(WebsiteEstimateSourceResolver, instance=True, spec_set=True)+ websites.resolve_selection.return_value = ExtractSetting(datasource_type=DatasourceType.WEBSITE)+ stored = create_autospec(StoredEstimateSourceResolver, instance=True, spec_set=True)+ stored.resolve.side_effect = lambda source: ExtractSetting(+ datasource_type=source.source_type, document_model=source.document_model+ )+ rules = create_autospec(ProcessRuleReader, instance=True, spec_set=True)+ rules.get_by_id.return_value = {"mode": "custom", "rules": {"segmentation": dict[str, object]()}}+ runner = create_autospec(IndexingEstimateRunner, instance=True, spec_set=True)+ runner.run.side_effect = lambda **kwargs: IndexingEstimate(+ total_segments=len(kwargs["extract_settings"]), preview=[]+ )+ service = IndexingEstimateApplicationService(+ dataset_access=access,+ datasets=datasets,+ documents=documents,+ files=files,+ notion=notion,+ websites=websites,+ stored_sources=stored,+ process_rules=rules,+ runner=runner,+ )+ return ServiceFixture(service, access, datasets, documents, files, notion, websites, stored, rules, runner)+++def test_estimate_new_sources_resolves_each_variant_and_normalizes_rule(fixture: ServiceFixture) -> None:++ result = fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(+ sources=(+ UploadFileEstimateSource("file-1"),+ NotionEstimateSource("notion-workspace", "page-1", NotionPageType.PAGE, "credential-1"),+ WebsiteEstimateSource("firecrawl", "job-1", "https://example.com", only_main_content=True),+ ),+ process_rule={"mode": "automatic", "summary_index_setting": {"enable": None}},+ dataset_id="dataset-1",+ ),+ )++ assert result.total_segments == 3+ fixture.access.require_accessible.assert_called_once_with(_context(), "dataset-1")+ fixture.files.resolve_selections.assert_called_once_with(+ workspace_id="workspace-1",+ file_ids=("file-1",),+ document_model="text_model",+ )+ fixture.notion.resolve_selection.assert_called_once_with(+ workspace_id="workspace-1",+ actor_id="account-1",+ credential_id="credential-1",+ notion_workspace_id="notion-workspace",+ page_id="page-1",+ page_type=NotionPageType.PAGE,+ document_model="text_model",+ )+ fixture.websites.resolve_selection.assert_called_once_with(+ workspace_id="workspace-1",+ provider="firecrawl",+ job_id="job-1",+ url="https://example.com",+ mode="crawl",+ only_main_content=True,+ document_model="text_model",+ )+ assert fixture.runner.run.call_args.kwargs["tmp_processing_rule"] == {"mode": "automatic", "rules": {}}+ assert fixture.runner.run.call_args.kwargs["dataset_id"] == "dataset-1"++[email protected]("dataset_id", [None, ""])+ "source",+ [+ NotionEstimateSource("notion-workspace", "page-1", NotionPageType.PAGE, "credential-1"),+ WebsiteEstimateSource("firecrawl", "job-1", "https://example.com"),+ ],+ ids=["notion", "website"],+)+def test_estimate_new_dataset_accepts_missing_or_empty_dataset_id(+ fixture: ServiceFixture,+ dataset_id: str | None,+ source: NotionEstimateSource | WebsiteEstimateSource,+) -> None:+ result = fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(+ sources=(source,),+ process_rule={"mode": "automatic"},+ dataset_id=dataset_id,+ ),+ )++ assert result.total_segments == 1+ fixture.access.require_accessible.assert_not_called()+ assert fixture.runner.run.call_args.kwargs["dataset_id"] is None+++def test_estimate_new_sources_stops_when_dataset_access_is_denied(fixture: ServiceFixture) -> None:+ fixture.access.require_accessible.side_effect = DatasetAccessDeniedError()++ with pytest.raises(DatasetAccessDeniedError):+ fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(+ sources=(UploadFileEstimateSource("file-1"),),+ process_rule={"mode": "automatic"},+ dataset_id="dataset-1",+ ),+ )++ fixture.files.resolve_selections.assert_not_called()+ fixture.runner.run.assert_not_called()+++def test_estimate_new_notion_source_maps_actor_credential_failure_without_running(fixture: ServiceFixture) -> None:+ fixture.notion.resolve_selection.side_effect = DatasourceCredentialNotFoundError()++ with pytest.raises(IndexingEstimateCredentialUnavailableError):+ fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(+ sources=(+ NotionEstimateSource(+ "notion-workspace",+ "page-1",+ NotionPageType.PAGE,+ "private-credential",+ ),+ ),+ process_rule={"mode": "automatic"},+ ),+ )++ fixture.runner.run.assert_not_called()+++def test_estimate_empty_new_sources_returns_zero_without_runner_call(fixture: ServiceFixture) -> None:++ result = fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(sources=(), process_rule={"mode": "automatic"}),+ )++ assert result == IndexingEstimate(total_segments=0, preview=[])+ fixture.runner.run.assert_not_called()+++def test_estimate_batch_preserves_first_document_contract_and_loads_its_rule(fixture: ServiceFixture) -> None:+ documents = (_document("document-1"), _document("document-2"))+ fixture.documents.list_estimate_documents_by_batch.return_value = documents++ fixture.service.estimate_batch(_context(), dataset_id="dataset-1", batch="batch-1")++ fixture.stored.resolve.assert_called_once_with(StoredSource.from_document(documents[0]))+ fixture.rules.get_by_id.assert_called_once_with(+ dataset_ref=DatasetRef("workspace-1", "dataset-1"),+ process_rule_id="rule-1",+ )+ fixture.documents.list_estimate_documents_by_batch.assert_called_once_with(+ DatasetRef("workspace-1", "dataset-1"),+ "batch-1",+ )+ assert fixture.runner.run.call_args.kwargs["indexing_technique"] == "high_quality"+++def test_estimate_uses_automatic_rule_when_document_has_no_saved_rule(fixture: ServiceFixture) -> None:+ fixture.documents.get_estimate_document.return_value = _document(process_rule_id=None)++ fixture.service.estimate_document(_context(), dataset_id="dataset-1", document_id="document-1")++ fixture.datasets.get_estimate_record.assert_called_once_with(DatasetRef("workspace-1", "dataset-1"))+ fixture.documents.get_estimate_document.assert_called_once_with(+ DatasetRef("workspace-1", "dataset-1").document("document-1")+ )+ fixture.rules.get_by_id.assert_not_called()+ assert fixture.runner.run.call_args.kwargs["tmp_processing_rule"] == {"mode": "automatic", "rules": {}}+++def test_estimate_rejects_missing_saved_process_rule(fixture: ServiceFixture) -> None:+ fixture.rules.get_by_id.return_value = None++ with pytest.raises(EstimateSourceNotFoundError, match="rule-1"):+ fixture.service.estimate_document(_context(), dataset_id="dataset-1", document_id="document-1")++ fixture.runner.run.assert_not_called()+++def test_estimate_empty_batch_raises_not_found_without_runner_call(fixture: ServiceFixture) -> None:++ with pytest.raises(EstimateDocumentNotFoundError):+ fixture.service.estimate_batch(_context(), dataset_id="dataset-1", batch="batch-1")++ fixture.runner.run.assert_not_called()++[email protected]("status", ["completed", "error"])+def test_estimate_rejects_finished_document(fixture: ServiceFixture, status: str) -> None:+ fixture.documents.get_estimate_document.return_value = _document(status=status)++ with pytest.raises(EstimateDocumentAlreadyFinishedError):+ fixture.service.estimate_document(_context(), dataset_id="dataset-1", document_id="document-1")++ fixture.runner.run.assert_not_called()+++def test_estimate_rejects_missing_document(fixture: ServiceFixture) -> None:+ fixture.documents.get_estimate_document.return_value = None++ with pytest.raises(EstimateDocumentNotFoundError):+ fixture.service.estimate_document(_context(), dataset_id="dataset-1", document_id="document-1")+++ ("source_error", "expected_error"),+ [+ (DatasourceCredentialNotFoundError(), IndexingEstimateCredentialUnavailableError),+ (IndexingInputSourceError("missing source"), EstimateSourceNotFoundError),+ (UnsupportedStoredSourceError("unknown"), UnsupportedEstimateSourceError),+ (RuntimeError("failed"), IndexingEstimateExecutionError),+ ],+)+def test_estimate_translates_source_boundary_errors(+ fixture: ServiceFixture, source_error: Exception, expected_error: type[Exception]+) -> None:+ fixture.stored.resolve.side_effect = source_error++ with pytest.raises(expected_error):+ fixture.service.estimate_document(_context(), dataset_id="dataset-1", document_id="document-1")+++def test_multi_file_estimate_reads_once_and_preserves_selection_order(fixture: ServiceFixture) -> None:+ fixture.service.estimate_new_sources(+ _context(),+ NewSourcesEstimateCommand(+ sources=(+ UploadFileEstimateSource("file-2"),+ UploadFileEstimateSource("file-1"),+ UploadFileEstimateSource("file-2"),+ ),+ process_rule={"mode": "automatic"},+ ),+ )+ fixture.files.resolve_selections.assert_called_once_with(+ workspace_id="workspace-1",+ file_ids=("file-2", "file-1"),+ document_model="text_model",+ )+ settings = fixture.runner.run.call_args.kwargs["extract_settings"]+ assert [setting.upload_file.id for setting in settings if setting.upload_file is not None] == [+ "file-2",+ "file-1",+ "file-2",+ ]diff --git a/api/tests/unit_tests/services/knowledge/test_indexing_estimate_entities.py b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_entities.pynew file mode 100644index 00000000000..a4b9c13cfbf--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_estimate_entities.py@@ -0,0 +1,126 @@+import pytest++from services.knowledge.entities.indexing_estimate import (+ EstimateValidationError,+ normalize_process_rule,+)+++def test_normalize_process_rule_sets_empty_rules_for_automatic_mode() -> None:+ normalized = normalize_process_rule({"mode": "automatic", "rules": {"ignored": True}})++ assert normalized["rules"] == {}++[email protected]("summary_setting", [None, {"enable": None}])+def test_normalize_process_rule_treats_absent_summary_enable_as_no_summary(summary_setting: object) -> None:+ normalized = normalize_process_rule(+ {+ "mode": "automatic",+ "summary_index_setting": summary_setting,+ }+ )++ assert "summary_index_setting" not in normalized+++def test_normalize_process_rule_rejects_unknown_pre_processing_rule_id() -> None:+ with pytest.raises(EstimateValidationError):+ normalize_process_rule(+ {+ "mode": "custom",+ "rules": {+ "pre_processing_rules": [{"id": "unknown", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 128},+ },+ }+ )+++def test_normalize_process_rule_deduplicates_custom_rules() -> None:+ normalized = normalize_process_rule(+ {+ "mode": "custom",+ "rules": {+ "pre_processing_rules": [+ {"id": "remove_stopwords", "enabled": True},+ {"id": "remove_stopwords", "enabled": False},+ ],+ "segmentation": {"separator": "\n", "max_tokens": 128},+ },+ }+ )++ rules = normalized["rules"]+ assert isinstance(rules, dict)+ assert rules["pre_processing_rules"] == [{"id": "remove_stopwords", "enabled": False}]+++def test_normalize_process_rule_drops_hierarchical_fields_from_custom_mode() -> None:+ normalized = normalize_process_rule(+ {+ "mode": "custom",+ "rules": {+ "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 128},+ "parent_mode": "full-doc",+ "subchunk_segmentation": {"separator": "###", "max_tokens": 64},+ },+ }+ )++ assert normalized["rules"] == {+ "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 128},+ }+++def test_normalize_process_rule_requires_summary_index_provider_name() -> None:+ with pytest.raises(EstimateValidationError, match="Field required"):+ normalize_process_rule(+ {+ "mode": "custom",+ "rules": {+ "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 128},+ },+ "summary_index_setting": {"enable": True, "model_name": "summary-model"},+ }+ )+++def test_normalize_process_rule_preserves_hierarchical_fields() -> None:+ normalized = normalize_process_rule(+ {+ "mode": "hierarchical",+ "rules": {+ "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 512},+ "parent_mode": "full-doc",+ "subchunk_segmentation": {"separator": "###", "max_tokens": 128},+ },+ }+ )++ rules = normalized["rules"]+ assert isinstance(rules, dict)+ assert rules["parent_mode"] == "full-doc"+ assert rules["subchunk_segmentation"] == {"separator": "###", "max_tokens": 128}+++def test_normalize_process_rule_defaults_hierarchical_parent_mode() -> None:+ normalized = normalize_process_rule(+ {+ "mode": "hierarchical",+ "rules": {+ "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],+ "segmentation": {"separator": "\n", "max_tokens": 512},+ "subchunk_segmentation": {"separator": "###", "max_tokens": 128},+ },+ }+ )++ rules = normalized["rules"]+ assert isinstance(rules, dict)+ assert rules["parent_mode"] == "paragraph"+ assert rules["subchunk_segmentation"] == {"separator": "###", "max_tokens": 128}diff --git a/api/tests/unit_tests/services/knowledge/test_indexing_execution.py b/api/tests/unit_tests/services/knowledge/test_indexing_execution.pynew file mode 100644index 00000000000..630e109a33b--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_execution.py@@ -0,0 +1,128 @@+from unittest.mock import MagicMock++import pytest++from core.rag.models.document import Document+from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError+from services.knowledge.indexing.estimate import StoredSource+from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument+from services.knowledge.resource_scope import DatasetRef++ExecutionFixture = tuple[DocumentIndexingService, MagicMock, IndexingDocument, list[Document]]+++def execution() -> ExecutionFixture:+ ref = DatasetRef("tenant", "dataset").document("document")+ document = IndexingDocument(+ StoredSource(ref, "upload_file", {}, "text_model"), {"mode": "automatic"}, "English", False+ )+ ports = MagicMock()+ ports.documents.get_indexing_document.return_value = document+ chunks = [Document(page_content="one"), Document(page_content="two")]+ ports.backend.extract.return_value = chunks+ ports.backend.transform.return_value = chunks+ ports.backend.count_tokens.return_value = [7, 11]+ ports.backend.describe_error.side_effect = str+ ports.segments.resume_indexing.return_value = (chunks[1:], 18)+ service = DocumentIndexingService(+ documents=ports.documents,+ segments=ports.segments,+ backend=ports.backend,+ enforce_vector_space_admission=True,+ clock=iter([10, 13]).__next__,+ )+ return service, ports, document, chunks++[email protected]("start", ["parsing", "splitting", "indexing"])+def test_completion_waits_for_persisted_segments_and_all_index_writes(execution: ExecutionFixture, start: str) -> None:+ service, ports, document, chunks = execution+ if start == "parsing":+ service.run([document.ref])+ elif start == "splitting":+ service.run_in_splitting_status(document.ref)+ else:+ service.run_in_indexing_status(document.ref)+ calls = [call[0] for call in ports.mock_calls]+ assert calls.index("backend.load") < calls.index("documents.complete_indexing")+ ports.documents.complete_indexing.assert_called_once_with(document.ref, tokens=18, latency=3)+ if start == "indexing":+ ports.backend.extract.assert_not_called()+ ports.segments.save_for_indexing.assert_not_called()+ ports.backend.load.assert_called_once_with(document, chunks[1:])+ else:+ assert calls.index("segments.save_for_indexing") < calls.index("backend.load")+ ports.segments.save_for_indexing.assert_called_once_with(document, chunks, [7, 11])+ if start == "splitting":+ assert calls.index("segments.clear_for_indexing") < calls.index("backend.extract")+ assert ports.backend.ensure_admission.call_count == (start == "parsing")++[email protected]("stage", ["extract", "transform", "ensure_admission", "count_tokens", "load"])+def test_failed_phase_records_error_without_completing(execution: ExecutionFixture, stage: str) -> None:+ service, ports, document, _ = execution+ operation = {+ "extract": ports.backend.extract,+ "transform": ports.backend.transform,+ "ensure_admission": ports.backend.ensure_admission,+ "count_tokens": ports.backend.count_tokens,+ "load": ports.backend.load,+ }[stage]+ operation.side_effect = RuntimeError("index unavailable")+ service.run([document.ref])+ ports.documents.complete_indexing.assert_not_called()+ ports.documents.fail_indexing.assert_called_once_with(document.ref, "index unavailable")+ if stage != "load":+ ports.backend.load.assert_not_called()++[email protected]("stage", ["get_indexing_document", "mark_splitting", "complete_indexing"])[email protected]("error", [DocumentIsPausedError, DocumentIsDeletedPausedError])+def test_pause_and_deletion_do_not_become_indexing_errors(+ execution: ExecutionFixture, stage: str, error: type[Exception]+) -> None:+ service, ports, document, _ = execution+ operation = {+ "get_indexing_document": ports.documents.get_indexing_document,+ "mark_splitting": ports.documents.mark_splitting,+ "complete_indexing": ports.documents.complete_indexing,+ }[stage]+ operation.side_effect = error()+ if error is DocumentIsPausedError:+ with pytest.raises(DocumentIsPausedError):+ service.run([document.ref])+ else:+ service.run([document.ref])+ ports.documents.fail_indexing.assert_not_called()+++def test_pause_after_worker_completion_prevents_document_completion(execution: ExecutionFixture) -> None:+ service, ports, document, _ = execution+ ports.backend.check_paused.side_effect = [None, None, DocumentIsPausedError()]+ with pytest.raises(DocumentIsPausedError):+ service.run([document.ref])+ ports.documents.complete_indexing.assert_not_called()+ ports.documents.fail_indexing.assert_not_called()+++def test_missing_document_is_skipped(execution: ExecutionFixture) -> None:+ service, ports, document, _ = execution+ ports.documents.get_indexing_document.return_value = None+ service.run([document.ref])+ ports.backend.extract.assert_not_called()+ ports.documents.fail_indexing.assert_not_called()+++def test_indexing_persists_the_backend_error_description(execution: ExecutionFixture) -> None:+ service, ports, document, _ = execution+ error = RuntimeError("backend details")+ ports.backend.extract.side_effect = error+ ports.backend.describe_error.side_effect = None+ ports.backend.describe_error.return_value = "source file missing"++ service.run([document.ref])++ ports.backend.describe_error.assert_called_once_with(error)+ ports.documents.fail_indexing.assert_called_once_with(document.ref, "source file missing")+ ports.documents.complete_indexing.assert_not_called()diff --git a/api/tests/unit_tests/services/knowledge/test_indexing_execution_adapters.py b/api/tests/unit_tests/services/knowledge/test_indexing_execution_adapters.pynew file mode 100644index 00000000000..45ec28a4ec8--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_execution_adapters.py@@ -0,0 +1,528 @@+import json+import threading+from collections.abc import Generator, Iterator+from contextlib import contextmanager+from dataclasses import replace+from datetime import datetime+from unittest.mock import MagicMock, patch++import pytest+from flask import Flask+from sqlalchemy import Connection, Engine, event, inspect, select+from sqlalchemy.orm import Session, sessionmaker+from werkzeug.exceptions import NotFound++from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError, QuotaExceededError+from core.plugin.impl.exc import PluginDaemonClientSideError+from core.rag.datasource.vdb.vector_factory import Vector+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from core.rag.models.document import AttachmentDocument, ChildDocument+from core.rag.models.document import Document as IndexDocument+from extensions.storage.storage_type import StorageType+from models.dataset import Dataset, DatasetKeywordTable, DatasetProcessRule, Document, DocumentSegment+from models.enums import CreatorUserRole, IndexingStatus, ProcessRuleMode, SegmentStatus+from models.model import UploadFile+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.indexing.adapters.execution import IndexingExecutionAdapter+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument+from services.knowledge.resource_scope import DatasetRef+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _document++MODULE = "services.knowledge.indexing.adapters.execution"+++BackendFixture = tuple[+ IndexingExecutionAdapter,+ SQLAlchemyDocumentRepository,+ SQLAlchemySegmentRepository,+ MagicMock,+ IndexingDocument,+ MagicMock,+]+++def backend(sqlite_session_factory: sessionmaker[Session]) -> Iterator[BackendFixture]:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [_dataset("dataset-1", "workspace-1"), _document("document-1", status=IndexingStatus.SPLITTING)]+ )+ documents = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)+ segments = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ sources = MagicMock()+ adapter = IndexingExecutionAdapter(+ session_factory=sqlite_session_factory,+ documents=documents,+ segments=segments,+ uploads=SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory),+ sources=sources,+ )+ job = replace(+ documents.get_indexing_document(DatasetRef("workspace-1", "dataset-1").document("document-1")),+ processing_rule={"mode": "automatic", "rules": {}},+ )+ with patch(f"{MODULE}.redis_client") as redis:+ redis.get.return_value = None+ yield adapter, documents, segments, sources, job, redis+++def active_transactions(sqlite_engine: Engine) -> Iterator[set[Connection]]:+ active: set[Connection] = set()+ begin, finish = active.add, active.discard+ event.listen(sqlite_engine, "begin", begin)+ event.listen(sqlite_engine, "commit", finish)+ event.listen(sqlite_engine, "rollback", finish)+ try:+ yield active+ finally:+ event.remove(sqlite_engine, "begin", begin)+ event.remove(sqlite_engine, "commit", finish)+ event.remove(sqlite_engine, "rollback", finish)++[email protected]("doc_form", list(IndexStructureType))+def test_vector_io_releases_transactions_and_preserves_chunk_structure(+ backend: BackendFixture,+ sqlite_session_factory: sessionmaker[Session],+ active_transactions: set[Connection],+ monkeypatch: pytest.MonkeyPatch,+ doc_form: IndexStructureType,+) -> None:+ adapter, _, segments, _, job, _ = backend+ apply_config_overrides(monkeypatch, VECTOR_STORE="qdrant", VECTOR_STORE_WHITELIST_ENABLE=True)+ files = [+ UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key=f"image-{tenant_id}",+ name="image.png",+ size=1,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime(2026, 1, 1),+ used=True,+ )+ for tenant_id in ("workspace-1", "workspace-2")+ ]+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ assert dataset is not None+ assert document is not None+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.is_multimodal = True+ document.doc_form = doc_form+ session.add_all(files)+ child = ChildDocument(page_content="child", metadata={"doc_id": "child-node", "doc_hash": "child-hash"})+ chunk = IndexDocument(+ page_content="parent",+ metadata={"doc_id": "node", "doc_hash": "hash"},+ children=[child] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else None,+ attachments=[+ AttachmentDocument(page_content="image", metadata={"doc_id": file.id, "doc_type": "image"})+ for file in files+ ],+ )+ segments.save_for_indexing(job, [chunk], [5])+ vector_backend = MagicMock()+ vector_factory = MagicMock()+ vector_factory.return_value.init_vector.return_value = vector_backend++ def external_call(*_args, **_kwargs) -> list[list[float]]:+ assert not active_transactions, "Index I/O must not hold a database transaction"+ return [[1.0]]++ def get_vector_factory(vector_type: str) -> MagicMock:+ assert not active_transactions, "Vector initialization must follow the whitelist read transaction"+ assert vector_type == "qdrant"+ return vector_factory++ def load_image(key: str) -> bytes:+ assert not active_transactions+ assert key == files[0].key+ return b"image"++ vector_backend.create.side_effect = external_call+ with (+ patch.object(Vector, "get_vector_factory", side_effect=get_vector_factory),+ patch(+ "core.rag.datasource.vdb.vector_factory._LazyEmbeddings.embed_documents", side_effect=external_call+ ) as embed,+ patch(+ "core.rag.datasource.vdb.vector_factory._LazyEmbeddings.embed_multimodal_documents",+ side_effect=external_call,+ ) as embed_images,+ patch("core.rag.datasource.vdb.vector_factory.storage") as storage,+ ):+ storage.load_once.side_effect = load_image+ adapter._process_chunk(Flask(__name__), job.ref, [chunk], False)+ embed.assert_called_once_with(["child" if doc_form == IndexStructureType.PARENT_CHILD_INDEX else "parent"])+ assert embed_images.call_args.args[0] == [{"content": "aW1hZ2U=", "content_type": "image", "file_id": files[0].id}]+ storage.load_once.assert_called_once_with(files[0].key)+ assert vector_backend.create.call_count == 2+ indexed_text = vector_backend.create.call_args_list[0].kwargs["texts"][0]+ assert indexed_text.metadata["doc_id"] == (+ "child-node" if doc_form == IndexStructureType.PARENT_CHILD_INDEX else "node"+ )+ assert segments.resume_indexing(job) == ([], 5)++[email protected]("storage_type", ["database", "file"])+def test_keyword_io_releases_transactions_and_commits_before_unlocking(+ backend: BackendFixture,+ sqlite_session_factory: sessionmaker[Session],+ active_transactions: set[Connection],+ storage_type: str,+) -> None:+ adapter, _, segments, _, job, redis = backend+ original = json.dumps({"__type__": "keyword_table", "__data__": {"table": {"existing": ["old-node"]}}})+ with sqlite_session_factory.begin() as session:+ session.add(DatasetKeywordTable(dataset_id="dataset-1", data_source_type=storage_type, keyword_table=original))+ chunk = IndexDocument(page_content="hello", metadata={"doc_id": "node", "doc_hash": "hash"})+ segments.save_for_indexing(job, [chunk], [5])+ locked = False++ @contextmanager+ def dataset_lock() -> Generator[None]:+ nonlocal locked+ locked = True+ yield+ assert not active_transactions+ # A separate session must see the write before the next lock holder can read.+ with sqlite_session_factory() as session:+ segment = session.scalar(select(DocumentSegment))+ assert segment is not None+ assert segment.keywords == ["hello"]+ locked = False++ redis.lock.side_effect = lambda *_args, **_kwargs: dataset_lock()++ def external_call(*_args, **_kwargs) -> None:+ assert locked+ assert not active_transactions, "Keyword I/O must not hold a database transaction"++ def load_keywords(*_args) -> bytes:+ external_call()+ return original.encode()++ def file_exists(*_args) -> bool:+ external_call()+ return True++ def extract_keywords(*_args) -> set[str]:+ external_call()+ return {"hello"}++ with (+ patch("core.rag.datasource.keyword.jieba.jieba.redis_client", redis),+ patch("core.rag.datasource.keyword.jieba.jieba.storage") as storage,+ patch("core.rag.datasource.keyword.jieba.jieba.JiebaKeywordTableHandler") as handler,+ ):+ storage.load_once.side_effect = load_keywords+ storage.exists.side_effect = file_exists+ storage.delete.side_effect = external_call+ storage.save.side_effect = external_call+ handler.return_value.extract_keywords.side_effect = extract_keywords+ adapter._process_chunk(Flask(__name__), job.ref, [chunk], True)+ assert not locked+ assert redis.lock.call_args.args == ("keyword_indexing_lock_dataset-1",)+ _, encoded = segments.get_keyword_table(job.ref.dataset)+ if storage_type == "file":+ storage.save.assert_called_once()+ encoded = storage.save.call_args.args[1]+ else:+ storage.save.assert_not_called()+ assert encoded is not None+ assert json.loads(encoded)["__data__"]["table"] == {"existing": ["old-node"], "hello": ["node"]}+ assert segments.resume_indexing(job) == ([], 5)+++def test_extract_attaches_document_metadata_and_owns_local_session(backend: BackendFixture) -> None:+ adapter, _, _, sources, job, _ = backend+ text = IndexDocument(page_content="source")+ with patch(f"{MODULE}.IndexProcessorFactory") as factory:+ processor = factory.return_value.init_index_processor.return_value++ def extract(setting: object, *, session: Session, process_rule_mode: str) -> list[IndexDocument]:+ assert setting is sources.resolve_for_indexing.return_value+ assert not session.in_transaction()+ assert process_rule_mode == "automatic"+ return [text]++ processor.extract.side_effect = extract+ assert adapter.extract(job) == [text]+ sources.resolve_for_indexing.assert_called_once_with(job.source)+ assert text.metadata == {"dataset_id": "dataset-1", "document_id": "document-1"}+++ ("error", "message"),+ [+ (ProviderTokenNotInitError(), "Provider Token Not Init"),+ (ProviderTokenNotInitError("configure an embedding provider"), "configure an embedding provider"),+ (LLMBadRequestError("invalid model input"), "invalid model input"),+ (QuotaExceededError(), "Quota Exceeded"),+ (PluginDaemonClientSideError("plugin unavailable"), "plugin unavailable"),+ (NotFound("source file missing"), "source file missing"),+ (RuntimeError("index unavailable"), "index unavailable"),+ ],+)+def test_indexing_preserves_provider_and_source_error_descriptions(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], error: Exception, message: str+) -> None:+ adapter, documents, segments, sources, job, _ = backend+ sources.resolve_for_indexing.side_effect = error+ with sqlite_session_factory.begin() as session:+ rule = DatasetProcessRule(+ dataset_id=job.ref.dataset.dataset_id, mode=ProcessRuleMode.AUTOMATIC, rules="{}", created_by="account-1"+ )+ session.add(rule)+ session.flush()+ document = session.get(Document, job.ref.document_id)+ assert document is not None+ document.dataset_process_rule_id = rule.id++ DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run([job.ref])++ sources.resolve_for_indexing.assert_called_once()+ with sqlite_session_factory() as session:+ document = session.get(Document, job.ref.document_id)+ assert document is not None+ assert document.indexing_status == IndexingStatus.ERROR+ assert document.error == message+ assert document.completed_at is None++[email protected]("provider", [None, "embedding-provider"])+def test_transform_selects_embedding_model_and_preserves_processor_options(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], provider: str | None+) -> None:+ adapter, documents, _, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.embedding_model_provider = provider+ dataset.embedding_model = "embedding-model"+ with (+ patch.object(documents, "get_indexing_user", return_value=MagicMock()) as user,+ patch(f"{MODULE}.ModelManager.for_tenant") as manager,+ patch(f"{MODULE}.IndexProcessorFactory") as factory,+ ):+ processor = factory.return_value.init_index_processor.return_value+ processor.transform.return_value = list[IndexDocument]()+ adapter.transform(job, [])+ selected = (+ manager.return_value.get_model_instance if provider else manager.return_value.get_default_model_instance+ )+ selected.assert_called_once()+ assert processor.transform.call_args.kwargs["embedding_model_instance"] is selected.return_value+ assert processor.transform.call_args.kwargs["process_rule"] == job.processing_rule+ assert processor.transform.call_args.args == ([], user.return_value)+++def test_hash_groups_load_worker_owned_inputs_before_completion(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]+) -> None:+ adapter, _, segments, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ persisted_dataset = session.get(Dataset, "dataset-1")+ assert persisted_dataset is not None+ persisted_dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ chunks = [+ IndexDocument(page_content=content, metadata={"doc_id": f"node-{i}", "doc_hash": f"hash-{i}"})+ for i, content in enumerate(["same", "other", "same"])+ ]+ segments.save_for_indexing(job, chunks, [3, 5, 7])+ calls: list[tuple[Dataset, list[str]]] = []+ main_thread = threading.get_ident()++ def vector(dataset: Dataset, *, session: Session | None, vector_type: str) -> MagicMock:+ assert threading.get_ident() != main_thread+ assert inspect(dataset).detached+ assert session is None+ assert vector_type == "qdrant"++ def create(group: list[IndexDocument]) -> None:+ with sqlite_session_factory() as reader:+ persisted = reader.scalars(select(DocumentSegment)).all()+ assert len(persisted) == 3+ assert all(+ row.status == SegmentStatus.INDEXING+ for row in persisted+ if row.index_node_id in {chunk.metadata["doc_id"] for chunk in group}+ )+ calls.append((dataset, [chunk.page_content for chunk in group]))++ backend = MagicMock()+ backend.create.side_effect = create+ return backend++ with Flask(__name__).app_context(), patch(f"{MODULE}.Vector", side_effect=vector) as factory:+ factory.resolve_vector_type.return_value = "qdrant"+ adapter.load(job, chunks)+ assert sum(group.count("same") for _, group in calls) == 2+ assert sum("same" in group for _, group in calls) == 1+ assert len({id(dataset) for dataset, _ in calls}) == len(calls)+ assert segments.resume_indexing(job) == ([], 15)++[email protected]("technique", ["economy", "high_quality"])+def test_worker_failure_marks_document_error_and_keeps_retryable_segments(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], technique: str+) -> None:+ adapter, documents, segments, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ persisted_dataset = session.get(Dataset, "dataset-1")+ assert persisted_dataset is not None+ persisted_dataset.indexing_technique = IndexTechniqueType(technique)+ chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]+ segments.save_for_indexing(job, chunks, [9])+ with (+ Flask(__name__).app_context(),+ patch(f"{MODULE}.Jieba") as keyword,+ patch(f"{MODULE}.Vector") as vector,+ ):+ keyword.return_value.update_texts.side_effect = RuntimeError("worker failed")+ vector.return_value.create.side_effect = RuntimeError("worker failed")+ DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run_in_indexing_status(job.ref)+ with sqlite_session_factory() as session:+ row = session.get(Document, job.ref.document_id)+ assert row is not None+ assert row.indexing_status == IndexingStatus.ERROR+ assert row.error == "worker failed"+ assert len(segments.resume_indexing(job)[0]) == 1+++def test_worker_pause_does_not_mark_segments_complete(backend: BackendFixture) -> None:+ adapter, _, segments, _, job, redis = backend+ chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]+ segments.save_for_indexing(job, chunks, [9])+ redis.get.return_value = b"paused"+ with Flask(__name__).app_context(), pytest.raises(DocumentIsPausedError):+ adapter.load(job, chunks)+ assert len(segments.resume_indexing(job)[0]) == 1++[email protected]("technique", ["economy", "high_quality"])[email protected]("doc_form", ["text_model", "qa_model", "hierarchical_model"])+def test_full_indexing_commits_phases_through_real_repositories(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], technique: str, doc_form: str+) -> None:+ adapter, documents, segments, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType(technique)+ row = session.get(Document, "document-1")+ assert row is not None+ row.doc_form = IndexStructureType(doc_form)+ rule = DatasetProcessRule(+ dataset_id="dataset-1", mode=ProcessRuleMode.AUTOMATIC, rules="{}", created_by="account-1"+ )+ rule.id = "rule-1"+ row.dataset_process_rule_id = rule.id+ session.add(rule)+ chunk = IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})+ with (+ Flask(__name__).app_context(),+ patch.object(documents, "get_indexing_user", return_value=MagicMock()),+ patch(f"{MODULE}.IndexProcessorFactory") as factory,+ patch(f"{MODULE}.ModelManager.for_tenant"),+ patch(f"{MODULE}.calculate_segment_token_counts", return_value=[19]),+ patch(f"{MODULE}.Jieba"),+ patch(f"{MODULE}.Vector"),+ ):+ processor = factory.return_value.init_index_processor.return_value+ processor.extract.return_value = [chunk]+ processor.transform.return_value = [chunk]+ DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run([job.ref])+ with sqlite_session_factory() as session:+ row = session.get(Document, "document-1")+ assert row is not None+ assert row.indexing_status == IndexingStatus.COMPLETED+ assert row.tokens == 19+ assert row.parsing_completed_at is not None+ assert row.splitting_completed_at is not None+ assert row.completed_at is not None++[email protected]("change", ["paused", "deleted"])+def test_document_changed_during_worker_io_cannot_be_completed(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], change: str+) -> None:+ adapter, documents, segments, _, job, _ = backend+ chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]+ segments.save_for_indexing(job, chunks, [9])++ def change_document(*_args, **_kwargs) -> None:+ with sqlite_session_factory.begin() as writer:+ row = writer.get(Document, "document-1")+ assert row is not None+ if change == "paused":+ row.is_paused = True+ else:+ writer.delete(row)++ service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)+ with Flask(__name__).app_context(), patch(f"{MODULE}.Jieba") as keyword:+ keyword.return_value.update_texts.side_effect = change_document+ if change == "paused":+ with pytest.raises(DocumentIsPausedError):+ service.run_in_indexing_status(job.ref)+ else:+ service.run_in_indexing_status(job.ref)+ with sqlite_session_factory() as session:+ segment = session.scalar(select(DocumentSegment))+ assert segment is not None+ assert segment.status == SegmentStatus.INDEXING+++def test_partial_worker_failure_retries_only_incomplete_segments_and_keeps_total(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]+) -> None:+ adapter, documents, segments, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ persisted_dataset = session.get(Dataset, "dataset-1")+ assert persisted_dataset is not None+ persisted_dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ chunks = [+ IndexDocument(page_content=content, metadata={"doc_id": content, "doc_hash": content})+ for content in ["good", "bad"]+ ]+ segments.save_for_indexing(job, chunks, [13, 17])+ service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)++ def load(group: list[IndexDocument]) -> None:+ if group[0].page_content == "bad":+ raise RuntimeError("one shard failed")++ with (+ Flask(__name__).app_context(),+ patch(f"{MODULE}.Vector") as factory,+ patch(f"{MODULE}.helper.generate_text_hash", side_effect=lambda content: "0" if content == "good" else "1"),+ ):+ vector = factory.return_value+ vector.create.side_effect = load+ service.run_in_indexing_status(job.ref)+ pending, tokens = segments.resume_indexing(job)+ assert [chunk.page_content for chunk in pending] == ["bad"]+ assert tokens == 30+ vector.create.reset_mock()+ vector.create.side_effect = None+ service.run_in_indexing_status(job.ref)+ vector.create.assert_called_once()+ assert [chunk.page_content for chunk in vector.create.call_args.args[0]] == ["bad"]+ with sqlite_session_factory() as session:+ row = session.get(Document, "document-1")+ assert row is not None+ assert row.indexing_status == IndexingStatus.COMPLETED+ assert row.tokens == 30+ assert row.error is Nonediff --git a/api/tests/unit_tests/services/knowledge/test_indexing_source_adapters.py b/api/tests/unit_tests/services/knowledge/test_indexing_source_adapters.pynew file mode 100644index 00000000000..8037a4d037d--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_indexing_source_adapters.py@@ -0,0 +1,342 @@+from collections.abc import Sequence+from dataclasses import dataclass, field+from unittest.mock import Mock++import pytest++from core.rag.extractor.entity.datasource_type import DatasourceType+from core.rag.extractor.entity.extract_setting import UploadFileExtractionInput+from core.rag.extractor.notion_extractor import NotionExtractor+from services.data_source.credential_gateway import (+ DatasourceCredentialNotFoundError,+ DatasourceCredentialRefreshError,+ DatasourceProviderCredentialStore,+)+from services.data_source.entities.notion_import import NotionPageType+from services.data_source.provider_service import DatasourceProviderService+from services.knowledge.indexing.adapters.sources import (+ CompositeStoredSourceResolver,+ FileSourceAdapter,+ NotionSourceResolver,+ WebsiteSourceAdapter,+)+from services.knowledge.indexing.errors import (+ IndexingInputSourceError,+ SourceCredentialUnavailableError,+ UnsupportedStoredSourceError,+)+from services.knowledge.indexing.estimate import StoredSource+from services.knowledge.resource_scope import DatasetRef+from tests.unit_tests.config_override import apply_config_overrides+++def _source(+ source_type: str,+ source_info: dict[str, object] | None,+ *,+ document_id: str = "document-1",+) -> StoredSource:+ return StoredSource(+ document_ref=DatasetRef("workspace-1", "dataset-1").document(document_id),+ source_type=source_type,+ source_info=source_info,+ document_model="text_model",+ )+++@dataclass+class RecordingUploads:+ uploads: dict[str, UploadFileExtractionInput]+ calls: list[tuple[str, tuple[str, ...]]] = field(default_factory=list)++ def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> dict[str, UploadFileExtractionInput]:+ self.calls.append((workspace_id, tuple(file_ids)))+ return {file_id: self.uploads[file_id] for file_id in file_ids if file_id in self.uploads}+++@dataclass+class RecordingActorCredentials:+ credentials: dict[str, object] | None = None+ error: Exception | None = None+ calls: list[dict[str, object]] = field(default_factory=list)++ def resolve(self, **kwargs: object) -> dict[str, object]:+ self.calls.append(kwargs)+ if self.error is not None:+ raise self.error+ return self.credentials or {}+++@dataclass+class RecordingStoredCredentials:+ credentials: dict[str, object] | None = None+ error: Exception | None = None+ calls: list[dict[str, object]] = field(default_factory=list)++ def resolve_for_document(self, **kwargs: object) -> dict[str, object]:+ self.calls.append(kwargs)+ if self.error is not None:+ raise self.error+ return self.credentials or {}+++def _notion_resolver(+ *,+ actor: RecordingActorCredentials | None = None,+ stored: RecordingStoredCredentials | None = None,+ integration_token: str | None = None,+) -> NotionSourceResolver:+ return NotionSourceResolver(+ actor_credentials=actor or RecordingActorCredentials({"integration_secret": "actor-secret"}),+ stored_credentials=stored or RecordingStoredCredentials({"integration_secret": "stored-secret"}),+ integration_token=integration_token,+ )+++def test_datasource_adapters_resolve_file_website_and_notion_inputs() -> None:+ upload = UploadFileExtractionInput(+ id="file-1",+ tenant_id="workspace-1",+ key="file.txt",+ created_by="account-1",+ )+ uploads = RecordingUploads({upload.id: upload})+ notion = _notion_resolver()+ resolver = CompositeStoredSourceResolver(+ adapters={+ "upload_file": FileSourceAdapter(uploads=uploads),+ "website_crawl": WebsiteSourceAdapter(),+ "notion_import": notion,+ }+ )++ file_extraction = resolver.resolve(+ _source("upload_file", {"upload_file_id": upload.id}),+ )+ website_extraction = resolver.resolve(+ _source(+ "website_crawl",+ {+ "provider": "firecrawl",+ "job_id": "job-1",+ "url": "https://example.com",+ "mode": "crawl",+ "only_main_content": True,+ },+ ),+ )+ notion_extraction = resolver.resolve(+ _source(+ "notion_import",+ {+ "credential_id": "credential-1",+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "page",+ },+ ),+ )++ assert file_extraction.datasource_type == DatasourceType.FILE+ assert file_extraction.upload_file == upload+ assert uploads.calls == [("workspace-1", ("file-1",))]+ assert website_extraction.website_info is not None+ assert website_extraction.website_info.tenant_id == "workspace-1"+ assert website_extraction.website_info.only_main_content is True+ assert notion_extraction.notion_info is not None+ assert notion_extraction.notion_info.notion_access_token == "stored-secret"+++def test_stored_notion_resolution_uses_trusted_document_owner_chain() -> None:+ actor = RecordingActorCredentials({"integration_secret": "actor-secret"})+ stored = RecordingStoredCredentials({"integration_secret": "stored-secret"})+ resolver = _notion_resolver(actor=actor, stored=stored)++ extraction = resolver.resolve(+ _source(+ "notion_import",+ {+ "credential_id": "private-credential",+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "database",+ },+ ),+ )++ assert actor.calls == []+ assert stored.calls == [+ {+ "workspace_id": "workspace-1",+ "dataset_id": "dataset-1",+ "document_id": "document-1",+ "credential_id": "private-credential",+ "provider": "notion_datasource",+ "plugin_id": "langgenius/notion_datasource",+ }+ ]+ assert extraction.notion_info is not None+ assert extraction.notion_info.notion_page_type == NotionPageType.DATABASE.value+++ ("credential_id", "credentials", "error", "expected_token"),+ [+ (None, {"integration_secret": "default-secret"}, None, "environment-secret"),+ ("credential-1", {"integration_secret": "saved-secret"}, None, "saved-secret"),+ ("deleted-credential", {}, DatasourceCredentialNotFoundError(), "environment-secret"),+ ("credential-1", {}, DatasourceCredentialRefreshError("credential-1"), "environment-secret"),+ ("credential-1", {}, None, "environment-secret"),+ ("credential-1", {"integration_secret": ""}, None, "environment-secret"),+ ("credential-1", {"integration_secret": 42}, None, "environment-secret"),+ ],+ ids=["legacy", "saved", "deleted", "refresh-failed", "missing-token", "empty-token", "invalid-token"],+)[email protected]("integration_token", ["environment-secret", None, ""])+def test_stored_notion_preview_and_indexing_use_same_integration(+ monkeypatch: pytest.MonkeyPatch,+ credential_id: str | None,+ credentials: dict[str, object],+ error: Exception | None,+ expected_token: str,+ integration_token: str | None,+) -> None:+ apply_config_overrides(monkeypatch, NOTION_INTEGRATION_TOKEN=integration_token)+ stored = RecordingStoredCredentials(credentials, error=error)+ resolver = _notion_resolver(stored=stored)+ indexing_credentials = Mock(return_value=credentials, side_effect=error)+ providers = DatasourceProviderService(credentials=Mock(spec=DatasourceProviderCredentialStore))+ monkeypatch.setattr(+ providers,+ "get_datasource_credentials",+ indexing_credentials,+ )+ source = _source(+ "notion_import",+ {+ "credential_id": credential_id,+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "page",+ },+ )++ def indexing_extractor() -> NotionExtractor:+ return NotionExtractor(+ notion_workspace_id="notion-workspace-1",+ notion_obj_id="notion-page-1",+ notion_page_type="page",+ tenant_id="workspace-1",+ credential_id=credential_id,+ notion_token_loader=lambda: providers.get_stored_notion_access_token(+ tenant_id="workspace-1", credential_id=credential_id+ ),+ )++ if expected_token == "environment-secret" and not integration_token:+ with pytest.raises(SourceCredentialUnavailableError):+ resolver.resolve(source)+ with pytest.raises(ValueError, match="Must specify `integration_token`"):+ indexing_extractor()+ else:+ preview = resolver.resolve(source)+ extractor = indexing_extractor()+ assert preview.notion_info is not None+ assert preview.notion_info.notion_access_token == extractor._notion_access_token == expected_token++ expected_ids: list[str] = [credential_id] if credential_id else []+ assert [call["credential_id"] for call in stored.calls] == expected_ids+ assert [call.kwargs["credential_id"] for call in indexing_credentials.call_args_list] == expected_ids+++def test_notion_resolution_falls_back_to_configured_integration_token() -> None:+ stored = RecordingStoredCredentials(error=DatasourceCredentialNotFoundError())+ resolver = _notion_resolver(stored=stored, integration_token="environment-secret")++ extraction = resolver.resolve(+ _source(+ "notion_import",+ {+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "page",+ },+ ),+ )++ assert extraction.notion_info is not None+ assert extraction.notion_info.notion_access_token == "environment-secret"+++def test_new_notion_selection_uses_actor_scoped_credential() -> None:+ actor = RecordingActorCredentials({"integration_secret": "actor-secret"})+ stored = RecordingStoredCredentials({"integration_secret": "stored-secret"})+ resolver = _notion_resolver(actor=actor, stored=stored)++ extraction = resolver.resolve_selection(+ workspace_id="workspace-1",+ actor_id="account-1",+ credential_id="credential-1",+ notion_workspace_id="notion-workspace-1",+ page_id="notion-page-1",+ page_type=NotionPageType.PAGE,+ document_model="text_model",+ )++ assert stored.calls == []+ assert actor.calls[0]["actor_id"] == "account-1"+ assert extraction.notion_info is not None+ assert extraction.notion_info.notion_access_token == "actor-secret"+++def test_new_notion_selection_does_not_fall_back_when_actor_cannot_access_credential() -> None:+ actor = RecordingActorCredentials(error=DatasourceCredentialNotFoundError())+ resolver = _notion_resolver(actor=actor, integration_token="environment-secret")++ with pytest.raises(DatasourceCredentialNotFoundError):+ resolver.resolve_selection(+ workspace_id="workspace-1",+ actor_id="account-1",+ credential_id="private-credential",+ notion_workspace_id="notion-workspace-1",+ page_id="notion-page-1",+ page_type=NotionPageType.PAGE,+ document_model="text_model",+ )+++def test_invalid_or_unknown_sources_are_rejected_at_the_adapter_boundary() -> None:+ resolver = CompositeStoredSourceResolver(adapters={"website_crawl": WebsiteSourceAdapter()})++ with pytest.raises(IndexingInputSourceError, match="provider"):+ resolver.resolve(+ _source("website_crawl", {"job_id": "job", "url": "url", "mode": "crawl"}),+ )+ with pytest.raises(UnsupportedStoredSourceError, match="mystery"):+ resolver.resolve(_source("mystery", {}))+++def test_formal_notion_extraction_carries_source_update_scope_but_preview_does_not() -> None:+ source = _source(+ "notion_import",+ {+ "credential_id": "credential-1",+ "notion_workspace_id": "notion-workspace-1",+ "notion_page_id": "notion-page-1",+ "type": "page",+ },+ )+ resolver = CompositeStoredSourceResolver(adapters={"notion_import": _notion_resolver()})+ preview = resolver.resolve(source)+ formal = resolver.resolve_for_indexing(source)+ assert preview.notion_info is not None+ assert formal.notion_info is not None+ assert preview.notion_info.document is None+ document = formal.notion_info.document+ assert document is not None+ assert (document.tenant_id, document.dataset_id, document.id) == (+ source.document_ref.dataset.tenant_id,+ source.document_ref.dataset.dataset_id,+ source.document_ref.document_id,+ )+ assert document.data_source_info_dict == source.source_infodiff --git a/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.pynew file mode 100644index 00000000000..23ce2b19dd3--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py@@ -0,0 +1,305 @@+"""Published applications sign admin-uploaded images only through valid knowledge bindings."""++from datetime import UTC, datetime+from unittest.mock import MagicMock+from uuid import uuid4++import pytest+from sqlalchemy import Engine, event+from sqlalchemy.orm import Session++from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity+from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.app.file_access import FileAccessScope, bind_file_access_scope, get_current_file_access_scope+from core.rag.models.document import Document as RagDocument+from core.rag.retrieval import dataset_retrieval as retrieval_module+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval+from core.workflow.nodes.knowledge_retrieval.retrieval import KnowledgeRetrievalRequest+from extensions.storage.storage_type import StorageType+from models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBinding+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus+from models.model import UploadFile+from services.knowledge.retrieval.attachments import authorize_retrieved_segment+++def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment, dict[str, str]]:+ dataset = Dataset(id=str(uuid4()), tenant_id=str(uuid4()), name="Dataset", created_by="admin")+ document = Document(+ id=str(uuid4()),+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom.WEB,+ created_by="admin",+ doc_form="text_model",+ doc_metadata={},+ )+ segment = DocumentSegment(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ position=1,+ content="",+ word_count=1,+ tokens=1,+ created_by="admin",+ index_node_id=str(uuid4()),+ index_node_hash="hash",+ status=SegmentStatus.COMPLETED,+ )+ file_ids: dict[str, str] = {}+ for kind in ("valid", "unbound", "tenant", "file_tenant", "dataset", "document", "segment"):+ upload = UploadFile(+ tenant_id=str(uuid4()) if kind == "file_tenant" else dataset.tenant_id,+ storage_type=StorageType.LOCAL,+ key=f"{kind}.png",+ name=f"{kind}.png",+ size=1,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="admin",+ created_at=datetime.now(UTC),+ used=True,+ )+ file_ids[kind] = upload.id+ sqlite_session.add(upload)+ if kind != "unbound":+ sqlite_session.add(+ SegmentAttachmentBinding(+ tenant_id=str(uuid4()) if kind == "tenant" else dataset.tenant_id,+ dataset_id=str(uuid4()) if kind == "dataset" else dataset.id,+ document_id=str(uuid4()) if kind == "document" else document.id,+ segment_id=str(uuid4()) if kind == "segment" else segment.id,+ attachment_id=upload.id,+ )+ )+ segment.content = "\n".join(+ f"" for kind, file_id in file_ids.items()+ )+ sqlite_session.add_all([dataset, document, segment])+ sqlite_session.commit()+ return dataset, document, segment, file_ids++[email protected]("entry", ["chat", "workflow"])[email protected]("vision_enabled", [False, True])+def test_published_retrieval_grants_bound_images_before_signing(+ graph: tuple[Dataset, Document, DocumentSegment, dict[str, str]],+ sqlite_session: Session,+ monkeypatch: pytest.MonkeyPatch,+ entry: str,+ vision_enabled: bool,+) -> None:+ dataset, document, segment, file_ids = graph+ retrieval = DatasetRetrieval()+ monkeypatch.setattr(retrieval, "_check_knowledge_rate_limit", MagicMock())+ monkeypatch.setattr(retrieval, "_get_available_datasets", MagicMock(return_value=[dataset]))+ monkeypatch.setattr(retrieval, "get_metadata_filter_condition", MagicMock(return_value=(None, None)))+ monkeypatch.setattr(+ retrieval,+ "multiple_retrieve",+ MagicMock(+ return_value=[+ RagDocument(+ page_content=segment.content,+ provider="dify",+ metadata={+ "doc_id": segment.index_node_id,+ "document_id": document.id,+ "dataset_id": dataset.id,+ "score": 0.8,+ },+ )+ ]+ ),+ )+ model = MagicMock()+ model.model_type_instance.get_model_schema.return_value.features = None+ monkeypatch.setattr(+ retrieval_module.ModelManager,+ "for_tenant",+ MagicMock(+ return_value=MagicMock(+ get_model_instance=MagicMock(return_value=model),+ )+ ),+ )+ scope = FileAccessScope(+ tenant_id=dataset.tenant_id,+ user_id=str(uuid4()),+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ )+ with bind_file_access_scope(scope):+ if entry == "chat":+ content, files = retrieval.retrieve(+ sqlite_session,+ app_id=str(uuid4()),+ user_id=scope.user_id,+ tenant_id=dataset.tenant_id,+ model_config=MagicMock(),+ config=DatasetEntity(+ dataset_ids=[dataset.id],+ retrieve_config=DatasetRetrieveConfigEntity(retrieve_strategy="multiple"),+ ),+ query="picture",+ invoke_from=InvokeFrom.WEB_APP,+ show_retrieve_source=False,+ hit_callback=MagicMock(),+ message_id=str(uuid4()),+ vision_enabled=vision_enabled,+ )+ assert files is not None+ assert [file.storage_key for file in files] == (["valid.png"] if vision_enabled else [])+ else:+ sources = retrieval.knowledge_retrieval(+ sqlite_session,+ KnowledgeRetrievalRequest(+ tenant_id=dataset.tenant_id,+ user_id=scope.user_id,+ app_id=str(uuid4()),+ user_from=UserFrom.END_USER.value,+ dataset_ids=[dataset.id],+ query="picture",+ retrieval_mode="multiple",+ ),+ )+ assert len(sources) == 1+ content = sources[0].content+ current_scope = get_current_file_access_scope()+ assert current_scope is not None+ assert current_scope.granted_upload_file_ids == frozenset([file_ids["valid"]])+ assert segment.id in current_scope.granted_retriever_segment_ids+ assert content is not None+ assert f"/files/{file_ids['valid']}/image-preview?timestamp=" in content+ assert "sign=old" not in content+ for kind, file_id in file_ids.items():+ if kind != "valid":+ assert f"/files/{file_id}/image-preview)" in content++[email protected]("broken_owner", ["tenant", "dataset", "document"])+def test_invalid_segment_owner_chain_grants_no_access(+ graph: tuple[Dataset, Document, DocumentSegment, dict[str, str]],+ sqlite_session: Session,+ broken_owner: str,+) -> None:+ dataset, document, segment, _ = graph+ if broken_owner == "tenant":+ document.tenant_id = str(uuid4())+ elif broken_owner == "dataset":+ document.dataset_id = str(uuid4())+ else:+ segment.document_id = str(uuid4())+ sqlite_session.commit()+ scope = FileAccessScope(+ tenant_id=dataset.tenant_id,+ user_id=str(uuid4()),+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ )+ with bind_file_access_scope(scope):+ assert (+ authorize_retrieved_segment(+ segment, tenant_id=dataset.tenant_id, dataset_ids=[dataset.id], session=sqlite_session+ )+ is None+ )+ current_scope = get_current_file_access_scope()+ assert current_scope is not None+ assert current_scope.granted_upload_file_ids == frozenset()+ assert current_scope.granted_retriever_segment_ids == frozenset()++[email protected]("with_image", [False, True])[email protected]("signing_fails", [False, True])+def test_workflow_retrieval_releases_owned_connections_and_preserves_caller_transaction(+ graph: tuple[Dataset, Document, DocumentSegment, dict[str, str]],+ sqlite_session: Session,+ monkeypatch: pytest.MonkeyPatch,+ with_image: bool,+ signing_fails: bool,+) -> None:+ dataset, document, segment, _ = graph+ if not with_image:+ segment.content = "Plain text"+ sqlite_session.commit()+ retrieval = DatasetRetrieval()+ monkeypatch.setattr(retrieval, "_check_knowledge_rate_limit", MagicMock())+ monkeypatch.setattr(retrieval, "_get_available_datasets", MagicMock(return_value=[dataset]))+ monkeypatch.setattr(+ retrieval,+ "multiple_retrieve",+ MagicMock(+ return_value=[+ RagDocument(+ page_content=segment.content,+ provider="dify",+ metadata={"doc_id": segment.index_node_id, "document_id": document.id, "dataset_id": dataset.id},+ )+ ]+ ),+ )+ if signing_fails:+ monkeypatch.setattr(+ retrieval_module, "sign_segment_content", MagicMock(side_effect=RuntimeError("signing failed"))+ )+ request = KnowledgeRetrievalRequest(+ tenant_id=dataset.tenant_id,+ user_id="visitor",+ app_id=str(uuid4()),+ user_from=UserFrom.END_USER.value,+ dataset_ids=[dataset.id],+ query="picture",+ retrieval_mode="multiple",+ )+ scope = FileAccessScope(+ tenant_id=dataset.tenant_id,+ user_id="visitor",+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ )+ engine = sqlite_session.get_bind()+ assert isinstance(engine, Engine)+ sessions: set[Session] = set()+ pool_events: list[str] = []++ def track_transaction(session: Session, *_args: object) -> None:+ # Hold references so garbage collection cannot hide an unclosed session.+ sessions.add(session)++ def track_checkout(*_args: object) -> None:+ pool_events.append("checkout")++ def track_checkin(*_args: object) -> None:+ pool_events.append("checkin")++ event.listen(Session, "after_begin", track_transaction)+ event.listen(engine, "checkout", track_checkout)+ event.listen(engine, "checkin", track_checkin)+ try:+ with bind_file_access_scope(scope), Session(engine) as caller_session, caller_session.begin() as caller_txn:+ if signing_fails:+ with pytest.raises(RuntimeError, match="signing failed"):+ retrieval.knowledge_retrieval(caller_session, request)+ else:+ sources = retrieval.knowledge_retrieval(caller_session, request)+ assert len(sources) == 1+ assert sources[0].metadata.segment_id == segment.id+ assert caller_session.get_transaction() is caller_txn+ assert caller_txn.is_active+ assert sessions+ assert all(not session.in_transaction() for session in sessions if session is not caller_session)+ assert pool_events.count("checkout") > 0+ assert pool_events.count("checkout") == pool_events.count("checkin")+ finally:+ event.remove(Session, "after_begin", track_transaction)+ event.remove(engine, "checkout", track_checkout)+ event.remove(engine, "checkin", track_checkin)+ for session in sessions:+ session.close()diff --git a/api/tests/unit_tests/services/knowledge/test_segment_adapters.py b/api/tests/unit_tests/services/knowledge/test_segment_adapters.pynew file mode 100644index 00000000000..a3cac9fa35e--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_segment_adapters.py@@ -0,0 +1,1729 @@+from collections.abc import Iterator, Mapping, Sequence+from contextlib import nullcontext+from dataclasses import dataclass, field+from datetime import datetime+from pathlib import Path+from typing import Protocol+from unittest.mock import create_autospec, patch++import pytest+from sqlalchemy import event, select+from sqlalchemy.dialects import postgresql+from sqlalchemy.engine import Connection+from sqlalchemy.orm import Session, sessionmaker++from core.credit_usage import CreditUsageCreatedBy+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError+from core.model_context import get_credit_usage_metadata+from core.model_manager import ModelManager+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from core.rag.index_processor.index_processor_base import SummaryIndexSettingDict+from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor+from core.rag.models.document import ChildDocument+from core.rag.models.document import Document as IndexDocument+from extensions.storage.storage_type import StorageType+from graphon.model_runtime.entities.llm_entities import LLMResult, LLMUsage+from graphon.model_runtime.entities.message_entities import AssistantPromptMessage+from libs.helper import generate_text_hash+from libs.pagination import PaginatedResult+from machinery.context import RequestContext+from models.dataset import ChildChunk, Dataset, Document, DocumentSegment, SegmentAttachmentBinding+from models.enums import CreatorUserRole, DocumentCreatedFrom, SegmentStatus, SummaryStatus+from models.model import UploadFile+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from repositories.workspace_member_query_repository import WorkspaceMemberQueryRepository+from services.knowledge.dataset_access import DatasetAccessService+from services.knowledge.dataset_service import SegmentService+from services.knowledge.entities.segments import ChildChunkUpdateArgs, SegmentUpdateArgs+from services.knowledge.resource_scope import DatasetRef+from services.knowledge.segments.adapters import (+ CelerySegmentBatchImportDispatcher,+ ModelManagerSegmentGuard,+ RedisSegmentIndexingState,+)+from services.knowledge.segments.application import (+ DatasetSegmentApplicationService,+ SegmentBatchImportDispatcher,+ SegmentDatasetRecord,+ SegmentDetail,+ SegmentIndex,+ SegmentIndexingState,+ SegmentListFilter,+ SegmentModelGuard,+ SegmentModelProviderError,+)+++@dataclass+class SegmentLimitsStub:+ SINGLE_CHUNK_ATTACHMENT_LIMIT: int = 10+++def test_celery_dispatcher_preserves_task_argument_order() -> None:+ calls: list[tuple[str, ...]] = []++ def delay(*args: str) -> None:+ calls.append(args)++ dispatcher = CelerySegmentBatchImportDispatcher(delay=delay)+ dispatcher.dispatch(+ job_id="job-1",+ upload_file_id="file-1",+ dataset_id="dataset-1",+ document_id="document-1",+ workspace_id="workspace-1",+ actor_id="account-1",+ )++ assert calls == [("job-1", "file-1", "dataset-1", "document-1", "workspace-1", "account-1")]+++def _dataset(dataset_id: str, workspace_id: str, *, permission: str = "partial_members") -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=workspace_id,+ name="Dataset",+ description="",+ provider="vendor",+ permission=permission,+ indexing_technique="economy",+ created_by="account-1",+ maintainer="maintainer-1",+ )+++def _document(document_id: str, workspace_id: str, dataset_id: str) -> Document:+ return Document(+ id=document_id,+ tenant_id=workspace_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type="upload_file",+ batch="batch-1",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by="account-1",+ doc_form="text_model",+ word_count=0,+ )+++def _segment(segment_id: str, workspace_id: str, dataset_id: str, document_id: str) -> DocumentSegment:+ segment = DocumentSegment(+ tenant_id=workspace_id,+ dataset_id=dataset_id,+ document_id=document_id,+ position=1,+ content="content",+ word_count=7,+ tokens=2,+ created_by="account-1",+ status=SegmentStatus.COMPLETED,+ )+ segment.id = segment_id+ return segment+++def _upload(file_id: str, workspace_id: str, name: str) -> UploadFile:+ upload = UploadFile(+ tenant_id=workspace_id,+ storage_type=StorageType.OPENDAL,+ key=f"key-{file_id}",+ name=name,+ size=1,+ extension="csv",+ mime_type="text/csv",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime(2026, 1, 1),+ used=False,+ )+ upload.id = file_id+ return upload+++def test_scope_lookup_enforces_owner_chain(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))++ store = SQLAlchemyDatasetRepository(session_factory=sqlite_session_factory)+ scope = store.get_segment_scope(+ dataset_ref=DatasetRef("workspace-1", "dataset-1"),+ document_id="document-1",+ actor_id="account-1",+ )++ assert scope.dataset is not None+ assert scope.document is not None+ assert (+ store.get_segment_scope(+ dataset_ref=DatasetRef("workspace-2", "dataset-1"),+ document_id="document-1",+ actor_id="account-1",+ ).dataset+ is None+ )+++def test_upload_lookup_is_tenant_scoped(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _upload("file-1", "workspace-1", "one.csv"),+ _upload("file-2", "workspace-2", "two.csv"),+ ]+ )++ store = SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory)++ assert store.get_file_name(workspace_id="workspace-1", upload_file_id="file-1") == "one.csv"+ assert store.get_file_name(workspace_id="workspace-1", upload_file_id="file-2") is None+++def test_segment_lookup_rejects_segment_from_another_owner_chain(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _dataset("dataset-1", "workspace-1"),+ _dataset("dataset-2", "workspace-2"),+ _document("document-1", "workspace-1", "dataset-1"),+ _document("document-2", "workspace-2", "dataset-2"),+ _segment("segment-1", "workspace-2", "dataset-2", "document-2"),+ ]+ )++ store = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ assert store.get_segment(DatasetRef("workspace-1", "dataset-1").document("document-1").segment("segment-2")) is None++[email protected]("owner_field", ["tenant_id", "dataset_id", "document_id"])+def test_segment_update_state_enforces_complete_owner_chain(+ sqlite_session_factory: sessionmaker[Session], owner_field: str+) -> None:+ with sqlite_session_factory.begin() as session:+ segment = _segment("segment-1", "workspace-1", "dataset-1", "document-1")+ setattr(segment, owner_field, "foreign-owner")+ session.add(segment)+ store = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1").segment("segment-1")++ assert store.get_segment_update_state(ref) is None+++def test_empty_segment_list_returns_materialized_page(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))++ store = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ with patch("repositories.knowledge.segment_repository.dify_config") as config:+ config.SQLALCHEMY_DATABASE_URI_SCHEME = "mysql"+ result = store.list_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"),+ SegmentListFilter(page=1, limit=20, keyword="missing"),+ )++ assert result.items == ()+ assert result.total == 0+ assert result.limit == 20+++def test_postgresql_keyword_filter_keeps_json_array_guard(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))++ page = PaginatedResult[DocumentSegment](items=[], total=0, page=1, per_page=20)+ store = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ with (+ patch("repositories.knowledge.segment_repository.dify_config") as config,+ patch("repositories.knowledge.segment_repository.paginate_query", return_value=page) as paginate,+ ):+ config.SQLALCHEMY_DATABASE_URI_SCHEME = "postgresql"+ store.list_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"),+ SegmentListFilter(keyword="知识"),+ )++ statement = paginate.call_args.args[0]+ sql = str(statement.compile(dialect=postgresql.dialect()))+ assert "jsonb_typeof" in sql+ assert "jsonb_array_elements_text" in sql+ assert "CASE WHEN" in sql+++def _application(+ factory: sessionmaker[Session], *, indexing: SegmentIndex | None = None+) -> DatasetSegmentApplicationService:+ datasets = SQLAlchemyDatasetRepository(session_factory=factory)+ index = create_autospec(SegmentIndex, instance=True, spec_set=True)+ index.count_tokens.return_value = 0+ state = create_autospec(SegmentIndexingState, instance=True, spec_set=True)+ state.is_segment_indexing.return_value = False+ state.is_document_indexing.return_value = False+ return DatasetSegmentApplicationService(+ dataset_access=DatasetAccessService(+ datasets=datasets,+ workspace_roles=WorkspaceMemberQueryRepository(session_factory=factory),+ legacy_permissions_enabled=False,+ ),+ scopes=datasets,+ store=SQLAlchemySegmentRepository(session_factory=factory),+ limits=SegmentLimitsStub(),+ text_hash=generate_text_hash,+ index=indexing or index,+ uploads=SQLAlchemyKnowledgeUploadRepository(session_factory=factory),+ model_guard=create_autospec(SegmentModelGuard, instance=True, spec_set=True),+ indexing_state=state,+ batch_dispatcher=create_autospec(SegmentBatchImportDispatcher, instance=True, spec_set=True),+ job_id_factory=lambda: "job-1",+ )+++def test_mutations_use_request_actor_without_a_flask_request(+ sqlite_session_factory: sessionmaker[Session],+) -> None:++ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))+ app = _application(sqlite_session_factory)+ context = RequestContext("request-1", None, "author-1", "workspace-1")+ created = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "original"}+ )+ assert created.data.created_by == "author-1"+ updated = app.update_segment(+ RequestContext("request-2", None, "editor-2", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "updated content"},+ )+ assert updated.data.updated_by == "editor-2"+ assert updated.data.content == "updated content"+ assert isinstance(updated.data.created_at, datetime)+ with sqlite_session_factory() as session:+ segment = session.get(DocumentSegment, created.data.id)+ assert segment is not None+ assert segment.created_by == "author-1"+ assert segment.updated_by == "editor-2"+ assert segment.status == SegmentStatus.COMPLETED+++def test_list_reads_dataset_and_document_once(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))+ statements: list[str] = []+ engine = sqlite_session_factory.kw["bind"]++ def record(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ statements.append(statement)++ event.listen(engine, "before_cursor_execute", record)+ try:+ page = _application(sqlite_session_factory).list_segments(+ RequestContext("request-1", None, "account-1", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ query=SegmentListFilter(),+ )+ finally:+ event.remove(engine, "before_cursor_execute", record)+ assert page.items == ()+ assert sum("FROM datasets" in statement for statement in statements) == 1+ assert sum("JOIN documents" in statement or "FROM documents" in statement for statement in statements) == 1+++ ("error", "kind"),+ [+ (LLMBadRequestError(), "bad_request"),+ (ProviderTokenNotInitError("token missing"), "token"),+ ],+)+def test_model_guard_translates_provider_errors(error: Exception, kind: str) -> None:+ from core.model_manager import ModelManager++ model = create_autospec(ModelManager, instance=True, spec_set=True)+ model.get_model_instance.side_effect = error+ dataset = SegmentDatasetRecord(+ id="dataset-1",+ workspace_id="workspace-1",+ indexing_technique="high_quality",+ embedding_model_provider="provider",+ embedding_model="model",+ )++ with patch("services.knowledge.segments.adapters.ModelManager.for_tenant", return_value=model):+ with pytest.raises(SegmentModelProviderError) as raised:+ ModelManagerSegmentGuard().check(dataset)++ assert raised.value.kind == kind+++def test_redis_state_uses_existing_keys_and_decodes_status() -> None:+ from services.knowledge.segments.adapters import RedisSegmentClient++ redis = create_autospec(RedisSegmentClient, instance=True, spec_set=True)+ redis.get.side_effect = [b"running", b"completed"]+ state = RedisSegmentIndexingState(redis)++ assert state.is_document_indexing("document-1") is True+ state.set_batch_waiting("job-1")+ assert state.get_batch_status("job-1") == "completed"+ redis.get.assert_any_call("document_document-1_indexing")+ redis.setnx.assert_called_once_with("segment_batch_import_job-1", "waiting")+++def test_child_chunk_mutations_preserve_explicit_actors(+ sqlite_session_factory: sessionmaker[Session],+) -> None:++ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ document = _document("document-1", "workspace-1", "dataset-1")+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ session.add(document)+ session.add(_segment("segment-1", "workspace-1", "dataset-1", "document-1"))+ app = _application(sqlite_session_factory)+ created = app.create_child_chunk(+ RequestContext("create", None, "creator", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ content="first",+ )+ app.update_child_chunk(+ RequestContext("update", None, "editor", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ child_chunk_id=created.id,+ content="second",+ )+ with sqlite_session_factory() as session:+ chunk = session.get(ChildChunk, created.id)+ assert chunk is not None+ assert chunk.created_by == "creator"+ assert chunk.updated_by == "editor"+ results = app.update_child_chunks(+ RequestContext("batch", None, "batch-editor", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ chunks=[ChildChunkUpdateArgs(id=created.id, content="third"), ChildChunkUpdateArgs(content="new child")],+ )+ with sqlite_session_factory() as session:+ chunks = [session.get(ChildChunk, result.id) for result in results]+ assert len(chunks) == 2+ existing, added = chunks+ assert existing is not None+ assert added is not None+ assert existing.updated_by == "batch-editor"+ assert added.created_by == "batch-editor"+++@dataclass+class IndexRedisStub:+ def get(self, name: str | bytes) -> None:+ del name++ def setnx(self, name: str | bytes, value: object) -> bool:+ del name, value+ return True++ def setex(self, name: str, time: int, value: object) -> bool:+ del name, time, value+ return True++ def lock(self, name: str, timeout: int) -> nullcontext[None]:+ del name, timeout+ return nullcontext()+++@dataclass+class VectorProbe:+ active_transactions: set[Connection]+ fail: bool = False+ writes: int = 0+ deleted_ids: list[str] = field(default_factory=list)+ tasks: list[tuple[str, tuple[object, ...]]] = field(default_factory=list)+ batches: list[tuple[str, ...]] = field(default_factory=list)++ def add_texts(self, documents: Sequence[IndexDocument], **_kwargs: object) -> None:+ assert not self.active_transactions, "Index I/O must not hold a DB transaction"+ self.writes += len(documents)+ if self.fail:+ raise RuntimeError("index unavailable")++ def create(self, documents: Sequence[IndexDocument], **kwargs: object) -> None:+ self.batches.append(tuple(document.page_content for document in documents))+ self.add_texts(documents, **kwargs)++ def delete_by_ids(self, ids: Sequence[str]) -> None:+ assert ids+ assert not self.active_transactions+ if self.fail:+ raise RuntimeError("index unavailable")+ self.deleted_ids.extend(ids)++ def create_multimodal(self, documents: Sequence[IndexDocument], **kwargs: object) -> None:+ assert not self.active_transactions+ assert kwargs["upload_files"]+ self.add_texts(documents)+++type IndexingProbe = tuple[DatasetSegmentApplicationService, SQLAlchemySegmentRepository, VectorProbe]+++def indexing_probe(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> Iterator[IndexingProbe]:+ from services.knowledge.segments import indexing as indexing_module++ with sqlite_session_factory.begin() as session:+ dataset = _dataset("dataset-1", "workspace-1")+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset.index_struct = '{"type": "qdrant"}'+ session.add(dataset)+ session.add(_document("document-1", "workspace-1", "dataset-1"))+ engine = sqlite_session_factory.kw["bind"]+ active: set[Connection] = set()+ begin = active.add+ finish = active.discard+ event.listen(engine, "begin", begin)+ event.listen(engine, "commit", finish)+ event.listen(engine, "rollback", finish)+ probe = VectorProbe(active)++ def vector(dataset: Dataset, *, session: Session | None, vector_type: str) -> VectorProbe:+ assert dataset.tenant_id == "workspace-1"+ assert session is None+ assert vector_type == "qdrant"+ assert not active+ return probe++ class TokenModel:+ def get_text_embedding_num_tokens(self, texts: Sequence[str]) -> list[int]:+ assert not active+ return [len(text) for text in texts]++ class ModelLookup:+ def get_model_instance(self, **kwargs: object) -> TokenModel:+ assert kwargs["tenant_id"] == "workspace-1"+ assert not active+ return TokenModel()++ monkeypatch.setattr(indexing_module, "Vector", vector)+ monkeypatch.setattr(indexing_module.ModelManager, "for_tenant", lambda **_kwargs: ModelLookup())+ repository = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)++ def dispatch(kind: str, *args: object) -> None:+ assert not active+ probe.tasks.append((kind, args))++ gateway = indexing_module.SegmentIndexingGateway(+ segments=repository,+ uploads=SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory),+ redis=IndexRedisStub(),+ new_session=sqlite_session_factory,+ delete_task=lambda *args: dispatch("delete", *args),+ enable_task=lambda *args: dispatch("enable", *args),+ disable_task=lambda *args: dispatch("disable", *args),+ )+ yield _application(sqlite_session_factory, indexing=gateway), repository, probe+ event.remove(engine, "begin", begin)+ event.remove(engine, "commit", finish)+ event.remove(engine, "rollback", finish)+++class SegmentUpdateEntry(Protocol):+ def __call__(+ self,+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ values: Mapping[str, object],+ ) -> SegmentDetail: ...++[email protected](params=["console", "service_api"])+def segment_update(+ request: pytest.FixtureRequest, indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session]+) -> SegmentUpdateEntry:+ app, repository, _ = indexing_probe+ if request.param == "console":+ return app.update_segment++ def update(+ context: RequestContext,+ *,+ dataset_id: str,+ document_id: str,+ segment_id: str,+ values: Mapping[str, object],+ ) -> SegmentDetail:+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, dataset_id)+ document = session.get(Document, document_id)+ segment = session.get(DocumentSegment, segment_id)+ assert dataset is not None+ assert document is not None+ assert segment is not None+ result = SegmentService.update_segment(+ SegmentUpdateArgs.model_validate(values),+ segment,+ document,+ dataset,+ session,+ actor_id=context.account_id,+ mutations=app.mutations,+ )+ assert result.content == (values.get("content") or segment.content)+ ref = DatasetRef(context.active_workspace_id, dataset_id).document(document_id).segment(segment_id)+ detail = repository.get_segment(ref)+ assert detail is not None+ return detail++ return update++[email protected]("entry", ["console", "service_api"])[email protected]("enabled", [True, False])+def test_delete_segment_with_legacy_scalar_keywords(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ entry: str,+ enabled: bool,+) -> None:+ app, _, probe = indexing_probe+ with sqlite_session_factory.begin() as session:+ segment = _segment("segment-1", "workspace-1", "dataset-1", "document-1")+ segment.keywords = "legacy-scalar"+ segment.enabled = enabled+ segment.index_node_id = "index-1"+ session.add(segment)+ document = session.get(Document, "document-1")+ assert document is not None+ document.word_count = segment.word_count++ if entry == "console":+ app.delete_segment(+ RequestContext("request", None, "author", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ )+ else:+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ segment = session.get(DocumentSegment, "segment-1")+ assert dataset is not None+ assert document is not None+ assert segment is not None+ SegmentService.delete_segment(segment, document, dataset, session, mutations=app.mutations)++ with sqlite_session_factory() as session:+ assert session.get(DocumentSegment, "segment-1") is None+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.word_count == 0+ if enabled:+ assert probe.tasks == [("delete", (["index-1"], "dataset-1", "document-1", ["segment-1"], []))]+ else:+ assert probe.tasks == []++[email protected]("keywords", ["legacy-scalar", 42, {"invalid": "object"}, [{"invalid": "item"}]])+def test_update_repairs_legacy_keywords_without_validating_presentation_first(+ segment_update: SegmentUpdateEntry,+ sqlite_session_factory: sessionmaker[Session],+ keywords: object,+) -> None:+ with sqlite_session_factory.begin() as session:+ segment = _segment("segment-1", "workspace-1", "dataset-1", "document-1")+ segment.keywords = keywords+ segment.index_node_id = "index-1"+ segment.index_node_hash = generate_text_hash(segment.content)+ session.add(segment)++ result = segment_update(+ RequestContext("request", None, "editor", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ values={"content": "content", "keywords": ["replacement"]},+ )++ assert result.data.status == "completed"+ assert result.data.keywords == ("replacement",)+ with sqlite_session_factory() as session:+ segment = session.get(DocumentSegment, "segment-1")+ assert segment is not None+ assert segment.keywords == ["replacement"]++[email protected]("content", ["original", "changed text"])+ ("attachment_values", "expected_ids", "removed_original"),+ [+ ({}, ["image-1"], False),+ ({"attachment_ids": None}, ["image-1"], False),+ ({"attachment_ids": []}, [], True),+ ({"attachment_ids": ["image-2"]}, ["image-2"], True),+ ],+ ids=["omitted", "null", "clear", "replace"],+)+def test_segment_update_preserves_or_replaces_attachments(+ segment_update: SegmentUpdateEntry,+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ content: str,+ attachment_values: dict[str, object],+ expected_ids: list[str],+ removed_original: bool,+) -> None:+ app, _, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [_upload("image-1", "workspace-1", "chart.png"), _upload("image-2", "workspace-1", "replacement.png")]+ )+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.is_multimodal = True+ created = app.create_segment(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ values={"content": "original", "attachment_ids": ["image-1"]},+ )+ assert [attachment.id for attachment in created.data.attachments] == ["image-1"]++ updated = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": content, **attachment_values},+ )++ assert updated.data.status == "completed"+ assert [attachment.id for attachment in updated.data.attachments] == expected_ids+ with sqlite_session_factory() as session:+ attachment_ids = list(+ session.scalars(+ select(SegmentAttachmentBinding.attachment_id).where(+ SegmentAttachmentBinding.segment_id == created.data.id+ )+ )+ )+ assert attachment_ids == expected_ids+ assert ("image-1" in probe.deleted_ids) is removed_original+++def test_segment_indexing_and_summary_use_closed_read_sessions(+ segment_update: SegmentUpdateEntry, indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session]+) -> None:+ from models.dataset import DocumentSegmentSummary++ app, repository, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "first"})+ updated = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "second", "summary": "manual summary"},+ )+ assert updated.data.status == "completed"+ assert updated.data.summary == "manual summary"+ with sqlite_session_factory() as session:+ summary = session.scalar(select(DocumentSegmentSummary))+ assert summary is not None+ assert summary.status == "completed"+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.word_count == len("second")+ assert probe.writes >= 3+ probe.fail = True+ failed = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "failed change"},+ )+ assert failed.data.status == "error"+ assert failed.data.enabled is False+ assert failed.data.error == "index unavailable"++[email protected]("operation", ["create", "update", "replace", "delete"])+def test_child_index_failure_leaves_database_unchanged(indexing_probe: IndexingProbe, operation: str) -> None:+ from services.knowledge.segments.application import (+ ChildChunkDeleteIndexApplicationError,+ ChildChunkIndexingApplicationError,+ )++ app, repository, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ parent = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "parent"})+ child = app.create_child_chunk(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ content="original",+ )+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(parent.data.id)+ before = repository.get_children(ref)+ probe.fail = True++ def mutate() -> None:+ match operation:+ case "create":+ app.create_child_chunk(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ content="new",+ )+ case "update":+ app.update_child_chunk(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ child_chunk_id=child.id,+ content="changed",+ )+ case "replace":+ app.update_child_chunks(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ chunks=[ChildChunkUpdateArgs(content="replacement")],+ )+ case "delete":+ app.delete_child_chunk(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ child_chunk_id=child.id,+ )++ with pytest.raises((ChildChunkIndexingApplicationError, ChildChunkDeleteIndexApplicationError)):+ mutate()+ assert repository.get_children(ref) == before+++def test_upload_batch_uses_one_scoped_query(sqlite_session_factory: sessionmaker[Session]) -> None:+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _upload("file-1", "workspace-1", "one.csv"),+ _upload("file-2", "workspace-1", "two.csv"),+ _upload("foreign", "workspace-2", "foreign.csv"),+ ]+ )+ engine = sqlite_session_factory.kw["bind"]+ statements = []++ def record_sql(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ statements.append(statement)++ event.listen(engine, "before_cursor_execute", record_sql)+ try:+ uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory).get_by_ids(+ workspace_id="workspace-1", file_ids=("file-2", "file-1", "foreign", "missing")+ )+ finally:+ event.remove(engine, "before_cursor_execute", record_sql)+ assert set(uploads) == {"file-1", "file-2"}+ assert len(statements) == 1+++def test_child_query_escapes_wildcards_and_enforces_complete_scope(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ from services.knowledge.segments.application import ChildChunkListFilter++ with sqlite_session_factory.begin() as session:+ session.add(_dataset("dataset-1", "workspace-1"))+ session.add(_document("document-1", "workspace-1", "dataset-1"))+ session.add(_segment("segment-1", "workspace-1", "dataset-1", "document-1"))+ for position, (workspace, text) in enumerate(+ [("workspace-1", "50%_literal"), ("workspace-1", "5000literal"), ("workspace-2", "50%_literal")], start=1+ ):+ session.add(+ ChildChunk(+ tenant_id=workspace,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ position=position,+ content=text,+ word_count=len(text),+ created_by="author",+ )+ )+ repo = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ page = repo.list_child_chunks(+ DatasetRef("workspace-1", "dataset-1").document("document-1").segment("segment-1"),+ ChildChunkListFilter(keyword="%_", limit=1),+ )+ assert page is not None+ assert page.total == 1+ assert [child.content for child in page.items] == ["50%_literal"]+++def test_batch_status_and_delete_only_modify_scoped_segments(+ indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session]+) -> None:+ app, repository, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ first = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "first"})+ second = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "second"})+ with sqlite_session_factory.begin() as session:+ session.add(_segment("foreign", "workspace-2", "dataset-1", "document-1"))+ segment_ids = [first.data.id, second.data.id, "foreign"]+ app.change_segment_status(+ context, dataset_id="dataset-1", document_id="document-1", segment_ids=segment_ids, action="disable"+ )+ kind, task_args = probe.tasks[-1]+ assert kind == "disable"+ assert isinstance(task_args[0], list)+ assert set(task_args[0]) == {first.data.id, second.data.id}+ assert task_args[1:] == ("dataset-1", "document-1")+ with sqlite_session_factory() as session:+ first_row = session.get(DocumentSegment, first.data.id)+ assert first_row is not None+ assert first_row.enabled is False+ foreign_row = session.get(DocumentSegment, "foreign")+ assert foreign_row is not None+ assert foreign_row.enabled is True+ app.delete_segments(context, dataset_id="dataset-1", document_id="document-1", segment_ids=segment_ids)+ with sqlite_session_factory() as session:+ assert session.get(DocumentSegment, first.data.id) is None+ assert session.get(DocumentSegment, second.data.id) is None+ assert session.get(DocumentSegment, "foreign") is not None+ document = session.get(Document, "document-1")+ assert document is not None+ assert document.word_count == 0+++def test_child_regeneration_reuses_splitter_outside_transaction(+ segment_update: SegmentUpdateEntry,+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import json++ from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor+ from models.dataset import DatasetProcessRule+ from models.enums import ProcessRuleMode++ app, repository, probe = indexing_probe+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.embedding_model_provider = "provider"+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.HIERARCHICAL,+ rules=json.dumps({"parent_mode": "full-doc", "subchunk_segmentation": {"max_tokens": 100}}),+ created_by="author",+ )+ session.add(rule)+ document = session.get(Document, "document-1")+ assert document is not None+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ document.dataset_process_rule_id = rule.id++ def split(_self, document: IndexDocument, _rule, _mode, _model) -> list[ChildDocument]:+ assert not probe.active_transactions+ return [+ ChildDocument(+ page_content=document.page_content, metadata={"doc_id": document.page_content, "doc_hash": "hash"}+ )+ ]++ monkeypatch.setattr(ParentChildIndexProcessor, "split_child_nodes", split)+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "first"})+ assert created.data.status == "completed"+ updated = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "second", "regenerate_child_chunks": True},+ )+ assert updated.data.status == "completed"+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(created.data.id)+ children = repository.get_children(ref)+ assert children is not None+ assert [child.data.content for child in children] == ["second"]+++ ("content", "submitted_summary", "summary_enabled", "expected_summary", "generated"),+ [+ ("new text", "old summary", True, "new summary", True),+ ("new text", None, True, "new summary", True),+ ("old text", "old summary", True, "old summary", False),+ ("new text", "manual summary", True, "manual summary", False),+ ("new text", "", True, None, False),+ ("new text", " \n\t ", True, None, False),+ ("new text", "old summary", False, "old summary", False),+ ("new text", None, False, "old summary", False),+ ],+)+def test_summary_updates_preserve_generation_and_manual_edit_rules(+ segment_update: SegmentUpdateEntry,+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ content: str,+ submitted_summary: str | None,+ summary_enabled: bool,+ expected_summary: str | None,+ generated: bool,+) -> None:+ app, repository, _ = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "old text"}+ )+ reference = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(created.data.id)+ repository.save_summary(reference, "old summary")+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.summary_index_setting = {+ "enable": summary_enabled,+ "model_name": "llm-model",+ "model_provider_name": "provider",+ }+ calls: list[str] = []++ def generate(_tenant_id: str, text: str, _setting: object, **_kwargs: object) -> tuple[str, LLMUsage]:+ calls.append(text)+ return "new summary", LLMUsage.empty_usage()++ monkeypatch.setattr(ParagraphIndexProcessor, "generate_summary_from_inputs", staticmethod(generate))+ updated = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": content, "summary": submitted_summary},+ )+ assert calls == ([content] if generated else [])+ assert updated.data.summary == expected_summary+++def test_automatic_summary_keeps_knowledge_credit_attribution(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ app, repository, _ = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "old text"}+ )+ reference = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(created.data.id)+ repository.save_summary(reference, "old summary")+ setting: SummaryIndexSettingDict = {"enable": True, "model_name": "llm-model", "model_provider_name": "provider"}+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.summary_index_setting = setting+ metadata: list[Mapping[str, object] | None] = []++ class ModelProbe:+ credentials: dict[str, str] = {}++ @property+ def model_type_instance(self) -> "ModelProbe":+ return self++ def get_model_schema(self, _name: str, _credentials: object) -> None:+ return None++ def get_text_embedding_num_tokens(self, texts: Sequence[str]) -> list[int]:+ return [len(text) for text in texts]++ def invoke_llm(self, **_kwargs: object) -> LLMResult:+ metadata.append(get_credit_usage_metadata())+ return LLMResult(+ model="llm-model", message=AssistantPromptMessage(content="new summary"), usage=LLMUsage.empty_usage()+ )++ def get_model_instance(self, **_kwargs: object) -> "ModelProbe":+ return self++ monkeypatch.setattr(ModelManager, "for_tenant", lambda **_kwargs: ModelProbe())+ # Both public summary entry points must preserve the credit-usage context.+ with sqlite_session_factory() as session:+ ParagraphIndexProcessor.generate_summary("workspace-1", "old text", setting, session=session)+ assert metadata.pop() == {"created_by": CreditUsageCreatedBy.KNOWLEDGE_INDEXING}+ updated = app.update_segment(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "new text"},+ )+ assert updated.data.summary == "new summary"+ assert metadata == [{"created_by": CreditUsageCreatedBy.KNOWLEDGE_INDEXING}]+ assert get_credit_usage_metadata() is None++[email protected]("storage_type", ["database", "file"])+def test_economy_index_shares_persistence_with_existing_keyword_calls(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ tmp_path: Path,+ storage_type: str,+) -> None:+ from core.rag.datasource.keyword.jieba import jieba as jieba_module+ from core.rag.datasource.keyword.keyword_factory import Keyword+ from repositories.knowledge import keyword_table_repository as table_module+ from tests.unit_tests.config_override import apply_config_overrides++ app, repository, probe = indexing_probe+ reads: list[str] = []+ writes: list[str] = []++ class FileStorageProbe:+ def load_once(self, key: str) -> bytes:+ assert not probe.active_transactions+ reads.append(key)+ return (tmp_path / key).read_bytes()++ def exists(self, key: str) -> bool:+ assert not probe.active_transactions+ return (tmp_path / key).exists()++ def save(self, key: str, data: bytes) -> None:+ assert not probe.active_transactions+ writes.append(key)+ path = tmp_path / key+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(data)++ def delete(self, key: str) -> None:+ assert not probe.active_transactions+ (tmp_path / key).unlink()++ class KeywordExtractor:+ def extract_keywords(self, text: str, _keyword_number: int = 10) -> set[str]:+ assert not probe.active_transactions+ return set(text.split())++ storage = FileStorageProbe()+ monkeypatch.setattr(jieba_module, "storage", storage)+ monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", KeywordExtractor)+ apply_config_overrides(monkeypatch, KEYWORD_DATA_SOURCE_TYPE=storage_type)+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.ECONOMY+ context = RequestContext("request", None, "author", "workspace-1")+ first = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "initial 中文"}+ )+ second = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "other", "keywords": ["manual"]}+ )+ updated = app.update_segment(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=first.data.id,+ values={"content": "replacement"},+ )+ assert first.data.status == second.data.status == updated.data.status == "completed"+ assert first.data.index_node_id is not None+ assert second.data.index_node_id is not None+ assert set(first.data.keywords or ()) == {"initial", "中文"}+ assert second.data.keywords == ("manual",)+ assert updated.data.keywords == ("replacement",)+ dataset_ref = DatasetRef("workspace-1", "dataset-1")+ source_type, data = repository.get_keyword_table(dataset_ref)+ payload = table_module.load_keyword_table(+ storage=storage, tenant_id="workspace-1", dataset_id="dataset-1", storage_type=source_type, data=data+ )+ assert payload is not None+ assert payload["__data__"]["table"] == {+ "replacement": {first.data.index_node_id},+ "manual": {second.data.index_node_id},+ }+ if storage_type == "database":+ assert reads == writes == []+ else:+ assert writes == ["keyword_files/workspace-1/dataset-1.txt"] * 3++ # The existing session-taking API must consume the exact payload written by+ # the repository-backed path. Only that existing API retains caller sessions.+ class SessionFileStorage:+ def load_once(self, key: str) -> bytes:+ return (tmp_path / key).read_bytes()++ def exists(self, key: str) -> bool:+ return (tmp_path / key).exists()++ def save(self, key: str, data: bytes) -> None:+ (tmp_path / key).write_bytes(data)++ def delete(self, key: str) -> None:+ (tmp_path / key).unlink()++ session_storage = SessionFileStorage()+ monkeypatch.setattr(jieba_module, "storage", session_storage)+ monkeypatch.setattr(jieba_module, "redis_client", IndexRedisStub())+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ keyword = Keyword(dataset)+ assert keyword.text_exists(first.data.index_node_id, session=session)+ assert keyword.text_exists(second.data.index_node_id, session=session)+ keyword.delete_by_ids([first.data.index_node_id], session)+ session.commit()+ source_type, data = repository.get_keyword_table(dataset_ref)+ payload = table_module.load_keyword_table(+ storage=session_storage, tenant_id="workspace-1", dataset_id="dataset-1", storage_type=source_type, data=data+ )+ assert payload is not None+ assert payload["__data__"]["table"] == {"manual": {second.data.index_node_id}}++[email protected]("generated_summary", [None, "", " \n\t "], ids=["exception", "empty", "whitespace"])+def test_summary_generation_failure_preserves_the_completed_segment(+ segment_update: SegmentUpdateEntry,+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ generated_summary: str | None,+) -> None:+ from models.dataset import DocumentSegmentSummary++ app, repository, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "original"}+ )+ reference = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(created.data.id)+ repository.save_summary(reference, "original summary")+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.summary_index_setting = {"enable": True, "model_name": "model", "model_provider_name": "provider"}+ summary = session.scalar(select(DocumentSegmentSummary))+ assert summary is not None+ summary.summary_index_node_id = "original-summary-node"++ def generate(*_args: object, **_kwargs: object) -> tuple[str, LLMUsage]:+ assert not probe.active_transactions+ if generated_summary is not None:+ return generated_summary, LLMUsage.empty_usage()+ raise RuntimeError("LLM unavailable")++ monkeypatch.setattr(ParagraphIndexProcessor, "generate_summary_from_inputs", staticmethod(generate))+ updated = segment_update(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "replacement", "summary": "original summary"},+ )+ assert updated.data.content == "replacement"+ assert updated.data.enabled is True+ assert updated.data.status == "completed"+ assert updated.data.summary == "original summary"+ assert "original-summary-node" not in probe.deleted_ids+ with sqlite_session_factory() as session:+ summary = session.scalar(select(DocumentSegmentSummary))+ assert summary is not None+ assert summary.status == "error"+ assert summary.summary_index_node_id == "original-summary-node"+ assert summary.error == ("Generated summary is empty" if generated_summary is not None else "LLM unavailable")++[email protected]("failed_commits", [1, 2, 3])+def test_summary_commit_retries_use_fresh_sessions_and_persist_outcome(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ failed_commits: int,+) -> None:+ from models.dataset import DocumentSegmentSummary+ from services.knowledge.summaries import adapters as summary_module++ app, _, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ created = app.create_segment(+ context, dataset_id="dataset-1", document_id="document-1", values={"content": "original"}+ )+ attempts: list[Session] = []+ delays: list[float] = []++ def fail_summary_commit(session: Session) -> None:+ if any(+ isinstance(record, DocumentSegmentSummary) and record.status == SummaryStatus.COMPLETED+ for record in session.new | session.dirty+ ):+ attempts.append(session)+ if len(attempts) <= failed_commits:+ raise ConnectionError("connection lost before commit")++ def wait(seconds: float) -> None:+ assert not probe.active_transactions+ delays.append(seconds)++ monkeypatch.setattr(summary_module.time, "sleep", wait)+ event.listen(sqlite_session_factory.class_, "before_commit", fail_summary_commit)+ try:+ updated = app.update_segment(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=created.data.id,+ values={"content": "original", "summary": "manual summary"},+ )+ assert updated.data.status == "completed"+ assert updated.data.enabled is True+ assert not probe.active_transactions+ finally:+ event.remove(sqlite_session_factory.class_, "before_commit", fail_summary_commit)+ for session in attempts:+ session.close()++ assert len(attempts) == min(failed_commits + 1, 3)+ assert len({id(session) for session in attempts}) == len(attempts)+ assert delays == [2.0, 4.0][: min(failed_commits, 2)]+ with sqlite_session_factory() as session:+ summary = session.scalar(select(DocumentSegmentSummary))+ assert summary is not None+ assert summary.summary_content == "manual summary"+ if failed_commits < 3:+ assert summary.status == SummaryStatus.COMPLETED+ assert summary.summary_index_node_id is not None+ assert summary.summary_index_node_hash is not None+ assert summary.error is None+ else:+ assert summary.status == SummaryStatus.ERROR+ assert summary.error == "Vectorization failed: connection lost before commit"+++def test_summary_noop_does_not_open_a_database_transaction(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ from services.knowledge.segments.indexing import SegmentIndexingGateway++ repository = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)+ gateway = SegmentIndexingGateway(+ segments=repository,+ uploads=SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory),+ redis=IndexRedisStub(),+ new_session=sqlite_session_factory,+ delete_task=lambda *_args: None,+ enable_task=lambda *_args: None,+ disable_task=lambda *_args: None,+ )+ engine = sqlite_session_factory.kw["bind"]+ connections: list[Connection] = []+ record_connection = connections.append+ event.listen(engine, "begin", record_connection)+ try:+ gateway.update_summary(+ DatasetRef("workspace-1", "dataset-1").document("document-1").segment("segment-1"),+ summary=None,+ content_changed=False,+ )+ finally:+ event.remove(engine, "begin", record_connection)+ assert connections == []+++def test_service_api_child_mutations_refresh_rows_and_release_read_transactions(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ app, repository, _ = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ parent = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "parent"})+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(parent.data.id)+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ segment = session.get(DocumentSegment, parent.data.id)+ assert dataset is not None+ assert document is not None+ assert segment is not None+ child = SegmentService.create_child_chunk(+ "first",+ segment,+ document,+ dataset,+ session,+ actor_id="creator",+ mutations=app.mutations,+ )+ child_id = child.id+ assert child.position == 1+ assert child.created_by == "creator"+ updated = SegmentService.update_child_chunk(+ "second",+ child,+ segment,+ document,+ dataset,+ session,+ actor_id="editor",+ mutations=app.mutations,+ )+ assert updated.content == "second"+ assert updated.updated_by == "editor"+ SegmentService.delete_child_chunk(updated, dataset, session, mutations=app.mutations)+ assert session.get(ChildChunk, child_id) is None+ SegmentService.delete_segment(segment, document, dataset, session, mutations=app.mutations)+ assert document.word_count == 0+ assert repository.get_segment(ref) is None++[email protected]("operation", ["create", "update", "delete"])+def test_service_api_child_index_failure_preserves_rows_and_legacy_errors(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ operation: str,+) -> None:+ from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError++ app, repository, probe = indexing_probe+ context = RequestContext("request", None, "author", "workspace-1")+ parent = app.create_segment(context, dataset_id="dataset-1", document_id="document-1", values={"content": "parent"})+ created = app.create_child_chunk(+ context,+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id=parent.data.id,+ content="original",+ )+ ref = DatasetRef("workspace-1", "dataset-1").document("document-1").segment(parent.data.id)+ before = repository.get_children(ref)+ probe.fail = True+ expected = ChildChunkDeleteIndexError if operation == "delete" else ChildChunkIndexingError+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ segment = session.get(DocumentSegment, parent.data.id)+ child = session.get(ChildChunk, created.id)+ assert dataset is not None+ assert document is not None+ assert segment is not None+ assert child is not None++ def mutate() -> None:+ if operation == "create":+ SegmentService.create_child_chunk(+ "new",+ segment,+ document,+ dataset,+ session,+ actor_id="editor",+ mutations=app.mutations,+ )+ elif operation == "update":+ SegmentService.update_child_chunk(+ "changed",+ child,+ segment,+ document,+ dataset,+ session,+ actor_id="editor",+ mutations=app.mutations,+ )+ else:+ SegmentService.delete_child_chunk(child, dataset, session, mutations=app.mutations)++ with pytest.raises(expected, match="index unavailable"):+ mutate()+ assert repository.get_children(ref) == before++[email protected]("values", [{"answer": "new answer"}, {"content": "new question", "answer": "new answer"}])+def test_service_api_partial_qa_update_uses_shared_word_count_and_indexing(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ values: dict[str, str],+) -> None:+ app, _, _ = indexing_probe+ with sqlite_session_factory.begin() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ document.doc_form = IndexStructureType.QA_INDEX+ created = app.create_segment(+ RequestContext("request", None, "author", "workspace-1"),+ dataset_id="dataset-1",+ document_id="document-1",+ values={"content": "question", "answer": "answer"},+ )+ with sqlite_session_factory() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ segment = session.get(DocumentSegment, created.data.id)+ assert dataset is not None+ assert document is not None+ assert segment is not None+ updated = SegmentService.update_segment(+ SegmentUpdateArgs(**values),+ segment,+ document,+ dataset,+ session,+ actor_id="editor",+ mutations=app.mutations,+ )+ assert updated.content == values.get("content", "question")+ assert updated.answer == "new answer"+ assert updated.word_count == len(updated.content) + len(updated.answer)+ assert document.word_count == updated.word_count+ if "content" in values:+ assert updated.tokens == updated.word_count++[email protected]("owner_field", ["tenant_id", "dataset_id", "document_id"])+def test_legacy_segment_mutation_rejects_mismatched_rows_before_dispatch(owner_field: str) -> None:+ from services.knowledge.segments.application import SegmentMutationService++ dataset = _dataset("dataset-1", "workspace-1")+ document = _document("document-1", "workspace-1", "dataset-1")+ segment = _segment("segment-1", "workspace-1", "dataset-1", "document-1")+ setattr(segment, owner_field, "foreign-owner")+ mutations = create_autospec(SegmentMutationService, instance=True, spec_set=True)+ session = create_autospec(Session, instance=True, spec_set=True)+ with pytest.raises(ValueError, match="Segment does not belong"):+ SegmentService.update_segment(+ SegmentUpdateArgs(content="new"),+ segment,+ document,+ dataset,+ session,+ actor_id="author",+ mutations=mutations,+ )+ mutations.update_segment.assert_not_called()+ session.commit.assert_not_called()+++def _persist_index_batch(+ session_factory: sessionmaker[Session], *, doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX+) -> None:+ with session_factory.begin() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ document.doc_form = doc_form+ for position, content in enumerate(("first", "second"), start=1):+ segment = _segment(content, "workspace-1", "dataset-1", "document-1")+ segment.position = position+ segment.content = content+ segment.index_node_id = f"node-{content}"+ segment.index_node_hash = f"hash-{content}"+ session.add(segment)++[email protected]("doc_form", [IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.QA_INDEX])+def test_batch_index_materializes_ordered_documents_before_one_vector_write(+ indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session], doc_form: IndexStructureType+) -> None:+ app, _, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory, doc_form=doc_form)++ app.mutations.index_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"), segment_ids=["second", "first"]+ )++ assert probe.batches == [("second", "first")]+ assert probe.writes == 2+ assert not probe.active_transactions+++def test_batch_index_failure_persists_errors_on_every_committed_segment(+ indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session]+) -> None:+ app, _, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory)+ probe.fail = True++ with pytest.raises(RuntimeError, match="index unavailable"):+ app.mutations.index_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"), segment_ids=["first", "second"]+ )++ with sqlite_session_factory() as session:+ segments = session.scalars(select(DocumentSegment)).all()+ assert len(segments) == 2+ assert all(segment.status == "error" and not segment.enabled for segment in segments)+ assert all(segment.error == "index unavailable" and segment.disabled_at is not None for segment in segments)++[email protected]("owner_field", ["tenant_id", "dataset_id", "document_id"])+def test_batch_index_validates_every_segment_owner_before_external_io(+ indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session], owner_field: str+) -> None:+ app, _, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ foreign = session.get(DocumentSegment, "second")+ assert foreign is not None+ setattr(foreign, owner_field, "foreign-owner")++ with pytest.raises(LookupError, match="Segment no longer exists"):+ app.mutations.index_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"), segment_ids=["first", "second"]+ )++ assert not probe.batches+ with sqlite_session_factory() as session:+ foreign = session.get(DocumentSegment, "second")+ assert foreign is not None+ assert foreign.status == "completed"+++def test_batch_index_preserves_owned_attachments(+ indexing_probe: IndexingProbe, sqlite_session_factory: sessionmaker[Session]+) -> None:+ app, _, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.is_multimodal = True+ session.add_all([_upload("owned", "workspace-1", "image.png"), _upload("foreign", "workspace-2", "other.png")])+ for file_id in ("owned", "foreign"):+ session.add(+ SegmentAttachmentBinding(+ tenant_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="first",+ attachment_id=file_id,+ )+ )++ app.mutations.index_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"), segment_ids=["first", "second"]+ )++ assert probe.writes == 3+ with sqlite_session_factory() as session:+ assert len(session.scalars(select(SegmentAttachmentBinding)).all()) == 2+++def test_batch_index_keeps_manual_keywords_aligned_with_segments(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from core.rag.datasource.keyword.jieba import jieba as jieba_module+ from tests.unit_tests.config_override import apply_config_overrides++ app, _, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.ECONOMY++ class Extractor:+ def extract_keywords(self, text: str, _keyword_number: int = 10) -> set[str]:+ assert not probe.active_transactions+ return {text}++ monkeypatch.setattr(jieba_module, "JiebaKeywordTableHandler", Extractor)+ apply_config_overrides(monkeypatch, KEYWORD_DATA_SOURCE_TYPE="database")+ app.mutations.index_segments(+ DatasetRef("workspace-1", "dataset-1").document("document-1"),+ segment_ids=["second", "first"],+ keywords_list=[["manual"], None],+ )++ with sqlite_session_factory() as session:+ assert {s.id: s.keywords for s in session.scalars(select(DocumentSegment))} == {+ "first": ["first"],+ "second": ["manual"],+ }+ assert not probe.batches+++def test_batch_parent_child_index_uses_shared_splitter_without_open_transactions(+ indexing_probe: IndexingProbe,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor+ from models.dataset import DatasetProcessRule+ from models.enums import ProcessRuleMode++ app, repository, probe = indexing_probe+ _persist_index_batch(sqlite_session_factory, doc_form=IndexStructureType.PARENT_CHILD_INDEX)+ with sqlite_session_factory.begin() as session:+ rule = DatasetProcessRule(+ dataset_id="dataset-1",+ mode=ProcessRuleMode.HIERARCHICAL,+ rules='{"parent_mode":"full-doc","subchunk_segmentation":{"max_tokens":100}}',+ created_by="author",+ )+ session.add(rule)+ document = session.get(Document, "document-1")+ dataset = session.get(Dataset, "dataset-1")+ assert document is not None+ assert dataset is not None+ dataset.embedding_model_provider = "provider"+ document.dataset_process_rule_id = rule.id++ def split(_self, document: IndexDocument, _rule, _mode, _model) -> list[ChildDocument]:+ assert not probe.active_transactions+ return [+ ChildDocument(+ page_content=document.page_content, metadata={"doc_id": document.page_content, "doc_hash": "hash"}+ )+ ]++ monkeypatch.setattr(ParentChildIndexProcessor, "split_child_nodes", split)+ document_ref = DatasetRef("workspace-1", "dataset-1").document("document-1")+ app.mutations.index_segments(document_ref, segment_ids=["first", "second"])++ assert probe.batches == [("first",), ("second",)]+ for segment_id in ("first", "second"):+ children = repository.get_children(document_ref.segment(segment_id))+ assert children is not None+ assert len(children) == 1+ assert children[0].data.content == segment_iddiff --git a/api/tests/unit_tests/services/knowledge/test_segment_application_service.py b/api/tests/unit_tests/services/knowledge/test_segment_application_service.pynew file mode 100644index 00000000000..9a7860ee7d6--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_segment_application_service.py@@ -0,0 +1,506 @@+from dataclasses import dataclass+from datetime import datetime+from typing import Literal+from unittest.mock import Mock, create_autospec++import pytest++from libs.helper import generate_text_hash+from machinery.context import RequestContext+from services.knowledge.dataset_access import (+ AccessibleDataset,+ DatasetAccess,+ DatasetAccessDeniedError,+ DatasetAccessRecord,+ DatasetAccessSnapshot,+)+from services.knowledge.entities.segments import SegmentRecord, SegmentUpdateArgs+from services.knowledge.resource_scope import DatasetRef+from services.knowledge.segments.application import (+ ChildChunkNotFoundError,+ DatasetSegmentApplicationService,+ SegmentBatchImportDispatcher,+ SegmentBatchImportDispatchError,+ SegmentBatchImportNotFoundError,+ SegmentDatasetModelUnavailableError,+ SegmentDatasetNotFoundError,+ SegmentDatasetRecord,+ SegmentDetail,+ SegmentDocumentIndexingError,+ SegmentDocumentNotFoundError,+ SegmentDocumentRecord,+ SegmentEmbeddingModelUnavailableError,+ SegmentIndex,+ SegmentIndexingState,+ SegmentInvalidFileTypeError,+ SegmentListFilter,+ SegmentModelGuard,+ SegmentModelProviderError,+ SegmentMutationScope,+ SegmentNotFoundError,+ SegmentPage,+ SegmentPermissionDeniedError,+ SegmentScope,+ SegmentScopeReader,+ SegmentStatusUpdateError,+ SegmentStore,+ SegmentUpdateState,+ SegmentUploadCatalog,+ SegmentUploadFileNotFoundError,+)+++@dataclass+class SegmentLimitsStub:+ SINGLE_CHUNK_ATTACHMENT_LIMIT: int = 10+++def _context() -> RequestContext:+ return RequestContext(+ request_id="request-1",+ trace_id=None,+ account_id="account-1",+ active_workspace_id="workspace-1",+ )+++def _dataset(+ *,+ indexing_technique: str = "economy",+) -> SegmentDatasetRecord:+ return SegmentDatasetRecord(+ id="dataset-1",+ workspace_id="workspace-1",+ indexing_technique=indexing_technique,+ embedding_model_provider="provider",+ embedding_model="model",+ )+++def _document() -> SegmentDocumentRecord:+ return SegmentDocumentRecord(+ id="document-1",+ dataset_id="dataset-1",+ workspace_id="workspace-1",+ doc_form="text_model",+ )+++def _snapshot() -> DatasetAccessSnapshot:+ return DatasetAccessSnapshot(+ DatasetAccessRecord("dataset-1", "workspace-1", "account-1", "all_team_members"), False+ )+++def _scope(+ *,+ dataset: SegmentDatasetRecord | None = None,+ document: SegmentDocumentRecord | None = None,+) -> SegmentScope:+ return SegmentScope(+ dataset=_dataset() if dataset is None else dataset,+ document=_document() if document is None else document,+ access_snapshot=_snapshot(),+ )+++type ServiceFixture = tuple[DatasetSegmentApplicationService, Mock, Mock, Mock, Mock, Mock, Mock]+++def _service(scope: SegmentScope | None = None, *, uploads: SegmentUploadCatalog | None = None) -> ServiceFixture:+ dataset_access = create_autospec(DatasetAccess, instance=True, spec_set=True)+ dataset_access.check_access.return_value = AccessibleDataset(id="dataset-1", workspace_id="workspace-1")+ store = create_autospec(SegmentStore, instance=True, spec_set=True)+ scopes = create_autospec(SegmentScopeReader, instance=True, spec_set=True)+ scopes.get_segment_scope.return_value = scope or _scope()+ model_guard = create_autospec(SegmentModelGuard, instance=True, spec_set=True)+ indexing_state = create_autospec(SegmentIndexingState, instance=True, spec_set=True)+ indexing_state.is_document_indexing.return_value = False+ indexing_state.is_segment_indexing.return_value = False+ dispatcher = create_autospec(SegmentBatchImportDispatcher, instance=True, spec_set=True)+ service = DatasetSegmentApplicationService(+ dataset_access=dataset_access,+ store=store,+ limits=SegmentLimitsStub(),+ text_hash=generate_text_hash,+ index=create_autospec(SegmentIndex, instance=True, spec_set=True),+ uploads=uploads or create_autospec(SegmentUploadCatalog, instance=True, spec_set=True),+ scopes=scopes,+ model_guard=model_guard,+ indexing_state=indexing_state,+ batch_dispatcher=dispatcher,+ job_id_factory=lambda: "job-1",+ )+ return service, dataset_access, store, scopes, model_guard, indexing_state, dispatcher+++def test_list_segments_uses_complete_owner_scope() -> None:+ service, dataset_access, store, scopes, _, _, _ = _service()+ expected = SegmentPage(items=(), total=0, total_pages=0, page=1, limit=20)+ store.list_segments.return_value = expected++ result = service.list_segments(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ query=SegmentListFilter(keyword="needle"),+ )++ assert result is expected+ assert scopes.get_segment_scope.call_args.kwargs == {+ "dataset_ref": DatasetRef("workspace-1", "dataset-1"),+ "document_id": "document-1",+ "actor_id": "account-1",+ }+ assert store.list_segments.call_args.args[0] == _document().ref+ dataset_access.check_access.assert_called_once_with(_context(), _snapshot())+++ ("scope", "expected"),+ [+ (SegmentScope(None, None), SegmentDatasetNotFoundError),+ (SegmentScope(_dataset(), None, _snapshot()), SegmentDocumentNotFoundError),+ ],+)+def test_mutations_reject_invalid_scope_before_store_operation(+ scope: SegmentScope,+ expected: type[Exception],+) -> None:+ service, _, store, scopes, _, _, _ = _service(scope)++ with pytest.raises(expected):+ service.delete_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ )++ store.delete_segments.assert_not_called()+++ ("access_error", "expected"),+ [+ (DatasetAccessDeniedError(), SegmentPermissionDeniedError),+ ],+)+def test_dataset_access_errors_are_translated(+ access_error: Exception,+ expected: type[Exception],+) -> None:+ service, dataset_access, store, scopes, _, _, _ = _service()+ dataset_access.check_access.side_effect = access_error++ with pytest.raises(expected):+ service.delete_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ )++ store.delete_segments.assert_not_called()+++ ("kind", "method", "expected", "message"),+ [+ ("bad_request", "create_segment", SegmentEmbeddingModelUnavailableError, "No Embedding Model"),+ ("token", "create_segment", SegmentEmbeddingModelUnavailableError, "token missing"),+ ("bad_request", "delete_segment", SegmentDatasetModelUnavailableError, "No Embedding Model"),+ ("token", "delete_segment", SegmentDatasetModelUnavailableError, "dataset is unavailable"),+ ],+)+def test_model_failures_are_translated_per_use_case(+ kind: Literal["bad_request", "token"],+ method: str,+ expected: type[Exception],+ message: str,+) -> None:+ service, _, store, scopes, model_guard, _, _ = _service(_scope(dataset=_dataset(indexing_technique="high_quality")))+ model_guard.check.side_effect = SegmentModelProviderError(kind=kind, description="token missing")+ store.get_segment.return_value = None++ if method == "create_segment":++ def invoke() -> object:+ return service.create_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ values={"content": "content"},+ )++ else:++ def invoke() -> object:+ return service.delete_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ )++ with pytest.raises(expected, match=message):+ invoke()+++def test_status_change_is_fenced_by_document_indexing_state() -> None:+ service, _, store, scopes, _, state, _ = _service()+ state.is_document_indexing.return_value = True++ with pytest.raises(SegmentDocumentIndexingError):+ service.change_segment_status(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_ids=["segment-1"],+ action="disable",+ )++ store.save_segment.assert_not_called()+++def test_status_change_translates_operation_failure() -> None:+ service, _, store, scopes, _, _, _ = _service()+ store.get_segments.side_effect = RuntimeError("failed")++ with pytest.raises(SegmentStatusUpdateError, match="failed"):+ service.change_segment_status(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_ids=["segment-1"],+ action="enable",+ )+++def test_update_and_delete_segment_map_missing_segment() -> None:+ service, _, store, scopes, _, _, _ = _service()+ store.get_segment_update_state.return_value = None+ store.get_segments.return_value = ()++ with pytest.raises(SegmentNotFoundError):+ service.update_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ values={"content": "content"},+ )+ with pytest.raises(SegmentNotFoundError):+ service.delete_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ )+++ ("filename", "expected"),+ [+ (None, SegmentUploadFileNotFoundError),+ ("segments.txt", SegmentInvalidFileTypeError),+ ],+)+def test_batch_import_validates_tenant_scoped_upload(filename: str | None, expected: type[Exception]) -> None:+ uploads = create_autospec(SegmentUploadCatalog, instance=True, spec_set=True)+ uploads.get_file_name.return_value = filename+ service, _, store, scopes, _, state, dispatcher = _service(uploads=uploads)++ with pytest.raises(expected):+ service.start_batch_import(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ upload_file_id="file-1",+ )++ assert uploads.get_file_name.call_args.kwargs["workspace_id"] == "workspace-1"+ state.set_batch_waiting.assert_not_called()+ dispatcher.dispatch.assert_not_called()+++def test_batch_import_records_status_before_dispatch() -> None:+ uploads = create_autospec(SegmentUploadCatalog, instance=True, spec_set=True)+ uploads.get_file_name.return_value = "segments.CSV"+ service, _, store, scopes, _, state, dispatcher = _service(uploads=uploads)+ calls: list[str] = []+ state.set_batch_waiting.side_effect = lambda _job_id: calls.append("status")+ dispatcher.dispatch.side_effect = lambda **_kwargs: calls.append("dispatch")++ result = service.start_batch_import(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ upload_file_id="file-1",+ )++ assert result.job_id == "job-1"+ assert calls == ["status", "dispatch"]+ dispatcher.dispatch.assert_called_once_with(+ job_id="job-1",+ upload_file_id="file-1",+ dataset_id="dataset-1",+ document_id="document-1",+ workspace_id="workspace-1",+ actor_id="account-1",+ )+++def test_batch_import_translates_dispatch_failure() -> None:+ uploads = create_autospec(SegmentUploadCatalog, instance=True, spec_set=True)+ uploads.get_file_name.return_value = "segments.csv"+ service, _, store, scopes, _, _, dispatcher = _service(uploads=uploads)+ dispatcher.dispatch.side_effect = RuntimeError("queue down")++ with pytest.raises(SegmentBatchImportDispatchError, match="queue down"):+ service.start_batch_import(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ upload_file_id="file-1",+ )+++def test_batch_import_status_distinguishes_unknown_job() -> None:+ service, _, _, scopes, _, state, _ = _service()+ state.get_batch_status.return_value = None+ with pytest.raises(SegmentBatchImportNotFoundError):+ service.get_batch_import_status("missing")++ state.get_batch_status.return_value = "completed"+ assert service.get_batch_import_status("job-1").job_status == "completed"+++def test_child_chunk_update_distinguishes_missing_parent_and_child() -> None:+ service, _, store, scopes, _, _, _ = _service()+ store.get_children.return_value = None+ with pytest.raises(SegmentNotFoundError):+ service.update_child_chunk(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ child_chunk_id="child-1",+ content="content",+ )++ store.get_children.return_value = ()+ with pytest.raises(ChildChunkNotFoundError):+ service.update_child_chunk(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ segment_id="segment-1",+ child_chunk_id="child-1",+ content="content",+ )+++def _segment_detail(*, enabled: bool = True) -> SegmentDetail:+ return SegmentDetail(+ data=SegmentRecord(+ id="segment-1",+ position=1,+ document_id="document-1",+ content="content",+ sign_content="content",+ answer=None,+ word_count=7,+ tokens=0,+ keywords=None,+ index_node_id=None,+ index_node_hash=None,+ hit_count=0,+ enabled=enabled,+ disabled_at=None,+ disabled_by=None,+ status="completed",+ created_by="account-1",+ created_at=datetime(2026, 1, 1),+ updated_at=datetime(2026, 1, 1),+ updated_by=None,+ indexing_at=None,+ completed_at=None,+ error=None,+ stopped_at=None,+ child_chunks=(),+ attachments=(),+ summary=None,+ ),+ doc_form="text_model",+ )+++def test_create_segment_returns_store_detail() -> None:+ service, _, store, scopes, _, _, _ = _service()+ detail = _segment_detail()+ store.get_segment.return_value = detail++ result = service.create_segment(+ _context(),+ dataset_id="dataset-1",+ document_id="document-1",+ values={"content": "content"},+ )++ assert result is detail+ assert store.save_segment.call_args.args[1]["created_by"] == _context().account_id+++ ("enabled", "indexing", "message"),+ [(False, False, "Can't update disabled segment"), (True, True, "Segment is indexing")],+)+def test_shared_update_rejects_disabled_or_indexing_segments(enabled: bool, indexing: bool, message: str) -> None:+ service, _, store, _, _, state, _ = _service()+ store.get_segment_update_state.return_value = SegmentUpdateState("content", enabled, None)+ state.is_segment_indexing.return_value = indexing+ with pytest.raises(ValueError, match=message):+ service.mutations.update_segment(+ SegmentMutationScope(_dataset(), _document()),+ actor_id="editor",+ segment_id="segment-1",+ args=SegmentUpdateArgs(content="changed"),+ )+ store.save_segment.assert_not_called()+++def test_shared_disable_patch_does_not_replace_content() -> None:+ service, _, store, _, _, state, _ = _service()+ store.get_segment_update_state.return_value = SegmentUpdateState("content", True, None)+ service.mutations.update_segment(+ SegmentMutationScope(_dataset(), _document()),+ actor_id="editor",+ segment_id="segment-1",+ args=SegmentUpdateArgs(enabled=False),+ )+ changes = store.save_segment.call_args.args[1]+ assert changes["enabled"] is False+ assert changes["disabled_by"] == "editor"+ assert "content" not in changes+ state.mark_segment_indexing.assert_called_once_with("segment-1")+ store.get_segment.assert_not_called()+++def test_shared_writes_do_not_build_segment_detail() -> None:+ service, _, store, _, _, _, _ = _service()+ store.get_segment.side_effect = RuntimeError("attachment URL signing unavailable")+ store.get_segment_update_state.return_value = SegmentUpdateState("content", True, None)+ scope = SegmentMutationScope(_dataset(), _document())++ created = service.mutations.create_segment(scope, actor_id="author", args=SegmentUpdateArgs(content="content"))+ service.mutations.update_segment(+ scope,+ actor_id="editor",+ segment_id=created.segment_id,+ args=SegmentUpdateArgs(content="updated"),+ )++ assert store.save_segment.call_count == 2+ assert store.save_segment.call_args.args[1]["content"] == "updated"+ store.get_segment.assert_not_called()diff --git a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.pyindex fabc093c9b0..939b54b1559 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py@@ -4,16 +4,22 @@ import pytestfrom pytest_mock import MockerFixturefrom sqlalchemy.orm import Session+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom models.dataset import Dataset, Document, Pipelinefrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import Account, App, AppMode, EndUserfrom models.workflow import Workflow, WorkflowType-from services.dataset_ref_service import DatasetRefService+from services.knowledge.resource_scope import DatasetReffrom services.rag_pipeline.pipeline_generate_service import PipelineGenerateService+def pipeline_generator(mocker: MockerFixture):+ return mocker.create_autospec(PipelineGenerator, instance=True, spec_set=True)++def _make_pipeline(*, tenant_id: str = "tenant-1") -> Pipeline:pipeline = Pipeline(tenant_id=tenant_id, name="Pipeline", description="")pipeline.id = "pipeline-1"@@ -128,7 +134,7 @@ def test_get_workflow(mocker: MockerFixture, invoke_from, workflow, expected_errdef test_generate_updates_document_status_and_returns_event_stream(- mocker: MockerFixture, sqlite_session: Session+ mocker: MockerFixture, sqlite_session: Session, pipeline_generator) -> None:dataset = _make_dataset()pipeline = _make_pipeline()@@ -141,7 +147,7 @@ def test_generate_updates_document_status_and_returns_event_stream(update_status_mock = mocker.patch.object(PipelineGenerateService, "update_document_status")generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator")- generator_instance = generator_cls.return_value+ generator_instance = pipeline_generatorgenerator_instance.generate.return_value = "raw-events"generator_cls.convert_to_event_stream.return_value = "stream-events"@@ -152,6 +158,7 @@ def test_generate_updates_document_status_and_returns_event_stream(invoke_from=InvokeFrom.WEB_APP,streaming=True,session=sqlite_session,+ generator=pipeline_generator,)assert result == "stream-events"@@ -163,7 +170,9 @@ def test_generate_updates_document_status_and_returns_event_stream(assert generator_instance.generate.call_args.kwargs["session"] is sqlite_session-def test_generate_rejects_pipeline_dataset_from_another_tenant(mocker: MockerFixture, sqlite_session: Session) -> None:+def test_generate_rejects_pipeline_dataset_from_another_tenant(+ mocker: MockerFixture, sqlite_session: Session, pipeline_generator+) -> None:dataset = _make_dataset(tenant_id="tenant-2")pipeline = _make_pipeline()sqlite_session.add_all([dataset, pipeline])@@ -178,6 +187,7 @@ def test_generate_rejects_pipeline_dataset_from_another_tenant(mocker: MockerFixargs={"original_document_id": "doc-1"},invoke_from=InvokeFrom.WEB_APP,session=sqlite_session,+ generator=pipeline_generator,)update_status_mock.assert_not_called()@@ -186,6 +196,7 @@ def test_generate_rejects_pipeline_dataset_from_another_tenant(mocker: MockerFixdef test_generate_rejects_original_document_outside_pipeline_dataset_before_dispatch(mocker: MockerFixture,sqlite_session: Session,+ pipeline_generator,) -> None:dataset = _make_dataset()pipeline = _make_pipeline()@@ -202,11 +213,12 @@ def test_generate_rejects_original_document_outside_pipeline_dataset_before_dispargs={"original_document_id": "foreign-doc"},invoke_from=InvokeFrom.PUBLISHED_PIPELINE,session=sqlite_session,+ generator=pipeline_generator,)sqlite_session.refresh(outside_document)assert outside_document.indexing_status == IndexingStatus.COMPLETED- generator_cls.assert_not_called()+ pipeline_generator.generate.assert_not_called()def test_update_document_status_updates_existing_document(sqlite_session: Session) -> None:@@ -214,8 +226,8 @@ def test_update_document_status_updates_existing_document(sqlite_session: Sessiodataset = _make_dataset()sqlite_session.add_all([dataset, document])sqlite_session.commit()- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, "doc-1")+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ document_ref = dataset_ref.document("doc-1")PipelineGenerateService.update_document_status(document_ref, session=sqlite_session)@@ -235,8 +247,8 @@ def test_update_document_status_rejects_document_outside_owner(sqlite_session: Session,) -> None:dataset = _make_dataset()- dataset_ref = DatasetRefService.create_dataset_ref(dataset)- document_ref = DatasetRefService.create_document_ref_from_id(dataset_ref, "doc-1")+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)+ document_ref = dataset_ref.document("doc-1")outside_document = _make_document(tenant_id=document_tenant_id, dataset_id=document_dataset_id)sqlite_session.add(outside_document)sqlite_session.commit()@@ -251,11 +263,13 @@ def test_update_document_status_rejects_document_outside_owner(# --- generate_single_iteration ----def test_generate_single_iteration_delegates(mocker: MockerFixture, sqlite_session: Session) -> None:+def test_generate_single_iteration_delegates(+ mocker: MockerFixture, sqlite_session: Session, pipeline_generator+) -> None:mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow())generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator")- generator_instance = generator_cls.return_value+ generator_instance = pipeline_generatorgenerator_instance.single_iteration_generate.return_value = "raw-iter"generator_cls.convert_to_event_stream.return_value = "stream-iter"@@ -264,7 +278,9 @@ def test_generate_single_iteration_delegates(mocker: MockerFixture, sqlite_sessiuser = _make_account(account_id="u1")session = sqlite_session- result = PipelineGenerateService.generate_single_iteration(pipeline, user, "node-1", {"key": "val"}, session)+ result = PipelineGenerateService.generate_single_iteration(+ pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator+ )assert result == "stream-iter"generator_instance.single_iteration_generate.assert_called_once()@@ -274,11 +290,11 @@ def test_generate_single_iteration_delegates(mocker: MockerFixture, sqlite_sessi# --- generate_single_loop ----def test_generate_single_loop_delegates(mocker: MockerFixture, sqlite_session: Session) -> None:+def test_generate_single_loop_delegates(mocker: MockerFixture, sqlite_session: Session, pipeline_generator) -> None:mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow())generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator")- generator_instance = generator_cls.return_value+ generator_instance = pipeline_generatorgenerator_instance.single_loop_generate.return_value = "raw-loop"generator_cls.convert_to_event_stream.return_value = "stream-loop"@@ -287,7 +303,9 @@ def test_generate_single_loop_delegates(mocker: MockerFixture, sqlite_session: Suser = _make_account(account_id="u1")session = sqlite_session- result = PipelineGenerateService.generate_single_loop(pipeline, user, "node-1", {"key": "val"}, session)+ result = PipelineGenerateService.generate_single_loop(+ pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator+ )assert result == "stream-loop"generator_instance.single_loop_generate.assert_called_once()diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.pyindex dbe3955033f..35e7c01dad6 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py@@ -40,7 +40,7 @@ def test_list_rag_pipeline_datasources_marks_authorized(mocker: MockerFixture) -None,]- result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")+ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1", datasource_providers=provider_instance)assert result == [datasource_1, datasource_2]assert datasource_1.is_authorized is True@@ -63,7 +63,7 @@ def test_list_rag_pipeline_datasources_marks_credential_free_providers_authorizeprovider_instance = provider_cls.return_valueprovider_instance.get_datasource_credentials.return_value = None- result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")+ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1", datasource_providers=provider_instance)assert result == [local_file, notion]assert local_file.is_authorized is True@@ -85,7 +85,7 @@ def test_oauth_only_provider_still_requires_authorization(provider_instance = provider_cls.return_valueprovider_instance.get_datasource_credentials.return_value = None- result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")+ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1", datasource_providers=provider_instance)assert result == [oauth_only]assert oauth_only.is_authorized is Falsediff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.pyindex 3687c0a298b..abfcf2e3437 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py@@ -5,12 +5,14 @@ from dataclasses import dataclassfrom datetime import datetimefrom types import SimpleNamespacefrom unittest import mock+from unittest.mock import create_autospecimport pytestfrom pytest_mock import MockerFixturefrom sqlalchemy.dialects import postgresqlfrom sqlalchemy.orm import Session, sessionmaker+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom graphon.enums import (@@ -32,12 +34,21 @@ from models.dataset import ()from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.workflow import Workflow, WorkflowRun+from repositories.knowledge.dataset_read_repository import get_pipeline_dataset+from services.credentials.query import CredentialQuery+from services.data_source.credential_gateway import DatasourceProviderCredentialStorefrom services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntityfrom services.errors.rag_pipeline import RagPipelineResourceNotFoundError+from services.rag_pipeline import rag_pipeline as rag_pipeline_modulefrom services.rag_pipeline.rag_pipeline import RagPipelineServicefrom services.workflow_ref_service import WorkflowRef+def pipeline_generator(mocker: MockerFixture):+ return mocker.create_autospec(PipelineGenerator, instance=True, spec_set=True)++@dataclassclass RagPipelineServiceTestContext:service: RagPipelineService@@ -611,7 +622,7 @@ def test_publish_workflow_success(mocker: MockerFixture, rag_pipeline_service: Rdataset = _make_dataset()_persist(rag_pipeline_service.session, draft_workflow, pipeline, dataset)mocker.patch("services.rag_pipeline.rag_pipeline.KnowledgeConfiguration.model_validate", return_value=mocker.Mock())- mock_dataset_service_class = mocker.patch("services.dataset_service.DatasetService")+ mock_dataset_service_class = mocker.patch("services.knowledge.dataset_service.DatasetService")result = rag_pipeline_service.service.publish_workflow(session=rag_pipeline_service.session, pipeline=pipeline, account=_make_account()@@ -716,6 +727,9 @@ def test_run_datasource_workflow_node_website_crawl(account=_make_account(),datasource_type="website_crawl",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)events = list(gen)@@ -797,6 +811,9 @@ def test_run_datasource_node_preview_online_document(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)# 4. Assertions@@ -916,7 +933,7 @@ def test_publish_customized_pipeline_template_success(# Mock RagPipelineDslServicemock_dsl_service = mocker.Mock()mock_dsl_service.export_rag_pipeline_dsl.return_value = "dsl: content"- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.RagPipelineDslService", return_value=mock_dsl_service)+ mocker.patch.object(rag_pipeline_module, "RagPipelineDslService", return_value=mock_dsl_service)account = _make_account(account_id="user-123")@@ -968,7 +985,15 @@ def test_get_datasource_plugins_success(mocker.patch("services.rag_pipeline.rag_pipeline.DatasourceProviderService", return_value=mock_provider_service)# 2. Run test- result = rag_pipeline_service.service.get_datasource_plugins("t1", "d1", True)+ result = rag_pipeline_service.service.get_datasource_plugins(+ "t1",+ "d1",+ True,+ credential_query=mock.create_autospec(CredentialQuery, instance=True),+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )# 3. Assertionsassert len(result) == 1@@ -980,7 +1005,7 @@ def test_get_datasource_plugins_success(def test_retry_error_document_success(- mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext+ mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, pipeline_generator) -> None:dataset = _make_dataset()document = _make_document()@@ -1003,12 +1028,11 @@ def test_retry_error_document_success(_persist(rag_pipeline_service.session, log, pipeline, workflow)# Mock PipelineGenerator- mock_gen_instance = mocker.Mock()- mocker.patch("services.rag_pipeline.rag_pipeline.PipelineGenerator", return_value=mock_gen_instance)+ mock_gen_instance = pipeline_generator# 2. Run testuser = mocker.Mock()- rag_pipeline_service.service.retry_error_document(dataset, document, user)+ rag_pipeline_service.service.retry_error_document(dataset, document, user, generator=pipeline_generator)# 3. Assertionsmock_gen_instance.generate.assert_called_once()@@ -1266,6 +1290,9 @@ def test_run_datasource_workflow_node_returns_error_when_workflow_missing(account=_make_account(),datasource_type="online_document",is_published=False,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -1314,6 +1341,9 @@ def test_run_datasource_workflow_node_online_document_success(account=_make_account(),datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -1366,6 +1396,9 @@ def test_run_datasource_workflow_node_online_drive_success(account=_make_account(),datasource_type=DatasourceProviderType.ONLINE_DRIVE,is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -1599,6 +1632,9 @@ def test_run_datasource_workflow_node_returns_error_when_node_missing(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -1641,7 +1677,8 @@ def test_run_datasource_workflow_node_online_document_exception(mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)events = list(@@ -1652,6 +1689,9 @@ def test_run_datasource_workflow_node_online_document_exception(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -1696,7 +1736,8 @@ def test_run_datasource_node_preview_raises_for_stream_non_string(mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)with pytest.raises(RuntimeError, match="must be a string"):@@ -1707,6 +1748,9 @@ def test_run_datasource_node_preview_raises_for_stream_non_string(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)@@ -1758,9 +1802,12 @@ def test_get_second_step_parameters_filters_first_step_variables(def test_retry_error_document_raises_when_execution_log_not_found(rag_pipeline_service: RagPipelineServiceTestContext,+ pipeline_generator,) -> None:with pytest.raises(ValueError, match="Document pipeline execution log not found"):- rag_pipeline_service.service.retry_error_document(_make_dataset(), _make_document(), _make_account())+ rag_pipeline_service.service.retry_error_document(+ _make_dataset(), _make_document(), _make_account(), generator=pipeline_generator+ )def test_get_datasource_plugins_raises_when_workflow_not_found(@@ -1771,7 +1818,15 @@ def test_get_datasource_plugins_raises_when_workflow_not_found(_persist(rag_pipeline_service.session, dataset, pipeline)with pytest.raises(ValueError, match="Pipeline or workflow not found"):- rag_pipeline_service.service.get_datasource_plugins("t1", "d1", True)+ rag_pipeline_service.service.get_datasource_plugins(+ "t1",+ "d1",+ True,+ credential_query=mock.create_autospec(CredentialQuery, instance=True),+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )def test_handle_node_run_result_raises_when_no_terminal_event(@@ -1864,7 +1919,8 @@ def test_run_datasource_node_preview_raises_for_unsupported_provider(runtime.datasource_provider_type.return_value = "unsupported"mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)with pytest.raises(RuntimeError, match="Unsupported datasource provider"):@@ -1875,6 +1931,9 @@ def test_run_datasource_node_preview_raises_for_unsupported_provider(account=_make_account(),datasource_type="website_crawl",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)@@ -1984,7 +2043,7 @@ def test_publish_workflow_skips_dataset_update_for_non_knowledge_nodes() -> None:draft = _make_workflow(graph={"nodes": [{"data": {"type": "start"}}]})_persist(rag_pipeline_service.session, draft)- dataset_service = mocker.patch("services.dataset_service.DatasetService")+ dataset_service = mocker.patch("services.knowledge.dataset_service.DatasetService")result = rag_pipeline_service.service.publish_workflow(session=rag_pipeline_service.session,@@ -2045,7 +2104,8 @@ def test_run_datasource_workflow_node_handles_variable_parameter_types(runtime.datasource_provider_type.return_value = DatasourceProviderType.WEBSITE_CRAWLmocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)events = list(@@ -2056,6 +2116,9 @@ def test_run_datasource_workflow_node_handles_variable_parameter_types(account=_make_account(),datasource_type="website_crawl",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -2093,7 +2156,8 @@ def test_run_datasource_workflow_node_online_drive_branch(runtime.datasource_provider_type.return_value = DatasourceProviderType.ONLINE_DRIVEmocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)events = list(@@ -2104,6 +2168,9 @@ def test_run_datasource_workflow_node_online_drive_branch(account=_make_account(),datasource_type="online_drive",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),))@@ -2143,7 +2210,8 @@ def test_run_datasource_node_preview_not_published_uses_draft(runtime.get_online_document_page_content.side_effect = doc_genmocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)result = rag_pipeline_service.service.run_datasource_node_preview(@@ -2153,6 +2221,9 @@ def test_run_datasource_node_preview_not_published_uses_draft(account=_make_account(),datasource_type="online_document",is_published=False,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)assert result == {"x": "v"}@@ -2187,7 +2258,7 @@ def test_publish_customized_pipeline_template_rejects_unowned_workflow_before_expipeline = _make_pipeline(workflow_id="wf-1")workflow = _make_workflow(workflow_id="wf-1", tenant_id=workflow_tenant_id, app_id=workflow_app_id)_persist(rag_pipeline_service.session, pipeline, workflow)- dsl_service = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.RagPipelineDslService")+ dsl_service = mocker.patch.object(rag_pipeline_module, "RagPipelineDslService")with pytest.raises(RagPipelineResourceNotFoundError, match="Workflow not found"):rag_pipeline_service.service.publish_customized_pipeline_template(@@ -2209,7 +2280,7 @@ def test_pipeline_retrieve_dataset_rejects_unowned_dataset(other_tenant_dataset = _make_dataset(tenant_id="t2")_persist(rag_pipeline_service.session, pipeline, other_tenant_dataset)- assert pipeline.retrieve_dataset(session=rag_pipeline_service.session) is None+ assert get_pipeline_dataset(pipeline, session=rag_pipeline_service.session) is None@pytest.mark.parametrize(@@ -2230,7 +2301,7 @@ def test_publish_customized_pipeline_template_rejects_missing_or_unowned_draft_bif draft_tenant_id and draft_app_id:resources.append(_make_workflow(workflow_id="wf-draft", tenant_id=draft_tenant_id, app_id=draft_app_id))_persist(session, *resources)- dsl_service = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.RagPipelineDslService")+ dsl_service = mocker.patch.object(rag_pipeline_module, "RagPipelineDslService")with pytest.raises(RagPipelineResourceNotFoundError, match="Draft workflow not found"):rag_pipeline_service.service.publish_customized_pipeline_template(@@ -2257,6 +2328,7 @@ def test_get_recommended_plugins_skips_manifest_when_missing(def test_retry_error_document_raises_when_pipeline_missing(rag_pipeline_service: RagPipelineServiceTestContext,+ pipeline_generator,) -> None:exec_log = DocumentPipelineExecutionLog(pipeline_id="p1",@@ -2270,11 +2342,14 @@ def test_retry_error_document_raises_when_pipeline_missing(_persist(rag_pipeline_service.session, exec_log)with pytest.raises(ValueError, match="Pipeline not found"):- rag_pipeline_service.service.retry_error_document(_make_dataset(), _make_document(), _make_account())+ rag_pipeline_service.service.retry_error_document(+ _make_dataset(), _make_document(), _make_account(), generator=pipeline_generator+ )def test_retry_error_document_raises_when_workflow_missing(rag_pipeline_service: RagPipelineServiceTestContext,+ pipeline_generator,) -> None:exec_log = DocumentPipelineExecutionLog(pipeline_id="p1",@@ -2289,7 +2364,9 @@ def test_retry_error_document_raises_when_workflow_missing(_persist(rag_pipeline_service.session, exec_log, pipeline)with pytest.raises(ValueError, match="Workflow not found"):- rag_pipeline_service.service.retry_error_document(_make_dataset(), _make_document(), _make_account())+ rag_pipeline_service.service.retry_error_document(+ _make_dataset(), _make_document(), _make_account(), generator=pipeline_generator+ )def test_get_datasource_plugins_returns_empty_for_non_datasource_nodes(@@ -2300,7 +2377,18 @@ def test_get_datasource_plugins_returns_empty_for_non_datasource_nodes(workflow = _make_workflow(graph={"nodes": [{"id": "n1", "data": {"type": "start"}}]})_persist(rag_pipeline_service.session, dataset, pipeline, workflow)- assert rag_pipeline_service.service.get_datasource_plugins("t1", "d1", True) == []+ assert (+ rag_pipeline_service.service.get_datasource_plugins(+ "t1",+ "d1",+ True,+ credential_query=mock.create_autospec(CredentialQuery, instance=True),+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )+ == []+ )def test_publish_workflow_raises_when_knowledge_index_dataset_missing(@@ -2330,6 +2418,9 @@ def test_run_datasource_node_preview_raises_when_workflow_missing(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)@@ -2348,6 +2439,9 @@ def test_run_datasource_node_preview_raises_when_node_missing(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)@@ -2385,7 +2479,8 @@ def test_run_datasource_node_preview_keeps_existing_user_input(runtime.get_online_document_page_content.side_effect = genmocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)result = rag_pipeline_service.service.run_datasource_node_preview(@@ -2395,6 +2490,9 @@ def test_run_datasource_node_preview_keeps_existing_user_input(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)assert result == {"ok": "1"}@@ -2426,7 +2524,8 @@ def test_run_datasource_node_preview_ignores_non_variable_messages(runtime.get_online_document_page_content.side_effect = genmocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", return_value=None+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials",+ return_value=None,)result = rag_pipeline_service.service.run_datasource_node_preview(@@ -2436,6 +2535,9 @@ def test_run_datasource_node_preview_ignores_non_variable_messages(account=_make_account(),datasource_type="online_document",is_published=True,+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),)assert result == {}@@ -2466,10 +2568,19 @@ def test_get_datasource_plugins_handles_empty_datasource_data_and_non_published(]_persist(rag_pipeline_service.session, dataset, pipeline, workflow)mocker.patch(- "services.rag_pipeline.rag_pipeline.DatasourceProviderService.list_datasource_credentials", return_value=[]+ "services.rag_pipeline.rag_pipeline.DatasourceProviderService.list_datasource_credentials",+ return_value=[],)- result = rag_pipeline_service.service.get_datasource_plugins("t1", "d1", False)+ result = rag_pipeline_service.service.get_datasource_plugins(+ "t1",+ "d1",+ False,+ credential_query=mock.create_autospec(CredentialQuery, instance=True),+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )assert len(result) == 1@@ -2510,7 +2621,15 @@ def test_get_datasource_plugins_extracts_user_inputs_and_credentials(return_value=[{"id": "c1", "name": "Cred", "type": "api", "is_default": True}],)- result = rag_pipeline_service.service.get_datasource_plugins("t1", "d1", True)+ result = rag_pipeline_service.service.get_datasource_plugins(+ "t1",+ "d1",+ True,+ credential_query=mock.create_autospec(CredentialQuery, instance=True),+ datasource_providers=rag_pipeline_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ ),+ )assert len(result) == 1assert len(result[0]["user_input_variables"]) == 2diff --git a/api/tests/unit_tests/services/test_credential_permission_service.py b/api/tests/unit_tests/services/test_credential_permission_service.pydeleted file mode 100644index 6964f702867..00000000000--- a/api/tests/unit_tests/services/test_credential_permission_service.py+++ /dev/null@@ -1,201 +0,0 @@-"""Unit tests for CredentialPermissionService.--Tests the visibility filtering logic, partial-member read path,-and admin bypass behavior.-"""--from uuid import uuid4--import pytest-from sqlalchemy import select-from sqlalchemy.orm import Session--from core.plugin.entities.plugin_daemon import CredentialType as TriggerCredentialType-from models.account import Account, TenantAccountRole-from models.credential_permission import CredentialPermission, CredentialType-from models.enums import PermissionEnum-from models.trigger import TriggerSubscription-from services.credential_permission_service import CredentialPermissionService---def tenant_id() -> str:- return str(uuid4())---def user_id() -> str:- return str(uuid4())---def other_user_id() -> str:- return str(uuid4())---def credential_id() -> str:- return str(uuid4())---def _subscription(- *,- tenant_id: str,- owner_id: str,- name: str,- visibility: PermissionEnum,-) -> TriggerSubscription:- return TriggerSubscription(- name=name,- tenant_id=tenant_id,- user_id=owner_id,- provider_id="test/provider",- endpoint_id=f"{name}-endpoint",- parameters={},- properties={},- credentials={},- credential_type=TriggerCredentialType.API_KEY,- visibility=visibility,- )---def _user(user_id: str, *, is_admin: bool) -> Account:- user = Account(name="Credential User", email=f"{user_id}@example.com")- user.id = user_id- user.role = TenantAccountRole.ADMIN if is_admin else TenantAccountRole.NORMAL- return user---class TestGetPartialMemberList:- def test_returns_empty_when_no_permissions(- self, sqlite_session: Session, credential_id: str, tenant_id: str, user_id: str- ) -> None:- unrelated_permission = CredentialPermission(- credential_id=str(uuid4()),- credential_type=CredentialType.TRIGGER_SUBSCRIPTION,- account_id=user_id,- tenant_id=tenant_id,- )- sqlite_session.add(unrelated_permission)- sqlite_session.commit()-- result = CredentialPermissionService.get_partial_member_list(- credential_id, CredentialType.TRIGGER_SUBSCRIPTION, session=sqlite_session- )-- assert result == []-- def test_returns_account_ids(- self,- sqlite_session: Session,- credential_id: str,- user_id: str,- other_user_id: str,- tenant_id: str,- ) -> None:- sqlite_session.add_all(- [- CredentialPermission(- credential_id=credential_id,- credential_type=CredentialType.TRIGGER_SUBSCRIPTION,- account_id=account_id,- tenant_id=tenant_id,- )- for account_id in (user_id, other_user_id)- ]- )- sqlite_session.commit()-- result = CredentialPermissionService.get_partial_member_list(- credential_id, CredentialType.TRIGGER_SUBSCRIPTION, session=sqlite_session- )-- assert set(result) == {user_id, other_user_id}---class TestApplyVisibilityFilter:- def test_admin_does_not_bypass_personal_visibility(- self,- sqlite_session: Session,- tenant_id: str,- user_id: str,- other_user_id: str,- ) -> None:- private_subscription = _subscription(- tenant_id=tenant_id,- owner_id=other_user_id,- name="private",- visibility=PermissionEnum.ONLY_ME,- )- sqlite_session.add(private_subscription)- sqlite_session.commit()-- query = CredentialPermissionService.apply_visibility_filter(- select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id),- model_id_column=TriggerSubscription.id,- model_user_id_column=TriggerSubscription.user_id,- model_visibility_column=TriggerSubscription.visibility,- credential_type=CredentialType.TRIGGER_SUBSCRIPTION,- user=_user(user_id, is_admin=True),- )-- assert sqlite_session.scalars(query).all() == []-- def test_non_admin_sees_team_owned_and_partial_member_subscriptions(- self,- sqlite_session: Session,- tenant_id: str,- user_id: str,- other_user_id: str,- ) -> None:- team_subscription = _subscription(- tenant_id=tenant_id,- owner_id=other_user_id,- name="team",- visibility=PermissionEnum.ALL_TEAM,- )- owned_subscription = _subscription(- tenant_id=tenant_id,- owner_id=user_id,- name="owned",- visibility=PermissionEnum.ONLY_ME,- )- shared_subscription = _subscription(- tenant_id=tenant_id,- owner_id=other_user_id,- name="shared",- visibility=PermissionEnum.PARTIAL_TEAM,- )- private_subscription = _subscription(- tenant_id=tenant_id,- owner_id=other_user_id,- name="private",- visibility=PermissionEnum.ONLY_ME,- )- sqlite_session.add_all(- [- team_subscription,- owned_subscription,- shared_subscription,- private_subscription,- CredentialPermission(- credential_id=shared_subscription.id,- credential_type=CredentialType.TRIGGER_SUBSCRIPTION,- account_id=user_id,- tenant_id=tenant_id,- ),- ]- )- sqlite_session.commit()-- query = CredentialPermissionService.apply_visibility_filter(- select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id),- model_id_column=TriggerSubscription.id,- model_user_id_column=TriggerSubscription.user_id,- model_visibility_column=TriggerSubscription.visibility,- credential_type=CredentialType.TRIGGER_SUBSCRIPTION,- user=_user(user_id, is_admin=False),- )-- visible_ids = {subscription.id for subscription in sqlite_session.scalars(query)}- assert visible_ids == {team_subscription.id, owned_subscription.id, shared_subscription.id}diff --git a/api/tests/unit_tests/services/test_data_source_oauth_service.py b/api/tests/unit_tests/services/test_data_source_oauth_service.pydeleted file mode 100644index f2f0189da8a..00000000000--- a/api/tests/unit_tests/services/test_data_source_oauth_service.py+++ /dev/null@@ -1,182 +0,0 @@-from unittest.mock import MagicMock, call--import pytest--from machinery.context import RequestContext-from services.data_source_oauth_service import (- DataSourceOAuthConfigurationError,- DataSourceOAuthError,- DataSourceOAuthService,- InvalidDataSourceOAuthCodeError,-)-from services.entities.data_source_oauth_entities import (- DataSourceOAuthAuthorization,- DataSourceOAuthBindingRecord,- DataSourceOAuthCallback,-)---def request_context() -> RequestContext:- return RequestContext(- request_id="request-1",- trace_id="trace-1",- account_id="account-1",- active_workspace_id="workspace-1",- )---def test_start_authorization_delegates_to_external_provider(request_context: RequestContext) -> None:- provider = MagicMock()- provider.get_authorization_url.return_value = "https://notion.example/authorize"- bindings = MagicMock()- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=provider,- bindings=bindings,- )-- result = service.start_authorization(request_context)-- assert result == "https://notion.example/authorize"- provider.get_authorization_url.assert_called_once_with()- bindings.upsert_authorization.assert_not_called()---def test_start_authorization_persists_internal_authorization(request_context: RequestContext) -> None:- authorization = DataSourceOAuthAuthorization(access_token="secret", source_info={"pages": []})- provider = MagicMock()- provider.authorize_internal.return_value = authorization- bindings = MagicMock()- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=provider,- bindings=bindings,- is_internal_provider=True,- internal_access_token="secret",- )-- result = service.start_authorization(request_context)-- assert result == "internal"- provider.authorize_internal.assert_called_once_with("secret", "workspace-1")- bindings.upsert_authorization.assert_called_once_with(- workspace_id="workspace-1",- provider="notion",- authorization=authorization,- )---def test_start_authorization_rejects_missing_internal_secret(request_context: RequestContext) -> None:- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=MagicMock(),- bindings=MagicMock(),- is_internal_provider=True,- )-- with pytest.raises(DataSourceOAuthConfigurationError, match="Internal secret is not set"):- service.start_authorization(request_context)--- ("code", "error", "expected_error"),- [- ("auth/code", None, None),- (None, "access denied", "access denied"),- (None, None, "Access denied"),- ],-)-def test_complete_callback_returns_framework_neutral_result(- code: str | None,- error: str | None,- expected_error: str | None,-) -> None:- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=MagicMock(),- bindings=MagicMock(),- )-- assert service.complete_callback(code=code, error=error) == DataSourceOAuthCallback(- provider="notion",- code=code,- error=expected_error,- )---def test_bind_authorizes_before_persisting(request_context: RequestContext) -> None:- events = MagicMock()- authorization = DataSourceOAuthAuthorization(access_token="token", source_info={"pages": []})- provider = MagicMock()- provider.authorize.side_effect = lambda code: (events.authorize(code), authorization)[1]- bindings = MagicMock()- bindings.upsert_authorization.side_effect = lambda **kwargs: events.persist(**kwargs)- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=provider,- bindings=bindings,- )-- service.bind(request_context, code="code-1")-- assert events.mock_calls == [- call.authorize("code-1"),- call.persist(workspace_id="workspace-1", provider="notion", authorization=authorization),- ]---def test_bind_rejects_empty_code(request_context: RequestContext) -> None:- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=MagicMock(),- bindings=MagicMock(),- )-- with pytest.raises(InvalidDataSourceOAuthCodeError):- service.bind(request_context, code="")---def test_sync_reads_before_provider_io_and_persists_afterward(request_context: RequestContext) -> None:- events = MagicMock()- binding = DataSourceOAuthBindingRecord(id="binding-1", access_token="token", source_info={"pages": []})- refreshed_source_info = {"pages": [{"page_id": "page-1"}]}- bindings = MagicMock()- bindings.get_enabled.side_effect = lambda **kwargs: (events.read(**kwargs), binding)[1]- bindings.update_source_info.side_effect = lambda **kwargs: (events.update(**kwargs), True)[1]- provider = MagicMock()- provider.refresh.side_effect = lambda *args: (events.refresh(*args), refreshed_source_info)[1]- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=provider,- bindings=bindings,- )-- service.sync(request_context, binding_id="binding-1")-- assert events.mock_calls == [- call.read(workspace_id="workspace-1", provider="notion", binding_id="binding-1"),- call.refresh("token", {"pages": []}),- call.update(- workspace_id="workspace-1",- provider="notion",- binding_id="binding-1",- source_info=refreshed_source_info,- ),- ]---def test_sync_rejects_missing_binding_without_provider_io(request_context: RequestContext) -> None:- provider = MagicMock()- bindings = MagicMock()- bindings.get_enabled.return_value = None- service = DataSourceOAuthService(- provider_name="notion",- provider_gateway=provider,- bindings=bindings,- )-- with pytest.raises(DataSourceOAuthError, match="Data source binding not found"):- service.sync(request_context, binding_id="missing")-- provider.refresh.assert_not_called()diff --git a/api/tests/unit_tests/services/test_dataset_service_dataset.py b/api/tests/unit_tests/services/test_dataset_service_dataset.pyindex 4f2dd4fc258..28df36f5ced 100644--- a/api/tests/unit_tests/services/test_dataset_service_dataset.py+++ b/api/tests/unit_tests/services/test_dataset_service_dataset.py@@ -1,4 +1,4 @@-"""SQLite-backed tests for dataset-level operations in :mod:`services.dataset_service`.+"""SQLite-backed tests for dataset-level operations in :mod:`services.knowledge.dataset_service`.Mapped objects in this module are real SQLAlchemy models. Provider runtimes,RBAC clients, Celery tasks, and model-manager results remain mocked at their@@ -28,13 +28,13 @@ from models.dataset import (ExternalKnowledgeBindings,Pipeline,)-from services.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetServicefrom services.entities.knowledge_entities.rag_pipeline_entities import (IconInfo,RagPipelineDatasetCreateEntity,)from services.errors.account import NoPermissionErrorfrom services.errors.dataset import DatasetNameDuplicateError+from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetServicefrom tests.unit_tests.model_factories import make_account, make_dataset, make_tenantfrom .dataset_service_test_helpers import (@@ -177,7 +177,7 @@ class TestDatasetServiceValidation:def test_check_dataset_model_setting_skips_non_high_quality_datasets(self) -> None:dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:DatasetService.check_dataset_model_setting(dataset)model_manager_cls.assert_not_called()@@ -185,7 +185,7 @@ class TestDatasetServiceValidation:def test_check_dataset_model_setting_validates_high_quality_embedding(self) -> None:dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:DatasetService.check_dataset_model_setting(dataset)model_manager_cls.for_tenant.return_value.get_model_instance.assert_called_once_with(@@ -205,7 +205,7 @@ class TestDatasetServiceValidation:def test_check_dataset_model_setting_wraps_provider_errors(self, error: Exception, message: str) -> None:dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = errorwith pytest.raises(ValueError, match=message):DatasetService.check_dataset_model_setting(dataset)@@ -220,7 +220,7 @@ class TestDatasetServiceValidation:credentials={"api_key": "secret"},)- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:model_manager_cls.for_tenant.return_value.get_model_instance.return_value = model_instanceresult = DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")@@ -235,7 +235,7 @@ class TestDatasetServiceValidation:credentials={},)- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:model_manager_cls.for_tenant.return_value.get_model_instance.return_value = model_instancewith pytest.raises(ValueError, match="Model schema not found"):DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")@@ -254,7 +254,7 @@ class TestDatasetServiceValidation:def test_direct_model_setting_checks_wrap_runtime_errors(self, method: Callable[[str, str, str], None], error: Exception, message: str) -> None:- with patch("services.dataset_service.ModelManager") as model_manager_cls:+ with patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls:model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = errorwith pytest.raises(ValueError, match=message):method("tenant-1", "provider", "model")@@ -285,7 +285,7 @@ class TestDatasetServiceRetrieval:with (patch(- "services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",+ "services.knowledge.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",return_value=SimpleNamespace(workspace=SimpleNamespace(permission_keys=[])),),):@@ -371,7 +371,7 @@ class TestDatasetServiceRetrieval:config_overrides(RBAC_ENABLED=True)with (patch(- "services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",+ "services.knowledge.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",return_value=SimpleNamespace(workspace=SimpleNamespace(permission_keys=["dataset.create_and_management"])),@@ -414,8 +414,10 @@ class TestDatasetServiceCreationAndUpdate:embedding_model = SimpleNamespace(provider="provider", model_name="default-embedding")with (- patch("services.dataset_service.ModelManager") as model_manager_cls,- patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,+ patch(+ "services.knowledge.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"+ ),):model_manager_cls.for_tenant.return_value.get_default_model_instance.return_value = embedding_modeldataset = DatasetService.create_empty_dataset(@@ -438,7 +440,9 @@ class TestDatasetServiceCreationAndUpdate:sqlite_session.add_all([_external_api(), _external_api(api_id="api-foreign", tenant_id="tenant-2")])sqlite_session.commit()- with patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"):+ with patch(+ "services.knowledge.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"+ ):dataset = DatasetService.create_empty_dataset("tenant-1","External",@@ -465,7 +469,9 @@ class TestDatasetServiceCreationAndUpdate:with (pytest.raises(ValueError, match="api template not found"),- patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"),+ patch(+ "services.knowledge.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"+ ),):DatasetService.create_empty_dataset("tenant-1",@@ -489,7 +495,7 @@ class TestDatasetServiceCreationAndUpdate:permission=DatasetPermissionEnum.ALL_TEAM,)- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):dataset = DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)assert dataset.name == "Untitled 2"@@ -508,7 +514,7 @@ class TestDatasetServiceCreationAndUpdate:)with (- patch("services.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.current_user", _account()),pytest.raises(DatasetNameDuplicateError, match="already exists"),):DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)@@ -524,7 +530,7 @@ class TestDatasetServiceCreationAndUpdate:)with (- patch("services.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.current_user", account),pytest.raises(ValueError, match="Current user or current user id not found"),):DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)@@ -640,8 +646,8 @@ class TestDatasetServiceCreationAndUpdate:with (patch.object(DatasetService, "_handle_indexing_technique_change", return_value="update"),patch.object(DatasetService, "_update_pipeline_knowledge_base_node_data"),- patch("services.dataset_service.deal_dataset_vector_index_task.delay") as vector_task,- patch("services.dataset_service.regenerate_summary_index_task.delay") as summary_task,+ patch("services.knowledge.dataset_service.deal_dataset_vector_index_task.delay") as vector_task,+ patch("services.knowledge.dataset_service.regenerate_summary_index_task.delay") as summary_task,):updated = DatasetService._update_internal_dataset(dataset,@@ -685,7 +691,7 @@ class TestDatasetServiceCreationAndUpdate:lambda _session, _previous_transaction: transaction_events.append("rollback"),)- with patch("services.dataset_service.RagPipelineService") as service_cls:+ with patch("services.knowledge.dataset_service.RagPipelineService") as service_cls:service_cls.return_value.get_published_workflow.side_effect = RuntimeError("boom")with pytest.raises(RuntimeError, match="boom"):DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", sqlite_session)@@ -765,8 +771,8 @@ class TestDatasetServiceEmbeddingSettings:def test_configure_high_quality_wraps_provider_error(self, unbound_session: Session) -> None:account = _account()with (- patch("services.dataset_service.current_user", account),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,):model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = LLMBadRequestError()with pytest.raises(ValueError, match="No Embedding Model available"):@@ -814,8 +820,8 @@ class TestDatasetServiceEmbeddingSettings:collection_binding.id = "collection-binding-1"with (- patch("services.dataset_service.current_user", account),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,patch.object(DatasetCollectionBindingService,"get_dataset_collection_binding",@@ -841,8 +847,8 @@ class TestDatasetServiceEmbeddingSettings:filtered_data: dict[str, object] = {}with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,):model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = ProviderTokenNotInitError("missing"@@ -886,7 +892,7 @@ class TestDatasetServiceRagPipelineSettings:account._current_tenant = Nonewith (- patch("services.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.current_user", account),pytest.raises(ValueError, match="Current user or current tenant not found"),):DatasetService.update_rag_pipeline_dataset_settings(@@ -910,8 +916,8 @@ class TestDatasetServiceRagPipelineSettings:collection_binding.id = "collection-binding-2"with (- patch("services.dataset_service.current_user", account),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,patch.object(DatasetService, "check_is_multimodal_model", return_value=True),patch.object(DatasetCollectionBindingService,@@ -942,7 +948,7 @@ class TestDatasetServiceRagPipelineSettings:sqlite_session.add(dataset)sqlite_session.commit()- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):DatasetService.update_rag_pipeline_dataset_settings(dataset,_make_knowledge_configuration(@@ -966,8 +972,8 @@ class TestDatasetServiceRagPipelineSettings:sqlite_session.commit()with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.deal_dataset_index_update_task.delay") as update_task,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.deal_dataset_index_update_task.delay") as update_task,):DatasetService.update_rag_pipeline_dataset_settings(dataset,@@ -1012,7 +1018,7 @@ class TestDatasetServiceRagPipelineSettings:sqlite_session.commit()with (- patch("services.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.current_user", _account()),pytest.raises(ValueError, match=message),):DatasetService.update_rag_pipeline_dataset_settings(diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.pyindex a450560d231..702e7902065 100644--- a/api/tests/unit_tests/services/test_dataset_service_document.py+++ b/api/tests/unit_tests/services/test_dataset_service_document.py@@ -6,12 +6,13 @@ from datetime import datetimefrom sqlalchemy import event, selectfrom sqlalchemy.orm import Session+from core.model_manager import ModelInstancefrom enums import DeploymentEditionfrom models.dataset import Dataset, DatasetCollectionBinding, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import UploadFilefrom models.source import DataSourceOauthBinding-from services.dataset_ref_service import DatasetRefService+from services.knowledge.resource_scope import DatasetReffrom tests.unit_tests.config_override import config_overrides_contextfrom tests.unit_tests.model_factories import make_account, make_tenant, make_upload_file@@ -41,7 +42,6 @@ from .dataset_service_test_helpers import (RetrievalModel,Rule,Segmentation,- SimpleNamespace,WebsiteInfo,_make_features,_make_lock_context,@@ -248,7 +248,7 @@ class TestDocumentServiceRetrieval:sqlite_session.commit()documents = DocumentService.get_document_by_ids(- DatasetRefService.create_dataset_ref(dataset),+ DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id),["expected", "disabled", "archived", "waiting", "other-dataset", "other-tenant"],sqlite_session,)@@ -276,8 +276,8 @@ class TestDocumentServiceMutations:sqlite_session.add_all([dataset, document, other_dataset, other_tenant])sqlite_session.commit()- with patch("services.dataset_service.batch_clean_document_task") as clean_task:- dataset_ref = DatasetRefService.create_dataset_ref(dataset)+ with patch("services.knowledge.dataset_service.batch_clean_document_task") as clean_task:+ dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)DocumentService.delete_documents(dataset_ref,[document.id, other_dataset.id, other_tenant.id],@@ -300,14 +300,18 @@ class TestDocumentServiceMutations:commits += 1event.listen(sqlite_session, "after_commit", count_commit)+ dataset = _dataset_row()DocumentService.delete_documents(- DatasetRefService.create_dataset_ref(_dataset_row()), [], IndexStructureType.PARAGRAPH_INDEX, sqlite_session+ DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id),+ [],+ IndexStructureType.PARAGRAPH_INDEX,+ sqlite_session,)event.remove(sqlite_session, "after_commit", count_commit)assert commits == 0def test_rename_document_raises_when_dataset_is_missing(self, sqlite_session: Session):- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):with pytest.raises(ValueError, match="Dataset not found"):DocumentService.rename_document("dataset-1", "doc-1", "New Name", sqlite_session)@@ -315,7 +319,7 @@ class TestDocumentServiceMutations:dataset = _dataset_row()sqlite_session.add(dataset)sqlite_session.commit()- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):with pytest.raises(ValueError, match="Document not found"):DocumentService.rename_document(dataset.id, "doc-1", "New Name", sqlite_session)@@ -324,7 +328,7 @@ class TestDocumentServiceMutations:document = _document_row(tenant_id="tenant-other")sqlite_session.add_all([dataset, document])sqlite_session.commit()- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):with pytest.raises(ValueError, match="No permission"):DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)@@ -357,7 +361,7 @@ class TestDocumentServiceMutations:commits += 1event.listen(sqlite_session, "after_commit", count_commit)- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):result = DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)event.remove(sqlite_session, "after_commit", count_commit)@@ -377,8 +381,8 @@ class TestDocumentServiceMutations:sqlite_session.add(document)sqlite_session.commit()with (- patch("services.dataset_service.redis_client") as redis,- patch("services.dataset_service.recover_document_indexing_task") as task,+ patch("services.knowledge.dataset_service.redis_client") as redis,+ patch("services.knowledge.dataset_service.recover_document_indexing_task") as task,):DocumentService.recover_document(document, sqlite_session)@@ -395,11 +399,11 @@ class TestDocumentServiceMutations:sqlite_session.commit()retry_flags = _RetryFlagStore({f"document_{document.id}_is_retried": "other-request"})with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client", retry_flags),+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.redis_client", retry_flags),):with pytest.raises(ValueError, match="being retried"):- DocumentService.retry_document("dataset-1", [document], sqlite_session)+ DocumentService.retry_document("dataset-1", [document], sqlite_session, actor_id="user-1")def test_retry_document_leaves_batch_unchanged_when_later_document_is_already_being_retried(self, sqlite_session: Session@@ -412,15 +416,13 @@ class TestDocumentServiceMutations:second_retry_key = "document_doc-2_is_retried"retry_flags = _RetryFlagStore({second_retry_key: "other-request"})with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client", retry_flags),- patch("services.dataset_service.retry_document_indexing_task") as retry_task,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.redis_client", retry_flags),+ patch("services.knowledge.dataset_service.retry_document_indexing_task") as retry_task,):with pytest.raises(ValueError, match="being retried"):DocumentService.retry_document(- "dataset-1",- [first_document, second_document],- sqlite_session,+ "dataset-1", [first_document, second_document], sqlite_session, actor_id="user-1")assert first_document.indexing_status == IndexingStatus.ERROR@@ -444,11 +446,11 @@ class TestDocumentServiceMutations:sqlite_session.commit()with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client", retry_flags),+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.redis_client", retry_flags),):with pytest.raises(ValueError, match="being retried"):- DocumentService.retry_document("dataset-1", documents, sqlite_session)+ DocumentService.retry_document("dataset-1", documents, sqlite_session, actor_id="user-1")assert retry_flags.values[first_retry_key] == "new-owner"assert retry_flags.values[second_retry_key] == "other-request"@@ -464,12 +466,12 @@ class TestDocumentServiceMutations:event.listen(sqlite_session, "before_commit", fail_commit)with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client", retry_flags),- patch("services.dataset_service.retry_document_indexing_task") as retry_task,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.redis_client", retry_flags),+ patch("services.knowledge.dataset_service.retry_document_indexing_task") as retry_task,):with pytest.raises(RuntimeError, match="database unavailable"):- DocumentService.retry_document("dataset-1", [document], sqlite_session)+ DocumentService.retry_document("dataset-1", [document], sqlite_session, actor_id="user-1")event.remove(sqlite_session, "before_commit", fail_commit)assert retry_flags.values == {}@@ -484,11 +486,11 @@ class TestDocumentServiceMutations:sqlite_session.commit()retry_flags = _RetryFlagStore()with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client", retry_flags),- patch("services.dataset_service.retry_document_indexing_task") as task,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.redis_client", retry_flags),+ patch("services.knowledge.dataset_service.retry_document_indexing_task") as task,):- DocumentService.retry_document("dataset-1", documents, sqlite_session)+ DocumentService.retry_document("dataset-1", documents, sqlite_session, actor_id="user-1")sqlite_session.expire_all()statuses = sqlite_session.scalars(select(Document.indexing_status).order_by(Document.id)).all()@@ -498,19 +500,25 @@ class TestDocumentServiceMutations:def test_sync_website_document_raises_when_sync_flag_exists(self, sqlite_session: Session):dataset = _dataset_row()document = _document_row()- with patch("services.dataset_service.redis_client") as mock_redis:+ with patch("services.knowledge.dataset_service.redis_client") as mock_redis:mock_redis.get.return_value = "1"with pytest.raises(ValueError, match="being synced"):DocumentService.sync_website_document(dataset, document, sqlite_session)- def test_sync_website_document_rejects_document_outside_dataset(self, sqlite_session: Session):+ @pytest.mark.parametrize(+ ("document_dataset_id", "document_tenant_id"),+ [("dataset-2", "tenant-1"), ("dataset-1", "tenant-2")],+ )+ def test_sync_website_document_rejects_document_outside_dataset(+ self, document_dataset_id: str, document_tenant_id: str, sqlite_session: Session+ ):dataset = _dataset_row()- document = _document_row(dataset_id="dataset-2")+ document = _document_row(dataset_id=document_dataset_id, tenant_id=document_tenant_id)with (pytest.raises(ValueError, match="Document not found"),- patch("services.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,):DocumentService.sync_website_document(dataset, document, sqlite_session)@@ -526,8 +534,8 @@ class TestDocumentServiceMutations:sqlite_session.commit()with (- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.sync_website_document_indexing_task") as sync_task,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.sync_website_document_indexing_task") as sync_task,):mock_redis.get.return_value = None@@ -549,7 +557,7 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:def account_context(self):account = _account()- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):yield account@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)@@ -579,9 +587,9 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:first_document = _document_row(name="VeryLongDocumentNameForDataset.txt")with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),patch(- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",+ "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",return_value=binding,),patch.object(@@ -643,7 +651,7 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:first_document = _document_row(name="Doc")with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),patch.object(DocumentService,"save_document_with_dataset_id",@@ -677,7 +685,7 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:with (patch(- "services.dataset_service.FeatureService.get_features",+ "services.knowledge.dataset_service.FeatureService.get_features",return_value=_make_features(plan=CloudPlan.SANDBOX),),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,@@ -697,7 +705,7 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:def account_context(self):account = _account()- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):yield accountdef test_update_document_with_dataset_id_raises_when_document_is_missing(@@ -805,8 +813,8 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:with (patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value=updated_at),- patch("services.dataset_service.document_indexing_update_task") as update_task,+ patch("services.knowledge.dataset_service.naive_utc_now", return_value=updated_at),+ patch("services.knowledge.dataset_service.document_indexing_update_task") as update_task,):result = DocumentService.update_document_with_dataset_id(dataset,@@ -906,8 +914,8 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:with (patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),- patch("services.dataset_service.document_indexing_update_task") as update_task,+ patch("services.knowledge.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),+ patch("services.knowledge.dataset_service.document_indexing_update_task") as update_task,):result = DocumentService.update_document_with_dataset_id(dataset,@@ -933,13 +941,22 @@ class TestDocumentServiceCreateValidation:"""Unit tests for document creation validation helpers."""def test_document_create_args_validate_requires_data_source_or_process_rule(self):- knowledge_config = SimpleNamespace(data_source=None, process_rule=None)+ knowledge_config = KnowledgeConfig(indexing_technique="economy")with pytest.raises(ValueError, match="Data source or Process rule is required"):DocumentService.document_create_args_validate(knowledge_config)def test_document_create_args_validate_delegates_to_sub_validators(self):- knowledge_config = SimpleNamespace(data_source=object(), process_rule=object())+ knowledge_config = KnowledgeConfig(+ indexing_technique="economy",+ data_source=DataSource(+ info_list=InfoList(+ data_source_type="upload_file",+ file_info_list=FileInfo(file_ids=["file-1"]),+ )+ ),+ process_rule=ProcessRule(mode="automatic"),+ )with (patch.object(DocumentService, "data_source_args_validate") as validate_data_source,@@ -950,39 +967,20 @@ class TestDocumentServiceCreateValidation:validate_data_source.assert_called_once_with(knowledge_config)validate_process_rule.assert_called_once_with(knowledge_config)- def test_data_source_args_validate_rejects_invalid_type(self):- knowledge_config = SimpleNamespace(- data_source=SimpleNamespace(- info_list=SimpleNamespace(- data_source_type="bad-source",- file_info_list=None,- notion_info_list=None,- website_info_list=None,- )- )- )-- with pytest.raises(ValueError, match="Data source type is invalid"):- DocumentService.data_source_args_validate(knowledge_config)-@pytest.mark.parametrize(- ("data_source_type", "field_name", "message"),+ ("data_source_type", "message"),[- ("upload_file", "file_info_list", "File source info is required"),- ("notion_import", "notion_info_list", "Notion source info is required"),- ("website_crawl", "website_info_list", "Website source info is required"),+ ("upload_file", "File source info is required"),+ ("notion_import", "Notion source info is required"),+ ("website_crawl", "Website source info is required"),],)- def test_data_source_args_validate_requires_source_specific_info(self, data_source_type, field_name, message):- info_values = {- "data_source_type": data_source_type,- "file_info_list": object(),- "notion_info_list": object(),- "website_info_list": object(),- }- info_values[field_name] = None- info_list = SimpleNamespace(**info_values)- knowledge_config = SimpleNamespace(data_source=SimpleNamespace(info_list=info_list))+ def test_data_source_args_validate_requires_source_specific_info(self, data_source_type, message):+ info_list = InfoList.model_validate({"data_source_type": data_source_type})+ knowledge_config = KnowledgeConfig(+ indexing_technique="economy",+ data_source=DataSource(info_list=info_list),+ )with pytest.raises(ValueError, match=message):DocumentService.data_source_args_validate(knowledge_config)@@ -1075,7 +1073,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:account = _account()with (- patch("services.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.current_user", account),patch.object(DatasetService, "check_doc_form"),):yield account@@ -1087,7 +1085,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(file_ids=None)- with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()):+ with patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()):with pytest.raises(ValueError, match="File source info is required"):DocumentService.save_document_with_dataset_id(dataset,@@ -1105,7 +1103,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:with (patch(- "services.dataset_service.FeatureService.get_features",+ "services.knowledge.dataset_service.FeatureService.get_features",return_value=_make_features(plan=CloudPlan.SANDBOX),),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,@@ -1132,7 +1130,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):with pytest.raises(ValueError, match="batch upload limit of 1"):@@ -1155,10 +1153,9 @@ class TestDocumentServiceSaveDocumentWithDatasetId:updated_document.batch = "batch-existing"with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch.object(- DocumentService, "update_document_with_dataset_id", return_value=updated_document- ) as update_document,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.document_indexing_update_task.delay"),+ patch.object(DocumentService, "update_prepared_document", return_value=updated_document) as update_document,):documents, batch = DocumentService.save_document_with_dataset_id(dataset,@@ -1179,7 +1176,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(data_source=None)- with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()):+ with patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()):with pytest.raises(ValueError, match="Data source is required when creating new documents"):DocumentService.save_document_with_dataset_id(dataset,@@ -1200,7 +1197,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:process_rule=ProcessRule(mode="custom"),)- with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()):+ with patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()):with pytest.raises(ValueError, match="No process rule found"):DocumentService.save_document_with_dataset_id(dataset,@@ -1209,46 +1206,6 @@ class TestDocumentServiceSaveDocumentWithDatasetId:session=sqlite_session,)- @config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)- def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(- self, account_context, unbound_session: Session- ):- dataset = _dataset_row(indexing_technique=None)- knowledge_config = SimpleNamespace(- doc_form=IndexStructureType.PARAGRAPH_INDEX,- original_document_id=None,- data_source=None,- indexing_technique="broken-technique",- )-- with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()):- with pytest.raises(ValueError, match="Indexing technique is invalid"):- DocumentService.save_document_with_dataset_id(- dataset,- knowledge_config,- account_context,- session=unbound_session,- )-- @config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)- def test_save_document_with_dataset_id_returns_empty_for_invalid_process_rule_mode(- self, account_context, unbound_session: Session- ):- dataset = _dataset_row()- knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"])- knowledge_config.process_rule = SimpleNamespace(mode="unsupported-mode", rules=None)-- with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()):- documents, batch = DocumentService.save_document_with_dataset_id(- dataset,- knowledge_config,- account_context,- session=unbound_session,- )-- assert documents == []- assert batch == ""-@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)def test_save_document_with_dataset_id_upload_file_creates_and_reindexes_documents(self, account_context, sqlite_session: Session@@ -1263,13 +1220,13 @@ class TestDocumentServiceSaveDocumentWithDatasetId:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,- patch("services.dataset_service.DuplicateDocumentIndexingTaskProxy") as duplicate_proxy_cls,- patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),- patch("services.dataset_service.time.strftime", return_value="20260101010101"),- patch("services.dataset_service.secrets.randbelow", return_value=23),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,+ patch("services.knowledge.dataset_service.DuplicateDocumentIndexingTaskProxy") as duplicate_proxy_cls,+ patch("services.knowledge.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),+ patch("services.knowledge.dataset_service.time.strftime", return_value="20260101010101"),+ patch("services.knowledge.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()documents, batch = DocumentService.save_document_with_dataset_id(@@ -1340,11 +1297,11 @@ class TestDocumentServiceSaveDocumentWithDatasetId:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.clean_notion_document_task") as clean_task,- patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,- patch("services.dataset_service.uuid.uuid4", return_value="doc-new"),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.clean_notion_document_task") as clean_task,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,+ patch("services.knowledge.dataset_service.uuid.uuid4", return_value="doc-new"),):mock_redis.lock.return_value = _make_lock_context()documents, _ = DocumentService.save_document_with_dataset_id(@@ -1390,9 +1347,9 @@ class TestDocumentServiceSaveDocumentWithDatasetId:doc_language="English",)with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,):mock_redis.lock.return_value = _make_lock_context()@@ -1448,7 +1405,7 @@ class TestDocumentServiceBatchUpdateStatus:sqlite_session.add_all([dataset, document])sqlite_session.commit()- with patch("services.dataset_service.redis_client") as mock_redis:+ with patch("services.knowledge.dataset_service.redis_client") as mock_redis:mock_redis.get.return_value = "1"with pytest.raises(DocumentIndexingError, match="Busy document is being indexed"):@@ -1470,7 +1427,7 @@ class TestDocumentServiceBatchUpdateStatus:event.listen(sqlite_session, "before_commit", fail_commit)with (- patch("services.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,):mock_redis.get.return_value = None@@ -1490,8 +1447,8 @@ class TestDocumentServiceBatchUpdateStatus:sqlite_session.commit()with (- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.add_document_to_index_task") as add_task,+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.add_document_to_index_task") as add_task,):mock_redis.get.return_value = Noneadd_task.delay.side_effect = RuntimeError("task failed")@@ -1513,7 +1470,7 @@ class TestDocumentServiceTenantAndUpdateEdges:def account_context(self):account = _account()- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):yield accountdef test_get_tenant_documents_count_scopes_state_and_tenant(self, account_context, sqlite_session: Session):@@ -1563,8 +1520,8 @@ class TestDocumentServiceTenantAndUpdateEdges:with (patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value=updated_at),- patch("services.dataset_service.document_indexing_update_task") as update_task,+ patch("services.knowledge.dataset_service.naive_utc_now", return_value=updated_at),+ patch("services.knowledge.dataset_service.document_indexing_update_task") as update_task,):result = DocumentService.update_document_with_dataset_id(dataset,@@ -1686,8 +1643,8 @@ class TestDocumentServiceTenantAndUpdateEdges:with (patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),- patch("services.dataset_service.document_indexing_update_task") as update_task,+ patch("services.knowledge.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),+ patch("services.knowledge.dataset_service.document_indexing_update_task") as update_task,):result = DocumentService.update_document_with_dataset_id(dataset,@@ -1719,7 +1676,7 @@ class TestDocumentServiceSaveWithoutDatasetBilling:def account_context(self):account = _account()- with patch("services.dataset_service.current_user", account):+ with patch("services.knowledge.dataset_service.current_user", account):yield account@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)@@ -1757,7 +1714,7 @@ class TestDocumentServiceSaveWithoutDatasetBilling:document = _document_row(name="Doc")with (- patch("services.dataset_service.FeatureService.get_features", return_value=features),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=features),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,patch.object(DocumentService,@@ -1799,7 +1756,7 @@ class TestDocumentServiceSaveWithoutDatasetBilling:)with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):with pytest.raises(ValueError, match="batch upload limit of 1"):@@ -1810,132 +1767,6 @@ class TestDocumentServiceSaveWithoutDatasetBilling:check_quota.assert_not_called()-class TestDocumentServiceEstimateValidation:- """Unit tests for estimate_args_validate branches."""-- def test_estimate_args_validate_rejects_missing_info_list(self):- with pytest.raises(ValueError, match="Field required"):- DocumentService.estimate_args_validate({})-- def test_estimate_args_validate_sets_empty_rules_for_automatic_mode(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {"mode": "automatic", "rules": {"ignored": True}},- }-- DocumentService.estimate_args_validate(args)-- assert args["process_rule"]["rules"] == {}-- def test_estimate_args_validate_rejects_unknown_pre_processing_rule_id(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "custom",- "rules": {- "pre_processing_rules": [{"id": "unknown", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 128},- },- },- }-- with pytest.raises(ValueError, match="pre_processing_rules id is invalid"):- DocumentService.estimate_args_validate(args)-- def test_estimate_args_validate_deduplicates_rules_for_custom_mode(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "custom",- "rules": {- "pre_processing_rules": [- {"id": "remove_stopwords", "enabled": True},- {"id": "remove_stopwords", "enabled": False},- ],- "segmentation": {"separator": "\n", "max_tokens": 128},- },- },- }-- DocumentService.estimate_args_validate(args)-- assert args["process_rule"]["rules"]["pre_processing_rules"] == [{"id": "remove_stopwords", "enabled": False}]-- def test_estimate_args_validate_custom_mode_drops_hierarchical_fields(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "custom",- "rules": {- "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 128},- "parent_mode": "full-doc",- "subchunk_segmentation": {"separator": "###", "max_tokens": 64},- },- },- }-- DocumentService.estimate_args_validate(args)-- assert args["process_rule"]["rules"] == {- "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 128},- }-- def test_estimate_args_validate_requires_summary_index_provider_name(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "custom",- "rules": {- "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 128},- },- "summary_index_setting": {"enable": True, "model_name": "summary-model"},- },- }-- with pytest.raises(ValueError, match="Field required"):- DocumentService.estimate_args_validate(args)-- def test_estimate_args_validate_preserves_hierarchical_fields(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "hierarchical",- "rules": {- "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 512},- "parent_mode": "full-doc",- "subchunk_segmentation": {"separator": "###", "max_tokens": 128},- },- },- }-- DocumentService.estimate_args_validate(args)-- assert args["process_rule"]["rules"]["parent_mode"] == "full-doc"- assert args["process_rule"]["rules"]["subchunk_segmentation"] == {"separator": "###", "max_tokens": 128}-- def test_estimate_args_validate_hierarchical_defaults_parent_mode_to_paragraph(self):- args = {- "info_list": {"data_source_type": "upload_file"},- "process_rule": {- "mode": "hierarchical",- "rules": {- "pre_processing_rules": [{"id": "remove_stopwords", "enabled": True}],- "segmentation": {"separator": "\n", "max_tokens": 512},- "subchunk_segmentation": {"separator": "###", "max_tokens": 128},- },- },- }-- DocumentService.estimate_args_validate(args)-- assert args["process_rule"]["rules"]["parent_mode"] == "paragraph"- assert args["process_rule"]["rules"]["subchunk_segmentation"] == {"separator": "###", "max_tokens": 128}--class TestDocumentServiceSaveDocumentAdditionalBranches:"""Additional unit tests for dataset bootstrap and process-rule branches."""@@ -1944,7 +1775,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:account = _account()with (- patch("services.dataset_service.current_user", account),+ patch("services.knowledge.dataset_service.current_user", account),patch.object(DatasetService, "check_doc_form"),):yield account@@ -1969,18 +1800,19 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:binding.id = "binding-1"with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,patch(- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",+ "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",return_value=binding,) as get_binding,- patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),+ patch.object(DocumentService, "update_prepared_document", return_value=updated_document),+ patch("services.knowledge.dataset_service.document_indexing_update_task.delay"),):- model_manager_cls.for_tenant.return_value.get_default_model_instance.return_value = SimpleNamespace(- model_name="default-embedding",- provider="default-provider",- )+ default_model = object.__new__(ModelInstance)+ default_model.model_name = "default-embedding"+ default_model.provider = "default-provider"+ model_manager_cls.for_tenant.return_value.get_default_model_instance.return_value = default_modeldocuments, batch = DocumentService.save_document_with_dataset_id(dataset,@@ -2035,13 +1867,14 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:updated_document = _document_row(document_id="doc-1")with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.ModelManager") as model_manager_cls,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls,patch(- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",+ "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",return_value=binding,) as get_binding,- patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),+ patch.object(DocumentService, "update_prepared_document", return_value=updated_document),+ patch("services.knowledge.dataset_service.document_indexing_update_task.delay"),):DocumentService.save_document_with_dataset_id(dataset, knowledge_config, account_context, session=sqlite_session@@ -2073,11 +1906,11 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,- patch("services.dataset_service.time.strftime", return_value="20260101010101"),- patch("services.dataset_service.secrets.randbelow", return_value=23),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,+ patch("services.knowledge.dataset_service.time.strftime", return_value="20260101010101"),+ patch("services.knowledge.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()documents, batch = DocumentService.save_document_with_dataset_id(@@ -2111,11 +1944,11 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DocumentIndexingTaskProxy"),- patch("services.dataset_service.time.strftime", return_value="20260101010101"),- patch("services.dataset_service.secrets.randbelow", return_value=23),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy"),+ patch("services.knowledge.dataset_service.time.strftime", return_value="20260101010101"),+ patch("services.knowledge.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()documents, _ = DocumentService.save_document_with_dataset_id(@@ -2141,11 +1974,11 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DocumentIndexingTaskProxy"),- patch("services.dataset_service.time.strftime", return_value="20260101010101"),- patch("services.dataset_service.secrets.randbelow", return_value=23),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.DocumentIndexingTaskProxy"),+ patch("services.knowledge.dataset_service.time.strftime", return_value="20260101010101"),+ patch("services.knowledge.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()documents, _ = DocumentService.save_document_with_dataset_id(@@ -2170,10 +2003,10 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:sqlite_session.commit()with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.time.strftime", return_value="20260101010101"),- patch("services.dataset_service.secrets.randbelow", return_value=23),+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.time.strftime", return_value="20260101010101"),+ patch("services.knowledge.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()with pytest.raises(FileNotExistsError, match="One or more files not found"):@@ -2200,8 +2033,8 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:)with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,):mock_redis.lock.return_value = _make_lock_context()with pytest.raises(ValueError, match="No notion info list found"):@@ -2229,8 +2062,8 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:)with (- patch("services.dataset_service.FeatureService.get_features", return_value=_make_features()),- patch("services.dataset_service.redis_client") as mock_redis,+ patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()),+ patch("services.knowledge.dataset_service.redis_client") as mock_redis,):mock_redis.lock.return_value = _make_lock_context()with pytest.raises(ValueError, match="No website info list found"):diff --git a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.pyindex 6049e76227a..d7236393504 100644--- a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py+++ b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py@@ -1,5 +1,5 @@import types-from unittest.mock import Mock+from unittest.mock import create_autospecimport pytestfrom redis.exceptions import LockNotOwnedError@@ -10,7 +10,8 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Indfrom models.account import Account, Tenantfrom models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegmentfrom models.enums import ProcessRuleMode-from services.dataset_service import DocumentService, SegmentService+from services.knowledge.dataset_service import DocumentService, SegmentService+from services.knowledge.segments.application import SegmentMutationServiceTENANT_ID = "11111111-1111-1111-1111-111111111111"USER_ID = "22222222-2222-2222-2222-222222222222"@@ -36,7 +37,7 @@ def fake_current_user(monkeypatch: pytest.MonkeyPatch):user = Account(name="Test User", email="[email protected]")user.id = USER_IDuser._current_tenant = tenant- monkeypatch.setattr("services.dataset_service.current_user", user)+ monkeypatch.setattr("services.knowledge.dataset_service.current_user", user)return user@@ -47,7 +48,7 @@ def fake_features(monkeypatch: pytest.MonkeyPatch):documents_upload_quota=types.SimpleNamespace(limit=10_000, size=0),)monkeypatch.setattr(- "services.dataset_service.FeatureService.get_features",+ "services.knowledge.dataset_service.FeatureService.get_features",lambda tenant_id, **_kwargs: features,)return features@@ -61,7 +62,7 @@ def fake_lock(monkeypatch: pytest.MonkeyPatch):return FakeLock()# DatasetService imports redis_client directly from extensions.ext_redis- monkeypatch.setattr("services.dataset_service.redis_client.lock", _fake_lock)+ monkeypatch.setattr("services.knowledge.dataset_service.redis_client.lock", _fake_lock)# ---------------------------------------------------------------------------@@ -115,7 +116,7 @@ def test_save_document_with_dataset_id_ignores_lock_not_owned(account = fake_current_user# Avoid touching real doc_form logic- monkeypatch.setattr("services.dataset_service.DatasetService.check_doc_form", lambda *a, **k: None)+ monkeypatch.setattr("services.knowledge.dataset_service.DatasetService.check_doc_form", lambda *a, **k: None)# Act: this would hit the redis lock, whose __enter__ raises LockNotOwnedError.# Our implementation should catch it and still return (documents, batch).@@ -143,60 +144,6 @@ def test_save_document_with_dataset_id_ignores_lock_not_owned(# ---------------------------------------------------------------------------[email protected]("sqlite_session", [(Account, Tenant, Dataset, Document, DocumentSegment)], indirect=True)-def test_add_segment_ignores_lock_not_owned(- monkeypatch: pytest.MonkeyPatch,- fake_current_user,- fake_lock,- sqlite_session: Session,-):- # Arrange- dataset = Dataset(- id=DATASET_ID,- tenant_id=TENANT_ID,- name="Test Dataset",- description="",- created_by=USER_ID,- indexing_technique=IndexTechniqueType.ECONOMY,- )- document = Document(- id=DOCUMENT_ID,- tenant_id=TENANT_ID,- dataset_id=DATASET_ID,- position=1,- data_source_type="upload_file",- data_source_info="{}",- batch="batch-1",- name="Test Document",- created_from="web",- created_by=USER_ID,- word_count=0,- doc_form=IndexStructureType.QA_INDEX,- )- sqlite_session.add_all([fake_current_user._current_tenant, fake_current_user, dataset, document])- sqlite_session.commit()-- # Minimal args required by add_segment- args = {- "content": "question text",- "answer": "answer text",- "keywords": ["k1", "k2"],- }-- monkeypatch.setattr("services.dataset_service.VectorService", Mock())-- # Act- result = SegmentService.create_segment(args=args, document=document, dataset=dataset, session=sqlite_session)-- # Assert- # Under LockNotOwnedError except, add_segment should swallow the error and return None.- assert result is None- assert not sqlite_session.in_transaction()- assert sqlite_session.scalar(select(func.count(DocumentSegment.id))) == 0- sqlite_session.refresh(document)- assert document.word_count == 0--# ---------------------------------------------------------------------------# 3. Multi-segment creation (multi_create_segment)# ---------------------------------------------------------------------------@@ -235,14 +182,17 @@ def test_multi_create_segment_ignores_lock_not_owned(sqlite_session.add_all([fake_current_user._current_tenant, fake_current_user, dataset, document])sqlite_session.commit()+ mutations = create_autospec(SegmentMutationService, instance=True)result = SegmentService.multi_create_segment(segments=[{"content": "question", "answer": "answer", "keywords": ["key"]}],document=document,dataset=dataset,session=sqlite_session,+ mutations=mutations,)assert result is None+ mutations.index_segments.assert_not_called()assert not sqlite_session.in_transaction()assert sqlite_session.scalar(select(func.count(DocumentSegment.id))) == 0sqlite_session.refresh(document)diff --git a/api/tests/unit_tests/services/test_dataset_service_segment.py b/api/tests/unit_tests/services/test_dataset_service_segment.pyindex c62c9acc680..f848684aee1 100644--- a/api/tests/unit_tests/services/test_dataset_service_segment.py+++ b/api/tests/unit_tests/services/test_dataset_service_segment.py@@ -5,12 +5,10 @@ from __future__ import annotationsfrom collections.abc import Callablefrom datetime import datetimefrom types import SimpleNamespace-from typing import Literal-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMock, create_autospec, patchimport pytest-from sqlalchemy import event, select-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models import Account@@ -18,17 +16,14 @@ from models.account import Tenantfrom models.dataset import (ChildChunk,Dataset,- DatasetProcessRule,Document,DocumentSegment,- DocumentSegmentSummary,- SegmentAttachmentBinding,)-from models.enums import DataSourceType, DocumentCreatedFrom, ProcessRuleMode, SegmentStatus-from services.dataset_ref_service import DatasetRef, DatasetRefService, DocumentRef, SegmentRef-from services.dataset_service import SegmentService-from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs-from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError+from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository+from services.knowledge.dataset_service import SegmentService+from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef+from services.knowledge.segments.application import SegmentIndex, SegmentIndexingState, SegmentMutationServicedef _account(*, account_id: str = "user-1", tenant_id: str = "tenant-1") -> Account:@@ -146,41 +141,6 @@ def _persist_chain(session: Session) -> tuple[Dataset, Document, DocumentSegmentreturn dataset, document, segment-class TestDatasetRefService:- def test_dataset_ref_is_plain_named_tuple(self) -> None:- assert DatasetRef("tenant-1", "dataset-1") == ("tenant-1", "dataset-1")-- @pytest.mark.parametrize(- ("document_dataset_id", "document_tenant_id"),- [("dataset-2", "tenant-1"), ("dataset-1", "tenant-2")],- )- def test_create_document_ref_rejects_document_outside_dataset(- self, document_dataset_id: str, document_tenant_id: str- ) -> None:- dataset_ref = DatasetRef("tenant-1", "dataset-1")- document = _document(dataset_id=document_dataset_id, tenant_id=document_tenant_id)-- assert DatasetRefService.create_document_ref(dataset_ref, document) is None-- def test_create_segment_ref_carries_full_parent_chain(self) -> None:- document_ref = DocumentRef(DatasetRef("tenant-1", "dataset-1"), "document-1")-- assert DatasetRefService.create_segment_ref(document_ref, "segment-1") == SegmentRef(document_ref, "segment-1")-- def test_get_document_by_ref_enforces_full_owner_chain(self, sqlite_session: Session) -> None:- owned = _document()- wrong_dataset = _document(document_id="wrong-dataset", dataset_id="dataset-2")- wrong_tenant = _document(document_id="wrong-tenant", tenant_id="tenant-2")- sqlite_session.add_all([owned, wrong_dataset, wrong_tenant])- sqlite_session.commit()-- owned_ref = DocumentRef(DatasetRef("tenant-1", "dataset-1"), owned.id)- assert DatasetRefService.get_document_by_ref(owned_ref, session=sqlite_session) is owned- for document in (wrong_dataset, wrong_tenant):- spoofed_ref = DocumentRef(DatasetRef("tenant-1", "dataset-1"), document.id)- assert DatasetRefService.get_document_by_ref(spoofed_ref, session=sqlite_session) is None--class TestSegmentServiceQueries:def test_get_child_chunks_filters_owner_keyword_and_paginates(self, sqlite_session: Session) -> None:_persist_chain(sqlite_session)@@ -194,7 +154,7 @@ class TestSegmentServiceQueries:)sqlite_session.commit()- with patch("services.dataset_service.current_user", _account()):+ with patch("services.knowledge.dataset_service.current_user", _account()):page = SegmentService.get_child_chunks("segment-1","document-1",@@ -277,6 +237,7 @@ class TestSegmentServiceQueries:_segment(segment_id="enabled"),_segment(segment_id="disabled", position=2, enabled=False),_segment(segment_id="other", document_id="document-2", position=1),+ _segment(segment_id="foreign", tenant_id="tenant-2"),])sqlite_session.commit()@@ -285,6 +246,7 @@ class TestSegmentServiceQueries:"document-1","dataset-1",sqlite_session,+ tenant_id="tenant-1",status=SegmentStatus.COMPLETED,enabled=True,)@@ -326,262 +288,49 @@ class TestSegmentServiceValidation:)-class TestSegmentServiceChildChunks:- def test_create_child_chunk_assigns_next_position_and_commits(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- sqlite_session.add(_child(position=1))- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.create_child_chunk_vector"),- ):- child = SegmentService.create_child_chunk("new child", segment, document, dataset, sqlite_session)-- assert child.position == 2- assert sqlite_session.get(ChildChunk, child.id) is child-- def test_create_child_chunk_rolls_back_on_vector_failure(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- rollback_events: list[str] = []- event.listen(sqlite_session, "after_rollback", lambda _session: rollback_events.append("rollback"))-- with (- patch("services.dataset_service.current_user", _account()),- patch(- "services.dataset_service.VectorService.create_child_chunk_vector",- side_effect=RuntimeError("vector failed"),- ),- pytest.raises(ChildChunkIndexingError, match="vector failed"),- ):- SegmentService.create_child_chunk("new child", segment, document, dataset, sqlite_session)-- assert rollback_events == ["rollback"]- assert sqlite_session.scalars(select(ChildChunk)).all() == []-- def test_update_child_chunks_updates_deletes_and_creates_real_rows(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- keep = _child(child_id="keep", content="old", position=1)- remove = _child(child_id="remove", content="remove", position=2)- sqlite_session.add_all([keep, remove])- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_child_chunk_vector"),- ):- chunks = SegmentService.update_child_chunks(- [- ChildChunkUpdateArgs(id=keep.id, content="updated"),- ChildChunkUpdateArgs(content="created"),- ],- segment,- document,- dataset,- sqlite_session,- )-- persisted = sqlite_session.scalars(select(ChildChunk).order_by(ChildChunk.position)).all()- assert {chunk.content for chunk in persisted} == {"updated", "created"}- assert sqlite_session.get(ChildChunk, remove.id) is None- assert {chunk.content for chunk in chunks} == {"updated", "created"}-- def test_update_child_chunk_rolls_back_on_vector_failure(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- child = _child()- sqlite_session.add(child)- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch(- "services.dataset_service.VectorService.update_child_chunk_vector",- side_effect=RuntimeError("vector failed"),- ),- pytest.raises(ChildChunkIndexingError, match="vector failed"),- ):- SegmentService.update_child_chunk("changed", child, segment, document, dataset, sqlite_session)-- sqlite_session.refresh(child)- assert child.content == "child content"-- def test_update_child_chunk_persists_and_updates_vector(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- child = _child()- sqlite_session.add(child)- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_child_chunk_vector") as vector_update,- ):- updated = SegmentService.update_child_chunk(- "changed child", child, segment, document, dataset, sqlite_session- )-- assert updated is child- assert child.content == "changed child"- assert child.word_count == len("changed child")- assert sqlite_session.get(ChildChunk, child.id) is child- vector_update.assert_called_once_with([], [child], [], dataset, session=sqlite_session)-- def test_update_child_chunks_rolls_back_all_database_changes_on_vector_failure(- self, sqlite_session: Session- ) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- child = _child(content="original")- sqlite_session.add(child)- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch(- "services.dataset_service.VectorService.update_child_chunk_vector",- side_effect=RuntimeError("vector failed"),- ),- pytest.raises(ChildChunkIndexingError, match="vector failed"),- ):- SegmentService.update_child_chunks(- [ChildChunkUpdateArgs(id=child.id, content="changed")],- segment,- document,- dataset,- sqlite_session,- )-- sqlite_session.refresh(child)- assert child.content == "original"-- def test_delete_child_chunk_commits_after_vector_delete(self, sqlite_session: Session) -> None:- dataset, _, _ = _persist_chain(sqlite_session)- child = _child()- sqlite_session.add(child)- sqlite_session.commit()-- with patch("services.dataset_service.VectorService.delete_child_chunk_vector"):- SegmentService.delete_child_chunk(child, dataset, sqlite_session)-- assert sqlite_session.get(ChildChunk, child.id) is None-- def test_delete_child_chunk_rolls_back_on_vector_failure(self, sqlite_session: Session) -> None:- dataset, _, _ = _persist_chain(sqlite_session)- child = _child()- sqlite_session.add(child)- sqlite_session.commit()-- with (- patch(- "services.dataset_service.VectorService.delete_child_chunk_vector",- side_effect=RuntimeError("vector failed"),- ),- pytest.raises(ChildChunkDeleteIndexError, match="vector failed"),- ):- SegmentService.delete_child_chunk(child, dataset, sqlite_session)-- assert sqlite_session.get(ChildChunk, child.id) is not None--class TestSegmentServiceMutations:- def test_create_segment_persists_position_and_updates_document_count(self, sqlite_session: Session) -> None:- dataset = _dataset()- document = _document(word_count=0)- sqlite_session.add_all([dataset, document, _segment(segment_id="existing", content="old")])- sqlite_session.commit()- embedding_model = SimpleNamespace(get_text_embedding_num_tokens=lambda *, texts: [len(texts) + 2])-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.create_segments_vector"),- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- segment = SegmentService.create_segment(- {"content": "new", "attachment_ids": [], "keywords": ["key"]},- document,- dataset,- sqlite_session,- )-- assert segment is not None- assert segment.position == 2- assert document.word_count == 3-- def test_create_segment_persists_qa_answer_and_attachment_bindings(self, sqlite_session: Session) -> None:- dataset = _dataset()- document = _document(doc_form=IndexStructureType.QA_INDEX, word_count=0)- sqlite_session.add_all([dataset, document])- sqlite_session.commit()- embedding_model = MagicMock()- embedding_model.get_text_embedding_num_tokens.return_value = [7]-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.create_segments_vector"),- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- segment = SegmentService.create_segment(- {- "content": "question",- "answer": "answer",- "attachment_ids": ["attachment-1", "attachment-2"],- },- document,- dataset,- sqlite_session,- )-- assert segment is not None- assert segment.answer == "answer"- assert segment.word_count == len("question") + len("answer")- assert document.word_count == segment.word_count- bindings = sqlite_session.scalars(- select(SegmentAttachmentBinding).where(SegmentAttachmentBinding.segment_id == segment.id)- ).all()- assert {binding.attachment_id for binding in bindings} == {"attachment-1", "attachment-2"}-- def test_create_segment_marks_real_row_error_on_vector_failure(self, sqlite_session: Session) -> None:+ @pytest.mark.parametrize(+ ("document_dataset_id", "document_tenant_id"),+ [("dataset-2", "tenant-1"), ("dataset-1", "tenant-2")],+ )+ def test_delete_segment_rejects_document_outside_dataset(+ self, document_dataset_id: str, document_tenant_id: str, sqlite_session: Session+ ) -> None:dataset = _dataset()- document = _document(word_count=0)- sqlite_session.add_all([dataset, document])- sqlite_session.commit()- embedding_model = SimpleNamespace(get_text_embedding_num_tokens=lambda *, texts: [len(texts) + 2])+ document = _document(dataset_id=document_dataset_id, tenant_id=document_tenant_id)+ segment = _segment(dataset_id=document_dataset_id, tenant_id=document_tenant_id)+ mutations = create_autospec(SegmentMutationService, instance=True, spec_set=True)- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch(- "services.dataset_service.VectorService.create_segments_vector",- side_effect=RuntimeError("vector failed"),- ),- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- segment = SegmentService.create_segment(- {"content": "new", "attachment_ids": []},- document,- dataset,- sqlite_session,- )+ with pytest.raises(ValueError, match="Document does not belong to the dataset"):+ SegmentService.delete_segment(segment, document, dataset, sqlite_session, mutations=mutations)- assert segment is not None- assert segment.status == SegmentStatus.ERROR- assert segment.enabled is False- assert segment.error == "vector failed"+ mutations.delete_segment.assert_not_called()- def test_multi_create_segment_marks_each_real_row_error_on_vector_failure(self, sqlite_session: Session) -> None:+ def test_multi_create_segment_marks_each_real_row_error_on_vector_failure(+ self, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]+ ) -> None:dataset = _dataset()document = _document(word_count=0)sqlite_session.add_all([dataset, document])sqlite_session.commit()embedding_model = SimpleNamespace(get_text_embedding_num_tokens=lambda *, texts: [len(texts) + 1])+ index = create_autospec(SegmentIndex, instance=True, spec_set=True)++ def fail(*_args: object, **_kwargs: object) -> None:+ assert not sqlite_session.in_transaction()+ raise RuntimeError("vector failed")++ index.create_many.side_effect = fail+ mutations = SegmentMutationService(+ store=SQLAlchemySegmentRepository(session_factory=sqlite_session_factory),+ index=index,+ text_hash=lambda text: text,+ indexing_state=create_autospec(SegmentIndexingState, instance=True, spec_set=True),+ )with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch(- "services.dataset_service.VectorService.create_segments_vector",- side_effect=RuntimeError("vector failed"),- ),+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.ModelManager") as manager_cls,):manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_modelsegments = SegmentService.multi_create_segment(@@ -589,6 +338,7 @@ class TestSegmentServiceMutations:document,dataset,sqlite_session,+ mutations=mutations,)assert segments is not None@@ -603,11 +353,11 @@ class TestSegmentServiceMutations:sqlite_session.commit()embedding_model = MagicMock()embedding_model.get_text_embedding_num_tokens.side_effect = [[11], [13]]+ mutations = create_autospec(SegmentMutationService, instance=True, spec_set=True)with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.create_segments_vector") as vector_create,+ patch("services.knowledge.dataset_service.current_user", _account()),+ patch("services.knowledge.dataset_service.ModelManager") as manager_cls,):manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_modelsegments = SegmentService.multi_create_segment(@@ -618,6 +368,7 @@ class TestSegmentServiceMutations:document,dataset,sqlite_session,+ mutations=mutations,)assert segments is not None@@ -632,531 +383,8 @@ class TestSegmentServiceMutations:assert embedding_model.get_text_embedding_num_tokens.call_args_list[1].kwargs == {"texts": ["question-2answer-2"]}- assert vector_create.call_args.args[:2] == ([["key"], None], segments)-- def test_update_segment_disables_and_dispatches_cleanup(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.disable_segment_from_index_task.delay") as cleanup,- ):- updated = SegmentService.update_segment(- SegmentUpdateArgs(enabled=False),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.enabled is False- cleanup.assert_called_once_with(segment.id)-- def test_update_segment_same_content_persists_keywords(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_segment_vector") as vector_update,- patch("services.dataset_service.VectorService.update_multimodel_vector"),- ):- updated = SegmentService.update_segment(- SegmentUpdateArgs(content=segment.content, keywords=["new-keyword"]),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.keywords == ["new-keyword"]- vector_update.assert_called_once()-- def test_update_segment_omitted_attachment_ids_preserves_existing_bindings(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- SegmentService.update_segment(- SegmentUpdateArgs(content=segment.content),- segment,- document,- dataset,- sqlite_session,- )-- multimodel_update.assert_not_called()-- def test_update_segment_explicit_empty_attachment_ids_clears_bindings(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- SegmentService.update_segment(- SegmentUpdateArgs(content=segment.content, attachment_ids=[]),- segment,- document,- dataset,- sqlite_session,- )-- multimodel_update.assert_called_once_with(segment, [], dataset, session=sqlite_session)-- def test_update_segment_explicit_attachment_ids_updates_multimodel_vector(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- attachment_ids = ["attachment-1", "attachment-2"]-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- SegmentService.update_segment(- SegmentUpdateArgs(content=segment.content, attachment_ids=attachment_ids),- segment,- document,- dataset,- sqlite_session,- )-- multimodel_update.assert_called_once_with(segment, attachment_ids, dataset, session=sqlite_session)-- def test_update_segment_content_change_uses_embedding_tokens_and_updates_document_count(- self, sqlite_session: Session- ) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- original_document_count = document.word_count- original_segment_count = segment.word_count- assert original_document_count is not None- assert original_segment_count is not None- embedding_model = SimpleNamespace(get_text_embedding_num_tokens=lambda *, texts: [len(texts) + 4])-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.update_segment_vector"),- patch("services.dataset_service.VectorService.update_multimodel_vector"),- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- updated = SegmentService.update_segment(- SegmentUpdateArgs(content="changed content"),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.content == "changed content"- assert updated.tokens == 5- assert document.word_count == original_document_count + len("changed content") - original_segment_count-- def test_update_segment_same_qa_content_updates_answer_and_word_counts(self, sqlite_session: Session) -> None:- dataset = _dataset()- document = _document(doc_form=IndexStructureType.QA_INDEX, word_count=20)- segment = _segment(content="question")- sqlite_session.add_all([dataset, document, segment])- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.VectorService.update_segment_vector") as vector_update,- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- updated = SegmentService.update_segment(- SegmentUpdateArgs(content="question", answer="new answer"),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.answer == "new answer"- assert updated.word_count == len("question") + len("new answer")- assert document.word_count == 20 + len("new answer")- vector_update.assert_not_called()- multimodel_update.assert_not_called()-- def test_update_segment_changed_qa_content_tokenizes_question_and_answer(self, sqlite_session: Session) -> None:- dataset = _dataset()- document = _document(doc_form=IndexStructureType.QA_INDEX, word_count=10)- segment = _segment(content="old")- sqlite_session.add_all([dataset, document, segment])- sqlite_session.commit()- embedding_model = MagicMock()- embedding_model.get_text_embedding_num_tokens.return_value = [21]-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.update_segment_vector") as vector_update,- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- updated = SegmentService.update_segment(- SegmentUpdateArgs(content="new question", answer="new answer", keywords=["key"]),- segment,- document,- dataset,- sqlite_session,- )-- embedding_model.get_text_embedding_num_tokens.assert_called_once_with(texts=["new questionnew answer"])- assert updated.answer == "new answer"- assert updated.tokens == 21- assert updated.word_count == len("new question") + len("new answer")- assert document.word_count == 10 + updated.word_count - len("old")- vector_update.assert_called_once_with(["key"], segment, dataset, session=sqlite_session)- multimodel_update.assert_not_called()-- @pytest.mark.parametrize("summary", [None, "same summary"])- def test_update_segment_changed_content_regenerates_existing_summary(- self, sqlite_session: Session, summary: str | None- ) -> None:- dataset = _dataset()- dataset.summary_index_setting = {"enable": True}- document = _document(word_count=10)- segment = _segment(content="old")- summary_record = DocumentSegmentSummary(- dataset_id=dataset.id,- document_id=document.id,- chunk_id=segment.id,- summary_content="same summary",- )- sqlite_session.add_all([dataset, document, segment, summary_record])- sqlite_session.commit()- embedding_model = MagicMock()- embedding_model.get_text_embedding_num_tokens.return_value = [9]-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService.update_segment_vector"),- patch("services.dataset_service.VectorService.update_multimodel_vector"),- patch(- "services.summary_index_service.SummaryIndexService.generate_and_vectorize_summary"- ) as generate_summary,- patch("services.summary_index_service.SummaryIndexService.update_summary_for_segment") as update_summary,- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- updated = SegmentService.update_segment(- SegmentUpdateArgs(content="new content", summary=summary),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.content == "new content"- generate_summary.assert_called_once_with(segment, dataset, {"enable": True}, session=sqlite_session)- update_summary.assert_not_called()-- def test_update_segment_same_parent_content_regenerates_children_and_manual_summary(- self, sqlite_session: Session- ) -> None:- dataset = _dataset()- document = _document(doc_form=IndexStructureType.PARENT_CHILD_INDEX)- segment = _segment(content="same content")- process_rule = DatasetProcessRule(- dataset_id=dataset.id,- mode=ProcessRuleMode.AUTOMATIC,- rules=None,- created_by="user-1",- )- process_rule.id = "rule-1"- document.dataset_process_rule_id = process_rule.id- summary_record = DocumentSegmentSummary(- dataset_id=dataset.id,- document_id=document.id,- chunk_id=segment.id,- summary_content="old summary",- )- sqlite_session.add_all([dataset, process_rule, document, segment, summary_record])- sqlite_session.commit()- embedding_model = object()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService") as vector_service,- patch("services.summary_index_service.SummaryIndexService.update_summary_for_segment") as update_summary,- ):- manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model- updated = SegmentService.update_segment(- SegmentUpdateArgs(- content="same content",- regenerate_child_chunks=True,- summary="new summary",- ),- segment,- document,- dataset,- sqlite_session,- )-- assert updated is segment- vector_service.generate_child_chunks.assert_called_once_with(- segment,- document,- dataset,- embedding_model,- process_rule,- True,- session=sqlite_session,+ mutations.index_segments.assert_called_once_with(+ DatasetRef("tenant-1", "dataset-1").document("document-1"),+ segment_ids=[segment.id for segment in segments],+ keywords_list=[["key"], None],)- update_summary.assert_called_once_with(segment, dataset, "new summary", session=sqlite_session)- vector_service.update_multimodel_vector.assert_not_called()-- def test_update_segment_changed_parent_content_uses_default_embedding_and_ignores_summary_failure(- self, sqlite_session: Session- ) -> None:- dataset = _dataset()- dataset.embedding_model_provider = None- document = _document(doc_form=IndexStructureType.PARENT_CHILD_INDEX, word_count=10)- segment = _segment(content="old")- process_rule = DatasetProcessRule(- dataset_id=dataset.id,- mode=ProcessRuleMode.AUTOMATIC,- rules=None,- created_by="user-1",- )- process_rule.id = "rule-1"- document.dataset_process_rule_id = process_rule.id- summary_record = DocumentSegmentSummary(- dataset_id=dataset.id,- document_id=document.id,- chunk_id=segment.id,- summary_content="old summary",- )- sqlite_session.add_all([dataset, process_rule, document, segment, summary_record])- sqlite_session.commit()- token_model = MagicMock()- token_model.get_text_embedding_num_tokens.return_value = [9]- default_embedding_model = object()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.ModelManager") as manager_cls,- patch("services.dataset_service.VectorService") as vector_service,- patch(- "services.summary_index_service.SummaryIndexService.update_summary_for_segment",- side_effect=RuntimeError("summary failed"),- ) as update_summary,- ):- manager = manager_cls.for_tenant.return_value- manager.get_model_instance.return_value = token_model- manager.get_default_model_instance.return_value = default_embedding_model- updated = SegmentService.update_segment(- SegmentUpdateArgs(- content="new parent content",- regenerate_child_chunks=True,- summary="new summary",- ),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.content == "new parent content"- manager.get_default_model_instance.assert_called_once()- vector_service.generate_child_chunks.assert_called_once_with(- segment,- document,- dataset,- default_embedding_model,- process_rule,- True,- session=sqlite_session,- )- update_summary.assert_called_once_with(segment, dataset, "new summary", session=sqlite_session)- vector_service.update_multimodel_vector.assert_not_called()-- def test_update_segment_parent_regeneration_marks_economy_segment_error(self, sqlite_session: Session) -> None:- dataset = _dataset()- dataset.indexing_technique = IndexTechniqueType.ECONOMY- document = _document(doc_form=IndexStructureType.PARENT_CHILD_INDEX)- segment = _segment(content="same content")- sqlite_session.add_all([dataset, document, segment])- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.VectorService.update_multimodel_vector") as multimodel_update,- ):- updated = SegmentService.update_segment(- SegmentUpdateArgs(content="same content", regenerate_child_chunks=True),- segment,- document,- dataset,- sqlite_session,- )-- assert updated.enabled is False- assert updated.status == SegmentStatus.ERROR- assert updated.error == "The knowledge base index technique is not high quality!"- multimodel_update.assert_not_called()-- def test_update_segment_rejects_disabled_or_indexing_segment(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- segment.enabled = False- sqlite_session.commit()-- with patch("services.dataset_service.current_user", _account()):- with pytest.raises(ValueError, match="Can't update disabled segment"):- SegmentService.update_segment(- SegmentUpdateArgs(content="changed"), segment, document, dataset, sqlite_session- )-- segment.enabled = True- sqlite_session.commit()- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.redis_client.get", return_value=b"1"),- pytest.raises(ValueError, match="Segment is indexing"),- ):- SegmentService.update_segment(- SegmentUpdateArgs(content="changed"), segment, document, dataset, sqlite_session- )-- def test_delete_segment_removes_row_and_updates_document_count(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- child = _child()- sqlite_session.add(child)- sqlite_session.commit()- original_count = document.word_count- assert original_count is not None- assert segment.word_count is not None-- with (- patch("services.dataset_service.redis_client.get", return_value=None),- patch("services.dataset_service.redis_client.setex") as set_cache,- patch("services.dataset_service.delete_segment_from_index_task.delay") as cleanup,- ):- SegmentService.delete_segment(segment, document, dataset, sqlite_session)-- assert sqlite_session.get(DocumentSegment, segment.id) is None- assert document.word_count == original_count - segment.word_count- set_cache.assert_called_once_with(f"segment_{segment.id}_delete_indexing", 600, 1)- cleanup.assert_called_once_with(- [segment.index_node_id],- dataset.id,- document.id,- [segment.id],- [child.index_node_id],- )-- def test_delete_segment_rejects_when_delete_is_already_in_progress(self, sqlite_session: Session) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.redis_client.get", return_value=b"1"),- pytest.raises(ValueError, match="Segment is deleting"),- ):- SegmentService.delete_segment(segment, document, dataset, sqlite_session)-- assert sqlite_session.get(DocumentSegment, segment.id) is segment-- def test_delete_segments_scopes_rows_and_clamps_document_count(self, sqlite_session: Session) -> None:- dataset = _dataset()- document = _document(word_count=5)- owned = _segment(segment_id="owned", content="123456789")- wrong_tenant = _segment(segment_id="wrong-tenant", tenant_id="tenant-2", content="foreign")- wrong_dataset = _segment(segment_id="wrong-dataset", dataset_id="dataset-2", content="foreign")- wrong_document = _segment(segment_id="wrong-document", document_id="document-2", content="foreign")- sqlite_session.add_all([dataset, document, owned, wrong_tenant, wrong_dataset, wrong_document])- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.delete_segment_from_index_task.delay"),- ):- SegmentService.delete_segments(- [owned.id, wrong_tenant.id, wrong_dataset.id, wrong_document.id],- document,- dataset,- sqlite_session,- )-- assert sqlite_session.get(DocumentSegment, owned.id) is None- assert sqlite_session.get(DocumentSegment, wrong_tenant.id) is wrong_tenant- assert sqlite_session.get(DocumentSegment, wrong_dataset.id) is wrong_dataset- assert sqlite_session.get(DocumentSegment, wrong_document.id) is wrong_document- assert document.word_count == 0-- @pytest.mark.parametrize("segment_ids", [[], ["missing"]])- def test_delete_segments_noops_at_empty_selection_boundary(- self, sqlite_session: Session, segment_ids: list[str]- ) -> None:- dataset, document, segment = _persist_chain(sqlite_session)-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.delete_segment_from_index_task.delay") as cleanup,- ):- SegmentService.delete_segments(segment_ids, document, dataset, sqlite_session)-- assert sqlite_session.get(DocumentSegment, segment.id) is segment- assert document.word_count == 20- cleanup.assert_not_called()-- @pytest.mark.parametrize(("action", "initial_enabled"), [("enable", False), ("disable", True)])- def test_update_segments_status_persists_only_owned_rows(- self,- sqlite_session: Session,- action: Literal["enable", "disable"],- initial_enabled: bool,- ) -> None:- dataset = _dataset()- document = _document()- owned = _segment(segment_id="owned", enabled=initial_enabled)- cached = _segment(segment_id="cached", position=2, enabled=initial_enabled)- decoy = _segment(segment_id="decoy", document_id="document-2", enabled=initial_enabled)- sqlite_session.add_all([dataset, document, owned, cached, decoy])- sqlite_session.commit()-- with (- patch("services.dataset_service.current_user", _account()),- patch(- "services.dataset_service.redis_client.get",- side_effect=lambda key: b"1" if key == f"segment_{cached.id}_indexing" else None,- ),- patch("services.dataset_service.enable_segments_to_index_task.delay") as enable_task,- patch("services.dataset_service.disable_segments_from_index_task.delay") as disable_task,- ):- SegmentService.update_segments_status(- [owned.id, cached.id, decoy.id], action, dataset, document, sqlite_session- )-- assert owned.enabled is (action == "enable")- assert cached.enabled is initial_enabled- assert decoy.enabled is initial_enabled- dispatched_task = enable_task if action == "enable" else disable_task- skipped_task = disable_task if action == "enable" else enable_task- dispatched_task.assert_called_once_with([owned.id], dataset.id, document.id)- skipped_task.assert_not_called()-- @pytest.mark.parametrize("action", ["enable", "disable"])- def test_update_segments_status_noops_at_empty_selection_boundary(- self, sqlite_session: Session, action: Literal["enable", "disable"]- ) -> None:- dataset, document, segment = _persist_chain(sqlite_session)- initial_enabled = segment.enabled-- with (- patch("services.dataset_service.current_user", _account()),- patch("services.dataset_service.enable_segments_to_index_task.delay") as enable_task,- patch("services.dataset_service.disable_segments_from_index_task.delay") as disable_task,- ):- SegmentService.update_segments_status([], action, dataset, document, sqlite_session)-- assert segment.enabled is initial_enabled- enable_task.assert_not_called()- disable_task.assert_not_called()diff --git a/api/tests/unit_tests/services/test_datasource_provider_service.py b/api/tests/unit_tests/services/test_datasource_provider_service.pyindex c52e59d8a98..277edf685fa 100644--- a/api/tests/unit_tests/services/test_datasource_provider_service.py+++ b/api/tests/unit_tests/services/test_datasource_provider_service.py@@ -14,8 +14,10 @@ from models.base import TypeBasefrom models.model import EndUserfrom models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProviderfrom models.provider_ids import DatasourceProviderID-from services import datasource_provider_service as service_module-from services.datasource_provider_service import DatasourceProviderService, get_current_user+from repositories.credentials.query_repository import CredentialQueryRepository+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from services.data_source import provider_service as service_module+from services.data_source.provider_service import DatasourceProviderService, get_current_user# ---------------------------------------------------------------------------# Helpers@@ -65,9 +67,46 @@ def persist(session: Session, *models: TypeBase) -> None:class TestDatasourceProviderService:"""Comprehensive tests for DatasourceProviderService targeting >95% coverage."""+ @pytest.mark.parametrize(+ ("credential_id", "credentials", "integration_token", "expected"),+ [+ ("cred", {"integration_secret": "saved-token"}, "env-token", "saved-token"),+ (None, {"integration_secret": "default-token"}, "env-token", "env-token"),+ ("cred", {}, "env-token", "env-token"),+ ("cred", {}, None, None),+ (None, {}, None, None),+ ],+ )+ def test_stored_notion_access_token_resolves_saved_credential_or_fallback(+ self, service, config_overrides, credential_id, credentials, integration_token, expected+ ):+ config_overrides(NOTION_INTEGRATION_TOKEN=integration_token)+ with patch.object(service, "get_datasource_credentials", return_value=credentials) as resolve:+ if expected is None:+ with pytest.raises(ValueError, match="Must specify `integration_token`"):+ service.get_stored_notion_access_token(tenant_id="tenant", credential_id=credential_id)+ else:+ assert (+ service.get_stored_notion_access_token(tenant_id="tenant", credential_id=credential_id) == expected+ )++ if credential_id is None:+ resolve.assert_not_called()+ else:+ resolve.assert_called_once_with(+ tenant_id="tenant",+ credential_id=credential_id,+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ )+@pytest.fixture- def service(self):- return DatasourceProviderService()+ def service(self, sqlite_session: Session):+ return DatasourceProviderService(+ credentials=SQLAlchemyDatasourceCredentialRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)+ )+ )@pytest.fixturedef sqlite_session(self, sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]:@@ -89,10 +128,10 @@ class TestDatasourceProviderService:patch("core.plugin.impl.base._httpx_client.request", side_effect=lambda **kw: httpx.request(**kw)),patch("core.plugin.impl.base._httpx_client.stream", side_effect=lambda **kw: httpx.stream(**kw)),patch("httpx.request") as mock_httpx,- patch("services.datasource_provider_service.encrypter") as mock_enc,- patch("services.datasource_provider_service.redis_client") as mock_redis,- patch("services.datasource_provider_service.generate_incremental_name") as mock_genname,- patch("services.datasource_provider_service.OAuthHandler") as mock_oauth,+ patch("services.data_source.provider_service.encrypter") as mock_enc,+ patch("services.data_source.provider_service.redis_client") as mock_redis,+ patch("services.data_source.provider_service.generate_incremental_name") as mock_genname,+ patch("services.data_source.provider_service.OAuthHandler") as mock_oauth,):mock_enc.encrypt_token.return_value = "enc_tok"mock_enc.decrypt_token.return_value = "dec_tok"@@ -299,7 +338,7 @@ class TestDatasourceProviderService:# -----------------------------------------------------------------------def test_should_return_empty_dict_when_credential_not_found(self, service, sqlite_session, mock_user):- with patch("services.datasource_provider_service.get_current_user", return_value=mock_user):+ with patch("services.data_source.provider_service.get_current_user", return_value=mock_user):assert service.get_datasource_credentials("t1", "prov", "org/plug") == {}def test_should_refresh_oauth_tokens_when_expired(self, service, sqlite_session, mock_user):@@ -307,7 +346,7 @@ class TestDatasourceProviderService:p = make_provider(auth_type="oauth2", expires_at=0, encrypted_credentials={"tok": "x"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "get_oauth_client", return_value={"oc": "v"}),patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),):@@ -324,8 +363,8 @@ class TestDatasourceProviderService:)persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),- patch("services.datasource_provider_service.OAuthHandler") as oauth_handler,+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.OAuthHandler") as oauth_handler,patch.object(service, "get_oauth_client", return_value={"oc": "v"}),patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),):@@ -338,7 +377,7 @@ class TestDatasourceProviderService:p = make_provider(encrypted_credentials={"k": "v"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "plain"}),):result = service.get_datasource_credentials("t1", "prov", "org/plug")@@ -346,28 +385,84 @@ class TestDatasourceProviderService:def test_should_fetch_by_credential_id_when_provided(self, service, sqlite_session, mock_user):"""When credential_id is passed, the credential_id filter path (line 113) is taken."""- p = make_provider(credential_id="cred-id", provider="other-provider", plugin_id="other-plugin")+ p = make_provider(credential_id="cred-id")persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "v"}),):result = service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id")assert result == {"k": "v"}+ @pytest.mark.parametrize("concurrent_update", [False, True])+ def test_refresh_closes_read_session_and_preserves_concurrent_credentials(+ self,+ service: DatasourceProviderService,+ sqlite_session: Session,+ mock_user: Account,+ monkeypatch: pytest.MonkeyPatch,+ concurrent_update: bool,+ ) -> None:+ import time++ from sqlalchemy import update++ row = make_provider(auth_type="oauth2", expires_at=0, encrypted_credentials={"token": "old"})+ persist(sqlite_session, row)+ opened: list[Session] = []++ class ObservedSession(Session):+ def __init__(self, **kwargs) -> None:+ super().__init__(**kwargs)+ opened.append(self)++ def refresh(**_kwargs: object) -> tuple[dict[str, str], int]:+ assert opened+ assert all(not session.in_transaction() for session in opened)+ if concurrent_update:+ with Session(sqlite_session.get_bind()) as session, session.begin():+ session.execute(+ update(DatasourceProvider)+ .where(DatasourceProvider.id == row.id)+ .values(encrypted_credentials={"token": "concurrent"}, expires_at=int(time.time()) + 3600)+ )+ return {"token": "refreshed"}, int(time.time()) + 3600++ service = DatasourceProviderService(+ credentials=SQLAlchemyDatasourceCredentialRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), class_=ObservedSession)+ )+ )+ monkeypatch.setattr(service_module, "get_current_user", lambda: mock_user)+ monkeypatch.setattr(service, "_refresh_datasource_credentials", refresh)+ monkeypatch.setattr(+ service,+ "decrypt_datasource_provider_credentials",+ lambda **kwargs: dict(kwargs["datasource_provider"].encrypted_credentials),+ )+ result = service.get_datasource_credentials("t1", "prov", "org/plug")+ assert result == {"token": "concurrent" if concurrent_update else "refreshed"}+ assert all(not session.in_transaction() for session in opened)++ def test_selected_credential_must_belong_to_the_requested_provider(+ self, service: DatasourceProviderService, sqlite_session: Session+ ) -> None:+ persist(sqlite_session, make_provider(provider="other-provider", plugin_id="other-plugin"))+ assert service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id") == {}+# -----------------------------------------------------------------------# get_all_datasource_credentials_by_provider (lines 176-228)# -----------------------------------------------------------------------def test_should_return_empty_list_when_no_provider_credentials_exist(self, service, sqlite_session, mock_user):- with patch("services.datasource_provider_service.get_current_user", return_value=mock_user):+ with patch("services.data_source.provider_service.get_current_user", return_value=mock_user):assert service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug") == []def test_should_refresh_and_return_credentials_when_oauth_expired(self, service, sqlite_session, mock_user):p = make_provider(auth_type="oauth2", expires_at=0, encrypted_credentials={"t": "x"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "get_oauth_client", return_value={"oc": "v"}),patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),):@@ -391,7 +486,7 @@ class TestDatasourceProviderService:)persist(sqlite_session, failed_provider, working_provider)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service,"_refresh_datasource_credentials",@@ -410,7 +505,7 @@ class TestDatasourceProviderService:p = make_provider(auth_type="oauth2", encrypted_credentials={"t": "x"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "_refresh_datasource_credentials") as refresh_credentials,patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),):@@ -514,7 +609,7 @@ class TestDatasourceProviderService:with (patch.object(service.provider_manager, "fetch_datasource_provider", return_value=pm),patch(- "services.datasource_provider_service.create_provider_encrypter",+ "services.data_source.provider_service.create_provider_encrypter",return_value=(MagicMock(), MagicMock()),),):@@ -567,7 +662,9 @@ class TestDatasourceProviderService:enabled=True,),)- with patch.object(service, "get_oauth_encrypter", return_value=(self._enc, None)):+ with patch(+ "services.data_source.credential_adapters.create_provider_encrypter", return_value=(self._enc, None)+ ):result = service.get_oauth_client("t1", make_id())assert result == {"k": "dec"}@@ -582,7 +679,7 @@ class TestDatasourceProviderService:)with (patch.object(service.provider_manager, "fetch_datasource_provider"),- patch("services.datasource_provider_service.PluginService.is_plugin_verified", return_value=True),+ patch("services.data_source.credential_adapters.PluginService.is_plugin_verified", return_value=True),):result = service.get_oauth_client("t1", make_id())assert result == {"k": "sys"}@@ -591,7 +688,7 @@ class TestDatasourceProviderService:"""Neither tenant nor system credentials → raises ValueError."""with (patch.object(service.provider_manager, "fetch_datasource_provider"),- patch("services.datasource_provider_service.PluginService.is_plugin_verified", return_value=False),+ patch("services.data_source.credential_adapters.PluginService.is_plugin_verified", return_value=False),):with pytest.raises(ValueError, match="Please configure oauth client params"):service.get_oauth_client("t1", make_id())@@ -734,13 +831,13 @@ class TestDatasourceProviderService:sqlite_session,make_provider(name="clash", provider="provider", plugin_id="org/plugin"),)- with patch("services.datasource_provider_service.get_current_user", return_value=mock_user):+ with patch("services.data_source.provider_service.get_current_user", return_value=mock_user):with pytest.raises(ValueError, match="already exists"):service.add_datasource_api_key_provider("clash", "t1", make_id(), {"sk": "v"})def test_should_raise_value_error_when_credentials_validation_fails(self, service, sqlite_session, mock_user):with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service.provider_manager, "validate_provider_credentials", side_effect=Exception("bad cred")),patch.object(service, "extract_secret_variables", return_value=[]),):@@ -749,7 +846,7 @@ class TestDatasourceProviderService:def test_should_add_api_key_provider_and_commit_when_valid(self, service, sqlite_session, mock_user):with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service.provider_manager, "validate_provider_credentials"),patch.object(service, "extract_secret_variables", return_value=["sk"]),):@@ -761,7 +858,7 @@ class TestDatasourceProviderService:def test_should_acquire_redis_lock_when_adding_api_key_provider(self, service, sqlite_session, mock_user):with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service.provider_manager, "validate_provider_credentials"),patch.object(service, "extract_secret_variables", return_value=[]),):@@ -805,13 +902,30 @@ class TestDatasourceProviderService:# -----------------------------------------------------------------------def test_should_return_empty_list_when_no_credentials_stored(self, service, sqlite_session):- assert service.list_datasource_credentials("t1", "prov", "org/plug", session=sqlite_session) == []+ assert (+ service.list_datasource_credentials(+ "t1",+ "prov",+ "org/plug",+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)+ ),+ )+ == []+ )def test_should_return_masked_credentials_list_when_credentials_exist(self, service, sqlite_session):p = make_provider(encrypted_credentials={"sk": "v"})persist(sqlite_session, p)with patch.object(service, "extract_secret_variables", return_value=["sk"]):- result = service.list_datasource_credentials("t1", "prov", "org/plug", session=sqlite_session)+ result = service.list_datasource_credentials(+ "t1",+ "prov",+ "org/plug",+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)+ ),+ )assert len(result) == 1assert result[0]["credential"] == {"sk": "obf"}@@ -820,7 +934,7 @@ class TestDatasourceProviderService:# -----------------------------------------------------------------------def test_should_aggregate_credentials_for_non_hardcoded_plugin(self, service, sqlite_session):- with patch("services.datasource_provider_service.PluginDatasourceManager") as mock_mgr:+ with patch("services.data_source.provider_service.PluginDatasourceManager") as mock_mgr:ds = MagicMock()ds.provider = "prov"ds.plugin_id = "org/plug"@@ -828,12 +942,18 @@ class TestDatasourceProviderService:mock_mgr.return_value.fetch_installed_datasource_providers.return_value = [ds]cred = {"credential": {"k": "v"}, "is_default": True}with patch.object(service, "list_datasource_credentials", return_value=[cred]):- results = service.get_all_datasource_credentials("t1", session=sqlite_session)+ results = service.get_all_datasource_credentials(+ "t1",+ session=sqlite_session,+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)+ ),+ )assert len(results) == 1def test_should_include_oauth_schema_for_hardcoded_plugin_ids(self, service, sqlite_session):"""Lines 819-871: get_all_datasource_credentials covers hardcoded langgenius plugin IDs."""- with patch("services.datasource_provider_service.PluginDatasourceManager") as mock_mgr:+ with patch("services.data_source.provider_service.PluginDatasourceManager") as mock_mgr:ds = MagicMock()ds.plugin_id = "langgenius/firecrawl_datasource"ds.provider = "firecrawl"@@ -853,7 +973,13 @@ class TestDatasourceProviderService:patch.object(service, "is_tenant_oauth_params_enabled", return_value=False),patch.object(service, "is_system_oauth_params_exist", return_value=False),):- results = service.get_all_datasource_credentials("t1", session=sqlite_session)+ results = service.get_all_datasource_credentials(+ "t1",+ session=sqlite_session,+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)+ ),+ )assert len(results) == 1assert results[0]["oauth_schema"] is not None@@ -877,7 +1003,7 @@ class TestDatasourceProviderService:# -----------------------------------------------------------------------def test_should_raise_value_error_when_credential_not_found_on_update(self, service, sqlite_session, mock_user):- with patch("services.datasource_provider_service.get_current_user", return_value=mock_user):+ with patch("services.data_source.provider_service.get_current_user", return_value=mock_user):with pytest.raises(ValueError, match="not found"):service.update_datasource_credentials("t1", "id", "prov", "org/plug", {}, "name")@@ -885,7 +1011,7 @@ class TestDatasourceProviderService:p = make_provider(credential_id="id", name="old_name", encrypted_credentials={"sk": "e"})conflict = make_provider(credential_id="conflict-id", name="new_name")persist(sqlite_session, p, conflict)- with patch("services.datasource_provider_service.get_current_user", return_value=mock_user):+ with patch("services.data_source.provider_service.get_current_user", return_value=mock_user):with pytest.raises(ValueError, match="already exists"):service.update_datasource_credentials("t1", "id", "prov", "org/plug", {}, "new_name")@@ -895,7 +1021,7 @@ class TestDatasourceProviderService:p = make_provider(credential_id="id", name="old_name", encrypted_credentials={"sk": "e"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "extract_secret_variables", return_value=["sk"]),patch.object(service.provider_manager, "validate_provider_credentials", side_effect=Exception("bad")),):@@ -907,7 +1033,7 @@ class TestDatasourceProviderService:p = make_provider(credential_id="id", name="old_name", encrypted_credentials={"sk": "old_enc"})persist(sqlite_session, p)with (- patch("services.datasource_provider_service.get_current_user", return_value=mock_user),+ patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(service, "extract_secret_variables", return_value=["sk"]),patch.object(service.provider_manager, "validate_provider_credentials"),):diff --git a/api/tests/unit_tests/services/test_external_dataset_service.py b/api/tests/unit_tests/services/test_external_dataset_service.pyindex c6a0032c227..30fdfeb50b5 100644--- a/api/tests/unit_tests/services/test_external_dataset_service.py+++ b/api/tests/unit_tests/services/test_external_dataset_service.py@@ -25,7 +25,7 @@ from services.entities.external_knowledge_entities.external_knowledge_entities i)from services.errors.dataset import DatasetNameDuplicateErrorfrom services.errors.knowledge_retrieval import ExternalKnowledgeRetrievalError-from services.external_knowledge_service import ExternalDatasetService+from services.knowledge.external.service import ExternalDatasetServiceclass ExternalDatasetServiceTestDataFactory:@@ -415,7 +415,7 @@ class TestExternalDatasetServiceValidateAPIList:class TestExternalDatasetServiceCreateAPI:"""Test create_external_knowledge_api operations."""- @patch("services.external_knowledge_service.ExternalDatasetService.check_endpoint_and_api_key")+ @patch("services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key")def test_create_external_knowledge_api_success_full(self, mock_check, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -442,7 +442,7 @@ class TestExternalDatasetServiceCreateAPI:persisted_api = sqlite_session.get(ExternalKnowledgeApis, result.id)assert persisted_api is result- @patch("services.external_knowledge_service.ExternalDatasetService.check_endpoint_and_api_key")+ @patch("services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key")def test_create_external_knowledge_api_minimal_fields(self, mock_check, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -485,7 +485,7 @@ class TestExternalDatasetServiceCreateAPI:with pytest.raises(ValueError, match="settings is required"):ExternalDatasetService.create_external_knowledge_api("tenant-123", "user-123", args, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.check_endpoint_and_api_key")+ @patch("services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key")def test_create_external_knowledge_api_settings_json_serialization(self, mock_check, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -508,7 +508,7 @@ class TestExternalDatasetServiceCreateAPI:parsed_settings = json.loads(result.settings)assert parsed_settings == settings- @patch("services.external_knowledge_service.ExternalDatasetService.check_endpoint_and_api_key")+ @patch("services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key")def test_create_external_knowledge_api_unicode_handling(self, mock_check, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -529,7 +529,7 @@ class TestExternalDatasetServiceCreateAPI:assert result.name == "测试API"assert result.description == "テストの説明"- @patch("services.external_knowledge_service.ExternalDatasetService.check_endpoint_and_api_key")+ @patch("services.knowledge.external.service.ExternalDatasetService.check_endpoint_and_api_key")def test_create_external_knowledge_api_long_description(self, mock_check, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -555,7 +555,7 @@ class TestExternalDatasetServiceCreateAPI:class TestExternalDatasetServiceCheckEndpoint:"""Test check_endpoint_and_api_key operations - extensive coverage."""- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_success_https(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test successful validation with HTTPS endpoint."""# Arrange@@ -569,7 +569,7 @@ class TestExternalDatasetServiceCheckEndpoint:ExternalDatasetService.check_endpoint_and_api_key(settings)mock_proxy.post.assert_called_once()- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_success_http(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test successful validation with HTTP endpoint."""# Arrange@@ -582,7 +582,7 @@ class TestExternalDatasetServiceCheckEndpoint:# Act & Assert - should not raiseExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_sends_json_body(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Regression for #39402: the validation probe must POST a JSON body matching theExternal Knowledge API retrieval contract, not a body-less request that providers@@ -688,7 +688,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="invalid endpoint"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_connection_timeout(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation fails on connection timeout."""# Arrange@@ -699,7 +699,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="failed to connect to the endpoint"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_network_error(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation fails on network error."""# Arrange@@ -710,7 +710,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="failed to connect to the endpoint"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_502_bad_gateway(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation fails with 502 Bad Gateway."""# Arrange@@ -724,7 +724,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="Bad Gateway.*failed to connect"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_404_not_found(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation fails with 404 Not Found."""# Arrange@@ -738,7 +738,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="Not Found.*failed to connect"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_403_forbidden(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation fails with 403 Forbidden (auth failure)."""# Arrange@@ -752,7 +752,7 @@ class TestExternalDatasetServiceCheckEndpoint:with pytest.raises(ValueError, match="Forbidden.*Authorization failed"):ExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_403_message_does_not_echo_api_key(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):@@ -789,7 +789,7 @@ class TestExternalDatasetServiceCheckEndpoint:assert "sk-abcdef" not in messageassert "CDEF" not in message- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_other_4xx_codes_pass(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test that other 4xx codes don't raise exceptions."""# Arrange@@ -803,7 +803,7 @@ class TestExternalDatasetServiceCheckEndpoint:# Act & Assert - should not raiseExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_5xx_codes_except_502_pass(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test that 5xx codes except 502 don't raise exceptions."""# Arrange@@ -817,7 +817,7 @@ class TestExternalDatasetServiceCheckEndpoint:# Act & Assert - should not raiseExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_with_port_number(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation with endpoint including port number."""# Arrange@@ -830,7 +830,7 @@ class TestExternalDatasetServiceCheckEndpoint:# Act & Assert - should not raiseExternalDatasetService.check_endpoint_and_api_key(settings)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_with_path(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test validation with endpoint including path."""# Arrange@@ -846,7 +846,7 @@ class TestExternalDatasetServiceCheckEndpoint:call_args = mock_proxy.post.call_argsassert "/retrieval" in call_args[0][0]- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_check_endpoint_authorization_header_format(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):@@ -900,7 +900,7 @@ class TestExternalDatasetServiceGetAPI:class TestExternalDatasetServiceUpdateAPI:"""Test update_external_knowledge_api operations."""- @patch("services.external_knowledge_service.naive_utc_now")+ @patch("services.knowledge.external.service.naive_utc_now")def test_update_external_knowledge_api_success_all_fields(self, mock_now, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1271,7 +1271,7 @@ class TestExternalDatasetServiceDocumentValidate:class TestExternalDatasetServiceProcessAPI:"""Test process_external_api operations - comprehensive HTTP method coverage."""- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_get_request(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing GET request."""# Arrange@@ -1287,7 +1287,7 @@ class TestExternalDatasetServiceProcessAPI:assert result == mock_responsemock_proxy.get.assert_called_once()- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_post_request_with_data(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):@@ -1307,7 +1307,7 @@ class TestExternalDatasetServiceProcessAPI:call_kwargs = mock_proxy.post.call_args.kwargsassert "data" in call_kwargs- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_put_request(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing PUT request."""# Arrange@@ -1323,7 +1323,7 @@ class TestExternalDatasetServiceProcessAPI:assert result == mock_responsemock_proxy.put.assert_called_once()- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_delete_request(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing DELETE request."""# Arrange@@ -1339,7 +1339,7 @@ class TestExternalDatasetServiceProcessAPI:assert result == mock_responsemock_proxy.delete.assert_called_once()- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_patch_request(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing PATCH request."""# Arrange@@ -1355,7 +1355,7 @@ class TestExternalDatasetServiceProcessAPI:assert result == mock_responsemock_proxy.patch.assert_called_once()- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_head_request(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing HEAD request."""# Arrange@@ -1380,7 +1380,7 @@ class TestExternalDatasetServiceProcessAPI:with pytest.raises(Exception, match="Invalid http method"):ExternalDatasetService.process_external_api(settings, None)- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_with_files(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test processing request with file uploads."""# Arrange@@ -1399,7 +1399,7 @@ class TestExternalDatasetServiceProcessAPI:assert "files" in call_kwargsassert call_kwargs["files"] == files- @patch("services.external_knowledge_service.ssrf_proxy")+ @patch("services.knowledge.external.service.ssrf_proxy")def test_process_external_api_follow_redirects(self, mock_proxy, factory: ExternalDatasetServiceTestDataFactory):"""Test that follow_redirects is enabled."""# Arrange@@ -1663,7 +1663,7 @@ class TestExternalDatasetServiceCreateDataset:class TestExternalDatasetServiceFetchRetrieval:"""Test fetch_external_knowledge_retrieval operations."""- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_success_with_results(self, mock_process, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1722,7 +1722,7 @@ class TestExternalDatasetServiceFetchRetrieval:"tenant-123", "dataset-123", "query", {}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_empty_results(self, mock_process, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1743,7 +1743,7 @@ class TestExternalDatasetServiceFetchRetrieval:# Assertassert len(result) == 0- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_with_score_threshold(self, mock_process, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1777,7 +1777,7 @@ class TestExternalDatasetServiceFetchRetrieval:call_args = mock_process.call_args[0][0]assert call_args.params["retrieval_setting"]["score_threshold"] == 0.75- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_non_200_status_raises_exception(self, mock_process, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1809,7 +1809,7 @@ class TestExternalDatasetServiceFetchRetrieval:(503, "Service Unavailable: Maintenance mode"),],)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_various_error_status_codes(self,mock_process,@@ -1836,7 +1836,7 @@ class TestExternalDatasetServiceFetchRetrieval:tenant_id, dataset_id, "query", {"top_k": 5}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_empty_response_text(self, mock_process, factory: ExternalDatasetServiceTestDataFactory, sqlite_session: Session):@@ -1855,7 +1855,7 @@ class TestExternalDatasetServiceFetchRetrieval:"tenant-123", "dataset-123", "query", {"top_k": 5}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_invalid_json_response(self, mock_process, factory, sqlite_session: Session):@@ -1872,7 +1872,7 @@ class TestExternalDatasetServiceFetchRetrieval:"tenant-123", "dataset-123", "query", {"top_k": 5}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_invalid_success_payload_shape(self, mock_process, factory, sqlite_session: Session):@@ -1889,7 +1889,7 @@ class TestExternalDatasetServiceFetchRetrieval:"tenant-123", "dataset-123", "query", {"top_k": 5}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_invalid_records_shape(self, mock_process, factory, sqlite_session: Session):@@ -1906,7 +1906,7 @@ class TestExternalDatasetServiceFetchRetrieval:"tenant-123", "dataset-123", "query", {"top_k": 5}, session=sqlite_session)- @patch("services.external_knowledge_service.ExternalDatasetService.process_external_api")+ @patch("services.knowledge.external.service.ExternalDatasetService.process_external_api")def test_fetch_external_knowledge_retrieval_wraps_transport_errors(self, mock_process, factory, sqlite_session: Session):diff --git a/api/tests/unit_tests/services/test_metadata_bug_complete.py b/api/tests/unit_tests/services/test_metadata_bug_complete.pyindex 6b22281fb48..7c5f978642d 100644--- a/api/tests/unit_tests/services/test_metadata_bug_complete.py+++ b/api/tests/unit_tests/services/test_metadata_bug_complete.py@@ -1,13 +1,25 @@from pathlib import Pathfrom typing import cast-from unittest.mock import Mock+from unittest.mock import Mock, create_autospecimport pytest-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom models import Account, Tenant-from services.entities.knowledge_entities.knowledge_entities import MetadataArgs-from services.metadata_service import MetadataService+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.knowledge_entities import MetadataArgs+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetRef+++def _metadata_service(session: Session) -> MetadataService:+ return MetadataService(+ store=SQLAlchemyMetadataRepository(+ session_factory=sessionmaker(bind=session.get_bind(), expire_on_commit=False)+ ),+ dataset_access=create_autospec(DatasetAccess, instance=True),+ )def _make_account(account_id: str = "user-456", tenant_id: str = "tenant-123") -> Account:@@ -50,15 +62,17 @@ class TestMetadataBugCompleteValidation:account = _make_account()# Should crash with TypeErrorwith pytest.raises(TypeError, match="object of type 'NoneType' has no len"):- MetadataService.create_metadata(- "dataset-123", mock_metadata_args, account, "tenant-123", session=sqlite_session+ _metadata_service(sqlite_session).create_metadata(+ DatasetRef("tenant-123", "dataset-123"), mock_metadata_args, actor_id=account.id)# Test update method as wellaccount = _make_account()none_name = cast(str, None)with pytest.raises(TypeError, match="object of type 'NoneType' has no len"):- MetadataService.update_metadata_name(Mock(), "metadata-456", none_name, account, session=sqlite_session)+ _metadata_service(sqlite_session).update_metadata_name(+ DatasetRef(Mock().tenant_id, Mock().id), "metadata-456", none_name, actor_id=account.id+ )assert not sqlite_session.in_transaction()def test_3_database_constraints_verification(self) -> None:@@ -103,8 +117,8 @@ class TestMetadataBugCompleteValidation:account = _make_account()with pytest.raises(TypeError, match="object of type 'NoneType' has no len"):- MetadataService.create_metadata(- "dataset-123", mock_metadata_args, account, "tenant-123", session=sqlite_session+ _metadata_service(sqlite_session).create_metadata(+ DatasetRef("tenant-123", "dataset-123"), mock_metadata_args, actor_id=account.id)assert not sqlite_session.in_transaction()diff --git a/api/tests/unit_tests/services/test_metadata_nullable_bug.py b/api/tests/unit_tests/services/test_metadata_nullable_bug.pyindex 4285c6f40d9..0347cee0263 100644--- a/api/tests/unit_tests/services/test_metadata_nullable_bug.py+++ b/api/tests/unit_tests/services/test_metadata_nullable_bug.py@@ -1,12 +1,25 @@from typing import cast-from unittest.mock import Mock+from unittest.mock import Mock, create_autospecimport pytest-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom models import Account, Tenant-from services.entities.knowledge_entities.knowledge_entities import MetadataArgs-from services.metadata_service import MetadataService+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.knowledge_entities import MetadataArgs+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetRef+++def _metadata_service(session: Session) -> MetadataService:+ return MetadataService(+ store=SQLAlchemyMetadataRepository(+ session_factory=sessionmaker(bind=session.get_bind(), expire_on_commit=False)+ ),+ dataset_access=create_autospec(DatasetAccess, instance=True),+ )+pytestmark = [pytest.mark.usefixtures("sqlite_session"),@@ -43,8 +56,8 @@ class TestMetadataNullableBug:account = _make_account()# This should crash with TypeError when calling len(None)with pytest.raises(TypeError, match="object of type 'NoneType' has no len"):- MetadataService.create_metadata(- "dataset-123", mock_metadata_args, account, "tenant-123", session=sqlite_session+ _metadata_service(sqlite_session).create_metadata(+ DatasetRef("tenant-123", "dataset-123"), mock_metadata_args, actor_id=account.id)assert not sqlite_session.in_transaction()@@ -54,7 +67,9 @@ class TestMetadataNullableBug:none_name = cast(str, None)# This should crash with TypeError when calling len(None)with pytest.raises(TypeError, match="object of type 'NoneType' has no len"):- MetadataService.update_metadata_name(Mock(), "metadata-456", none_name, account, session=sqlite_session)+ _metadata_service(sqlite_session).update_metadata_name(+ DatasetRef(Mock().tenant_id, Mock().id), "metadata-456", none_name, actor_id=account.id+ )assert not sqlite_session.in_transaction()def test_api_layer_now_uses_pydantic_validation(self) -> None:diff --git a/api/tests/unit_tests/services/test_metadata_service_session_boundary.py b/api/tests/unit_tests/services/test_metadata_service_session_boundary.pyindex eea773da175..2f3a6951576 100644--- a/api/tests/unit_tests/services/test_metadata_service_session_boundary.py+++ b/api/tests/unit_tests/services/test_metadata_service_session_boundary.py@@ -1,293 +1,197 @@+"""Metadata operations persist through scoped, independently owned transactions."""++from collections.abc import Iteratorfrom datetime import datetime-from unittest.mock import patch+from unittest.mock import create_autospecimport pytestfrom sqlalchemy import event, selectfrom sqlalchemy.orm import Session, sessionmaker-from core.rag.index_processor.constant.built_in_field import BuiltInField-from models import Account-from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Document+from models.dataset import DatasetMetadataBinding, Documentfrom models.enums import DocumentCreatedFrom-from services.dataset_service import DocumentService-from services.entities.knowledge_entities.knowledge_entities import (+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository+from services.errors.metadata import MetadataResourceNotFoundError+from services.knowledge.dataset_access import DatasetAccess+from services.knowledge.entities.knowledge_entities import (DocumentMetadataOperation,MetadataArgs,MetadataDetail,MetadataOperationData,)-from services.errors.metadata import MetadataResourceNotFoundError-from services.metadata_service import MetadataService+from services.knowledge.metadata.application import MetadataService+from services.knowledge.resource_scope import DatasetReffrom tests.unit_tests.model_factories import make_account, make_dataset, make_document+REF = DatasetRef("tenant-1", "dataset-1")DOCUMENT_ID = "11111111-1111-1111-1111-111111111111"-FOREIGN_DOCUMENT_ID = "22222222-2222-2222-2222-222222222222"-METADATA_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"-FOREIGN_METADATA_ID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"---def _account() -> Account:- return make_account(name="User", email="[email protected]")---def test_create_metadata_flushes_without_committing_caller_session(sqlite_session: Session) -> None:- transaction_events: list[str] = []- event.listen(sqlite_session, "after_commit", lambda _session: transaction_events.append("commit"))- event.listen(sqlite_session, "after_rollback", lambda _session: transaction_events.append("rollback"))+FOREIGN_ID = "22222222-2222-2222-2222-222222222222"- metadata = MetadataService.create_metadata(- "dataset-1",- MetadataArgs(type="string", name="author"),- _account(),- "tenant-1",- session=sqlite_session,- )-- assert metadata.name == "author"- assert sqlite_session.get(DatasetMetadata, metadata.id) is metadata- assert transaction_events == []--def _dataset(*, built_in_field_enabled: bool) -> Dataset:- return make_dataset(- description="",- provider="vendor",- built_in_field_enabled=built_in_field_enabled,- )---def _document() -> Document:- return make_document(- document_id=DOCUMENT_ID,- created_from=DocumentCreatedFrom.API,- created_at=datetime(2026, 1, 1),- updated_at=datetime(2026, 1, 2),- doc_metadata={},+def metadata_service(sqlite_session_factory: sessionmaker[Session]) -> Iterator[MetadataService]:+ with sqlite_session_factory.begin() as session:+ account = make_account(account_id="actor", name="User", email="[email protected]")+ session.add(account)+ for ref, document_id in [(REF, DOCUMENT_ID), (DatasetRef("tenant-2", "dataset-2"), FOREIGN_ID)]:+ session.add(+ make_dataset(dataset_id=ref.dataset_id, tenant_id=ref.tenant_id, name="Dataset", created_by="actor")+ )+ document = make_document(+ document_id=document_id,+ tenant_id=ref.tenant_id,+ dataset_id=ref.dataset_id,+ batch="batch",+ created_from=DocumentCreatedFrom.API,+ created_by="actor",+ created_at=datetime(2026, 1, 1),+ updated_at=datetime(2026, 1, 2),+ )+ document.enabled = True+ document.archived = False+ document.indexing_status = "completed"+ session.add(document)+ return MetadataService(+ store=SQLAlchemyMetadataRepository(session_factory=sqlite_session_factory),+ dataset_access=create_autospec(DatasetAccess, instance=True),)-def test_enable_built_in_field_uses_caller_session_for_uploader(sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=False)- document = _document()- sqlite_session.add_all([_account(), dataset, document])- sqlite_session.commit()-- with (- patch.object(MetadataService, "knowledge_base_metadata_lock_check"),- patch.object(DocumentService, "get_working_documents_by_dataset_id", return_value=[document]),- patch("services.metadata_service.redis_client.delete"),- ):- MetadataService.enable_built_in_field(dataset, sqlite_session)-- assert document.doc_metadata[BuiltInField.uploader] == "User"+def test_create_commits_through_owned_session(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session]+) -> None:+ result = metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")+ fields = metadata_service.get_dataset_metadatas(REF)+ assert fields["doc_metadata"] == [{"id": result.id, "type": "string", "name": "author", "count": 0}]+ with sqlite_session_factory() as session:+ assert session.scalar(select(DatasetMetadataBinding)) is None-def test_update_documents_metadata_uses_caller_session_for_uploader(sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=True)- document = _document()- sqlite_session.add_all([_account(), dataset, document])- sqlite_session.commit()- metadata_args = MetadataOperationData(- operation_data=[- DocumentMetadataOperation(document_id=document.id, metadata_list=[], partial_update=False),- ]- )[email protected]("name", ["x" * 256, "document_name", "uploader", "upload_date", "last_update_date", "source"])+def test_invalid_name_is_rejected(metadata_service: MetadataService, name: str) -> None:+ with pytest.raises(ValueError):+ metadata_service.create_metadata(REF, MetadataArgs(type="string", name=name), actor_id="actor")+ assert metadata_service.get_dataset_metadatas(REF)["doc_metadata"] == []- with (- patch.object(MetadataService, "knowledge_base_metadata_lock_check"),- patch("services.metadata_service.redis_client.delete"),- ):- MetadataService.update_documents_metadata(- dataset,- metadata_args,- _account(),- session=sqlite_session,- )- assert document.doc_metadata[BuiltInField.uploader] == "User"+def test_duplicate_name_is_rejected(metadata_service: MetadataService) -> None:+ metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")+ with pytest.raises(ValueError, match="already exists"):+ metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")-def test_update_documents_metadata_rejects_foreign_metadata_before_writes(sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=False)- document = _document()- foreign_metadata = DatasetMetadata(- tenant_id="tenant-2",- dataset_id="dataset-2",- type="string",- name="spoofed",- created_by="account-2",- )- foreign_metadata.id = FOREIGN_METADATA_ID- sqlite_session.add_all([_account(), dataset, document, foreign_metadata])- sqlite_session.commit()- transaction_events: list[str] = []- event.listen(sqlite_session, "after_flush", lambda _session, _context: transaction_events.append("flush"))- event.listen(sqlite_session, "after_commit", lambda _session: transaction_events.append("commit"))- metadata_args = MetadataOperationData(+def _operation(document_id: str, metadata_id: str, *, partial: bool = False) -> MetadataOperationData:+ return MetadataOperationData(operation_data=[DocumentMetadataOperation(- document_id=DOCUMENT_ID,- metadata_list=[MetadataDetail(id=FOREIGN_METADATA_ID, name="spoofed", value="value")],- partial_update=False,+ document_id=document_id,+ metadata_list=[MetadataDetail(id=metadata_id, name="spoofed", value="Alice")],+ partial_update=partial,)])- with (- pytest.raises(MetadataResourceNotFoundError, match="Metadata not found"),- patch.object(MetadataService, "knowledge_base_metadata_lock_check") as lock_check,- ):- MetadataService.update_documents_metadata(dataset, metadata_args, _account(), session=sqlite_session)-- lock_check.assert_not_called()- assert transaction_events == []- assert sqlite_session.scalars(select(DatasetMetadataBinding)).all() == []- assert sqlite_session.get(Document, DOCUMENT_ID).doc_metadata == {}---def test_update_documents_metadata_validates_all_documents_before_writes(sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=False)- document = _document()- metadata = DatasetMetadata(- tenant_id=dataset.tenant_id,- dataset_id=dataset.id,- type="string",- name="canonical",- created_by="account-1",- )- metadata.id = METADATA_ID- sqlite_session.add_all([_account(), dataset, document, metadata])- sqlite_session.commit()- transaction_events: list[str] = []- event.listen(sqlite_session, "after_flush", lambda _session, _context: transaction_events.append("flush"))- event.listen(sqlite_session, "after_commit", lambda _session: transaction_events.append("commit"))- metadata_detail = MetadataDetail(id=metadata.id, name="spoofed", value="value")- metadata_args = MetadataOperationData(- operation_data=[- DocumentMetadataOperation(document_id=DOCUMENT_ID, metadata_list=[metadata_detail], partial_update=False),- DocumentMetadataOperation(- document_id=FOREIGN_DOCUMENT_ID, metadata_list=[metadata_detail], partial_update=False- ),- ]- )-- with (- pytest.raises(MetadataResourceNotFoundError, match="Document not found"),- patch.object(MetadataService, "knowledge_base_metadata_lock_check") as lock_check,- patch("services.metadata_service.redis_client.delete"),- ):- MetadataService.update_documents_metadata(dataset, metadata_args, _account(), session=sqlite_session)-- lock_check.assert_not_called()- assert transaction_events == []- assert sqlite_session.scalars(select(DatasetMetadataBinding)).all() == []- assert sqlite_session.get(Document, DOCUMENT_ID).doc_metadata == {}--def test_update_documents_metadata_uses_canonical_metadata_name(- sqlite_session: Session, sqlite_session_factory: sessionmaker[Session][email protected]("partial", [True, False])+def test_update_uses_canonical_names_and_preserves_partial_values(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session], partial: bool) -> None:- dataset = _dataset(built_in_field_enabled=False)- document = _document()- metadata = DatasetMetadata(- tenant_id=dataset.tenant_id,- dataset_id=dataset.id,- type="string",- name="canonical",- created_by="account-1",- )- metadata.id = METADATA_ID- sqlite_session.add_all([_account(), dataset, document, metadata])- sqlite_session.commit()- metadata_args = MetadataOperationData(- operation_data=[- DocumentMetadataOperation(- document_id=document.id,- metadata_list=[MetadataDetail(id=metadata.id, name="spoofed", value="value")],- partial_update=False,- )- ]+ field = metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")+ with sqlite_session_factory.begin() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ document.doc_metadata = {"existing": "kept"}+ metadata_service.update_documents_metadata(+ REF, _operation(DOCUMENT_ID, field.id, partial=partial), actor_id="actor")-- with (- patch.object(MetadataService, "knowledge_base_metadata_lock_check"),- patch("services.metadata_service.redis_client.delete"),- ):- MetadataService.update_documents_metadata(dataset, metadata_args, _account(), session=sqlite_session)-- with sqlite_session_factory() as observer:- persisted_document = observer.get(Document, document.id)- assert persisted_document is not None- assert persisted_document.doc_metadata == {"canonical": "value"}- binding = observer.scalar(select(DatasetMetadataBinding))- assert binding is not None- assert binding.metadata_id == metadata.id---def test_metadata_operation_normalizes_uuid_ids() -> None:- operation = DocumentMetadataOperation(- document_id=DOCUMENT_ID.upper(),- metadata_list=[MetadataDetail(id=METADATA_ID.upper(), name="ignored", value="value")],+ metadata_service.update_documents_metadata(+ REF, _operation(DOCUMENT_ID, field.id, partial=partial), actor_id="actor")+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ assert document.doc_metadata == ({"existing": "kept", "author": "Alice"} if partial else {"author": "Alice"})+ assert len(session.scalars(select(DatasetMetadataBinding)).all()) == 1- assert operation.document_id == DOCUMENT_ID- assert operation.metadata_list[0].id == METADATA_ID--def test_document_metadata_details_scopes_binding_to_document_owner(sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=False)- document = _document()- document.doc_metadata = {"canonical": "value"}- foreign_metadata = DatasetMetadata(- tenant_id="tenant-2",- dataset_id="dataset-2",- type="string",- name="canonical",- created_by="account-2",- )- foreign_metadata.id = FOREIGN_METADATA_ID- foreign_binding = DatasetMetadataBinding(- tenant_id="tenant-2",- dataset_id="dataset-2",- document_id=document.id,- metadata_id=foreign_metadata.id,- created_by="account-2",[email protected]("foreign", ["dataset", "document", "metadata"])+def test_owner_chain_is_validated_before_any_batch_write(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session], foreign: str+) -> None:+ field = metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")+ other = metadata_service.create_metadata(+ DatasetRef("tenant-2", "dataset-2"), MetadataArgs(type="string", name="foreign"), actor_id="actor")- sqlite_session.add_all([_account(), dataset, document, foreign_metadata, foreign_binding])- sqlite_session.commit()-- details = document.get_doc_metadata_details(session=sqlite_session)-- assert details is not None- assert [detail for detail in details if detail["id"] != "built-in"] == []---def test_get_dataset_metadatas_uses_caller_session(monkeypatch, sqlite_session: Session) -> None:- dataset = _dataset(built_in_field_enabled=False)- sqlite_session.add_all(- [- DatasetMetadataBinding(- tenant_id="tenant-1",- dataset_id="dataset-1",- document_id=f"document-{index}",- metadata_id="metadata-1",- created_by="account-1",- )- for index in range(2)- ]+ args = _operation(DOCUMENT_ID, field.id)+ args.operation_data.extend(+ _operation(+ FOREIGN_ID if foreign == "document" else DOCUMENT_ID, other.id if foreign == "metadata" else field.id+ ).operation_data)- sqlite_session.commit()-- def get_doc_metadata(_dataset: Dataset, *, session: Session) -> list[dict[str, str]]:- assert session is sqlite_session- return [{"id": "metadata-1", "name": "author", "type": "string"}]-- monkeypatch.setattr(Dataset, "get_doc_metadata", get_doc_metadata)+ ref = DatasetRef("tenant-2", REF.dataset_id) if foreign == "dataset" else REF+ with pytest.raises(MetadataResourceNotFoundError):+ metadata_service.update_documents_metadata(ref, args, actor_id="actor")+ with sqlite_session_factory() as session:+ assert all(document.doc_metadata is None for document in session.scalars(select(Document)))+ assert not session.scalars(select(DatasetMetadataBinding)).all()- result = MetadataService.get_dataset_metadatas(dataset, sqlite_session)- assert result == {- "doc_metadata": [{"id": "metadata-1", "name": "author", "type": "string", "count": 2}],- "built_in_field_enabled": False,- }- assert sqlite_session.scalar(select(DatasetMetadataBinding).limit(1)) is not None+def test_rename_delete_and_counts_share_scoped_bindings(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session]+) -> None:+ field = metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")+ metadata_service.update_documents_metadata(REF, _operation(DOCUMENT_ID, field.id), actor_id="actor")+ renamed = metadata_service.update_metadata_name(REF, field.id, "writer", actor_id="actor")+ assert renamed.name == "writer"+ assert metadata_service.get_dataset_metadatas(REF)["doc_metadata"] == [+ {"id": field.id, "name": "writer", "type": "string", "count": 1}+ ]+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ assert document.doc_metadata == {"writer": "Alice"}+ metadata_service.delete_metadata(REF, field.id)+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ assert document.doc_metadata == {}+ assert not session.scalars(select(DatasetMetadataBinding)).all()+++def test_builtin_fields_use_owned_document_uploader(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session]+) -> None:+ metadata_service.enable_built_in_field(REF)+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ foreign = session.get(Document, FOREIGN_ID)+ assert document is not None+ assert foreign is not None+ assert document.doc_metadata["uploader"] == "User"+ assert foreign.doc_metadata is None+ metadata_service.disable_built_in_field(REF)+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ assert document.doc_metadata == {}+++def test_commit_failure_rolls_back_document_and_bindings(+ metadata_service: MetadataService, sqlite_session_factory: sessionmaker[Session]+) -> None:+ field = metadata_service.create_metadata(REF, MetadataArgs(type="string", name="author"), actor_id="actor")++ def fail(_session: Session) -> None:+ raise RuntimeError("commit failed")++ event.listen(sqlite_session_factory.class_, "before_commit", fail)+ try:+ with pytest.raises(RuntimeError, match="commit failed"):+ metadata_service.update_documents_metadata(REF, _operation(DOCUMENT_ID, field.id), actor_id="actor")+ finally:+ event.remove(sqlite_session_factory.class_, "before_commit", fail)+ with sqlite_session_factory() as session:+ document = session.get(Document, DOCUMENT_ID)+ assert document is not None+ assert document.doc_metadata is None+ assert not session.scalars(select(DatasetMetadataBinding)).all()diff --git a/api/tests/unit_tests/services/test_model_provider_service.py b/api/tests/unit_tests/services/test_model_provider_service.pyindex ce2dd7c1bcb..6659d5ac7d0 100644--- a/api/tests/unit_tests/services/test_model_provider_service.py+++ b/api/tests/unit_tests/services/test_model_provider_service.py@@ -1,6 +1,6 @@from types import SimpleNamespacefrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import MagicMock, create_autospecimport pytestfrom sqlalchemy import event@@ -15,6 +15,7 @@ from enums import DeploymentEditionfrom graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.model_entities import FetchFrom, ModelType, ParameterRule, ParameterTypefrom graphon.model_runtime.entities.provider_entities import ConfigurateMethod+from models.account import Account, TenantAccountRolefrom models.provider import (Provider,ProviderCredential,@@ -23,9 +24,10 @@ from models.provider import (TenantDefaultModel,TenantPreferredModelProvider,)-from services import model_provider_service as service_module+from services.credentials.query import CredentialQuery, ModelCredentialRecordfrom services.errors.app_model_config import ProviderNotFoundError-from services.model_provider_service import ModelProviderService, _ProviderSummaryState+from services.model_provider import service as service_module+from services.model_provider.service import ModelProviderService, _ProviderSummaryStatedef _create_service_with_mocked_manager() -> tuple[ModelProviderService, MagicMock]:@@ -35,6 +37,21 @@ def _create_service_with_mocked_manager() -> tuple[ModelProviderService, MagicMoreturn service, manager+def test_available_credentials_use_injected_query_for_admin_viewer() -> None:+ service, manager = _create_service_with_mocked_manager()+ user = Account(name="Admin", email="[email protected]")+ user.id = "actor"+ user.role = TenantAccountRole.ADMIN+ query = create_autospec(CredentialQuery, instance=True, spec_set=True)+ query.list_models.return_value = [ModelCredentialRecord(id="credential", name="Shared credential")]++ result = service.get_provider_available_credentials("tenant", "openai", user=user, credential_query=query)++ query.list_models.assert_called_once_with(workspace_id="tenant", provider="openai", actor_id="actor")+ assert result == [CredentialConfiguration(credential_id="credential", credential_name="Shared credential")]+ assert manager.mock_calls == []++def _build_provider_configuration(*,provider_name: str = "openai",diff --git a/api/tests/unit_tests/services/test_model_provider_service_sanitization.py b/api/tests/unit_tests/services/test_model_provider_service_sanitization.pyindex 97f3bd6f013..1c75b0e059b 100644--- a/api/tests/unit_tests/services/test_model_provider_service_sanitization.py+++ b/api/tests/unit_tests/services/test_model_provider_service_sanitization.py@@ -7,7 +7,7 @@ from graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.model_entities import ModelTypefrom graphon.model_runtime.entities.provider_entities import ConfigurateMethodfrom models.provider import ProviderType-from services.model_provider_service import ModelProviderService+from services.model_provider.service import ModelProviderServiceclass _FakeConfigurations:diff --git a/api/tests/unit_tests/services/test_notion_data_source_gateway.py b/api/tests/unit_tests/services/test_notion_data_source_gateway.pydeleted file mode 100644index b3754eb799b..00000000000--- a/api/tests/unit_tests/services/test_notion_data_source_gateway.py+++ /dev/null@@ -1,156 +0,0 @@-import json-from unittest.mock import MagicMock--import httpx-import pytest--from core.helper.ssrf_proxy import SSRFProxy-from core.tools.errors import ToolSSRFError-from services.data_source_oauth_service import DataSourceOAuthError-from services.notion_data_source_gateway import NotionDataSourceGateway---def _gateway(handler: httpx.MockTransport) -> tuple[NotionDataSourceGateway, httpx.Client]:- client = httpx.Client(transport=handler)- return (- NotionDataSourceGateway(- client_id="client-id",- client_secret="client-secret",- redirect_uri="https://api.example/oauth/data-source/callback/notion",- http_client=client,- ),- client,- )---def test_get_authorization_url_encodes_configured_values() -> None:- gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(500)))- try:- result = gateway.get_authorization_url()- finally:- client.close()-- assert result == (- "https://api.notion.com/v1/oauth/authorize?client_id=client-id&response_type=code&"- "redirect_uri=https%3A%2F%2Fapi.example%2Foauth%2Fdata-source%2Fcallback%2Fnotion&owner=user"- )---def test_authorize_exchanges_code_and_builds_source_info_without_persistence() -> None:- search_types: list[str] = []-- def handler(request: httpx.Request) -> httpx.Response:- if request.url.path == "/v1/oauth/token":- assert request.headers["authorization"].startswith("Basic ")- return httpx.Response(- 200,- json={- "access_token": "notion-token",- "workspace_name": "Notion Workspace",- "workspace_icon": "https://example/icon.png",- "workspace_id": "notion-workspace",- },- )- if request.url.path == "/v1/search":- payload = json.loads(request.content)- search_types.append(payload["filter"]["value"])- return httpx.Response(200, json={"results": [], "has_more": False})- raise AssertionError(f"Unexpected Notion request: {request.url}")-- gateway, client = _gateway(httpx.MockTransport(handler))- try:- authorization = gateway.authorize("authorization-code")- finally:- client.close()-- assert authorization.access_token == "notion-token"- assert authorization.source_info == {- "workspace_name": "Notion Workspace",- "workspace_icon": "https://example/icon.png",- "workspace_id": "notion-workspace",- "pages": [],- "total": 0,- }- assert search_types == ["page", "database"]---def test_refresh_preserves_workspace_metadata() -> None:- gateway, client = _gateway(- httpx.MockTransport(- lambda request: (- httpx.Response(200, json={"results": [], "has_more": False})- if request.url.path == "/v1/search"- else httpx.Response(404)- )- )- )- try:- source_info = gateway.refresh(- "notion-token",- {- "workspace_name": "Workspace",- "workspace_icon": None,- "workspace_id": "notion-workspace",- "pages": [- {- "page_id": "old",- "page_name": "Old page",- "page_icon": None,- "parent_id": "root",- "type": "page",- }- ],- "total": 1,- },- )- finally:- client.close()-- assert source_info == {- "workspace_name": "Workspace",- "workspace_icon": None,- "workspace_id": "notion-workspace",- "pages": [],- "total": 0,- }---def test_search_rejects_missing_cursor_when_more_results_are_declared() -> None:- requests = 0-- def handler(_request: httpx.Request) -> httpx.Response:- nonlocal requests- requests += 1- return httpx.Response(200, json={"results": [], "has_more": True, "next_cursor": None})-- gateway, client = _gateway(httpx.MockTransport(handler))- try:- with pytest.raises(DataSourceOAuthError, match="valid next_cursor"):- gateway.notion_page_search("notion-token")- finally:- client.close()-- assert requests == 1---def test_provider_http_failure_is_translated_to_application_boundary_error() -> None:- gateway, client = _gateway(httpx.MockTransport(lambda _request: httpx.Response(503, text="unavailable")))- try:- with pytest.raises(DataSourceOAuthError, match="unsuccessful response"):- gateway.authorize("authorization-code")- finally:- client.close()---def test_ssrf_rejection_is_translated_to_application_boundary_error() -> None:- client = MagicMock(spec=SSRFProxy)- client.post.side_effect = ToolSSRFError("blocked")- gateway = NotionDataSourceGateway(- client_id="client-id",- client_secret="client-secret",- redirect_uri="https://api.example/oauth/data-source/callback/notion",- http_client=client,- )-- with pytest.raises(DataSourceOAuthError, match="Notion request failed"):- gateway.authorize("authorization-code")diff --git a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.pyindex 2b73d643387..0905c78deba 100644--- a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py+++ b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py@@ -226,14 +226,13 @@ class TestRagPipelineTaskProxy:assert parsed_json[0]["pipeline_id"] == "pipeline-1"assert parsed_json[1]["pipeline_id"] == "pipeline-2"- @patch("services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task")- def test_send_to_direct_queue(self, mock_task: MagicMock):+ def test_send_to_direct_queue(self):"""Test _send_to_direct_queue method."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()proxy._tenant_isolated_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue()upload_file_id = "file-123"- mock_task.delay = Mock()+ mock_task = Mock()# Actproxy._send_to_direct_queue(upload_file_id, mock_task)@@ -246,8 +245,7 @@ class TestRagPipelineTaskProxy:rag_pipeline_invoke_entities_file_id=upload_file_id, tenant_id="tenant-123")- @patch("services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task")- def test_send_to_tenant_queue_with_existing_task_key(self, mock_task: MagicMock):+ def test_send_to_tenant_queue_with_existing_task_key(self):"""Test _send_to_tenant_queue when task key exists."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()@@ -255,7 +253,7 @@ class TestRagPipelineTaskProxy:has_task_key=True)upload_file_id = "file-123"- mock_task.delay = Mock()+ mock_task = Mock()# Actproxy._send_to_tenant_queue(upload_file_id, mock_task)@@ -265,8 +263,7 @@ class TestRagPipelineTaskProxy:# Celery should not be called directlymock_task.delay.assert_not_called()- @patch("services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task")- def test_send_to_tenant_queue_without_task_key(self, mock_task: MagicMock):+ def test_send_to_tenant_queue_without_task_key(self):"""Test _send_to_tenant_queue when no task key exists."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()@@ -274,7 +271,7 @@ class TestRagPipelineTaskProxy:has_task_key=False)upload_file_id = "file-123"- mock_task.delay = Mock()+ mock_task = Mock()# Actproxy._send_to_tenant_queue(upload_file_id, mock_task)@@ -288,8 +285,7 @@ class TestRagPipelineTaskProxy:# The first task should be sent to celery directly, so push tasks should not be calledproxy._tenant_isolated_task_queue.push_tasks.assert_not_called()- @patch("services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task")- def test_send_to_default_tenant_queue(self, mock_task: MagicMock):+ def test_send_to_default_tenant_queue(self):"""Test _send_to_default_tenant_queue method."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()@@ -300,10 +296,13 @@ class TestRagPipelineTaskProxy:proxy._send_to_default_tenant_queue(upload_file_id)# Assert- proxy._send_to_tenant_queue.assert_called_once_with(upload_file_id, mock_task)+ proxy._send_to_tenant_queue.assert_called_once()+ sent_file_id, task = proxy._send_to_tenant_queue.call_args.args+ assert sent_file_id == upload_file_id+ assert task.task == "tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task"+ assert task.options["queue"] == "pipeline"- @patch("services.rag_pipeline.rag_pipeline_task_proxy.priority_rag_pipeline_run_task")- def test_send_to_priority_tenant_queue(self, mock_task: MagicMock):+ def test_send_to_priority_tenant_queue(self):"""Test _send_to_priority_tenant_queue method."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()@@ -314,10 +313,13 @@ class TestRagPipelineTaskProxy:proxy._send_to_priority_tenant_queue(upload_file_id)# Assert- proxy._send_to_tenant_queue.assert_called_once_with(upload_file_id, mock_task)+ proxy._send_to_tenant_queue.assert_called_once()+ sent_file_id, task = proxy._send_to_tenant_queue.call_args.args+ assert sent_file_id == upload_file_id+ assert task.task == "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task"+ assert task.options["queue"] == "priority_pipeline"- @patch("services.rag_pipeline.rag_pipeline_task_proxy.priority_rag_pipeline_run_task")- def test_send_to_priority_direct_queue(self, mock_task: MagicMock):+ def test_send_to_priority_direct_queue(self):"""Test _send_to_priority_direct_queue method."""# Arrangeproxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()@@ -328,7 +330,11 @@ class TestRagPipelineTaskProxy:proxy._send_to_priority_direct_queue(upload_file_id)# Assert- proxy._send_to_direct_queue.assert_called_once_with(upload_file_id, mock_task)+ proxy._send_to_direct_queue.assert_called_once()+ sent_file_id, task = proxy._send_to_direct_queue.call_args.args+ assert sent_file_id == upload_file_id+ assert task.task == "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task"+ assert task.options["queue"] == "priority_pipeline"@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)@patch("services.rag_pipeline.rag_pipeline_task_proxy.FeatureService")diff --git a/api/tests/unit_tests/services/test_rbac_resource_service.py b/api/tests/unit_tests/services/test_rbac_resource_service.pyindex d93834b8eb6..a11eddcbd9b 100644--- a/api/tests/unit_tests/services/test_rbac_resource_service.py+++ b/api/tests/unit_tests/services/test_rbac_resource_service.py@@ -1,11 +1,13 @@"""RBACResourceService reads through the session its caller passes in (#37403)."""+import pytestfrom sqlalchemy.orm import Sessionfrom models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatusfrom models.dataset import Datasetfrom models.model import App, AppModefrom services.rbac_resource_service import RBACResourceService+from tests.unit_tests.model_factories import make_documentTENANT_ID = "tenant-1"OTHER_TENANT_ID = "tenant-2"@@ -133,3 +135,30 @@ class TestGetDatasetIdByPipeline:sqlite_session.flush()assert RBACResourceService.get_dataset_id_by_pipeline(sqlite_session, TENANT_ID, "pipeline-1") is None+++class TestGetDatasetIdByDocument:+ @pytest.mark.parametrize(+ ("dataset_tenant", "document_tenant", "expected"),+ [+ (TENANT_ID, TENANT_ID, "dataset-1"),+ (OTHER_TENANT_ID, OTHER_TENANT_ID, None),+ (OTHER_TENANT_ID, TENANT_ID, None),+ (TENANT_ID, OTHER_TENANT_ID, None),+ ],+ )+ def test_scopes_both_dataset_and_document(+ self, sqlite_session: Session, dataset_tenant: str, document_tenant: str, expected: str | None+ ) -> None:+ sqlite_session.add_all(+ [+ _dataset(dataset_id="dataset-1", tenant_id=dataset_tenant),+ make_document(document_id="doc-1", dataset_id="dataset-1", tenant_id=document_tenant),+ ]+ )+ sqlite_session.flush()++ assert RBACResourceService.get_dataset_id_by_document(sqlite_session, TENANT_ID, "doc-1") == expected++ def test_missing_document(self, sqlite_session: Session) -> None:+ assert RBACResourceService.get_dataset_id_by_document(sqlite_session, TENANT_ID, "missing") is Nonediff --git a/api/tests/unit_tests/services/test_summary_index_service.py b/api/tests/unit_tests/services/test_summary_index_service.pyindex 4dab7608964..c8fb1a36e65 100644--- a/api/tests/unit_tests/services/test_summary_index_service.py+++ b/api/tests/unit_tests/services/test_summary_index_service.py@@ -1,4 +1,4 @@-"""SQLite-backed tests for :mod:`services.summary_index_service`."""+"""SQLite-backed tests for :mod:`services.knowledge.summaries.adapters`."""from __future__ import annotations@@ -11,11 +11,11 @@ import pytestfrom sqlalchemy import event, func, selectfrom sqlalchemy.orm import Session, sessionmaker-import services.summary_index_service as summary_module+import services.knowledge.summaries.adapters as summary_modulefrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummaryfrom models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatus-from services.summary_index_service import SummaryIndexService+from services.knowledge.summaries.adapters import SummaryIndexAdapterTENANT_ID = "tenant-1"OTHER_TENANT_ID = "tenant-2"@@ -168,7 +168,7 @@ def _install_summary_generator(usage: MagicMock | None = None,) -> MagicMock:generate = MagicMock(return_value=(content, usage or _usage()))- paragraph_module = SimpleNamespace(ParagraphIndexProcessor=SimpleNamespace(generate_summary=generate))+ paragraph_module = SimpleNamespace(ParagraphIndexProcessor=SimpleNamespace(generate_summary_from_inputs=generate))monkeypatch.setitem(sys.modules,"core.rag.index_processor.processor.paragraph_index_processor",@@ -196,14 +196,21 @@ class TestGenerateAndCreate:usage = _usage()generate = _install_summary_generator(monkeypatch, usage=usage)- content, result_usage = SummaryIndexService.generate_summary_for_segment(+ def call_model(**kwargs):+ assert not sqlite_session.in_transaction()+ return "generated summary", usage++ generate.side_effect = call_model++ content, result_usage = SummaryIndexAdapter.generate_summary_for_segment(segment, dataset, {"enable": True}, session=sqlite_session)assert content == "generated summary"assert result_usage is usageassert generate.call_args.kwargs["document_language"] == "en"- assert generate.call_args.kwargs["session"] is sqlite_session+ assert callable(generate.call_args.kwargs["image_loader"])+ assert not sqlite_session.in_transaction()def test_generate_allows_missing_document_but_rejects_empty_summary(self, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch@@ -216,7 +223,7 @@ class TestGenerateAndCreate:_install_summary_generator(monkeypatch, content="")with pytest.raises(ValueError, match="Generated summary is empty"):- SummaryIndexService.generate_summary_for_segment(segment, dataset, {"enable": True}, session=sqlite_session)+ SummaryIndexAdapter.generate_summary_for_segment(segment, dataset, {"enable": True}, session=sqlite_session)def test_create_updates_only_matching_dataset_and_reenables(self, sqlite_session: Session) -> None:dataset, document, segment = _graph(sqlite_session)@@ -237,7 +244,7 @@ class TestGenerateAndCreate:existing.disabled_by = "account-2"sqlite_session.commit()- result = SummaryIndexService.create_summary_record(+ result = SummaryIndexAdapter.create_summary_record(segment, dataset, "new", status=SummaryStatus.GENERATING, session=sqlite_session)@@ -255,7 +262,7 @@ class TestGenerateAndCreate:def test_create_persists_new_record_in_current_transaction(self, sqlite_session: Session) -> None:dataset, document, segment = _graph(sqlite_session)- result = SummaryIndexService.create_summary_record(+ result = SummaryIndexAdapter.create_summary_record(segment, dataset, "new", status=SummaryStatus.NOT_STARTED, session=sqlite_session)@@ -279,7 +286,7 @@ class TestVectorizeSummary:vector_class = MagicMock()monkeypatch.setattr(summary_module, "Vector", vector_class)- SummaryIndexService.vectorize_summary(summary, segment, dataset, session=sqlite_session)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset, session=sqlite_session)vector_class.assert_not_called()@@ -288,19 +295,25 @@ class TestVectorizeSummary:summary = _persist_summary(sqlite_session, dataset, document, segment, content=" ")with pytest.raises(ValueError, match="Summary content is empty"):- SummaryIndexService.vectorize_summary(summary, segment, dataset, session=sqlite_session)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset, session=sqlite_session)- def test_provided_session_retries_and_flushes_mapped_record(+ def test_provided_session_releases_transaction_before_every_vector_attempt(self, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> None:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment)vector = _install_vector_dependencies(monkeypatch)- vector.add_texts.side_effect = [ConnectionError("connection timeout"), None]++ def write_vector(*args, **kwargs):+ assert not sqlite_session.in_transaction()+ if vector.add_texts.call_count == 1:+ raise ConnectionError("connection timeout")++ vector.add_texts.side_effect = write_vectorsleep = MagicMock()monkeypatch.setattr(summary_module.time, "sleep", sleep)- SummaryIndexService.vectorize_summary(summary, segment, dataset, session=sqlite_session)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset, session=sqlite_session)assert vector.add_texts.call_count == 2sleep.assert_called_once_with(2.0)@@ -321,7 +334,7 @@ class TestVectorizeSummary:summary = _persist_summary(sqlite_session, dataset, document, segment)_install_vector_dependencies(monkeypatch)- SummaryIndexService.vectorize_summary(summary, segment, dataset)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset)with sqlite_session_factory() as observer:stored = observer.get(DocumentSegmentSummary, summary.id)@@ -350,7 +363,7 @@ class TestVectorizeSummary:detached.id = "missing-id"_install_vector_dependencies(monkeypatch)- SummaryIndexService.vectorize_summary(detached, segment, dataset)+ SummaryIndexAdapter.vectorize_summary(detached, segment, dataset)with sqlite_session_factory() as observer:refreshed = observer.get(DocumentSegmentSummary, stored.id)@@ -375,14 +388,14 @@ class TestVectorizeSummary:detached.id = "new-summary"_install_vector_dependencies(monkeypatch)- SummaryIndexService.vectorize_summary(detached, segment, dataset)+ SummaryIndexAdapter.vectorize_summary(detached, segment, dataset)with sqlite_session_factory() as observer:stored = observer.get(DocumentSegmentSummary, detached.id)assert stored is not Noneassert stored.status == SummaryStatus.COMPLETED- def test_provided_session_failure_records_error_without_opening_another_session(+ def test_provided_session_is_committed_before_independent_error_persistence(self,sqlite_session: Session,monkeypatch: pytest.MonkeyPatch,@@ -391,22 +404,60 @@ class TestVectorizeSummary:summary = _persist_summary(sqlite_session, dataset, document, segment)vector = _install_vector_dependencies(monkeypatch)vector.add_texts.side_effect = RuntimeError("fatal vector failure")- create_session = MagicMock()+ create_session = MagicMock(wraps=summary_module.session_factory.create_session)monkeypatch.setattr(summary_module.session_factory, "create_session", create_session)with pytest.raises(RuntimeError, match="fatal vector failure"):- SummaryIndexService.vectorize_summary(summary, segment, dataset, session=sqlite_session)+ SummaryIndexAdapter.vectorize_summary(summary, segment, dataset, session=sqlite_session)- create_session.assert_not_called()+ assert create_session.call_count == 2+ assert not sqlite_session.in_transaction()assert summary.status == SummaryStatus.ERRORassert summary.error == "Vectorization failed: fatal vector failure"class TestBatchAndGeneration:+ @pytest.mark.parametrize("foreign_document", [False, True])+ def test_task_resolves_owned_document_and_releases_read_transaction(+ self,+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ foreign_document: bool,+ ) -> None:+ from tasks.generate_summary_index_task import generate_summary_index_task++ dataset, document, _ = _graph(sqlite_session)+ dataset.summary_index_setting = {"enable": True}+ document.need_summary = True+ if foreign_document:+ other = _persist_dataset(sqlite_session, dataset_id="other", tenant_id=OTHER_TENANT_ID)+ document.dataset_id, document.tenant_id = other.id, other.tenant_id+ sqlite_session.commit()+ sessions = []++ def create_session():+ session = sqlite_session_factory()+ sessions.append(session)+ return session++ def generate(**kwargs):+ assert sessions+ assert all(not session.in_transaction() for session in sessions)+ return []++ generate_mock = MagicMock(side_effect=generate)+ monkeypatch.setattr(summary_module.session_factory, "create_session", create_session)+ monkeypatch.setattr(SummaryIndexAdapter, "generate_summaries_for_document", generate_mock)++ generate_summary_index_task(dataset.id, document.id)++ assert generate_mock.call_count == (0 if foreign_document else 1)+def test_batch_create_no_segments_is_noop(self, sqlite_session: Session) -> None:dataset = _persist_dataset(sqlite_session)- SummaryIndexService.batch_create_summary_records([], dataset)+ SummaryIndexAdapter.batch_create_summary_records([], dataset)assert sqlite_session.scalar(select(func.count()).select_from(DocumentSegmentSummary)) == 0@@ -420,7 +471,7 @@ class TestBatchAndGeneration:existing.disabled_by = "account-2"sqlite_session.commit()- SummaryIndexService.batch_create_summary_records([first, second], dataset, status=SummaryStatus.NOT_STARTED)+ SummaryIndexAdapter.batch_create_summary_records([first, second], dataset, status=SummaryStatus.NOT_STARTED)with sqlite_session_factory() as observer:rows = observer.scalars(@@ -438,14 +489,14 @@ class TestBatchAndGeneration:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment)- SummaryIndexService.update_summary_record_error(segment, dataset, "generation failed")+ SummaryIndexAdapter.update_summary_record_error(segment, dataset, "generation failed")with sqlite_session_factory() as observer:stored = observer.get(DocumentSegmentSummary, summary.id)assert stored is not Noneassert (stored.status, stored.error) == (SummaryStatus.ERROR, "generation failed")other = _persist_segment(sqlite_session, dataset, document, segment_id="seg-2", position=2)- SummaryIndexService.update_summary_record_error(other, dataset, "ignored")+ SummaryIndexAdapter.update_summary_record_error(other, dataset, "ignored")assert sqlite_session.scalar(select(func.count()).select_from(DocumentSegmentSummary)) == 1def test_generate_and_vectorize_creates_commits_and_returns_row(@@ -453,7 +504,7 @@ class TestBatchAndGeneration:) -> None:dataset, _, segment = _graph(sqlite_session)monkeypatch.setattr(- SummaryIndexService,+ SummaryIndexAdapter,"generate_summary_for_segment",MagicMock(return_value=("generated", _usage())),)@@ -468,9 +519,9 @@ class TestBatchAndGeneration:record.summary_index_node_id = "node-1"sqlite_session.flush()- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", vectorize)+ monkeypatch.setattr(SummaryIndexAdapter, "vectorize_summary", vectorize)- result = SummaryIndexService.generate_and_vectorize_summary(+ result = SummaryIndexAdapter.generate_and_vectorize_summary(segment, dataset, {"enable": True}, session=sqlite_session)@@ -486,13 +537,13 @@ class TestBatchAndGeneration:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment, status=SummaryStatus.NOT_STARTED)monkeypatch.setattr(- SummaryIndexService,+ SummaryIndexAdapter,"generate_summary_for_segment",MagicMock(side_effect=RuntimeError("LLM failed")),)with pytest.raises(RuntimeError, match="LLM failed"):- SummaryIndexService.generate_and_vectorize_summary(+ SummaryIndexAdapter.generate_and_vectorize_summary(segment, dataset, {"enable": True}, session=sqlite_session)@@ -517,7 +568,7 @@ class TestBatchAndGeneration:dataset = _persist_dataset(sqlite_session, indexing_technique=technique)document = _persist_document(sqlite_session, dataset, doc_form=doc_form)- assert SummaryIndexService.generate_summaries_for_document(dataset, document, {"enable": enabled}) == []+ assert SummaryIndexAdapter.generate_summaries_for_document(dataset, document, {"enable": enabled}) == []def test_generate_document_filters_segments_and_continues_after_failure(self, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch@@ -534,10 +585,10 @@ class TestBatchAndGeneration:)generate = MagicMock(side_effect=[generated, RuntimeError("boom")])update_error = MagicMock()- monkeypatch.setattr(SummaryIndexService, "generate_and_vectorize_summary", generate)- monkeypatch.setattr(SummaryIndexService, "update_summary_record_error", update_error)+ monkeypatch.setattr(SummaryIndexAdapter, "generate_and_vectorize_summary", generate)+ monkeypatch.setattr(SummaryIndexAdapter, "update_summary_record_error", update_error)- result = SummaryIndexService.generate_summaries_for_document(+ result = SummaryIndexAdapter.generate_summaries_for_document(dataset,document,{"enable": True},@@ -564,7 +615,7 @@ class TestEnableDisableDelete:vector = MagicMock(name="vector")monkeypatch.setattr(summary_module, "Vector", MagicMock(return_value=vector))- SummaryIndexService.disable_summaries_for_segments(dataset, segment_ids=[first.id], disabled_by="account-2")+ SummaryIndexAdapter.disable_summaries_for_segments(dataset, segment_ids=[first.id], disabled_by="account-2")vector.delete_by_ids.assert_called_once_with(["node-1"])with sqlite_session_factory() as observer:@@ -586,7 +637,7 @@ class TestEnableDisableDelete:vector.delete_by_ids.side_effect = RuntimeError("vector unavailable")monkeypatch.setattr(summary_module, "Vector", MagicMock(return_value=vector))- SummaryIndexService.disable_summaries_for_segments(dataset)+ SummaryIndexAdapter.disable_summaries_for_segments(dataset)sqlite_session.refresh(summary)assert summary.enabled is False@@ -621,9 +672,9 @@ class TestEnableDisableDelete:sqlite_session.flush()vectorize_mock = MagicMock(side_effect=vectorize)- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", vectorize_mock)+ monkeypatch.setattr(SummaryIndexAdapter, "vectorize_summary", vectorize_mock)- SummaryIndexService.enable_summaries_for_segments(dataset)+ SummaryIndexAdapter.enable_summaries_for_segments(dataset)sqlite_session.refresh(good_summary)assert good_summary.enabled is True@@ -632,16 +683,16 @@ class TestEnableDisableDelete:def test_enable_skips_economy_dataset(self, sqlite_session: Session) -> None:dataset = _persist_dataset(sqlite_session, indexing_technique=IndexTechniqueType.ECONOMY)- SummaryIndexService.enable_summaries_for_segments(dataset)+ SummaryIndexAdapter.enable_summaries_for_segments(dataset)def test_enable_keeps_failed_summary_disabled(self, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> None:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment, enabled=False)- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", MagicMock(side_effect=RuntimeError("boom")))+ monkeypatch.setattr(SummaryIndexAdapter, "vectorize_summary", MagicMock(side_effect=RuntimeError("boom")))- SummaryIndexService.enable_summaries_for_segments(dataset)+ SummaryIndexAdapter.enable_summaries_for_segments(dataset)sqlite_session.refresh(summary)assert summary.enabled is False@@ -656,7 +707,7 @@ class TestEnableDisableDelete:vector = MagicMock(name="vector")monkeypatch.setattr(summary_module, "Vector", MagicMock(return_value=vector))- SummaryIndexService.delete_summaries_for_segments(dataset, segment_ids=[first.id], session=sqlite_session)+ SummaryIndexAdapter.delete_summaries_for_segments(dataset, segment_ids=[first.id], session=sqlite_session)assert sqlite_session.get(DocumentSegmentSummary, deleted.id) is Noneassert sqlite_session.get(DocumentSegmentSummary, kept.id) is not None@@ -668,7 +719,7 @@ class TestManualUpdate:economy = _persist_dataset(sqlite_session, indexing_technique=IndexTechniqueType.ECONOMY)document = _persist_document(sqlite_session, economy)segment = _persist_segment(sqlite_session, economy, document)- assert SummaryIndexService.update_summary_for_segment(segment, economy, "new", session=sqlite_session) is None+ assert SummaryIndexAdapter.update_summary_for_segment(segment, economy, "new", session=sqlite_session) is Nonequality = _persist_dataset(sqlite_session, dataset_id="dataset-2")qa_document = _persist_document(@@ -676,7 +727,7 @@ class TestManualUpdate:)qa_segment = _persist_segment(sqlite_session, quality, qa_document, segment_id="seg-2")assert (- SummaryIndexService.update_summary_for_segment(qa_segment, quality, "new", session=sqlite_session) is None+ SummaryIndexAdapter.update_summary_for_segment(qa_segment, quality, "new", session=sqlite_session) is None)def test_empty_content_deletes_record_even_when_vector_delete_fails(@@ -688,14 +739,14 @@ class TestManualUpdate:vector.delete_by_ids.side_effect = RuntimeError("boom")monkeypatch.setattr(summary_module, "Vector", MagicMock(return_value=vector))- result = SummaryIndexService.update_summary_for_segment(segment, dataset, " ", session=sqlite_session)+ result = SummaryIndexAdapter.update_summary_for_segment(segment, dataset, " ", session=sqlite_session)assert result is Noneassert sqlite_session.get(DocumentSegmentSummary, summary.id) is Nonedef test_empty_content_without_record_is_noop(self, sqlite_session: Session) -> None:dataset, _, segment = _graph(sqlite_session)- assert SummaryIndexService.update_summary_for_segment(segment, dataset, "", session=sqlite_session) is None+ assert SummaryIndexAdapter.update_summary_for_segment(segment, dataset, "", session=sqlite_session) is Nonedef test_existing_record_updates_vectorizes_and_commits(self,@@ -705,22 +756,9 @@ class TestManualUpdate:) -> None:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment, content="old", node_id="old-node")- vector = MagicMock(name="vector")- monkeypatch.setattr(summary_module, "Vector", MagicMock(return_value=vector))-- def vectorize(- record: DocumentSegmentSummary,- _segment: DocumentSegment,- _dataset: Dataset,- session: Session | None = None,- ) -> None:- record.status = SummaryStatus.COMPLETED- record.summary_index_node_hash = "new-hash"- sqlite_session.flush()-- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", vectorize)+ vector = _install_vector_dependencies(monkeypatch)- result = SummaryIndexService.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)+ result = SummaryIndexAdapter.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)assert result is not Nonevector.delete_by_ids.assert_called_once_with(["old-node"])@@ -730,7 +768,7 @@ class TestManualUpdate:assert (stored.summary_content, stored.status, stored.summary_index_node_hash) == ("new",SummaryStatus.COMPLETED,- "new-hash",+ "hash-1",)def test_new_record_is_persisted(self, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> None:@@ -745,9 +783,9 @@ class TestManualUpdate:record.status = SummaryStatus.COMPLETEDsqlite_session.flush()- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", vectorize)+ monkeypatch.setattr(SummaryIndexAdapter, "vectorize_summary", vectorize)- result = SummaryIndexService.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)+ result = SummaryIndexAdapter.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)assert result is not Noneassert sqlite_session.get(DocumentSegmentSummary, result.id) is result@@ -758,9 +796,9 @@ class TestManualUpdate:) -> None:dataset, document, segment = _graph(sqlite_session)summary = _persist_summary(sqlite_session, dataset, document, segment, content="old")- monkeypatch.setattr(SummaryIndexService, "vectorize_summary", MagicMock(side_effect=RuntimeError("boom")))+ monkeypatch.setattr(SummaryIndexAdapter, "vectorize_summary", MagicMock(side_effect=RuntimeError("boom")))- result = SummaryIndexService.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)+ result = SummaryIndexAdapter.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)assert result is summaryassert (result.summary_content, result.status, result.error) == (@@ -785,7 +823,7 @@ class TestManualUpdate:event.listen(sqlite_session, "before_flush", fail_first_flush)try:with pytest.raises(RuntimeError, match="flush boom"):- SummaryIndexService.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)+ SummaryIndexAdapter.update_summary_for_segment(segment, dataset, "new", session=sqlite_session)finally:event.remove(sqlite_session, "before_flush", fail_first_flush)@@ -804,13 +842,13 @@ class TestReadModels:other_segment = _persist_segment(sqlite_session, other_dataset, other_document, segment_id="seg-3")_persist_summary(sqlite_session, other_dataset, other_document, other_segment, summary_id="sum-3")- assert SummaryIndexService.get_segment_summary(first.id, dataset.id, session=sqlite_session) is first_summary- assert SummaryIndexService.get_segment_summary(second.id, dataset.id, session=sqlite_session) is None- assert SummaryIndexService.get_segments_summaries(+ assert SummaryIndexAdapter.get_segment_summary(first.id, dataset.id, session=sqlite_session) is first_summary+ assert SummaryIndexAdapter.get_segment_summary(second.id, dataset.id, session=sqlite_session) is None+ assert SummaryIndexAdapter.get_segments_summaries([first.id, second.id, other_segment.id], dataset.id, session=sqlite_session) == {first.id: first_summary}- assert SummaryIndexService.get_segments_summaries([], dataset.id, session=sqlite_session) == {}- assert SummaryIndexService.get_document_summaries(+ assert SummaryIndexAdapter.get_segments_summaries([], dataset.id, session=sqlite_session) == {}+ assert SummaryIndexAdapter.get_document_summaries(document.id, dataset.id, [first.id, second.id], session=sqlite_session) == [first_summary]@@ -819,7 +857,7 @@ class TestReadModels:summary = _persist_summary(sqlite_session, dataset, document, segment, status=SummaryStatus.GENERATING)assert (- SummaryIndexService.get_document_summary_index_status(+ SummaryIndexAdapter.get_document_summary_index_status(document.id, dataset.id, TENANT_ID, session=sqlite_session)== "SUMMARIZING"@@ -827,13 +865,13 @@ class TestReadModels:summary.status = SummaryStatus.COMPLETEDsqlite_session.commit()assert (- SummaryIndexService.get_document_summary_index_status(+ SummaryIndexAdapter.get_document_summary_index_status(document.id, dataset.id, TENANT_ID, session=sqlite_session)is None)assert (- SummaryIndexService.get_document_summary_index_status(+ SummaryIndexAdapter.get_document_summary_index_status(document.id, dataset.id, OTHER_TENANT_ID, session=sqlite_session)is None@@ -855,13 +893,13 @@ class TestReadModels:status=SummaryStatus.COMPLETED,)- result = SummaryIndexService.get_documents_summary_index_status(+ result = SummaryIndexAdapter.get_documents_summary_index_status([first_doc.id, second_doc.id, "missing"], dataset.id, TENANT_ID, session=sqlite_session)assert result == {first_doc.id: "SUMMARIZING", second_doc.id: None, "missing": None}assert (- SummaryIndexService.get_documents_summary_index_status([], dataset.id, TENANT_ID, session=sqlite_session)+ SummaryIndexAdapter.get_documents_summary_index_status([], dataset.id, TENANT_ID, session=sqlite_session)== {})@@ -879,9 +917,13 @@ class TestReadModels:created_at=created_at,)segment_service = SimpleNamespace(get_segments_by_document_and_dataset=MagicMock(return_value=[first, second]))- monkeypatch.setitem(sys.modules, "services.dataset_service", SimpleNamespace(SegmentService=segment_service))+ monkeypatch.setitem(+ sys.modules, "services.knowledge.dataset_service", SimpleNamespace(SegmentService=segment_service)+ )- detail = SummaryIndexService.get_document_summary_status_detail(document.id, dataset.id, sqlite_session)+ detail = SummaryIndexAdapter.get_document_summary_status_detail(+ document.id, dataset.id, sqlite_session, tenant_id=dataset.tenant_id+ )assert detail["total_segments"] == 2assert detail["summary_status"]["completed"] == 1diff --git a/api/tests/unit_tests/services/test_trigger_provider_service.py b/api/tests/unit_tests/services/test_trigger_provider_service.pyindex 90abc9671d9..bb0507203bf 100644--- a/api/tests/unit_tests/services/test_trigger_provider_service.py+++ b/api/tests/unit_tests/services/test_trigger_provider_service.py@@ -32,6 +32,7 @@ from models.trigger import (TriggerSubscription,WorkflowPluginTrigger,)+from repositories.credentials.query_repository import CredentialQueryRepositoryfrom services.trigger import trigger_provider_service as service_modulefrom services.trigger.trigger_provider_service import TriggerProviderService@@ -183,7 +184,12 @@ def test_provider_manager_entities_are_forwarded(mocker, trigger_db: TriggerDatadef test_list_subscriptions_empty_state(trigger_db: TriggerDatabase) -> None:assert (- TriggerProviderService.list_trigger_provider_subscriptions(trigger_db.tenant_id, trigger_db.provider_id) == []+ TriggerProviderService.list_trigger_provider_subscriptions(+ trigger_db.tenant_id,+ trigger_db.provider_id,+ credential_query=CredentialQueryRepository(session_factory=trigger_db.session_maker),+ )+ == [])@@ -231,7 +237,9 @@ def test_list_subscriptions_masks_and_counts_distinct_apps()subscriptions = TriggerProviderService.list_trigger_provider_subscriptions(- trigger_db.tenant_id, trigger_db.provider_id+ trigger_db.tenant_id,+ trigger_db.provider_id,+ credential_query=CredentialQueryRepository(session_factory=trigger_db.session_maker),)assert [item.id for item in subscriptions] == [target.id]diff --git a/api/tests/unit_tests/services/test_vector_service.py b/api/tests/unit_tests/services/test_vector_service.pydeleted file mode 100644index a8914f41f1c..00000000000--- a/api/tests/unit_tests/services/test_vector_service.py+++ /dev/null@@ -1,790 +0,0 @@-"""Unit tests for `api/services/vector_service.py`."""--from __future__ import annotations--import logging-from dataclasses import dataclass-from datetime import datetime-from typing import Any-from unittest.mock import MagicMock--import pytest-from sqlalchemy import event-from sqlalchemy.orm import Session--import services.vector_service as vector_service_module-from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from models import UploadFile-from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment, SegmentAttachmentBinding-from models.dataset import Document as DatasetDocument-from models.enums import DataSourceType, DocumentCreatedFrom, ProcessRuleMode-from services.vector_service import VectorService-from tests.unit_tests.model_factories import make_upload_file---@dataclass(frozen=True)-class _ChildDocStub:- page_content: str- metadata: dict[str, Any]---@dataclass-class _ParentDocStub:- children: list[_ChildDocStub]---def _make_dataset(- *,- indexing_technique: str = IndexTechniqueType.HIGH_QUALITY,- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,- tenant_id: str = "tenant-1",- dataset_id: str = "dataset-1",- is_multimodal: bool = False,- embedding_model_provider: str | None = "openai",- embedding_model: str = "text-embedding",-) -> Dataset:- dataset = Dataset(- id=dataset_id,- tenant_id=tenant_id,- name="Dataset",- created_by="account-1",- indexing_technique=indexing_technique,- chunk_structure=doc_form,- is_multimodal=is_multimodal,- embedding_model_provider=embedding_model_provider,- embedding_model=embedding_model,- )- return dataset---def _make_segment(- *,- segment_id: str = "seg-1",- tenant_id: str = "tenant-1",- dataset_id: str = "dataset-1",- document_id: str = "doc-1",- content: str = "hello",- index_node_id: str = "node-1",- index_node_hash: str = "hash-1",- session: Session | None = None,- attachments: list[dict[str, str]] | None = None,-) -> DocumentSegment:- segment = DocumentSegment(- tenant_id=tenant_id,- dataset_id=dataset_id,- document_id=document_id,- position=1,- content=content,- word_count=len(content),- tokens=len(content),- created_by="account-1",- index_node_id=index_node_id,- index_node_hash=index_node_hash,- )- segment.id = segment_id- if attachments:- assert session is not None- for attachment in attachments:- upload_file = _upload_file(- file_id=attachment["id"],- name=attachment.get("name", f"{attachment['id']}.png"),- tenant_id=tenant_id,- )- session.add_all(- [- upload_file,- SegmentAttachmentBinding(- tenant_id=tenant_id,- dataset_id=dataset_id,- document_id=document_id,- segment_id=segment_id,- attachment_id=upload_file.id,- ),- ]- )- session.flush()- return segment---def _upload_file(*, file_id: str = "file-1", name: str = "img.png", tenant_id: str = "tenant-1") -> UploadFile:- return make_upload_file(- file_id=file_id,- tenant_id=tenant_id,- key=f"uploads/{file_id}",- name=name,- extension="png",- mime_type="image/png",- created_at=datetime(2026, 1, 1),- )---def _make_child_chunk(- *,- index_node_id: str,- content: str = "child",- index_node_hash: str = "hash",- tenant_id: str = "tenant-1",- dataset_id: str = "dataset-1",- document_id: str = "doc-1",- segment_id: str = "seg-1",-) -> ChildChunk:- return ChildChunk(- tenant_id=tenant_id,- dataset_id=dataset_id,- document_id=document_id,- segment_id=segment_id,- position=1,- content=content,- word_count=len(content),- created_by="account-1",- index_node_id=index_node_id,- index_node_hash=index_node_hash,- )---def test_create_segments_vector_regular_indexing_loads_documents_and_keywords(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(is_multimodal=False)- segment = _make_segment()-- index_processor = MagicMock(name="index_processor")- factory_instance = MagicMock(name="IndexProcessorFactory-instance")- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.create_segments_vector(- [["k1"]], [segment], dataset, IndexStructureType.PARAGRAPH_INDEX, session=sqlite_session- )-- index_processor.load.assert_called_once()- args, kwargs = index_processor.load.call_args- assert args[0] == dataset- assert len(args[1]) == 1- assert args[2] is None- assert kwargs["with_keywords"] is True- assert kwargs["keywords_list"] == [["k1"]]---def test_create_segments_vector_regular_indexing_loads_multimodal_documents(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(is_multimodal=True)- segment = _make_segment(- session=sqlite_session,- attachments=[- {"id": "img-1", "name": "a.png"},- {"id": "img-2", "name": "b.png"},- ],- )-- index_processor = MagicMock(name="index_processor")- factory_instance = MagicMock(name="IndexProcessorFactory-instance")- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.create_segments_vector(- [["k1"]], [segment], dataset, IndexStructureType.PARAGRAPH_INDEX, session=sqlite_session- )-- assert index_processor.load.call_count == 2- first_args, first_kwargs = index_processor.load.call_args_list[0]- assert first_args[0] == dataset- assert len(first_args[1]) == 1- assert first_kwargs["with_keywords"] is True-- second_args, second_kwargs = index_processor.load.call_args_list[1]- assert second_args[0] == dataset- assert second_args[1] == []- assert len(second_args[2]) == 2- assert second_kwargs["with_keywords"] is False- assert {document.page_content for document in second_args[2]} == {"a.png", "b.png"}---def test_create_segments_vector_with_no_segments_does_not_load(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset()- index_processor = MagicMock(name="index_processor")- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.create_segments_vector(None, [], dataset, IndexStructureType.PARAGRAPH_INDEX, session=sqlite_session)- index_processor.load.assert_not_called()---def _persist_parent_child_rows(- session: Session,- *,- segment: DocumentSegment,- include_document: bool = True,- include_rule: bool = True,-) -> tuple[DatasetDocument | None, DatasetProcessRule | None]:- document = None- rule = None- if include_document:- document = DatasetDocument(- id=segment.document_id,- tenant_id=segment.tenant_id,- dataset_id=segment.dataset_id,- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- dataset_process_rule_id="rule-1",- batch="batch-1",- name="Document",- created_from=DocumentCreatedFrom.API,- created_by="user-1",- doc_language="en",- )- session.add(document)- if include_rule:- rule = DatasetProcessRule(- dataset_id=segment.dataset_id,- mode=ProcessRuleMode.HIERARCHICAL,- rules='{"parent_mode":"full-doc"}',- created_by="user-1",- )- rule.id = "rule-1"- session.add(rule)- session.flush()- return document, rule---def test_create_segments_vector_parent_child_calls_generate_child_chunks_with_explicit_model(- monkeypatch: pytest.MonkeyPatch,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(- doc_form=vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- embedding_model_provider="openai",- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- segment = _make_segment()-- dataset_document, processing_rule = _persist_parent_child_rows(sqlite_session, segment=segment)- assert dataset_document is not None- assert processing_rule is not None-- embedding_model_instance = MagicMock(name="embedding_model_instance")- model_manager_instance = MagicMock(name="model_manager_instance")- model_manager_instance.get_model_instance.return_value = embedding_model_instance- monkeypatch.setattr(- vector_service_module.ModelManager, "for_tenant", MagicMock(return_value=model_manager_instance)- )-- generate_child_chunks_mock = MagicMock()- monkeypatch.setattr(VectorService, "generate_child_chunks", generate_child_chunks_mock)-- index_processor = MagicMock()- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.create_segments_vector(- None,- [segment],- dataset,- vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- session=sqlite_session,- )-- model_manager_instance.get_model_instance.assert_called_once()- generate_child_chunks_mock.assert_called_once_with(- segment, dataset_document, dataset, embedding_model_instance, processing_rule, False, session=sqlite_session- )- index_processor.load.assert_not_called()---def test_create_segments_vector_parent_child_uses_default_embedding_model_when_provider_missing(- monkeypatch: pytest.MonkeyPatch,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(- doc_form=vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- embedding_model_provider=None,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- segment = _make_segment()-- _persist_parent_child_rows(sqlite_session, segment=segment)-- embedding_model_instance = MagicMock()- model_manager_instance = MagicMock()- model_manager_instance.get_default_model_instance.return_value = embedding_model_instance- monkeypatch.setattr(- vector_service_module.ModelManager, "for_tenant", MagicMock(return_value=model_manager_instance)- )-- generate_child_chunks_mock = MagicMock()- monkeypatch.setattr(VectorService, "generate_child_chunks", generate_child_chunks_mock)-- index_processor = MagicMock()- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.create_segments_vector(- None,- [segment],- dataset,- vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- session=sqlite_session,- )-- model_manager_instance.get_default_model_instance.assert_called_once()- generate_child_chunks_mock.assert_called_once()---def test_create_segments_vector_parent_child_missing_document_logs_warning_and_continues(- monkeypatch: pytest.MonkeyPatch,- caplog: pytest.LogCaptureFixture,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(doc_form=vector_service_module.IndexStructureType.PARENT_CHILD_INDEX)- segment = _make_segment()-- index_processor = MagicMock()- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- with caplog.at_level(logging.WARNING, logger="services.vector_service"):- VectorService.create_segments_vector(- None,- [segment],- dataset,- vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- session=sqlite_session,- )- assert any(r.levelno >= logging.WARNING for r in caplog.records)- index_processor.load.assert_not_called()---def test_create_segments_vector_parent_child_missing_processing_rule_raises(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(doc_form=vector_service_module.IndexStructureType.PARENT_CHILD_INDEX)- segment = _make_segment()-- _persist_parent_child_rows(sqlite_session, segment=segment, include_rule=False)-- with pytest.raises(ValueError, match="No processing rule found"):- VectorService.create_segments_vector(- None,- [segment],- dataset,- vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- session=sqlite_session,- )---def test_create_segments_vector_parent_child_non_high_quality_raises(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(- doc_form=vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- indexing_technique=IndexTechniqueType.ECONOMY,- )- segment = _make_segment()- _persist_parent_child_rows(sqlite_session, segment=segment)-- with pytest.raises(ValueError, match="not high quality"):- VectorService.create_segments_vector(- None,- [segment],- dataset,- vector_service_module.IndexStructureType.PARENT_CHILD_INDEX,- session=sqlite_session,- )---def test_update_segment_vector_high_quality_uses_vector(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)- segment = _make_segment()-- vector_instance = MagicMock()- vector_cls = MagicMock(return_value=vector_instance)- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.update_segment_vector(["k"], segment, dataset, session=sqlite_session)-- vector_cls.assert_called_once_with(dataset=dataset, session=sqlite_session)- vector_instance.delete_by_ids.assert_called_once_with([segment.index_node_id])- vector_instance.add_texts.assert_called_once()- add_args, add_kwargs = vector_instance.add_texts.call_args- assert len(add_args[0]) == 1- assert add_kwargs["duplicate_check"] is True---def test_update_segment_vector_economy_uses_keyword_with_keywords_list(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.ECONOMY)- segment = _make_segment()-- keyword_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Keyword", MagicMock(return_value=keyword_instance))-- VectorService.update_segment_vector(["a", "b"], segment, dataset, session=sqlite_session)-- keyword_instance.delete_by_ids.assert_called_once_with([segment.index_node_id], sqlite_session)- keyword_instance.add_texts.assert_called_once()- args, kwargs = keyword_instance.add_texts.call_args- assert len(args[0]) == 1- assert args[1] is sqlite_session- assert kwargs["keywords_list"] == [["a", "b"]]---def test_update_segment_vector_economy_uses_keyword_without_keywords_list(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.ECONOMY)- segment = _make_segment()-- keyword_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Keyword", MagicMock(return_value=keyword_instance))-- VectorService.update_segment_vector(None, segment, dataset, session=sqlite_session)- keyword_instance.add_texts.assert_called_once()- args, kwargs = keyword_instance.add_texts.call_args- assert len(args[0]) == 1- assert args[1] is sqlite_session- assert "keywords_list" not in kwargs---def test_generate_child_chunks_regenerate_cleans_then_saves_children(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(doc_form=IndexStructureType.PARAGRAPH_INDEX, tenant_id="tenant-1", dataset_id="dataset-1")- segment = _make_segment(segment_id="seg-1")-- dataset_document = DatasetDocument(- id=segment.document_id,- tenant_id=segment.tenant_id,- dataset_id=segment.dataset_id,- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- batch="batch-1",- name="Document",- created_from=DocumentCreatedFrom.API,- created_by="user-1",- doc_language="en",- )- processing_rule = DatasetProcessRule(- dataset_id=segment.dataset_id,- mode=ProcessRuleMode.HIERARCHICAL,- rules="{}",- created_by="user-1",- )-- child1 = _ChildDocStub(page_content="c1", metadata={"doc_id": "c1-id", "doc_hash": "c1-h"})- child2 = _ChildDocStub(page_content="c2", metadata={"doc_id": "c2-id", "doc_hash": "c2-h"})- transformed = [_ParentDocStub(children=[child1, child2])]-- index_processor = MagicMock()- index_processor.transform.return_value = transformed- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.generate_child_chunks(- segment=segment,- dataset_document=dataset_document,- dataset=dataset,- embedding_model_instance=MagicMock(),- processing_rule=processing_rule,- regenerate=True,- session=sqlite_session,- )-- index_processor.clean.assert_called_once()- _, transform_kwargs = index_processor.transform.call_args- assert transform_kwargs["process_rule"]["rules"]["parent_mode"] == vector_service_module.ParentMode.FULL_DOC- index_processor.load.assert_called_once()- stored = sqlite_session.query(ChildChunk).order_by(ChildChunk.position).all()- assert [chunk.content for chunk in stored] == ["c1", "c2"]---def test_generate_child_chunks_flushes_even_when_no_children(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(doc_form=IndexStructureType.PARAGRAPH_INDEX)- segment = _make_segment()- dataset_document = DatasetDocument(- id=segment.document_id,- tenant_id=segment.tenant_id,- dataset_id=segment.dataset_id,- position=1,- data_source_type=DataSourceType.UPLOAD_FILE,- batch="batch-1",- name="Document",- created_from=DocumentCreatedFrom.API,- created_by="user-1",- doc_language="en",- )- processing_rule = DatasetProcessRule(- dataset_id=segment.dataset_id,- mode=ProcessRuleMode.HIERARCHICAL,- rules="{}",- created_by="user-1",- )-- index_processor = MagicMock()- index_processor.transform.return_value = [_ParentDocStub(children=[])]- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = index_processor- monkeypatch.setattr(vector_service_module, "IndexProcessorFactory", MagicMock(return_value=factory_instance))-- VectorService.generate_child_chunks(- segment=segment,- dataset_document=dataset_document,- dataset=dataset,- embedding_model_instance=MagicMock(),- processing_rule=processing_rule,- regenerate=False,- session=sqlite_session,- )-- index_processor.load.assert_not_called()- assert sqlite_session.query(ChildChunk).count() == 0---def test_create_child_chunk_vector_high_quality_adds_texts(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)- child_chunk = _make_child_chunk(index_node_id="id", index_node_hash="h")-- vector_instance = MagicMock()- vector_cls = MagicMock(return_value=vector_instance)- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.create_child_chunk_vector(child_chunk, dataset, session=sqlite_session)- vector_cls.assert_called_once_with(dataset=dataset, session=sqlite_session)- vector_instance.add_texts.assert_called_once()---def test_create_child_chunk_vector_economy_noop(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.ECONOMY)- vector_cls = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- child_chunk = _make_child_chunk(index_node_id="id", index_node_hash="h")-- VectorService.create_child_chunk_vector(child_chunk, dataset, session=sqlite_session)- vector_cls.assert_not_called()---def test_update_child_chunk_vector_high_quality_updates_vector(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)-- new_chunk = _make_child_chunk(- content="n", index_node_id="nid", index_node_hash="nh", document_id="d", dataset_id="ds"- )- upd_chunk = _make_child_chunk(- content="u", index_node_id="uid", index_node_hash="uh", document_id="d", dataset_id="ds"- )- del_chunk = _make_child_chunk(index_node_id="did")-- vector_instance = MagicMock()- vector_cls = MagicMock(return_value=vector_instance)- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.update_child_chunk_vector([new_chunk], [upd_chunk], [del_chunk], dataset, session=sqlite_session)-- vector_cls.assert_called_once_with(dataset=dataset, session=sqlite_session)- vector_instance.delete_by_ids.assert_called_once_with(["uid", "did"])- vector_instance.add_texts.assert_called_once()- docs = vector_instance.add_texts.call_args.args[0]- assert len(docs) == 2---def test_update_child_chunk_vector_economy_noop(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.ECONOMY)- vector_cls = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)- VectorService.update_child_chunk_vector([], [], [], dataset, session=sqlite_session)- vector_cls.assert_not_called()---def test_delete_child_chunk_vector_deletes_by_id(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:- dataset = _make_dataset()- child_chunk = _make_child_chunk(index_node_id="cid")-- vector_instance = MagicMock()- vector_cls = MagicMock(return_value=vector_instance)- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.delete_child_chunk_vector(child_chunk, dataset, session=sqlite_session)- vector_cls.assert_called_once_with(dataset=dataset, session=sqlite_session)- vector_instance.delete_by_ids.assert_called_once_with(["cid"])---# ----------------------------------------------------------------------------# update_multimodel_vector (missing coverage in previous suites)-# ------------------------------------------------------------------------------def test_update_multimodel_vector_returns_when_not_high_quality(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.ECONOMY, is_multimodal=True)- segment = _make_segment(tenant_id="t")-- vector_cls = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["a"], dataset=dataset- )- vector_cls.assert_not_called()- assert not sqlite_session.in_transaction()---def test_update_multimodel_vector_returns_when_no_actual_change(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=True)- segment = _make_segment(tenant_id="t", session=sqlite_session, attachments=[{"id": "a"}, {"id": "b"}])-- vector_cls = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["b", "a"], dataset=dataset- )- vector_cls.assert_not_called()- assert sqlite_session.in_transaction()---def test_update_multimodel_vector_deletes_bindings_and_commits_on_empty_new_ids(- monkeypatch: pytest.MonkeyPatch,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=True)- segment = _make_segment(- tenant_id="tenant-1",- session=sqlite_session,- attachments=[{"id": "old-1"}, {"id": "old-2"}],- )-- vector_instance = MagicMock(name="vector_instance")- vector_cls = MagicMock(return_value=vector_instance)-- monkeypatch.setattr(vector_service_module, "Vector", vector_cls)-- VectorService.update_multimodel_vector(segment=segment, attachment_ids=[], dataset=dataset, session=sqlite_session)-- vector_cls.assert_called_once_with(dataset=dataset, session=sqlite_session)- vector_instance.delete_by_ids.assert_called_once_with(["old-1", "old-2"])- assert sqlite_session.query(SegmentAttachmentBinding).count() == 0- vector_instance.add_texts.assert_not_called()---def test_update_multimodel_vector_flushes_when_no_upload_files_found(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=True)- segment = _make_segment(tenant_id="tenant-1", session=sqlite_session, attachments=[{"id": "old-1"}])-- vector_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", MagicMock(return_value=vector_instance))- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["new-1"], dataset=dataset- )-- assert sqlite_session.query(SegmentAttachmentBinding).count() == 0- vector_instance.add_texts.assert_not_called()---def test_update_multimodel_vector_adds_bindings_and_vectors_and_skips_missing_upload_files(- monkeypatch: pytest.MonkeyPatch,- caplog: pytest.LogCaptureFixture,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=True)- segment = _make_segment(- segment_id="seg-1",- tenant_id="tenant-1",- session=sqlite_session,- attachments=[{"id": "old-1"}],- )-- vector_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", MagicMock(return_value=vector_instance))- sqlite_session.add(_upload_file())- sqlite_session.flush()-- with caplog.at_level(logging.WARNING, logger="services.vector_service"):- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["file-1", "missing"], dataset=dataset- )- assert any(r.levelno >= logging.WARNING for r in caplog.records)- bindings = sqlite_session.query(SegmentAttachmentBinding).all()- assert len(bindings) == 1- assert bindings[0].attachment_id == "file-1"-- vector_instance.create_multimodal.assert_called_once()- documents = vector_instance.create_multimodal.call_args.args[0]- assert len(documents) == 1- assert documents[0].page_content == "img.png"- assert documents[0].metadata["doc_id"] == "file-1"---def test_update_multimodel_vector_updates_bindings_without_multimodal_vector_ops(- monkeypatch: pytest.MonkeyPatch,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=False)- segment = _make_segment(tenant_id="tenant-1", session=sqlite_session, attachments=[{"id": "old-1"}])-- vector_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", MagicMock(return_value=vector_instance))- sqlite_session.add(_upload_file())- sqlite_session.flush()-- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["file-1"], dataset=dataset- )-- vector_instance.delete_by_ids.assert_not_called()- vector_instance.add_texts.assert_not_called()- binding = sqlite_session.query(SegmentAttachmentBinding).one()- assert binding.attachment_id == "file-1"---def test_update_multimodel_vector_rolls_back_and_reraises_on_error(- monkeypatch: pytest.MonkeyPatch,- caplog: pytest.LogCaptureFixture,- sqlite_session: Session,-) -> None:- dataset = _make_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, is_multimodal=True)- segment = _make_segment(- segment_id="seg-1",- tenant_id="tenant-1",- session=sqlite_session,- attachments=[{"id": "old-1"}],- )-- vector_instance = MagicMock()- monkeypatch.setattr(vector_service_module, "Vector", MagicMock(return_value=vector_instance))- sqlite_session.add(_upload_file())- sqlite_session.flush()- rollback_events: list[str] = []- event.listen(sqlite_session, "after_rollback", lambda _session: rollback_events.append("rollback"))- monkeypatch.setattr(sqlite_session, "flush", MagicMock(side_effect=RuntimeError("boom")))-- with caplog.at_level(logging.ERROR, logger="services.vector_service"):- with sqlite_session.no_autoflush:- with pytest.raises(RuntimeError, match="boom"):- VectorService.update_multimodel_vector(- session=sqlite_session, segment=segment, attachment_ids=["file-1"], dataset=dataset- )-- assert any(r.levelno >= logging.ERROR for r in caplog.records)- assert rollback_events == ["rollback"]diff --git a/api/tests/unit_tests/services/test_website_service.py b/api/tests/unit_tests/services/test_website_service.pyindex 571e8e99903..cde5d32a8ab 100644--- a/api/tests/unit_tests/services/test_website_service.py+++ b/api/tests/unit_tests/services/test_website_service.py@@ -1,4 +1,4 @@-"""Unit tests for services.website_service.+"""Unit tests for services.data_source.website_service.Focuses on provider dispatching, argument validation, and provider-specific brancheswithout making any real network/storage/redis calls.@@ -10,12 +10,13 @@ import jsonfrom dataclasses import dataclassfrom datetime import UTC, datetimefrom typing import Any-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMock, create_autospec, patchimport pytest-import services.website_service as website_service_module-from services.website_service import (+import services.data_source.website_service as website_service_module+from services.data_source.credential_gateway import DatasourceProviderCredentialStore+from services.data_source.website_service import (CrawlOptions,WebsiteCrawlApiRequest,WebsiteCrawlStatusApiRequest,@@ -111,7 +112,11 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_firecrawl(monkeypaservice_instance.get_datasource_credentials.return_value = {"firecrawl_api_key": "k", "base_url": "b"}monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))- api_key, config = WebsiteService._get_credentials_and_config("tenant-1", "firecrawl")+ api_key, config = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ )._get_credentials_and_config("tenant-1", "firecrawl")assert api_key == "k"assert config["base_url"] == "b"@@ -136,7 +141,11 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(service_instance.get_datasource_credentials.return_value = {"api_key": "enc-key", "base_url": "b"}monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))- api_key, config = WebsiteService._get_credentials_and_config("tenant-1", provider)+ api_key, config = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ )._get_credentials_and_config("tenant-1", provider)assert api_key == "enc-key"assert config["base_url"] == "b"@@ -149,7 +158,11 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(def test_get_credentials_and_config_rejects_invalid_provider() -> None:with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService._get_credentials_and_config("tenant-1", "unknown")+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ )._get_credentials_and_config("tenant-1", "unknown")def test_get_credentials_and_config_hits_unreachable_guard_branch(monkeypatch: pytest.MonkeyPatch) -> None:@@ -174,7 +187,11 @@ def test_get_credentials_and_config_hits_unreachable_guard_branch(monkeypatch: pmonkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ )._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]def test_get_decrypted_api_key_requires_api_key(monkeypatch: pytest.MonkeyPatch) -> None:@@ -204,7 +221,11 @@ def test_crawl_url_dispatches_by_provider(monkeypatch: pytest.MonkeyPatch) -> Nofirecrawl_mock = MagicMock(return_value={"status": "active", "job_id": "j1"})monkeypatch.setattr(WebsiteService, "_crawl_with_firecrawl", firecrawl_mock)- result = WebsiteService.crawl_url(api_request)+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)assert result == {"status": "active", "job_id": "j1"}firecrawl_mock.assert_called_once()@@ -225,7 +246,11 @@ def test_crawl_url_dispatches_other_providers(monkeypatch: pytest.MonkeyPatch, pimpl_mock = MagicMock(return_value={"status": "active"})monkeypatch.setattr(WebsiteService, method_name, impl_mock)- assert WebsiteService.crawl_url(api_request) == {"status": "active"}+ assert WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request) == {"status": "active"}impl_mock.assert_called_once()@@ -234,7 +259,11 @@ def test_crawl_url_rejects_invalid_provider(monkeypatch: pytest.MonkeyPatch) ->monkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.crawl_url(api_request)+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).crawl_url(api_request)def test_crawl_with_firecrawl_builds_params_single_page_and_sets_redis(monkeypatch: pytest.MonkeyPatch) -> None:@@ -406,25 +435,41 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:firecrawl_status = MagicMock(return_value={"status": "active"})monkeypatch.setattr(WebsiteService, "_get_firecrawl_status", firecrawl_status)- result = WebsiteService.get_crawl_status("job-1", "firecrawl")+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_status("job-1", "firecrawl")assert result == {"status": "active"}firecrawl_status.assert_called_once_with("job-1", "k", {"base_url": "b"})watercrawl_status = MagicMock(return_value={"status": "active", "job_id": "w"})monkeypatch.setattr(WebsiteService, "_get_watercrawl_status", watercrawl_status)- assert WebsiteService.get_crawl_status("job-2", "watercrawl") == {"status": "active", "job_id": "w"}+ assert WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_status("job-2", "watercrawl") == {"status": "active", "job_id": "w"}watercrawl_status.assert_called_once_with("job-2", "k", {"base_url": "b"})jinareader_status = MagicMock(return_value={"status": "active", "job_id": "j"})monkeypatch.setattr(WebsiteService, "_get_jinareader_status", jinareader_status)- assert WebsiteService.get_crawl_status("job-3", "jinareader") == {"status": "active", "job_id": "j"}+ assert WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_status("job-3", "jinareader") == {"status": "active", "job_id": "j"}jinareader_status.assert_called_once_with("job-3", "k")def test_get_crawl_status_typed_rejects_invalid_provider(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.get_crawl_status_typed(WebsiteCrawlStatusApiRequest(provider="bad", job_id="j"))+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_status_typed(WebsiteCrawlStatusApiRequest(provider="bad", job_id="j"))def test_get_firecrawl_status_adds_time_consuming_when_completed_and_cached(monkeypatch: pytest.MonkeyPatch) -> None:@@ -532,7 +577,11 @@ def test_get_jinareader_status_completed_formats_processed_items(monkeypatch: pydef test_get_crawl_url_data_dispatches_invalid_provider() -> None:with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.get_crawl_url_data("job-1", "bad", "https://example.com", "tenant-1")+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_url_data("job-1", "bad", "https://example.com", "tenant-1")def test_get_crawl_url_data_hits_invalid_provider_branch_when_credentials_stubbed(@@ -540,7 +589,11 @@ def test_get_crawl_url_data_hits_invalid_provider_branch_when_credentials_stubbe) -> None:monkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.get_crawl_url_data("job-1", object(), "u", "tenant-1") # type: ignore[arg-type]+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_url_data("job-1", object(), "u", "tenant-1") # type: ignore[arg-type]@pytest.mark.parametrize(@@ -556,7 +609,11 @@ def test_get_crawl_url_data_dispatches(monkeypatch: pytest.MonkeyPatch, providerimpl_mock = MagicMock(return_value={"ok": True})monkeypatch.setattr(WebsiteService, method_name, impl_mock)- result = WebsiteService.get_crawl_url_data("job-1", provider, "u", "tenant-1")+ result = WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_crawl_url_data("job-1", provider, "u", "tenant-1")assert result == {"ok": True}impl_mock.assert_called_once()@@ -676,16 +733,28 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatchscrape_mock = MagicMock(return_value={"data": "x"})monkeypatch.setattr(WebsiteService, "_scrape_with_firecrawl", scrape_mock)- assert WebsiteService.get_scrape_url_data("firecrawl", "u", "tenant-1", True) == {"data": "x"}+ assert WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_scrape_url_data("firecrawl", "u", "tenant-1", True) == {"data": "x"}scrape_mock.assert_called_once()watercrawl_mock = MagicMock(return_value={"data": "y"})monkeypatch.setattr(WebsiteService, "_scrape_with_watercrawl", watercrawl_mock)- assert WebsiteService.get_scrape_url_data("watercrawl", "u", "tenant-1", False) == {"data": "y"}+ assert WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_scrape_url_data("watercrawl", "u", "tenant-1", False) == {"data": "y"}watercrawl_mock.assert_called_once()with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService.get_scrape_url_data("jinareader", "u", "tenant-1", True)+ WebsiteService(+ providers=website_service_module.DatasourceProviderService(+ credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)+ )+ ).get_scrape_url_data("jinareader", "u", "tenant-1", True)def test_scrape_with_firecrawl_calls_app(monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/api/tests/unit_tests/services/tools/test_builtin_tools_manage_service.py b/api/tests/unit_tests/services/tools/test_builtin_tools_manage_service.pyindex bcaabd7b815..4fef00d32a4 100644--- a/api/tests/unit_tests/services/tools/test_builtin_tools_manage_service.py+++ b/api/tests/unit_tests/services/tools/test_builtin_tools_manage_service.py@@ -6,10 +6,11 @@ from unittest.mock import MagicMockimport pytestfrom sqlalchemy import select-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom core.plugin.entities.plugin_daemon import CredentialTypefrom models.tools import BuiltinToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient+from repositories.credentials.query_repository import CredentialQueryRepositoryfrom services.tools import builtin_tools_manage_service as service_modulefrom services.tools.builtin_tools_manage_service import BuiltinToolManageService@@ -391,7 +392,11 @@ class TestGetBuiltinToolProviderCredentialInfo:)result = BuiltinToolManageService.get_builtin_tool_provider_credential_info(- "tenant-1", "google", session=repository_session+ "tenant-1",+ "google",+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=repository_session.get_bind(), expire_on_commit=False)+ ),)assert result.credentials == []@@ -404,7 +409,11 @@ class TestGetBuiltinToolProviderCredentials:_persist_provider(repository_session, credential_id="other", tenant_id="other-tenant")result = BuiltinToolManageService.get_builtin_tool_provider_credentials(- "tenant-1", "google", session=repository_session+ "tenant-1",+ "google",+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=repository_session.get_bind(), expire_on_commit=False)+ ),)assert result == []@@ -426,23 +435,23 @@ class TestGetBuiltinToolProviderCredentials:"create_tool_encrypter",MagicMock(return_value=(encrypter, MagicMock())),)- credential_entity = MagicMock()- convert = MagicMock(return_value=credential_entity)- monkeypatch.setattr(- service_module.ToolTransformService,- "convert_builtin_provider_to_credential_entity",- convert,- )result = BuiltinToolManageService.get_builtin_tool_provider_credentials(- "tenant-1", "google", session=repository_session+ "tenant-1",+ "google",+ credential_query=CredentialQueryRepository(+ session_factory=sessionmaker(bind=repository_session.get_bind(), expire_on_commit=False)+ ),)- assert result == [credential_entity]- converted_provider = convert.call_args.kwargs["provider"]- assert isinstance(converted_provider, BuiltinToolProvider)- assert converted_provider.tenant_id == "tenant-1"- assert converted_provider.is_default is True+ assert len(result) == 1+ assert result[0].id == "cred-1"+ assert result[0].credentials == {"key": "***"}+ assert result[0].is_default is True+ repository_session.expire_all()+ stored = repository_session.get(BuiltinToolProvider, "cred-1")+ assert stored is not None+ assert stored.is_default is Falseclass TestGetBuiltinProvider:diff --git a/api/tests/unit_tests/tasks/test_batch_clean_document_task.py b/api/tests/unit_tests/tasks/test_batch_clean_document_task.pyindex 8f80c5a7ec2..588148d1c60 100644--- a/api/tests/unit_tests/tasks/test_batch_clean_document_task.py+++ b/api/tests/unit_tests/tasks/test_batch_clean_document_task.py@@ -53,7 +53,7 @@ def test_successful_vector_cleanup_schedules_billing_refresh(cleanup_rows: tuplewith (patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),- patch("tasks.batch_clean_document_task.IndexProcessorFactory") as processor_factory,+ patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id) as index_cleanup,patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,):batch_clean_document_task(@@ -63,7 +63,7 @@ def test_successful_vector_cleanup_schedules_billing_refresh(cleanup_rows: tuplefile_ids=[],)- processor_factory.return_value.init_index_processor.return_value.clean.assert_called_once()+ index_cleanup.assert_called_once()schedule_refresh.assert_called_once_with(tenant_id)@@ -72,12 +72,10 @@ def test_failed_vector_cleanup_does_not_schedule_billing_refresh(cleanup_rows: twith (patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),- patch("tasks.batch_clean_document_task.IndexProcessorFactory") as processor_factory,+ patch("tasks.batch_clean_document_task.clean_document_indexes") as index_cleanup,patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,):- processor_factory.return_value.init_index_processor.return_value.clean.side_effect = RuntimeError(- "vector cleanup failed"- )+ index_cleanup.side_effect = RuntimeError("vector cleanup failed")batch_clean_document_task(document_ids=[document_id],dataset_id=dataset_id,@@ -119,7 +117,7 @@ def test_cleans_segment_attachment_bindings_and_files(cleanup_rows: tuple[str, swith (patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),- patch("tasks.batch_clean_document_task.IndexProcessorFactory"),+ patch("tasks.batch_clean_document_task.clean_document_indexes"),patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),patch("tasks.batch_clean_document_task.storage.delete") as storage_delete,):diff --git a/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.py b/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.pyindex c3ea579e365..5df433759d7 100644--- a/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.py+++ b/api/tests/unit_tests/tasks/test_batch_create_segment_to_index_task.py@@ -3,6 +3,7 @@ from pathlib import Pathfrom unittest.mock import patchfrom uuid import uuid4+import pytestfrom sqlalchemy import event, selectfrom sqlalchemy.engine import Enginefrom sqlalchemy.orm import Session@@ -12,12 +13,15 @@ from extensions.storage.storage_type import StorageTypefrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatusfrom models.model import UploadFile+from services.knowledge.resource_scope import DatasetRef, DocumentReffrom tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task-def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch([email protected]("foreign_owner", [None, "dataset", "document_tenant", "document_dataset", "upload"])+def test_batch_create_segment_to_index_task_validates_scope_and_indexes_committed_batch(sqlite_session: Session,sqlite_engine: Engine,+ foreign_owner: str | None,) -> None:tenant_id = str(uuid4())user_id = str(uuid4())@@ -71,6 +75,14 @@ def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch(index_node_hash="existing-hash",status=SegmentStatus.COMPLETED,)+ if foreign_owner == "dataset":+ dataset.tenant_id = str(uuid4())+ elif foreign_owner == "document_tenant":+ document.tenant_id = str(uuid4())+ elif foreign_owner == "document_dataset":+ document.dataset_id = str(uuid4())+ elif foreign_owner == "upload":+ upload_file.tenant_id = str(uuid4())sqlite_session.add_all([dataset, document, upload_file, existing_segment])sqlite_session.commit()@@ -94,8 +106,8 @@ def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch(event.listen(sqlite_engine, "before_cursor_execute", count_max_position_query)try:with (- patch("tasks.batch_create_segment_to_index_task.storage.download", side_effect=mock_download),- patch("tasks.batch_create_segment_to_index_task.VectorService.create_segments_vector"),+ patch("tasks.batch_create_segment_to_index_task.storage.download", side_effect=mock_download) as download,+ patch("extensions.ext_application_services.application_services") as services,):batch_create_segment_to_index_task(job_id=str(uuid4()),@@ -108,6 +120,13 @@ def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch(finally:event.remove(sqlite_engine, "before_cursor_execute", count_max_position_query)+ if foreign_owner is not None:+ download.assert_not_called()+ services.assert_not_called()+ assert not max_position_queries+ assert sqlite_session.scalars(select(DocumentSegment.id)).all() == [existing_segment.id]+ return+positions = sqlite_session.scalars(select(DocumentSegment.position).where(DocumentSegment.document_id == document.id)@@ -115,3 +134,7 @@ def test_batch_create_segment_to_index_task_queries_max_position_once_per_batch().all()assert positions == [7, 8, 9, 10]assert len(max_position_queries) == 1+ indexing = services.return_value.knowledge.segments.mutations.index_segments+ indexing.assert_called_once()+ assert indexing.call_args.args == (DocumentRef(DatasetRef(tenant_id, dataset.id), document.id),)+ assert len(indexing.call_args.kwargs["segment_ids"]) == 3diff --git a/api/tests/unit_tests/tasks/test_clean_document_task.py b/api/tests/unit_tests/tasks/test_clean_document_task.pyindex 259dfd6e6a1..a9d7b646c4f 100644--- a/api/tests/unit_tests/tasks/test_clean_document_task.py+++ b/api/tests/unit_tests/tasks/test_clean_document_task.py@@ -6,7 +6,7 @@ starts from the production incident shape: the caller has already deleted the"""import uuid-from unittest.mock import MagicMock, patch+from unittest.mock import patchimport pytestfrom sqlalchemy import select@@ -71,20 +71,10 @@ def mock_storage():@pytest.fixture-def mock_index_processor_factory():- """Mock the vector/index boundary so cleanup behavior is deterministic."""- with patch("tasks.clean_document_task.IndexProcessorFactory", autospec=True) as factory_cls:- processor = MagicMock()- processor.clean.return_value = None- factory_instance = MagicMock()- factory_instance.init_index_processor.return_value = processor- factory_cls.return_value = factory_instance-- yield {- "factory_cls": factory_cls,- "factory_instance": factory_instance,- "processor": processor,- }+def mock_index_cleanup(tenant_id: str):+ """Keep task resilience tests independent of the external index backend."""+ with patch("tasks.clean_document_task.clean_document_indexes", return_value=tenant_id) as cleanup:+ yield cleanupdef _document(*, document_id: str, dataset_id: str, tenant_id: str, created_by: str) -> Document:@@ -209,7 +199,7 @@ class TestVectorCleanupResilience:sqlite_session: Session,bind_task_sessions: None,mock_storage,- mock_index_processor_factory,+ mock_index_cleanup,) -> None:"""A transient billing failure leaves only the unrelated document's rows."""other_document_id, survivor_segment_id = _persist_deleted_document_state(@@ -219,7 +209,7 @@ class TestVectorCleanupResilience:tenant_id=tenant_id,target_segment_ids=["seg-1", "seg-2"],)- mock_index_processor_factory["processor"].clean.side_effect = ValueError(+ mock_index_cleanup.side_effect = ValueError("Unable to retrieve billing information. Please try again later or contact support.")@@ -232,7 +222,7 @@ class TestVectorCleanupResilience:file_id=None,)- mock_index_processor_factory["processor"].clean.assert_called_once()+ mock_index_cleanup.assert_called_once()_assert_relational_cleanup(sqlite_session,document_id=document_id,@@ -249,7 +239,7 @@ class TestVectorCleanupResilience:sqlite_session: Session,bind_task_sessions: None,mock_storage,- mock_index_processor_factory,+ mock_index_cleanup,) -> None:"""The happy path calls the index boundary and completes scoped cleanup."""other_document_id, survivor_segment_id = _persist_deleted_document_state(@@ -268,15 +258,15 @@ class TestVectorCleanupResilience:file_id=None,)- mock_index_processor_factory["processor"].clean.assert_called_once()- _, kwargs = mock_index_processor_factory["processor"].clean.call_args- cleanup_session = kwargs.pop("session")- assert isinstance(cleanup_session, Session)- assert cleanup_session.get_bind() is sqlite_session.get_bind()+ mock_index_cleanup.assert_called_once()+ _, kwargs = mock_index_cleanup.call_args+ new_session = kwargs.pop("new_session")+ with new_session() as cleanup_session:+ assert cleanup_session.get_bind() is sqlite_session.get_bind()assert kwargs == {- "with_keywords": True,- "delete_child_chunks": True,- "delete_summaries": True,+ "dataset_id": dataset_id,+ "document_ids": [document_id],+ "doc_form": "paragraph",}_assert_relational_cleanup(sqlite_session,@@ -286,7 +276,7 @@ class TestVectorCleanupResilience:)schedule_refresh.assert_called_once_with(tenant_id)- def test_no_segments_skips_vector_cleanup(+ def test_no_index_state_does_not_refresh_billing(self,document_id: str,dataset_id: str,@@ -294,9 +284,10 @@ class TestVectorCleanupResilience:sqlite_session: Session,bind_task_sessions: None,mock_storage,- mock_index_processor_factory,+ mock_index_cleanup,) -> None:- """A target document without segments skips the vector/index boundary."""+ """Cleanup checks for orphan summaries even when no segments remain."""+ mock_index_cleanup.return_value = Noneother_document_id, survivor_segment_id = _persist_deleted_document_state(sqlite_session,document_id=document_id,@@ -313,7 +304,7 @@ class TestVectorCleanupResilience:file_id=None,)- mock_index_processor_factory["factory_cls"].assert_not_called()+ mock_index_cleanup.assert_called_once()_assert_relational_cleanup(sqlite_session,document_id=document_id,diff --git a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py b/api/tests/unit_tests/tasks/test_dataset_indexing_task.pyindex 7ef6aaac1bf..3b9b5103947 100644--- a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_dataset_indexing_task.py@@ -8,18 +8,19 @@ ORM rows so each phase observes only committed database state.import uuidfrom contextlib import nullcontextfrom types import SimpleNamespace-from unittest.mock import MagicMock, Mock, patch+from unittest.mock import ANY, MagicMock, Mock, patchimport pytestfrom sqlalchemy.orm import Session-from core.indexing_runner import DocumentIsPausedErrorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom enums import CloudPlan, DeploymentEditionfrom extensions.ext_redis import redis_clientfrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatusfrom services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy+from services.knowledge.indexing.errors import DocumentIsPausedError+from services.knowledge.resource_scope import DocumentReffrom tasks.document_indexing_task import (_document_indexing,_document_indexing_with_tenant_queue,@@ -27,9 +28,50 @@ from tasks.document_indexing_task import (normal_document_indexing_task,priority_document_indexing_task,)+from tasks.recover_document_indexing_task import recover_document_indexing_taskfrom tests.unit_tests.config_override import apply_config_overrides, config_overrides_context+ ("status", "entry"),+ [+ ("waiting", "run"),+ ("parsing", "run"),+ ("cleaning", "run"),+ ("splitting", "run_in_splitting_status"),+ ("indexing", "run_in_indexing_status"),+ ("completed", None),+ ],+)+def test_recovery_dispatches_detached_owner_reference(+ sqlite_session, tenant_id, dataset_id, document_ids, status, entry+):+ _persist_indexing_rows(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id, document_ids=document_ids[:1])+ row = sqlite_session.get(Document, document_ids[0])+ row.indexing_status = status+ sqlite_session.commit()+ with patch("tasks.recover_document_indexing_task.build_document_indexing_service") as factory:+ recover_document_indexing_task.run(dataset_id, document_ids[0])+ service = factory.return_value+ if entry is None:+ assert service.mock_calls == []+ else:+ method = {+ "run": service.run,+ "run_in_splitting_status": service.run_in_splitting_status,+ "run_in_indexing_status": service.run_in_indexing_status,+ }[entry]+ method.assert_called_once()+ (argument,) = method.call_args.args+ ref = argument[0] if entry == "run" else argument+ assert isinstance(ref, DocumentRef)+ assert (ref.dataset.tenant_id, ref.dataset.dataset_id, ref.document_id) == (+ tenant_id,+ dataset_id,+ document_ids[0],+ )++@pytest.fixturedef tenant_id() -> str:return str(uuid.uuid4())@@ -61,7 +103,7 @@ def mock_redis() -> MagicMock:def indexing_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:runner = MagicMock()runner_class = MagicMock(return_value=runner)- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", runner_class)+ monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)runner._constructor_mock = runner_classreturn runner@@ -213,19 +255,22 @@ class TestDocumentIndexing:)_patch_features(monkeypatch, _features())- def assert_committed_parsing(documents: list[Document], session: Session) -> None:- assert all(document.indexing_status == IndexingStatus.PARSING for document in documents)- assert all(document.processing_started_at is not None for document in documents)- assert all(session.get(Document, document.id) is document for document in documents)+ def assert_committed_parsing(refs: list[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as observer:+ documents = [observer.get(Document, ref.document_id) for ref in refs]+ assert all(document.indexing_status == IndexingStatus.PARSING for document in documents)+ assert all(document.processing_started_at is not None for document in documents)indexing_runner.run.side_effect = assert_committed_parsingdocument_indexing_task.run(dataset_id, document_ids)persisted = _persisted_documents(sqlite_session, document_ids)assert [document.indexing_status for document in persisted] == [IndexingStatus.PARSING] * 3- indexing_runner._constructor_mock.assert_called_once_with(enforce_vector_space_admission=True)+ indexing_runner._constructor_mock.assert_called_once_with(+ session_factory=ANY, enforce_vector_space_admission=True+ )indexing_runner.run.assert_called_once()- assert isinstance(indexing_runner.run.call_args.args[1], Session)+ assert len(indexing_runner.run.call_args.args) == 1def test_only_existing_documents_are_processed(self,@@ -248,7 +293,7 @@ class TestDocumentIndexing:_document_indexing(dataset_id, document_ids)processed = indexing_runner.run.call_args.args[0]- assert {document.id for document in processed} == set(existing_ids)+ assert {ref.document_id for ref in processed} == set(existing_ids)assert sqlite_session.get(Document, document_ids[1]) is Nonedef test_empty_batch_still_reaches_runner(@@ -270,14 +315,14 @@ class TestDocumentIndexing:_document_indexing(dataset_id, [])assert indexing_runner.run.call_args.args[0] == []- assert isinstance(indexing_runner.run.call_args.args[1], Session)+ assert len(indexing_runner.run.call_args.args) == 1def test_missing_dataset_returns_before_feature_lookup(self, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch) -> None:get_features = _patch_features(monkeypatch, _features())runner_class = MagicMock()- monkeypatch.setattr("tasks.document_indexing_task.IndexingRunner", runner_class)+ monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)_document_indexing(dataset_id, document_ids)@@ -387,10 +432,12 @@ class TestSummaryDispatch:)_patch_features(monkeypatch, _features())- def finish_documents(documents: list[Document], _session: Session) -> None:- documents[0].indexing_status = IndexingStatus.COMPLETED- documents[1].indexing_status = IndexingStatus.COMPLETED- documents[2].indexing_status = IndexingStatus.INDEXING+ def finish_documents(refs: list[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as writer, writer.begin():+ for ref, status in zip(+ refs, [IndexingStatus.COMPLETED, IndexingStatus.COMPLETED, IndexingStatus.INDEXING], strict=True+ ):+ writer.get(Document, ref.document_id).indexing_status = statusindexing_runner.run.side_effect = finish_documentssummary_delay = MagicMock()@@ -418,9 +465,12 @@ class TestSummaryDispatch:need_summary=[True],)_patch_features(monkeypatch, _features())- indexing_runner.run.side_effect = lambda documents, _session: setattr(- documents[0], "indexing_status", IndexingStatus.COMPLETED- )++ def complete_documents(refs: list[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as writer, writer.begin():+ writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED++ indexing_runner.run.side_effect = complete_documentssummary_delay = MagicMock(side_effect=RuntimeError("queue unavailable"))monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)@@ -449,9 +499,12 @@ class TestSummaryDispatch:need_summary=[True],)_patch_features(monkeypatch, _features())- indexing_runner.run.side_effect = lambda documents, _session: setattr(- documents[0], "indexing_status", IndexingStatus.COMPLETED- )++ def complete_documents(refs: list[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as writer, writer.begin():+ writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED++ indexing_runner.run.side_effect = complete_documentssummary_delay = MagicMock()monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)@@ -478,10 +531,11 @@ class TestSummaryDispatch:)_patch_features(monkeypatch, _features())- def remove_dataset(_documents: list[Document], session: Session) -> None:- dataset = session.get(Dataset, dataset_id)- assert dataset is not None- session.delete(dataset)+ def remove_dataset(_refs: list[DocumentRef]) -> None:+ with Session(sqlite_session.get_bind()) as writer, writer.begin():+ dataset = writer.get(Dataset, dataset_id)+ assert dataset is not None+ writer.delete(dataset)indexing_runner.run.side_effect = remove_datasetsummary_delay = MagicMock()diff --git a/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py b/api/tests/unit_tests/tasks/test_document_indexing_sync_task.pyindex 4fe36b11b81..be8e8b3c022 100644--- a/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py+++ b/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py@@ -6,7 +6,7 @@ Notion extraction, index cleanup, and indexing remain mocked I/O boundaries.import jsonimport uuid-from unittest.mock import MagicMock, patch+from unittest.mock import ANY, MagicMock, patchimport pytestfrom sqlalchemy.orm import Session, sessionmaker@@ -126,10 +126,10 @@ def _bind_sqlite_session_factory(monkeypatch: pytest.MonkeyPatch, sqlite_session@pytest.fixturedef mock_datasource_provider_service():"""Mock datasource credential provider."""- with patch("tasks.document_indexing_sync_task.DatasourceProviderService", autospec=True) as mock_service_class:+ with patch("tasks.document_indexing_sync_task.build_data_source_credentials", autospec=True) as mock_service_class:mock_service = MagicMock()mock_service.get_datasource_credentials.return_value = {"integration_secret": "test_token"}- mock_service_class.return_value = mock_service+ mock_service_class.return_value.providers = mock_serviceyield mock_service@@ -165,6 +165,7 @@ class TestDocumentIndexingSyncTaskCollaboratorParams:# Assertmock_notion_extractor["class"].assert_called_once_with(+ notion_token_loader=ANY,notion_workspace_id=notion_workspace_id,notion_obj_id=notion_page_id,notion_page_type="page",@@ -246,15 +247,15 @@ class TestDataSourceInfoSerialization:):"""data_source_info must be serialized with json.dumps before DB write."""with (- patch("tasks.document_indexing_sync_task.DatasourceProviderService") as mock_service_class,+ patch("tasks.document_indexing_sync_task.build_data_source_credentials") as mock_service_class,patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_extractor_class,patch("tasks.document_indexing_sync_task.IndexProcessorFactory") as mock_ipf,- patch("tasks.document_indexing_sync_task.IndexingRunner") as mock_runner_class,+ patch("tasks.document_indexing_sync_task.build_document_indexing_service") as mock_runner_class,):# External collaboratorsmock_service = MagicMock()mock_service.get_datasource_credentials.return_value = {"integration_secret": "token"}- mock_service_class.return_value = mock_service+ mock_service_class.return_value.providers = mock_servicemock_extractor = MagicMock()# Return a *different* timestamp so the task enters the sync/update branchdiff --git a/api/tests/unit_tests/tasks/test_document_indexing_update_task.py b/api/tests/unit_tests/tasks/test_document_indexing_update_task.pyindex 3756a02a194..c38b3440b64 100644--- a/api/tests/unit_tests/tasks/test_document_indexing_update_task.py+++ b/api/tests/unit_tests/tasks/test_document_indexing_update_task.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport uuidfrom collections.abc import Callablefrom datetime import UTC, datetime+from functools import partialfrom unittest.mock import MagicMockimport pytest@@ -12,12 +13,12 @@ from sqlalchemy import selectfrom sqlalchemy.orm import Sessionimport tasks.document_indexing_update_task as task_module-from core.indexing_runner import DocumentIsPausedErrorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.storage.storage_type import StorageTypefrom models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBindingfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import UploadFile+from services.knowledge.indexing.errors import DocumentIsPausedErrorfrom tasks.document_indexing_update_task import document_indexing_update_task@@ -35,7 +36,7 @@ def task_harness()runner = MagicMock()processor = MagicMock()- monkeypatch.setattr(task_module, "IndexingRunner", MagicMock(return_value=runner))+ monkeypatch.setattr(task_module, "build_document_indexing_service", MagicMock(return_value=runner))monkeypatch.setattr(task_module,"IndexProcessorFactory",@@ -100,9 +101,10 @@ def _persist_rows(return dataset, document-def _complete_indexing(documents: list[Document], _session: Session) -> None:- for document in documents:- document.indexing_status = IndexingStatus.COMPLETED+def _complete_indexing(refs, *, bind) -> None:+ with Session(bind) as writer, writer.begin():+ for ref in refs:+ writer.get(Document, ref.document_id).indexing_status = IndexingStatus.COMPLETEDdef _persist_attachment(@@ -145,7 +147,7 @@ def test_queues_summary_when_all_persisted_conditions_match() -> None:runner, _processor = task_harnessdataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})- runner.run.side_effect = _complete_indexing+ runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())delay = MagicMock()monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)@@ -186,7 +188,7 @@ def test_skips_summary_when_persisted_eligibility_does_not_match(for key, value in document_changes.items():setattr(document, key, value)sqlite_session.commit()- runner.run.side_effect = _complete_indexing+ runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())delay = MagicMock()monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)@@ -276,7 +278,7 @@ def test_queue_failure_is_swallowed_after_successful_indexing() -> None:runner, _processor = task_harnessdataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})- runner.run.side_effect = _complete_indexing+ runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())monkeypatch.setattr(task_module.generate_summary_index_task,"delay",@@ -300,7 +302,7 @@ def test_cleans_and_deletes_persisted_segments_with_real_session(summary_index_setting={"enable": True},with_segment=True,)- runner.run.side_effect = _complete_indexing+ runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())delay = MagicMock()monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)diff --git a/api/tests/unit_tests/tasks/test_enable_segment_index_tasks.py b/api/tests/unit_tests/tasks/test_enable_segment_index_tasks.pyindex fb64f5b5ba4..260e8a15ba2 100644--- a/api/tests/unit_tests/tasks/test_enable_segment_index_tasks.py+++ b/api/tests/unit_tests/tasks/test_enable_segment_index_tasks.py@@ -94,7 +94,7 @@ def test_enable_segment_commits_index_rows_after_loading(_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,patch(- "services.summary_index_service.SummaryIndexService.enable_summaries_for_segments",+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",enable_summaries,),patch("tasks.enable_segment_to_index_task.redis_client.delete"),@@ -126,7 +126,9 @@ def test_enable_segment_rolls_back_before_error_compensation(with (_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,- patch("services.summary_index_service.SummaryIndexService.enable_summaries_for_segments") as enable_summaries,+ patch(+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"+ ) as enable_summaries,patch("tasks.enable_segment_to_index_task.redis_client.delete"),):processor_factory.return_value.init_index_processor.return_value = index_processor@@ -157,7 +159,7 @@ def test_enable_segments_commits_index_rows_after_loading(_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,patch(- "services.summary_index_service.SummaryIndexService.enable_summaries_for_segments",+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",enable_summaries,),patch("tasks.enable_segments_to_index_task.redis_client.delete"),@@ -189,7 +191,9 @@ def test_enable_segments_rolls_back_before_error_compensation(with (_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,- patch("services.summary_index_service.SummaryIndexService.enable_summaries_for_segments") as enable_summaries,+ patch(+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"+ ) as enable_summaries,patch("tasks.enable_segments_to_index_task.redis_client.delete"),):processor_factory.return_value.init_index_processor.return_value = index_processordiff --git a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.pyindex 47d02381da9..2adf8cd37e5 100644--- a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py@@ -1,8 +1,10 @@-from unittest.mock import MagicMock, patch+from unittest.mock import ANY, MagicMock, patchfrom uuid import uuid4+import pytestfrom sqlalchemy.orm import Session+from core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom enums import DeploymentEditionfrom models import Account, Tenant, TenantAccountJoin@@ -14,7 +16,8 @@ from tests.unit_tests.config_override import config_overrides_context@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)-def test_retry_enforces_vector_space_admission(sqlite_session: Session) -> None:[email protected]("runtime_mode", [DatasetRuntimeMode.GENERAL, DatasetRuntimeMode.RAG_PIPELINE])+def test_retry_uses_dataset_runtime(sqlite_session: Session, runtime_mode: DatasetRuntimeMode) -> None:tenant = Tenant(name="Retry tenant")user = Account(name="Retry user", email=f"retry-{uuid4()}@example.com")membership = TenantAccountJoin(@@ -31,7 +34,7 @@ def test_retry_enforces_vector_space_admission(sqlite_session: Session) -> None:data_source_type=DataSourceType.UPLOAD_FILE,indexing_technique=IndexTechniqueType.ECONOMY,chunk_structure=IndexStructureType.PARAGRAPH_INDEX,- runtime_mode=DatasetRuntimeMode.GENERAL,+ runtime_mode=runtime_mode,)document = Document(id=str(uuid4()),@@ -56,12 +59,22 @@ def test_retry_enforces_vector_space_admission(sqlite_session: Session) -> None:with (patch("tasks.retry_document_indexing_task.FeatureService.get_features", return_value=features),patch("tasks.retry_document_indexing_task.IndexProcessorFactory"),- patch("tasks.retry_document_indexing_task.IndexingRunner") as indexing_runner,+ patch("tasks.retry_document_indexing_task.build_document_indexing_service") as indexing_runner,+ patch("tasks.retry_document_indexing_task.RagPipelineService") as pipeline_service,patch("tasks.retry_document_indexing_task.redis_client"),):retry_document_indexing_task.run(dataset.id, [document.id], user.id)- indexing_runner.assert_called_once_with(enforce_vector_space_admission=True)- run_documents, run_session = indexing_runner.return_value.run.call_args.args- assert [item.id for item in run_documents] == [document.id]- assert isinstance(run_session, Session)+ if runtime_mode == DatasetRuntimeMode.GENERAL:+ pipeline_service.assert_not_called()+ indexing_runner.assert_called_once_with(session_factory=ANY, enforce_vector_space_admission=True)+ (run_documents,) = indexing_runner.return_value.run.call_args.args+ assert [item.document_id for item in run_documents] == [document.id]+ assert run_documents[0].dataset.tenant_id == dataset.tenant_id+ else:+ indexing_runner.assert_not_called()+ retry = pipeline_service.return_value.retry_error_document+ retry.assert_called_once()+ retry_dataset, retry_document, retry_user = retry.call_args.args+ assert (retry_dataset.id, retry_document.id, retry_user.id) == (dataset.id, document.id, user.id)+ assert isinstance(retry.call_args.kwargs["generator"], PipelineGenerator)diff --git a/api/tests/unit_tests/tasks/test_segment_index_cleanup_tasks.py b/api/tests/unit_tests/tasks/test_segment_index_cleanup_tasks.pyindex 6a256656ad3..910cf8abf34 100644--- a/api/tests/unit_tests/tasks/test_segment_index_cleanup_tasks.py+++ b/api/tests/unit_tests/tasks/test_segment_index_cleanup_tasks.py@@ -94,7 +94,7 @@ def test_disable_segment_commits_index_cleanup(_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.disable_segment_from_index_task.IndexProcessorFactory") as processor_factory,patch(- "services.summary_index_service.SummaryIndexService.disable_summaries_for_segments",+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",disable_summaries,),patch("tasks.disable_segment_from_index_task.redis_client.delete"),@@ -123,7 +123,7 @@ def test_disable_segments_commits_index_cleanup(_record_transaction_events(sqlite_session_factory, phase_events),patch("tasks.disable_segments_from_index_task.IndexProcessorFactory") as processor_factory,patch(- "services.summary_index_service.SummaryIndexService.disable_summaries_for_segments",+ "services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",disable_summaries,),patch("tasks.disable_segments_from_index_task.redis_client.delete"),diff --git a/api/tests/unit_tests/tasks/test_sync_website_document_indexing_task.py b/api/tests/unit_tests/tasks/test_sync_website_document_indexing_task.pyindex 594c4295305..0cfa3d62742 100644--- a/api/tests/unit_tests/tasks/test_sync_website_document_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_sync_website_document_indexing_task.py@@ -87,7 +87,7 @@ def test_cleanup_is_owner_scoped_and_skips_empty_vector_ids(sqlite_session: Sesswith (patch("tasks.sync_website_document_indexing_task.FeatureService.get_features", return_value=features),patch("tasks.sync_website_document_indexing_task.IndexProcessorFactory") as processor_factory,- patch("tasks.sync_website_document_indexing_task.IndexingRunner") as indexing_runner,+ patch("tasks.sync_website_document_indexing_task.build_document_indexing_service") as indexing_runner,patch("tasks.sync_website_document_indexing_task.redis_client"),):sync_website_document_indexing_task(dataset.id, document.id)diff --git a/packages/contracts/generated/api/console/data-source/orpc.gen.ts b/packages/contracts/generated/api/console/data-source/orpc.gen.tsindex 692089d1845..69711d6a5b7 100644--- a/packages/contracts/generated/api/console/data-source/orpc.gen.ts+++ b/packages/contracts/generated/api/console/data-source/orpc.gen.ts@@ -3,25 +3,11 @@import { oc } from '@orpc/contract'import * as z from 'zod'import {- zGetDataSourceIntegratesByBindingIdByActionPath,- zGetDataSourceIntegratesByBindingIdByActionResponse,zGetDataSourceIntegratesResponse,zPatchDataSourceIntegratesByBindingIdByActionPath,zPatchDataSourceIntegratesByBindingIdByActionResponse,- zPatchDataSourceIntegratesResponse,} from './zod.gen.ts'-export const get = oc- .route({- inputStructure: 'detailed',- method: 'GET',- operationId: 'getDataSourceIntegratesByBindingIdByAction',- path: '/data-source/integrates/{binding_id}/{action}',- tags: ['console'],- })- .input(z.object({ params: zGetDataSourceIntegratesByBindingIdByActionPath }))- .output(zGetDataSourceIntegratesByBindingIdByActionResponse)-export const patch = oc.route({inputStructure: 'detailed',@@ -34,7 +20,6 @@ export const patch = oc.output(zPatchDataSourceIntegratesByBindingIdByActionResponse)export const byAction = {- get,patch,}@@ -42,7 +27,7 @@ export const byBindingId = {byAction,}-export const get2 = oc+export const get = oc.route({inputStructure: 'detailed',method: 'GET',@@ -52,19 +37,8 @@ export const get2 = oc}).output(zGetDataSourceIntegratesResponse)-export const patch2 = oc- .route({- inputStructure: 'detailed',- method: 'PATCH',- operationId: 'patchDataSourceIntegrates',- path: '/data-source/integrates',- tags: ['console'],- })- .output(zPatchDataSourceIntegratesResponse)-export const integrates = {- get: get2,- patch: patch2,+ get,byBindingId,}diff --git a/packages/contracts/generated/api/console/data-source/types.gen.ts b/packages/contracts/generated/api/console/data-source/types.gen.tsindex 1c1051369c4..a19681a65e1 100644--- a/packages/contracts/generated/api/console/data-source/types.gen.ts+++ b/packages/contracts/generated/api/console/data-source/types.gen.ts@@ -4,56 +4,61 @@ export type ClientOptions = {baseUrl: `${string}://${string}/console/api` | (string & {})}-export type GetDataSourceIntegratesData = {- body?: never- path?: never- query?: never- url: '/data-source/integrates'+export type DataSourceIntegrateListResponse = {+ data: Array<DataSourceIntegrateResponse>}-export type GetDataSourceIntegratesResponses = {- 200: {- [key: string]: unknown- }+export type SimpleResultResponse = {+ result: string}-export type GetDataSourceIntegratesResponse =- GetDataSourceIntegratesResponses[keyof GetDataSourceIntegratesResponses]+export type DataSourceIntegrateResponse = {+ created_at: number | null+ disabled: boolean | null+ id: string | null+ is_bound: boolean+ link: string+ provider: string+ source_info: DataSourceIntegrateWorkspaceResponse | null+}-export type PatchDataSourceIntegratesData = {- body?: never- path?: never- query?: never- url: '/data-source/integrates'+export type DataSourceIntegrateWorkspaceResponse = {+ pages: Array<DataSourceIntegratePageResponse>+ total: number+ workspace_icon: string | null+ workspace_id: string | null+ workspace_name: string | null}-export type PatchDataSourceIntegratesResponses = {- 200: {- [key: string]: unknown- }+export type DataSourceIntegratePageResponse = {+ page_icon: DataSourceIntegrateIconResponse | null+ page_id: string+ page_name: string+ parent_id: string+ type: NotionPageType}-export type PatchDataSourceIntegratesResponse =- PatchDataSourceIntegratesResponses[keyof PatchDataSourceIntegratesResponses]+export type DataSourceIntegrateIconResponse = {+ emoji?: string | null+ type?: string | null+ url?: string | null+}++export type NotionPageType = 'database' | 'page'-export type GetDataSourceIntegratesByBindingIdByActionData = {+export type GetDataSourceIntegratesData = {body?: never- path: {- action: string- binding_id: string- }+ path?: neverquery?: never- url: '/data-source/integrates/{binding_id}/{action}'+ url: '/data-source/integrates'}-export type GetDataSourceIntegratesByBindingIdByActionResponses = {- 200: {- [key: string]: unknown- }+export type GetDataSourceIntegratesResponses = {+ 200: DataSourceIntegrateListResponse}-export type GetDataSourceIntegratesByBindingIdByActionResponse =- GetDataSourceIntegratesByBindingIdByActionResponses[keyof GetDataSourceIntegratesByBindingIdByActionResponses]+export type GetDataSourceIntegratesResponse =+ GetDataSourceIntegratesResponses[keyof GetDataSourceIntegratesResponses]export type PatchDataSourceIntegratesByBindingIdByActionData = {body?: never@@ -66,9 +71,7 @@ export type PatchDataSourceIntegratesByBindingIdByActionData = {}export type PatchDataSourceIntegratesByBindingIdByActionResponses = {- 200: {- [key: string]: unknown- }+ 200: SimpleResultResponse}export type PatchDataSourceIntegratesByBindingIdByActionResponse =diff --git a/packages/contracts/generated/api/console/data-source/zod.gen.ts b/packages/contracts/generated/api/console/data-source/zod.gen.tsindex 8236a06d2e4..3e4b58997ce 100644--- a/packages/contracts/generated/api/console/data-source/zod.gen.ts+++ b/packages/contracts/generated/api/console/data-source/zod.gen.ts@@ -3,24 +3,72 @@import * as z from 'zod'/**- * Success+ * SimpleResultResponse*/-export const zGetDataSourceIntegratesResponse = z.record(z.string(), z.unknown())+export const zSimpleResultResponse = z.object({+ result: z.string(),+})/**- * Success+ * DataSourceIntegrateIconResponse*/-export const zPatchDataSourceIntegratesResponse = z.record(z.string(), z.unknown())+export const zDataSourceIntegrateIconResponse = z.object({+ emoji: z.string().nullish(),+ type: z.string().nullish(),+ url: z.string().nullish(),+})-export const zGetDataSourceIntegratesByBindingIdByActionPath = z.object({- action: z.string(),- binding_id: z.uuid(),+/**+ * NotionPageType+ */+export const zNotionPageType = z.enum(['database', 'page'])++/**+ * DataSourceIntegratePageResponse+ */+export const zDataSourceIntegratePageResponse = z.object({+ page_icon: zDataSourceIntegrateIconResponse.nullable(),+ page_id: z.string(),+ page_name: z.string(),+ parent_id: z.string(),+ type: zNotionPageType,+})++/**+ * DataSourceIntegrateWorkspaceResponse+ */+export const zDataSourceIntegrateWorkspaceResponse = z.object({+ pages: z.array(zDataSourceIntegratePageResponse),+ total: z.int(),+ workspace_icon: z.string().nullable(),+ workspace_id: z.string().nullable(),+ workspace_name: z.string().nullable(),+})++/**+ * DataSourceIntegrateResponse+ */+export const zDataSourceIntegrateResponse = z.object({+ created_at: z.int().nullable(),+ disabled: z.boolean().nullable(),+ id: z.string().nullable(),+ is_bound: z.boolean(),+ link: z.string(),+ provider: z.string(),+ source_info: zDataSourceIntegrateWorkspaceResponse.nullable(),+})++/**+ * DataSourceIntegrateListResponse+ */+export const zDataSourceIntegrateListResponse = z.object({+ data: z.array(zDataSourceIntegrateResponse),})/*** Success*/-export const zGetDataSourceIntegratesByBindingIdByActionResponse = z.record(z.string(), z.unknown())+export const zGetDataSourceIntegratesResponse = zDataSourceIntegrateListResponseexport const zPatchDataSourceIntegratesByBindingIdByActionPath = z.object({action: z.string(),@@ -30,7 +78,4 @@ export const zPatchDataSourceIntegratesByBindingIdByActionPath = z.object({/*** Success*/-export const zPatchDataSourceIntegratesByBindingIdByActionResponse = z.record(- z.string(),- z.unknown(),-)+export const zPatchDataSourceIntegratesByBindingIdByActionResponse = zSimpleResultResponsediff --git a/packages/contracts/generated/api/console/datasets/orpc.gen.ts b/packages/contracts/generated/api/console/datasets/orpc.gen.tsindex 2cbafe6d030..8ed04583d05 100644--- a/packages/contracts/generated/api/console/datasets/orpc.gen.ts+++ b/packages/contracts/generated/api/console/datasets/orpc.gen.ts@@ -47,8 +47,6 @@ import {zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse,zGetDatasetsByDatasetIdDocumentsByDocumentIdQuery,zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse,- zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportPath,- zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse,zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksPath,zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksQuery,zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksResponse,@@ -127,9 +125,6 @@ import {zPatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponse,zPostDatasetsApiKeysBody,zPostDatasetsApiKeysResponse,- zPostDatasetsBatchImportStatusByJobIdBody,- zPostDatasetsBatchImportStatusByJobIdPath,- zPostDatasetsBatchImportStatusByJobIdResponse,zPostDatasetsBody,zPostDatasetsByDatasetIdApiKeysByStatusPath,zPostDatasetsByDatasetIdApiKeysByStatusResponse,@@ -269,25 +264,8 @@ export const get3 = oc.input(z.object({ params: zGetDatasetsBatchImportStatusByJobIdPath })).output(zGetDatasetsBatchImportStatusByJobIdResponse)-export const post2 = oc- .route({- inputStructure: 'detailed',- method: 'POST',- operationId: 'postDatasetsBatchImportStatusByJobId',- path: '/datasets/batch_import_status/{job_id}',- tags: ['console'],- })- .input(- z.object({- body: zPostDatasetsBatchImportStatusByJobIdBody,- params: zPostDatasetsBatchImportStatusByJobIdPath,- }),- )- .output(zPostDatasetsBatchImportStatusByJobIdResponse)-export const byJobId = {get: get3,- post: post2,}export const batchImportStatus = {@@ -297,7 +275,7 @@ export const batchImportStatus = {/*** Create external knowledge dataset*/-export const post3 = oc+export const post2 = oc.route({description: 'Create external knowledge dataset',inputStructure: 'detailed',@@ -311,7 +289,7 @@ export const post3 = oc.output(zPostDatasetsExternalResponse)export const external = {- post: post3,+ post: post2,}/**@@ -405,7 +383,7 @@ export const get6 = oc/*** Create external knowledge API template*/-export const post4 = oc+export const post3 = oc.route({description: 'Create external knowledge API template',inputStructure: 'detailed',@@ -420,14 +398,14 @@ export const post4 = ocexport const externalKnowledgeApi = {get: get6,- post: post4,+ post: post3,byExternalKnowledgeApiId,}/*** Estimate dataset indexing cost*/-export const post5 = oc+export const post4 = oc.route({description: 'Estimate dataset indexing cost',inputStructure: 'detailed',@@ -440,13 +418,13 @@ export const post5 = oc.output(zPostDatasetsIndexingEstimateResponse)export const indexingEstimate = {- post: post5,+ post: post4,}/*** Initialize dataset with documents*/-export const post6 = oc+export const post5 = oc.route({description: 'Initialize dataset with documents',inputStructure: 'detailed',@@ -459,7 +437,7 @@ export const post6 = oc.output(zPostDatasetsInitResponse)export const init = {- post: post6,+ post: post5,}export const get7 = oc@@ -480,7 +458,7 @@ export const metadata = {builtIn,}-export const post7 = oc+export const post6 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -492,7 +470,7 @@ export const post7 = oc.output(zPostDatasetsNotionIndexingEstimateResponse)export const notionIndexingEstimate = {- post: post7,+ post: post6,}/**@@ -552,7 +530,7 @@ export const retrievalSetting = {byVectorType,}-export const post8 = oc+export const post7 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -564,7 +542,7 @@ export const post8 = oc.output(zPostDatasetsByDatasetIdApiKeysByStatusResponse)export const byStatus = {- post: post8,+ post: post7,}export const apiKeys2 = {@@ -634,7 +612,7 @@ export const batch = {** Download selected dataset documents as a single ZIP archive (upload-file only)*/-export const post9 = oc+export const post8 = oc.route({description: 'Download selected dataset documents as a single ZIP archive (upload-file only)',inputStructure: 'detailed',@@ -653,7 +631,7 @@ export const post9 = oc.output(zPostDatasetsByDatasetIdDocumentsDownloadZipResponse)export const downloadZip = {- post: post9,+ post: post8,}/**@@ -664,7 +642,7 @@ export const downloadZip = {* (indexing_technique must be 'high_quality' and summary_index_setting.enable must be true),* then asynchronously generates summary indexes for the provided documents.*/-export const post10 = oc+export const post9 = oc.route({description:"Generate summary index for documents\nThis endpoint checks if the dataset configuration supports summary generation\n(indexing_technique must be 'high_quality' and summary_index_setting.enable must be true),\nthen asynchronously generates summary indexes for the provided documents.",@@ -684,10 +662,10 @@ export const post10 = oc.output(zPostDatasetsByDatasetIdDocumentsGenerateSummaryResponse)export const generateSummary = {- post: post10,+ post: post9,}-export const post11 = oc+export const post10 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -705,7 +683,7 @@ export const post11 = oc.output(zPostDatasetsByDatasetIdDocumentsMetadataResponse)export const metadata2 = {- post: post11,+ post: post10,}export const patch2 = oc@@ -911,7 +889,7 @@ export const processing = {byAction: byAction2,}-export const post12 = oc+export const post11 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -928,7 +906,7 @@ export const post12 = oc.output(zPostDatasetsByDatasetIdDocumentsByDocumentIdRenameResponse)export const rename = {- post: post12,+ post: post11,}export const patch6 = oc@@ -951,7 +929,7 @@ export const byAction3 = {patch: patch6,}-export const post13 = oc+export const post12 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -968,22 +946,11 @@ export const post13 = oc.output(zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentResponse)export const segment = {- post: post13,+ post: post12,byAction: byAction3,}-export const get19 = oc- .route({- inputStructure: 'detailed',- method: 'GET',- operationId: 'getDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImport',- path: '/datasets/{dataset_id}/documents/{document_id}/segments/batch_import',- tags: ['console'],- })- .input(z.object({ params: zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportPath }))- .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse)--export const post14 = oc+export const post13 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1000,8 +967,7 @@ export const post14 = oc.output(zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse)export const batchImport = {- get: get19,- post: post14,+ post: post13,}export const delete3 = oc@@ -1049,7 +1015,7 @@ export const byChildChunkId = {patch: patch7,}-export const get20 = oc+export const get19 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1082,7 +1048,7 @@ export const patch8 = oc).output(zPatchDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksResponse)-export const post15 = oc+export const post14 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1099,9 +1065,9 @@ export const post15 = oc.output(zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksResponse)export const childChunks = {- get: get20,+ get: get19,patch: patch8,- post: post15,+ post: post14,byChildChunkId,}@@ -1158,7 +1124,7 @@ export const delete5 = oc).output(zDeleteDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponse)-export const get21 = oc+export const get20 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1176,7 +1142,7 @@ export const get21 = ocexport const segments = {delete: delete5,- get: get21,+ get: get20,batchImport,bySegmentId,}@@ -1195,7 +1161,7 @@ export const segments = {* - timeout: Number of summaries that timed out* - summaries: List of summary records with status and content preview*/-export const get22 = oc+export const get21 = oc.route({description:'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n - completed: Number of summaries completed\n - generating: Number of summaries being generated\n - error: Number of summaries with errors\n - not_started: Number of segments without summary records\n - timeout: Number of summaries that timed out\n- summaries: List of summary records with status and content preview',@@ -1210,13 +1176,13 @@ export const get22 = oc.output(zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse)export const summaryStatus = {- get: get22,+ get: get21,}/*** sync website document*/-export const get23 = oc+export const get22 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1229,7 +1195,7 @@ export const get23 = oc.output(zGetDatasetsByDatasetIdDocumentsByDocumentIdWebsiteSyncResponse)export const websiteSync = {- get: get23,+ get: get22,}export const delete6 = oc@@ -1247,7 +1213,7 @@ export const delete6 = oc/*** Get document details*/-export const get24 = oc+export const get23 = oc.route({description: 'Get document details',inputStructure: 'detailed',@@ -1266,7 +1232,7 @@ export const get24 = ocexport const byDocumentId = {delete: delete6,- get: get24,+ get: get23,download,indexingEstimate: indexingEstimate3,indexingStatus: indexingStatus2,@@ -1296,7 +1262,7 @@ export const delete7 = oc/*** Get documents in a dataset*/-export const get25 = oc+export const get24 = oc.route({description: 'Get documents in a dataset',inputStructure: 'detailed',@@ -1313,7 +1279,7 @@ export const get25 = oc).output(zGetDatasetsByDatasetIdDocumentsResponse)-export const post16 = oc+export const post15 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1331,8 +1297,8 @@ export const post16 = ocexport const documents = {delete: delete7,- get: get25,- post: post16,+ get: get24,+ post: post15,downloadZip,generateSummary,metadata: metadata2,@@ -1343,7 +1309,7 @@ export const documents = {/*** Get dataset error documents*/-export const get26 = oc+export const get25 = oc.route({description: 'Get dataset error documents',inputStructure: 'detailed',@@ -1356,13 +1322,13 @@ export const get26 = oc.output(zGetDatasetsByDatasetIdErrorDocsResponse)export const errorDocs = {- get: get26,+ get: get25,}/*** Test external knowledge retrieval for dataset*/-export const post17 = oc+export const post16 = oc.route({description: 'Test external knowledge retrieval for dataset',inputStructure: 'detailed',@@ -1380,13 +1346,13 @@ export const post17 = oc.output(zPostDatasetsByDatasetIdExternalHitTestingResponse)export const externalHitTesting = {- post: post17,+ post: post16,}/*** Test dataset knowledge retrieval*/-export const post18 = oc+export const post17 = oc.route({description: 'Test dataset knowledge retrieval',inputStructure: 'detailed',@@ -1404,13 +1370,13 @@ export const post18 = oc.output(zPostDatasetsByDatasetIdHitTestingResponse)export const hitTesting = {- post: post18,+ post: post17,}/*** Get dataset indexing status*/-export const get27 = oc+export const get26 = oc.route({description: 'Get dataset indexing status',inputStructure: 'detailed',@@ -1423,10 +1389,10 @@ export const get27 = oc.output(zGetDatasetsByDatasetIdIndexingStatusResponse)export const indexingStatus3 = {- get: get27,+ get: get26,}-export const post19 = oc+export const post18 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1439,7 +1405,7 @@ export const post19 = oc.output(zPostDatasetsByDatasetIdMetadataBuiltInByActionResponse)export const byAction4 = {- post: post19,+ post: post18,}export const builtIn2 = {@@ -1479,7 +1445,7 @@ export const byMetadataId = {patch: patch10,}-export const get28 = oc+export const get27 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1490,7 +1456,7 @@ export const get28 = oc.input(z.object({ params: zGetDatasetsByDatasetIdMetadataPath })).output(zGetDatasetsByDatasetIdMetadataResponse)-export const post20 = oc+export const post19 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1508,13 +1474,13 @@ export const post20 = oc.output(zPostDatasetsByDatasetIdMetadataResponse)export const metadata4 = {- get: get28,- post: post20,+ get: get27,+ post: post19,builtIn: builtIn2,byMetadataId,}-export const get29 = oc+export const get28 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1526,7 +1492,7 @@ export const get29 = oc.output(zGetDatasetsByDatasetIdNotionSyncResponse)export const sync2 = {- get: get29,+ get: get28,}export const notion2 = {@@ -1536,7 +1502,7 @@ export const notion2 = {/*** Get dataset permission user list*/-export const get30 = oc+export const get29 = oc.route({description: 'Get dataset permission user list',inputStructure: 'detailed',@@ -1549,13 +1515,13 @@ export const get30 = oc.output(zGetDatasetsByDatasetIdPermissionPartUsersResponse)export const permissionPartUsers = {- get: get30,+ get: get29,}/*** Get dataset query history*/-export const get31 = oc+export const get30 = oc.route({description: 'Get dataset query history',inputStructure: 'detailed',@@ -1568,13 +1534,13 @@ export const get31 = oc.output(zGetDatasetsByDatasetIdQueriesResponse)export const queries = {- get: get31,+ get: get30,}/*** Get applications related to dataset*/-export const get32 = oc+export const get31 = oc.route({description: 'Get applications related to dataset',inputStructure: 'detailed',@@ -1587,13 +1553,13 @@ export const get32 = oc.output(zGetDatasetsByDatasetIdRelatedAppsResponse)export const relatedApps = {- get: get32,+ get: get31,}/*** retry document*/-export const post21 = oc+export const post20 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1612,13 +1578,13 @@ export const post21 = oc.output(zPostDatasetsByDatasetIdRetryResponse)export const retry = {- post: post21,+ post: post20,}/*** Check if dataset is in use*/-export const get33 = oc+export const get32 = oc.route({description: 'Check if dataset is in use',inputStructure: 'detailed',@@ -1631,7 +1597,7 @@ export const get33 = oc.output(zGetDatasetsByDatasetIdUseCheckResponse)export const useCheck2 = {- get: get33,+ get: get32,}export const delete9 = oc@@ -1649,7 +1615,7 @@ export const delete9 = oc/*** Get dataset details*/-export const get34 = oc+export const get33 = oc.route({description: 'Get dataset details',inputStructure: 'detailed',@@ -1678,7 +1644,7 @@ export const patch11 = ocexport const byDatasetId = {delete: delete9,- get: get34,+ get: get33,patch: patch11,apiKeys: apiKeys2,autoDisableLogs,@@ -1720,7 +1686,7 @@ export const byApiKeyId2 = {/*** Get all API keys for a dataset*/-export const get35 = oc+export const get34 = oc.route({description: 'Get all API keys for a dataset',inputStructure: 'detailed',@@ -1735,7 +1701,7 @@ export const get35 = oc/*** Create a new API key for a dataset*/-export const post22 = oc+export const post21 = oc.route({description: 'Create a new API key for a dataset',inputStructure: 'detailed',@@ -1749,8 +1715,8 @@ export const post22 = oc.output(zPostDatasetsByResourceIdApiKeysResponse)export const apiKeys3 = {- get: get35,- post: post22,+ get: get34,+ post: post21,byApiKeyId: byApiKeyId2,}@@ -1761,7 +1727,7 @@ export const byResourceId = {/*** Get list of datasets*/-export const get36 = oc+export const get35 = oc.route({description: 'Get list of datasets',inputStructure: 'detailed',@@ -1776,7 +1742,7 @@ export const get36 = oc/*** Create a new dataset*/-export const post23 = oc+export const post22 = oc.route({description: 'Create a new dataset',inputStructure: 'detailed',@@ -1790,8 +1756,8 @@ export const post23 = oc.output(zPostDatasetsResponse)export const datasets = {- get: get36,- post: post23,+ get: get35,+ post: post22,apiBaseInfo,apiKeys,batchImportStatus,diff --git a/packages/contracts/generated/api/console/datasets/types.gen.ts b/packages/contracts/generated/api/console/datasets/types.gen.tsindex fda9408dcd2..6c84febc239 100644--- a/packages/contracts/generated/api/console/datasets/types.gen.ts+++ b/packages/contracts/generated/api/console/datasets/types.gen.ts@@ -22,7 +22,7 @@ export type DatasetCreatePayload = {provider?: string}-export type DatasetDetailResponse = {+export type DatasetDetailWithPartialMembersResponse = {app_count: numberauthor_name: string | nullbuilt_in_field_enabled: boolean@@ -47,6 +47,7 @@ export type DatasetDetailResponse = {is_published: booleanmaintainer?: string | nullname: string+ partial_member_list?: Array<string> | nullpermission: stringpermission_keys?: Array<string>pipeline_id: string | null@@ -88,10 +89,6 @@ export type SegmentBatchImportStatusResponse = {job_status: string}-export type BatchImportPayload = {- upload_file_id: string-}-export type ExternalDatasetCreatePayload = {description?: string | nullexternal_knowledge_api_id: string@@ -102,6 +99,46 @@ export type ExternalDatasetCreatePayload = {name: string}+export type DatasetDetailResponse = {+ app_count: number+ author_name: string | null+ built_in_field_enabled: boolean+ chunk_structure: string | null+ created_at: number+ created_by: string+ data_source_type: string | null+ description: string | null+ doc_form: string | null+ doc_metadata: Array<DatasetDocMetadataResponse>+ document_count: number+ embedding_available?: boolean | null+ embedding_model: string | null+ embedding_model_provider: string | null+ enable_api: boolean+ external_knowledge_info?: DatasetExternalKnowledgeInfoResponse+ external_retrieval_model: DatasetExternalRetrievalModelResponse | null+ icon_info?: DatasetIconInfoResponse+ id: string+ indexing_technique: string | null+ is_multimodal: boolean+ is_published: boolean+ maintainer?: string | null+ name: string+ permission: string+ permission_keys?: Array<string>+ pipeline_id: string | null+ provider: string+ retrieval_model_dict: DatasetRetrievalModelResponse+ runtime_mode: string | null+ summary_index_setting?: DatasetSummaryIndexSettingResponse+ tags: Array<DatasetTagResponse>+ total_available_documents: number+ total_documents: number+ updated_at: number+ updated_by: string | null+ word_count: number+}+export type ExternalKnowledgeApiListResponse = {data: Array<ExternalKnowledgeApiResponse>has_more: boolean@@ -191,9 +228,7 @@ export type DatasetMetadataBuiltInFieldsResponse = {export type NotionEstimatePayload = {doc_form?: stringdoc_language?: string- notion_info_list: Array<{- [key: string]: unknown- }>+ notion_info_list: Array<NotionEstimateWorkspacePayload>process_rule: {[key: string]: unknown}@@ -217,47 +252,6 @@ export type RetrievalSettingResponse = {retrieval_method: Array<string>}-export type DatasetDetailWithPartialMembersResponse = {- app_count: number- author_name: string | null- built_in_field_enabled: boolean- chunk_structure: string | null- created_at: number- created_by: string- data_source_type: string | null- description: string | null- doc_form: string | null- doc_metadata: Array<DatasetDocMetadataResponse>- document_count: number- embedding_available?: boolean | null- embedding_model: string | null- embedding_model_provider: string | null- enable_api: boolean- external_knowledge_info?: DatasetExternalKnowledgeInfoResponse- external_retrieval_model: DatasetExternalRetrievalModelResponse | null- icon_info?: DatasetIconInfoResponse- id: string- indexing_technique: string | null- is_multimodal: boolean- is_published: boolean- maintainer?: string | null- name: string- partial_member_list?: Array<string> | null- permission: string- permission_keys?: Array<string>- pipeline_id: string | null- provider: string- retrieval_model_dict: DatasetRetrievalModelResponse- runtime_mode: string | null- summary_index_setting?: DatasetSummaryIndexSettingResponse- tags: Array<DatasetTagResponse>- total_available_documents: number- total_documents: number- updated_at: number- updated_by: string | null- word_count: number-}-export type DatasetUpdatePayload = {description?: string | nullembedding_model?: string | null@@ -443,6 +437,10 @@ export type ConsoleSegmentListResponse = {total_pages: number}+export type BatchImportPayload = {+ upload_file_id: string+}+export type SegmentUpdatePayload = {answer?: string | nullattachment_ids?: Array<string> | null@@ -713,6 +711,12 @@ export type DatasetMetadataBuiltInFieldResponse = {type: string}+export type NotionEstimateWorkspacePayload = {+ credential_id: string+ pages: Array<NotionEstimatePagePayload>+ workspace_id: string+}+export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical'export type Rule = {@@ -929,9 +933,14 @@ export type WeightModel = {weight_type?: 'customized' | 'keyword_first' | 'semantic_first' | null}+export type NotionEstimatePagePayload = {+ page_id: string+ type: NotionPageType+}+export type PreProcessingRule = {enabled: boolean- id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'+ id: PreProcessingRuleKey}export type Segmentation = {@@ -1064,6 +1073,10 @@ export type WeightVectorSetting = {vector_weight: number}+export type NotionPageType = 'database' | 'page'++export type PreProcessingRuleKey = 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'+export type HitTestingDocument = {data_source_type: stringdoc_metadata: unknown | null@@ -1126,7 +1139,7 @@ export type PostDatasetsErrors = {}export type PostDatasetsResponses = {- 201: DatasetDetailResponse+ 201: DatasetDetailWithPartialMembersResponse}export type PostDatasetsResponse = PostDatasetsResponses[keyof PostDatasetsResponses]@@ -1209,22 +1222,6 @@ export type GetDatasetsBatchImportStatusByJobIdResponses = {export type GetDatasetsBatchImportStatusByJobIdResponse =GetDatasetsBatchImportStatusByJobIdResponses[keyof GetDatasetsBatchImportStatusByJobIdResponses]-export type PostDatasetsBatchImportStatusByJobIdData = {- body: BatchImportPayload- path: {- job_id: string- }- query?: never- url: '/datasets/batch_import_status/{job_id}'-}--export type PostDatasetsBatchImportStatusByJobIdResponses = {- 200: SegmentBatchImportStatusResponse-}--export type PostDatasetsBatchImportStatusByJobIdResponse =- PostDatasetsBatchImportStatusByJobIdResponses[keyof PostDatasetsBatchImportStatusByJobIdResponses]-export type PostDatasetsExternalData = {body: ExternalDatasetCreatePayloadpath?: never@@ -2028,23 +2025,6 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponses = {export type GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponse =GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponses[keyof GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponses]-export type GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportData = {- body?: never- path: {- dataset_id: string- document_id: string- }- query?: never- url: '/datasets/{dataset_id}/documents/{document_id}/segments/batch_import'-}--export type GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponses = {- 200: SegmentBatchImportStatusResponse-}--export type GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse =- GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponses[keyof GetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponses]-export type PostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportData = {body: BatchImportPayloadpath: {diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.tsindex ee911690420..0d7a89c3f85 100644--- a/packages/contracts/generated/api/console/datasets/zod.gen.ts+++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts@@ -43,13 +43,6 @@ export const zSegmentBatchImportStatusResponse = z.object({job_status: z.string(),})-/**- * BatchImportPayload- */-export const zBatchImportPayload = z.object({- upload_file_id: z.string(),-})-/*** ExternalDatasetCreatePayload*@@ -96,16 +89,6 @@ export const zIndexingEstimatePayload = z.object({process_rule: z.record(z.string(), z.unknown()),})-/**- * NotionEstimatePayload- */-export const zNotionEstimatePayload = z.object({- doc_form: z.string().optional().default('text_model'),- doc_language: z.string().optional().default('English'),- notion_info_list: z.array(z.record(z.string(), z.unknown())),- process_rule: z.record(z.string(), z.unknown()),-})-/*** RetrievalSettingResponse*/@@ -209,6 +192,13 @@ export const zSegmentCreatePayload = z.object({keywords: z.array(z.string()).nullish(),})+/**+ * BatchImportPayload+ */+export const zBatchImportPayload = z.object({+ upload_file_id: z.string(),+})+/*** SegmentUpdatePayload*/@@ -828,14 +818,6 @@ export const zRetrievalMethod = z.enum(['semantic_search',])-/**- * PreProcessingRule- */-export const zPreProcessingRule = z.object({- enabled: z.boolean(),- id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),-})-/*** Segmentation*/@@ -845,33 +827,6 @@ export const zSegmentation = z.object({separator: z.string().optional().default('\n'),})-/**- * Rule- */-export const zRule = z.object({- parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),- pre_processing_rules: z.array(zPreProcessingRule).nullish(),- segmentation: zSegmentation.nullish(),- subchunk_segmentation: zSegmentation.nullish(),-})--/**- * ProcessRuleResponse- */-export const zProcessRuleResponse = z.object({- limits: z.record(z.string(), z.unknown()),- mode: zProcessRuleMode,- rules: zRule.nullish(),-})--/**- * ProcessRule- */-export const zProcessRule = z.object({- mode: zProcessRuleMode,- rules: zRule.nullish(),-})-/*** MetadataDetail*/@@ -1025,9 +980,9 @@ export const zDatasetRetrievalModelResponse = z.object({})/**- * DatasetDetailResponse+ * DatasetDetailWithPartialMembersResponse*/-export const zDatasetDetailResponse = z.object({+export const zDatasetDetailWithPartialMembersResponse = z.object({app_count: z.int(),author_name: z.string().nullable(),built_in_field_enabled: z.boolean(),@@ -1052,6 +1007,7 @@ export const zDatasetDetailResponse = z.object({is_published: z.boolean(),maintainer: z.string().nullish(),name: z.string(),+ partial_member_list: z.array(z.string()).nullish(),permission: z.string(),permission_keys: z.array(z.string()).optional(),pipeline_id: z.string().nullable(),@@ -1068,9 +1024,9 @@ export const zDatasetDetailResponse = z.object({})/**- * DatasetDetailWithPartialMembersResponse+ * DatasetDetailResponse*/-export const zDatasetDetailWithPartialMembersResponse = z.object({+export const zDatasetDetailResponse = z.object({app_count: z.int(),author_name: z.string().nullable(),built_in_field_enabled: z.boolean(),@@ -1095,7 +1051,6 @@ export const zDatasetDetailWithPartialMembersResponse = z.object({is_published: z.boolean(),maintainer: z.string().nullish(),name: z.string(),- partial_member_list: z.array(z.string()).nullish(),permission: z.string(),permission_keys: z.array(z.string()).optional(),pipeline_id: z.string().nullable(),@@ -1279,6 +1234,79 @@ export const zHitTestingPayload = z.object({retrieval_model: zRetrievalModel.nullish(),})+/**+ * NotionPageType+ */+export const zNotionPageType = z.enum(['database', 'page'])++/**+ * NotionEstimatePagePayload+ */+export const zNotionEstimatePagePayload = z.object({+ page_id: z.string().min(1),+ type: zNotionPageType,+})++/**+ * NotionEstimateWorkspacePayload+ */+export const zNotionEstimateWorkspacePayload = z.object({+ credential_id: z.string().min(1),+ pages: z.array(zNotionEstimatePagePayload).min(1),+ workspace_id: z.string().min(1),+})++/**+ * NotionEstimatePayload+ */+export const zNotionEstimatePayload = z.object({+ doc_form: z.string().optional().default('text_model'),+ doc_language: z.string().optional().default('English'),+ notion_info_list: z.array(zNotionEstimateWorkspacePayload).min(1),+ process_rule: z.record(z.string(), z.unknown()),+})++export const zPreProcessingRuleKey = z.enum([+ 'remove_extra_spaces',+ 'remove_stopwords',+ 'remove_urls_emails',+])++/**+ * PreProcessingRule+ */+export const zPreProcessingRule = z.object({+ enabled: z.boolean(),+ id: zPreProcessingRuleKey,+})++/**+ * Rule+ */+export const zRule = z.object({+ parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),+ pre_processing_rules: z.array(zPreProcessingRule).nullish(),+ segmentation: zSegmentation.nullish(),+ subchunk_segmentation: zSegmentation.nullish(),+})++/**+ * ProcessRuleResponse+ */+export const zProcessRuleResponse = z.object({+ limits: z.record(z.string(), z.unknown()),+ mode: zProcessRuleMode,+ rules: zRule.nullish(),+})++/**+ * ProcessRule+ */+export const zProcessRule = z.object({+ mode: zProcessRuleMode,+ rules: zRule.nullish(),+})+/*** HitTestingDocument*/@@ -1477,7 +1505,7 @@ export const zPostDatasetsBody = zDatasetCreatePayload/*** Dataset created successfully*/-export const zPostDatasetsResponse = zDatasetDetailResponse+export const zPostDatasetsResponse = zDatasetDetailWithPartialMembersResponse/*** API base info retrieved successfully@@ -1514,17 +1542,6 @@ export const zGetDatasetsBatchImportStatusByJobIdPath = z.object({*/export const zGetDatasetsBatchImportStatusByJobIdResponse = zSegmentBatchImportStatusResponse-export const zPostDatasetsBatchImportStatusByJobIdBody = zBatchImportPayload--export const zPostDatasetsBatchImportStatusByJobIdPath = z.object({- job_id: z.uuid(),-})--/**- * Batch import started- */-export const zPostDatasetsBatchImportStatusByJobIdResponse = zSegmentBatchImportStatusResponse-export const zPostDatasetsExternalBody = zExternalDatasetCreatePayload/**@@ -1990,17 +2007,6 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsQuery = z.objecexport const zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponse =zConsoleSegmentListResponse-export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportPath = z.object({- dataset_id: z.uuid(),- document_id: z.uuid(),-})--/**- * Batch import status- */-export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse =- zSegmentBatchImportStatusResponse-export const zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportBody =zBatchImportPayloaddiff --git a/packages/contracts/generated/api/console/notion/types.gen.ts b/packages/contracts/generated/api/console/notion/types.gen.tsindex 43545a9721d..92a1ebe6011 100644--- a/packages/contracts/generated/api/console/notion/types.gen.ts+++ b/packages/contracts/generated/api/console/notion/types.gen.ts@@ -25,7 +25,7 @@ export type NotionIntegratePageResponse = {page_id: stringpage_name: stringparent_id: string | null- type: string+ type: NotionPageType}export type DataSourceIntegrateIconResponse = {@@ -34,6 +34,8 @@ export type DataSourceIntegrateIconResponse = {url?: string | null}+export type NotionPageType = 'database' | 'page'+export type GetNotionPagesByPageIdByPageTypePreviewData = {body?: neverpath: {diff --git a/packages/contracts/generated/api/console/notion/zod.gen.ts b/packages/contracts/generated/api/console/notion/zod.gen.tsindex 7f228414f19..f144e14845e 100644--- a/packages/contracts/generated/api/console/notion/zod.gen.ts+++ b/packages/contracts/generated/api/console/notion/zod.gen.ts@@ -18,6 +18,11 @@ export const zDataSourceIntegrateIconResponse = z.object({url: z.string().nullish(),})+/**+ * NotionPageType+ */+export const zNotionPageType = z.enum(['database', 'page'])+/*** NotionIntegratePageResponse*/@@ -27,7 +32,7 @@ export const zNotionIntegratePageResponse = z.object({page_id: z.string(),page_name: z.string(),parent_id: z.string().nullable(),- type: z.string(),+ type: zNotionPageType,})/**diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.tsindex b51d4c34d93..31940741fcd 100644--- a/packages/contracts/generated/api/service/types.gen.ts+++ b/packages/contracts/generated/api/service/types.gen.ts@@ -1777,9 +1777,11 @@ export type PipelineUploadFileResponse = {export type PreProcessingRule = {enabled: boolean- id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'+ id: PreProcessingRuleKey}+export type PreProcessingRuleKey = 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'+export type ProcessRule = {mode: ProcessRuleModerules?: Rule | nulldiff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.tsindex 9b2bcb161b9..24cfe744072 100644--- a/packages/contracts/generated/api/service/zod.gen.ts+++ b/packages/contracts/generated/api/service/zod.gen.ts@@ -1664,12 +1664,18 @@ export const zPipelineUploadFileResponse = z.object({size: z.int(),})+export const zPreProcessingRuleKey = z.enum([+ 'remove_extra_spaces',+ 'remove_stopwords',+ 'remove_urls_emails',+])+/*** PreProcessingRule*/export const zPreProcessingRule = z.object({enabled: z.boolean(),- id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),+ id: zPreProcessingRuleKey,})/**diff --git a/web/app/components/base/notion-page-selector/__tests__/base.spec.tsx b/web/app/components/base/notion-page-selector/__tests__/base.spec.tsxindex 4995e87fc40..8975198bbc8 100644--- a/web/app/components/base/notion-page-selector/__tests__/base.spec.tsx+++ b/web/app/components/base/notion-page-selector/__tests__/base.spec.tsx@@ -145,6 +145,7 @@ describe('NotionPageSelector Base', () => {expect(screen.getByTestId('notion-page-selector-base')).toBeInTheDocument()expect(screen.getByText('Root 1')).toBeInTheDocument()+ expect(screen.getByText('External 1')).toBeInTheDocument()const checkbox = screen.getByRole('checkbox', { name: 'Root 1' })await user.click(checkbox)diff --git a/web/app/components/base/notion-page-selector/base.tsx b/web/app/components/base/notion-page-selector/base.tsxindex 6fccd5e6b13..c0762bb0cef 100644--- a/web/app/components/base/notion-page-selector/base.tsx+++ b/web/app/components/base/notion-page-selector/base.tsx@@ -100,6 +100,7 @@ const NotionPageSelector = ({isFetching: isFetchingNotionPages,isError: isFetchingNotionPagesError,} = usePreImportNotionPages({ datasetId, credentialId: currentCredentialId })+ const notionPages = notionsPages?.notion_info?.flatMap((workspace) => workspace.pages) || []const pagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] =useMemo(() => {@@ -206,7 +207,7 @@ const NotionPageSelector = ({value={selectedPagesId}disabledValue={pagesMapAndSelectedPagesId[2]}searchValue={searchValue}- list={notionsPages!.notion_info?.[0]!.pages || []}+ list={notionPages}pagesMap={pagesMapAndSelectedPagesId[0]}onSelect={handleSelectPages}canPreview={canPreview}