Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor: migrate Dataset to TypeBase (#40817)
Asuka Minatoa4eb8363be29
diff --git a/api/models/dataset.py b/api/models/dataset.pyindex 77211b59242..e6535d6e16e 100644--- a/api/models/dataset.py+++ b/api/models/dataset.py@@ -146,7 +146,7 @@ class DocumentDict(TypedDict):DatasetPermissionEnum = PermissionEnum-class Dataset(Base):+class Dataset(TypeBase, kw_only=True):__tablename__ = "datasets"__table_args__ = (sa.PrimaryKeyConstraint("id", name="dataset_pkey"),@@ -159,40 +159,63 @@ class Dataset(Base):PROVIDER_LIST = ["vendor", "external", None]DOC_FORM_LIST = [member.value for member in IndexStructureType]- id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))- tenant_id: Mapped[str] = mapped_column(StringUUID)- name: Mapped[str] = mapped_column(String(255))- description = mapped_column(LongText, nullable=True)- provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'"))- permission: Mapped[DatasetPermissionEnum] = mapped_column(+ id: Mapped[str] = mapped_column(+ StringUUID,+ insert_default=lambda: str(uuid4()),+ default_factory=lambda: str(uuid4()),+ )+ tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False, default=cast(str, None))+ name: Mapped[str] = mapped_column(String(255), nullable=False, default=cast(str, None))+ description: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)+ provider: Mapped[str] = mapped_column(+ String(255), nullable=False, server_default=sa.text("'vendor'"), default=cast(str, None)+ )+ permission: Mapped[Any] = mapped_column(EnumText(DatasetPermissionEnum, length=255),default=DatasetPermissionEnum.ONLY_ME,)- data_source_type = mapped_column(EnumText(DataSourceType, length=255))- indexing_technique: Mapped[IndexTechniqueType | None] = mapped_column(EnumText(IndexTechniqueType, length=255))- index_struct = mapped_column(LongText, nullable=True)- created_by = mapped_column(StringUUID, nullable=False)- maintainer: Mapped[str | None] = mapped_column(StringUUID, nullable=True)- created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())- updated_by = mapped_column(StringUUID, nullable=True)- updated_at = mapped_column(- sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()- )- embedding_model = mapped_column(sa.String(255), nullable=True)- embedding_model_provider = mapped_column(sa.String(255), nullable=True)- keyword_number = mapped_column(sa.Integer, nullable=True, server_default=sa.text("10"))- collection_binding_id = mapped_column(StringUUID, nullable=True)- retrieval_model = mapped_column(AdjustedJSON, nullable=True)- summary_index_setting = mapped_column(AdjustedJSON, nullable=True)- built_in_field_enabled = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())- icon_info = mapped_column(AdjustedJSON, nullable=True)- runtime_mode = mapped_column(- EnumText(DatasetRuntimeMode, length=255), nullable=True, server_default=sa.text("'general'")- )- pipeline_id = mapped_column(StringUUID, nullable=True)- chunk_structure = mapped_column(sa.String(255), nullable=True)- enable_api = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())- is_multimodal = mapped_column(sa.Boolean, default=False, nullable=False)+ data_source_type: Mapped[Any] = mapped_column(EnumText(DataSourceType, length=255), nullable=True, default=None)+ indexing_technique: Mapped[Any] = mapped_column(+ EnumText(IndexTechniqueType, length=255), nullable=True, default=None+ )+ index_struct: Mapped[Any] = mapped_column(LongText, nullable=True, default=None)+ created_by: Mapped[str] = mapped_column(StringUUID, nullable=False, default=cast(str, None))+ maintainer: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ created_at: Mapped[datetime] = mapped_column(+ DateTime, nullable=False, server_default=func.current_timestamp(), init=False+ )+ updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ updated_at: Mapped[datetime] = mapped_column(+ sa.DateTime,+ nullable=False,+ server_default=func.current_timestamp(),+ onupdate=func.current_timestamp(),+ init=False,+ )+ embedding_model: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)+ embedding_model_provider: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)+ keyword_number: Mapped[Any] = mapped_column(sa.Integer, nullable=True, server_default=sa.text("10"), default=None)+ collection_binding_id: Mapped[Any] = mapped_column(StringUUID, nullable=True, default=None)+ retrieval_model: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)+ summary_index_setting: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)+ built_in_field_enabled: Mapped[bool] = mapped_column(+ sa.Boolean, nullable=False, server_default=sa.false(), default=cast(bool, None)+ )+ icon_info: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)+ runtime_mode: Mapped[Any] = mapped_column(+ EnumText(DatasetRuntimeMode, length=255), nullable=True, server_default=sa.text("'general'"), default=None+ )+ pipeline_id: Mapped[Any] = mapped_column(StringUUID, nullable=True, default=None)+ chunk_structure: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)+ enable_api: Mapped[bool] = mapped_column(+ sa.Boolean, nullable=False, server_default=sa.true(), default=cast(bool, None)+ )+ is_multimodal: Mapped[bool] = mapped_column(+ sa.Boolean,+ insert_default=False,+ default=False,+ nullable=False,+ )@propertydef index_struct_dict(self):diff --git a/api/services/rag_pipeline/rag_pipeline_dsl_service.py b/api/services/rag_pipeline/rag_pipeline_dsl_service.pyindex d149346b0a7..00372527040 100644--- a/api/services/rag_pipeline/rag_pipeline_dsl_service.py+++ b/api/services/rag_pipeline/rag_pipeline_dsl_service.py@@ -295,7 +295,7 @@ class RagPipelineDslService:names = [dataset.name for dataset in datasets]generate_name = generate_incremental_name(names, name)dataset = Dataset(- tenant_id=account.current_tenant_id,+ tenant_id=cast(str, account.current_tenant_id),name=generate_name,description=description,icon_info={@@ -440,7 +440,7 @@ class RagPipelineDslService:if not dataset:dataset_created = Truedataset = Dataset(- tenant_id=account.current_tenant_id,+ tenant_id=cast(str, account.current_tenant_id),name=name,description=description,icon_info={diff --git a/api/services/rag_pipeline/rag_pipeline_transform_service.py b/api/services/rag_pipeline/rag_pipeline_transform_service.pyindex fbf4dd526c3..4aa959b6d16 100644--- a/api/services/rag_pipeline/rag_pipeline_transform_service.py+++ b/api/services/rag_pipeline/rag_pipeline_transform_service.py@@ -325,7 +325,7 @@ class RagPipelineTransformService:pipeline = Pipeline(tenant_id=dataset.tenant_id,name=dataset.name,- description=dataset.description,+ description=dataset.description or "",created_by=account_id,)session.add(pipeline)diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.pyindex d311cbfc4c8..6584ad5bfbb 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py@@ -160,9 +160,9 @@ class TestCleanDatasetTask:index_struct='{"type": "paragraph"}',collection_binding_id=str(uuid.uuid4()),created_by=account.id,- created_at=datetime.now(),- updated_at=datetime.now(),)+ dataset.created_at = datetime.now()+ dataset.updated_at = datetime.now()db_session_with_containers.add(dataset)db_session_with_containers.commit()@@ -906,9 +906,9 @@ class TestCleanDatasetTask:index_struct='{"type": "paragraph", "max_length": 10000}',collection_binding_id=str(uuid.uuid4()),created_by=account.id,- created_at=datetime.now(),- updated_at=datetime.now(),)+ dataset.created_at = datetime.now()+ dataset.updated_at = datetime.now()db_session_with_containers.add(dataset)db_session_with_containers.commit()diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py b/api/tests/unit_tests/controllers/console/explore/test_trial_preview.pyindex 80054e075a1..3db174ef0c1 100644--- a/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py+++ b/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py@@ -161,8 +161,8 @@ def harness(data_source_type="upload_file",indexing_technique=None,created_by=creator_id,- created_at=_CREATED_AT,)+ dataset.created_at = _CREATED_ATwith sqlite_session_factory.begin() as session:session.add_all([owner, target, listing, config, site, dataset])session.flush()diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.pyindex 03aebf9a47b..5ec542c7a56 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py@@ -8,6 +8,7 @@ import uuidfrom datetime import UTC, datetimefrom inspect import unwrapfrom types import SimpleNamespace+from typing import castfrom unittest.mock import MagicMock, patchimport pytest@@ -102,7 +103,11 @@ def make_dataset("is_multimodal": False,}base.update(overrides)- dataset = Dataset(**base)+ created_at = cast(datetime, base.pop("created_at"))+ updated_at = cast(datetime, base.pop("updated_at"))+ dataset = Dataset(**base) # type: ignore[bad-argument-type]+ dataset.created_at = created_at+ dataset.updated_at = updated_atsession.add(dataset)session.flush()return datasetdiff --git a/api/tests/unit_tests/model_factories.py b/api/tests/unit_tests/model_factories.pyindex 3aeb0f061b5..0d6e28835fb 100644--- a/api/tests/unit_tests/model_factories.py+++ b/api/tests/unit_tests/model_factories.py@@ -311,28 +311,42 @@ def make_dataset(unset, matching a bare ``Dataset(...)``. ``dataset_id=None`` keeps thegenerated identifier."""- values: dict[str, object] = {"tenant_id": tenant_id, "name": name, "created_by": created_by}- optional: dict[str, object | None] = {- "id": dataset_id,- "description": description,- "provider": provider,- "permission": permission,- "data_source_type": data_source_type,- "indexing_technique": indexing_technique,- "index_struct": index_struct,- "maintainer": maintainer,- "created_at": created_at,- "embedding_model": embedding_model,- "embedding_model_provider": embedding_model_provider,- "keyword_number": keyword_number,- "built_in_field_enabled": built_in_field_enabled,- "icon_info": icon_info,- "pipeline_id": pipeline_id,- "chunk_structure": chunk_structure,- "enable_api": enable_api,- }- values.update({key: value for key, value in optional.items() if value is not None})- return Dataset(**values)+ result = Dataset(tenant_id=tenant_id, name=name, created_by=created_by)+ if dataset_id is not None:+ result.id = dataset_id+ if description is not None:+ result.description = description+ if provider is not None:+ result.provider = provider+ if permission is not None:+ result.permission = permission+ if data_source_type is not None:+ result.data_source_type = data_source_type+ if indexing_technique is not None:+ result.indexing_technique = indexing_technique+ if index_struct is not None:+ result.index_struct = index_struct+ if maintainer is not None:+ result.maintainer = maintainer+ if created_at is not None:+ result.created_at = created_at+ if embedding_model is not None:+ result.embedding_model = embedding_model+ if embedding_model_provider is not None:+ result.embedding_model_provider = embedding_model_provider+ if keyword_number is not None:+ result.keyword_number = keyword_number+ if built_in_field_enabled is not None:+ result.built_in_field_enabled = built_in_field_enabled+ if icon_info is not None:+ result.icon_info = icon_info+ if pipeline_id is not None:+ result.pipeline_id = pipeline_id+ if chunk_structure is not None:+ result.chunk_structure = chunk_structure+ if enable_api is not None:+ result.enable_api = enable_api+ return resultdef make_conversation(diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.pyindex e691a7d623b..0a39d4b4440 100644--- a/api/tests/unit_tests/models/test_dataset_models.py+++ b/api/tests/unit_tests/models/test_dataset_models.py@@ -23,6 +23,7 @@ from core.rag.entities import ParentModefrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.storage.storage_type import StorageTypefrom models.account import Account+from models.base import Base, TypeBasefrom models.dataset import (AppDatasetJoin,ChildChunk,@@ -161,6 +162,16 @@ def _make_segments(document: Document, hit_counts: list[int]) -> list[DocumentSeclass TestDatasetModelValidation:"""Test suite for Dataset model validation and basic operations."""+ def test_dataset_uses_typebase_registry_and_generates_ids(self):+ generated_dataset = Dataset()+ another_dataset = Dataset()+ explicit_id = str(uuid4())++ assert Dataset.__mapper__.registry is TypeBase.registry+ assert Dataset.__mapper__.registry is not Base.registry+ assert generated_dataset.id != another_dataset.id+ assert Dataset(id=explicit_id).id == explicit_id+def test_dataset_creation_with_required_fields(self):"""Test creating a dataset with all required fields."""# Arrangediff --git a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py b/api/tests/unit_tests/repositories/test_app_preview_query_repository.pyindex 31ea2abadfc..fd37d0e613d 100644--- a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py+++ b/api/tests/unit_tests/repositories/test_app_preview_query_repository.py@@ -73,19 +73,18 @@ def _add_site(session: Session, *, app_id: str, title: str) -> None:def _add_dataset(session: Session, *, dataset_id: str, tenant_id: str) -> None:- session.add(- Dataset(- id=dataset_id,- tenant_id=tenant_id,- name="Knowledge",- description="Knowledge description",- permission="only_me",- data_source_type="upload_file",- indexing_technique="high_quality",- created_by=_CREATOR_ID,- created_at=_CREATED_AT,- )+ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ name="Knowledge",+ description="Knowledge description",+ permission="only_me",+ data_source_type="upload_file",+ indexing_technique="high_quality",+ created_by=_CREATOR_ID,)+ dataset.created_at = _CREATED_AT+ session.add(dataset)@pytest.fixturediff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.pyindex 07364ea972c..c4410988259 100644--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py@@ -182,7 +182,6 @@ class DatasetServiceUnitDataFactory:chunk_structure=doc_form,enable_api=enable_api,updated_by=None,- updated_at=None,summary_index_setting=summary_index_setting,**kwargs,)diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.pyindex f9fb45e2a01..56f8a01a302 100644--- a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py+++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py@@ -41,18 +41,18 @@ REF = DatasetRef("tenant", "dataset")def dataset(**values: object) -> Dataset:- return Dataset(- **{- "id": "dataset",- "tenant_id": "tenant",- "name": "Dataset",- "created_by": "actor",- "maintainer": "actor",- "indexing_technique": "economy",- "permission": "all_team_members",- **values,- }+ row = Dataset(+ id="dataset",+ tenant_id="tenant",+ name="Dataset",+ created_by="actor",+ maintainer="actor",+ indexing_technique="economy",+ permission="all_team_members",)+ for key, value in values.items():+ setattr(row, key, value)+ return rowdef document(**values: object) -> Document:diff --git a/api/tests/unit_tests/services/knowledge/test_document_adapters.py b/api/tests/unit_tests/services/knowledge/test_document_adapters.pyindex eb85952f5b0..a655a19e9e5 100644--- a/api/tests/unit_tests/services/knowledge/test_document_adapters.py+++ b/api/tests/unit_tests/services/knowledge/test_document_adapters.py@@ -18,18 +18,18 @@ REF = DatasetRef("tenant-1", "dataset-1")def _dataset(**values: object) -> Dataset:- return Dataset(- **{- "id": "dataset-1",- "tenant_id": "tenant-1",- "name": "Dataset",- "created_by": "actor-1",- "maintainer": "actor-1",- "indexing_technique": "economy",- "chunk_structure": "text_model",- **values,- }+ row = Dataset(+ id="dataset-1",+ tenant_id="tenant-1",+ name="Dataset",+ created_by="actor-1",+ maintainer="actor-1",+ indexing_technique="economy",+ chunk_structure="text_model",)+ for key, value in values.items():+ setattr(row, key, value)+ return rowdef _document(**values: object) -> Document: