Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor: migrate Dataset to TypeBase (#40817)

Asuka Minatoa4eb8363be29
diff --git a/api/models/dataset.py b/api/models/dataset.py
index 77211b59242..e6535d6e16e 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -146,7 +146,7 @@ class DocumentDict(TypedDict):
DatasetPermissionEnum = PermissionEnum
-class Dataset(Base):
+class Dataset(TypeBase, kw_only=True):
__tablename__ = "datasets"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_pkey"),
@@ -159,40 +159,63 @@ class Dataset(Base):
PROVIDER_LIST = ["vendor", "external", None]
DOC_FORM_LIST = [member.value for member in IndexStructureType]
- id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))
- tenant_id: Mapped[str] = mapped_column(StringUUID)
- name: Mapped[str] = mapped_column(String(255))
- description = mapped_column(LongText, nullable=True)
- provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'"))
- permission: Mapped[DatasetPermissionEnum] = mapped_column(
+ id: Mapped[str] = mapped_column(
+ StringUUID,
+ insert_default=lambda: str(uuid4()),
+ default_factory=lambda: str(uuid4()),
+ )
+ tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False, default=cast(str, None))
+ name: Mapped[str] = mapped_column(String(255), nullable=False, default=cast(str, None))
+ description: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
+ provider: Mapped[str] = mapped_column(
+ String(255), nullable=False, server_default=sa.text("'vendor'"), default=cast(str, None)
+ )
+ permission: Mapped[Any] = mapped_column(
EnumText(DatasetPermissionEnum, length=255),
default=DatasetPermissionEnum.ONLY_ME,
)
- data_source_type = mapped_column(EnumText(DataSourceType, length=255))
- indexing_technique: Mapped[IndexTechniqueType | None] = mapped_column(EnumText(IndexTechniqueType, length=255))
- index_struct = mapped_column(LongText, nullable=True)
- created_by = mapped_column(StringUUID, nullable=False)
- maintainer: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
- created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
- updated_by = mapped_column(StringUUID, nullable=True)
- updated_at = mapped_column(
- sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
- )
- embedding_model = mapped_column(sa.String(255), nullable=True)
- embedding_model_provider = mapped_column(sa.String(255), nullable=True)
- keyword_number = mapped_column(sa.Integer, nullable=True, server_default=sa.text("10"))
- collection_binding_id = mapped_column(StringUUID, nullable=True)
- retrieval_model = mapped_column(AdjustedJSON, nullable=True)
- summary_index_setting = mapped_column(AdjustedJSON, nullable=True)
- built_in_field_enabled = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())
- icon_info = mapped_column(AdjustedJSON, nullable=True)
- runtime_mode = mapped_column(
- EnumText(DatasetRuntimeMode, length=255), nullable=True, server_default=sa.text("'general'")
- )
- pipeline_id = mapped_column(StringUUID, nullable=True)
- chunk_structure = mapped_column(sa.String(255), nullable=True)
- enable_api = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())
- is_multimodal = mapped_column(sa.Boolean, default=False, nullable=False)
+ data_source_type: Mapped[Any] = mapped_column(EnumText(DataSourceType, length=255), nullable=True, default=None)
+ indexing_technique: Mapped[Any] = mapped_column(
+ EnumText(IndexTechniqueType, length=255), nullable=True, default=None
+ )
+ index_struct: Mapped[Any] = mapped_column(LongText, nullable=True, default=None)
+ created_by: Mapped[str] = mapped_column(StringUUID, nullable=False, default=cast(str, None))
+ maintainer: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+ )
+ updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ updated_at: Mapped[datetime] = mapped_column(
+ sa.DateTime,
+ nullable=False,
+ server_default=func.current_timestamp(),
+ onupdate=func.current_timestamp(),
+ init=False,
+ )
+ embedding_model: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)
+ embedding_model_provider: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)
+ keyword_number: Mapped[Any] = mapped_column(sa.Integer, nullable=True, server_default=sa.text("10"), default=None)
+ collection_binding_id: Mapped[Any] = mapped_column(StringUUID, nullable=True, default=None)
+ retrieval_model: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)
+ summary_index_setting: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)
+ built_in_field_enabled: Mapped[bool] = mapped_column(
+ sa.Boolean, nullable=False, server_default=sa.false(), default=cast(bool, None)
+ )
+ icon_info: Mapped[Any] = mapped_column(AdjustedJSON, nullable=True, default=None)
+ runtime_mode: Mapped[Any] = mapped_column(
+ EnumText(DatasetRuntimeMode, length=255), nullable=True, server_default=sa.text("'general'"), default=None
+ )
+ pipeline_id: Mapped[Any] = mapped_column(StringUUID, nullable=True, default=None)
+ chunk_structure: Mapped[Any] = mapped_column(sa.String(255), nullable=True, default=None)
+ enable_api: Mapped[bool] = mapped_column(
+ sa.Boolean, nullable=False, server_default=sa.true(), default=cast(bool, None)
+ )
+ is_multimodal: Mapped[bool] = mapped_column(
+ sa.Boolean,
+ insert_default=False,
+ default=False,
+ nullable=False,
+ )
@property
def index_struct_dict(self):
diff --git a/api/services/rag_pipeline/rag_pipeline_dsl_service.py b/api/services/rag_pipeline/rag_pipeline_dsl_service.py
index d149346b0a7..00372527040 100644
--- a/api/services/rag_pipeline/rag_pipeline_dsl_service.py
+++ b/api/services/rag_pipeline/rag_pipeline_dsl_service.py
@@ -295,7 +295,7 @@ class RagPipelineDslService:
names = [dataset.name for dataset in datasets]
generate_name = generate_incremental_name(names, name)
dataset = Dataset(
- tenant_id=account.current_tenant_id,
+ tenant_id=cast(str, account.current_tenant_id),
name=generate_name,
description=description,
icon_info={
@@ -440,7 +440,7 @@ class RagPipelineDslService:
if not dataset:
dataset_created = True
dataset = Dataset(
- tenant_id=account.current_tenant_id,
+ tenant_id=cast(str, account.current_tenant_id),
name=name,
description=description,
icon_info={
diff --git a/api/services/rag_pipeline/rag_pipeline_transform_service.py b/api/services/rag_pipeline/rag_pipeline_transform_service.py
index fbf4dd526c3..4aa959b6d16 100644
--- a/api/services/rag_pipeline/rag_pipeline_transform_service.py
+++ b/api/services/rag_pipeline/rag_pipeline_transform_service.py
@@ -325,7 +325,7 @@ class RagPipelineTransformService:
pipeline = Pipeline(
tenant_id=dataset.tenant_id,
name=dataset.name,
- description=dataset.description,
+ description=dataset.description or "",
created_by=account_id,
)
session.add(pipeline)
diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
index d311cbfc4c8..6584ad5bfbb 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
@@ -160,9 +160,9 @@ class TestCleanDatasetTask:
index_struct='{"type": "paragraph"}',
collection_binding_id=str(uuid.uuid4()),
created_by=account.id,
- created_at=datetime.now(),
- updated_at=datetime.now(),
)
+ dataset.created_at = datetime.now()
+ dataset.updated_at = datetime.now()
db_session_with_containers.add(dataset)
db_session_with_containers.commit()
@@ -906,9 +906,9 @@ class TestCleanDatasetTask:
index_struct='{"type": "paragraph", "max_length": 10000}',
collection_binding_id=str(uuid.uuid4()),
created_by=account.id,
- created_at=datetime.now(),
- updated_at=datetime.now(),
)
+ dataset.created_at = datetime.now()
+ dataset.updated_at = datetime.now()
db_session_with_containers.add(dataset)
db_session_with_containers.commit()
diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py b/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py
index 80054e075a1..3db174ef0c1 100644
--- a/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py
+++ b/api/tests/unit_tests/controllers/console/explore/test_trial_preview.py
@@ -161,8 +161,8 @@ def harness(
data_source_type="upload_file",
indexing_technique=None,
created_by=creator_id,
- created_at=_CREATED_AT,
)
+ dataset.created_at = _CREATED_AT
with sqlite_session_factory.begin() as session:
session.add_all([owner, target, listing, config, site, dataset])
session.flush()
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
index 03aebf9a47b..5ec542c7a56 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
@@ -8,6 +8,7 @@ import uuid
from datetime import UTC, datetime
from inspect import unwrap
from types import SimpleNamespace
+from typing import cast
from unittest.mock import MagicMock, patch
import pytest
@@ -102,7 +103,11 @@ def make_dataset(
"is_multimodal": False,
}
base.update(overrides)
- dataset = Dataset(**base)
+ created_at = cast(datetime, base.pop("created_at"))
+ updated_at = cast(datetime, base.pop("updated_at"))
+ dataset = Dataset(**base) # type: ignore[bad-argument-type]
+ dataset.created_at = created_at
+ dataset.updated_at = updated_at
session.add(dataset)
session.flush()
return dataset
diff --git a/api/tests/unit_tests/model_factories.py b/api/tests/unit_tests/model_factories.py
index 3aeb0f061b5..0d6e28835fb 100644
--- a/api/tests/unit_tests/model_factories.py
+++ b/api/tests/unit_tests/model_factories.py
@@ -311,28 +311,42 @@ def make_dataset(
unset, matching a bare ``Dataset(...)``. ``dataset_id=None`` keeps the
generated identifier.
"""
- values: dict[str, object] = {"tenant_id": tenant_id, "name": name, "created_by": created_by}
- optional: dict[str, object | None] = {
- "id": dataset_id,
- "description": description,
- "provider": provider,
- "permission": permission,
- "data_source_type": data_source_type,
- "indexing_technique": indexing_technique,
- "index_struct": index_struct,
- "maintainer": maintainer,
- "created_at": created_at,
- "embedding_model": embedding_model,
- "embedding_model_provider": embedding_model_provider,
- "keyword_number": keyword_number,
- "built_in_field_enabled": built_in_field_enabled,
- "icon_info": icon_info,
- "pipeline_id": pipeline_id,
- "chunk_structure": chunk_structure,
- "enable_api": enable_api,
- }
- values.update({key: value for key, value in optional.items() if value is not None})
- return Dataset(**values)
+ result = Dataset(tenant_id=tenant_id, name=name, created_by=created_by)
+ if dataset_id is not None:
+ result.id = dataset_id
+ if description is not None:
+ result.description = description
+ if provider is not None:
+ result.provider = provider
+ if permission is not None:
+ result.permission = permission
+ if data_source_type is not None:
+ result.data_source_type = data_source_type
+ if indexing_technique is not None:
+ result.indexing_technique = indexing_technique
+ if index_struct is not None:
+ result.index_struct = index_struct
+ if maintainer is not None:
+ result.maintainer = maintainer
+ if created_at is not None:
+ result.created_at = created_at
+ if embedding_model is not None:
+ result.embedding_model = embedding_model
+ if embedding_model_provider is not None:
+ result.embedding_model_provider = embedding_model_provider
+ if keyword_number is not None:
+ result.keyword_number = keyword_number
+ if built_in_field_enabled is not None:
+ result.built_in_field_enabled = built_in_field_enabled
+ if icon_info is not None:
+ result.icon_info = icon_info
+ if pipeline_id is not None:
+ result.pipeline_id = pipeline_id
+ if chunk_structure is not None:
+ result.chunk_structure = chunk_structure
+ if enable_api is not None:
+ result.enable_api = enable_api
+ return result
def make_conversation(
diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.py
index e691a7d623b..0a39d4b4440 100644
--- a/api/tests/unit_tests/models/test_dataset_models.py
+++ b/api/tests/unit_tests/models/test_dataset_models.py
@@ -23,6 +23,7 @@ from core.rag.entities import ParentMode
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.storage.storage_type import StorageType
from models.account import Account
+from models.base import Base, TypeBase
from models.dataset import (
AppDatasetJoin,
ChildChunk,
@@ -161,6 +162,16 @@ def _make_segments(document: Document, hit_counts: list[int]) -> list[DocumentSe
class TestDatasetModelValidation:
"""Test suite for Dataset model validation and basic operations."""
+ def test_dataset_uses_typebase_registry_and_generates_ids(self):
+ generated_dataset = Dataset()
+ another_dataset = Dataset()
+ explicit_id = str(uuid4())
+
+ assert Dataset.__mapper__.registry is TypeBase.registry
+ assert Dataset.__mapper__.registry is not Base.registry
+ assert generated_dataset.id != another_dataset.id
+ assert Dataset(id=explicit_id).id == explicit_id
+
def test_dataset_creation_with_required_fields(self):
"""Test creating a dataset with all required fields."""
# Arrange
diff --git a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py b/api/tests/unit_tests/repositories/test_app_preview_query_repository.py
index 31ea2abadfc..fd37d0e613d 100644
--- a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py
+++ b/api/tests/unit_tests/repositories/test_app_preview_query_repository.py
@@ -73,19 +73,18 @@ def _add_site(session: Session, *, app_id: str, title: str) -> None:
def _add_dataset(session: Session, *, dataset_id: str, tenant_id: str) -> None:
- session.add(
- Dataset(
- id=dataset_id,
- tenant_id=tenant_id,
- name="Knowledge",
- description="Knowledge description",
- permission="only_me",
- data_source_type="upload_file",
- indexing_technique="high_quality",
- created_by=_CREATOR_ID,
- created_at=_CREATED_AT,
- )
+ dataset = Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name="Knowledge",
+ description="Knowledge description",
+ permission="only_me",
+ data_source_type="upload_file",
+ indexing_technique="high_quality",
+ created_by=_CREATOR_ID,
)
+ dataset.created_at = _CREATED_AT
+ session.add(dataset)
@pytest.fixture
diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.py
index 07364ea972c..c4410988259 100644
--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py
+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py
@@ -182,7 +182,6 @@ class DatasetServiceUnitDataFactory:
chunk_structure=doc_form,
enable_api=enable_api,
updated_by=None,
- updated_at=None,
summary_index_setting=summary_index_setting,
**kwargs,
)
diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
index f9fb45e2a01..56f8a01a302 100644
--- a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
@@ -41,18 +41,18 @@ REF = DatasetRef("tenant", "dataset")
def dataset(**values: object) -> Dataset:
- return Dataset(
- **{
- "id": "dataset",
- "tenant_id": "tenant",
- "name": "Dataset",
- "created_by": "actor",
- "maintainer": "actor",
- "indexing_technique": "economy",
- "permission": "all_team_members",
- **values,
- }
+ row = Dataset(
+ id="dataset",
+ tenant_id="tenant",
+ name="Dataset",
+ created_by="actor",
+ maintainer="actor",
+ indexing_technique="economy",
+ permission="all_team_members",
)
+ for key, value in values.items():
+ setattr(row, key, value)
+ return row
def document(**values: object) -> Document:
diff --git a/api/tests/unit_tests/services/knowledge/test_document_adapters.py b/api/tests/unit_tests/services/knowledge/test_document_adapters.py
index eb85952f5b0..a655a19e9e5 100644
--- a/api/tests/unit_tests/services/knowledge/test_document_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/test_document_adapters.py
@@ -18,18 +18,18 @@ REF = DatasetRef("tenant-1", "dataset-1")
def _dataset(**values: object) -> Dataset:
- return Dataset(
- **{
- "id": "dataset-1",
- "tenant_id": "tenant-1",
- "name": "Dataset",
- "created_by": "actor-1",
- "maintainer": "actor-1",
- "indexing_technique": "economy",
- "chunk_structure": "text_model",
- **values,
- }
+ row = Dataset(
+ id="dataset-1",
+ tenant_id="tenant-1",
+ name="Dataset",
+ created_by="actor-1",
+ maintainer="actor-1",
+ indexing_technique="economy",
+ chunk_structure="text_model",
)
+ for key, value in values.items():
+ setattr(row, key, value)
+ return row
def _document(**values: object) -> Document: