Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
chore: port Document to TypeBase (#36084)
Asuka Minato7445e4832303
diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.pyindex 5f3528b2d9a..af056d7e525 100644--- a/api/controllers/service_api/dataset/document.py+++ b/api/controllers/service_api/dataset/document.py@@ -75,6 +75,7 @@ from libs.helper import dump_responsefrom libs.login import current_userfrom libs.pagination import clamp_pagination, paginate_queryfrom models.dataset import Dataset, Document+from models.enums import DocumentCreatedFromfrom repositories.knowledge.dataset_read_repository import (get_dataset_creator,get_document_hit_count,@@ -459,7 +460,7 @@ def _create_document_by_text(session: Session, tenant_id: str, dataset_id: UUID)dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,- created_from="api",+ created_from=DocumentCreatedFrom.API,session=session,)except ProviderTokenNotInitError as ex:@@ -523,7 +524,7 @@ def _update_document_by_text(dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,- created_from="api",+ created_from=DocumentCreatedFrom.API,session=session,)except ProviderTokenNotInitError as ex:@@ -852,7 +853,7 @@ class DocumentAddByFileApi(DatasetApiResource):knowledge_config=knowledge_config,account=get_dataset_creator(dataset, session=session),dataset_process_rule=dataset_process_rule,- created_from="api",+ created_from=DocumentCreatedFrom.API,session=session,)except ProviderTokenNotInitError as ex:@@ -933,7 +934,7 @@ def _update_document_by_file(dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)if "process_rule" not in argselse None,- created_from="api",+ created_from=DocumentCreatedFrom.API,session=session,)except ProviderTokenNotInitError as ex:@@ -1324,6 +1325,7 @@ class DocumentApi(DatasetApiResource):metadata = query_params.metadataresponse_include: set[str] | None = Noneresponse_exclude: set[str] | None = None+ response: dict[str, Any]# Calculate summary_index_status if neededsummary_index_status = Nonediff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.pyindex b0e28ce5b69..58f86248ea3 100644--- a/api/core/app/apps/pipeline/pipeline_generator.py+++ b/api/core/app/apps/pipeline/pipeline_generator.py@@ -37,6 +37,7 @@ from core.datasource.entities.datasource_entities import ()from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePluginfrom core.entities.knowledge_entities import PipelineDataset, PipelineDocument+from core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.repositories.factory import (DifyCoreRepositoryFactory,WorkflowExecutionRepository,@@ -49,7 +50,7 @@ from libs.datetime_utils import naive_utc_nowfrom libs.flask_utils import preserve_flask_contextsfrom models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFromfrom models.dataset import Document, DocumentPipelineExecutionLog, Pipeline, build_built_in_field_values-from models.enums import WorkflowRunTriggeredFrom+from models.enums import DataSourceType, DocumentCreatedFrom, WorkflowRunTriggeredFromfrom models.model import AppModefrom services.data_source.provider_service import DatasourceProviderServicefrom services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy@@ -161,13 +162,14 @@ class PipelineGenerator(BaseAppGenerator):for datasource_info in datasource_info_list:position = DocumentService.get_documents_position(dataset.id, session)+ document_created_from: DocumentCreatedFrom = DocumentCreatedFrom("rag-pipeline")document = self._build_document(tenant_id=pipeline.tenant_id,dataset_id=dataset.id,built_in_field_enabled=dataset.built_in_field_enabled,datasource_type=datasource_type,datasource_info=datasource_info,- created_from="rag-pipeline",+ created_from=document_created_from,position=position,account=user,batch=batch,@@ -717,11 +719,11 @@ class PipelineGenerator(BaseAppGenerator):built_in_field_enabled: bool,datasource_type: DatasourceProviderType,datasource_info: Mapping[str, Any],- created_from: str,+ created_from: DocumentCreatedFrom,position: int,account: Account | EndUser,batch: str,- document_form: str,+ document_form: IndexStructureType,):match datasource_type:case DatasourceProviderType.LOCAL_FILE:@@ -734,11 +736,12 @@ class PipelineGenerator(BaseAppGenerator):name = datasource_info.get("name", "untitled")case _:raise ValueError(f"Unsupported datasource type: {datasource_type}")+ document_data_source_type: DataSourceType = DataSourceType(datasource_type)document = Document(tenant_id=tenant_id,dataset_id=dataset_id,position=position,- data_source_type=datasource_type,+ data_source_type=document_data_source_type,data_source_info=json.dumps(datasource_info),batch=batch,name=name,diff --git a/api/core/rag/index_processor/index_processor.py b/api/core/rag/index_processor/index_processor.pyindex 311f929ae23..16267a3907d 100644--- a/api/core/rag/index_processor/index_processor.py+++ b/api/core/rag/index_processor/index_processor.py@@ -15,7 +15,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor, Sufrom core.workflow.nodes.knowledge_index.exc import KnowledgeIndexNodeErrorfrom core.workflow.nodes.knowledge_index.protocols import IndexingResultDict, Preview, PreviewItem, QaPreviewfrom models.dataset import Dataset, Document, DocumentSegment-from models.enums import SegmentStatus+from models.enums import IndexingStatus, SegmentStatusfrom services.vector_space_admission_service import VectorSpaceAdmissionServicefrom .processor.paragraph_index_processor import ParagraphIndexProcessor@@ -157,7 +157,7 @@ class IndexProcessor:indexing_end_at = time.perf_counter()document.indexing_latency = indexing_end_at - indexing_start_at- document.indexing_status = "completed"+ document.indexing_status = IndexingStatus.COMPLETEDdocument.completed_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)document.word_count = (session.scalar(diff --git a/api/models/dataset.py b/api/models/dataset.pyindex 9c95fbf23a9..72af9ae19b8 100644--- a/api/models/dataset.py+++ b/api/models/dataset.py@@ -17,7 +17,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Indfrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom libs.datetime_utils import ensure_naive_utcfrom libs.uuid_utils import uuidv7-from models.base import Base, TypeBase+from models.base import TypeBasefrom models.enums import (CollectionBindingType,CreatorUserRole,@@ -367,7 +367,7 @@ def build_built_in_field_values(}-class Document(Base):+class Document(TypeBase):__tablename__ = "documents"__table_args__ = (sa.PrimaryKeyConstraint("id", name="document_pkey"),@@ -378,69 +378,81 @@ class Document(Base):)# initial fields- id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))- tenant_id = mapped_column(StringUUID, nullable=False)- dataset_id = mapped_column(StringUUID, nullable=False)+ id: Mapped[str] = mapped_column(StringUUID, nullable=False, default_factory=lambda: str(uuid4()), kw_only=True)+ tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)+ dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)position: Mapped[int] = mapped_column(sa.Integer, nullable=False)- data_source_type: Mapped[str] = mapped_column(EnumText(DataSourceType, length=255), nullable=False)- data_source_info = mapped_column(LongText, nullable=True)- dataset_process_rule_id = mapped_column(StringUUID, nullable=True)+ data_source_type: Mapped[DataSourceType] = mapped_column(EnumText(DataSourceType, length=255), nullable=False)batch: Mapped[str] = mapped_column(String(255), nullable=False)name: Mapped[str] = mapped_column(String(255), nullable=False)- created_from: Mapped[str] = mapped_column(EnumText(DocumentCreatedFrom, length=255), nullable=False)- created_by = mapped_column(StringUUID, nullable=False)- created_api_request_id = mapped_column(StringUUID, nullable=True)- created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())+ created_from: Mapped[DocumentCreatedFrom] = mapped_column(EnumText(DocumentCreatedFrom, length=255), nullable=False)+ created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)+ data_source_info: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)+ dataset_process_rule_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ created_api_request_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ created_at: Mapped[datetime] = mapped_column(+ DateTime, nullable=False, server_default=func.current_timestamp(), init=False+ )# start processing- processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# parsing- file_id = mapped_column(LongText, nullable=True)- word_count: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) # TODO: make this not nullable- parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ file_id: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)+ word_count: Mapped[int | None] = mapped_column(+ sa.Integer, nullable=True, default=None+ ) # TODO: make this not nullable+ parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# cleaning- cleaning_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ cleaning_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# split- splitting_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ splitting_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# indexing- tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True)- indexing_latency: Mapped[float | None] = mapped_column(sa.Float, nullable=True)- completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None)+ indexing_latency: Mapped[float | None] = mapped_column(sa.Float, nullable=True, default=None)+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# pause- is_paused: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.false())- paused_by = mapped_column(StringUUID, nullable=True)- paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ is_paused: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.false(), default=False)+ paused_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# error- error: Mapped[str | None] = mapped_column(LongText, nullable=True)- stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ error: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)+ stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)# basic fields- indexing_status: Mapped[str] = mapped_column(- EnumText(IndexingStatus, length=255), nullable=False, server_default=sa.text("'waiting'")- )- enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())- disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)- disabled_by = mapped_column(StringUUID, nullable=True)- archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())- archived_reason = mapped_column(String(255), nullable=True)- archived_by = mapped_column(StringUUID, nullable=True)- archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+ indexing_status: Mapped[IndexingStatus] = mapped_column(+ EnumText(IndexingStatus, length=255),+ nullable=False,+ server_default=sa.text("'waiting'"),+ default=IndexingStatus.WAITING,+ )+ enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.true(), default=True)+ disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)+ disabled_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false(), default=False)+ archived_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)+ archived_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)+ archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)updated_at: Mapped[datetime] = mapped_column(- DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()+ DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False+ )+ doc_type: Mapped[DocumentDocType | None] = mapped_column(+ EnumText(DocumentDocType, length=40), nullable=True, default=None)- doc_type = mapped_column(EnumText(DocumentDocType, length=40), nullable=True)- doc_metadata = mapped_column(AdjustedJSON, nullable=True)+ doc_metadata: Mapped[dict[str, Any] | None] = mapped_column(AdjustedJSON, nullable=True, default=None)doc_form: Mapped[IndexStructureType] = mapped_column(- EnumText(IndexStructureType, length=255), nullable=False, server_default=sa.text("'text_model'")+ EnumText(IndexStructureType, length=255),+ nullable=False,+ server_default=sa.text("'text_model'"),+ default=IndexStructureType.PARAGRAPH_INDEX,)- doc_language = mapped_column(String(255), nullable=True)- need_summary: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())+ doc_language: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)+ need_summary: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false(), default=False)DATA_SOURCES = ["upload_file", "notion_import", "website_crawl"]@@ -534,51 +546,6 @@ class Document(Base):}return result- @classmethod- def from_dict(cls, data: dict[str, Any]):- return cls(- id=data.get("id"),- tenant_id=data.get("tenant_id"),- dataset_id=data.get("dataset_id"),- position=data.get("position"),- data_source_type=data.get("data_source_type"),- data_source_info=data.get("data_source_info"),- dataset_process_rule_id=data.get("dataset_process_rule_id"),- batch=data.get("batch"),- name=data.get("name"),- created_from=data.get("created_from"),- created_by=data.get("created_by"),- created_api_request_id=data.get("created_api_request_id"),- created_at=data.get("created_at"),- processing_started_at=data.get("processing_started_at"),- file_id=data.get("file_id"),- word_count=data.get("word_count"),- parsing_completed_at=data.get("parsing_completed_at"),- cleaning_completed_at=data.get("cleaning_completed_at"),- splitting_completed_at=data.get("splitting_completed_at"),- tokens=data.get("tokens"),- indexing_latency=data.get("indexing_latency"),- completed_at=data.get("completed_at"),- is_paused=data.get("is_paused"),- paused_by=data.get("paused_by"),- paused_at=data.get("paused_at"),- error=data.get("error"),- stopped_at=data.get("stopped_at"),- indexing_status=data.get("indexing_status"),- enabled=data.get("enabled"),- disabled_at=data.get("disabled_at"),- disabled_by=data.get("disabled_by"),- archived=data.get("archived"),- archived_reason=data.get("archived_reason"),- archived_by=data.get("archived_by"),- archived_at=data.get("archived_at"),- updated_at=data.get("updated_at"),- doc_type=data.get("doc_type"),- doc_metadata=data.get("doc_metadata"),- doc_form=data.get("doc_form"),- doc_language=data.get("doc_language"),- )-class DocumentSegment(TypeBase):__tablename__ = "document_segments"diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.pyindex a2d738ce7a9..97aba42cb61 100644--- a/api/services/knowledge/dataset_read_service.py+++ b/api/services/knowledge/dataset_read_service.py@@ -213,7 +213,11 @@ def format_document_metadata(def document_response_values(document: Document, batch: DocumentReadBatch) -> dict[str, Any]:"""Build response-only values from a loaded snapshot without database access."""file_id = document.data_source_info_dict.get("upload_file_id")- process_rule = batch.process_rules.get(document.dataset_process_rule_id)+ process_rule = (+ batch.process_rules.get(document.dataset_process_rule_id)+ if document.dataset_process_rule_id is not None+ else None+ )if process_rule is not None and process_rule.dataset_id != document.dataset_id:process_rule = Nonereturn {diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.pyindex 818a8e9811f..25e7a9af2ce 100644--- a/api/services/knowledge/dataset_service.py+++ b/api/services/knowledge/dataset_service.py@@ -2151,7 +2151,7 @@ class DocumentService:knowledge_config: KnowledgeConfig,account: Account | Any,dataset_process_rule: DatasetProcessRule | None = None,- created_from: str = DocumentCreatedFrom.WEB,+ created_from: DocumentCreatedFrom = DocumentCreatedFrom.WEB,*,session: Session,) -> tuple[list[Document], str]:@@ -2418,7 +2418,7 @@ class DocumentService:if knowledge_config.duplicate and document:document.dataset_process_rule_id = dataset_process_rule.iddocument.updated_at = naive_utc_now()- document.created_from = created_from+ document.created_from = DocumentCreatedFrom(created_from)document.doc_form = IndexStructureType(knowledge_config.doc_form)document.doc_language = knowledge_config.doc_languagedocument.data_source_info = json.dumps(data_source_info)@@ -2465,6 +2465,8 @@ class DocumentService:)if documents:for document in documents:+ if document.data_source_info is None:+ raise ValueError("Notion document data source information is missing.")data_source_info = json.loads(document.data_source_info)exist_page_ids.append(data_source_info["notion_page_id"])exist_document[data_source_info["notion_page_id"]] = document.id@@ -2866,18 +2868,21 @@ class DocumentService:if dataset.summary_index_setting and dataset.summary_index_setting.get("enable") is True:need_summary = True+ document_data_source_type: DataSourceType = DataSourceType(data_source_type)+ document_created_from: DocumentCreatedFrom = DocumentCreatedFrom(created_from)+ document_form_type: IndexStructureType = IndexStructureType(document_form)document = Document(tenant_id=dataset.tenant_id,dataset_id=dataset.id,position=position,- data_source_type=data_source_type,+ data_source_type=document_data_source_type,data_source_info=json.dumps(data_source_info),dataset_process_rule_id=process_rule_id,batch=batch,name=name,- created_from=created_from,+ created_from=document_created_from,created_by=account.id,- doc_form=document_form,+ doc_form=document_form_type,doc_language=document_language,need_summary=need_summary,)@@ -3029,7 +3034,10 @@ class DocumentService:"only_main_content": website_info.only_main_content,"mode": "crawl",}- document.data_source_type = document_data.data_source.info_list.data_source_type+ updated_data_source_type: DataSourceType = DataSourceType(+ document_data.data_source.info_list.data_source_type+ )+ document.data_source_type = updated_data_source_typedocument.data_source_info = json.dumps(data_source_info)document.name = file_name@@ -3044,8 +3052,10 @@ class DocumentService:document.cleaning_completed_at = Nonedocument.splitting_completed_at = Nonedocument.updated_at = naive_utc_now()- document.created_from = created_from- document.doc_form = IndexStructureType(document_data.doc_form)+ updated_created_from: DocumentCreatedFrom = DocumentCreatedFrom(created_from)+ updated_doc_form: IndexStructureType = IndexStructureType(document_data.doc_form)+ document.created_from = updated_created_from+ document.doc_form = updated_doc_formsession.add(document)session.commit()# update document segmentdiff --git a/api/services/knowledge/datasets/adapters.py b/api/services/knowledge/datasets/adapters.pyindex 8a4645e986b..6e702a00ad0 100644--- a/api/services/knowledge/datasets/adapters.py+++ b/api/services/knowledge/datasets/adapters.py@@ -63,7 +63,7 @@ def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:- result = {+ result: dict[str, Any] = {"id": document.id,"indexing_status": document.indexing_status,"processing_started_at": document.processing_started_at,diff --git a/api/services/knowledge/documents/adapters.py b/api/services/knowledge/documents/adapters.pyindex 6caa431621c..ee0e18181bc 100644--- a/api/services/knowledge/documents/adapters.py+++ b/api/services/knowledge/documents/adapters.py@@ -218,7 +218,7 @@ class SQLAlchemyDocumentOperations:def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]:with self._sessions() as session:document = _require_document(session, ref)- result = {+ result: dict[str, Any] = {"id": document.id,"doc_type": document.doc_type,"doc_metadata": get_document_metadata_details(document, session=session),diff --git a/api/tasks/remove_document_from_index_task.py b/api/tasks/remove_document_from_index_task.pyindex 55e10f907c2..35213fc96ed 100644--- a/api/tasks/remove_document_from_index_task.py+++ b/api/tasks/remove_document_from_index_task.py@@ -10,6 +10,7 @@ from core.rag.index_processor.index_processor import IndexProcessorFactoryfrom extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Document, DocumentSegment+from models.enums import IndexingStatusfrom repositories.knowledge.dataset_read_repository import get_document_datasetlogger = logging.getLogger(__name__)@@ -32,7 +33,7 @@ def remove_document_from_index_task(document_id: str):logger.info(click.style(f"Document not found: {document_id}", fg="red"))return- if document.indexing_status != "completed":+ if document.indexing_status != IndexingStatus.COMPLETED:logger.info(click.style(f"Document is not completed, remove is not allowed: {document_id}", fg="red"))returndiff --git a/api/tasks/sync_website_document_indexing_task.py b/api/tasks/sync_website_document_indexing_task.pyindex 34b892cc2cf..6f873899afd 100644--- a/api/tasks/sync_website_document_indexing_task.py+++ b/api/tasks/sync_website_document_indexing_task.py@@ -12,7 +12,7 @@ from enums import DeploymentEditionfrom extensions.ext_redis import redis_clientfrom libs.datetime_utils import naive_utc_nowfrom models.dataset import Dataset, DocumentSegment-from models.enums import IndexingStatus+from models.enums import DataSourceType, IndexingStatusfrom repositories.knowledge.document_repository import _get_documentfrom services.feature_service import FeatureServicefrom services.knowledge.indexing.adapters.execution import build_document_indexing_service@@ -43,7 +43,7 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):if document is None:logger.info(click.style(f"Document not found: {document_id}", fg="yellow"))return- if document.data_source_type != "website_crawl":+ if document.data_source_type != DataSourceType.WEBSITE_CRAWL:raise ValueError("Document is not a website document")sync_indexing_cache_key = f"document_{document_id}_is_sync"diff --git a/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py b/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.pyindex 215e1e46704..e67f23d4473 100644--- a/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py+++ b/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py@@ -478,7 +478,6 @@ class TestKnowledgeRetrievalIntegration:db_session_with_containers.add(dataset)document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.pyindex dc57992a999..4ec0329757b 100644--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py@@ -6,6 +6,7 @@ from uuid import uuid4from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType+from models import AccountStatus, TenantStatusfrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import Dataset, Documentfrom models.enums import DataSourceType, DocumentCreatedFrom@@ -23,14 +24,14 @@ class DatasetDeleteIntegrationDataFactory:email=f"owner-{uuid4()}@example.com",name="Owner",interface_language="en-US",- status="active",+ status=AccountStatus.ACTIVE,)db_session_with_containers.add(account)db_session_with_containers.commit()tenant = Tenant(name=f"tenant-{uuid4()}",- status="normal",+ status=TenantStatus.NORMAL,)db_session_with_containers.add(tenant)db_session_with_containers.commit()@@ -82,7 +83,7 @@ class DatasetDeleteIntegrationDataFactory:tenant_id: str,dataset_id: str,created_by: str,- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,+ doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,) -> Document:"""Persist a document so dataset.get_doc_form resolves through the real document path."""document = Document(diff --git a/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py b/api/tests/test_containers_integration_tests/services/test_hit_testing_service.pyindex 249a8f98cf8..0d989d0f3b0 100644--- a/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py+++ b/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py@@ -15,7 +15,7 @@ from core.rag.models.document import Documentfrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom models.dataset import Dataset, DatasetQuery, DocumentSegmentfrom models.dataset import Document as DatasetDocument-from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus+from models.enums import DataSourceType, DocumentCreatedFrom, DocumentDocType, SegmentStatusfrom services.hit_testing_service import HitTestingService@@ -105,12 +105,12 @@ def _create_dataset_document(tenant_id=tenant_id,dataset_id=dataset.id,position=1,- data_source_type=data_source_type,+ data_source_type=DataSourceType(data_source_type),batch=f"batch-{uuid4()}",name=name,created_from=DocumentCreatedFrom.WEB,created_by=created_by,- doc_type=doc_type,+ doc_type=DocumentDocType(doc_type) if doc_type else None,doc_metadata=doc_metadata,)db_session.add(document)diff --git a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.pyindex 016c85e9dad..64f1287dc84 100644--- a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py+++ b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py@@ -131,6 +131,7 @@ class TestMetadataPartialUpdate:updated_doc = db_session_with_containers.get(Document, document.id)assert updated_doc is not None+ assert updated_doc.doc_metadataassert updated_doc.doc_metadata["existing_key"] == "existing_value"assert updated_doc.doc_metadata["new_key"] == "new_value"@@ -245,7 +246,7 @@ class TestMetadataPartialUpdate:updated_doc = db_session_with_containers.get(Document, document.id)assert updated_doc is not Noneassert updated_doc.doc_metadata == {"new_key": "new_value"}- assert "existing_key" not in updated_doc.doc_metadata+ assert "existing_key" not in (updated_doc.doc_metadata or {})def test_partial_update_skips_existing_binding(self,diff --git a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.pyindex 30857e22c0e..a8e22e88a97 100644--- a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py@@ -90,7 +90,6 @@ class TestAddDocumentToIndexTask:# Create documentdocument = Document(- id=fake.uuid4(),tenant_id=tenant.id,dataset_id=dataset.id,position=1,diff --git a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.pyindex d325867a391..2384e3bf2e0 100644--- a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py@@ -140,7 +140,6 @@ class TestBatchCleanDocumentTask:fake = Faker()document = Document(- id=str(uuid.uuid4()),tenant_id=account.current_tenant.id,dataset_id=dataset.id,position=0,diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.pyindex dc4fc8d7c09..4cd3160bb44 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py@@ -183,7 +183,6 @@ class TestCleanDatasetTask:Document: Created document instance"""document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=1,@@ -197,9 +196,9 @@ class TestCleanDatasetTask:archived=False,doc_form=IndexStructureType.PARAGRAPH_INDEX,word_count=100,- created_at=datetime.now(),- updated_at=datetime.now(),)+ document.created_at = datetime.now()+ document.updated_at = datetime.now()db_session_with_containers.add(document)db_session_with_containers.commit()@@ -882,7 +881,6 @@ class TestCleanDatasetTask:special_content = "Special chars: !@#$%^&*()_+-=[]{}|;':\",./<>?`~"document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=1,@@ -892,9 +890,9 @@ class TestCleanDatasetTask:name=f"test_doc_{special_content}",created_from=DocumentCreatedFrom.WEB,created_by=account.id,- created_at=datetime.now(),- updated_at=datetime.now(),)+ document.created_at = datetime.now()+ document.updated_at = datetime.now()db_session_with_containers.add(document)db_session_with_containers.commit()diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.pyindex ac43b65acdf..ed1c37c7317 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py@@ -341,7 +341,6 @@ class TestCleanNotionDocumentTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -527,7 +526,6 @@ class TestCleanNotionDocumentTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -629,7 +627,6 @@ class TestCleanNotionDocumentTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -1045,7 +1042,6 @@ class TestCleanNotionDocumentTask:# Create document with rich metadatadocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,diff --git a/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.pyindex 91131d9970f..b301da469df 100644--- a/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py@@ -102,7 +102,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -161,7 +160,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -181,7 +179,6 @@ class TestDealDatasetVectorIndexTask:# Create documentsdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -262,7 +259,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -282,7 +278,6 @@ class TestDealDatasetVectorIndexTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -415,7 +410,6 @@ class TestDealDatasetVectorIndexTask:# Create document without segmentsdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -510,7 +504,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -530,7 +523,6 @@ class TestDealDatasetVectorIndexTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -606,7 +598,6 @@ class TestDealDatasetVectorIndexTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -682,7 +673,6 @@ class TestDealDatasetVectorIndexTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -758,7 +748,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -862,7 +851,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -882,7 +870,6 @@ class TestDealDatasetVectorIndexTask:# Create documentdocument = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -959,7 +946,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -979,7 +965,6 @@ class TestDealDatasetVectorIndexTask:# Create enabled documentenabled_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -998,7 +983,6 @@ class TestDealDatasetVectorIndexTask:# Create disabled documentdisabled_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=1,@@ -1080,7 +1064,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -1100,7 +1083,6 @@ class TestDealDatasetVectorIndexTask:# Create active documentactive_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -1119,7 +1101,6 @@ class TestDealDatasetVectorIndexTask:# Create archived documentarchived_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=1,@@ -1201,7 +1182,6 @@ class TestDealDatasetVectorIndexTask:# Create a document to set the doc_form propertydocument_for_doc_form = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -1221,7 +1201,6 @@ class TestDealDatasetVectorIndexTask:# Create completed documentcompleted_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=0,@@ -1240,7 +1219,6 @@ class TestDealDatasetVectorIndexTask:# Create incomplete documentincomplete_document = Document(- id=str(uuid.uuid4()),tenant_id=tenant.id,dataset_id=dataset.id,position=1,diff --git a/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.pyindex 9ab88957de0..41162ba0a0a 100644--- a/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py@@ -150,7 +150,16 @@ class TestDeleteSegmentFromIndexTask:Document: Created test document instance"""fake = fake or Faker()- document = Document()+ document = Document(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=kwargs.get("position", 1),+ data_source_type=kwargs.get("data_source_type", "upload_file"),+ batch=kwargs.get("batch", fake.uuid4()),+ name=kwargs.get("name", f"Test Document {fake.word()}"),+ created_from=kwargs.get("created_from", DocumentCreatedFrom.API),+ created_by=account.id,+ )document.id = fake.uuid4()document.tenant_id = dataset.tenant_iddocument.dataset_id = dataset.iddiff --git a/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.pyindex e0dd84ee67e..adc34f8bb39 100644--- a/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py@@ -114,7 +114,7 @@ class TestDisableSegmentFromIndexTask:dataset: Dataset,tenant: Tenant,account: Account,- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,+ doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,) -> Document:"""Helper method to create a test document.diff --git a/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.pyindex a218336deef..a2517e0715b 100644--- a/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py@@ -135,7 +135,16 @@ class TestDisableSegmentsFromIndexTask:DatasetDocument: Created test document instance"""fake = fake or Faker()- document = DatasetDocument()+ document = DatasetDocument(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch=fake.uuid4(),+ name=f"Test Document {fake.word()}.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by=account.id,+ )document.id = fake.uuid4()document.tenant_id = dataset.tenant_iddiff --git a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.pyindex c48929bb83c..a0cdf1980ea 100644--- a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py@@ -409,7 +409,6 @@ class TestDocumentIndexingTasks:# Document with different indexing statusdoc1 = Document(- id=fake.uuid4(),tenant_id=dataset.tenant_id,dataset_id=dataset.id,position=2,@@ -426,7 +425,6 @@ class TestDocumentIndexingTasks:# Document with disabled statusdoc2 = Document(- id=fake.uuid4(),tenant_id=dataset.tenant_id,dataset_id=dataset.id,position=3,diff --git a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.pyindex c2fe023cc38..8b6062f3877 100644--- a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py@@ -7,7 +7,7 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom enums import CloudPlan, DeploymentEdition-from models import Account, Tenant, TenantAccountJoin, TenantAccountRole+from models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatusfrom models.dataset import Dataset, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatusfrom services.knowledge.indexing.errors import DocumentIsPausedError@@ -99,14 +99,14 @@ class TestDuplicateDocumentIndexingTasks:email=fake.email(),name=fake.name(),interface_language="en-US",- status="active",+ status=AccountStatus.ACTIVE,)db_session_with_containers.add(account)db_session_with_containers.commit()tenant = Tenant(name=fake.company(),- status="normal",+ status=TenantStatus.NORMAL,)db_session_with_containers.add(tenant)db_session_with_containers.commit()@@ -138,7 +138,6 @@ class TestDuplicateDocumentIndexingTasks:documents = []for i in range(document_count):document = Document(- id=fake.uuid4(),tenant_id=tenant.id,dataset_id=dataset.id,position=i,diff --git a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.pyindex 525deeca421..88db0d67b20 100644--- a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py@@ -6,6 +6,7 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.ext_redis import redis_client+from models import AccountStatus, TenantStatusfrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.dataset import ChildChunk, Dataset, DatasetProcessRule, Document, DocumentSegmentfrom models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatus@@ -52,14 +53,14 @@ class TestEnableSegmentsToIndexTask:email=fake.email(),name=fake.name(),interface_language="en-US",- status="active",+ status=AccountStatus.ACTIVE,)db_session_with_containers.add(account)db_session_with_containers.commit()tenant = Tenant(name=fake.company(),- status="normal",+ status=TenantStatus.NORMAL,)db_session_with_containers.add(tenant)db_session_with_containers.commit()@@ -89,7 +90,6 @@ class TestEnableSegmentsToIndexTask:# Create documentdocument = Document(- id=fake.uuid4(),tenant_id=tenant.id,dataset_id=dataset.id,position=1,@@ -291,7 +291,7 @@ class TestEnableSegmentsToIndexTask:# Reset document statusdocument.enabled = Truedocument.archived = False- document.indexing_status = "completed"+ document.indexing_status = IndexingStatus.COMPLETEDdb_session_with_containers.commit()# Set invalid statusdiff --git a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.pyindex 83a840b0e67..4b4c881c53b 100644--- a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py+++ b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py@@ -10,6 +10,7 @@ from typing import Literal, NotRequired, TypedDictfrom unittest.mock import MagicMockfrom models.dataset import Document+from models.enums import DataSourceType, DocumentCreatedFromfrom services.knowledge.dataset_read_service import get_document_source_detail@@ -67,6 +68,20 @@ case_2: LocalFileCase = {cases: list[AnyCase] = [case_1]+def make_document(*, data_source_type: str, data_source_info: str) -> Document:+ return Document(+ tenant_id="tenant-id",+ dataset_id="dataset-id",+ position=1,+ data_source_type=DataSourceType(data_source_type),+ batch="batch-id",+ name="document.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by="account-id",+ data_source_info=data_source_info,+ )++class TestDocumentDetailDataSourceInfo:"""Test cases for document detail API data_source_info serialization."""@@ -74,7 +89,7 @@ class TestDocumentDetailDataSourceInfo:"""Test that data_source_info_dict returns raw JSON data for all data_source_type values."""# Test data for different data_source_type valuesfor case in cases:- document = Document(+ document = make_document(data_source_type=case["data_source_type"],data_source_info=case["data_source_info"],)@@ -94,7 +109,7 @@ class TestDocumentDetailDataSourceInfo:"created_at": "2024-01-01T00:00:00Z",}- document = Document(+ document = make_document(data_source_type="local_file",data_source_info=json.dumps(test_data),)@@ -112,7 +127,7 @@ class TestDocumentDetailDataSourceInfo:"""Test that notion_import and website_crawl return raw data in data_source_detail_dict."""# Test notion_importnotion_data: NotionImportInfo = {"notion_page_id": "page-123", "workspace_id": "ws-456"}- document = Document(+ document = make_document(data_source_type="notion_import",data_source_info=json.dumps(notion_data),)@@ -123,7 +138,7 @@ class TestDocumentDetailDataSourceInfo:# Test website_crawlwebsite_data: WebsiteCrawlInfo = {"url": "https://example.com", "job_id": "job-789"}- document = Document(+ document = make_document(data_source_type="website_crawl",data_source_info=json.dumps(website_data),)@@ -135,7 +150,7 @@ class TestDocumentDetailDataSourceInfo:def test_local_file_data_source_detail_dict_without_db(self):"""Test that local_file returns empty data_source_detail_dict (this doesn't need DB context)."""# Test local_file - this should work without database context since it returns {} early- document = Document(+ document = make_document(data_source_type="local_file",data_source_info=json.dumps({"file_path": "/tmp/test.txt"}),)diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.pyindex cffa59042d1..be28636eae7 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py@@ -41,7 +41,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureTypefrom libs.datetime_utils import naive_utc_nowfrom models.account import Account, Tenantfrom models.dataset import ChildChunk, Dataset, Document, DocumentSegment, DocumentSegmentSummary-from models.enums import IndexingStatus, SegmentType+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentTypefrom services.api_token_service import CachedApiTokenfrom services.knowledge.dataset_service import DocumentService, SegmentServicefrom services.knowledge.segments.application import SegmentMutationService@@ -131,6 +131,30 @@ def _child_chunk() -> ChildChunk:return child_chunk+def _document(+ *,+ document_id: str | None = None,+ dataset_id: str = "dataset-id",+ indexing_status: IndexingStatus | str = IndexingStatus.WAITING,+ enabled: bool = True,+) -> Document:+ document = Document(+ tenant_id="tenant-id",+ dataset_id=dataset_id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-id",+ name="document.txt",+ created_from=DocumentCreatedFrom.API,+ created_by="account-id",+ indexing_status=IndexingStatus(indexing_status),+ enabled=enabled,+ )+ if document_id is not None:+ document.id = document_id+ return document++def _document_for_dataset(dataset: Dataset, document_id: str = "doc-id", doc_form: str = IndexStructureType.PARAGRAPH_INDEX) -> Document:@@ -139,10 +163,10 @@ def _document_for_dataset(dataset_id=dataset.id,tenant_id=dataset.tenant_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="Document",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by="account-1",indexing_status="completed",enabled=True,@@ -396,8 +420,8 @@ class TestSegmentServiceMockedBehavior:@pytest.fixturedef mock_document(self):"""Create mock document."""- document = Document(- id=str(uuid.uuid4()),+ document = _document(+ document_id=str(uuid.uuid4()),dataset_id=str(uuid.uuid4()),indexing_status="completed",enabled=True,@@ -562,7 +586,7 @@ class TestChildChunkServiceMockedBehavior:result = SegmentService.create_child_chunk(content="New chunk content",segment=mock_segment,- document=Document(),+ document=_document(),dataset=Dataset(),session=unbound_session,actor_id="account-id",@@ -635,7 +659,7 @@ class TestChildChunkServiceMockedBehavior:tokens=0,created_by="account-id",),- document=Document(),+ document=_document(),dataset=Dataset(),session=unbound_session,actor_id="account-id",@@ -649,28 +673,28 @@ class TestDocumentValidation:def test_document_indexing_status_completed_is_valid(self):"""Test that completed indexing status is valid."""- document = Document(+ document = _document(indexing_status="completed",)assert document.indexing_status == "completed"def test_document_indexing_status_indexing_is_invalid(self):"""Test that indexing status is invalid for segment operations."""- document = Document(+ document = _document(indexing_status="indexing",)assert document.indexing_status != "completed"def test_document_enabled_true_is_valid(self):"""Test that enabled=True is valid."""- document = Document(+ document = _document(enabled=True,)assert document.enableddef test_document_enabled_false_is_invalid(self):"""Test that enabled=False is invalid for segment operations."""- document = Document(+ document = _document(enabled=False,)assert not document.enabled@@ -926,7 +950,7 @@ class TestSegmentIndexingRequirements:)def test_valid_indexing_statuses(self, status):"""Test valid document indexing statuses."""- document = Document(+ document = _document(indexing_status=status,)assert document.indexing_status in {@@ -939,7 +963,7 @@ class TestSegmentIndexingRequirements:def test_completed_status_required_for_segments(self):"""Test that completed status is required for segment operations."""- document = Document(+ document = _document(indexing_status="completed",enabled=True,)diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.pyindex 159dddfbbc2..a89b15bd2eb 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py@@ -20,6 +20,7 @@ import jsonimport uuidfrom collections.abc import Iteratorfrom datetime import UTC, datetime+from typing import castfrom unittest.mock import Mock, PropertyMock, patchimport pytest@@ -184,7 +185,11 @@ def make_serializable_document(**overrides: object) -> Document:"word_count": None,}attrs.update(overrides)+ created_at = cast(datetime, attrs.pop("created_at"))+ updated_at = cast(datetime, attrs.pop("updated_at"))document = Document(**attrs)+ document.created_at = created_at+ document.updated_at = updated_atdocument.summary_index_status = summary_index_status # type: ignore[attr-defined]return documentdiff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.pyindex 5e943e1b4d8..ea1f12e4570 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py@@ -17,6 +17,7 @@ from core.rag.index_processor.processor.qa_index_processor import QAIndexProcessfrom core.rag.models.document import AttachmentDocument, Documentfrom models.dataset import Dataset, DocumentSegmentfrom models.dataset import Document as DatasetDocument+from models.enums import DataSourceType, DocumentCreatedFromclass _ImmediateThread:@@ -50,7 +51,17 @@ class TestQAIndexProcessor:@pytest.fixturedef dataset_document(self) -> DatasetDocument:- return DatasetDocument(id="doc-1", created_by="user-1")+ return DatasetDocument(+ id="doc-1",+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name="qa.txt",+ created_from=DocumentCreatedFrom.WEB,+ created_by="user-1",+ )@pytest.fixturedef process_rule(self) -> dict:diff --git a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.pyindex 525314f8cde..aff0cfdeb40 100644--- a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py+++ b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py@@ -17,11 +17,11 @@ from core.rag.models.document import AttachmentDocument, Documentfrom extensions.storage.storage_type import StorageTypefrom models.account import Accountfrom models.dataset import Dataset-from models.dataset import Document as DatasetDocumentfrom models.enums import CreatorUserRolefrom models.model import UploadFilefrom models.tools import ToolFilefrom tests.unit_tests.config_override import config_overrides_context+from tests.unit_tests.model_factories import make_documentdef _upload(@@ -132,7 +132,7 @@ class TestBaseIndexProcessor:with pytest.raises(NotImplementedError):processor.clean(Dataset(), None, session=unbound_session)with pytest.raises(NotImplementedError):- processor.index(Dataset(), DatasetDocument(), {}, unbound_session)+ processor.index(Dataset(), make_document(), {}, unbound_session)with pytest.raises(NotImplementedError):processor.format_preview([])diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.pyindex 9b1646e4f59..66dda219616 100644--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py@@ -55,7 +55,7 @@ from models.dataset import (SegmentAttachmentBinding,)from models.dataset import Document as DatasetDocument-from models.enums import CreatorUserRole+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom# ==================== Helper Functions ====================@@ -4258,10 +4258,10 @@ class TestDatasetRetrievalAdditionalHelpers:tenant_id=tenant_id,dataset_id=dataset_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="document",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by=user_id,indexing_status="completed",enabled=True,@@ -4386,10 +4386,10 @@ class TestDatasetRetrievalAdditionalHelpers:tenant_id=tenant_id,dataset_id=internal_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="available document",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by=creator_id,indexing_status="completed",enabled=True,@@ -4810,10 +4810,10 @@ class TestRetrieveCoverage:tenant_id=tenant_id,dataset_id=dataset_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="Document One",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by=creator_id,indexing_status="completed",enabled=True,@@ -5391,10 +5391,10 @@ class TestInternalHooksCoverage:tenant_id=tenant_id,dataset_id=dataset_id,position=position,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name=f"document-{position}",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by=creator_id,indexing_status="completed",enabled=True,@@ -5745,10 +5745,10 @@ class TestInternalHooksCoverage:tenant_id=tenant_id,dataset_id=available_id,position=position,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name=f"document-{position}",- created_from="api",+ created_from=DocumentCreatedFrom("api"),created_by=creator_id,indexing_status="completed",enabled=True,diff --git a/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.pyindex fc918dbc29a..e6313b03d98 100644--- a/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py+++ b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py@@ -13,6 +13,7 @@ from core.rag.models.document import Document as RetrievedDocumentfrom core.rag.retrieval.dataset_retrieval import DatasetRetrievalfrom core.workflow.nodes.knowledge_retrieval.retrieval import KnowledgeRetrievalRequestfrom models.dataset import Dataset, Document, DocumentSegment+from models.enums import DataSourceType, DocumentCreatedFromfrom repositories.knowledge.retrieval_source_repository import get_retrieval_source_documents@@ -24,10 +25,10 @@ def test_source_citations_do_not_fetch_unused_document_fields(sqlite_session: Setenant_id=tenant_id,dataset_id=dataset_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="source.txt",- created_from="web",+ created_from=DocumentCreatedFrom("web"),created_by=creator_id,enabled=True,archived=False,@@ -112,10 +113,10 @@ def test_source_document_read_is_scoped_and_has_no_lazy_queries(sqlite_session:tenant_id=owner,dataset_id=dataset,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="source.txt",- created_from="web",+ created_from=DocumentCreatedFrom("web"),created_by=str(uuid4()),doc_metadata=None,)diff --git a/api/tests/unit_tests/model_factories.py b/api/tests/unit_tests/model_factories.pyindex 0d6e28835fb..3f79f797657 100644--- a/api/tests/unit_tests/model_factories.py+++ b/api/tests/unit_tests/model_factories.py@@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequencefrom datetime import datetimefrom decimal import Decimal+from core.rag.index_processor.constant.index_type import IndexStructureTypefrom extensions.storage.storage_type import StorageTypefrom graphon.variables import VariableBasefrom models.account import Account, AccountStatus, Tenant, TenantAccountRole, TenantStatus@@ -465,27 +466,32 @@ def make_document(created_at: datetime | None = None,updated_at: datetime | None = None,) -> Document:- """Build a transient ``Document``; unset keywords leave their column unset."""- values: dict[str, object] = {- "tenant_id": tenant_id,- "dataset_id": dataset_id,- "position": position,- "data_source_type": data_source_type,- "batch": batch,- "name": name,- "created_from": created_from,- "created_by": created_by,- }- optional: dict[str, object | None] = {- "id": document_id,- "data_source_info": data_source_info,- "doc_form": doc_form,- "doc_metadata": doc_metadata,- "created_at": created_at,- "updated_at": updated_at,- }- values.update({key: value for key, value in optional.items() if value is not None})- return Document(**values)+ """Build a transient Document with constructor defaults.++ None leaves the document ID and optional form at their model defaults.+ None timestamps leave database-generated fields unset.+ """+ document = Document(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=position,+ data_source_type=data_source_type,+ batch=batch,+ name=name,+ created_from=DocumentCreatedFrom(created_from),+ created_by=created_by,+ data_source_info=data_source_info,+ doc_metadata=doc_metadata,+ )+ if document_id is not None:+ document.id = document_id+ if doc_form is not None:+ document.doc_form = IndexStructureType(doc_form)+ if created_at is not None:+ document.created_at = created_at+ if updated_at is not None:+ document.updated_at = updated_at+ return document__all__ = [diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.pyindex 0a39d4b4440..7bccc69fb24 100644--- a/api/tests/unit_tests/models/test_dataset_models.py+++ b/api/tests/unit_tests/models/test_dataset_models.py@@ -110,8 +110,7 @@ def _make_document(word_count: int | None = None,indexing_status: IndexingStatus = IndexingStatus.WAITING,) -> Document:- return Document(- id=document_id,+ document = Document(tenant_id=tenant_id,dataset_id=dataset_id,position=position,@@ -124,6 +123,8 @@ def _make_document(word_count=word_count,indexing_status=indexing_status,)+ document.id = document_id+ return documentdef _make_app(*, app_id: str, tenant_id: str = "tenant-1") -> App:@@ -1595,8 +1596,8 @@ class TestModelIntegration:created_from=DocumentCreatedFrom.WEB,created_by=created_by,word_count=100,- id=document_id,)+ document.id = document_id# Create segmentsegment = DocumentSegment(diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.pyindex b2e67607f93..1b6e235d223 100644--- a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py@@ -28,7 +28,7 @@ from models.dataset import (ExternalKnowledgeApis,ExternalKnowledgeBindings,)-from models.enums import CreatorUserRole+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import UploadFilefrom models.tools import ToolFilefrom repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch@@ -52,10 +52,10 @@ def _document(index: int = 1) -> Document:tenant_id="tenant-1",dataset_id="dataset-1",position=index,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="source.txt",- created_from="web",+ created_from=DocumentCreatedFrom("web"),created_by="account-1",)@@ -228,7 +228,7 @@ def test_document_metadata_respects_dataset_builtin_flag(sqlite_session: Sessiondef test_waiting_document_reports_paused_until_resumed() -> None:document = _document()- document.indexing_status = "waiting"+ document.indexing_status = IndexingStatus.WAITINGdocument.is_paused = Trueassert document.display_status == "paused"document.is_paused = Falsediff --git a/api/tests/unit_tests/repositories/knowledge/test_document_repository.py b/api/tests/unit_tests/repositories/knowledge/test_document_repository.pyindex 5820f173e93..98f7a4dc85d 100644--- a/api/tests/unit_tests/repositories/knowledge/test_document_repository.py+++ b/api/tests/unit_tests/repositories/knowledge/test_document_repository.py@@ -31,10 +31,10 @@ def _document(*,workspace_id: str = "workspace-1",dataset_id: str = "dataset-1",- source_type: str = DataSourceType.NOTION_IMPORT,+ source_type: DataSourceType = DataSourceType.NOTION_IMPORT,enabled: bool = True,archived: bool = False,- status: str = IndexingStatus.COMPLETED,+ status: IndexingStatus = IndexingStatus.COMPLETED,batch: str = "batch-1",) -> Document:source_info = (@@ -42,7 +42,7 @@ def _document(if source_type == DataSourceType.NOTION_IMPORTelse {"upload_file_id": f"file-{document_id}"})- return Document(+ row = Document(id=document_id,tenant_id=workspace_id,dataset_id=dataset_id,@@ -53,8 +53,6 @@ def _document(name=f"Document {document_id}",created_from=DocumentCreatedFrom.API,created_by="account-1",- created_at=datetime(2026, 1, 1),- updated_at=datetime(2026, 1, 1),indexing_status=status,doc_form=IndexStructureType.PARAGRAPH_INDEX,word_count=1,@@ -62,6 +60,9 @@ def _document(archived=archived,is_paused=False,)+ row.created_at = datetime(2026, 1, 1)+ row.updated_at = datetime(2026, 1, 1)+ return rowdef test_document_queries_enforce_the_complete_owner_chain(diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.pyindex c4410988259..8864c9a046f 100644--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py@@ -30,6 +30,7 @@ from models.dataset import (Document,DocumentSegment,)+from models.enums import DataSourceTypefrom models.model import UploadFilefrom services.entities.knowledge_entities.rag_pipeline_entities import (IconInfo as PipelineIconInfo,@@ -315,10 +316,11 @@ def _make_document(archived: bool = False,indexing_status: str = "completed",) -> Document:- return Document(+ document = Document(id=document_id,dataset_id=dataset_id,tenant_id=tenant_id,+ position=1,batch=batch,doc_form=doc_form,word_count=word_count,@@ -326,17 +328,19 @@ def _make_document(enabled=enabled,archived=archived,indexing_status=indexing_status,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),data_source_info="{}",completed_at=SimpleNamespace(),processing_started_at="started",parsing_completed_at="parsed",cleaning_completed_at="cleaned",splitting_completed_at="split",- updated_at=None,created_from=None,+ created_by="user-1",dataset_process_rule_id="process-rule-1",)+ document.updated_at = None # type: ignore[assignment]+ return documentdef _make_segment(diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.pyindex 56f8a01a302..0f77bf48d14 100644--- a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py+++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py@@ -21,7 +21,14 @@ from models.dataset import (DatasetQuery,DocumentSegment,)-from models.enums import CreatorUserRole, DatasetQuerySource, IndexingStatus, SegmentStatus+from models.enums import (+ CreatorUserRole,+ DatasetQuerySource,+ DataSourceType,+ DocumentCreatedFrom,+ IndexingStatus,+ SegmentStatus,+)from repositories.app.console_repository import ConsoleAppRepositoryfrom services.enterprise import rbac_servicefrom services.knowledge.dataset_access import DatasetNotFoundError@@ -56,21 +63,21 @@ def dataset(**values: object) -> Dataset:def document(**values: object) -> Document:- return Document(- **{- "id": "document",- "tenant_id": "tenant",- "dataset_id": "dataset",- "position": 1,- "data_source_type": "local_file",- "batch": "batch",- "name": "Document",- "created_from": "web",- "created_by": "actor",- "indexing_status": IndexingStatus.ERROR,- **values,- }+ row = Document(+ tenant_id="tenant",+ dataset_id="dataset",+ position=1,+ data_source_type=DataSourceType("local_file"),+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom("web"),+ created_by="actor",)+ row.id = "document"+ row.indexing_status = IndexingStatus.ERROR+ for key, value in values.items():+ setattr(row, key, value)+ return row@pytest.fixturediff --git a/api/tests/unit_tests/services/knowledge/test_document_adapters.py b/api/tests/unit_tests/services/knowledge/test_document_adapters.pyindex a655a19e9e5..a56f5374f96 100644--- a/api/tests/unit_tests/services/knowledge/test_document_adapters.py+++ b/api/tests/unit_tests/services/knowledge/test_document_adapters.py@@ -7,9 +7,9 @@ from sqlalchemy import Engine, eventfrom sqlalchemy.orm import Session, sessionmakerfrom controllers.console.datasets.datasets_document import DocumentDetailResponse, DocumentWithSegmentsListResponse-from core.rag.index_processor.constant.index_type import IndexTechniqueType+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models.dataset import Dataset, DatasetProcessRule, Document, DocumentPipelineExecutionLog, DocumentSegment-from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatusfrom services.knowledge.documents.adapters import SQLAlchemyDocumentOperationsfrom services.knowledge.documents.application import DocumentListFilter, DocumentNotFoundErrorfrom services.knowledge.resource_scope import DatasetRef@@ -33,27 +33,27 @@ def _dataset(**values: object) -> Dataset:def _document(**values: object) -> Document:- return Document(- **{- "id": "document-1",- "tenant_id": "tenant-1",- "dataset_id": "dataset-1",- "position": 1,- "data_source_type": "local_file",- "data_source_info": json.dumps({"file_path": "/path/to/file"}),- "batch": "batch-1",- "name": "Document",- "created_from": "web",- "created_by": "actor-1",- "indexing_status": IndexingStatus.COMPLETED,- "doc_form": "text_model",- "enabled": True,- "archived": False,- "word_count": 10,- "created_at": datetime(2024, 1, 1),- **values,- }+ row = Document(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ position=1,+ data_source_type=DataSourceType("local_file"),+ batch="batch-1",+ name="Document",+ created_from=DocumentCreatedFrom("web"),+ created_by="actor-1",)+ row.id = "document-1"+ row.data_source_info = json.dumps({"file_path": "/path/to/file"})+ row.indexing_status = IndexingStatus.COMPLETED+ row.doc_form = IndexStructureType.PARAGRAPH_INDEX+ row.enabled = True+ row.archived = False+ row.word_count = 10+ row.created_at = datetime(2024, 1, 1)+ for key, value in values.items():+ setattr(row, key, value)+ return rowdef _segment(diff --git a/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.pyindex b0885001843..f9f99d11e88 100644--- a/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py+++ b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py@@ -37,7 +37,7 @@ def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment,name="Document",created_from=DocumentCreatedFrom.WEB,created_by="admin",- doc_form="text_model",+ doc_form=IndexStructureType("text_model"),)segment = DocumentSegment(tenant_id=dataset.tenant_id,diff --git a/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.pyindex 23ce2b19dd3..d149c1d266d 100644--- a/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py+++ b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py@@ -11,6 +11,7 @@ from sqlalchemy.orm import Sessionfrom core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntityfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.app.file_access import FileAccessScope, bind_file_access_scope, get_current_file_access_scope+from core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.rag.models.document import Document as RagDocumentfrom core.rag.retrieval import dataset_retrieval as retrieval_modulefrom core.rag.retrieval.dataset_retrieval import DatasetRetrieval@@ -35,7 +36,7 @@ def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment,name="Document",created_from=DocumentCreatedFrom.WEB,created_by="admin",- doc_form="text_model",+ doc_form=IndexStructureType("text_model"),doc_metadata={},)segment = DocumentSegment(diff --git a/api/tests/unit_tests/services/knowledge/test_segment_adapters.py b/api/tests/unit_tests/services/knowledge/test_segment_adapters.pyindex 89a6b491316..427b76127d2 100644--- a/api/tests/unit_tests/services/knowledge/test_segment_adapters.py+++ b/api/tests/unit_tests/services/knowledge/test_segment_adapters.py@@ -28,7 +28,7 @@ from libs.helper import generate_text_hashfrom libs.pagination import PaginatedResultfrom machinery.context import RequestContextfrom models.dataset import ChildChunk, Dataset, Document, DocumentSegment, SegmentAttachmentBinding-from models.enums import CreatorUserRole, DocumentCreatedFrom, SegmentStatus, SummaryStatus+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatusfrom models.model import UploadFilefrom repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepositoryfrom repositories.knowledge.segment_repository import SQLAlchemySegmentRepository@@ -100,12 +100,12 @@ def _document(document_id: str, workspace_id: str, dataset_id: str) -> Document:tenant_id=workspace_id,dataset_id=dataset_id,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),batch="batch-1",name="Document",created_from=DocumentCreatedFrom.API,created_by="account-1",- doc_form="text_model",+ doc_form=IndexStructureType("text_model"),word_count=0,)diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.pyindex 702e7902065..40bce008f4f 100644--- a/api/tests/unit_tests/services/test_dataset_service_document.py+++ b/api/tests/unit_tests/services/test_dataset_service_document.py@@ -97,7 +97,7 @@ def _document_row(archived: bool = False,is_paused: bool = False,) -> Document:- return Document(+ document = Document(id=document_id,tenant_id=tenant_id,dataset_id=dataset_id,@@ -108,8 +108,6 @@ def _document_row(name=name,created_from=DocumentCreatedFrom.API,created_by="user-1",- created_at=datetime(2026, 1, 1),- updated_at=datetime(2026, 1, 2),indexing_status=indexing_status,doc_form=IndexStructureType.PARAGRAPH_INDEX,word_count=10,@@ -118,6 +116,9 @@ def _document_row(is_paused=is_paused,completed_at=datetime(2026, 1, 2) if indexing_status == IndexingStatus.COMPLETED else None,)+ document.created_at = datetime(2026, 1, 1)+ document.updated_at = datetime(2026, 1, 2)+ return documentdef _upload_file(*, file_id: str, tenant_id: str = "tenant-1", name: str = "upload.txt") -> UploadFile:diff --git a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.pyindex a143ee76e81..19599e6fe9f 100644--- a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py+++ b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py@@ -9,7 +9,7 @@ from sqlalchemy.orm import Sessionfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models.account import Account, Tenantfrom models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment-from models.enums import ProcessRuleMode+from models.enums import DataSourceType, DocumentCreatedFrom, ProcessRuleModefrom services.knowledge.dataset_service import DocumentService, SegmentServicefrom services.knowledge.entities.knowledge_entities import DataSource, InfoList, KnowledgeConfigfrom services.knowledge.segments.application import SegmentMutationService@@ -166,11 +166,11 @@ def test_multi_create_segment_ignores_lock_not_owned(tenant_id=TENANT_ID,dataset_id=DATASET_ID,position=1,- data_source_type="upload_file",+ data_source_type=DataSourceType("upload_file"),data_source_info="{}",batch="batch-1",name="Test Document",- created_from="web",+ created_from=DocumentCreatedFrom("web"),created_by=USER_ID,word_count=0,doc_form=IndexStructureType.QA_INDEX,diff --git a/api/tests/unit_tests/services/test_dataset_service_segment.py b/api/tests/unit_tests/services/test_dataset_service_segment.pyindex f848684aee1..527041a6497 100644--- a/api/tests/unit_tests/services/test_dataset_service_segment.py+++ b/api/tests/unit_tests/services/test_dataset_service_segment.py@@ -59,7 +59,7 @@ def _document(doc_form: str = IndexStructureType.PARAGRAPH_INDEX,word_count: int = 20,) -> Document:- return Document(+ document = Document(id=document_id,tenant_id=tenant_id,dataset_id=dataset_id,@@ -69,13 +69,15 @@ def _document(name="Document",created_from=DocumentCreatedFrom.API,created_by="user-1",- created_at=datetime(2026, 1, 1),- updated_at=datetime(2026, 1, 2),indexing_status="completed",doc_form=doc_form,word_count=word_count,)+ document.created_at = datetime(2026, 1, 1)+ document.updated_at = datetime(2026, 1, 2)+ return document+def _segment(*,diff --git a/api/tests/unit_tests/services/test_summary_index_service.py b/api/tests/unit_tests/services/test_summary_index_service.pyindex c8fb1a36e65..c60a9ee7bf0 100644--- a/api/tests/unit_tests/services/test_summary_index_service.py+++ b/api/tests/unit_tests/services/test_summary_index_service.py@@ -14,7 +14,7 @@ from sqlalchemy.orm import Session, sessionmakerimport services.knowledge.summaries.adapters as summary_modulefrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummary-from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatus+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus, SummaryStatusfrom services.knowledge.summaries.adapters import SummaryIndexAdapterTENANT_ID = "tenant-1"@@ -75,7 +75,7 @@ def _persist_document(name=f"Document {document_id}",created_from=DocumentCreatedFrom.WEB,created_by="account-1",- indexing_status="completed",+ indexing_status=IndexingStatus.COMPLETED,enabled=True,archived=False,doc_form=doc_form,