Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

chore: port Document to TypeBase (#36084)

Asuka Minato7445e4832303
diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py
index 5f3528b2d9a..af056d7e525 100644
--- a/api/controllers/service_api/dataset/document.py
+++ b/api/controllers/service_api/dataset/document.py
@@ -75,6 +75,7 @@ from libs.helper import dump_response
from libs.login import current_user
from libs.pagination import clamp_pagination, paginate_query
from models.dataset import Dataset, Document
+from models.enums import DocumentCreatedFrom
from repositories.knowledge.dataset_read_repository import (
get_dataset_creator,
get_document_hit_count,
@@ -459,7 +460,7 @@ def _create_document_by_text(session: Session, tenant_id: str, dataset_id: UUID)
dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)
if "process_rule" not in args
else None,
- created_from="api",
+ created_from=DocumentCreatedFrom.API,
session=session,
)
except ProviderTokenNotInitError as ex:
@@ -523,7 +524,7 @@ def _update_document_by_text(
dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)
if "process_rule" not in args
else None,
- created_from="api",
+ created_from=DocumentCreatedFrom.API,
session=session,
)
except ProviderTokenNotInitError as ex:
@@ -852,7 +853,7 @@ class DocumentAddByFileApi(DatasetApiResource):
knowledge_config=knowledge_config,
account=get_dataset_creator(dataset, session=session),
dataset_process_rule=dataset_process_rule,
- created_from="api",
+ created_from=DocumentCreatedFrom.API,
session=session,
)
except ProviderTokenNotInitError as ex:
@@ -933,7 +934,7 @@ def _update_document_by_file(
dataset_process_rule=get_latest_dataset_process_rule(dataset, session=session)
if "process_rule" not in args
else None,
- created_from="api",
+ created_from=DocumentCreatedFrom.API,
session=session,
)
except ProviderTokenNotInitError as ex:
@@ -1324,6 +1325,7 @@ class DocumentApi(DatasetApiResource):
metadata = query_params.metadata
response_include: set[str] | None = None
response_exclude: set[str] | None = None
+ response: dict[str, Any]
# Calculate summary_index_status if needed
summary_index_status = None
diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.py
index b0e28ce5b69..58f86248ea3 100644
--- a/api/core/app/apps/pipeline/pipeline_generator.py
+++ b/api/core/app/apps/pipeline/pipeline_generator.py
@@ -37,6 +37,7 @@ from core.datasource.entities.datasource_entities import (
)
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.entities.knowledge_entities import PipelineDataset, PipelineDocument
+from core.rag.index_processor.constant.index_type import IndexStructureType
from core.repositories.factory import (
DifyCoreRepositoryFactory,
WorkflowExecutionRepository,
@@ -49,7 +50,7 @@ from libs.datetime_utils import naive_utc_now
from libs.flask_utils import preserve_flask_contexts
from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom
from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline, build_built_in_field_values
-from models.enums import WorkflowRunTriggeredFrom
+from models.enums import DataSourceType, DocumentCreatedFrom, WorkflowRunTriggeredFrom
from models.model import AppMode
from services.data_source.provider_service import DatasourceProviderService
from services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy
@@ -161,13 +162,14 @@ class PipelineGenerator(BaseAppGenerator):
for datasource_info in datasource_info_list:
position = DocumentService.get_documents_position(dataset.id, session)
+ document_created_from: DocumentCreatedFrom = DocumentCreatedFrom("rag-pipeline")
document = self._build_document(
tenant_id=pipeline.tenant_id,
dataset_id=dataset.id,
built_in_field_enabled=dataset.built_in_field_enabled,
datasource_type=datasource_type,
datasource_info=datasource_info,
- created_from="rag-pipeline",
+ created_from=document_created_from,
position=position,
account=user,
batch=batch,
@@ -717,11 +719,11 @@ class PipelineGenerator(BaseAppGenerator):
built_in_field_enabled: bool,
datasource_type: DatasourceProviderType,
datasource_info: Mapping[str, Any],
- created_from: str,
+ created_from: DocumentCreatedFrom,
position: int,
account: Account | EndUser,
batch: str,
- document_form: str,
+ document_form: IndexStructureType,
):
match datasource_type:
case DatasourceProviderType.LOCAL_FILE:
@@ -734,11 +736,12 @@ class PipelineGenerator(BaseAppGenerator):
name = datasource_info.get("name", "untitled")
case _:
raise ValueError(f"Unsupported datasource type: {datasource_type}")
+ document_data_source_type: DataSourceType = DataSourceType(datasource_type)
document = Document(
tenant_id=tenant_id,
dataset_id=dataset_id,
position=position,
- data_source_type=datasource_type,
+ data_source_type=document_data_source_type,
data_source_info=json.dumps(datasource_info),
batch=batch,
name=name,
diff --git a/api/core/rag/index_processor/index_processor.py b/api/core/rag/index_processor/index_processor.py
index 311f929ae23..16267a3907d 100644
--- a/api/core/rag/index_processor/index_processor.py
+++ b/api/core/rag/index_processor/index_processor.py
@@ -15,7 +15,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor, Su
from core.workflow.nodes.knowledge_index.exc import KnowledgeIndexNodeError
from core.workflow.nodes.knowledge_index.protocols import IndexingResultDict, Preview, PreviewItem, QaPreview
from models.dataset import Dataset, Document, DocumentSegment
-from models.enums import SegmentStatus
+from models.enums import IndexingStatus, SegmentStatus
from services.vector_space_admission_service import VectorSpaceAdmissionService
from .processor.paragraph_index_processor import ParagraphIndexProcessor
@@ -157,7 +157,7 @@ class IndexProcessor:
indexing_end_at = time.perf_counter()
document.indexing_latency = indexing_end_at - indexing_start_at
- document.indexing_status = "completed"
+ document.indexing_status = IndexingStatus.COMPLETED
document.completed_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
document.word_count = (
session.scalar(
diff --git a/api/models/dataset.py b/api/models/dataset.py
index 9c95fbf23a9..72af9ae19b8 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -17,7 +17,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureType, Ind
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from libs.datetime_utils import ensure_naive_utc
from libs.uuid_utils import uuidv7
-from models.base import Base, TypeBase
+from models.base import TypeBase
from models.enums import (
CollectionBindingType,
CreatorUserRole,
@@ -367,7 +367,7 @@ def build_built_in_field_values(
}
-class Document(Base):
+class Document(TypeBase):
__tablename__ = "documents"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="document_pkey"),
@@ -378,69 +378,81 @@ class Document(Base):
)
# initial fields
- id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))
- tenant_id = mapped_column(StringUUID, nullable=False)
- dataset_id = mapped_column(StringUUID, nullable=False)
+ id: Mapped[str] = mapped_column(StringUUID, nullable=False, default_factory=lambda: str(uuid4()), kw_only=True)
+ tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+ dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
- data_source_type: Mapped[str] = mapped_column(EnumText(DataSourceType, length=255), nullable=False)
- data_source_info = mapped_column(LongText, nullable=True)
- dataset_process_rule_id = mapped_column(StringUUID, nullable=True)
+ data_source_type: Mapped[DataSourceType] = mapped_column(EnumText(DataSourceType, length=255), nullable=False)
batch: Mapped[str] = mapped_column(String(255), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
- created_from: Mapped[str] = mapped_column(EnumText(DocumentCreatedFrom, length=255), nullable=False)
- created_by = mapped_column(StringUUID, nullable=False)
- created_api_request_id = mapped_column(StringUUID, nullable=True)
- created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+ created_from: Mapped[DocumentCreatedFrom] = mapped_column(EnumText(DocumentCreatedFrom, length=255), nullable=False)
+ created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
+ data_source_info: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
+ dataset_process_rule_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ created_api_request_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+ )
# start processing
- processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# parsing
- file_id = mapped_column(LongText, nullable=True)
- word_count: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) # TODO: make this not nullable
- parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ file_id: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
+ word_count: Mapped[int | None] = mapped_column(
+ sa.Integer, nullable=True, default=None
+ ) # TODO: make this not nullable
+ parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# cleaning
- cleaning_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ cleaning_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# split
- splitting_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ splitting_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# indexing
- tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True)
- indexing_latency: Mapped[float | None] = mapped_column(sa.Float, nullable=True)
- completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None)
+ indexing_latency: Mapped[float | None] = mapped_column(sa.Float, nullable=True, default=None)
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# pause
- is_paused: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.false())
- paused_by = mapped_column(StringUUID, nullable=True)
- paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ is_paused: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.false(), default=False)
+ paused_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# error
- error: Mapped[str | None] = mapped_column(LongText, nullable=True)
- stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ error: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
+ stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# basic fields
- indexing_status: Mapped[str] = mapped_column(
- EnumText(IndexingStatus, length=255), nullable=False, server_default=sa.text("'waiting'")
- )
- enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.true())
- disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
- disabled_by = mapped_column(StringUUID, nullable=True)
- archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())
- archived_reason = mapped_column(String(255), nullable=True)
- archived_by = mapped_column(StringUUID, nullable=True)
- archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+ indexing_status: Mapped[IndexingStatus] = mapped_column(
+ EnumText(IndexingStatus, length=255),
+ nullable=False,
+ server_default=sa.text("'waiting'"),
+ default=IndexingStatus.WAITING,
+ )
+ enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.true(), default=True)
+ disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
+ disabled_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false(), default=False)
+ archived_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
+ archived_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
+ archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
updated_at: Mapped[datetime] = mapped_column(
- DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+ DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
+ )
+ doc_type: Mapped[DocumentDocType | None] = mapped_column(
+ EnumText(DocumentDocType, length=40), nullable=True, default=None
)
- doc_type = mapped_column(EnumText(DocumentDocType, length=40), nullable=True)
- doc_metadata = mapped_column(AdjustedJSON, nullable=True)
+ doc_metadata: Mapped[dict[str, Any] | None] = mapped_column(AdjustedJSON, nullable=True, default=None)
doc_form: Mapped[IndexStructureType] = mapped_column(
- EnumText(IndexStructureType, length=255), nullable=False, server_default=sa.text("'text_model'")
+ EnumText(IndexStructureType, length=255),
+ nullable=False,
+ server_default=sa.text("'text_model'"),
+ default=IndexStructureType.PARAGRAPH_INDEX,
)
- doc_language = mapped_column(String(255), nullable=True)
- need_summary: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false())
+ doc_language: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
+ need_summary: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.false(), default=False)
DATA_SOURCES = ["upload_file", "notion_import", "website_crawl"]
@@ -534,51 +546,6 @@ class Document(Base):
}
return result
- @classmethod
- def from_dict(cls, data: dict[str, Any]):
- return cls(
- id=data.get("id"),
- tenant_id=data.get("tenant_id"),
- dataset_id=data.get("dataset_id"),
- position=data.get("position"),
- data_source_type=data.get("data_source_type"),
- data_source_info=data.get("data_source_info"),
- dataset_process_rule_id=data.get("dataset_process_rule_id"),
- batch=data.get("batch"),
- name=data.get("name"),
- created_from=data.get("created_from"),
- created_by=data.get("created_by"),
- created_api_request_id=data.get("created_api_request_id"),
- created_at=data.get("created_at"),
- processing_started_at=data.get("processing_started_at"),
- file_id=data.get("file_id"),
- word_count=data.get("word_count"),
- parsing_completed_at=data.get("parsing_completed_at"),
- cleaning_completed_at=data.get("cleaning_completed_at"),
- splitting_completed_at=data.get("splitting_completed_at"),
- tokens=data.get("tokens"),
- indexing_latency=data.get("indexing_latency"),
- completed_at=data.get("completed_at"),
- is_paused=data.get("is_paused"),
- paused_by=data.get("paused_by"),
- paused_at=data.get("paused_at"),
- error=data.get("error"),
- stopped_at=data.get("stopped_at"),
- indexing_status=data.get("indexing_status"),
- enabled=data.get("enabled"),
- disabled_at=data.get("disabled_at"),
- disabled_by=data.get("disabled_by"),
- archived=data.get("archived"),
- archived_reason=data.get("archived_reason"),
- archived_by=data.get("archived_by"),
- archived_at=data.get("archived_at"),
- updated_at=data.get("updated_at"),
- doc_type=data.get("doc_type"),
- doc_metadata=data.get("doc_metadata"),
- doc_form=data.get("doc_form"),
- doc_language=data.get("doc_language"),
- )
-
class DocumentSegment(TypeBase):
__tablename__ = "document_segments"
diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.py
index a2d738ce7a9..97aba42cb61 100644
--- a/api/services/knowledge/dataset_read_service.py
+++ b/api/services/knowledge/dataset_read_service.py
@@ -213,7 +213,11 @@ def format_document_metadata(
def document_response_values(document: Document, batch: DocumentReadBatch) -> dict[str, Any]:
"""Build response-only values from a loaded snapshot without database access."""
file_id = document.data_source_info_dict.get("upload_file_id")
- process_rule = batch.process_rules.get(document.dataset_process_rule_id)
+ process_rule = (
+ batch.process_rules.get(document.dataset_process_rule_id)
+ if document.dataset_process_rule_id is not None
+ else None
+ )
if process_rule is not None and process_rule.dataset_id != document.dataset_id:
process_rule = None
return {
diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py
index 818a8e9811f..25e7a9af2ce 100644
--- a/api/services/knowledge/dataset_service.py
+++ b/api/services/knowledge/dataset_service.py
@@ -2151,7 +2151,7 @@ class DocumentService:
knowledge_config: KnowledgeConfig,
account: Account | Any,
dataset_process_rule: DatasetProcessRule | None = None,
- created_from: str = DocumentCreatedFrom.WEB,
+ created_from: DocumentCreatedFrom = DocumentCreatedFrom.WEB,
*,
session: Session,
) -> tuple[list[Document], str]:
@@ -2418,7 +2418,7 @@ class DocumentService:
if knowledge_config.duplicate and document:
document.dataset_process_rule_id = dataset_process_rule.id
document.updated_at = naive_utc_now()
- document.created_from = created_from
+ document.created_from = DocumentCreatedFrom(created_from)
document.doc_form = IndexStructureType(knowledge_config.doc_form)
document.doc_language = knowledge_config.doc_language
document.data_source_info = json.dumps(data_source_info)
@@ -2465,6 +2465,8 @@ class DocumentService:
)
if documents:
for document in documents:
+ if document.data_source_info is None:
+ raise ValueError("Notion document data source information is missing.")
data_source_info = json.loads(document.data_source_info)
exist_page_ids.append(data_source_info["notion_page_id"])
exist_document[data_source_info["notion_page_id"]] = document.id
@@ -2866,18 +2868,21 @@ class DocumentService:
if dataset.summary_index_setting and dataset.summary_index_setting.get("enable") is True:
need_summary = True
+ document_data_source_type: DataSourceType = DataSourceType(data_source_type)
+ document_created_from: DocumentCreatedFrom = DocumentCreatedFrom(created_from)
+ document_form_type: IndexStructureType = IndexStructureType(document_form)
document = Document(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
position=position,
- data_source_type=data_source_type,
+ data_source_type=document_data_source_type,
data_source_info=json.dumps(data_source_info),
dataset_process_rule_id=process_rule_id,
batch=batch,
name=name,
- created_from=created_from,
+ created_from=document_created_from,
created_by=account.id,
- doc_form=document_form,
+ doc_form=document_form_type,
doc_language=document_language,
need_summary=need_summary,
)
@@ -3029,7 +3034,10 @@ class DocumentService:
"only_main_content": website_info.only_main_content,
"mode": "crawl",
}
- document.data_source_type = document_data.data_source.info_list.data_source_type
+ updated_data_source_type: DataSourceType = DataSourceType(
+ document_data.data_source.info_list.data_source_type
+ )
+ document.data_source_type = updated_data_source_type
document.data_source_info = json.dumps(data_source_info)
document.name = file_name
@@ -3044,8 +3052,10 @@ class DocumentService:
document.cleaning_completed_at = None
document.splitting_completed_at = None
document.updated_at = naive_utc_now()
- document.created_from = created_from
- document.doc_form = IndexStructureType(document_data.doc_form)
+ updated_created_from: DocumentCreatedFrom = DocumentCreatedFrom(created_from)
+ updated_doc_form: IndexStructureType = IndexStructureType(document_data.doc_form)
+ document.created_from = updated_created_from
+ document.doc_form = updated_doc_form
session.add(document)
session.commit()
# update document segment
diff --git a/api/services/knowledge/datasets/adapters.py b/api/services/knowledge/datasets/adapters.py
index 8a4645e986b..6e702a00ad0 100644
--- a/api/services/knowledge/datasets/adapters.py
+++ b/api/services/knowledge/datasets/adapters.py
@@ -63,7 +63,7 @@ def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:
- result = {
+ result: dict[str, Any] = {
"id": document.id,
"indexing_status": document.indexing_status,
"processing_started_at": document.processing_started_at,
diff --git a/api/services/knowledge/documents/adapters.py b/api/services/knowledge/documents/adapters.py
index 6caa431621c..ee0e18181bc 100644
--- a/api/services/knowledge/documents/adapters.py
+++ b/api/services/knowledge/documents/adapters.py
@@ -218,7 +218,7 @@ class SQLAlchemyDocumentOperations:
def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]:
with self._sessions() as session:
document = _require_document(session, ref)
- result = {
+ result: dict[str, Any] = {
"id": document.id,
"doc_type": document.doc_type,
"doc_metadata": get_document_metadata_details(document, session=session),
diff --git a/api/tasks/remove_document_from_index_task.py b/api/tasks/remove_document_from_index_task.py
index 55e10f907c2..35213fc96ed 100644
--- a/api/tasks/remove_document_from_index_task.py
+++ b/api/tasks/remove_document_from_index_task.py
@@ -10,6 +10,7 @@ from core.rag.index_processor.index_processor import IndexProcessorFactory
from extensions.ext_redis import redis_client
from libs.datetime_utils import naive_utc_now
from models.dataset import Document, DocumentSegment
+from models.enums import IndexingStatus
from repositories.knowledge.dataset_read_repository import get_document_dataset
logger = logging.getLogger(__name__)
@@ -32,7 +33,7 @@ def remove_document_from_index_task(document_id: str):
logger.info(click.style(f"Document not found: {document_id}", fg="red"))
return
- if document.indexing_status != "completed":
+ if document.indexing_status != IndexingStatus.COMPLETED:
logger.info(click.style(f"Document is not completed, remove is not allowed: {document_id}", fg="red"))
return
diff --git a/api/tasks/sync_website_document_indexing_task.py b/api/tasks/sync_website_document_indexing_task.py
index 34b892cc2cf..6f873899afd 100644
--- a/api/tasks/sync_website_document_indexing_task.py
+++ b/api/tasks/sync_website_document_indexing_task.py
@@ -12,7 +12,7 @@ from enums import DeploymentEdition
from extensions.ext_redis import redis_client
from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, DocumentSegment
-from models.enums import IndexingStatus
+from models.enums import DataSourceType, IndexingStatus
from repositories.knowledge.document_repository import _get_document
from services.feature_service import FeatureService
from services.knowledge.indexing.adapters.execution import build_document_indexing_service
@@ -43,7 +43,7 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):
if document is None:
logger.info(click.style(f"Document not found: {document_id}", fg="yellow"))
return
- if document.data_source_type != "website_crawl":
+ if document.data_source_type != DataSourceType.WEBSITE_CRAWL:
raise ValueError("Document is not a website document")
sync_indexing_cache_key = f"document_{document_id}_is_sync"
diff --git a/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py b/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py
index 215e1e46704..e67f23d4473 100644
--- a/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py
+++ b/api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py
@@ -478,7 +478,6 @@ class TestKnowledgeRetrievalIntegration:
db_session_with_containers.add(dataset)
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py
index dc57992a999..4ec0329757b 100644
--- a/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py
+++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_delete_dataset.py
@@ -6,6 +6,7 @@ from uuid import uuid4
from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
+from models import AccountStatus, TenantStatus
from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
from models.dataset import Dataset, Document
from models.enums import DataSourceType, DocumentCreatedFrom
@@ -23,14 +24,14 @@ class DatasetDeleteIntegrationDataFactory:
email=f"owner-{uuid4()}@example.com",
name="Owner",
interface_language="en-US",
- status="active",
+ status=AccountStatus.ACTIVE,
)
db_session_with_containers.add(account)
db_session_with_containers.commit()
tenant = Tenant(
name=f"tenant-{uuid4()}",
- status="normal",
+ status=TenantStatus.NORMAL,
)
db_session_with_containers.add(tenant)
db_session_with_containers.commit()
@@ -82,7 +83,7 @@ class DatasetDeleteIntegrationDataFactory:
tenant_id: str,
dataset_id: str,
created_by: str,
- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,
+ doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,
) -> Document:
"""Persist a document so dataset.get_doc_form resolves through the real document path."""
document = Document(
diff --git a/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py b/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py
index 249a8f98cf8..0d989d0f3b0 100644
--- a/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py
+++ b/api/tests/test_containers_integration_tests/services/test_hit_testing_service.py
@@ -15,7 +15,7 @@ from core.rag.models.document import Document
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from models.dataset import Dataset, DatasetQuery, DocumentSegment
from models.dataset import Document as DatasetDocument
-from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus
+from models.enums import DataSourceType, DocumentCreatedFrom, DocumentDocType, SegmentStatus
from services.hit_testing_service import HitTestingService
@@ -105,12 +105,12 @@ def _create_dataset_document(
tenant_id=tenant_id,
dataset_id=dataset.id,
position=1,
- data_source_type=data_source_type,
+ data_source_type=DataSourceType(data_source_type),
batch=f"batch-{uuid4()}",
name=name,
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
- doc_type=doc_type,
+ doc_type=DocumentDocType(doc_type) if doc_type else None,
doc_metadata=doc_metadata,
)
db_session.add(document)
diff --git a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py
index 016c85e9dad..64f1287dc84 100644
--- a/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py
+++ b/api/tests/test_containers_integration_tests/services/test_metadata_partial_update.py
@@ -131,6 +131,7 @@ class TestMetadataPartialUpdate:
updated_doc = db_session_with_containers.get(Document, document.id)
assert updated_doc is not None
+ assert updated_doc.doc_metadata
assert updated_doc.doc_metadata["existing_key"] == "existing_value"
assert updated_doc.doc_metadata["new_key"] == "new_value"
@@ -245,7 +246,7 @@ class TestMetadataPartialUpdate:
updated_doc = db_session_with_containers.get(Document, document.id)
assert updated_doc is not None
assert updated_doc.doc_metadata == {"new_key": "new_value"}
- assert "existing_key" not in updated_doc.doc_metadata
+ assert "existing_key" not in (updated_doc.doc_metadata or {})
def test_partial_update_skips_existing_binding(
self,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py
index 30857e22c0e..a8e22e88a97 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_add_document_to_index_task.py
@@ -90,7 +90,6 @@ class TestAddDocumentToIndexTask:
# Create document
document = Document(
- id=fake.uuid4(),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py
index d325867a391..2384e3bf2e0 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py
@@ -140,7 +140,6 @@ class TestBatchCleanDocumentTask:
fake = Faker()
document = Document(
- id=str(uuid.uuid4()),
tenant_id=account.current_tenant.id,
dataset_id=dataset.id,
position=0,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
index dc4fc8d7c09..4cd3160bb44 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
@@ -183,7 +183,6 @@ class TestCleanDatasetTask:
Document: Created document instance
"""
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
@@ -197,9 +196,9 @@ class TestCleanDatasetTask:
archived=False,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
word_count=100,
- created_at=datetime.now(),
- updated_at=datetime.now(),
)
+ document.created_at = datetime.now()
+ document.updated_at = datetime.now()
db_session_with_containers.add(document)
db_session_with_containers.commit()
@@ -882,7 +881,6 @@ class TestCleanDatasetTask:
special_content = "Special chars: !@#$%^&*()_+-=[]{}|;':\",./<>?`~"
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
@@ -892,9 +890,9 @@ class TestCleanDatasetTask:
name=f"test_doc_{special_content}",
created_from=DocumentCreatedFrom.WEB,
created_by=account.id,
- created_at=datetime.now(),
- updated_at=datetime.now(),
)
+ document.created_at = datetime.now()
+ document.updated_at = datetime.now()
db_session_with_containers.add(document)
db_session_with_containers.commit()
diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py
index ac43b65acdf..ed1c37c7317 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py
@@ -341,7 +341,6 @@ class TestCleanNotionDocumentTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -527,7 +526,6 @@ class TestCleanNotionDocumentTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -629,7 +627,6 @@ class TestCleanNotionDocumentTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -1045,7 +1042,6 @@ class TestCleanNotionDocumentTask:
# Create document with rich metadata
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py
index 91131d9970f..b301da469df 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py
@@ -102,7 +102,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -161,7 +160,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -181,7 +179,6 @@ class TestDealDatasetVectorIndexTask:
# Create documents
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -262,7 +259,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -282,7 +278,6 @@ class TestDealDatasetVectorIndexTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -415,7 +410,6 @@ class TestDealDatasetVectorIndexTask:
# Create document without segments
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -510,7 +504,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -530,7 +523,6 @@ class TestDealDatasetVectorIndexTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -606,7 +598,6 @@ class TestDealDatasetVectorIndexTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -682,7 +673,6 @@ class TestDealDatasetVectorIndexTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -758,7 +748,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -862,7 +851,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -882,7 +870,6 @@ class TestDealDatasetVectorIndexTask:
# Create document
document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -959,7 +946,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -979,7 +965,6 @@ class TestDealDatasetVectorIndexTask:
# Create enabled document
enabled_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -998,7 +983,6 @@ class TestDealDatasetVectorIndexTask:
# Create disabled document
disabled_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
@@ -1080,7 +1064,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -1100,7 +1083,6 @@ class TestDealDatasetVectorIndexTask:
# Create active document
active_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -1119,7 +1101,6 @@ class TestDealDatasetVectorIndexTask:
# Create archived document
archived_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
@@ -1201,7 +1182,6 @@ class TestDealDatasetVectorIndexTask:
# Create a document to set the doc_form property
document_for_doc_form = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -1221,7 +1201,6 @@ class TestDealDatasetVectorIndexTask:
# Create completed document
completed_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
@@ -1240,7 +1219,6 @@ class TestDealDatasetVectorIndexTask:
# Create incomplete document
incomplete_document = Document(
- id=str(uuid.uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py
index 9ab88957de0..41162ba0a0a 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_delete_segment_from_index_task.py
@@ -150,7 +150,16 @@ class TestDeleteSegmentFromIndexTask:
Document: Created test document instance
"""
fake = fake or Faker()
- document = Document()
+ document = Document(
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ position=kwargs.get("position", 1),
+ data_source_type=kwargs.get("data_source_type", "upload_file"),
+ batch=kwargs.get("batch", fake.uuid4()),
+ name=kwargs.get("name", f"Test Document {fake.word()}"),
+ created_from=kwargs.get("created_from", DocumentCreatedFrom.API),
+ created_by=account.id,
+ )
document.id = fake.uuid4()
document.tenant_id = dataset.tenant_id
document.dataset_id = dataset.id
diff --git a/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py
index e0dd84ee67e..adc34f8bb39 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_disable_segment_from_index_task.py
@@ -114,7 +114,7 @@ class TestDisableSegmentFromIndexTask:
dataset: Dataset,
tenant: Tenant,
account: Account,
- doc_form: str = IndexStructureType.PARAGRAPH_INDEX,
+ doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,
) -> Document:
"""
Helper method to create a test document.
diff --git a/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py
index a218336deef..a2517e0715b 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_disable_segments_from_index_task.py
@@ -135,7 +135,16 @@ class TestDisableSegmentsFromIndexTask:
DatasetDocument: Created test document instance
"""
fake = fake or Faker()
- document = DatasetDocument()
+ document = DatasetDocument(
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch=fake.uuid4(),
+ name=f"Test Document {fake.word()}.txt",
+ created_from=DocumentCreatedFrom.WEB,
+ created_by=account.id,
+ )
document.id = fake.uuid4()
document.tenant_id = dataset.tenant_id
diff --git a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py
index c48929bb83c..a0cdf1980ea 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py
@@ -409,7 +409,6 @@ class TestDocumentIndexingTasks:
# Document with different indexing status
doc1 = Document(
- id=fake.uuid4(),
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
position=2,
@@ -426,7 +425,6 @@ class TestDocumentIndexingTasks:
# Document with disabled status
doc2 = Document(
- id=fake.uuid4(),
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
position=3,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py
index c2fe023cc38..8b6062f3877 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_duplicate_document_indexing_task.py
@@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from enums import CloudPlan, DeploymentEdition
-from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
+from models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
from services.knowledge.indexing.errors import DocumentIsPausedError
@@ -99,14 +99,14 @@ class TestDuplicateDocumentIndexingTasks:
email=fake.email(),
name=fake.name(),
interface_language="en-US",
- status="active",
+ status=AccountStatus.ACTIVE,
)
db_session_with_containers.add(account)
db_session_with_containers.commit()
tenant = Tenant(
name=fake.company(),
- status="normal",
+ status=TenantStatus.NORMAL,
)
db_session_with_containers.add(tenant)
db_session_with_containers.commit()
@@ -138,7 +138,6 @@ class TestDuplicateDocumentIndexingTasks:
documents = []
for i in range(document_count):
document = Document(
- id=fake.uuid4(),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=i,
diff --git a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py
index 525deeca421..88db0d67b20 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_enable_segments_to_index_task.py
@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.ext_redis import redis_client
+from models import AccountStatus, TenantStatus
from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
from models.dataset import ChildChunk, Dataset, DatasetProcessRule, Document, DocumentSegment
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatus
@@ -52,14 +53,14 @@ class TestEnableSegmentsToIndexTask:
email=fake.email(),
name=fake.name(),
interface_language="en-US",
- status="active",
+ status=AccountStatus.ACTIVE,
)
db_session_with_containers.add(account)
db_session_with_containers.commit()
tenant = Tenant(
name=fake.company(),
- status="normal",
+ status=TenantStatus.NORMAL,
)
db_session_with_containers.add(tenant)
db_session_with_containers.commit()
@@ -89,7 +90,6 @@ class TestEnableSegmentsToIndexTask:
# Create document
document = Document(
- id=fake.uuid4(),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
@@ -291,7 +291,7 @@ class TestEnableSegmentsToIndexTask:
# Reset document status
document.enabled = True
document.archived = False
- document.indexing_status = "completed"
+ document.indexing_status = IndexingStatus.COMPLETED
db_session_with_containers.commit()
# Set invalid status
diff --git a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py
index 83a840b0e67..4b4c881c53b 100644
--- a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py
+++ b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py
@@ -10,6 +10,7 @@ from typing import Literal, NotRequired, TypedDict
from unittest.mock import MagicMock
from models.dataset import Document
+from models.enums import DataSourceType, DocumentCreatedFrom
from services.knowledge.dataset_read_service import get_document_source_detail
@@ -67,6 +68,20 @@ case_2: LocalFileCase = {
cases: list[AnyCase] = [case_1]
+def make_document(*, data_source_type: str, data_source_info: str) -> Document:
+ return Document(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ position=1,
+ data_source_type=DataSourceType(data_source_type),
+ batch="batch-id",
+ name="document.txt",
+ created_from=DocumentCreatedFrom.WEB,
+ created_by="account-id",
+ data_source_info=data_source_info,
+ )
+
+
class TestDocumentDetailDataSourceInfo:
"""Test cases for document detail API data_source_info serialization."""
@@ -74,7 +89,7 @@ class TestDocumentDetailDataSourceInfo:
"""Test that data_source_info_dict returns raw JSON data for all data_source_type values."""
# Test data for different data_source_type values
for case in cases:
- document = Document(
+ document = make_document(
data_source_type=case["data_source_type"],
data_source_info=case["data_source_info"],
)
@@ -94,7 +109,7 @@ class TestDocumentDetailDataSourceInfo:
"created_at": "2024-01-01T00:00:00Z",
}
- document = Document(
+ document = make_document(
data_source_type="local_file",
data_source_info=json.dumps(test_data),
)
@@ -112,7 +127,7 @@ class TestDocumentDetailDataSourceInfo:
"""Test that notion_import and website_crawl return raw data in data_source_detail_dict."""
# Test notion_import
notion_data: NotionImportInfo = {"notion_page_id": "page-123", "workspace_id": "ws-456"}
- document = Document(
+ document = make_document(
data_source_type="notion_import",
data_source_info=json.dumps(notion_data),
)
@@ -123,7 +138,7 @@ class TestDocumentDetailDataSourceInfo:
# Test website_crawl
website_data: WebsiteCrawlInfo = {"url": "https://example.com", "job_id": "job-789"}
- document = Document(
+ document = make_document(
data_source_type="website_crawl",
data_source_info=json.dumps(website_data),
)
@@ -135,7 +150,7 @@ class TestDocumentDetailDataSourceInfo:
def test_local_file_data_source_detail_dict_without_db(self):
"""Test that local_file returns empty data_source_detail_dict (this doesn't need DB context)."""
# Test local_file - this should work without database context since it returns {} early
- document = Document(
+ document = make_document(
data_source_type="local_file",
data_source_info=json.dumps({"file_path": "/tmp/test.txt"}),
)
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py
index cffa59042d1..be28636eae7 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_segment.py
@@ -41,7 +41,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureType
from libs.datetime_utils import naive_utc_now
from models.account import Account, Tenant
from models.dataset import ChildChunk, Dataset, Document, DocumentSegment, DocumentSegmentSummary
-from models.enums import IndexingStatus, SegmentType
+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentType
from services.api_token_service import CachedApiToken
from services.knowledge.dataset_service import DocumentService, SegmentService
from services.knowledge.segments.application import SegmentMutationService
@@ -131,6 +131,30 @@ def _child_chunk() -> ChildChunk:
return child_chunk
+def _document(
+ *,
+ document_id: str | None = None,
+ dataset_id: str = "dataset-id",
+ indexing_status: IndexingStatus | str = IndexingStatus.WAITING,
+ enabled: bool = True,
+) -> Document:
+ document = Document(
+ tenant_id="tenant-id",
+ dataset_id=dataset_id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch-id",
+ name="document.txt",
+ created_from=DocumentCreatedFrom.API,
+ created_by="account-id",
+ indexing_status=IndexingStatus(indexing_status),
+ enabled=enabled,
+ )
+ if document_id is not None:
+ document.id = document_id
+ return document
+
+
def _document_for_dataset(
dataset: Dataset, document_id: str = "doc-id", doc_form: str = IndexStructureType.PARAGRAPH_INDEX
) -> Document:
@@ -139,10 +163,10 @@ def _document_for_dataset(
dataset_id=dataset.id,
tenant_id=dataset.tenant_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="Document",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by="account-1",
indexing_status="completed",
enabled=True,
@@ -396,8 +420,8 @@ class TestSegmentServiceMockedBehavior:
@pytest.fixture
def mock_document(self):
"""Create mock document."""
- document = Document(
- id=str(uuid.uuid4()),
+ document = _document(
+ document_id=str(uuid.uuid4()),
dataset_id=str(uuid.uuid4()),
indexing_status="completed",
enabled=True,
@@ -562,7 +586,7 @@ class TestChildChunkServiceMockedBehavior:
result = SegmentService.create_child_chunk(
content="New chunk content",
segment=mock_segment,
- document=Document(),
+ document=_document(),
dataset=Dataset(),
session=unbound_session,
actor_id="account-id",
@@ -635,7 +659,7 @@ class TestChildChunkServiceMockedBehavior:
tokens=0,
created_by="account-id",
),
- document=Document(),
+ document=_document(),
dataset=Dataset(),
session=unbound_session,
actor_id="account-id",
@@ -649,28 +673,28 @@ class TestDocumentValidation:
def test_document_indexing_status_completed_is_valid(self):
"""Test that completed indexing status is valid."""
- document = Document(
+ document = _document(
indexing_status="completed",
)
assert document.indexing_status == "completed"
def test_document_indexing_status_indexing_is_invalid(self):
"""Test that indexing status is invalid for segment operations."""
- document = Document(
+ document = _document(
indexing_status="indexing",
)
assert document.indexing_status != "completed"
def test_document_enabled_true_is_valid(self):
"""Test that enabled=True is valid."""
- document = Document(
+ document = _document(
enabled=True,
)
assert document.enabled
def test_document_enabled_false_is_invalid(self):
"""Test that enabled=False is invalid for segment operations."""
- document = Document(
+ document = _document(
enabled=False,
)
assert not document.enabled
@@ -926,7 +950,7 @@ class TestSegmentIndexingRequirements:
)
def test_valid_indexing_statuses(self, status):
"""Test valid document indexing statuses."""
- document = Document(
+ document = _document(
indexing_status=status,
)
assert document.indexing_status in {
@@ -939,7 +963,7 @@ class TestSegmentIndexingRequirements:
def test_completed_status_required_for_segments(self):
"""Test that completed status is required for segment operations."""
- document = Document(
+ document = _document(
indexing_status="completed",
enabled=True,
)
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
index 159dddfbbc2..a89b15bd2eb 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
@@ -20,6 +20,7 @@ import json
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime
+from typing import cast
from unittest.mock import Mock, PropertyMock, patch
import pytest
@@ -184,7 +185,11 @@ def make_serializable_document(**overrides: object) -> Document:
"word_count": None,
}
attrs.update(overrides)
+ created_at = cast(datetime, attrs.pop("created_at"))
+ updated_at = cast(datetime, attrs.pop("updated_at"))
document = Document(**attrs)
+ document.created_at = created_at
+ document.updated_at = updated_at
document.summary_index_status = summary_index_status # type: ignore[attr-defined]
return document
diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py
index 5e943e1b4d8..ea1f12e4570 100644
--- a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py
+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py
@@ -17,6 +17,7 @@ from core.rag.index_processor.processor.qa_index_processor import QAIndexProcess
from core.rag.models.document import AttachmentDocument, Document
from models.dataset import Dataset, DocumentSegment
from models.dataset import Document as DatasetDocument
+from models.enums import DataSourceType, DocumentCreatedFrom
class _ImmediateThread:
@@ -50,7 +51,17 @@ class TestQAIndexProcessor:
@pytest.fixture
def dataset_document(self) -> DatasetDocument:
- return DatasetDocument(id="doc-1", created_by="user-1")
+ return DatasetDocument(
+ id="doc-1",
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch-1",
+ name="qa.txt",
+ created_from=DocumentCreatedFrom.WEB,
+ created_by="user-1",
+ )
@pytest.fixture
def process_rule(self) -> dict:
diff --git a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py
index 525314f8cde..aff0cfdeb40 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_index_processor_base.py
@@ -17,11 +17,11 @@ from core.rag.models.document import AttachmentDocument, Document
from extensions.storage.storage_type import StorageType
from models.account import Account
from models.dataset import Dataset
-from models.dataset import Document as DatasetDocument
from models.enums import CreatorUserRole
from models.model import UploadFile
from models.tools import ToolFile
from tests.unit_tests.config_override import config_overrides_context
+from tests.unit_tests.model_factories import make_document
def _upload(
@@ -132,7 +132,7 @@ class TestBaseIndexProcessor:
with pytest.raises(NotImplementedError):
processor.clean(Dataset(), None, session=unbound_session)
with pytest.raises(NotImplementedError):
- processor.index(Dataset(), DatasetDocument(), {}, unbound_session)
+ processor.index(Dataset(), make_document(), {}, unbound_session)
with pytest.raises(NotImplementedError):
processor.format_preview([])
diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
index 9b1646e4f59..66dda219616 100644
--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
@@ -55,7 +55,7 @@ from models.dataset import (
SegmentAttachmentBinding,
)
from models.dataset import Document as DatasetDocument
-from models.enums import CreatorUserRole
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom
# ==================== Helper Functions ====================
@@ -4258,10 +4258,10 @@ class TestDatasetRetrievalAdditionalHelpers:
tenant_id=tenant_id,
dataset_id=dataset_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="document",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by=user_id,
indexing_status="completed",
enabled=True,
@@ -4386,10 +4386,10 @@ class TestDatasetRetrievalAdditionalHelpers:
tenant_id=tenant_id,
dataset_id=internal_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="available document",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by=creator_id,
indexing_status="completed",
enabled=True,
@@ -4810,10 +4810,10 @@ class TestRetrieveCoverage:
tenant_id=tenant_id,
dataset_id=dataset_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="Document One",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by=creator_id,
indexing_status="completed",
enabled=True,
@@ -5391,10 +5391,10 @@ class TestInternalHooksCoverage:
tenant_id=tenant_id,
dataset_id=dataset_id,
position=position,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name=f"document-{position}",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by=creator_id,
indexing_status="completed",
enabled=True,
@@ -5745,10 +5745,10 @@ class TestInternalHooksCoverage:
tenant_id=tenant_id,
dataset_id=available_id,
position=position,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name=f"document-{position}",
- created_from="api",
+ created_from=DocumentCreatedFrom("api"),
created_by=creator_id,
indexing_status="completed",
enabled=True,
diff --git a/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py
index fc918dbc29a..e6313b03d98 100644
--- a/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py
+++ b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py
@@ -13,6 +13,7 @@ from core.rag.models.document import Document as RetrievedDocument
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
from core.workflow.nodes.knowledge_retrieval.retrieval import KnowledgeRetrievalRequest
from models.dataset import Dataset, Document, DocumentSegment
+from models.enums import DataSourceType, DocumentCreatedFrom
from repositories.knowledge.retrieval_source_repository import get_retrieval_source_documents
@@ -24,10 +25,10 @@ def test_source_citations_do_not_fetch_unused_document_fields(sqlite_session: Se
tenant_id=tenant_id,
dataset_id=dataset_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="source.txt",
- created_from="web",
+ created_from=DocumentCreatedFrom("web"),
created_by=creator_id,
enabled=True,
archived=False,
@@ -112,10 +113,10 @@ def test_source_document_read_is_scoped_and_has_no_lazy_queries(sqlite_session:
tenant_id=owner,
dataset_id=dataset,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="source.txt",
- created_from="web",
+ created_from=DocumentCreatedFrom("web"),
created_by=str(uuid4()),
doc_metadata=None,
)
diff --git a/api/tests/unit_tests/model_factories.py b/api/tests/unit_tests/model_factories.py
index 0d6e28835fb..3f79f797657 100644
--- a/api/tests/unit_tests/model_factories.py
+++ b/api/tests/unit_tests/model_factories.py
@@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence
from datetime import datetime
from decimal import Decimal
+from core.rag.index_processor.constant.index_type import IndexStructureType
from extensions.storage.storage_type import StorageType
from graphon.variables import VariableBase
from models.account import Account, AccountStatus, Tenant, TenantAccountRole, TenantStatus
@@ -465,27 +466,32 @@ def make_document(
created_at: datetime | None = None,
updated_at: datetime | None = None,
) -> Document:
- """Build a transient ``Document``; unset keywords leave their column unset."""
- values: dict[str, object] = {
- "tenant_id": tenant_id,
- "dataset_id": dataset_id,
- "position": position,
- "data_source_type": data_source_type,
- "batch": batch,
- "name": name,
- "created_from": created_from,
- "created_by": created_by,
- }
- optional: dict[str, object | None] = {
- "id": document_id,
- "data_source_info": data_source_info,
- "doc_form": doc_form,
- "doc_metadata": doc_metadata,
- "created_at": created_at,
- "updated_at": updated_at,
- }
- values.update({key: value for key, value in optional.items() if value is not None})
- return Document(**values)
+ """Build a transient Document with constructor defaults.
+
+ None leaves the document ID and optional form at their model defaults.
+ None timestamps leave database-generated fields unset.
+ """
+ document = Document(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ position=position,
+ data_source_type=data_source_type,
+ batch=batch,
+ name=name,
+ created_from=DocumentCreatedFrom(created_from),
+ created_by=created_by,
+ data_source_info=data_source_info,
+ doc_metadata=doc_metadata,
+ )
+ if document_id is not None:
+ document.id = document_id
+ if doc_form is not None:
+ document.doc_form = IndexStructureType(doc_form)
+ if created_at is not None:
+ document.created_at = created_at
+ if updated_at is not None:
+ document.updated_at = updated_at
+ return document
__all__ = [
diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.py
index 0a39d4b4440..7bccc69fb24 100644
--- a/api/tests/unit_tests/models/test_dataset_models.py
+++ b/api/tests/unit_tests/models/test_dataset_models.py
@@ -110,8 +110,7 @@ def _make_document(
word_count: int | None = None,
indexing_status: IndexingStatus = IndexingStatus.WAITING,
) -> Document:
- return Document(
- id=document_id,
+ document = Document(
tenant_id=tenant_id,
dataset_id=dataset_id,
position=position,
@@ -124,6 +123,8 @@ def _make_document(
word_count=word_count,
indexing_status=indexing_status,
)
+ document.id = document_id
+ return document
def _make_app(*, app_id: str, tenant_id: str = "tenant-1") -> App:
@@ -1595,8 +1596,8 @@ class TestModelIntegration:
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
word_count=100,
- id=document_id,
)
+ document.id = document_id
# Create segment
segment = DocumentSegment(
diff --git a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py
index b2e67607f93..1b6e235d223 100644
--- a/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py
+++ b/api/tests/unit_tests/repositories/knowledge/test_dataset_read_boundaries.py
@@ -28,7 +28,7 @@ from models.dataset import (
ExternalKnowledgeApis,
ExternalKnowledgeBindings,
)
-from models.enums import CreatorUserRole
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus
from models.model import UploadFile
from models.tools import ToolFile
from repositories.knowledge.dataset_read_repository import get_external_api_bindings_batch
@@ -52,10 +52,10 @@ def _document(index: int = 1) -> Document:
tenant_id="tenant-1",
dataset_id="dataset-1",
position=index,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="source.txt",
- created_from="web",
+ created_from=DocumentCreatedFrom("web"),
created_by="account-1",
)
@@ -228,7 +228,7 @@ def test_document_metadata_respects_dataset_builtin_flag(sqlite_session: Session
def test_waiting_document_reports_paused_until_resumed() -> None:
document = _document()
- document.indexing_status = "waiting"
+ document.indexing_status = IndexingStatus.WAITING
document.is_paused = True
assert document.display_status == "paused"
document.is_paused = False
diff --git a/api/tests/unit_tests/repositories/knowledge/test_document_repository.py b/api/tests/unit_tests/repositories/knowledge/test_document_repository.py
index 5820f173e93..98f7a4dc85d 100644
--- a/api/tests/unit_tests/repositories/knowledge/test_document_repository.py
+++ b/api/tests/unit_tests/repositories/knowledge/test_document_repository.py
@@ -31,10 +31,10 @@ def _document(
*,
workspace_id: str = "workspace-1",
dataset_id: str = "dataset-1",
- source_type: str = DataSourceType.NOTION_IMPORT,
+ source_type: DataSourceType = DataSourceType.NOTION_IMPORT,
enabled: bool = True,
archived: bool = False,
- status: str = IndexingStatus.COMPLETED,
+ status: IndexingStatus = IndexingStatus.COMPLETED,
batch: str = "batch-1",
) -> Document:
source_info = (
@@ -42,7 +42,7 @@ def _document(
if source_type == DataSourceType.NOTION_IMPORT
else {"upload_file_id": f"file-{document_id}"}
)
- return Document(
+ row = Document(
id=document_id,
tenant_id=workspace_id,
dataset_id=dataset_id,
@@ -53,8 +53,6 @@ def _document(
name=f"Document {document_id}",
created_from=DocumentCreatedFrom.API,
created_by="account-1",
- created_at=datetime(2026, 1, 1),
- updated_at=datetime(2026, 1, 1),
indexing_status=status,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
word_count=1,
@@ -62,6 +60,9 @@ def _document(
archived=archived,
is_paused=False,
)
+ row.created_at = datetime(2026, 1, 1)
+ row.updated_at = datetime(2026, 1, 1)
+ return row
def test_document_queries_enforce_the_complete_owner_chain(
diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.py
index c4410988259..8864c9a046f 100644
--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py
+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py
@@ -30,6 +30,7 @@ from models.dataset import (
Document,
DocumentSegment,
)
+from models.enums import DataSourceType
from models.model import UploadFile
from services.entities.knowledge_entities.rag_pipeline_entities import (
IconInfo as PipelineIconInfo,
@@ -315,10 +316,11 @@ def _make_document(
archived: bool = False,
indexing_status: str = "completed",
) -> Document:
- return Document(
+ document = Document(
id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
+ position=1,
batch=batch,
doc_form=doc_form,
word_count=word_count,
@@ -326,17 +328,19 @@ def _make_document(
enabled=enabled,
archived=archived,
indexing_status=indexing_status,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
data_source_info="{}",
completed_at=SimpleNamespace(),
processing_started_at="started",
parsing_completed_at="parsed",
cleaning_completed_at="cleaned",
splitting_completed_at="split",
- updated_at=None,
created_from=None,
+ created_by="user-1",
dataset_process_rule_id="process-rule-1",
)
+ document.updated_at = None # type: ignore[assignment]
+ return document
def _make_segment(
diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
index 56f8a01a302..0f77bf48d14 100644
--- a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py
@@ -21,7 +21,14 @@ from models.dataset import (
DatasetQuery,
DocumentSegment,
)
-from models.enums import CreatorUserRole, DatasetQuerySource, IndexingStatus, SegmentStatus
+from models.enums import (
+ CreatorUserRole,
+ DatasetQuerySource,
+ DataSourceType,
+ DocumentCreatedFrom,
+ IndexingStatus,
+ SegmentStatus,
+)
from repositories.app.console_repository import ConsoleAppRepository
from services.enterprise import rbac_service
from services.knowledge.dataset_access import DatasetNotFoundError
@@ -56,21 +63,21 @@ def dataset(**values: object) -> Dataset:
def document(**values: object) -> Document:
- return Document(
- **{
- "id": "document",
- "tenant_id": "tenant",
- "dataset_id": "dataset",
- "position": 1,
- "data_source_type": "local_file",
- "batch": "batch",
- "name": "Document",
- "created_from": "web",
- "created_by": "actor",
- "indexing_status": IndexingStatus.ERROR,
- **values,
- }
+ row = Document(
+ tenant_id="tenant",
+ dataset_id="dataset",
+ position=1,
+ data_source_type=DataSourceType("local_file"),
+ batch="batch",
+ name="Document",
+ created_from=DocumentCreatedFrom("web"),
+ created_by="actor",
)
+ row.id = "document"
+ row.indexing_status = IndexingStatus.ERROR
+ for key, value in values.items():
+ setattr(row, key, value)
+ return row
@pytest.fixture
diff --git a/api/tests/unit_tests/services/knowledge/test_document_adapters.py b/api/tests/unit_tests/services/knowledge/test_document_adapters.py
index a655a19e9e5..a56f5374f96 100644
--- a/api/tests/unit_tests/services/knowledge/test_document_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/test_document_adapters.py
@@ -7,9 +7,9 @@ from sqlalchemy import Engine, event
from sqlalchemy.orm import Session, sessionmaker
from controllers.console.datasets.datasets_document import DocumentDetailResponse, DocumentWithSegmentsListResponse
-from core.rag.index_processor.constant.index_type import IndexTechniqueType
+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from models.dataset import Dataset, DatasetProcessRule, Document, DocumentPipelineExecutionLog, DocumentSegment
-from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus
+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatus
from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations
from services.knowledge.documents.application import DocumentListFilter, DocumentNotFoundError
from services.knowledge.resource_scope import DatasetRef
@@ -33,27 +33,27 @@ def _dataset(**values: object) -> Dataset:
def _document(**values: object) -> Document:
- return Document(
- **{
- "id": "document-1",
- "tenant_id": "tenant-1",
- "dataset_id": "dataset-1",
- "position": 1,
- "data_source_type": "local_file",
- "data_source_info": json.dumps({"file_path": "/path/to/file"}),
- "batch": "batch-1",
- "name": "Document",
- "created_from": "web",
- "created_by": "actor-1",
- "indexing_status": IndexingStatus.COMPLETED,
- "doc_form": "text_model",
- "enabled": True,
- "archived": False,
- "word_count": 10,
- "created_at": datetime(2024, 1, 1),
- **values,
- }
+ row = Document(
+ tenant_id="tenant-1",
+ dataset_id="dataset-1",
+ position=1,
+ data_source_type=DataSourceType("local_file"),
+ batch="batch-1",
+ name="Document",
+ created_from=DocumentCreatedFrom("web"),
+ created_by="actor-1",
)
+ row.id = "document-1"
+ row.data_source_info = json.dumps({"file_path": "/path/to/file"})
+ row.indexing_status = IndexingStatus.COMPLETED
+ row.doc_form = IndexStructureType.PARAGRAPH_INDEX
+ row.enabled = True
+ row.archived = False
+ row.word_count = 10
+ row.created_at = datetime(2024, 1, 1)
+ for key, value in values.items():
+ setattr(row, key, value)
+ return row
def _segment(
diff --git a/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py
index b0885001843..f9f99d11e88 100644
--- a/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py
+++ b/api/tests/unit_tests/services/knowledge/test_document_index_cleanup.py
@@ -37,7 +37,7 @@ def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment,
name="Document",
created_from=DocumentCreatedFrom.WEB,
created_by="admin",
- doc_form="text_model",
+ doc_form=IndexStructureType("text_model"),
)
segment = DocumentSegment(
tenant_id=dataset.tenant_id,
diff --git a/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py
index 23ce2b19dd3..d149c1d266d 100644
--- a/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py
+++ b/api/tests/unit_tests/services/knowledge/test_retrieval_attachment_signing.py
@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
from core.app.file_access import FileAccessScope, bind_file_access_scope, get_current_file_access_scope
+from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.models.document import Document as RagDocument
from core.rag.retrieval import dataset_retrieval as retrieval_module
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
@@ -35,7 +36,7 @@ def graph(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment,
name="Document",
created_from=DocumentCreatedFrom.WEB,
created_by="admin",
- doc_form="text_model",
+ doc_form=IndexStructureType("text_model"),
doc_metadata={},
)
segment = DocumentSegment(
diff --git a/api/tests/unit_tests/services/knowledge/test_segment_adapters.py b/api/tests/unit_tests/services/knowledge/test_segment_adapters.py
index 89a6b491316..427b76127d2 100644
--- a/api/tests/unit_tests/services/knowledge/test_segment_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/test_segment_adapters.py
@@ -28,7 +28,7 @@ from libs.helper import generate_text_hash
from libs.pagination import PaginatedResult
from machinery.context import RequestContext
from models.dataset import ChildChunk, Dataset, Document, DocumentSegment, SegmentAttachmentBinding
-from models.enums import CreatorUserRole, DocumentCreatedFrom, SegmentStatus, SummaryStatus
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatus
from models.model import UploadFile
from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
@@ -100,12 +100,12 @@ def _document(document_id: str, workspace_id: str, dataset_id: str) -> Document:
tenant_id=workspace_id,
dataset_id=dataset_id,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
batch="batch-1",
name="Document",
created_from=DocumentCreatedFrom.API,
created_by="account-1",
- doc_form="text_model",
+ doc_form=IndexStructureType("text_model"),
word_count=0,
)
diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.py
index 702e7902065..40bce008f4f 100644
--- a/api/tests/unit_tests/services/test_dataset_service_document.py
+++ b/api/tests/unit_tests/services/test_dataset_service_document.py
@@ -97,7 +97,7 @@ def _document_row(
archived: bool = False,
is_paused: bool = False,
) -> Document:
- return Document(
+ document = Document(
id=document_id,
tenant_id=tenant_id,
dataset_id=dataset_id,
@@ -108,8 +108,6 @@ def _document_row(
name=name,
created_from=DocumentCreatedFrom.API,
created_by="user-1",
- created_at=datetime(2026, 1, 1),
- updated_at=datetime(2026, 1, 2),
indexing_status=indexing_status,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
word_count=10,
@@ -118,6 +116,9 @@ def _document_row(
is_paused=is_paused,
completed_at=datetime(2026, 1, 2) if indexing_status == IndexingStatus.COMPLETED else None,
)
+ document.created_at = datetime(2026, 1, 1)
+ document.updated_at = datetime(2026, 1, 2)
+ return document
def _upload_file(*, file_id: str, tenant_id: str = "tenant-1", name: str = "upload.txt") -> UploadFile:
diff --git a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py
index a143ee76e81..19599e6fe9f 100644
--- a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py
+++ b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from models.account import Account, Tenant
from models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment
-from models.enums import ProcessRuleMode
+from models.enums import DataSourceType, DocumentCreatedFrom, ProcessRuleMode
from services.knowledge.dataset_service import DocumentService, SegmentService
from services.knowledge.entities.knowledge_entities import DataSource, InfoList, KnowledgeConfig
from services.knowledge.segments.application import SegmentMutationService
@@ -166,11 +166,11 @@ def test_multi_create_segment_ignores_lock_not_owned(
tenant_id=TENANT_ID,
dataset_id=DATASET_ID,
position=1,
- data_source_type="upload_file",
+ data_source_type=DataSourceType("upload_file"),
data_source_info="{}",
batch="batch-1",
name="Test Document",
- created_from="web",
+ created_from=DocumentCreatedFrom("web"),
created_by=USER_ID,
word_count=0,
doc_form=IndexStructureType.QA_INDEX,
diff --git a/api/tests/unit_tests/services/test_dataset_service_segment.py b/api/tests/unit_tests/services/test_dataset_service_segment.py
index f848684aee1..527041a6497 100644
--- a/api/tests/unit_tests/services/test_dataset_service_segment.py
+++ b/api/tests/unit_tests/services/test_dataset_service_segment.py
@@ -59,7 +59,7 @@ def _document(
doc_form: str = IndexStructureType.PARAGRAPH_INDEX,
word_count: int = 20,
) -> Document:
- return Document(
+ document = Document(
id=document_id,
tenant_id=tenant_id,
dataset_id=dataset_id,
@@ -69,13 +69,15 @@ def _document(
name="Document",
created_from=DocumentCreatedFrom.API,
created_by="user-1",
- created_at=datetime(2026, 1, 1),
- updated_at=datetime(2026, 1, 2),
indexing_status="completed",
doc_form=doc_form,
word_count=word_count,
)
+ document.created_at = datetime(2026, 1, 1)
+ document.updated_at = datetime(2026, 1, 2)
+ return document
+
def _segment(
*,
diff --git a/api/tests/unit_tests/services/test_summary_index_service.py b/api/tests/unit_tests/services/test_summary_index_service.py
index c8fb1a36e65..c60a9ee7bf0 100644
--- a/api/tests/unit_tests/services/test_summary_index_service.py
+++ b/api/tests/unit_tests/services/test_summary_index_service.py
@@ -14,7 +14,7 @@ from sqlalchemy.orm import Session, sessionmaker
import services.knowledge.summaries.adapters as summary_module
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummary
-from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatus
+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus, SummaryStatus
from services.knowledge.summaries.adapters import SummaryIndexAdapter
TENANT_ID = "tenant-1"
@@ -75,7 +75,7 @@ def _persist_document(
name=f"Document {document_id}",
created_from=DocumentCreatedFrom.WEB,
created_by="account-1",
- indexing_status="completed",
+ indexing_status=IndexingStatus.COMPLETED,
enabled=True,
archived=False,
doc_form=doc_form,