Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): write built-in document metadata in a single format (#44218)
Tianyu Caie7c2c6fad5ac
diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.pyindex 3b4773bc748..b0e28ce5b69 100644--- a/api/core/app/apps/pipeline/pipeline_generator.py+++ b/api/core/app/apps/pipeline/pipeline_generator.py@@ -1,5 +1,4 @@import contextvars-import datetimeimport jsonimport loggingimport secrets@@ -38,7 +37,6 @@ from core.datasource.entities.datasource_entities import ()from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePluginfrom core.entities.knowledge_entities import PipelineDataset, PipelineDocument-from core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom core.repositories.factory import (DifyCoreRepositoryFactory,WorkflowExecutionRepository,@@ -47,9 +45,10 @@ from core.repositories.factory import (from extensions.ext_database import dbfrom graphon.model_runtime.errors.invoke import InvokeAuthorizationErrorfrom graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader+from libs.datetime_utils import naive_utc_nowfrom libs.flask_utils import preserve_flask_contextsfrom models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom-from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline+from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline, build_built_in_field_valuesfrom models.enums import WorkflowRunTriggeredFromfrom models.model import AppModefrom services.data_source.provider_service import DatasourceProviderService@@ -747,17 +746,15 @@ class PipelineGenerator(BaseAppGenerator):created_by=account.id,doc_form=document_form,)- doc_metadata = {}if built_in_field_enabled:- doc_metadata = {- BuiltInField.document_name: name,- BuiltInField.uploader: account.name,- BuiltInField.upload_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),- BuiltInField.last_update_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),- BuiltInField.source: datasource_type,- }- if doc_metadata:- document.doc_metadata = doc_metadata+ now = naive_utc_now()+ document.doc_metadata = build_built_in_field_values(+ document_name=name,+ uploader=account.name,+ upload_date=now,+ last_update_date=now,+ data_source_type=datasource_type,+ )return documentdef _format_datasource_info_list(diff --git a/api/models/dataset.py b/api/models/dataset.pyindex e6535d6e16e..9c95fbf23a9 100644--- a/api/models/dataset.py+++ b/api/models/dataset.py@@ -1,7 +1,7 @@import jsonimport picklefrom collections.abc import Sequence-from datetime import datetime+from datetime import UTC, datetimefrom json import JSONDecodeErrorfrom typing import Any, ClassVar, TypedDict, cast, overridefrom uuid import uuid4@@ -12,9 +12,10 @@ from sqlalchemy.orm import Mapped, mapped_columnfrom configs import dify_configfrom core.rag.entities import PreProcessingRuleKey-from core.rag.index_processor.constant.built_in_field import BuiltInField+from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSourcefrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethod+from libs.datetime_utils import ensure_naive_utcfrom libs.uuid_utils import uuidv7from models.base import Base, TypeBasefrom models.enums import (@@ -337,6 +338,35 @@ class DatasetProcessRule(TypeBase):return None+def _utc_epoch_seconds(value: datetime) -> float:+ return ensure_naive_utc(value).replace(tzinfo=UTC).timestamp()+++def build_built_in_field_values(+ *,+ document_name: str,+ uploader: str | None,+ upload_date: datetime,+ last_update_date: datetime,+ data_source_type: str,+) -> dict[str, str | float | None]:+ """Return built-in metadata in the single format stored in ``Document.doc_metadata``.++ Retrieval filters compare time fields numerically and ``source`` by exact string, so every+ writer must store UTC epoch seconds and the ``MetadataDataSource`` label shown to users.+ Epoch seconds keep their fractional part so strict and equality comparisons at an+ integer-second boundary behave as they did for stored ``datetime.timestamp()`` values.+ Naive datetimes are treated as UTC.+ """+ return {+ BuiltInField.document_name: document_name,+ BuiltInField.uploader: uploader,+ BuiltInField.upload_date: _utc_epoch_seconds(upload_date),+ BuiltInField.last_update_date: _utc_epoch_seconds(last_update_date),+ BuiltInField.source: MetadataDataSource[data_source_type],+ }++class Document(Base):__tablename__ = "documents"__table_args__ = (diff --git a/api/repositories/knowledge/metadata_repository.py b/api/repositories/knowledge/metadata_repository.pyindex 1610fd3ab0d..fb28a19ec7b 100644--- a/api/repositories/knowledge/metadata_repository.py+++ b/api/repositories/knowledge/metadata_repository.py@@ -5,9 +5,15 @@ from collections.abc import Sequencefrom sqlalchemy import func, selectfrom sqlalchemy.orm import Session, sessionmaker-from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource+from core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom libs.datetime_utils import naive_utc_now-from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Document+from models.dataset import (+ Dataset,+ DatasetMetadata,+ DatasetMetadataBinding,+ Document,+ build_built_in_field_values,+)from repositories.knowledge.dataset_read_repository import get_document_uploaderfrom services.errors.metadata import MetadataResourceNotFoundErrorfrom services.knowledge.metadata.application import MetadataList, MetadataOperation, MetadataRecord@@ -115,14 +121,14 @@ class SQLAlchemyMetadataRepository:return self._record(row)@staticmethod- def _built_in(session: Session, document: Document) -> dict[str, str | int | float | None]:- return {- BuiltInField.document_name: document.name,- BuiltInField.uploader: get_document_uploader(document, session=session),- BuiltInField.upload_date: document.upload_date.timestamp(),- BuiltInField.last_update_date: document.last_update_date.timestamp(),- BuiltInField.source: MetadataDataSource[document.data_source_type],- }+ def _built_in(session: Session, document: Document) -> dict[str, str | float | None]:+ return build_built_in_field_values(+ document_name=document.name,+ uploader=get_document_uploader(document, session=session),+ upload_date=document.upload_date,+ last_update_date=document.last_update_date,+ data_source_type=document.data_source_type,+ )def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None:with self._sessions.begin() as session:diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.pyindex 64a33ac308f..818a8e9811f 100644--- a/api/services/knowledge/dataset_service.py+++ b/api/services/knowledge/dataset_service.py@@ -51,6 +51,7 @@ from models.dataset import (DocumentSegment,ExternalKnowledgeBindings,Pipeline,+ build_built_in_field_values,)from models.enums import (DatasetRuntimeMode,@@ -2880,17 +2881,15 @@ class DocumentService:doc_language=document_language,need_summary=need_summary,)- doc_metadata = {}if dataset.built_in_field_enabled:- doc_metadata = {- BuiltInField.document_name: name,- BuiltInField.uploader: account.name,- BuiltInField.upload_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),- BuiltInField.last_update_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),- BuiltInField.source: data_source_type,- }- if doc_metadata:- document.doc_metadata = doc_metadata+ now = naive_utc_now()+ document.doc_metadata = build_built_in_field_values(+ document_name=name,+ uploader=account.name,+ upload_date=now,+ last_update_date=now,+ data_source_type=data_source_type,+ )return document@staticmethoddiff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.pyindex 71cf0564e34..734f82f47b7 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py@@ -1,5 +1,6 @@import contextlibimport json+from datetime import datetimefrom types import SimpleNamespacefrom unittest.mock import MagicMock, PropertyMock, create_autospec@@ -13,6 +14,7 @@ import core.app.apps.pipeline.pipeline_generator as modulefrom core.app.apps.exc import GenerateTaskStoppedErrorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.datasource.entities.datasource_entities import DatasourceProviderType+from core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipelinefrom models.enums import DataSourceType, EndUserTypefrom models.model import EndUser@@ -660,7 +662,10 @@ def test_handle_response_value_error_triggers_generate_task_stopped(generator, m)-def test_build_document_sets_metadata_for_builtin_fields(generator):+def test_build_document_sets_metadata_for_builtin_fields(generator, monkeypatch):+ created_at = datetime(2026, 10, 16, 8, 0, 0)+ monkeypatch.setattr(module, "naive_utc_now", lambda: created_at)+document = generator._build_document(tenant_id="tenant",dataset_id="ds",@@ -675,7 +680,13 @@ def test_build_document_sets_metadata_for_builtin_fields(generator):)assert document.name == "file"- assert document.doc_metadata+ assert document.doc_metadata == {+ BuiltInField.document_name: "file",+ BuiltInField.uploader: "User",+ BuiltInField.upload_date: 1792137600,+ BuiltInField.last_update_date: 1792137600,+ BuiltInField.source: "file_upload",+ }def test_build_document_supports_online_drive_datasource_type(generator):diff --git a/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.py b/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.pynew file mode 100644index 00000000000..02cc45958f4--- /dev/null+++ b/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.py@@ -0,0 +1,163 @@+import time+from collections.abc import Iterator+from datetime import datetime++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.rag.index_processor.constant.built_in_field import BuiltInField+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval+from models.dataset import Document, build_built_in_field_values+from models.enums import DataSourceType+from services.knowledge import dataset_service+from services.knowledge.dataset_service import DocumentService+from tests.unit_tests.model_factories import make_account, make_dataset, make_document++CREATED_AT = datetime(2026, 10, 16, 8, 0, 0)+CREATED_AT_EPOCH = 1792137600+++def non_utc_host_timezone(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:+ monkeypatch.setenv("TZ", "Asia/Tokyo")+ time.tzset()+ yield+ monkeypatch.undo()+ time.tzset()++[email protected]("non_utc_host_timezone")+def test_build_built_in_field_values_treats_naive_datetimes_as_utc() -> None:+ values = build_built_in_field_values(+ document_name="doc.txt",+ uploader="Alice",+ upload_date=CREATED_AT,+ last_update_date=CREATED_AT,+ data_source_type=DataSourceType.UPLOAD_FILE,+ )++ assert values == {+ BuiltInField.document_name: "doc.txt",+ BuiltInField.uploader: "Alice",+ BuiltInField.upload_date: CREATED_AT_EPOCH,+ BuiltInField.last_update_date: CREATED_AT_EPOCH,+ BuiltInField.source: "file_upload",+ }+++ ("data_source_type", "source"),+ [+ (DataSourceType.UPLOAD_FILE, "file_upload"),+ (DataSourceType.LOCAL_FILE, "file_upload"),+ (DataSourceType.NOTION_IMPORT, "notion"),+ (DataSourceType.WEBSITE_CRAWL, "website"),+ (DataSourceType.ONLINE_DOCUMENT, "online_document"),+ (DataSourceType.ONLINE_DRIVE, "online_drive"),+ ],+)+def test_build_built_in_field_values_maps_source_to_metadata_label(+ data_source_type: DataSourceType, source: str+) -> None:+ values = build_built_in_field_values(+ document_name="doc.txt",+ uploader=None,+ upload_date=CREATED_AT,+ last_update_date=CREATED_AT,+ data_source_type=data_source_type,+ )++ assert values[BuiltInField.source] == source+++ "data_source_type",+ [DataSourceType.UPLOAD_FILE, DataSourceType.NOTION_IMPORT, DataSourceType.WEBSITE_CRAWL],+)+def test_build_document_writes_built_in_fields_in_shared_format(+ monkeypatch: pytest.MonkeyPatch, data_source_type: DataSourceType+) -> None:+ monkeypatch.setattr(dataset_service, "naive_utc_now", lambda: CREATED_AT)+ dataset = make_dataset(built_in_field_enabled=True)++ document = DocumentService.build_document(+ dataset=dataset,+ process_rule_id=None,+ data_source_type=data_source_type,+ document_form="text_model",+ document_language="English",+ data_source_info={},+ created_from="web",+ position=1,+ account=make_account(name="Alice"),+ name="doc.txt",+ batch="batch-1",+ )++ assert document.doc_metadata == build_built_in_field_values(+ document_name="doc.txt",+ uploader="Alice",+ upload_date=CREATED_AT,+ last_update_date=CREATED_AT,+ data_source_type=data_source_type,+ )+++def test_build_document_skips_built_in_fields_when_disabled() -> None:+ dataset = make_dataset(built_in_field_enabled=False)++ document = DocumentService.build_document(+ dataset=dataset,+ process_rule_id=None,+ data_source_type=DataSourceType.UPLOAD_FILE,+ document_form="text_model",+ document_language="English",+ data_source_info={},+ created_from="web",+ position=1,+ account=make_account(name="Alice"),+ name="doc.txt",+ batch="batch-1",+ )++ assert not document.doc_metadata+++FRACTIONAL_CREATED_AT = datetime(2026, 10, 16, 8, 0, 0, 900000)+++def test_build_built_in_field_values_keeps_fractional_epoch_seconds() -> None:+ values = build_built_in_field_values(+ document_name="doc.txt",+ uploader=None,+ upload_date=FRACTIONAL_CREATED_AT,+ last_update_date=FRACTIONAL_CREATED_AT,+ data_source_type=DataSourceType.UPLOAD_FILE,+ )++ assert values[BuiltInField.upload_date] == pytest.approx(CREATED_AT_EPOCH + 0.9, abs=1e-6)+ assert values[BuiltInField.last_update_date] == pytest.approx(CREATED_AT_EPOCH + 0.9, abs=1e-6)++[email protected]("field", [BuiltInField.upload_date, BuiltInField.last_update_date])[email protected](("condition", "matches"), [("after", True), (">", True), ("=", False), ("is", False)])+def test_fractional_built_in_dates_keep_integer_second_boundary_semantics(+ sqlite_session_factory: sessionmaker[Session], field: BuiltInField, condition: str, matches: bool+) -> None:+ values = build_built_in_field_values(+ document_name="doc.txt",+ uploader=None,+ upload_date=FRACTIONAL_CREATED_AT,+ last_update_date=FRACTIONAL_CREATED_AT,+ data_source_type=DataSourceType.UPLOAD_FILE,+ )+ doc_metadata: dict[str, object] = dict(values)+ with sqlite_session_factory.begin() as session:+ session.add(make_document(document_id="document-1", doc_metadata=doc_metadata))++ filters = DatasetRetrieval.process_metadata_filter_func(0, condition, field, CREATED_AT_EPOCH, [])+ with sqlite_session_factory() as session:+ found = session.scalars(select(Document.id).where(*filters)).all()++ assert found == (["document-1"] if matches else [])