Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): write built-in document metadata in a single format (#44218)

Tianyu Caie7c2c6fad5ac
diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.py
index 3b4773bc748..b0e28ce5b69 100644
--- a/api/core/app/apps/pipeline/pipeline_generator.py
+++ b/api/core/app/apps/pipeline/pipeline_generator.py
@@ -1,5 +1,4 @@
import contextvars
-import datetime
import json
import logging
import secrets
@@ -38,7 +37,6 @@ from core.datasource.entities.datasource_entities import (
)
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.entities.knowledge_entities import PipelineDataset, PipelineDocument
-from core.rag.index_processor.constant.built_in_field import BuiltInField
from core.repositories.factory import (
DifyCoreRepositoryFactory,
WorkflowExecutionRepository,
@@ -47,9 +45,10 @@ from core.repositories.factory import (
from extensions.ext_database import db
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader
+from libs.datetime_utils import naive_utc_now
from libs.flask_utils import preserve_flask_contexts
from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom
-from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline
+from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline, build_built_in_field_values
from models.enums import WorkflowRunTriggeredFrom
from models.model import AppMode
from services.data_source.provider_service import DatasourceProviderService
@@ -747,17 +746,15 @@ class PipelineGenerator(BaseAppGenerator):
created_by=account.id,
doc_form=document_form,
)
- doc_metadata = {}
if built_in_field_enabled:
- doc_metadata = {
- BuiltInField.document_name: name,
- BuiltInField.uploader: account.name,
- BuiltInField.upload_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),
- BuiltInField.last_update_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),
- BuiltInField.source: datasource_type,
- }
- if doc_metadata:
- document.doc_metadata = doc_metadata
+ now = naive_utc_now()
+ document.doc_metadata = build_built_in_field_values(
+ document_name=name,
+ uploader=account.name,
+ upload_date=now,
+ last_update_date=now,
+ data_source_type=datasource_type,
+ )
return document
def _format_datasource_info_list(
diff --git a/api/models/dataset.py b/api/models/dataset.py
index e6535d6e16e..9c95fbf23a9 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -1,7 +1,7 @@
import json
import pickle
from collections.abc import Sequence
-from datetime import datetime
+from datetime import UTC, datetime
from json import JSONDecodeError
from typing import Any, ClassVar, TypedDict, cast, override
from uuid import uuid4
@@ -12,9 +12,10 @@ from sqlalchemy.orm import Mapped, mapped_column
from configs import dify_config
from core.rag.entities import PreProcessingRuleKey
-from core.rag.index_processor.constant.built_in_field import BuiltInField
+from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
+from libs.datetime_utils import ensure_naive_utc
from libs.uuid_utils import uuidv7
from models.base import Base, TypeBase
from models.enums import (
@@ -337,6 +338,35 @@ class DatasetProcessRule(TypeBase):
return None
+def _utc_epoch_seconds(value: datetime) -> float:
+ return ensure_naive_utc(value).replace(tzinfo=UTC).timestamp()
+
+
+def build_built_in_field_values(
+ *,
+ document_name: str,
+ uploader: str | None,
+ upload_date: datetime,
+ last_update_date: datetime,
+ data_source_type: str,
+) -> dict[str, str | float | None]:
+ """Return built-in metadata in the single format stored in ``Document.doc_metadata``.
+
+ Retrieval filters compare time fields numerically and ``source`` by exact string, so every
+ writer must store UTC epoch seconds and the ``MetadataDataSource`` label shown to users.
+ Epoch seconds keep their fractional part so strict and equality comparisons at an
+ integer-second boundary behave as they did for stored ``datetime.timestamp()`` values.
+ Naive datetimes are treated as UTC.
+ """
+ return {
+ BuiltInField.document_name: document_name,
+ BuiltInField.uploader: uploader,
+ BuiltInField.upload_date: _utc_epoch_seconds(upload_date),
+ BuiltInField.last_update_date: _utc_epoch_seconds(last_update_date),
+ BuiltInField.source: MetadataDataSource[data_source_type],
+ }
+
+
class Document(Base):
__tablename__ = "documents"
__table_args__ = (
diff --git a/api/repositories/knowledge/metadata_repository.py b/api/repositories/knowledge/metadata_repository.py
index 1610fd3ab0d..fb28a19ec7b 100644
--- a/api/repositories/knowledge/metadata_repository.py
+++ b/api/repositories/knowledge/metadata_repository.py
@@ -5,9 +5,15 @@ from collections.abc import Sequence
from sqlalchemy import func, select
from sqlalchemy.orm import Session, sessionmaker
-from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource
+from core.rag.index_processor.constant.built_in_field import BuiltInField
from libs.datetime_utils import naive_utc_now
-from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Document
+from models.dataset import (
+ Dataset,
+ DatasetMetadata,
+ DatasetMetadataBinding,
+ Document,
+ build_built_in_field_values,
+)
from repositories.knowledge.dataset_read_repository import get_document_uploader
from services.errors.metadata import MetadataResourceNotFoundError
from services.knowledge.metadata.application import MetadataList, MetadataOperation, MetadataRecord
@@ -115,14 +121,14 @@ class SQLAlchemyMetadataRepository:
return self._record(row)
@staticmethod
- def _built_in(session: Session, document: Document) -> dict[str, str | int | float | None]:
- return {
- BuiltInField.document_name: document.name,
- BuiltInField.uploader: get_document_uploader(document, session=session),
- BuiltInField.upload_date: document.upload_date.timestamp(),
- BuiltInField.last_update_date: document.last_update_date.timestamp(),
- BuiltInField.source: MetadataDataSource[document.data_source_type],
- }
+ def _built_in(session: Session, document: Document) -> dict[str, str | float | None]:
+ return build_built_in_field_values(
+ document_name=document.name,
+ uploader=get_document_uploader(document, session=session),
+ upload_date=document.upload_date,
+ last_update_date=document.last_update_date,
+ data_source_type=document.data_source_type,
+ )
def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None:
with self._sessions.begin() as session:
diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py
index 64a33ac308f..818a8e9811f 100644
--- a/api/services/knowledge/dataset_service.py
+++ b/api/services/knowledge/dataset_service.py
@@ -51,6 +51,7 @@ from models.dataset import (
DocumentSegment,
ExternalKnowledgeBindings,
Pipeline,
+ build_built_in_field_values,
)
from models.enums import (
DatasetRuntimeMode,
@@ -2880,17 +2881,15 @@ class DocumentService:
doc_language=document_language,
need_summary=need_summary,
)
- doc_metadata = {}
if dataset.built_in_field_enabled:
- doc_metadata = {
- BuiltInField.document_name: name,
- BuiltInField.uploader: account.name,
- BuiltInField.upload_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),
- BuiltInField.last_update_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"),
- BuiltInField.source: data_source_type,
- }
- if doc_metadata:
- document.doc_metadata = doc_metadata
+ now = naive_utc_now()
+ document.doc_metadata = build_built_in_field_values(
+ document_name=name,
+ uploader=account.name,
+ upload_date=now,
+ last_update_date=now,
+ data_source_type=data_source_type,
+ )
return document
@staticmethod
diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
index 71cf0564e34..734f82f47b7 100644
--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
@@ -1,5 +1,6 @@
import contextlib
import json
+from datetime import datetime
from types import SimpleNamespace
from unittest.mock import MagicMock, PropertyMock, create_autospec
@@ -13,6 +14,7 @@ import core.app.apps.pipeline.pipeline_generator as module
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.entities.app_invoke_entities import InvokeFrom
from core.datasource.entities.datasource_entities import DatasourceProviderType
+from core.rag.index_processor.constant.built_in_field import BuiltInField
from models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipeline
from models.enums import DataSourceType, EndUserType
from models.model import EndUser
@@ -660,7 +662,10 @@ def test_handle_response_value_error_triggers_generate_task_stopped(generator, m
)
-def test_build_document_sets_metadata_for_builtin_fields(generator):
+def test_build_document_sets_metadata_for_builtin_fields(generator, monkeypatch):
+ created_at = datetime(2026, 10, 16, 8, 0, 0)
+ monkeypatch.setattr(module, "naive_utc_now", lambda: created_at)
+
document = generator._build_document(
tenant_id="tenant",
dataset_id="ds",
@@ -675,7 +680,13 @@ def test_build_document_sets_metadata_for_builtin_fields(generator):
)
assert document.name == "file"
- assert document.doc_metadata
+ assert document.doc_metadata == {
+ BuiltInField.document_name: "file",
+ BuiltInField.uploader: "User",
+ BuiltInField.upload_date: 1792137600,
+ BuiltInField.last_update_date: 1792137600,
+ BuiltInField.source: "file_upload",
+ }
def test_build_document_supports_online_drive_datasource_type(generator):
diff --git a/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.py b/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.py
new file mode 100644
index 00000000000..02cc45958f4
--- /dev/null
+++ b/api/tests/unit_tests/services/knowledge/test_metadata_built_in_fields.py
@@ -0,0 +1,163 @@
+import time
+from collections.abc import Iterator
+from datetime import datetime
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.rag.index_processor.constant.built_in_field import BuiltInField
+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
+from models.dataset import Document, build_built_in_field_values
+from models.enums import DataSourceType
+from services.knowledge import dataset_service
+from services.knowledge.dataset_service import DocumentService
+from tests.unit_tests.model_factories import make_account, make_dataset, make_document
+
+CREATED_AT = datetime(2026, 10, 16, 8, 0, 0)
+CREATED_AT_EPOCH = 1792137600
+
+
+def non_utc_host_timezone(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ monkeypatch.setenv("TZ", "Asia/Tokyo")
+ time.tzset()
+ yield
+ monkeypatch.undo()
+ time.tzset()
+
+
[email protected]("non_utc_host_timezone")
+def test_build_built_in_field_values_treats_naive_datetimes_as_utc() -> None:
+ values = build_built_in_field_values(
+ document_name="doc.txt",
+ uploader="Alice",
+ upload_date=CREATED_AT,
+ last_update_date=CREATED_AT,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ )
+
+ assert values == {
+ BuiltInField.document_name: "doc.txt",
+ BuiltInField.uploader: "Alice",
+ BuiltInField.upload_date: CREATED_AT_EPOCH,
+ BuiltInField.last_update_date: CREATED_AT_EPOCH,
+ BuiltInField.source: "file_upload",
+ }
+
+
+ ("data_source_type", "source"),
+ [
+ (DataSourceType.UPLOAD_FILE, "file_upload"),
+ (DataSourceType.LOCAL_FILE, "file_upload"),
+ (DataSourceType.NOTION_IMPORT, "notion"),
+ (DataSourceType.WEBSITE_CRAWL, "website"),
+ (DataSourceType.ONLINE_DOCUMENT, "online_document"),
+ (DataSourceType.ONLINE_DRIVE, "online_drive"),
+ ],
+)
+def test_build_built_in_field_values_maps_source_to_metadata_label(
+ data_source_type: DataSourceType, source: str
+) -> None:
+ values = build_built_in_field_values(
+ document_name="doc.txt",
+ uploader=None,
+ upload_date=CREATED_AT,
+ last_update_date=CREATED_AT,
+ data_source_type=data_source_type,
+ )
+
+ assert values[BuiltInField.source] == source
+
+
+ "data_source_type",
+ [DataSourceType.UPLOAD_FILE, DataSourceType.NOTION_IMPORT, DataSourceType.WEBSITE_CRAWL],
+)
+def test_build_document_writes_built_in_fields_in_shared_format(
+ monkeypatch: pytest.MonkeyPatch, data_source_type: DataSourceType
+) -> None:
+ monkeypatch.setattr(dataset_service, "naive_utc_now", lambda: CREATED_AT)
+ dataset = make_dataset(built_in_field_enabled=True)
+
+ document = DocumentService.build_document(
+ dataset=dataset,
+ process_rule_id=None,
+ data_source_type=data_source_type,
+ document_form="text_model",
+ document_language="English",
+ data_source_info={},
+ created_from="web",
+ position=1,
+ account=make_account(name="Alice"),
+ name="doc.txt",
+ batch="batch-1",
+ )
+
+ assert document.doc_metadata == build_built_in_field_values(
+ document_name="doc.txt",
+ uploader="Alice",
+ upload_date=CREATED_AT,
+ last_update_date=CREATED_AT,
+ data_source_type=data_source_type,
+ )
+
+
+def test_build_document_skips_built_in_fields_when_disabled() -> None:
+ dataset = make_dataset(built_in_field_enabled=False)
+
+ document = DocumentService.build_document(
+ dataset=dataset,
+ process_rule_id=None,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ document_form="text_model",
+ document_language="English",
+ data_source_info={},
+ created_from="web",
+ position=1,
+ account=make_account(name="Alice"),
+ name="doc.txt",
+ batch="batch-1",
+ )
+
+ assert not document.doc_metadata
+
+
+FRACTIONAL_CREATED_AT = datetime(2026, 10, 16, 8, 0, 0, 900000)
+
+
+def test_build_built_in_field_values_keeps_fractional_epoch_seconds() -> None:
+ values = build_built_in_field_values(
+ document_name="doc.txt",
+ uploader=None,
+ upload_date=FRACTIONAL_CREATED_AT,
+ last_update_date=FRACTIONAL_CREATED_AT,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ )
+
+ assert values[BuiltInField.upload_date] == pytest.approx(CREATED_AT_EPOCH + 0.9, abs=1e-6)
+ assert values[BuiltInField.last_update_date] == pytest.approx(CREATED_AT_EPOCH + 0.9, abs=1e-6)
+
+
[email protected]("field", [BuiltInField.upload_date, BuiltInField.last_update_date])
[email protected](("condition", "matches"), [("after", True), (">", True), ("=", False), ("is", False)])
+def test_fractional_built_in_dates_keep_integer_second_boundary_semantics(
+ sqlite_session_factory: sessionmaker[Session], field: BuiltInField, condition: str, matches: bool
+) -> None:
+ values = build_built_in_field_values(
+ document_name="doc.txt",
+ uploader=None,
+ upload_date=FRACTIONAL_CREATED_AT,
+ last_update_date=FRACTIONAL_CREATED_AT,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ )
+ doc_metadata: dict[str, object] = dict(values)
+ with sqlite_session_factory.begin() as session:
+ session.add(make_document(document_id="document-1", doc_metadata=doc_metadata))
+
+ filters = DatasetRetrieval.process_metadata_filter_func(0, condition, field, CREATED_AT_EPOCH, [])
+ with sqlite_session_factory() as session:
+ found = session.scalars(select(Document.id).where(*filters)).all()
+
+ assert found == (["document-1"] if matches else [])