Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(service_api): dep-inject dataset payloads with @model_validate (#41490)

Eddy ZHANG884ece570e80
diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py
index 51667e1fbde..da17036689f 100644
--- a/api/controllers/service_api/dataset/dataset.py
+++ b/api/controllers/service_api/dataset/dataset.py
@@ -25,7 +25,7 @@ from controllers.common.schema import (
register_schema_models,
)
from controllers.common.session import with_session
-from controllers.console.wraps import edit_permission_required
+from controllers.console.wraps import edit_permission_required, model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.dataset.error import DatasetInUseError, DatasetNameDuplicateError, InvalidActionError
from controllers.service_api.wraps import (
@@ -669,14 +669,13 @@ class DatasetApi(DatasetApiResource):
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
- def patch(self, session: Session, _, dataset_id: UUID):
+ @model_validate(DatasetUpdatePayload)
+ def patch(self, payload: DatasetUpdatePayload, session: Session, _, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str, session)
if dataset is None:
raise NotFound("Dataset not found.")
- payload_dict = service_api_ns.payload or {}
- payload = DatasetUpdatePayload.model_validate(payload_dict)
update_data = payload.model_dump(exclude_unset=True)
if payload.permission is not None:
update_data["permission"] = str(payload.permission)
@@ -944,13 +943,13 @@ class DatasetTagsApi(DatasetApiResource):
service_api_ns.models[KnowledgeTagResponse.__name__],
)
@with_session
- def post(self, session: Session, _):
+ @model_validate(TagCreatePayload)
+ def post(self, payload: TagCreatePayload, session: Session, _):
"""Add a knowledge type tag."""
assert isinstance(current_user, Account)
if not (current_user.has_edit_permission or current_user.is_dataset_editor):
raise Forbidden()
- payload = TagCreatePayload.model_validate(service_api_ns.payload or {})
tag = TagService.save_tags(SaveTagPayload(name=payload.name, type=TagType.KNOWLEDGE), session)
response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count="0")
@@ -982,12 +981,12 @@ class DatasetTagsApi(DatasetApiResource):
service_api_ns.models[KnowledgeTagResponse.__name__],
)
@with_session
- def patch(self, session: Session, _):
+ @model_validate(TagUpdatePayload)
+ def patch(self, payload: TagUpdatePayload, session: Session, _):
assert isinstance(current_user, Account)
if not (current_user.has_edit_permission or current_user.is_dataset_editor):
raise Forbidden()
- payload = TagUpdatePayload.model_validate(service_api_ns.payload or {})
tag_id = payload.tag_id
tag = TagService.update_tags(
UpdateTagServicePayload(name=payload.name), tag_id, session, tag_type=TagType.KNOWLEDGE
@@ -1019,9 +1018,9 @@ class DatasetTagsApi(DatasetApiResource):
)
@edit_permission_required
@with_session
- def delete(self, session: Session, _):
+ @model_validate(TagDeletePayload)
+ def delete(self, payload: TagDeletePayload, session: Session, _):
"""Delete a knowledge type tag."""
- payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
TagService.delete_tag(payload.tag_id, session, tag_type=TagType.KNOWLEDGE)
return "", 204
@@ -1049,13 +1048,13 @@ class DatasetTagBindingApi(DatasetApiResource):
}
)
@with_session
- def post(self, session: Session, _):
+ @model_validate(TagBindingPayload)
+ def post(self, payload: TagBindingPayload, session: Session, _):
# The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
assert isinstance(current_user, Account)
if not (current_user.has_edit_permission or current_user.is_dataset_editor):
raise Forbidden()
- payload = TagBindingPayload.model_validate(service_api_ns.payload or {})
TagService.save_tag_binding(
TagBindingCreatePayload(tag_ids=payload.tag_ids, target_id=payload.target_id, type=TagType.KNOWLEDGE),
session,
@@ -1086,13 +1085,13 @@ class DatasetTagUnbindingApi(DatasetApiResource):
}
)
@with_session
- def post(self, session: Session, _):
+ @model_validate(TagUnbindingPayload)
+ def post(self, payload: TagUnbindingPayload, session: Session, _):
# The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
assert isinstance(current_user, Account)
if not (current_user.has_edit_permission or current_user.is_dataset_editor):
raise Forbidden()
- payload = TagUnbindingPayload.model_validate(service_api_ns.payload or {})
TagService.delete_tag_binding(
TagBindingDeletePayload(tag_ids=payload.tag_ids, target_id=payload.target_id, type=TagType.KNOWLEDGE),
session,
diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py
index 216e3143339..5049c2a3782 100644
--- a/api/controllers/service_api/dataset/document.py
+++ b/api/controllers/service_api/dataset/document.py
@@ -45,6 +45,7 @@ from controllers.common.schema import (
register_schema_models,
)
from controllers.common.session import with_session
+from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.app.error import ProviderNotInitializeError
from controllers.service_api.dataset.error import (
@@ -1069,9 +1070,8 @@ class DocumentBatchDownloadZipApi(DatasetApiResource):
@service_api_ns.response(200, "ZIP archive generated successfully")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session(write=False)
- def post(self, session: Session, tenant_id, dataset_id: UUID):
- payload = DocumentBatchDownloadZipPayload.model_validate(service_api_ns.payload or {})
-
+ @model_validate(DocumentBatchDownloadZipPayload)
+ def post(self, payload: DocumentBatchDownloadZipPayload, session: Session, tenant_id, dataset_id: UUID):
upload_files, download_name = DocumentService.prepare_document_batch_download_zip(
dataset_id=str(dataset_id),
document_ids=[str(document_id) for document_id in payload.document_ids],
diff --git a/api/controllers/service_api/dataset/metadata.py b/api/controllers/service_api/dataset/metadata.py
index 52fa8f370d5..693ba8377b9 100644
--- a/api/controllers/service_api/dataset/metadata.py
+++ b/api/controllers/service_api/dataset/metadata.py
@@ -316,15 +316,14 @@ class DocumentMetadataEditServiceApi(DatasetApiResource):
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
- def post(self, session: Session, tenant_id, dataset_id: UUID):
+ @model_validate(MetadataOperationData)
+ def post(self, metadata_args: MetadataOperationData, session: Session, tenant_id, dataset_id: UUID):
"""Update metadata for multiple documents."""
dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), str(tenant_id), session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
- metadata_args = MetadataOperationData.model_validate(service_api_ns.payload or {})
-
try:
MetadataService.update_documents_metadata(
dataset, metadata_args, cast(Account, current_user), session=session
diff --git a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py
index 35a083a3714..245466367b7 100644
--- a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py
+++ b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py
@@ -24,6 +24,7 @@ from controllers.common.schema import (
register_schema_model,
)
from controllers.console.app.wraps import with_session
+from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.dataset.error import PipelineRunError
from controllers.service_api.schema import event_stream_response, json_or_event_stream_response, multipart_file_params
@@ -215,7 +216,8 @@ class DatasourceNodeRunApi(DatasetApiResource):
}
)
@service_api_ns.expect(service_api_ns.models[DatasourceNodeRunPayload.__name__])
- def post(self, tenant_id: str, dataset_id: UUID, node_id: str):
+ @model_validate(DatasourceNodeRunPayload)
+ def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UUID, node_id: str):
"""Resource for getting datasource plugins."""
dataset_id_str = str(dataset_id)
# Verify dataset ownership
@@ -224,7 +226,6 @@ class DatasourceNodeRunApi(DatasetApiResource):
if not dataset:
raise NotFound("Dataset not found.")
- payload = DatasourceNodeRunPayload.model_validate(service_api_ns.payload or {})
assert isinstance(current_user, Account)
rag_pipeline_service: RagPipelineService = RagPipelineService(db.session())
pipeline: Pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
index df1c9ff00d2..4a87b4a8ff6 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
@@ -506,7 +506,10 @@ class TestDatasourceNodeRunApiPost:
The source asserts ``isinstance(current_user, Account)`` and delegates to
``RagPipelineService`` and ``PipelineGenerator``, so we patch those plus
- ``current_user`` and ``service_api_ns``.
+ ``current_user``. ``post`` is wrapped in ``@model_validate``, which parses
+ the JSON request body live, so payloads are supplied via
+ ``test_request_context(json=...)`` and validation runs before the dataset
+ ownership guard.
"""
@patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.helper")
@@ -516,10 +519,8 @@ class TestDatasourceNodeRunApiPost:
new_callable=lambda: Account(name="Test Account", email="[email protected]"),
)
@patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.RagPipelineService")
- @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.service_api_ns")
def test_post_success(
self,
- mock_ns,
mock_svc_cls,
current_account,
mock_gen,
@@ -534,12 +535,6 @@ class TestDatasourceNodeRunApiPost:
_persist_dataset(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id)
- mock_ns.payload = {
- "inputs": {"url": "https://example.com"},
- "datasource_type": "online_document",
- "is_published": True,
- }
-
pipeline = _persist_pipeline(sqlite_session, tenant_id=tenant_id)
mock_svc_instance = Mock()
mock_svc_instance.get_pipeline.return_value = pipeline
@@ -549,7 +544,15 @@ class TestDatasourceNodeRunApiPost:
mock_gen.convert_to_event_stream.return_value = iter(["stream_event"])
mock_helper.compact_generate_response.return_value = {"result": "ok"}
- with app.test_request_context("/datasets/test/pipeline/datasource/nodes/node_abc/run", method="POST"):
+ with app.test_request_context(
+ "/datasets/test/pipeline/datasource/nodes/node_abc/run",
+ method="POST",
+ json={
+ "inputs": {"url": "https://example.com"},
+ "datasource_type": "online_document",
+ "is_published": True,
+ },
+ ):
api = DatasourceNodeRunApi()
response = api.post(tenant_id=tenant_id, dataset_id=dataset_id, node_id=node_id)
@@ -561,7 +564,13 @@ class TestDatasourceNodeRunApiPost:
def test_post_not_found(self, app: Flask, sqlite_session: Session):
"""Test NotFound when dataset check fails."""
- with app.test_request_context("/datasets/test/pipeline/datasource/nodes/n1/run", method="POST"):
+ # `@model_validate` parses the body before the ownership guard, so a
+ # valid payload is required to reach the NotFound branch.
+ with app.test_request_context(
+ "/datasets/test/pipeline/datasource/nodes/n1/run",
+ method="POST",
+ json={"inputs": {}, "datasource_type": "online_document", "is_published": True},
+ ):
api = DatasourceNodeRunApi()
with pytest.raises(NotFound):
api.post(tenant_id=str(uuid.uuid4()), dataset_id=str(uuid.uuid4()), node_id="n1")
@@ -570,19 +579,17 @@ class TestDatasourceNodeRunApiPost:
"controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.current_user",
new="not_account",
)
- @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.service_api_ns")
- def test_post_fails_when_current_user_not_account(self, mock_ns, app: Flask, sqlite_session: Session):
+ def test_post_fails_when_current_user_not_account(self, app: Flask, sqlite_session: Session):
"""Test AssertionError when current_user is not an Account instance."""
tenant_id = str(uuid.uuid4())
dataset_id = str(uuid.uuid4())
_persist_dataset(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id)
- mock_ns.payload = {
- "inputs": {},
- "datasource_type": "local_file",
- "is_published": True,
- }
- with app.test_request_context("/datasets/test/pipeline/datasource/nodes/n1/run", method="POST"):
+ with app.test_request_context(
+ "/datasets/test/pipeline/datasource/nodes/n1/run",
+ method="POST",
+ json={"inputs": {}, "datasource_type": "local_file", "is_published": True},
+ ):
api = DatasourceNodeRunApi()
with pytest.raises(AssertionError):
api.post(tenant_id=tenant_id, dataset_id=dataset_id, node_id="n1")
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
index fc17d166994..ade48fd2c03 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
@@ -10,7 +10,7 @@ from inspect import unwrap
from unittest.mock import MagicMock, patch
import pytest
-from flask import Flask
+from flask import Flask, request
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
@@ -462,7 +462,7 @@ class TestDatasetApiPatch:
dataset: Dataset,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetApi
+ from controllers.service_api.dataset.dataset import DatasetApi, DatasetUpdatePayload
dataset.name = "Updated Dataset"
mock_dataset_svc.get_dataset.return_value = dataset
@@ -481,8 +481,12 @@ class TestDatasetApiPatch:
json=payload,
):
api = DatasetApi()
+ # `patch` is wrapped in @model_validate, so the unwrapped view expects
+ # the validated model where the decorator would have injected it.
+ validated_payload = DatasetUpdatePayload.model_validate(request.get_json() or {})
response, status = unwrap(api.patch)(
api,
+ validated_payload,
controller_session,
_=dataset.tenant_id,
dataset_id=dataset.id,
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_tag_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_tag_apis.py
index 1f20b74180f..55aae9be652 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_tag_apis.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_tag_apis.py
@@ -10,7 +10,7 @@ from inspect import unwrap
from unittest.mock import MagicMock, patch
import pytest
-from flask import Flask
+from flask import Flask, request
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden
@@ -109,7 +109,7 @@ class TestDatasetTagsApiPost:
tenant: Tenant,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagsApi
+ from controllers.service_api.dataset.dataset import DatasetTagsApi, TagCreatePayload
tag = make_tag(controller_session, tenant, account, id="tag-new", name="New Tag")
mock_tag_svc.save_tags.return_value = tag
@@ -120,7 +120,8 @@ class TestDatasetTagsApiPost:
json={"name": "New Tag"},
):
api = DatasetTagsApi()
- response, status = unwrap(api.post)(api, controller_session, _=None)
+ payload = TagCreatePayload.model_validate(request.get_json() or {})
+ response, status = unwrap(api.post)(api, payload, controller_session, _=None)
assert status == 200
assert response == {"id": "tag-new", "name": "New Tag", "type": "knowledge", "binding_count": "0"}
@@ -155,7 +156,7 @@ class TestDatasetTagsApiPatch:
tenant: Tenant,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagsApi
+ from controllers.service_api.dataset.dataset import DatasetTagsApi, TagUpdatePayload
tag = make_tag(controller_session, tenant, account, id="tag-1", name="Updated Tag")
mock_tag_svc.update_tags.return_value = tag
@@ -168,7 +169,8 @@ class TestDatasetTagsApiPatch:
json={"name": "Updated Tag", "tag_id": "tag-1"},
):
api = DatasetTagsApi()
- response, status = unwrap(api.patch)(api, controller_session, _=None)
+ payload = TagUpdatePayload.model_validate(request.get_json() or {})
+ response, status = unwrap(api.patch)(api, payload, controller_session, _=None)
assert status == 200
assert response == {"id": "tag-1", "name": "Updated Tag", "type": "knowledge", "binding_count": "5"}
@@ -206,7 +208,7 @@ class TestDatasetTagsApiDelete:
app: Flask,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagsApi
+ from controllers.service_api.dataset.dataset import DatasetTagsApi, TagDeletePayload
mock_tag_svc.delete_tag.return_value = None
mock_service_api_ns.payload = {"tag_id": "tag-1"}
@@ -217,7 +219,8 @@ class TestDatasetTagsApiDelete:
json={"tag_id": "tag-1"},
):
api = DatasetTagsApi()
- result = unwrap(api.delete)(api, controller_session, _=None)
+ payload = TagDeletePayload.model_validate(request.get_json() or {})
+ result = unwrap(api.delete)(api, payload, controller_session, _=None)
assert result == ("", 204)
mock_tag_svc.delete_tag.assert_called_once_with("tag-1", controller_session, tag_type=TagType.KNOWLEDGE)
@@ -263,7 +266,7 @@ class TestDatasetTagBindingApiPost:
app: Flask,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagBindingApi
+ from controllers.service_api.dataset.dataset import DatasetTagBindingApi, TagBindingPayload
mock_tag_svc.save_tag_binding.return_value = None
@@ -273,7 +276,8 @@ class TestDatasetTagBindingApiPost:
json={"tag_ids": ["tag-1"], "target_id": "ds-1"},
):
api = DatasetTagBindingApi()
- result = unwrap(api.post)(api, controller_session, _=None)
+ payload = TagBindingPayload.model_validate(request.get_json() or {})
+ result = unwrap(api.post)(api, payload, controller_session, _=None)
assert result == ("", 204)
from services.tag_service import TagBindingCreatePayload
@@ -309,7 +313,7 @@ class TestDatasetTagUnbindingApiPost:
app: Flask,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagUnbindingApi
+ from controllers.service_api.dataset.dataset import DatasetTagUnbindingApi, TagUnbindingPayload
mock_tag_svc.delete_tag_binding.return_value = None
@@ -319,7 +323,8 @@ class TestDatasetTagUnbindingApiPost:
json={"tag_ids": ["tag-1"], "target_id": "ds-1"},
):
api = DatasetTagUnbindingApi()
- result = unwrap(api.post)(api, controller_session, _=None)
+ payload = TagUnbindingPayload.model_validate(request.get_json() or {})
+ result = unwrap(api.post)(api, payload, controller_session, _=None)
assert result == ("", 204)
from services.tag_service import TagBindingDeletePayload
@@ -337,7 +342,7 @@ class TestDatasetTagUnbindingApiPost:
app: Flask,
controller_session: Session,
) -> None:
- from controllers.service_api.dataset.dataset import DatasetTagUnbindingApi
+ from controllers.service_api.dataset.dataset import DatasetTagUnbindingApi, TagUnbindingPayload
mock_tag_svc.delete_tag_binding.return_value = None
@@ -347,7 +352,8 @@ class TestDatasetTagUnbindingApiPost:
json={"tag_id": "tag-1", "target_id": "ds-1"},
):
api = DatasetTagUnbindingApi()
- result = unwrap(api.post)(api, controller_session, _=None)
+ payload = TagUnbindingPayload.model_validate(request.get_json() or {})
+ result = unwrap(api.post)(api, payload, controller_session, _=None)
assert result == ("", 204)
from services.tag_service import TagBindingDeletePayload
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py b/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py
index 64e331ea072..e138a5b3c77 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_metadata.py
@@ -38,7 +38,7 @@ from controllers.service_api.dataset.metadata import (
from models.account import Account, Tenant
from models.dataset import Dataset
from models.enums import PermissionEnum
-from services.entities.knowledge_entities.knowledge_entities import MetadataArgs
+from services.entities.knowledge_entities.knowledge_entities import MetadataArgs, MetadataOperationData
from services.errors.metadata import MetadataResourceNotFoundError
@@ -537,7 +537,8 @@ class TestDocumentMetadataEditPost(_UsesSQLiteSession):
@staticmethod
def _call_post(api, session: Session, **kwargs):
- return unwrap(api.post)(api, session, **kwargs)
+ metadata_args = MetadataOperationData.model_validate(request.get_json() or {})
+ return unwrap(api.post)(api, metadata_args, session, **kwargs)
@patch("controllers.service_api.dataset.metadata.MetadataService")
@patch("controllers.service_api.dataset.metadata.DatasetService")