Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): normalize blank legacy dataset retrieval settings (#45020)

hdu_zhoub8304c80f2d2
diff --git a/api/models/dataset.py b/api/models/dataset.py
index 72af9ae19b8..360b46455cf 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -234,9 +234,9 @@ class Dataset(TypeBase, kw_only=True):
def retrieval_model_dict(self):
"""Return a normalized retrieval model payload for API responses.
- Older rows may only persist a partial retrieval model dict. Merge the
- stored value over the current defaults so response validation still sees
- the required baseline fields.
+ Older rows may persist partial settings or blank values for disabled
+ options. Fill missing or blank baseline fields without rewriting stored
+ settings; leave other values for response validation.
"""
default_retrieval_model = {
"search_method": RetrievalMethod.SEMANTIC_SEARCH,
@@ -248,7 +248,11 @@ class Dataset(TypeBase, kw_only=True):
if not self.retrieval_model:
return default_retrieval_model
- return {**default_retrieval_model, **self.retrieval_model}
+ retrieval_model = {**default_retrieval_model, **self.retrieval_model}
+ for key in ("search_method", "reranking_enable", "reranking_model", "score_threshold_enabled"):
+ if retrieval_model[key] is None or retrieval_model[key] == "":
+ retrieval_model[key] = default_retrieval_model[key]
+ return retrieval_model
@staticmethod
def build_external_knowledge_info(
diff --git a/api/tests/unit_tests/models/test_dataset.py b/api/tests/unit_tests/models/test_dataset.py
index cc25be69258..fe6467dbd60 100644
--- a/api/tests/unit_tests/models/test_dataset.py
+++ b/api/tests/unit_tests/models/test_dataset.py
@@ -17,11 +17,13 @@ from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
+from pydantic import ValidationError
from sqlalchemy.orm import Session
from core.rag.entities import ParentMode
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.storage.storage_type import StorageType
+from fields.dataset_fields import DatasetRetrievalModelResponse
from models.account import Account
from models.base import Base, TypeBase
from models.dataset import (
@@ -569,6 +571,58 @@ class TestDatasetModelValidation:
assert result["score_threshold_enabled"] is True
assert result["score_threshold"] == 0.42
+ @pytest.mark.parametrize("blank", [None, ""])
+ def test_dataset_retrieval_model_dict_normalizes_blank_legacy_values(self, blank: str | None) -> None:
+ stored = {
+ "search_method": blank,
+ "reranking_enable": blank,
+ "reranking_model": blank,
+ "score_threshold_enabled": blank,
+ "top_k": 4,
+ }
+ dataset = Dataset(retrieval_model=stored.copy())
+
+ response = DatasetRetrievalModelResponse.model_validate(dataset.retrieval_model_dict)
+
+ assert response.search_method == "semantic_search"
+ assert response.reranking_enable is False
+ assert response.score_threshold_enabled is False
+ assert response.reranking_model.reranking_provider_name == ""
+ assert response.reranking_model.reranking_model_name == ""
+ assert response.top_k == 4
+ assert dataset.retrieval_model == stored
+
+ @pytest.mark.parametrize("flag", [True, False, "true", "false"])
+ def test_dataset_retrieval_model_dict_preserves_explicit_settings(self, flag: bool | str) -> None:
+ stored = {
+ "search_method": "keyword_search",
+ "reranking_enable": flag,
+ "reranking_model": {"reranking_provider_name": "provider", "reranking_model_name": "model"},
+ "score_threshold_enabled": flag,
+ "score_threshold": 0,
+ "top_k": 4,
+ }
+ dataset = Dataset(retrieval_model=stored.copy())
+
+ response = DatasetRetrievalModelResponse.model_validate(dataset.retrieval_model_dict)
+
+ assert response.search_method == "keyword_search"
+ assert response.reranking_enable is (flag is True or flag == "true")
+ assert response.score_threshold_enabled is response.reranking_enable
+ assert response.reranking_model.reranking_model_name == "model"
+ assert response.score_threshold == 0
+ assert response.top_k == 4
+ assert dataset.retrieval_model == stored
+
+ @pytest.mark.parametrize("field", ["search_method", "reranking_enable", "score_threshold_enabled"])
+ def test_dataset_retrieval_model_dict_does_not_hide_unknown_values(self, field: str) -> None:
+ dataset = Dataset(retrieval_model={field: "invalid"})
+
+ with pytest.raises(ValidationError) as error:
+ DatasetRetrievalModelResponse.model_validate(dataset.retrieval_model_dict)
+
+ assert error.value.errors()[0]["loc"] == (field,)
+
def test_dataset_gen_collection_name_by_id(self):
"""Test static method for generating collection name."""
# Arrange
diff --git a/api/tests/unit_tests/services/knowledge/datasets/test_adapters.py b/api/tests/unit_tests/services/knowledge/datasets/test_adapters.py
index 1224323a466..01fb45f27af 100644
--- a/api/tests/unit_tests/services/knowledge/datasets/test_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/datasets/test_adapters.py
@@ -135,6 +135,37 @@ def test_listing_materializes_page_and_owner_scoped_partial_members(
assert response["data"][0]["retrieval_model_dict"]["top_k"] == 2
+def test_listing_serializes_legacy_retrieval_settings_without_rewriting_rows(
+ operations: SQLAlchemyDatasetOperations, sqlite_session_factory: sessionmaker[Session]
+) -> None:
+ stored = {
+ "search_method": "",
+ "reranking_enable": "",
+ "reranking_model": None,
+ "score_threshold_enabled": "false",
+ "top_k": 4,
+ }
+ with sqlite_session_factory.begin() as session:
+ row = session.get(Dataset, "dataset")
+ assert row is not None
+ row.retrieval_model = stored
+
+ result = operations.list_datasets(CONTEXT, DatasetListFilter(ids=["dataset", "foreign"]), None)
+ response = DatasetListResponse.model_validate(result).model_dump(mode="json")
+
+ assert response["total"] == 1
+ assert response["data"][0]["id"] == "dataset"
+ retrieval = response["data"][0]["retrieval_model_dict"]
+ assert retrieval["search_method"] == "semantic_search"
+ assert retrieval["reranking_enable"] is False
+ assert retrieval["score_threshold_enabled"] is False
+ assert retrieval["top_k"] == 4
+ with sqlite_session_factory() as session:
+ row = session.get(Dataset, "dataset")
+ assert row is not None
+ assert row.retrieval_model == stored
+
+
@pytest.mark.parametrize(
("edition", "community_dev_enabled", "can_upgrade"),
[