Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test(vdb): use real ORM models in provider tests (#40687)

Asuka Minato39ce9f68fc84
diff --git a/api/providers/vdb/vdb-alibabacloud-mysql/tests/unit_tests/test_alibabacloud_mysql_factory.py b/api/providers/vdb/vdb-alibabacloud-mysql/tests/unit_tests/test_alibabacloud_mysql_factory.py
index 37b2331f0f8..ba18801f82b 100644
--- a/api/providers/vdb/vdb-alibabacloud-mysql/tests/unit_tests/test_alibabacloud_mysql_factory.py
+++ b/api/providers/vdb/vdb-alibabacloud-mysql/tests/unit_tests/test_alibabacloud_mysql_factory.py
@@ -1,10 +1,12 @@
-from types import SimpleNamespace
+import json
from unittest.mock import MagicMock, patch
import dify_vdb_alibabacloud_mysql.alibabacloud_mysql_vector as alibaba_module
import pytest
from dify_vdb_alibabacloud_mysql.alibabacloud_mysql_vector import AlibabaCloudMySQLVectorFactory
+from models.dataset import Dataset
+
def test_validate_distance_function_accepts_supported_values():
factory = AlibabaCloudMySQLVectorFactory()
@@ -22,10 +24,8 @@ def test_validate_distance_function_rejects_unsupported_values():
def test_factory_init_vector_uses_existing_index_struct_class_prefix(monkeypatch: pytest.MonkeyPatch):
factory = AlibabaCloudMySQLVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "existing_collection"}},
- index_struct=None,
+ dataset = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "existing_collection"}})
)
monkeypatch.setattr(alibaba_module.dify_config, "ALIBABACLOUD_MYSQL_HOST", "host")
@@ -47,11 +47,7 @@ def test_factory_init_vector_uses_existing_index_struct_class_prefix(monkeypatch
def test_factory_init_vector_generates_collection_name_when_index_struct_is_missing(monkeypatch: pytest.MonkeyPatch):
factory = AlibabaCloudMySQLVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-2",
- index_struct_dict=None,
- index_struct=None,
- )
+ dataset = Dataset(id="dataset-2")
monkeypatch.setattr(alibaba_module.Dataset, "gen_collection_name_by_id", lambda dataset_id: f"COL_{dataset_id}")
monkeypatch.setattr(alibaba_module.dify_config, "ALIBABACLOUD_MYSQL_HOST", "host")
diff --git a/api/providers/vdb/vdb-analyticdb/tests/unit_tests/test_analyticdb_vector.py b/api/providers/vdb/vdb-analyticdb/tests/unit_tests/test_analyticdb_vector.py
index 2e8052b7dcd..57ca12f4082 100644
--- a/api/providers/vdb/vdb-analyticdb/tests/unit_tests/test_analyticdb_vector.py
+++ b/api/providers/vdb/vdb-analyticdb/tests/unit_tests/test_analyticdb_vector.py
@@ -1,4 +1,4 @@
-from types import SimpleNamespace
+import json
from unittest.mock import MagicMock, patch
import dify_vdb_analyticdb.analyticdb_vector as analyticdb_module
@@ -8,6 +8,7 @@ from dify_vdb_analyticdb.analyticdb_vector_openapi import AnalyticdbVectorOpenAP
from dify_vdb_analyticdb.analyticdb_vector_sql import AnalyticdbVectorBySqlConfig
from core.rag.models.document import Document
+from models.dataset import Dataset
def test_init_prefers_openapi_when_api_config_is_provided():
@@ -85,7 +86,7 @@ def test_get_type_is_analyticdb():
def test_factory_builds_openapi_config_when_host_is_missing(monkeypatch: pytest.MonkeyPatch):
factory = AnalyticdbVectorFactory()
- dataset = SimpleNamespace(id="dataset-1", index_struct_dict=None, index_struct=None)
+ dataset = Dataset(id="dataset-1")
monkeypatch.setattr(analyticdb_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(analyticdb_module.dify_config, "ANALYTICDB_HOST", None)
@@ -111,9 +112,7 @@ def test_factory_builds_openapi_config_when_host_is_missing(monkeypatch: pytest.
def test_factory_builds_sql_config_when_host_is_present(monkeypatch: pytest.MonkeyPatch):
factory = AnalyticdbVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-2", index_struct_dict={"vector_store": {"class_prefix": "EXISTING"}}, index_struct=None
- )
+ dataset = Dataset(id="dataset-2", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING"}}))
monkeypatch.setattr(analyticdb_module.dify_config, "ANALYTICDB_HOST", "127.0.0.1")
monkeypatch.setattr(analyticdb_module.dify_config, "ANALYTICDB_PORT", 5432)
diff --git a/api/providers/vdb/vdb-baidu/tests/unit_tests/test_baidu_vector.py b/api/providers/vdb/vdb-baidu/tests/unit_tests/test_baidu_vector.py
index f0dddee3b90..d1d548dfd03 100644
--- a/api/providers/vdb/vdb-baidu/tests/unit_tests/test_baidu_vector.py
+++ b/api/providers/vdb/vdb-baidu/tests/unit_tests/test_baidu_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -7,6 +8,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_pymochow_modules():
@@ -256,7 +258,7 @@ def test_search_methods_delegate_to_database_table(baidu_module):
def test_factory_initializes_collection_name_and_index_struct(baidu_module, monkeypatch: pytest.MonkeyPatch):
factory = baidu_module.BaiduVectorFactory()
- dataset = SimpleNamespace(id="dataset-1", index_struct_dict=None, index_struct=None)
+ dataset = Dataset(id="dataset-1")
monkeypatch.setattr(baidu_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(baidu_module.dify_config, "BAIDU_VECTOR_DB_ENDPOINT", "https://endpoint")
monkeypatch.setattr(baidu_module.dify_config, "BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS", 1000)
@@ -528,10 +530,8 @@ def test_factory_uses_existing_collection_prefix_when_index_struct_exists(
baidu_module, monkeypatch: pytest.MonkeyPatch
):
factory = baidu_module.BaiduVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
monkeypatch.setattr(baidu_module.dify_config, "BAIDU_VECTOR_DB_ENDPOINT", "https://endpoint")
monkeypatch.setattr(baidu_module.dify_config, "BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS", 1000)
diff --git a/api/providers/vdb/vdb-chroma/tests/unit_tests/test_chroma_vector.py b/api/providers/vdb/vdb-chroma/tests/unit_tests/test_chroma_vector.py
index f18f9a65614..b58002795c0 100644
--- a/api/providers/vdb/vdb-chroma/tests/unit_tests/test_chroma_vector.py
+++ b/api/providers/vdb/vdb-chroma/tests/unit_tests/test_chroma_vector.py
@@ -1,13 +1,14 @@
import importlib
+import json
import sys
import types
from collections import UserDict
-from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_chroma_modules():
@@ -175,10 +176,10 @@ def test_search_by_full_text_returns_empty_list(chroma_module):
def test_factory_init_vector_uses_existing_or_generated_collection(chroma_module, monkeypatch: pytest.MonkeyPatch):
factory = chroma_module.ChromaVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1", index_struct_dict={"vector_store": {"class_prefix": "EXISTING"}}, index_struct=None
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(chroma_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(chroma_module.dify_config, "CHROMA_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-clickzetta/tests/unit_tests/test_clickzetta_vector.py b/api/providers/vdb/vdb-clickzetta/tests/unit_tests/test_clickzetta_vector.py
index 4f8395e475e..be476d9175e 100644
--- a/api/providers/vdb/vdb-clickzetta/tests/unit_tests/test_clickzetta_vector.py
+++ b/api/providers/vdb/vdb-clickzetta/tests/unit_tests/test_clickzetta_vector.py
@@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_clickzetta_module():
@@ -220,7 +221,7 @@ def test_search_by_like_returns_documents_with_default_score(clickzetta_module):
def test_factory_initializes_clickzetta_vector(clickzetta_module, monkeypatch: pytest.MonkeyPatch):
factory = clickzetta_module.ClickzettaVectorFactory()
- dataset = SimpleNamespace(id="dataset-1")
+ dataset = Dataset(id="dataset-1")
monkeypatch.setattr(clickzetta_module.Dataset, "gen_collection_name_by_id", lambda _id: "COLLECTION")
monkeypatch.setattr(clickzetta_module.dify_config, "CLICKZETTA_USERNAME", "username")
diff --git a/api/providers/vdb/vdb-couchbase/tests/unit_tests/test_couchbase_vector.py b/api/providers/vdb/vdb-couchbase/tests/unit_tests/test_couchbase_vector.py
index d474b566d39..0390b75c128 100644
--- a/api/providers/vdb/vdb-couchbase/tests/unit_tests/test_couchbase_vector.py
+++ b/api/providers/vdb/vdb-couchbase/tests/unit_tests/test_couchbase_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_couchbase_modules():
@@ -331,12 +333,10 @@ def test_delete_collection_and_factory(couchbase_module, monkeypatch: pytest.Mon
vector._bucket.collections().drop_collection.assert_called_once_with("_default", "collection_1")
factory = couchbase_module.CouchbaseVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(couchbase_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(
diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
index 91cc2e0fdbb..16a4e363bfb 100644
--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -6,6 +7,8 @@ from unittest.mock import MagicMock, patch
import pytest
+from models.dataset import Dataset
+
def _build_fake_elasticsearch_modules():
elasticsearch = types.ModuleType("elasticsearch")
@@ -89,12 +92,10 @@ def test_create_collection_create_and_exists_paths(elasticsearch_ja_module, monk
def test_ja_factory_uses_existing_or_generated_collection(elasticsearch_ja_module, monkeypatch: pytest.MonkeyPatch):
factory = elasticsearch_ja_module.ElasticSearchJaVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(elasticsearch_ja_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(
diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
index d54c105a0fa..8953928a5c9 100644
--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_elasticsearch_modules():
@@ -333,12 +335,10 @@ def test_create_and_create_collection_paths(elasticsearch_module, monkeypatch: p
def test_elasticsearch_factory_branches(elasticsearch_module, monkeypatch: pytest.MonkeyPatch):
factory = elasticsearch_module.ElasticSearchVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(elasticsearch_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
diff --git a/api/providers/vdb/vdb-hologres/tests/unit_tests/test_hologres_vector.py b/api/providers/vdb/vdb-hologres/tests/unit_tests/test_hologres_vector.py
index 8b197662e30..eff8b4184f2 100644
--- a/api/providers/vdb/vdb-hologres/tests/unit_tests/test_hologres_vector.py
+++ b/api/providers/vdb/vdb-hologres/tests/unit_tests/test_hologres_vector.py
@@ -9,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_hologres_modules():
@@ -333,12 +334,10 @@ def test_create_collection_raises_when_table_never_becomes_ready(hologres_module
def test_hologres_factory_uses_existing_or_generated_collection(hologres_module, monkeypatch: pytest.MonkeyPatch):
factory = hologres_module.HologresVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "existing_collection"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "existing_collection"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(hologres_module.Dataset, "gen_collection_name_by_id", lambda _id: "generated_collection")
monkeypatch.setattr(hologres_module.dify_config, "HOLOGRES_HOST", "127.0.0.1")
diff --git a/api/providers/vdb/vdb-huawei-cloud/tests/unit_tests/test_huawei_cloud_vector.py b/api/providers/vdb/vdb-huawei-cloud/tests/unit_tests/test_huawei_cloud_vector.py
index a1617b6d43c..b057a6cb348 100644
--- a/api/providers/vdb/vdb-huawei-cloud/tests/unit_tests/test_huawei_cloud_vector.py
+++ b/api/providers/vdb/vdb-huawei-cloud/tests/unit_tests/test_huawei_cloud_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_elasticsearch_modules():
@@ -220,12 +222,10 @@ def test_create_and_create_collection_paths(huawei_module, monkeypatch: pytest.M
def test_huawei_factory_branches(huawei_module, monkeypatch: pytest.MonkeyPatch):
factory = huawei_module.HuaweiCloudVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(huawei_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(huawei_module.dify_config, "HUAWEI_CLOUD_HOSTS", "http://huawei-es:9200")
diff --git a/api/providers/vdb/vdb-iris/tests/unit_tests/test_iris_vector.py b/api/providers/vdb/vdb-iris/tests/unit_tests/test_iris_vector.py
index b4ea6ea6c1a..2364626d5eb 100644
--- a/api/providers/vdb/vdb-iris/tests/unit_tests/test_iris_vector.py
+++ b/api/providers/vdb/vdb-iris/tests/unit_tests/test_iris_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from contextlib import contextmanager
@@ -8,6 +9,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_iris_module():
@@ -381,12 +383,10 @@ def test_iris_vector_delete_create_collection_and_factory(iris_module, monkeypat
assert cursor.execute.call_count == 2
factory = iris_module.IrisVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(iris_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(iris_module.dify_config, "IRIS_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-lindorm/tests/unit_tests/test_lindorm_vector.py b/api/providers/vdb/vdb-lindorm/tests/unit_tests/test_lindorm_vector.py
index 182a4f4ea97..a7e4ca76a10 100644
--- a/api/providers/vdb/vdb-lindorm/tests/unit_tests/test_lindorm_vector.py
+++ b/api/providers/vdb/vdb-lindorm/tests/unit_tests/test_lindorm_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_opensearch_modules():
@@ -342,7 +344,7 @@ def test_lindorm_factory_branches(lindorm_module, monkeypatch: pytest.MonkeyPatc
monkeypatch.setattr(lindorm_module.dify_config, "LINDORM_DISTANCE_TYPE", "l2")
monkeypatch.setattr(lindorm_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
- dataset = SimpleNamespace(id="dataset-1", index_struct=None, index_struct_dict={})
+ dataset = Dataset(id="dataset-1")
embeddings = SimpleNamespace(embed_query=lambda _q: [0.1, 0.2, 0.3])
monkeypatch.setattr(lindorm_module.dify_config, "LINDORM_USING_UGC", None)
@@ -351,33 +353,32 @@ def test_lindorm_factory_branches(lindorm_module, monkeypatch: pytest.MonkeyPatc
monkeypatch.setattr(lindorm_module.dify_config, "LINDORM_USING_UGC", False)
- dataset_existing_plain = SimpleNamespace(
- id="dataset-1",
- index_struct="{}",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING"}, "using_ugc": False},
+ dataset_existing_plain = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING"}, "using_ugc": False})
)
with patch.object(lindorm_module, "LindormVectorStore", return_value="vector") as store_cls:
result = factory.init_vector(dataset_existing_plain, attributes=[], embeddings=embeddings)
assert result == "vector"
assert store_cls.call_args.args[0] == "existing"
- dataset_existing_ugc = SimpleNamespace(
+ dataset_existing_ugc = Dataset(
id="dataset-1",
- index_struct="{}",
- index_struct_dict={
- "vector_store": {"class_prefix": "ROUTING"},
- "using_ugc": True,
- "dimension": 1536,
- "index_type": "hnsw",
- "distance_type": "l2",
- },
+ index_struct=json.dumps(
+ {
+ "vector_store": {"class_prefix": "ROUTING"},
+ "using_ugc": True,
+ "dimension": 1536,
+ "index_type": "hnsw",
+ "distance_type": "l2",
+ }
+ ),
)
with patch.object(lindorm_module, "LindormVectorStore", return_value="vector") as store_cls:
factory.init_vector(dataset_existing_ugc, attributes=[], embeddings=embeddings)
assert store_cls.call_args.args[0] == "ugc_index_1536_hnsw_l2"
assert store_cls.call_args.kwargs["routing_value"] == "ROUTING"
- dataset_new = SimpleNamespace(id="dataset-2", index_struct=None, index_struct_dict={})
+ dataset_new = Dataset(id="dataset-2")
monkeypatch.setattr(lindorm_module.dify_config, "LINDORM_USING_UGC", True)
with patch.object(lindorm_module, "LindormVectorStore", return_value="vector") as store_cls:
@@ -386,7 +387,7 @@ def test_lindorm_factory_branches(lindorm_module, monkeypatch: pytest.MonkeyPatc
assert store_cls.call_args.kwargs["routing_value"] == "auto_collection"
assert dataset_new.index_struct is not None
- dataset_new_plain = SimpleNamespace(id="dataset-3", index_struct=None, index_struct_dict={})
+ dataset_new_plain = Dataset(id="dataset-3")
monkeypatch.setattr(lindorm_module.dify_config, "LINDORM_USING_UGC", False)
with patch.object(lindorm_module, "LindormVectorStore", return_value="vector") as store_cls:
factory.init_vector(dataset_new_plain, attributes=[], embeddings=embeddings)
diff --git a/api/providers/vdb/vdb-matrixone/tests/unit_tests/test_matrixone_vector.py b/api/providers/vdb/vdb-matrixone/tests/unit_tests/test_matrixone_vector.py
index c14c9529fdf..40ef49387bd 100644
--- a/api/providers/vdb/vdb-matrixone/tests/unit_tests/test_matrixone_vector.py
+++ b/api/providers/vdb/vdb-matrixone/tests/unit_tests/test_matrixone_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_mo_vector_modules():
@@ -230,12 +232,10 @@ def test_search_by_vector_applies_score_threshold(matrixone_module):
def test_matrixone_factory_uses_existing_or_generated_collection(matrixone_module, monkeypatch: pytest.MonkeyPatch):
factory = matrixone_module.MatrixoneVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(matrixone_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(matrixone_module.dify_config, "MATRIXONE_HOST", "127.0.0.1")
diff --git a/api/providers/vdb/vdb-milvus/tests/unit_tests/test_milvus.py b/api/providers/vdb/vdb-milvus/tests/unit_tests/test_milvus.py
index 028842a7d6d..8e97ea26090 100644
--- a/api/providers/vdb/vdb-milvus/tests/unit_tests/test_milvus.py
+++ b/api/providers/vdb/vdb-milvus/tests/unit_tests/test_milvus.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_pymilvus_modules():
@@ -416,12 +418,10 @@ def test_create_collection_builds_schema_and_indexes(milvus_module, monkeypatch:
def test_factory_initializes_milvus_vector(milvus_module, monkeypatch: pytest.MonkeyPatch):
factory = milvus_module.MilvusVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(milvus_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(milvus_module.dify_config, "MILVUS_URI", "http://localhost:19530")
diff --git a/api/providers/vdb/vdb-myscale/tests/unit_tests/test_myscale_vector.py b/api/providers/vdb/vdb-myscale/tests/unit_tests/test_myscale_vector.py
index e2f4eee0769..28acb640073 100644
--- a/api/providers/vdb/vdb-myscale/tests/unit_tests/test_myscale_vector.py
+++ b/api/providers/vdb/vdb-myscale/tests/unit_tests/test_myscale_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -7,6 +8,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_clickhouse_connect_module():
@@ -92,12 +94,10 @@ def test_delete_by_ids_short_circuits_on_empty_list(myscale_module):
def test_factory_initializes_lower_case_collection_name(myscale_module, monkeypatch: pytest.MonkeyPatch):
factory = myscale_module.MyScaleVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(myscale_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(myscale_module.dify_config, "MYSCALE_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-oceanbase/tests/unit_tests/test_oceanbase_vector.py b/api/providers/vdb/vdb-oceanbase/tests/unit_tests/test_oceanbase_vector.py
index c5ca9d0a873..a6e147cfb35 100644
--- a/api/providers/vdb/vdb-oceanbase/tests/unit_tests/test_oceanbase_vector.py
+++ b/api/providers/vdb/vdb-oceanbase/tests/unit_tests/test_oceanbase_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -9,6 +10,7 @@ from pydantic import ValidationError
from sqlalchemy.exc import SQLAlchemyError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_pyobvector_module():
@@ -519,12 +521,10 @@ def test_delete_success_and_exception(oceanbase_module):
def test_oceanbase_factory_uses_existing_or_generated_collection(oceanbase_module, monkeypatch: pytest.MonkeyPatch):
factory = oceanbase_module.OceanBaseVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(oceanbase_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(oceanbase_module.dify_config, "OCEANBASE_VECTOR_HOST", "127.0.0.1")
diff --git a/api/providers/vdb/vdb-opengauss/tests/unit_tests/test_opengauss.py b/api/providers/vdb/vdb-opengauss/tests/unit_tests/test_opengauss.py
index 57c9b14d9f7..b2cd5d5306e 100644
--- a/api/providers/vdb/vdb-opengauss/tests/unit_tests/test_opengauss.py
+++ b/api/providers/vdb/vdb-opengauss/tests/unit_tests/test_opengauss.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from contextlib import contextmanager
@@ -9,6 +10,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_psycopg2_modules():
@@ -372,12 +374,10 @@ def test_create_collection_cache_and_create_path(opengauss_module, monkeypatch:
def test_opengauss_factory_uses_existing_or_generated_collection(opengauss_module, monkeypatch: pytest.MonkeyPatch):
factory = opengauss_module.OpenGaussFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(opengauss_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(opengauss_module.dify_config, "OPENGAUSS_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-opensearch/tests/unit_tests/test_opensearch_vector.py b/api/providers/vdb/vdb-opensearch/tests/unit_tests/test_opensearch_vector.py
index 80bf20e8206..3622bf77639 100644
--- a/api/providers/vdb/vdb-opensearch/tests/unit_tests/test_opensearch_vector.py
+++ b/api/providers/vdb/vdb-opensearch/tests/unit_tests/test_opensearch_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
# TODO(wylswz): There's a known issue with namespace collision
@@ -333,12 +335,10 @@ def test_create_collection_cache_and_create_path(opensearch_module, monkeypatch:
def test_opensearch_factory_initializes_expected_collection_name(opensearch_module, monkeypatch: pytest.MonkeyPatch):
factory = opensearch_module.OpenSearchVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(opensearch_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(opensearch_module.dify_config, "OPENSEARCH_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-oracle/tests/unit_tests/test_oraclevector.py b/api/providers/vdb/vdb-oracle/tests/unit_tests/test_oraclevector.py
index 46027c7e449..97af40b646e 100644
--- a/api/providers/vdb/vdb-oracle/tests/unit_tests/test_oraclevector.py
+++ b/api/providers/vdb/vdb-oracle/tests/unit_tests/test_oraclevector.py
@@ -1,5 +1,6 @@
import array
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -10,6 +11,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_oracle_modules():
@@ -350,12 +352,10 @@ def test_oracle_factory_init_vector_uses_existing_or_generated_collection(
oracle_module, monkeypatch: pytest.MonkeyPatch
):
factory = oracle_module.OracleVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(oracle_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(oracle_module.dify_config, "ORACLE_USER", "system")
diff --git a/api/providers/vdb/vdb-pgvecto-rs/tests/unit_tests/test_pgvecto_rs.py b/api/providers/vdb/vdb-pgvecto-rs/tests/unit_tests/test_pgvecto_rs.py
index 1841e881397..cfc910aa6b0 100644
--- a/api/providers/vdb/vdb-pgvecto-rs/tests/unit_tests/test_pgvecto_rs.py
+++ b/api/providers/vdb/vdb-pgvecto-rs/tests/unit_tests/test_pgvecto_rs.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -9,6 +10,7 @@ from pydantic import ValidationError
from sqlalchemy.types import UserDefinedType
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_pgvecto_modules():
@@ -316,12 +318,10 @@ def test_text_exists_search_and_full_text(pgvecto_module, monkeypatch: pytest.Mo
def test_factory_uses_existing_or_generated_collection(pgvecto_module, monkeypatch: pytest.MonkeyPatch):
module, _ = pgvecto_module
factory = module.PGVectoRSFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(module.dify_config, "PGVECTO_RS_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-pgvector/tests/unit_tests/test_pgvector.py b/api/providers/vdb/vdb-pgvector/tests/unit_tests/test_pgvector.py
index 38e472df636..b6505285398 100644
--- a/api/providers/vdb/vdb-pgvector/tests/unit_tests/test_pgvector.py
+++ b/api/providers/vdb/vdb-pgvector/tests/unit_tests/test_pgvector.py
@@ -1,3 +1,4 @@
+import json
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -7,6 +8,7 @@ import pytest
from dify_vdb_pgvector.pgvector import PGVector, PGVectorConfig
from core.rag.models.document import Document
+from models.dataset import Dataset
class TestPGVector:
@@ -466,12 +468,10 @@ def test_search_by_full_text_branches_for_bigm_and_standard():
def test_pgvector_factory_initializes_expected_collection_name(monkeypatch: pytest.MonkeyPatch):
factory = pgvector_module.PGVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(pgvector_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(pgvector_module.dify_config, "PGVECTOR_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
index 011c63b8042..3f88219e6fc 100644
--- a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
+++ b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import os
import sys
import types
@@ -8,8 +9,11 @@ from typing import override
from unittest.mock import MagicMock, patch
import pytest
+from sqlalchemy.orm import Session
from core.rag.models.document import Document
+from models.dataset import Dataset, DatasetCollectionBinding
+from models.enums import CollectionBindingType
def _build_fake_qdrant_modules():
@@ -290,15 +294,14 @@ def test_search_and_helper_methods(qdrant_module):
assert doc.page_content == "doc"
-def test_qdrant_factory_paths(qdrant_module, monkeypatch: pytest.MonkeyPatch):
[email protected]("sqlite3_session", [(DatasetCollectionBinding,)], indirect=True)
+def test_qdrant_factory_paths(
+ qdrant_module,
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite3_session: Session,
+):
factory = qdrant_module.QdrantVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-1",
- tenant_id="tenant-1",
- collection_binding_id=None,
- index_struct_dict=None,
- index_struct=None,
- )
+ dataset = Dataset(id="dataset-1", tenant_id="tenant-1", collection_binding_id=None)
monkeypatch.setattr(qdrant_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(qdrant_module, "current_app", SimpleNamespace(config=SimpleNamespace(root_path="/root")))
monkeypatch.setattr(qdrant_module.dify_config, "QDRANT_URL", "http://localhost:6333")
@@ -315,16 +318,23 @@ def test_qdrant_factory_paths(qdrant_module, monkeypatch: pytest.MonkeyPatch):
assert dataset.index_struct is not None
# collection binding lookup path
- dataset.collection_binding_id = "binding-1"
- dataset.index_struct_dict = {"vector_store": {"class_prefix": "existing"}}
- monkeypatch.setattr(qdrant_module, "select", lambda _model: SimpleNamespace(where=lambda *_args: "stmt"))
- qdrant_module.db.session.scalars = MagicMock(
- return_value=SimpleNamespace(one_or_none=lambda: SimpleNamespace(collection_name="BOUND_COLLECTION"))
+ binding = DatasetCollectionBinding(
+ provider_name="provider",
+ model_name="model",
+ type=CollectionBindingType.DATASET,
+ collection_name="BOUND_COLLECTION",
)
+ sqlite3_session.add(binding)
+ sqlite3_session.commit()
+ dataset.collection_binding_id = binding.id
+ dataset.index_struct = json.dumps({"vector_store": {"class_prefix": "existing"}})
+ monkeypatch.setattr(qdrant_module.db, "session", sqlite3_session)
+
with patch.object(qdrant_module, "QdrantVector", return_value="vector") as vector_cls:
factory.init_vector(dataset, attributes=[], embeddings=MagicMock())
assert vector_cls.call_args.kwargs["collection_name"] == "BOUND_COLLECTION"
- qdrant_module.db.session.scalars = MagicMock(return_value=SimpleNamespace(one_or_none=lambda: None))
+ sqlite3_session.delete(binding)
+ sqlite3_session.commit()
with pytest.raises(ValueError, match="Dataset Collection Bindings does not exist"):
factory.init_vector(dataset, attributes=[], embeddings=MagicMock())
diff --git a/api/providers/vdb/vdb-relyt/tests/unit_tests/test_relyt_vector.py b/api/providers/vdb/vdb-relyt/tests/unit_tests/test_relyt_vector.py
index c5f3a9f847a..95e63910b3e 100644
--- a/api/providers/vdb/vdb-relyt/tests/unit_tests/test_relyt_vector.py
+++ b/api/providers/vdb/vdb-relyt/tests/unit_tests/test_relyt_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -9,6 +10,7 @@ from pydantic import ValidationError
from sqlalchemy.types import UserDefinedType
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_relyt_modules():
@@ -297,12 +299,10 @@ def test_delete_drops_table(relyt_module, monkeypatch: pytest.MonkeyPatch):
def test_relyt_factory_existing_and_generated_collection(relyt_module, monkeypatch: pytest.MonkeyPatch):
factory = relyt_module.RelytVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(relyt_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(relyt_module.dify_config, "RELYT_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-tablestore/tests/unit_tests/test_tablestore_vector.py b/api/providers/vdb/vdb-tablestore/tests/unit_tests/test_tablestore_vector.py
index 49d4b160cf9..2acd214472c 100644
--- a/api/providers/vdb/vdb-tablestore/tests/unit_tests/test_tablestore_vector.py
+++ b/api/providers/vdb/vdb-tablestore/tests/unit_tests/test_tablestore_vector.py
@@ -9,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_tablestore_module():
@@ -291,12 +292,10 @@ def test_write_row_and_search_helpers(tablestore_module):
def test_tablestore_factory_uses_existing_or_generated_collection(tablestore_module, monkeypatch: pytest.MonkeyPatch):
factory = tablestore_module.TableStoreVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(tablestore_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(tablestore_module.dify_config, "TABLESTORE_ENDPOINT", "endpoint")
diff --git a/api/providers/vdb/vdb-tencent/tests/unit_tests/test_tencent_vector.py b/api/providers/vdb/vdb-tencent/tests/unit_tests/test_tencent_vector.py
index e1fe227a29a..7ad76e1960f 100644
--- a/api/providers/vdb/vdb-tencent/tests/unit_tests/test_tencent_vector.py
+++ b/api/providers/vdb/vdb-tencent/tests/unit_tests/test_tencent_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -7,6 +8,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_tencent_modules():
@@ -281,12 +283,10 @@ def test_create_add_delete_and_search_behaviour(tencent_module):
def test_tencent_factory_existing_and_generated_collection(tencent_module, monkeypatch: pytest.MonkeyPatch):
factory = tencent_module.TencentVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(tencent_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(tencent_module.dify_config, "TENCENT_VECTOR_DB_URL", "http://vdb.local")
diff --git a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_service.py b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_service.py
index 20a42f6cc34..7c3f828e73d 100644
--- a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_service.py
+++ b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_service.py
@@ -1,9 +1,11 @@
-from types import SimpleNamespace
from unittest.mock import MagicMock, patch
+import dify_vdb_tidb_on_qdrant.tidb_service as tidb_service_module
import pytest
from dify_vdb_tidb_on_qdrant.tidb_service import TidbService
+from sqlalchemy.orm import Session
+from models.dataset import TidbAuthBinding
from models.enums import TidbAuthBindingStatus
@@ -221,18 +223,34 @@ class TestBatchCreateEdgeCases:
)
+def _persist_tidb_auth_binding(sqlite3_session: Session) -> TidbAuthBinding:
+ binding = TidbAuthBinding(
+ tenant_id=None,
+ cluster_id="c-1",
+ cluster_name="cluster-1",
+ active=False,
+ status=TidbAuthBindingStatus.CREATING,
+ account="root",
+ password="password",
+ )
+ sqlite3_session.add(binding)
+ sqlite3_session.commit()
+ return binding
+
+
[email protected]("sqlite3_session", [(TidbAuthBinding,)], indirect=True)
class TestBatchUpdateTidbServerlessClusterStatus:
"""Verify that status updates only expose clusters after qdrant endpoint is ready."""
- @patch("dify_vdb_tidb_on_qdrant.tidb_service.db")
@patch("dify_vdb_tidb_on_qdrant.tidb_service._tidb_http_client")
- def test_sets_active_when_batch_response_contains_endpoint(self, mock_http, mock_db):
- binding = SimpleNamespace(
- cluster_id="c-1",
- status=TidbAuthBindingStatus.CREATING,
- account="root",
- qdrant_endpoint=None,
- )
+ def test_sets_active_when_batch_response_contains_endpoint(
+ self,
+ mock_http,
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite3_session: Session,
+ ):
+ binding = _persist_tidb_auth_binding(sqlite3_session)
+ monkeypatch.setattr(tidb_service_module.db, "session", sqlite3_session)
mock_http.get.return_value = MagicMock(
status_code=200,
json=lambda: {
@@ -249,22 +267,24 @@ class TestBatchUpdateTidbServerlessClusterStatus:
TidbService.batch_update_tidb_serverless_cluster_status([binding], "proj", "url", "iam", "pub", "priv")
- assert binding.account == "pfx.root"
- assert binding.qdrant_endpoint == "https://qdrant-gw.tidbcloud.com"
- assert binding.status == TidbAuthBindingStatus.ACTIVE
- mock_db.session.add.assert_called_once_with(binding)
- mock_db.session.commit.assert_called_once()
+ sqlite3_session.expire_all()
+ persisted = sqlite3_session.get(TidbAuthBinding, binding.id)
+ assert persisted is not None
+ assert persisted.account == "pfx.root"
+ assert persisted.qdrant_endpoint == "https://qdrant-gw.tidbcloud.com"
+ assert persisted.status == TidbAuthBindingStatus.ACTIVE
@patch.object(TidbService, "fetch_qdrant_endpoint", return_value="https://qdrant-gw.tidbcloud.com")
- @patch("dify_vdb_tidb_on_qdrant.tidb_service.db")
@patch("dify_vdb_tidb_on_qdrant.tidb_service._tidb_http_client")
- def test_fetches_endpoint_when_batch_response_omits_it(self, mock_http, mock_db, mock_fetch_endpoint):
- binding = SimpleNamespace(
- cluster_id="c-1",
- status=TidbAuthBindingStatus.CREATING,
- account="root",
- qdrant_endpoint=None,
- )
+ def test_fetches_endpoint_when_batch_response_omits_it(
+ self,
+ mock_http,
+ mock_fetch_endpoint,
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite3_session: Session,
+ ):
+ binding = _persist_tidb_auth_binding(sqlite3_session)
+ monkeypatch.setattr(tidb_service_module.db, "session", sqlite3_session)
mock_http.get.return_value = MagicMock(
status_code=200,
json=lambda: {"clusters": [{"clusterId": "c-1", "state": "ACTIVE", "userPrefix": "pfx", "endpoints": {}}]},
@@ -272,23 +292,25 @@ class TestBatchUpdateTidbServerlessClusterStatus:
TidbService.batch_update_tidb_serverless_cluster_status([binding], "proj", "url", "iam", "pub", "priv")
- assert binding.account == "pfx.root"
- assert binding.qdrant_endpoint == "https://qdrant-gw.tidbcloud.com"
- assert binding.status == TidbAuthBindingStatus.ACTIVE
+ sqlite3_session.expire_all()
+ persisted = sqlite3_session.get(TidbAuthBinding, binding.id)
+ assert persisted is not None
+ assert persisted.account == "pfx.root"
+ assert persisted.qdrant_endpoint == "https://qdrant-gw.tidbcloud.com"
+ assert persisted.status == TidbAuthBindingStatus.ACTIVE
mock_fetch_endpoint.assert_called_once_with("url", "pub", "priv", "c-1")
- mock_db.session.add.assert_called_once_with(binding)
- mock_db.session.commit.assert_called_once()
@patch.object(TidbService, "fetch_qdrant_endpoint", return_value=None)
- @patch("dify_vdb_tidb_on_qdrant.tidb_service.db")
@patch("dify_vdb_tidb_on_qdrant.tidb_service._tidb_http_client")
- def test_keeps_creating_when_endpoint_is_not_ready(self, mock_http, mock_db, mock_fetch_endpoint):
- binding = SimpleNamespace(
- cluster_id="c-1",
- status=TidbAuthBindingStatus.CREATING,
- account="root",
- qdrant_endpoint=None,
- )
+ def test_keeps_creating_when_endpoint_is_not_ready(
+ self,
+ mock_http,
+ mock_fetch_endpoint,
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite3_session: Session,
+ ):
+ binding = _persist_tidb_auth_binding(sqlite3_session)
+ monkeypatch.setattr(tidb_service_module.db, "session", sqlite3_session)
mock_http.get.return_value = MagicMock(
status_code=200,
json=lambda: {"clusters": [{"clusterId": "c-1", "state": "ACTIVE", "userPrefix": "pfx", "endpoints": {}}]},
@@ -296,9 +318,10 @@ class TestBatchUpdateTidbServerlessClusterStatus:
TidbService.batch_update_tidb_serverless_cluster_status([binding], "proj", "url", "iam", "pub", "priv")
- assert binding.account == "pfx.root"
- assert binding.qdrant_endpoint is None
- assert binding.status == TidbAuthBindingStatus.CREATING
+ sqlite3_session.expire_all()
+ persisted = sqlite3_session.get(TidbAuthBinding, binding.id)
+ assert persisted is not None
+ assert persisted.account == "pfx.root"
+ assert persisted.qdrant_endpoint is None
+ assert persisted.status == TidbAuthBindingStatus.CREATING
mock_fetch_endpoint.assert_called_once_with("url", "pub", "priv", "c-1")
- mock_db.session.add.assert_called_once_with(binding)
- mock_db.session.commit.assert_called_once()
diff --git a/api/providers/vdb/vdb-tidb-vector/tests/unit_tests/test_tidb_vector.py b/api/providers/vdb/vdb-tidb-vector/tests/unit_tests/test_tidb_vector.py
index 47d63f0641d..52ec6fc9999 100644
--- a/api/providers/vdb/vdb-tidb-vector/tests/unit_tests/test_tidb_vector.py
+++ b/api/providers/vdb/vdb-tidb-vector/tests/unit_tests/test_tidb_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
@pytest.fixture
@@ -531,12 +533,10 @@ def test_delete_drops_table(tidb_module, monkeypatch: pytest.MonkeyPatch):
def test_tidb_factory_uses_existing_or_generated_collection(tidb_module, monkeypatch: pytest.MonkeyPatch):
factory = tidb_module.TiDBVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(tidb_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(tidb_module.dify_config, "TIDB_VECTOR_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-upstash/tests/unit_tests/test_upstash_vector.py b/api/providers/vdb/vdb-upstash/tests/unit_tests/test_upstash_vector.py
index 55d27ad264d..5412a4bce33 100644
--- a/api/providers/vdb/vdb-upstash/tests/unit_tests/test_upstash_vector.py
+++ b/api/providers/vdb/vdb-upstash/tests/unit_tests/test_upstash_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from types import SimpleNamespace
@@ -8,6 +9,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_upstash_module():
@@ -164,12 +166,10 @@ def test_search_by_vector_filter_threshold_and_delete(upstash_module):
def test_upstash_factory_uses_existing_or_generated_collection(upstash_module, monkeypatch: pytest.MonkeyPatch):
factory = upstash_module.UpstashVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(upstash_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(upstash_module.dify_config, "UPSTASH_VECTOR_URL", "https://upstash.example")
diff --git a/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py b/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
index 32f47c67edb..271dbc415e5 100644
--- a/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
+++ b/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
@@ -1,4 +1,5 @@
import importlib
+import json
import sys
import types
from contextlib import contextmanager
@@ -9,6 +10,7 @@ import pytest
from pydantic import ValidationError
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_psycopg2_modules():
@@ -242,12 +244,10 @@ def test_create_collection_cache_and_dimension_branches(vastbase_module, monkeyp
def test_vastbase_factory_uses_existing_or_generated_collection(vastbase_module, monkeypatch: pytest.MonkeyPatch):
factory = vastbase_module.VastbaseVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(vastbase_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
monkeypatch.setattr(vastbase_module.dify_config, "VASTBASE_HOST", "localhost")
diff --git a/api/providers/vdb/vdb-vikingdb/tests/unit_tests/test_vikingdb_vector.py b/api/providers/vdb/vdb-vikingdb/tests/unit_tests/test_vikingdb_vector.py
index 6559ad97d28..efa02bc4ac3 100644
--- a/api/providers/vdb/vdb-vikingdb/tests/unit_tests/test_vikingdb_vector.py
+++ b/api/providers/vdb/vdb-vikingdb/tests/unit_tests/test_vikingdb_vector.py
@@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch
import pytest
from core.rag.models.document import Document
+from models.dataset import Dataset
def _build_fake_vikingdb_modules():
@@ -255,12 +256,10 @@ def test_delete_drops_index_and_collection_when_present(vikingdb_module):
def test_vikingdb_factory_validates_config_and_builds_vector(vikingdb_module, monkeypatch: pytest.MonkeyPatch):
factory = vikingdb_module.VikingDBVectorFactory()
- dataset_with_index = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "EXISTING_COLLECTION"}},
- index_struct=None,
+ dataset_with_index = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
)
- dataset_without_index = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset_without_index = Dataset(id="dataset-2")
monkeypatch.setattr(vikingdb_module.Dataset, "gen_collection_name_by_id", lambda _id: "AUTO_COLLECTION")
@@ -297,9 +296,7 @@ def test_vikingdb_factory_raises_when_required_config_missing(
vikingdb_module, monkeypatch: pytest.MonkeyPatch, field, message
):
factory = vikingdb_module.VikingDBVectorFactory()
- dataset = SimpleNamespace(
- id="dataset-1", index_struct_dict={"vector_store": {"class_prefix": "existing"}}, index_struct=None
- )
+ dataset = Dataset(id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "existing"}}))
monkeypatch.setattr(vikingdb_module.dify_config, "VIKINGDB_ACCESS_KEY", "ak")
monkeypatch.setattr(vikingdb_module.dify_config, "VIKINGDB_SECRET_KEY", "sk")
diff --git a/api/providers/vdb/vdb-weaviate/tests/unit_tests/test_weaviate_vector.py b/api/providers/vdb/vdb-weaviate/tests/unit_tests/test_weaviate_vector.py
index 84dbddbeb3a..fa1a70e18d8 100644
--- a/api/providers/vdb/vdb-weaviate/tests/unit_tests/test_weaviate_vector.py
+++ b/api/providers/vdb/vdb-weaviate/tests/unit_tests/test_weaviate_vector.py
@@ -18,8 +18,10 @@ from unittest.mock import MagicMock, patch
import pytest
from dify_vdb_weaviate import weaviate_vector as weaviate_vector_module
from dify_vdb_weaviate.weaviate_vector import WeaviateConfig, WeaviateVector
+from sqlalchemy.orm import Session
from core.rag.models.document import Document
+from models.dataset import Dataset
class TestWeaviateVector(unittest.TestCase):
@@ -164,13 +166,13 @@ class TestWeaviateVector(unittest.TestCase):
}
def test_get_collection_name_uses_existing_class_prefix_and_appends_suffix(self):
- dataset = SimpleNamespace(index_struct_dict={"vector_store": {"class_prefix": "ExistingCollection"}}, id="ds-1")
+ dataset = Dataset(id="ds-1", index_struct=json.dumps({"vector_store": {"class_prefix": "ExistingCollection"}}))
wv = WeaviateVector.__new__(WeaviateVector)
assert wv.get_collection_name(dataset) == "ExistingCollection_Node"
def test_get_collection_name_generates_name_from_dataset_id(self):
- dataset = SimpleNamespace(index_struct_dict=None, id="ds-2")
+ dataset = Dataset(id="ds-2")
wv = WeaviateVector.__new__(WeaviateVector)
with patch.object(weaviate_vector_module.Dataset, "gen_collection_name_by_id", return_value="Generated_Node"):
@@ -851,32 +853,35 @@ class TestWeaviateVector(unittest.TestCase):
assert wv._json_serializable("plain") == "plain"
-class TestVectorDefaultAttributes(unittest.TestCase):
[email protected]("sqlite3_session", [(Dataset,)], indirect=True)
+class TestVectorDefaultAttributes:
"""Tests for Vector class default attributes list."""
@patch("core.rag.datasource.vdb.vector_factory.Vector._get_embeddings")
@patch("core.rag.datasource.vdb.vector_factory.Vector._init_vector")
- def test_default_attributes_include_doc_type(self, mock_init_vector, mock_get_embeddings):
+ def test_default_attributes_include_doc_type(
+ self,
+ mock_init_vector,
+ mock_get_embeddings,
+ sqlite3_session: Session,
+ ):
"""Test that Vector class default attributes include doc_type."""
from core.rag.datasource.vdb.vector_factory import Vector
mock_get_embeddings.return_value = MagicMock()
mock_init_vector.return_value = MagicMock()
- mock_dataset = MagicMock()
- mock_dataset.index_struct_dict = None
+ mock_dataset = Dataset(id="dataset-default")
- vector = Vector(dataset=mock_dataset, session=MagicMock())
+ vector = Vector(dataset=mock_dataset, session=sqlite3_session)
assert "doc_type" in vector._attributes, f"doc_type should be in default attributes, got: {vector._attributes}"
class TestWeaviateVectorFactory(unittest.TestCase):
def test_init_vector_uses_existing_dataset_index_struct(self):
- dataset = SimpleNamespace(
- id="dataset-1",
- index_struct_dict={"vector_store": {"class_prefix": "ExistingCollection_Node"}},
- index_struct=None,
+ dataset = Dataset(
+ id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "ExistingCollection_Node"}})
)
attributes = ["doc_id"]
@@ -898,10 +903,10 @@ class TestWeaviateVectorFactory(unittest.TestCase):
assert config.grpc_endpoint == "localhost:50051"
assert config.api_key == "api-key"
assert config.batch_size == 88
- assert dataset.index_struct is None
+ assert json.loads(dataset.index_struct) == {"vector_store": {"class_prefix": "ExistingCollection_Node"}}
def test_init_vector_generates_collection_and_updates_index_struct(self):
- dataset = SimpleNamespace(id="dataset-2", index_struct_dict=None, index_struct=None)
+ dataset = Dataset(id="dataset-2")
attributes = ["doc_id", "doc_type"]
with (