Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): annotate Tablestore, VikingDB and MatrixOne tests (#44704)

Eddy ZHANGfecf1a20ce45
diff --git a/api/providers/vdb/vdb-matrixone/tests/unit_tests/src/dify_vdb_matrixone/test_matrixone_vector.py b/api/providers/vdb/vdb-matrixone/tests/unit_tests/src/dify_vdb_matrixone/test_matrixone_vector.py
index 856b16b44bc..2bcdbb1fc82 100644
--- a/api/providers/vdb/vdb-matrixone/tests/unit_tests/src/dify_vdb_matrixone/test_matrixone_vector.py
+++ b/api/providers/vdb/vdb-matrixone/tests/unit_tests/src/dify_vdb_matrixone/test_matrixone_vector.py
@@ -1,7 +1,8 @@
import importlib
import json
import sys
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
+from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, patch
import pytest
@@ -11,14 +12,17 @@ from core.rag.datasource.vdb.vector_integration_test_support import FakeModule
from core.rag.models.document import Document
from models.dataset import Dataset
+if TYPE_CHECKING:
+ from dify_vdb_matrixone.matrixone_vector import MatrixoneConfig
-def _build_fake_mo_vector_modules():
+
+def _build_fake_mo_vector_modules() -> dict[str, FakeModule]:
mo_vector = FakeModule("mo_vector")
mo_vector.__path__ = []
mo_vector_client = FakeModule("mo_vector.client")
class MoVectorClient:
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
self.kwargs = kwargs
self.create_full_text_index = MagicMock()
self.insert = MagicMock()
@@ -34,7 +38,7 @@ def _build_fake_mo_vector_modules():
@pytest.fixture
-def matrixone_module(monkeypatch: pytest.MonkeyPatch):
+def matrixone_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
for name, module in _build_fake_mo_vector_modules().items():
monkeypatch.setitem(sys.modules, name, module)
@@ -43,7 +47,7 @@ def matrixone_module(monkeypatch: pytest.MonkeyPatch):
return importlib.reload(module)
-def _valid_config(module):
+def _valid_config(module: ModuleType) -> "MatrixoneConfig":
return module.MatrixoneConfig(
host="localhost",
port=6001,
@@ -64,7 +68,7 @@ def _valid_config(module):
("database", "", "config database is required"),
],
)
-def test_matrixone_config_validation(matrixone_module, field, value, message):
+def test_matrixone_config_validation(matrixone_module: ModuleType, field: str, value: str | int, message: str) -> None:
values = _valid_config(matrixone_module).model_dump()
values[field] = value
@@ -72,7 +76,9 @@ def test_matrixone_config_validation(matrixone_module, field, value, message):
matrixone_module.MatrixoneConfig.model_validate(values)
-def test_get_client_creates_full_text_index_when_cache_misses(matrixone_module, monkeypatch: pytest.MonkeyPatch):
+def test_get_client_creates_full_text_index_when_cache_misses(
+ matrixone_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
lock = MagicMock()
lock.__enter__.return_value = None
lock.__exit__.return_value = None
@@ -88,7 +94,9 @@ def test_get_client_creates_full_text_index_when_cache_misses(matrixone_module,
matrixone_module.redis_client.set.assert_called_once()
-def test_get_client_skips_index_creation_when_cache_hits(matrixone_module, monkeypatch: pytest.MonkeyPatch):
+def test_get_client_skips_index_creation_when_cache_hits(
+ matrixone_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
lock = MagicMock()
lock.__enter__.return_value = None
lock.__exit__.return_value = None
@@ -103,7 +111,7 @@ def test_get_client_skips_index_creation_when_cache_hits(matrixone_module, monke
matrixone_module.redis_client.set.assert_not_called()
-def test_ensure_client_initializes_client_for_decorated_methods(matrixone_module):
+def test_ensure_client_initializes_client_for_decorated_methods(matrixone_module: ModuleType) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
vector.client = None
fake_client = MagicMock()
@@ -116,7 +124,7 @@ def test_ensure_client_initializes_client_for_decorated_methods(matrixone_module
vector._get_client.assert_called_once_with(None, False)
-def test_search_by_full_text_parses_metadata_and_applies_threshold(matrixone_module):
+def test_search_by_full_text_parses_metadata_and_applies_threshold(matrixone_module: ModuleType) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
vector.client = MagicMock()
vector.client.full_text_query.return_value = [
@@ -133,7 +141,7 @@ def test_search_by_full_text_parses_metadata_and_applies_threshold(matrixone_mod
assert vector.client.full_text_query.call_args.kwargs["filter"] == {"document_id": {"$in": ["doc-1"]}}
-def test_get_type_and_create_delegate_to_add_texts(matrixone_module):
+def test_get_type_and_create_delegate_to_add_texts(matrixone_module: ModuleType) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
fake_client = MagicMock()
vector._get_client = MagicMock(return_value=fake_client)
@@ -148,7 +156,9 @@ def test_get_type_and_create_delegate_to_add_texts(matrixone_module):
vector.add_texts.assert_called_once_with(docs, [[0.1, 0.2]])
-def test_get_client_handles_full_text_index_creation_error(matrixone_module, monkeypatch: pytest.MonkeyPatch):
+def test_get_client_handles_full_text_index_creation_error(
+ matrixone_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
lock = MagicMock()
lock.__enter__.return_value = None
lock.__exit__.return_value = None
@@ -167,7 +177,7 @@ def test_get_client_handles_full_text_index_creation_error(matrixone_module, mon
matrixone_module.redis_client.set.assert_not_called()
-def test_add_texts_generates_ids_and_inserts(matrixone_module, monkeypatch: pytest.MonkeyPatch):
+def test_add_texts_generates_ids_and_inserts(matrixone_module: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
vector.client = MagicMock()
generated_ids = iter(["generated-id-b", "generated-id-c"])
@@ -191,7 +201,7 @@ def test_add_texts_generates_ids_and_inserts(matrixone_module, monkeypatch: pyte
assert ids_insert == ids
-def test_delete_and_metadata_methods(matrixone_module):
+def test_delete_and_metadata_methods(matrixone_module: ModuleType) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
vector.client = MagicMock()
vector.client.query_by_metadata.return_value = [SimpleNamespace(id="seg-1"), SimpleNamespace(id="seg-2")]
@@ -208,7 +218,7 @@ def test_delete_and_metadata_methods(matrixone_module):
assert vector.client.delete.call_count == 3
-def test_search_by_vector_applies_score_threshold(matrixone_module):
+def test_search_by_vector_applies_score_threshold(matrixone_module: ModuleType) -> None:
vector = matrixone_module.MatrixoneVector("collection_1", _valid_config(matrixone_module))
vector.client = MagicMock()
vector.client.query.return_value = [
@@ -230,7 +240,9 @@ def test_search_by_vector_applies_score_threshold(matrixone_module):
assert vector.client.query.call_args.kwargs["filter"] == {"document_id": {"$in": ["d-1"]}}
-def test_matrixone_factory_uses_existing_or_generated_collection(matrixone_module, monkeypatch: pytest.MonkeyPatch):
+def test_matrixone_factory_uses_existing_or_generated_collection(
+ matrixone_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
factory = matrixone_module.MatrixoneVectorFactory()
dataset_with_index = Dataset(
id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
diff --git a/api/providers/vdb/vdb-tablestore/tests/unit_tests/src/dify_vdb_tablestore/test_tablestore_vector.py b/api/providers/vdb/vdb-tablestore/tests/unit_tests/src/dify_vdb_tablestore/test_tablestore_vector.py
index dfa4d4657da..c1393f47ff1 100644
--- a/api/providers/vdb/vdb-tablestore/tests/unit_tests/src/dify_vdb_tablestore/test_tablestore_vector.py
+++ b/api/providers/vdb/vdb-tablestore/tests/unit_tests/src/dify_vdb_tablestore/test_tablestore_vector.py
@@ -1,7 +1,9 @@
import importlib
import json
import sys
-from types import SimpleNamespace
+from dataclasses import InitVar, dataclass
+from types import ModuleType, SimpleNamespace
+from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, patch
import pytest
@@ -11,30 +13,37 @@ from core.rag.datasource.vdb.vector_integration_test_support import FakeModule
from core.rag.models.document import Document
from models.dataset import Dataset
+if TYPE_CHECKING:
+ from dify_vdb_tablestore.tablestore_vector import TableStoreConfig
-def _build_fake_tablestore_module():
+
+def _build_fake_tablestore_module() -> FakeModule:
tablestore = FakeModule("tablestore")
class _BatchGetRowRequest:
- def __init__(self):
- self.items = []
+ def __init__(self) -> None:
+ self.items: list[_TableInBatchGetRowItem] = []
- def add(self, item):
+ def add(self, item: "_TableInBatchGetRowItem") -> None:
self.items.append(item)
+ @dataclass(eq=False, repr=False)
class _TableInBatchGetRowItem:
- def __init__(self, table_name, rows_to_get, columns_to_get, _unused, _ver):
- self.table_name = table_name
- self.rows_to_get = rows_to_get
- self.columns_to_get = columns_to_get
+ table_name: str
+ rows_to_get: list[list[tuple[str, str]]]
+ columns_to_get: list[str]
+ _unused: InitVar[None]
+ _ver: InitVar[int]
class _Row:
- def __init__(self, primary_key, attribute_columns=None):
+ def __init__(
+ self, primary_key: list[tuple[str, str]], attribute_columns: list[tuple[str, object]] | None = None
+ ) -> None:
self.primary_key = primary_key
self.attribute_columns = attribute_columns or []
class _Client:
- def __init__(self, *_args):
+ def __init__(self, *_args: object) -> None:
self.list_table = MagicMock(return_value=[])
self.create_table = MagicMock()
self.list_search_index = MagicMock(return_value=[])
@@ -78,7 +87,7 @@ def _build_fake_tablestore_module():
@pytest.fixture
-def tablestore_module(monkeypatch: pytest.MonkeyPatch):
+def tablestore_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
fake_module = _build_fake_tablestore_module()
monkeypatch.setitem(sys.modules, "tablestore", fake_module)
@@ -87,7 +96,7 @@ def tablestore_module(monkeypatch: pytest.MonkeyPatch):
return importlib.reload(module)
-def _config(module, **overrides):
+def _config(module: ModuleType, **overrides: Any) -> "TableStoreConfig":
values = {
"access_key_id": "ak",
"access_key_secret": "sk",
@@ -108,14 +117,14 @@ def _config(module, **overrides):
("endpoint", "", "config ENDPOINT is required"),
],
)
-def test_tablestore_config_validation(tablestore_module, field, value, message):
+def test_tablestore_config_validation(tablestore_module: ModuleType, field: str, value: str, message: str) -> None:
values = _config(tablestore_module).model_dump()
values[field] = value
with pytest.raises(ValidationError, match=message):
tablestore_module.TableStoreConfig.model_validate(values)
-def test_init_and_basic_delegation(tablestore_module):
+def test_init_and_basic_delegation(tablestore_module: ModuleType) -> None:
vector = tablestore_module.TableStoreVector("collection_1", _config(tablestore_module))
assert vector.get_type() == tablestore_module.VectorType.TABLESTORE
assert vector._table_name == "collection_1"
@@ -132,7 +141,7 @@ def test_init_and_basic_delegation(tablestore_module):
assert vector._create_collection.call_count == 2
-def test_get_by_ids_text_exists_delete_and_wrappers(tablestore_module):
+def test_get_by_ids_text_exists_delete_and_wrappers(tablestore_module: ModuleType) -> None:
vector = tablestore_module.TableStoreVector("collection_1", _config(tablestore_module))
# get_by_ids
@@ -178,7 +187,9 @@ def test_get_by_ids_text_exists_delete_and_wrappers(tablestore_module):
vector._delete_table_if_exist.assert_called_once()
-def test_create_collection_and_table_index_lifecycle(tablestore_module, monkeypatch: pytest.MonkeyPatch):
+def test_create_collection_and_table_index_lifecycle(
+ tablestore_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
vector = tablestore_module.TableStoreVector("collection_1", _config(tablestore_module))
lock = MagicMock()
lock.__enter__.return_value = None
@@ -201,13 +212,13 @@ def test_create_collection_and_table_index_lifecycle(tablestore_module, monkeypa
vector = tablestore_module.TableStoreVector("collection_2", _config(tablestore_module))
vector._tablestore_client.list_table.return_value = ["collection_2"]
assert vector._create_table_if_not_exist() is None
- vector._tablestore_client.list_table.return_value = []
+ vector._tablestore_client.list_table.return_value = list[str]()
vector._create_table_if_not_exist()
vector._tablestore_client.create_table.assert_called_once()
vector._tablestore_client.list_search_index.return_value = [("collection_2", "collection_2_idx")]
assert vector._create_search_index_if_not_exist(3) is None
- vector._tablestore_client.list_search_index.return_value = []
+ vector._tablestore_client.list_search_index.return_value = list[tuple[str, str]]()
vector._create_search_index_if_not_exist(3)
vector._tablestore_client.create_search_index.assert_called_once()
@@ -220,7 +231,7 @@ def test_create_collection_and_table_index_lifecycle(tablestore_module, monkeypa
vector._tablestore_client.delete_search_index.assert_called_with("collection_2", "collection_2_idx")
-def test_write_row_and_search_helpers(tablestore_module):
+def test_write_row_and_search_helpers(tablestore_module: ModuleType) -> None:
vector = tablestore_module.TableStoreVector("collection_1", _config(tablestore_module))
vector._write_row(
@@ -290,7 +301,9 @@ def test_write_row_and_search_helpers(tablestore_module):
assert "score" not in docs[0].metadata
-def test_tablestore_factory_uses_existing_or_generated_collection(tablestore_module, monkeypatch: pytest.MonkeyPatch):
+def test_tablestore_factory_uses_existing_or_generated_collection(
+ tablestore_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
factory = tablestore_module.TableStoreVectorFactory()
dataset_with_index = Dataset(
id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
diff --git a/api/providers/vdb/vdb-vikingdb/tests/unit_tests/src/dify_vdb_vikingdb/test_vikingdb_vector.py b/api/providers/vdb/vdb-vikingdb/tests/unit_tests/src/dify_vdb_vikingdb/test_vikingdb_vector.py
index ba9a217de7c..bce095b4f33 100644
--- a/api/providers/vdb/vdb-vikingdb/tests/unit_tests/src/dify_vdb_vikingdb/test_vikingdb_vector.py
+++ b/api/providers/vdb/vdb-vikingdb/tests/unit_tests/src/dify_vdb_vikingdb/test_vikingdb_vector.py
@@ -2,7 +2,8 @@ import importlib
import json
import sys
from collections import UserDict
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
+from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, patch
import pytest
@@ -11,14 +12,17 @@ from core.rag.datasource.vdb.vector_integration_test_support import FakeModule
from core.rag.models.document import Document
from models.dataset import Dataset
+if TYPE_CHECKING:
+ from dify_vdb_vikingdb.vikingdb_vector import VikingDBConfig
-def _build_fake_vikingdb_modules():
+
+def _build_fake_vikingdb_modules() -> dict[str, FakeModule]:
volcengine = FakeModule("volcengine")
volcengine.__path__ = []
viking_db = FakeModule("volcengine.viking_db")
class Data(UserDict):
- def __init__(self, payload):
+ def __init__(self, payload: dict[str, object]) -> None:
super().__init__(payload)
self.fields = payload
@@ -37,26 +41,26 @@ def _build_fake_vikingdb_modules():
Vector = "vector"
class Field:
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
self.kwargs = kwargs
class VectorIndexParams:
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
self.kwargs = kwargs
class _Collection:
- def __init__(self):
+ def __init__(self) -> None:
self.upsert_data = MagicMock()
self.fetch_data = MagicMock(return_value=None)
self.delete_data = MagicMock()
class _Index:
- def __init__(self):
+ def __init__(self) -> None:
self.search = MagicMock(return_value=[])
self.search_by_vector = MagicMock(return_value=[])
class VikingDBService:
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
self.kwargs = kwargs
self.create_collection = MagicMock()
self.create_index = MagicMock()
@@ -80,7 +84,7 @@ def _build_fake_vikingdb_modules():
@pytest.fixture
-def vikingdb_module(monkeypatch: pytest.MonkeyPatch):
+def vikingdb_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
for name, module in _build_fake_vikingdb_modules().items():
monkeypatch.setitem(sys.modules, name, module)
@@ -89,7 +93,7 @@ def vikingdb_module(monkeypatch: pytest.MonkeyPatch):
return importlib.reload(module)
-def _config(module):
+def _config(module: ModuleType) -> "VikingDBConfig":
return module.VikingDBConfig(
access_key="ak",
secret_key="sk",
@@ -101,7 +105,7 @@ def _config(module):
)
-def test_init_get_type_and_has_checks(vikingdb_module):
+def test_init_get_type_and_has_checks(vikingdb_module: ModuleType) -> None:
vector = vikingdb_module.VikingDBVector("collection_1", "group-1", _config(vikingdb_module))
assert vector.get_type() == vikingdb_module.VectorType.VIKINGDB
@@ -118,7 +122,9 @@ def test_init_get_type_and_has_checks(vikingdb_module):
assert vector._has_index() is False
-def test_create_collection_cache_and_creation_paths(vikingdb_module, monkeypatch: pytest.MonkeyPatch):
+def test_create_collection_cache_and_creation_paths(
+ vikingdb_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
lock = MagicMock()
lock.__enter__.return_value = None
lock.__exit__.return_value = None
@@ -142,7 +148,7 @@ def test_create_collection_cache_and_creation_paths(vikingdb_module, monkeypatch
vikingdb_module.redis_client.set.assert_called_once()
-def test_create_and_add_texts(vikingdb_module):
+def test_create_and_add_texts(vikingdb_module: ModuleType) -> None:
vector = vikingdb_module.VikingDBVector("collection_1", "group-1", _config(vikingdb_module))
vector._create_collection = MagicMock()
vector.add_texts = MagicMock()
@@ -166,7 +172,7 @@ def test_create_and_add_texts(vikingdb_module):
assert upsert_docs[0][vikingdb_module.vdb_Field.GROUP_KEY] == "group-2"
-def test_text_exists_and_delete_operations(vikingdb_module):
+def test_text_exists_and_delete_operations(vikingdb_module: ModuleType) -> None:
vector = vikingdb_module.VikingDBVector("collection_1", "group-1", _config(vikingdb_module))
vector._client.get_collection.return_value.fetch_data.return_value = SimpleNamespace(fields={"message": "ok"})
@@ -189,10 +195,10 @@ def test_text_exists_and_delete_operations(vikingdb_module):
vector.delete_by_ids.assert_called_once_with(["id-2"])
-def test_get_ids_and_search_helpers(vikingdb_module):
+def test_get_ids_and_search_helpers(vikingdb_module: ModuleType) -> None:
vector = vikingdb_module.VikingDBVector("collection_1", "group-1", _config(vikingdb_module))
- vector._client.get_index.return_value.search.return_value = []
+ vector._client.get_index.return_value.search.return_value = list[object]()
assert vector.get_ids_by_metadata_field("doc_id", "x") == []
vector._client.get_index.return_value.search.return_value = [
@@ -234,7 +240,7 @@ def test_get_ids_and_search_helpers(vikingdb_module):
assert vector.search_by_full_text("query") == []
-def test_delete_drops_index_and_collection_when_present(vikingdb_module):
+def test_delete_drops_index_and_collection_when_present(vikingdb_module: ModuleType) -> None:
vector = vikingdb_module.VikingDBVector("collection_1", "group-1", _config(vikingdb_module))
vector._has_index = MagicMock(return_value=True)
vector._has_collection = MagicMock(return_value=True)
@@ -254,7 +260,9 @@ def test_delete_drops_index_and_collection_when_present(vikingdb_module):
vector._client.drop_collection.assert_not_called()
-def test_vikingdb_factory_validates_config_and_builds_vector(vikingdb_module, monkeypatch: pytest.MonkeyPatch):
+def test_vikingdb_factory_validates_config_and_builds_vector(
+ vikingdb_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
factory = vikingdb_module.VikingDBVectorFactory()
dataset_with_index = Dataset(
id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "EXISTING_COLLECTION"}})
@@ -293,8 +301,8 @@ def test_vikingdb_factory_validates_config_and_builds_vector(vikingdb_module, mo
],
)
def test_vikingdb_factory_raises_when_required_config_missing(
- vikingdb_module, monkeypatch: pytest.MonkeyPatch, field, message
-):
+ vikingdb_module: ModuleType, monkeypatch: pytest.MonkeyPatch, field: str, message: str
+) -> None:
factory = vikingdb_module.VikingDBVectorFactory()
dataset = Dataset(id="dataset-1", index_struct=json.dumps({"vector_store": {"class_prefix": "existing"}}))