Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): improve Vastbase vector store with Graph_Index and BM25 full-text (#42922)

chenbd123f6a1878ceb93
diff --git a/api/providers/vdb/vdb-vastbase/pyproject.toml b/api/providers/vdb/vdb-vastbase/pyproject.toml
index 8fccc3b4234..7e888ee2ff3 100644
--- a/api/providers/vdb/vdb-vastbase/pyproject.toml
+++ b/api/providers/vdb/vdb-vastbase/pyproject.toml
@@ -2,9 +2,7 @@
name = "dify-vdb-vastbase"
version = "0.0.1"
-dependencies = [
- "pyobvector==0.2.25",
-]
+dependencies = []
description = "Dify vector store backend (dify-vdb-vastbase)."
[project.entry-points."dify.vector_backends"]
diff --git a/api/providers/vdb/vdb-vastbase/src/dify_vdb_vastbase/vastbase_vector.py b/api/providers/vdb/vdb-vastbase/src/dify_vdb_vastbase/vastbase_vector.py
index c562a75e086..0e4c36a3923 100644
--- a/api/providers/vdb/vdb-vastbase/src/dify_vdb_vastbase/vastbase_vector.py
+++ b/api/providers/vdb/vdb-vastbase/src/dify_vdb_vastbase/vastbase_vector.py
@@ -1,8 +1,11 @@
+import hashlib
import json
+import logging
import uuid
from contextlib import contextmanager
from typing import Any, override
+import psycopg2.errors
import psycopg2.extras
import psycopg2.pool
from pydantic import BaseModel, model_validator
@@ -16,6 +19,8 @@ from core.rag.models.document import Document
from extensions.ext_redis import redis_client
from models.dataset import Dataset
+logger = logging.getLogger(__name__)
+
class VastbaseVectorConfig(BaseModel):
host: str
@@ -48,6 +53,8 @@ class VastbaseVectorConfig(BaseModel):
return values
+# Vastbase G100 stores embeddings in its native floatvector type. Vector retrieval
+# is only supported on the ASTORE (row) storage engine, which is the table default.
SQL_CREATE_TABLE = """
CREATE TABLE IF NOT EXISTS {table_name} (
id UUID PRIMARY KEY,
@@ -57,17 +64,30 @@ CREATE TABLE IF NOT EXISTS {table_name} (
);
"""
+# Graph_Index is Vastbase's recommended ANN index and supersedes the deprecated
+# native HNSW index. Cosine distance matches Dify's `1 - distance` scoring.
SQL_CREATE_INDEX = """
-CREATE INDEX IF NOT EXISTS embedding_cosine_v1_idx ON {table_name}
-USING hnsw (embedding floatvector_cosine_ops) WITH (m = 16, ef_construction = 64);
+CREATE INDEX IF NOT EXISTS {index_name} ON {table_name}
+USING graph_index (embedding floatvector_cosine_ops) WITH (m = 16, ef_construction = 64);
+"""
+
+# Native BM25 full-text index, queried with the `@~@` operator and bm25_score().
+# It tokenizes CJK text correctly, unlike a plain to_tsvector() scan.
+SQL_CREATE_FULLTEXT_INDEX = """
+CREATE INDEX IF NOT EXISTS {index_name} ON {table_name}
+USING fulltext (text);
"""
+# Graph_Index supports up to 16384 dimensions; other ANN indexes cap at 2000.
+MAX_GRAPH_INDEX_DIMENSION = 16384
+
class VastbaseVector(BaseVector):
def __init__(self, collection_name: str, config: VastbaseVectorConfig):
super().__init__(collection_name)
self.pool = self._create_connection_pool(config)
self.table_name = f"embedding_{collection_name}"
+ self.index_hash = hashlib.md5(self.table_name.encode()).hexdigest()[:8]
@override
def get_type(self) -> str:
@@ -99,7 +119,11 @@ class VastbaseVector(BaseVector):
def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
dimension = len(embeddings[0])
self._create_collection(dimension)
- return self.add_texts(texts, embeddings)
+ ids = self.add_texts(texts, embeddings)
+ # Build indexes after the initial bulk load; Graph_Index and fulltext both
+ # accept incremental inserts afterwards.
+ self._create_index(dimension)
+ return ids
@override
def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
@@ -145,7 +169,11 @@ class VastbaseVector(BaseVector):
if not ids:
return
with self._get_cursor() as cur:
- cur.execute(f"DELETE FROM {self.table_name} WHERE id IN %s", (tuple(ids),))
+ try:
+ cur.execute(f"DELETE FROM {self.table_name} WHERE id IN %s", (tuple(ids),))
+ except psycopg2.errors.UndefinedTable:
+ logger.warning("Table %s not found, skipping delete operation.", self.table_name)
+ return
@override
def delete_by_metadata_field(self, key: str, value: str):
@@ -158,18 +186,25 @@ class VastbaseVector(BaseVector):
Search the nearest neighbors to a vector.
:param query_vector: The input vector to search for similar items.
- :param top_k: The number of nearest neighbors to return, default is 5.
:return: List of Documents that are nearest to the query vector.
"""
top_k = kwargs.get("top_k", 4)
-
if not isinstance(top_k, int) or top_k <= 0:
raise ValueError("top_k must be a positive integer")
+
+ document_ids_filter = kwargs.get("document_ids_filter")
+ where_clause = ""
+ params: list[Any] = [json.dumps(query_vector)]
+ if document_ids_filter:
+ where_clause = " WHERE meta->>'document_id' IN %s"
+ params.append(tuple(document_ids_filter))
+
with self._get_cursor() as cur:
cur.execute(
f"SELECT meta, text, embedding <=> %s AS distance FROM {self.table_name}"
+ f"{where_clause}"
f" ORDER BY distance LIMIT {top_k}",
- (json.dumps(query_vector),),
+ params,
)
docs = []
score_threshold = float(kwargs.get("score_threshold") or 0.0)
@@ -184,27 +219,30 @@ class VastbaseVector(BaseVector):
@override
def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
top_k = kwargs.get("top_k", 5)
-
if not isinstance(top_k, int) or top_k <= 0:
raise ValueError("top_k must be a positive integer")
+
+ document_ids_filter = kwargs.get("document_ids_filter")
+ where_clause = "WHERE text @~@ %s"
+ params: list[Any] = [query]
+ if document_ids_filter:
+ where_clause += " AND meta->>'document_id' IN %s"
+ params.append(tuple(document_ids_filter))
+
with self._get_cursor() as cur:
cur.execute(
- f"""SELECT meta, text, ts_rank(to_tsvector(coalesce(text, '')), plainto_tsquery(%s)) AS score
+ f"""SELECT meta, text, bm25_score() AS score
FROM {self.table_name}
- WHERE to_tsvector(text) @@ plainto_tsquery(%s)
- ORDER BY score DESC
+ {where_clause}
+ ORDER BY score DESC NULLS LAST
LIMIT {top_k}""",
- # f"'{query}'" is required in order to account for whitespace in query
- (f"'{query}'", f"'{query}'"),
+ params,
)
-
docs = []
-
for record in cur:
metadata, text, score = record
metadata["score"] = score
docs.append(Document(page_content=text, metadata=metadata))
-
return docs
@override
@@ -222,11 +260,31 @@ class VastbaseVector(BaseVector):
with self._get_cursor() as cur:
cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension))
- # Vastbase supports vector dimensions in the range [1, 16,000]
- if dimension <= 16000:
- cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name))
redis_client.set(collection_exist_cache_key, 1, ex=3600)
+ def _create_index(self, dimension: int):
+ index_cache_key = f"vector_index_{self._collection_name}"
+ lock_name = f"{index_cache_key}_lock"
+ with redis_client.lock(lock_name, timeout=60):
+ if redis_client.get(index_cache_key):
+ return
+
+ with self._get_cursor() as cur:
+ if dimension <= MAX_GRAPH_INDEX_DIMENSION:
+ cur.execute(
+ SQL_CREATE_INDEX.format(
+ index_name=f"embedding_cosine_idx_{self.index_hash}",
+ table_name=self.table_name,
+ )
+ )
+ cur.execute(
+ SQL_CREATE_FULLTEXT_INDEX.format(
+ index_name=f"embedding_fulltext_idx_{self.index_hash}",
+ table_name=self.table_name,
+ )
+ )
+ redis_client.set(index_cache_key, 1, ex=3600)
+
class VastbaseVectorFactory(AbstractVectorFactory):
@override
diff --git a/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py b/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
index d9c074c10cc..97efad8dad0 100644
--- a/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
+++ b/api/providers/vdb/vdb-vastbase/tests/unit_tests/test_vastbase_vector.py
@@ -1,3 +1,4 @@
+import hashlib
import importlib
import json
import sys
@@ -18,6 +19,7 @@ def _build_fake_psycopg2_modules():
psycopg2.__path__ = []
psycopg2_extras = types.ModuleType("psycopg2.extras")
psycopg2_pool = types.ModuleType("psycopg2.pool")
+ psycopg2_errors = types.ModuleType("psycopg2.errors")
class SimpleConnectionPool:
def __init__[**P](self, *args: P.args, **kwargs: P.kwargs):
@@ -26,15 +28,21 @@ def _build_fake_psycopg2_modules():
self.getconn = MagicMock()
self.putconn = MagicMock()
+ class UndefinedTableError(Exception):
+ pass
+
psycopg2_pool.SimpleConnectionPool = SimpleConnectionPool
psycopg2_extras.execute_values = MagicMock()
+ psycopg2_errors.UndefinedTable = UndefinedTableError
+
psycopg2.pool = psycopg2_pool
psycopg2.extras = psycopg2_extras
-
+ psycopg2.errors = psycopg2_errors
return {
"psycopg2": psycopg2,
"psycopg2.pool": psycopg2_pool,
"psycopg2.extras": psycopg2_extras,
+ "psycopg2.errors": psycopg2_errors,
}
@@ -60,6 +68,14 @@ def _config(module):
)
+def _cursor_ctx(cursor):
+ @contextmanager
+ def _ctx():
+ yield cursor
+
+ return _ctx
+
+
@pytest.mark.parametrize(
("field", "value", "message"),
[
@@ -95,18 +111,26 @@ def test_vastbase_config_rejects_invalid_connection_window(vastbase_module):
)
-def test_init_and_get_cursor_context_manager(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+def test_init_sets_table_name_index_hash_and_type(vastbase_module, monkeypatch: pytest.MonkeyPatch):
pool = MagicMock()
monkeypatch.setattr(vastbase_module.psycopg2.pool, "SimpleConnectionPool", MagicMock(return_value=pool))
+ vector = vastbase_module.VastbaseVector("collection_1", _config(vastbase_module))
+
+ assert vector.table_name == "embedding_collection_1"
+ assert vector.get_type() == "vastbase"
+ assert vector.pool is pool
+ assert vector.index_hash == hashlib.md5(b"embedding_collection_1").hexdigest()[:8]
+
+
+def test_get_cursor_closes_commits_and_returns_connection(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ pool = MagicMock()
conn = MagicMock()
cur = MagicMock()
pool.getconn.return_value = conn
conn.cursor.return_value = cur
-
- vector = vastbase_module.VastbaseVector("collection_1", _config(vastbase_module))
- assert vector.get_type() == "vastbase"
- assert vector.table_name == "embedding_collection_1"
+ vector.pool = pool
with vector._get_cursor() as got_cur:
assert got_cur is cur
@@ -116,130 +140,229 @@ def test_init_and_get_cursor_context_manager(vastbase_module, monkeypatch: pytes
pool.putconn.assert_called_once_with(conn)
-def test_create_and_add_texts(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+def test_create_calls_collection_insert_and_index(vastbase_module):
vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
- vector.table_name = "embedding_collection_1"
vector._create_collection = MagicMock()
+ vector.add_texts = MagicMock(return_value=["seg-1"])
+ vector._create_index = MagicMock()
+ docs = [Document(page_content="text", metadata={"doc_id": "seg-1"})]
- cursor = MagicMock()
+ result = vector.create(docs, [[0.1, 0.2]])
- @contextmanager
- def _cursor_ctx():
- yield cursor
+ vector._create_collection.assert_called_once_with(2)
+ vector.add_texts.assert_called_once_with(docs, [[0.1, 0.2]])
+ vector._create_index.assert_called_once_with(2)
+ assert result == ["seg-1"]
- vector._get_cursor = _cursor_ctx
- monkeypatch.setattr(vastbase_module.uuid, "uuid4", lambda: "generated-uuid")
+
+def test_add_texts_uses_execute_values(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+ pool = MagicMock()
+ monkeypatch.setattr(vastbase_module.psycopg2.pool, "SimpleConnectionPool", MagicMock(return_value=pool))
+ vector = vastbase_module.VastbaseVector("collection_1", _config(vastbase_module))
+ cursor = MagicMock()
+ vastbase_module.psycopg2.extras.execute_values.reset_mock()
+ vector._get_cursor = _cursor_ctx(cursor)
docs = [
- Document(page_content="a", metadata={"doc_id": "doc-a"}),
- Document(page_content="b", metadata={"document_id": "doc-b"}),
- SimpleNamespace(page_content="c", metadata=None),
+ Document(page_content="text-1", metadata={"doc_id": "seg-1", "document_id": "d-1"}),
+ SimpleNamespace(page_content="text-2", metadata=None),
]
+ monkeypatch.setattr(vastbase_module.uuid, "uuid4", lambda: "generated-uuid")
- ids = vector.add_texts(docs, [[0.1], [0.2], [0.3]])
- assert ids == ["doc-a", "generated-uuid"]
- vastbase_module.psycopg2.extras.execute_values.assert_called_once()
+ ids = vector.add_texts(docs, [[0.1], [0.2]])
- vector.add_texts = MagicMock(return_value=["doc-a"])
- result = vector.create(docs, [[0.1], [0.2], [0.3]])
- vector._create_collection.assert_called_once_with(1)
- assert result == ["doc-a"]
+ assert ids == ["seg-1"]
+ vastbase_module.psycopg2.extras.execute_values.assert_called_once()
-def test_text_get_delete_and_metadata_methods(vastbase_module):
+def test_text_exists_and_get_by_ids(vastbase_module):
vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
vector.table_name = "embedding_collection_1"
cursor = MagicMock()
- cursor.fetchone.return_value = ("id-1",)
+ cursor.fetchone.return_value = ("seg-1",)
cursor.__iter__.return_value = iter([({"doc_id": "1"}, "text-1"), ({"doc_id": "2"}, "text-2")])
+ vector._get_cursor = _cursor_ctx(cursor)
- @contextmanager
- def _cursor_ctx():
- yield cursor
-
- vector._get_cursor = _cursor_ctx
-
- assert vector.text_exists("id-1") is True
- docs = vector.get_by_ids(["id-1", "id-2"])
+ assert vector.text_exists("seg-1") is True
+ docs = vector.get_by_ids(["seg-1", "seg-2"])
assert len(docs) == 2
assert docs[0].page_content == "text-1"
- vector.delete_by_ids([])
- vector.delete_by_ids(["id-1"])
+
+def test_delete_and_metadata_field_queries(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ vector.table_name = "embedding_collection_1"
+ cursor = MagicMock()
+ vector._get_cursor = _cursor_ctx(cursor)
+
+ vector.delete_by_ids(["seg-1", "seg-2"])
vector.delete_by_metadata_field("document_id", "doc-1")
vector.delete()
- executed_sql = [call.args[0] for call in cursor.execute.call_args_list]
- assert any("DELETE FROM embedding_collection_1 WHERE id IN %s" in sql for sql in executed_sql)
- assert any("meta->>%s = %s" in sql for sql in executed_sql)
- assert any("DROP TABLE IF EXISTS embedding_collection_1" in sql for sql in executed_sql)
+
+ sql = [call.args[0] for call in cursor.execute.call_args_list]
+ assert any("DELETE FROM embedding_collection_1 WHERE id IN %s" in query for query in sql)
+ assert any("meta->>%s = %s" in query for query in sql)
+ assert any("DROP TABLE IF EXISTS embedding_collection_1" in query for query in sql)
-def test_search_by_vector_and_full_text(vastbase_module):
+def test_delete_by_ids_short_circuits_with_empty_input(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ vector._get_cursor = MagicMock()
+
+ vector.delete_by_ids([])
+
+ vector._get_cursor.assert_not_called()
+
+
+def test_delete_by_ids_swallows_undefined_table(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ vector.table_name = "embedding_collection_1"
+ cursor = MagicMock()
+ cursor.execute.side_effect = vastbase_module.psycopg2.errors.UndefinedTable("missing")
+ vector._get_cursor = _cursor_ctx(cursor)
+
+ # Should not raise even though the table does not exist.
+ vector.delete_by_ids(["seg-1"])
+
+
+def test_search_by_vector_validates_top_k(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ with pytest.raises(ValueError, match="top_k must be a positive integer"):
+ vector.search_by_vector([0.1, 0.2], top_k=0)
+
+
+def test_search_by_vector_applies_threshold_and_metadata_filter(vastbase_module):
vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
vector.table_name = "embedding_collection_1"
cursor = MagicMock()
cursor.__iter__.return_value = iter(
[
({"doc_id": "1"}, "text-1", 0.1),
- ({"doc_id": "2"}, "text-2", 0.8),
+ ({"doc_id": "2"}, "text-2", 0.6),
]
)
+ vector._get_cursor = _cursor_ctx(cursor)
- @contextmanager
- def _cursor_ctx():
- yield cursor
+ docs = vector.search_by_vector([0.1, 0.2], top_k=2, score_threshold=0.5, document_ids_filter=["doc-a", "doc-b"])
- vector._get_cursor = _cursor_ctx
+ assert len(docs) == 1
+ assert docs[0].metadata["score"] == pytest.approx(0.9)
+ sql, params = cursor.execute.call_args.args
+ assert "embedding <=> %s AS distance" in sql
+ assert "WHERE meta->>'document_id' IN %s" in sql
+ assert params[1] == ("doc-a", "doc-b")
- with pytest.raises(ValueError, match="top_k must be a positive integer"):
- vector.search_by_vector([0.1, 0.2], top_k=0)
- docs = vector.search_by_vector([0.1, 0.2], top_k=2, score_threshold=0.5)
+def test_search_by_vector_without_filter_omits_where_clause(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ vector.table_name = "embedding_collection_1"
+ cursor = MagicMock()
+ cursor.__iter__.return_value = iter([({"doc_id": "1"}, "text-1", 0.2)])
+ vector._get_cursor = _cursor_ctx(cursor)
+
+ docs = vector.search_by_vector([0.1, 0.2], top_k=4)
+
assert len(docs) == 1
- assert docs[0].metadata["score"] == pytest.approx(0.9)
+ sql, params = cursor.execute.call_args.args
+ assert "WHERE" not in sql
+ assert params == [json.dumps([0.1, 0.2])]
+
+def test_search_by_full_text_validates_top_k(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
with pytest.raises(ValueError, match="top_k must be a positive integer"):
- vector.search_by_full_text("hello", top_k=0)
+ vector.search_by_full_text("query", top_k=0)
+
+def test_search_by_full_text_uses_bm25_and_metadata_filter(vastbase_module):
+ vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
+ vector.table_name = "embedding_collection_1"
+ cursor = MagicMock()
cursor.__iter__.return_value = iter([({"doc_id": "3"}, "full-text", 0.7)])
- full_docs = vector.search_by_full_text("hello world", top_k=2)
- assert len(full_docs) == 1
- assert full_docs[0].page_content == "full-text"
+ vector._get_cursor = _cursor_ctx(cursor)
+
+ docs = vector.search_by_full_text("hello world", top_k=2, document_ids_filter=["doc-a"])
+
+ assert len(docs) == 1
+ assert docs[0].page_content == "full-text"
+ assert docs[0].metadata["score"] == pytest.approx(0.7)
+ sql, params = cursor.execute.call_args.args
+ assert "bm25_score()" in sql
+ assert "WHERE text @~@ %s" in sql
+ assert "AND meta->>'document_id' IN %s" in sql
+ assert params == ["hello world", ("doc-a",)]
-def test_create_collection_cache_and_dimension_branches(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+def test_create_collection_cache_and_create_path(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+ pool = MagicMock()
+ monkeypatch.setattr(vastbase_module.psycopg2.pool, "SimpleConnectionPool", MagicMock(return_value=pool))
lock = MagicMock()
lock.__enter__.return_value = None
lock.__exit__.return_value = None
monkeypatch.setattr(vastbase_module.redis_client, "lock", MagicMock(return_value=lock))
monkeypatch.setattr(vastbase_module.redis_client, "set", MagicMock())
- vector = vastbase_module.VastbaseVector.__new__(vastbase_module.VastbaseVector)
- vector._collection_name = "collection_1"
- vector.table_name = "embedding_collection_1"
+ vector = vastbase_module.VastbaseVector("collection_1", _config(vastbase_module))
cursor = MagicMock()
-
- @contextmanager
- def _cursor_ctx():
- yield cursor
-
- vector._get_cursor = _cursor_ctx
+ vector._get_cursor = _cursor_ctx(cursor)
monkeypatch.setattr(vastbase_module.redis_client, "get", MagicMock(return_value=1))
- vector._create_collection(3)
+ vector._create_collection(1536)
cursor.execute.assert_not_called()
monkeypatch.setattr(vastbase_module.redis_client, "get", MagicMock(return_value=None))
- vector._create_collection(17000)
- executed_sql = [call.args[0] for call in cursor.execute.call_args_list]
- assert any("CREATE TABLE IF NOT EXISTS embedding_collection_1" in sql for sql in executed_sql)
- assert all("embedding_cosine_v1_idx" not in sql for sql in executed_sql)
-
- cursor.execute.reset_mock()
- vector._create_collection(3)
- executed_sql = [call.args[0] for call in cursor.execute.call_args_list]
- assert any("embedding_cosine_v1_idx" in sql for sql in executed_sql)
- vastbase_module.redis_client.set.assert_called()
+ vector._create_collection(1536)
+ sql = [call.args[0] for call in cursor.execute.call_args_list]
+ assert any("CREATE TABLE IF NOT EXISTS embedding_collection_1" in query for query in sql)
+ assert any("floatvector(1536)" in query for query in sql)
+ vastbase_module.redis_client.set.assert_called_once()
+
+
+def _index_vector(vastbase_module, monkeypatch, *, cache_hit=False):
+ pool = MagicMock()
+ monkeypatch.setattr(vastbase_module.psycopg2.pool, "SimpleConnectionPool", MagicMock(return_value=pool))
+ lock = MagicMock()
+ lock.__enter__.return_value = None
+ lock.__exit__.return_value = None
+ monkeypatch.setattr(vastbase_module.redis_client, "lock", MagicMock(return_value=lock))
+ monkeypatch.setattr(vastbase_module.redis_client, "get", MagicMock(return_value=1 if cache_hit else None))
+ monkeypatch.setattr(vastbase_module.redis_client, "set", MagicMock())
+ vector = vastbase_module.VastbaseVector("collection_1", _config(vastbase_module))
+ cursor = MagicMock()
+ vector._get_cursor = _cursor_ctx(cursor)
+ return vector, cursor
+
+
+def test_create_index_builds_graph_and_fulltext_indexes(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+ vector, cursor = _index_vector(vastbase_module, monkeypatch)
+
+ vector._create_index(1536)
+
+ sql = [call.args[0] for call in cursor.execute.call_args_list]
+ assert any("USING graph_index (embedding floatvector_cosine_ops)" in query for query in sql)
+ assert any("USING fulltext (text)" in query for query in sql)
+ assert not any("USING hnsw" in query for query in sql)
+ vastbase_module.redis_client.set.assert_called_once()
+
+
+def test_create_index_skips_graph_index_for_large_dimension(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+ vector, cursor = _index_vector(vastbase_module, monkeypatch)
+
+ vector._create_index(vastbase_module.MAX_GRAPH_INDEX_DIMENSION + 1)
+
+ sql = [call.args[0] for call in cursor.execute.call_args_list]
+ assert not any("graph_index" in query for query in sql)
+ # Full-text index is dimension independent and still created.
+ assert any("USING fulltext (text)" in query for query in sql)
+
+
+def test_create_index_returns_early_on_cache_hit(vastbase_module, monkeypatch: pytest.MonkeyPatch):
+ vector, cursor = _index_vector(vastbase_module, monkeypatch, cache_hit=True)
+
+ vector._create_index(1536)
+
+ cursor.execute.assert_not_called()
+ vastbase_module.redis_client.set.assert_not_called()
def test_vastbase_factory_uses_existing_or_generated_collection(vastbase_module, monkeypatch: pytest.MonkeyPatch):
diff --git a/api/uv.lock b/api/uv.lock
index 784a8fa42fd..932423f87ea 100644
--- a/api/uv.lock
+++ b/api/uv.lock
@@ -2303,12 +2303,6 @@ requires-dist = [{ name = "upstash-vector", specifier = "==0.8.0" }]
name = "dify-vdb-vastbase"
version = "0.0.1"
source = { editable = "providers/vdb/vdb-vastbase" }
-dependencies = [
- { name = "pyobvector" },
-]
-
-[package.metadata]
-requires-dist = [{ name = "pyobvector", specifier = "==0.2.25" }]
[[package]]
name = "dify-vdb-vikingdb"