Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor(vectordb): remove legacy vectorize path (#3024)

Qin Haojieb931f4d63f4f
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 2c32e9d48..8b271e411 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -48,9 +48,6 @@ from openviking.storage.vectordb.utils.path_safety import (
safe_join_name,
)
from openviking.storage.vectordb.utils.str_to_uint64 import str_to_uint64
-from openviking.storage.vectordb.vectorize.base import BaseVectorizer
-from openviking.storage.vectordb.vectorize.vectorizer import VectorizerAdapter
-from openviking.storage.vectordb.vectorize.vectorizer_factory import VectorizerFactory
from openviking_cli.utils.logger import default_logger as logger
# Use imported constants, no longer defined here
@@ -60,7 +57,7 @@ AUTO_ID_KEY = SpecialFields.AUTO_ID.value
def get_or_create_local_collection(
meta_data: Optional[Dict[str, Any]] = None,
path: str = "",
- vectorizer: Optional[BaseVectorizer] = None,
+ *,
config: Optional[Dict[str, Any]] = None,
):
"""Create or retrieve a local Collection.
@@ -68,7 +65,6 @@ def get_or_create_local_collection(
Args:
meta_data: Collection metadata configuration
path: Persistence path. If empty, creates an in-memory collection
- vectorizer: Vectorizer for embedding generation
config: Configuration parameters, optional settings include:
- "ttl_cleanup_seconds": Interval (in seconds) for TTL expiration data cleanup
- "index_maintenance_seconds": Interval (in seconds) for index maintenance tasks
@@ -102,30 +98,18 @@ def get_or_create_local_collection(
collection: ICollection
if not path:
meta = create_collection_meta(path, meta_data)
- vectorizer = (
- VectorizerFactory.create(meta.vectorize)
- if meta.vectorize and vectorizer is None
- else vectorizer
- )
store_mgr = create_store_manager("local")
- collection = VolatileCollection(
- meta=meta, store=store_mgr, vectorizer=vectorizer, config=config
- )
+ collection = VolatileCollection(meta=meta, store=store_mgr, config=config)
return Collection(collection)
else:
collection_dir = str(resolve_storage_path(path))
os.makedirs(collection_dir, exist_ok=True)
meta_path = str(safe_join(collection_dir, "collection_meta.json"))
meta = create_collection_meta(meta_path, meta_data)
- vectorizer = (
- VectorizerFactory.create(meta.vectorize)
- if meta.vectorize and vectorizer is None
- else vectorizer
- )
storage_path = str(safe_join(collection_dir, STORAGE_DIR_NAME))
store_mgr = create_store_manager("local", storage_path)
collection = PersistCollection(
- path=collection_dir, meta=meta, store=store_mgr, vectorizer=vectorizer, config=config
+ path=collection_dir, meta=meta, store=store_mgr, config=config
)
return Collection(collection)
@@ -135,7 +119,6 @@ class LocalCollection(ICollection):
self,
meta: CollectionMeta,
store_mgr: StoreManager,
- vectorizer: Optional[BaseVectorizer] = None,
config: Optional[Dict[str, Any]] = None,
):
self.indexes = ThreadSafeDictManager[IIndex]()
@@ -156,10 +139,6 @@ class LocalCollection(ICollection):
self.data_processor = DataProcessor(
self.meta.fields_dict, collection_name=self.meta.collection_name
)
- self.vectorizer_adapter = None
- if meta.vectorize and vectorizer:
- self.vectorizer_adapter = VectorizerAdapter(vectorizer, meta.vectorize)
- self.meta.vector_dim = self.vectorizer_adapter.get_dim()
self.ttl_cleanup_job_id: Optional[str] = None
self.index_manage_job_id: Optional[str] = None
@@ -405,7 +384,7 @@ class LocalCollection(ICollection):
filters: Optional[Dict[str, Any]] = None,
output_fields: Optional[List[str]] = None,
) -> SearchResult:
- """Search using multimodal data by generating vectors and calling search_by_vector.
+ """Local backend does not generate vectors from multimodal input.
Args:
index_name: Name of the index to search
@@ -420,18 +399,9 @@ class LocalCollection(ICollection):
Returns:
SearchResult: Search results
"""
- if not self.vectorizer_adapter:
- raise ValueError("vectorizer is not initialized")
-
- # Currently mainly supports text vectorization
- if not text and not image and not video:
- raise ValueError("At least one of text, image, or video must be provided")
-
- dense_vector, sparse_vector = self.vectorizer_adapter.vectorize_one(
- text=text, image=image, video=video
- )
- return self.search_by_vector(
- index_name, dense_vector, limit, offset, filters, sparse_vector, output_fields
+ raise NotImplementedError(
+ "LocalCollection.search_by_multimodal no longer performs automatic vectorization; "
+ "call search_by_vector with caller-provided vectors"
)
def search_by_random(
@@ -457,7 +427,7 @@ class LocalCollection(ICollection):
filters: Optional[Dict[str, Any]] = None,
output_fields: Optional[List[str]] = None,
) -> SearchResult:
- """Search by keywords by generating vectors and calling search_by_vector.
+ """Local backend does not generate vectors from keywords.
Args:
index_name: Name of the index to search
@@ -471,26 +441,9 @@ class LocalCollection(ICollection):
Returns:
SearchResult: Search results
"""
- if not self.vectorizer_adapter:
- raise ValueError("vectorizer is not initialized")
-
- if not keywords and not query:
- raise ValueError("At least one of keywords or query must be provided")
-
- # Construct query text
- if query:
- query_text = query
- elif keywords:
- # Join keyword list into a string
- query_text = " ".join(keywords)
- else:
- raise ValueError("No valid query input provided")
-
- # Call vectorization interface to generate vectors
- dense_vector, sparse_vector = self.vectorizer_adapter.vectorize_one(text=query_text)
-
- return self.search_by_vector(
- index_name, dense_vector, limit, offset, filters, sparse_vector, output_fields
+ raise NotImplementedError(
+ "LocalCollection.search_by_keywords no longer performs automatic vectorization; "
+ "call search_by_vector with caller-provided vectors"
)
def search_by_scalar(
@@ -588,12 +541,6 @@ class LocalCollection(ICollection):
def _write_data_list(self, data_list: List[Dict[str, Any]], ttl=0):
result = UpsertDataResult()
- dense_emb, sparse_emb = (
- self.vectorizer_adapter.vectorize_raw_data(data_list)
- if self.vectorizer_adapter
- else ([], [])
- )
-
cands_list = [CandidateData() for _ in range(len(data_list))]
pk = self.meta.primary_key
vk = self.meta.vector_key
@@ -608,19 +555,12 @@ class LocalCollection(ICollection):
data[AUTO_ID_KEY] = label
cands_list[i].label = label
- if self.vectorizer_adapter:
- if dense_emb:
- cands_list[i].vector = dense_emb[i]
- if sparse_emb:
- cands_list[i].sparse_raw_terms = list(sparse_emb[i].keys())
- cands_list[i].sparse_values = list(sparse_emb[i].values())
- else:
- cands_list[i].vector = data.pop(vk, None)
- if svk:
- sparse_dict = data.pop(svk, None)
- if sparse_dict and isinstance(sparse_dict, dict):
- cands_list[i].sparse_raw_terms = list(sparse_dict.keys())
- cands_list[i].sparse_values = list(sparse_dict.values())
+ cands_list[i].vector = data.pop(vk, None)
+ if svk:
+ sparse_dict = data.pop(svk, None)
+ if sparse_dict and isinstance(sparse_dict, dict):
+ cands_list[i].sparse_raw_terms = list(sparse_dict.keys())
+ cands_list[i].sparse_values = list(sparse_dict.values())
cands_list[i].fields = safe_json_dumps(data, ensure_ascii=False)
cands_list[i].expire_ns_ts = time.time_ns() + ttl * 1000000000 if ttl > 0 else 0
@@ -634,9 +574,8 @@ class LocalCollection(ICollection):
self.indexes.iterate(upsert_to_index)
- if not self.vectorizer_adapter:
- for i, data in enumerate(data_list):
- data[vk] = list(cands_list[i].vector) if cands_list[i].vector else []
+ for i, data in enumerate(data_list):
+ data[vk] = list(cands_list[i].vector) if cands_list[i].vector else []
if pk != AUTO_ID_KEY:
primary_keys = [data.get(pk) for data in data_list]
@@ -665,12 +604,11 @@ class LocalCollection(ICollection):
raw_data_list.append(None)
continue
raw_data = json.loads(cand_data.fields)
- if not self.vectorizer_adapter:
- raw_data[vk] = list(cand_data.vector)
- if svk and cand_data.sparse_raw_terms and cand_data.sparse_values:
- raw_data[svk] = dict(
- zip(cand_data.sparse_raw_terms, cand_data.sparse_values, strict=False)
- )
+ raw_data[vk] = list(cand_data.vector)
+ if svk and cand_data.sparse_raw_terms and cand_data.sparse_values:
+ raw_data[svk] = dict(
+ zip(cand_data.sparse_raw_terms, cand_data.sparse_values, strict=False)
+ )
raw_data = validation.fix_fields_data(raw_data, self.meta.fields_dict)
raw_data_list.append(raw_data)
@@ -956,10 +894,9 @@ class VolatileCollection(LocalCollection):
self,
meta: CollectionMeta,
store: StoreManager,
- vectorizer: Optional[BaseVectorizer] = None,
config: Optional[Dict[str, Any]] = None,
):
- super().__init__(meta, store, vectorizer, config)
+ super().__init__(meta, store, config=config)
LocalCollection._register_scheduler_job(self)
def _new_index(
@@ -987,14 +924,13 @@ class PersistCollection(LocalCollection):
path: str,
meta: CollectionMeta,
store: StoreManager,
- vectorizer: Optional[BaseVectorizer] = None,
config: Optional[Dict[str, Any]] = None,
):
self.collection_dir = str(resolve_storage_path(path))
os.makedirs(self.collection_dir, exist_ok=True)
self.index_dir = str(safe_join(self.collection_dir, "index"))
os.makedirs(self.index_dir, exist_ok=True)
- super().__init__(meta, store, vectorizer, config)
+ super().__init__(meta, store, config=config)
self._recover()
LocalCollection._register_scheduler_job(self) # TTL expiration data cleanup
diff --git a/openviking/storage/vectordb/meta/collection_meta.py b/openviking/storage/vectordb/meta/collection_meta.py
index 191127161..3809d6bb0 100644
--- a/openviking/storage/vectordb/meta/collection_meta.py
+++ b/openviking/storage/vectordb/meta/collection_meta.py
@@ -61,6 +61,7 @@ class CollectionMeta:
Dict[str, Any]: The internal metadata structure.
"""
inner_meta = copy.deepcopy(user_meta)
+ inner_meta.pop("Vectorize", None)
fields = inner_meta.get("Fields", [])
has_pk = next(
(True for item in fields if item.get("IsPrimaryKey", False)),
@@ -129,6 +130,7 @@ class CollectionMeta:
user_meta.pop("PrimaryKey", None)
user_meta.pop("VectorKey", None)
user_meta.pop("SparseVectorKey", None)
+ user_meta.pop("Vectorize", None)
for item in user_meta.get("Fields", []):
item.pop("FieldID", None)
return user_meta
@@ -188,11 +190,6 @@ class CollectionMeta:
"""Get the dictionary of fields definitions."""
return self.inner_meta.get("FieldsDict", {})
- @property
- def vectorize(self) -> Dict[str, Any]:
- """Get the vectorization configuration."""
- return self.inner_meta.get("Vectorize", {})
-
@property
def vector_key(self) -> str:
"""Get the vector field name."""
@@ -203,11 +200,6 @@ class CollectionMeta:
"""Get the sparse vector field name."""
return self.inner_meta.get("SparseVectorKey", "")
- @property
- def has_sparse(self) -> bool:
- """Check if sparse vector is enabled."""
- return "Sparse" in self.inner_meta.get("Vectorize", {})
-
@property
def vector_dim(self) -> int:
"""Get or set the vector dimension."""
diff --git a/openviking/storage/vectordb/service/api_fastapi.py b/openviking/storage/vectordb/service/api_fastapi.py
index c4c16deb6..ff2877d49 100644
--- a/openviking/storage/vectordb/service/api_fastapi.py
+++ b/openviking/storage/vectordb/service/api_fastapi.py
@@ -134,7 +134,6 @@ async def create_collection(request: CollectionCreateRequest, req: Request):
project_name = request.ProjectName or "default"
description = request.Description or ""
fields = data_utils.convert_dict(request.Fields)
- vectorize = data_utils.convert_dict(request.Vectorize)
project = get_project(project_name)
@@ -146,7 +145,6 @@ async def create_collection(request: CollectionCreateRequest, req: Request):
"CollectionName": collection_name,
"Description": description,
"Fields": fields,
- "Vectorize": vectorize,
}
logger.info(f"Creating collection: {collection_name} in project: {project_name}")
diff --git a/openviking/storage/vectordb/service/app_models.py b/openviking/storage/vectordb/service/app_models.py
index 81a622c60..2cde62eb6 100644
--- a/openviking/storage/vectordb/service/app_models.py
+++ b/openviking/storage/vectordb/service/app_models.py
@@ -12,7 +12,6 @@ class CollectionCreateRequest(BaseModel):
ProjectName: Optional[str] = Field("default", description="Project name")
Description: Optional[str] = Field("", description="Collection description")
Fields: Optional[Any] = Field(None, description="Field definitions")
- Vectorize: Optional[Any] = Field(None, description="Vectorize configuration")
class CollectionUpdateRequest(BaseModel):
diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.py
index 90864eac7..33175fe38 100644
--- a/openviking/storage/vectordb/utils/validation.py
+++ b/openviking/storage/vectordb/utils/validation.py
@@ -60,54 +60,6 @@ class FieldTypeEnum(str, Enum):
GEO_POINT = "geo_point"
-class DenseVectorize(BaseModel):
- model_config = ConfigDict()
-
- ModelName: str
- ModelVersion: Optional[str] = None
- TextField: Optional[str] = None
- ImageField: Optional[str] = None
- VideoField: Optional[str] = None
- Dim: Optional[int] = None
- Dimension: Optional[int] = None
-
- @field_validator("TextField", "ImageField", "VideoField")
- @classmethod
- def check_fields(cls, v):
- # We enforce presence logic in model_validator if needed,
- # but the original code had strict checks inside validate_dense_vectorize
- # Original: "TextField" is required
- return v
-
- @model_validator(mode="after")
- def check_required(self):
- if self.TextField is None and self.ImageField is None and self.VideoField is None:
- # Original logic: if "text_field" not in vectorize["dense"]
- # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")
- # Actually, old code: if "text_field" not in vectorize["dense"]: return False
- # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.
- # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK
- # Let's assume PascalCase as per ALLOWED dictionary keys.
- if not self.TextField:
- raise ValueError("vectorize dense must contain TextField")
- return self
-
-
-class SparseVectorize(BaseModel):
- model_config = ConfigDict()
-
- ModelName: str
- ModelVersion: Optional[str] = None
- TextField: Optional[str] = None
-
-
-class VectorizeConfig(BaseModel):
- model_config = ConfigDict()
-
- Dense: Optional[DenseVectorize] = None
- Sparse: Optional[SparseVectorize] = None
-
-
class CollectionField(BaseModel):
model_config = ConfigDict()
@@ -151,7 +103,6 @@ class CollectionMetaConfig(BaseModel):
Fields: List[CollectionField]
ProjectName: Optional[str] = None
Description: Optional[str] = Field(None, max_length=65535)
- Vectorize: Optional[VectorizeConfig] = None
FullText: Optional[List[dict]] = (
None # e.g. [{"Field": "content", "Analyzer": {"Tokenizer": "standard"}}]
)
@@ -272,7 +223,6 @@ class CollectionMetaUpdateConfig(BaseModel):
Fields: Optional[List[CollectionField]] = None
ProjectName: Optional[str] = None
Description: Optional[str] = Field(None, max_length=65535)
- Vectorize: Optional[VectorizeConfig] = None
@field_validator("CollectionName", "ProjectName")
@classmethod
diff --git a/openviking/storage/vectordb/vectorize/__init__.py b/openviking/storage/vectordb/vectorize/__init__.py
deleted file mode 100644
index 236633478..000000000
--- a/openviking/storage/vectordb/vectorize/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
diff --git a/openviking/storage/vectordb/vectorize/base.py b/openviking/storage/vectordb/vectorize/base.py
deleted file mode 100644
index 88ad17641..000000000
--- a/openviking/storage/vectordb/vectorize/base.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""
-Base interface definition for vectorization module
-"""
-
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
-
-
-class VectorizeResult:
- """Vectorization result"""
-
- def __init__(
- self,
- dense_vectors: Optional[List[List[float]]] = None,
- sparse_vectors: Optional[List[Dict[str, float]]] = None,
- request_id: str = "",
- token_usage: Optional[Dict[str, Any]] = None,
- ):
- self.dense_vectors = dense_vectors or []
- self.sparse_vectors = sparse_vectors or []
- self.request_id = request_id
- self.token_usage = token_usage
-
- def __repr__(self):
- return (
- f"VectorizeResult(dense={len(self.dense_vectors)}, sparse={len(self.sparse_vectors)}, "
- f"request_id='{self.request_id}')"
- )
-
-
-class BaseVectorizer(ABC):
- """Base vectorizer class"""
-
- def __init__(self, config: Dict[str, Any]):
- """
- Initialize vectorizer
-
- Args:
- config: Configuration dictionary containing model-related settings
- """
- self.config = config
- self.model_name = config.get("ModelName", "")
- pass
-
- @abstractmethod
- def vectorize_query(self, texts: List[str]) -> VectorizeResult:
- """
- Vectorize query texts
-
- Args:
- texts: List of texts to vectorize
-
- Returns:
- VectorizeResult: Vectorization results
- """
- pass
-
- @abstractmethod
- def vectorize_document(
- self,
- data: List[Any],
- dense_model: Dict[str, Any],
- sparse_model: Optional[Dict[str, Any]] = None,
- ) -> VectorizeResult:
- """
- Vectorize documents
-
- Args:
- data: List of data items to vectorize
- dense_model: Configuration for dense model
- sparse_model: Configuration for sparse model (optional)
-
- Returns:
- VectorizeResult: Vectorization results
- """
- pass
-
- def close(self):
- """Close resources"""
- pass
diff --git a/openviking/storage/vectordb/vectorize/vectorizer.py b/openviking/storage/vectordb/vectorize/vectorizer.py
deleted file mode 100644
index be3e2d043..000000000
--- a/openviking/storage/vectordb/vectorize/vectorizer.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-from typing import Any, Dict, List, Optional, Tuple, TypedDict
-
-
-class DenseMeta(TypedDict, total=False):
- ModelName: str
- Version: str
- Dim: int
- TextField: str
- ImageField: str
- VideoField: str
-
-
-class SparseMeta(TypedDict, total=False):
- ModelName: str
- Version: str
-
-
-class VectorizeMeta(TypedDict, total=False):
- Dense: DenseMeta
- Sparse: SparseMeta
-
-
-class VectorizerAdapter:
- """Adapter for vectorizer to handle data vectorization.
-
- Adapts the base vectorizer to work with specific collection configuration,
- managing field mapping and model parameters.
- """
-
- def __init__(self, vectorizer: Any, vectorize_meta: VectorizeMeta):
- """Initialize the VectorizerAdapter.
-
- Args:
- vectorizer: The underlying vectorizer instance.
- vectorize_meta (VectorizeMeta): Configuration for vectorization,
- including model names, versions, and field mappings.
- """
- dense_meta = vectorize_meta.get("Dense", {})
- self.text_field = dense_meta.get("TextField", "")
- self.image_field = dense_meta.get("ImageField", "")
- self.video_field = dense_meta.get("VideoField", "")
- self.vectorizer = vectorizer
- sparse_meta = vectorize_meta.get("Sparse", {})
- self.dense_model = {
- "name": dense_meta.get("ModelName", ""),
- "version": dense_meta.get("Version", "default"),
- }
- if "Dim" in dense_meta:
- self.dense_model["dim"] = int(dense_meta["Dim"])
- self.sparse_model = (
- {
- "name": sparse_meta.get("ModelName", ""),
- "version": sparse_meta.get("Version", "default"),
- }
- if sparse_meta
- else {}
- )
- self.dim = self.vectorizer.get_dense_vector_dim(self.dense_model, self.sparse_model)
-
- def get_dim(self) -> int:
- """Get the dimension of the dense vector.
-
- Returns:
- int: The dimension of the dense vector.
- """
- return self.dim
-
- def vectorize_raw_data(
- self, raw_data_list: List[Dict[str, Any]]
- ) -> Tuple[List[List[float]], List[Dict[str, float]]]:
- """Vectorize a list of raw data items.
-
- Args:
- raw_data_list (List[Dict[str, Any]]): List of data dictionaries to vectorize.
-
- Returns:
- Tuple[List[List[float]], List[Dict[str, float]]]: A tuple containing:
- - List of dense vectors.
- - List of sparse vectors (dictionaries of term-weight pairs).
- """
- data_list = []
- for raw_data in raw_data_list:
- data = {}
- if self.text_field in raw_data:
- data["text"] = raw_data[self.text_field]
- if self.image_field in raw_data:
- data["image"] = raw_data[self.image_field]
- if self.video_field in raw_data:
- data["video"] = raw_data[self.video_field]
- data_list.append(data)
- result = self.vectorizer.vectorize_document(data_list, self.dense_model, self.sparse_model)
- return result.dense_vectors, result.sparse_vectors
-
- def vectorize_one(
- self, text: Optional[str] = None, image: Optional[Any] = None, video: Optional[Any] = None
- ) -> Tuple[Optional[List[float]], Optional[Dict[str, float]]]:
- """Vectorize a single item.
-
- Args:
- text (Optional[str]): Text content to vectorize.
- image (Optional[Any]): Image content to vectorize.
- video (Optional[Any]): Video content to vectorize.
-
- Returns:
- Tuple[Optional[List[float]], Optional[Dict[str, float]]]: A tuple containing:
- - Dense vector (or None if not generated).
- - Sparse vector (or None if not generated).
- """
- data = {}
- if text:
- data["text"] = text
- if image:
- data["image"] = image
- if video:
- data["video"] = video
- result = self.vectorizer.vectorize_document([data], self.dense_model, self.sparse_model)
- return result.dense_vectors[0] if result.dense_vectors else None, (
- result.sparse_vectors[0] if result.sparse_vectors else None
- )
diff --git a/openviking/storage/vectordb/vectorize/vectorizer_factory.py b/openviking/storage/vectordb/vectorize/vectorizer_factory.py
deleted file mode 100644
index 4d3d4b639..000000000
--- a/openviking/storage/vectordb/vectorize/vectorizer_factory.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-from enum import Enum
-from typing import Any, Dict
-
-from openviking.storage.vectordb.vectorize.base import BaseVectorizer
-from openviking.storage.vectordb.vectorize.volcengine_vectorizer import VolcengineVectorizer
-
-
-class ModelType(Enum):
- """Model type enumeration."""
-
- LOCAL = "local" # Local model
- HTTP = "http" # HTTP remote model
- VOLCENGINE = "Volcengine" # Volcengine remote model
-
-
-class VectorizerFactory:
- """Vectorizer factory."""
-
- _registry: Dict[str, type] = {}
-
- @classmethod
- def register(cls, model_type: ModelType, vectorizer_class: type):
- """Register vectorizer class."""
- cls._registry[model_type.value.lower()] = vectorizer_class
- # print(f"Register vectorizer {vectorizer_class.__name__} for model type {model_type.value.lower()}")
-
- @classmethod
- def create(
- cls, config: Dict[str, Any], model_type: ModelType = ModelType.VOLCENGINE
- ) -> BaseVectorizer:
- """
- Create vectorizer instance.
-
- Args:
- model_type: Model type (local/http/grpc)
- config: Configuration dictionary
-
- Returns:
- BaseVectorizer instance
- """
- vectorizer_class = cls._registry.get(model_type.value.lower())
- if not vectorizer_class:
- print(
- f"Unknown model type: {model_type.value.lower()}. Available: {list(cls._registry.keys())}"
- )
- raise ValueError(
- f"Unknown model type: {model_type}. Available: {list(cls._registry.keys())}"
- )
-
- return vectorizer_class(config)
-
-
-VectorizerFactory.register(ModelType.VOLCENGINE, VolcengineVectorizer)
diff --git a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py
deleted file mode 100644
index 4d62b5c92..000000000
--- a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py
+++ /dev/null
@@ -1,268 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-import json
-import os
-import time
-from typing import Any, Dict, List, Optional
-
-import requests
-from volcengine.auth.SignerV4 import SignerV4
-from volcengine.base.Request import Request
-from volcengine.Credentials import Credentials
-
-from openviking.storage.vectordb.vectorize.base import BaseVectorizer, VectorizeResult
-
-
-class ClientForDataApi:
- def __init__(self, ak, sk, host, region):
- self.ak = ak
- self.sk = sk
- self.host = host
- self.region = region
-
- def prepare_request(self, method, path, params=None, data=None):
- r = Request()
- r.set_shema("https")
- r.set_method(method)
- r.set_connection_timeout(10)
- r.set_socket_timeout(10)
- mheaders = {
- "Accept": "application/json",
- "Content-Type": "application/json",
- "Host": self.host,
- }
- r.set_headers(mheaders)
- if params:
- r.set_query(params)
- r.set_host(self.host)
- r.set_path(path)
- if data is not None:
- r.set_body(json.dumps(data))
- credentials = Credentials(self.ak, self.sk, "vikingdb", self.region)
- SignerV4.sign(r, credentials)
- return r
-
- def do_req(self, req_method, req_path, req_params, req_body):
- req = self.prepare_request(
- method=req_method, path=req_path, params=req_params, data=req_body
- )
- return requests.request(
- method=req.method,
- url=f"https://{self.host}{req.path}",
- headers=req.headers,
- data=req.body,
- timeout=10000,
- )
-
-
-class VolcengineVectorizer(BaseVectorizer):
- def __init__(self, config: Dict[str, Any]):
- """
- Initialize Volcengine vectorizer
-
- Args:
- config: Configuration dictionary
- - AK: Access Key (required)
- - SK: Secret Key (required)
- - Host: API domain (required)
- - APIPath: API path (default /api/vikingdb/embedding)
- - DenseModelName: Dense model name
- - DenseModelVersion: Dense model version
- - SparseModelName: Sparse model name
- - SparseModelVersion: Sparse model version
- - Dim: Dense vector dimension
- - RetryTimes: Retry count
- - RetryDelay: Base retry delay
- """
- # Merge default config and user config
- self.full_config = config
- super().__init__(self.full_config)
-
- self.ak = self.full_config.get("AK", os.environ.get("VOLC_AK"))
- self.sk = self.full_config.get("SK", os.environ.get("VOLC_SK"))
- self.host = self.full_config.get("Host", os.environ.get("VOLC_HOST"))
- self.region = self.full_config.get("Region", os.environ.get("VOLC_REGION"))
-
- if not self.ak or not self.sk or not self.host or not self.region:
- raise ValueError("AK, SK, Host, Region must set")
-
- # Initialize AK/SK signature client
- self.api_client = ClientForDataApi(
- ak=self.ak, sk=self.sk, host=self.host, region=self.region
- )
-
- # Extract core configuration
- self.api_path = self.full_config.get("APIPath", "/api/vikingdb/embedding")
- self.retry_times = self.full_config.get("RetryTimes", 3)
- self.retry_delay = self.full_config.get("RetryDelay", 1)
- self.dim = self.full_config.get("Dim", 0)
-
- def vectorize_query(self, texts: List[str]) -> VectorizeResult:
- """
- Vectorize query text
-
- Args:
- texts: Text list
-
- Returns:
- VectorizeResult: Vectorization result
- """
- dense_model = {
- "name": self.full_config.get("DenseModelName", ""),
- "version": self.full_config.get("DenseModelVersion", "default"),
- }
- sparse_model = None
- if self.full_config.get("SparseModelName"):
- sparse_model = {
- "name": self.full_config.get("SparseModelName", ""),
- "version": self.full_config.get("SparseModelVersion", "default"),
- }
-
- data = [{"text": t} for t in texts]
- return self.vectorize_document(data, dense_model, sparse_model)
-
- def _build_request_body(
- self, data: List[Any], dense_model: Dict[str, Any], sparse_model: Dict[str, Any] = None
- ) -> Dict[str, Any]:
- """Build embedding request body"""
- if not isinstance(data, list) or not all(isinstance(t, dict) for t in data):
- raise ValueError("data must be a list of dictionaries")
-
- req: Dict[str, Any] = {
- "dense_model": {
- "name": dense_model["name"],
- "version": dense_model.get("version", "default"),
- },
- "data": data,
- }
- if "dim" in dense_model:
- req["dense_model"]["dim"] = dense_model["dim"]
-
- if sparse_model:
- req["sparse_model"] = {
- "name": sparse_model["name"],
- "version": sparse_model.get("version", "default"),
- }
- return req
-
- def _parse_response(
- self,
- response_dict: Dict[str, Any],
- dense_model: Dict[str, Any],
- sparse_model: Dict[str, Any] = None,
- ) -> VectorizeResult:
- """Parse API response"""
- # Basic validation
- if not isinstance(response_dict, dict):
- raise ValueError("response must be dictionary type")
-
- # Check call status
- if response_dict.get("code") != "Success":
- raise RuntimeError(
- f"API call failed: {response_dict.get('message', 'unknown error')}, "
- f"request_id: {response_dict.get('request_id', 'none')}"
- )
-
- # Wrap result
- result = VectorizeResult()
- result.request_id = response_dict.get("request_id", "")
-
- # Parse embedding data
- embedding_data = response_dict["result"]["data"]
- result.dense_vectors = [item["dense"] for item in embedding_data]
- result.sparse_vectors = [item["sparse"] for item in embedding_data] if sparse_model else []
-
- # Parse token usage information
- result.token_usage = response_dict["result"]["token_usage"]
- return result
-
- def vectorize_document(
- self,
- data: List[Any],
- dense_model: Dict[str, Any],
- sparse_model: Optional[Dict[str, Any]] = None,
- ) -> VectorizeResult:
- """
- Text vectorization core method
-
- Args:
- data: Data list to vectorize
-
- Returns:
- VectorizeResult: Vectorization result
-
- Raises:
- ValueError: Input parameter error
- RuntimeError: Request failed or response parsing failed
- """
- if not data:
- raise ValueError("data list cannot be empty")
-
- # Build request body
- req_body = self._build_request_body(data, dense_model, sparse_model)
-
- # Request logic with retry
- retry_count = 0
- last_exception = None
-
- while retry_count <= self.retry_times:
- try:
- # Send request using AK/SK signature client
- resp = self.api_client.do_req(
- req_method="POST",
- req_path=self.api_path,
- req_params=None,
- req_body=req_body,
- )
-
- # Check HTTP status code
- resp.raise_for_status()
-
- # Parse JSON response
- resp_dict = resp.json()
-
- # Parse and return result
- return self._parse_response(resp_dict, dense_model, sparse_model)
-
- except (
- requests.exceptions.RequestException,
- json.JSONDecodeError,
- RuntimeError,
- ) as e:
- last_exception = e
- retry_count += 1
-
- # Raise exception if retry limit exceeded
- if retry_count > self.retry_times:
- raise RuntimeError(
- f"request failed (after {self.retry_times} retries): {str(e)}"
- ) from e
-
- # Exponential backoff delay
- delay = self.retry_delay * (2 ** (retry_count - 1))
- time.sleep(delay)
-
- # Fallback exception (should not reach here)
- raise RuntimeError(f"request exception: {str(last_exception)}")
-
- def get_dense_vector_dim(
- self, dense_model: Dict[str, Any], sparse_model: Optional[Dict[str, Any]] = None
- ) -> int:
- """Get dense vector dimension"""
- if self.dim > 0:
- return self.dim
- test_data = [{"text": "volcengine vectorizer health check"}]
- try:
- result = self.vectorize_document(test_data, dense_model, sparse_model)
- # Validate result validity
- return len(result.dense_vectors[0]) if result.dense_vectors else 0
- except Exception:
- return 0
-
- def close(self):
- """Close resources (interface compatibility)"""
- pass
-
- def __del__(self):
- """Destructor"""
- self.close()