Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(vectordb): remove legacy vectorize path (#3024)
Qin Haojieb931f4d63f4f
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 2c32e9d48..8b271e411 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -48,9 +48,6 @@ from openviking.storage.vectordb.utils.path_safety import (safe_join_name,)from openviking.storage.vectordb.utils.str_to_uint64 import str_to_uint64-from openviking.storage.vectordb.vectorize.base import BaseVectorizer-from openviking.storage.vectordb.vectorize.vectorizer import VectorizerAdapter-from openviking.storage.vectordb.vectorize.vectorizer_factory import VectorizerFactoryfrom openviking_cli.utils.logger import default_logger as logger# Use imported constants, no longer defined here@@ -60,7 +57,7 @@ AUTO_ID_KEY = SpecialFields.AUTO_ID.valuedef get_or_create_local_collection(meta_data: Optional[Dict[str, Any]] = None,path: str = "",- vectorizer: Optional[BaseVectorizer] = None,+ *,config: Optional[Dict[str, Any]] = None,):"""Create or retrieve a local Collection.@@ -68,7 +65,6 @@ def get_or_create_local_collection(Args:meta_data: Collection metadata configurationpath: Persistence path. If empty, creates an in-memory collection- vectorizer: Vectorizer for embedding generationconfig: Configuration parameters, optional settings include:- "ttl_cleanup_seconds": Interval (in seconds) for TTL expiration data cleanup- "index_maintenance_seconds": Interval (in seconds) for index maintenance tasks@@ -102,30 +98,18 @@ def get_or_create_local_collection(collection: ICollectionif not path:meta = create_collection_meta(path, meta_data)- vectorizer = (- VectorizerFactory.create(meta.vectorize)- if meta.vectorize and vectorizer is None- else vectorizer- )store_mgr = create_store_manager("local")- collection = VolatileCollection(- meta=meta, store=store_mgr, vectorizer=vectorizer, config=config- )+ collection = VolatileCollection(meta=meta, store=store_mgr, config=config)return Collection(collection)else:collection_dir = str(resolve_storage_path(path))os.makedirs(collection_dir, exist_ok=True)meta_path = str(safe_join(collection_dir, "collection_meta.json"))meta = create_collection_meta(meta_path, meta_data)- vectorizer = (- VectorizerFactory.create(meta.vectorize)- if meta.vectorize and vectorizer is None- else vectorizer- )storage_path = str(safe_join(collection_dir, STORAGE_DIR_NAME))store_mgr = create_store_manager("local", storage_path)collection = PersistCollection(- path=collection_dir, meta=meta, store=store_mgr, vectorizer=vectorizer, config=config+ path=collection_dir, meta=meta, store=store_mgr, config=config)return Collection(collection)@@ -135,7 +119,6 @@ class LocalCollection(ICollection):self,meta: CollectionMeta,store_mgr: StoreManager,- vectorizer: Optional[BaseVectorizer] = None,config: Optional[Dict[str, Any]] = None,):self.indexes = ThreadSafeDictManager[IIndex]()@@ -156,10 +139,6 @@ class LocalCollection(ICollection):self.data_processor = DataProcessor(self.meta.fields_dict, collection_name=self.meta.collection_name)- self.vectorizer_adapter = None- if meta.vectorize and vectorizer:- self.vectorizer_adapter = VectorizerAdapter(vectorizer, meta.vectorize)- self.meta.vector_dim = self.vectorizer_adapter.get_dim()self.ttl_cleanup_job_id: Optional[str] = Noneself.index_manage_job_id: Optional[str] = None@@ -405,7 +384,7 @@ class LocalCollection(ICollection):filters: Optional[Dict[str, Any]] = None,output_fields: Optional[List[str]] = None,) -> SearchResult:- """Search using multimodal data by generating vectors and calling search_by_vector.+ """Local backend does not generate vectors from multimodal input.Args:index_name: Name of the index to search@@ -420,18 +399,9 @@ class LocalCollection(ICollection):Returns:SearchResult: Search results"""- if not self.vectorizer_adapter:- raise ValueError("vectorizer is not initialized")-- # Currently mainly supports text vectorization- if not text and not image and not video:- raise ValueError("At least one of text, image, or video must be provided")-- dense_vector, sparse_vector = self.vectorizer_adapter.vectorize_one(- text=text, image=image, video=video- )- return self.search_by_vector(- index_name, dense_vector, limit, offset, filters, sparse_vector, output_fields+ raise NotImplementedError(+ "LocalCollection.search_by_multimodal no longer performs automatic vectorization; "+ "call search_by_vector with caller-provided vectors")def search_by_random(@@ -457,7 +427,7 @@ class LocalCollection(ICollection):filters: Optional[Dict[str, Any]] = None,output_fields: Optional[List[str]] = None,) -> SearchResult:- """Search by keywords by generating vectors and calling search_by_vector.+ """Local backend does not generate vectors from keywords.Args:index_name: Name of the index to search@@ -471,26 +441,9 @@ class LocalCollection(ICollection):Returns:SearchResult: Search results"""- if not self.vectorizer_adapter:- raise ValueError("vectorizer is not initialized")-- if not keywords and not query:- raise ValueError("At least one of keywords or query must be provided")-- # Construct query text- if query:- query_text = query- elif keywords:- # Join keyword list into a string- query_text = " ".join(keywords)- else:- raise ValueError("No valid query input provided")-- # Call vectorization interface to generate vectors- dense_vector, sparse_vector = self.vectorizer_adapter.vectorize_one(text=query_text)-- return self.search_by_vector(- index_name, dense_vector, limit, offset, filters, sparse_vector, output_fields+ raise NotImplementedError(+ "LocalCollection.search_by_keywords no longer performs automatic vectorization; "+ "call search_by_vector with caller-provided vectors")def search_by_scalar(@@ -588,12 +541,6 @@ class LocalCollection(ICollection):def _write_data_list(self, data_list: List[Dict[str, Any]], ttl=0):result = UpsertDataResult()- dense_emb, sparse_emb = (- self.vectorizer_adapter.vectorize_raw_data(data_list)- if self.vectorizer_adapter- else ([], [])- )-cands_list = [CandidateData() for _ in range(len(data_list))]pk = self.meta.primary_keyvk = self.meta.vector_key@@ -608,19 +555,12 @@ class LocalCollection(ICollection):data[AUTO_ID_KEY] = labelcands_list[i].label = label- if self.vectorizer_adapter:- if dense_emb:- cands_list[i].vector = dense_emb[i]- if sparse_emb:- cands_list[i].sparse_raw_terms = list(sparse_emb[i].keys())- cands_list[i].sparse_values = list(sparse_emb[i].values())- else:- cands_list[i].vector = data.pop(vk, None)- if svk:- sparse_dict = data.pop(svk, None)- if sparse_dict and isinstance(sparse_dict, dict):- cands_list[i].sparse_raw_terms = list(sparse_dict.keys())- cands_list[i].sparse_values = list(sparse_dict.values())+ cands_list[i].vector = data.pop(vk, None)+ if svk:+ sparse_dict = data.pop(svk, None)+ if sparse_dict and isinstance(sparse_dict, dict):+ cands_list[i].sparse_raw_terms = list(sparse_dict.keys())+ cands_list[i].sparse_values = list(sparse_dict.values())cands_list[i].fields = safe_json_dumps(data, ensure_ascii=False)cands_list[i].expire_ns_ts = time.time_ns() + ttl * 1000000000 if ttl > 0 else 0@@ -634,9 +574,8 @@ class LocalCollection(ICollection):self.indexes.iterate(upsert_to_index)- if not self.vectorizer_adapter:- for i, data in enumerate(data_list):- data[vk] = list(cands_list[i].vector) if cands_list[i].vector else []+ for i, data in enumerate(data_list):+ data[vk] = list(cands_list[i].vector) if cands_list[i].vector else []if pk != AUTO_ID_KEY:primary_keys = [data.get(pk) for data in data_list]@@ -665,12 +604,11 @@ class LocalCollection(ICollection):raw_data_list.append(None)continueraw_data = json.loads(cand_data.fields)- if not self.vectorizer_adapter:- raw_data[vk] = list(cand_data.vector)- if svk and cand_data.sparse_raw_terms and cand_data.sparse_values:- raw_data[svk] = dict(- zip(cand_data.sparse_raw_terms, cand_data.sparse_values, strict=False)- )+ raw_data[vk] = list(cand_data.vector)+ if svk and cand_data.sparse_raw_terms and cand_data.sparse_values:+ raw_data[svk] = dict(+ zip(cand_data.sparse_raw_terms, cand_data.sparse_values, strict=False)+ )raw_data = validation.fix_fields_data(raw_data, self.meta.fields_dict)raw_data_list.append(raw_data)@@ -956,10 +894,9 @@ class VolatileCollection(LocalCollection):self,meta: CollectionMeta,store: StoreManager,- vectorizer: Optional[BaseVectorizer] = None,config: Optional[Dict[str, Any]] = None,):- super().__init__(meta, store, vectorizer, config)+ super().__init__(meta, store, config=config)LocalCollection._register_scheduler_job(self)def _new_index(@@ -987,14 +924,13 @@ class PersistCollection(LocalCollection):path: str,meta: CollectionMeta,store: StoreManager,- vectorizer: Optional[BaseVectorizer] = None,config: Optional[Dict[str, Any]] = None,):self.collection_dir = str(resolve_storage_path(path))os.makedirs(self.collection_dir, exist_ok=True)self.index_dir = str(safe_join(self.collection_dir, "index"))os.makedirs(self.index_dir, exist_ok=True)- super().__init__(meta, store, vectorizer, config)+ super().__init__(meta, store, config=config)self._recover()LocalCollection._register_scheduler_job(self) # TTL expiration data cleanupdiff --git a/openviking/storage/vectordb/meta/collection_meta.py b/openviking/storage/vectordb/meta/collection_meta.pyindex 191127161..3809d6bb0 100644--- a/openviking/storage/vectordb/meta/collection_meta.py+++ b/openviking/storage/vectordb/meta/collection_meta.py@@ -61,6 +61,7 @@ class CollectionMeta:Dict[str, Any]: The internal metadata structure."""inner_meta = copy.deepcopy(user_meta)+ inner_meta.pop("Vectorize", None)fields = inner_meta.get("Fields", [])has_pk = next((True for item in fields if item.get("IsPrimaryKey", False)),@@ -129,6 +130,7 @@ class CollectionMeta:user_meta.pop("PrimaryKey", None)user_meta.pop("VectorKey", None)user_meta.pop("SparseVectorKey", None)+ user_meta.pop("Vectorize", None)for item in user_meta.get("Fields", []):item.pop("FieldID", None)return user_meta@@ -188,11 +190,6 @@ class CollectionMeta:"""Get the dictionary of fields definitions."""return self.inner_meta.get("FieldsDict", {})- @property- def vectorize(self) -> Dict[str, Any]:- """Get the vectorization configuration."""- return self.inner_meta.get("Vectorize", {})-@propertydef vector_key(self) -> str:"""Get the vector field name."""@@ -203,11 +200,6 @@ class CollectionMeta:"""Get the sparse vector field name."""return self.inner_meta.get("SparseVectorKey", "")- @property- def has_sparse(self) -> bool:- """Check if sparse vector is enabled."""- return "Sparse" in self.inner_meta.get("Vectorize", {})-@propertydef vector_dim(self) -> int:"""Get or set the vector dimension."""diff --git a/openviking/storage/vectordb/service/api_fastapi.py b/openviking/storage/vectordb/service/api_fastapi.pyindex c4c16deb6..ff2877d49 100644--- a/openviking/storage/vectordb/service/api_fastapi.py+++ b/openviking/storage/vectordb/service/api_fastapi.py@@ -134,7 +134,6 @@ async def create_collection(request: CollectionCreateRequest, req: Request):project_name = request.ProjectName or "default"description = request.Description or ""fields = data_utils.convert_dict(request.Fields)- vectorize = data_utils.convert_dict(request.Vectorize)project = get_project(project_name)@@ -146,7 +145,6 @@ async def create_collection(request: CollectionCreateRequest, req: Request):"CollectionName": collection_name,"Description": description,"Fields": fields,- "Vectorize": vectorize,}logger.info(f"Creating collection: {collection_name} in project: {project_name}")diff --git a/openviking/storage/vectordb/service/app_models.py b/openviking/storage/vectordb/service/app_models.pyindex 81a622c60..2cde62eb6 100644--- a/openviking/storage/vectordb/service/app_models.py+++ b/openviking/storage/vectordb/service/app_models.py@@ -12,7 +12,6 @@ class CollectionCreateRequest(BaseModel):ProjectName: Optional[str] = Field("default", description="Project name")Description: Optional[str] = Field("", description="Collection description")Fields: Optional[Any] = Field(None, description="Field definitions")- Vectorize: Optional[Any] = Field(None, description="Vectorize configuration")class CollectionUpdateRequest(BaseModel):diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.pyindex 90864eac7..33175fe38 100644--- a/openviking/storage/vectordb/utils/validation.py+++ b/openviking/storage/vectordb/utils/validation.py@@ -60,54 +60,6 @@ class FieldTypeEnum(str, Enum):GEO_POINT = "geo_point"-class DenseVectorize(BaseModel):- model_config = ConfigDict()-- ModelName: str- ModelVersion: Optional[str] = None- TextField: Optional[str] = None- ImageField: Optional[str] = None- VideoField: Optional[str] = None- Dim: Optional[int] = None- Dimension: Optional[int] = None-- @field_validator("TextField", "ImageField", "VideoField")- @classmethod- def check_fields(cls, v):- # We enforce presence logic in model_validator if needed,- # but the original code had strict checks inside validate_dense_vectorize- # Original: "TextField" is required- return v-- @model_validator(mode="after")- def check_required(self):- if self.TextField is None and self.ImageField is None and self.VideoField is None:- # Original logic: if "text_field" not in vectorize["dense"]- # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")- # Actually, old code: if "text_field" not in vectorize["dense"]: return False- # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.- # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK- # Let's assume PascalCase as per ALLOWED dictionary keys.- if not self.TextField:- raise ValueError("vectorize dense must contain TextField")- return self---class SparseVectorize(BaseModel):- model_config = ConfigDict()-- ModelName: str- ModelVersion: Optional[str] = None- TextField: Optional[str] = None---class VectorizeConfig(BaseModel):- model_config = ConfigDict()-- Dense: Optional[DenseVectorize] = None- Sparse: Optional[SparseVectorize] = None--class CollectionField(BaseModel):model_config = ConfigDict()@@ -151,7 +103,6 @@ class CollectionMetaConfig(BaseModel):Fields: List[CollectionField]ProjectName: Optional[str] = NoneDescription: Optional[str] = Field(None, max_length=65535)- Vectorize: Optional[VectorizeConfig] = NoneFullText: Optional[List[dict]] = (None # e.g. [{"Field": "content", "Analyzer": {"Tokenizer": "standard"}}])@@ -272,7 +223,6 @@ class CollectionMetaUpdateConfig(BaseModel):Fields: Optional[List[CollectionField]] = NoneProjectName: Optional[str] = NoneDescription: Optional[str] = Field(None, max_length=65535)- Vectorize: Optional[VectorizeConfig] = None@field_validator("CollectionName", "ProjectName")@classmethoddiff --git a/openviking/storage/vectordb/vectorize/__init__.py b/openviking/storage/vectordb/vectorize/__init__.pydeleted file mode 100644index 236633478..000000000--- a/openviking/storage/vectordb/vectorize/__init__.py+++ /dev/null@@ -1,2 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0diff --git a/openviking/storage/vectordb/vectorize/base.py b/openviking/storage/vectordb/vectorize/base.pydeleted file mode 100644index 88ad17641..000000000--- a/openviking/storage/vectordb/vectorize/base.py+++ /dev/null@@ -1,82 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""-Base interface definition for vectorization module-"""--from abc import ABC, abstractmethod-from typing import Any, Dict, List, Optional---class VectorizeResult:- """Vectorization result"""-- def __init__(- self,- dense_vectors: Optional[List[List[float]]] = None,- sparse_vectors: Optional[List[Dict[str, float]]] = None,- request_id: str = "",- token_usage: Optional[Dict[str, Any]] = None,- ):- self.dense_vectors = dense_vectors or []- self.sparse_vectors = sparse_vectors or []- self.request_id = request_id- self.token_usage = token_usage-- def __repr__(self):- return (- f"VectorizeResult(dense={len(self.dense_vectors)}, sparse={len(self.sparse_vectors)}, "- f"request_id='{self.request_id}')"- )---class BaseVectorizer(ABC):- """Base vectorizer class"""-- def __init__(self, config: Dict[str, Any]):- """- Initialize vectorizer-- Args:- config: Configuration dictionary containing model-related settings- """- self.config = config- self.model_name = config.get("ModelName", "")- pass-- @abstractmethod- def vectorize_query(self, texts: List[str]) -> VectorizeResult:- """- Vectorize query texts-- Args:- texts: List of texts to vectorize-- Returns:- VectorizeResult: Vectorization results- """- pass-- @abstractmethod- def vectorize_document(- self,- data: List[Any],- dense_model: Dict[str, Any],- sparse_model: Optional[Dict[str, Any]] = None,- ) -> VectorizeResult:- """- Vectorize documents-- Args:- data: List of data items to vectorize- dense_model: Configuration for dense model- sparse_model: Configuration for sparse model (optional)-- Returns:- VectorizeResult: Vectorization results- """- pass-- def close(self):- """Close resources"""- passdiff --git a/openviking/storage/vectordb/vectorize/vectorizer.py b/openviking/storage/vectordb/vectorize/vectorizer.pydeleted file mode 100644index be3e2d043..000000000--- a/openviking/storage/vectordb/vectorize/vectorizer.py+++ /dev/null@@ -1,121 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-from typing import Any, Dict, List, Optional, Tuple, TypedDict---class DenseMeta(TypedDict, total=False):- ModelName: str- Version: str- Dim: int- TextField: str- ImageField: str- VideoField: str---class SparseMeta(TypedDict, total=False):- ModelName: str- Version: str---class VectorizeMeta(TypedDict, total=False):- Dense: DenseMeta- Sparse: SparseMeta---class VectorizerAdapter:- """Adapter for vectorizer to handle data vectorization.-- Adapts the base vectorizer to work with specific collection configuration,- managing field mapping and model parameters.- """-- def __init__(self, vectorizer: Any, vectorize_meta: VectorizeMeta):- """Initialize the VectorizerAdapter.-- Args:- vectorizer: The underlying vectorizer instance.- vectorize_meta (VectorizeMeta): Configuration for vectorization,- including model names, versions, and field mappings.- """- dense_meta = vectorize_meta.get("Dense", {})- self.text_field = dense_meta.get("TextField", "")- self.image_field = dense_meta.get("ImageField", "")- self.video_field = dense_meta.get("VideoField", "")- self.vectorizer = vectorizer- sparse_meta = vectorize_meta.get("Sparse", {})- self.dense_model = {- "name": dense_meta.get("ModelName", ""),- "version": dense_meta.get("Version", "default"),- }- if "Dim" in dense_meta:- self.dense_model["dim"] = int(dense_meta["Dim"])- self.sparse_model = (- {- "name": sparse_meta.get("ModelName", ""),- "version": sparse_meta.get("Version", "default"),- }- if sparse_meta- else {}- )- self.dim = self.vectorizer.get_dense_vector_dim(self.dense_model, self.sparse_model)-- def get_dim(self) -> int:- """Get the dimension of the dense vector.-- Returns:- int: The dimension of the dense vector.- """- return self.dim-- def vectorize_raw_data(- self, raw_data_list: List[Dict[str, Any]]- ) -> Tuple[List[List[float]], List[Dict[str, float]]]:- """Vectorize a list of raw data items.-- Args:- raw_data_list (List[Dict[str, Any]]): List of data dictionaries to vectorize.-- Returns:- Tuple[List[List[float]], List[Dict[str, float]]]: A tuple containing:- - List of dense vectors.- - List of sparse vectors (dictionaries of term-weight pairs).- """- data_list = []- for raw_data in raw_data_list:- data = {}- if self.text_field in raw_data:- data["text"] = raw_data[self.text_field]- if self.image_field in raw_data:- data["image"] = raw_data[self.image_field]- if self.video_field in raw_data:- data["video"] = raw_data[self.video_field]- data_list.append(data)- result = self.vectorizer.vectorize_document(data_list, self.dense_model, self.sparse_model)- return result.dense_vectors, result.sparse_vectors-- def vectorize_one(- self, text: Optional[str] = None, image: Optional[Any] = None, video: Optional[Any] = None- ) -> Tuple[Optional[List[float]], Optional[Dict[str, float]]]:- """Vectorize a single item.-- Args:- text (Optional[str]): Text content to vectorize.- image (Optional[Any]): Image content to vectorize.- video (Optional[Any]): Video content to vectorize.-- Returns:- Tuple[Optional[List[float]], Optional[Dict[str, float]]]: A tuple containing:- - Dense vector (or None if not generated).- - Sparse vector (or None if not generated).- """- data = {}- if text:- data["text"] = text- if image:- data["image"] = image- if video:- data["video"] = video- result = self.vectorizer.vectorize_document([data], self.dense_model, self.sparse_model)- return result.dense_vectors[0] if result.dense_vectors else None, (- result.sparse_vectors[0] if result.sparse_vectors else None- )diff --git a/openviking/storage/vectordb/vectorize/vectorizer_factory.py b/openviking/storage/vectordb/vectorize/vectorizer_factory.pydeleted file mode 100644index 4d3d4b639..000000000--- a/openviking/storage/vectordb/vectorize/vectorizer_factory.py+++ /dev/null@@ -1,55 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-from enum import Enum-from typing import Any, Dict--from openviking.storage.vectordb.vectorize.base import BaseVectorizer-from openviking.storage.vectordb.vectorize.volcengine_vectorizer import VolcengineVectorizer---class ModelType(Enum):- """Model type enumeration."""-- LOCAL = "local" # Local model- HTTP = "http" # HTTP remote model- VOLCENGINE = "Volcengine" # Volcengine remote model---class VectorizerFactory:- """Vectorizer factory."""-- _registry: Dict[str, type] = {}-- @classmethod- def register(cls, model_type: ModelType, vectorizer_class: type):- """Register vectorizer class."""- cls._registry[model_type.value.lower()] = vectorizer_class- # print(f"Register vectorizer {vectorizer_class.__name__} for model type {model_type.value.lower()}")-- @classmethod- def create(- cls, config: Dict[str, Any], model_type: ModelType = ModelType.VOLCENGINE- ) -> BaseVectorizer:- """- Create vectorizer instance.-- Args:- model_type: Model type (local/http/grpc)- config: Configuration dictionary-- Returns:- BaseVectorizer instance- """- vectorizer_class = cls._registry.get(model_type.value.lower())- if not vectorizer_class:- print(- f"Unknown model type: {model_type.value.lower()}. Available: {list(cls._registry.keys())}"- )- raise ValueError(- f"Unknown model type: {model_type}. Available: {list(cls._registry.keys())}"- )-- return vectorizer_class(config)---VectorizerFactory.register(ModelType.VOLCENGINE, VolcengineVectorizer)diff --git a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.pydeleted file mode 100644index 4d62b5c92..000000000--- a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py+++ /dev/null@@ -1,268 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-import json-import os-import time-from typing import Any, Dict, List, Optional--import requests-from volcengine.auth.SignerV4 import SignerV4-from volcengine.base.Request import Request-from volcengine.Credentials import Credentials--from openviking.storage.vectordb.vectorize.base import BaseVectorizer, VectorizeResult---class ClientForDataApi:- def __init__(self, ak, sk, host, region):- self.ak = ak- self.sk = sk- self.host = host- self.region = region-- def prepare_request(self, method, path, params=None, data=None):- r = Request()- r.set_shema("https")- r.set_method(method)- r.set_connection_timeout(10)- r.set_socket_timeout(10)- mheaders = {- "Accept": "application/json",- "Content-Type": "application/json",- "Host": self.host,- }- r.set_headers(mheaders)- if params:- r.set_query(params)- r.set_host(self.host)- r.set_path(path)- if data is not None:- r.set_body(json.dumps(data))- credentials = Credentials(self.ak, self.sk, "vikingdb", self.region)- SignerV4.sign(r, credentials)- return r-- def do_req(self, req_method, req_path, req_params, req_body):- req = self.prepare_request(- method=req_method, path=req_path, params=req_params, data=req_body- )- return requests.request(- method=req.method,- url=f"https://{self.host}{req.path}",- headers=req.headers,- data=req.body,- timeout=10000,- )---class VolcengineVectorizer(BaseVectorizer):- def __init__(self, config: Dict[str, Any]):- """- Initialize Volcengine vectorizer-- Args:- config: Configuration dictionary- - AK: Access Key (required)- - SK: Secret Key (required)- - Host: API domain (required)- - APIPath: API path (default /api/vikingdb/embedding)- - DenseModelName: Dense model name- - DenseModelVersion: Dense model version- - SparseModelName: Sparse model name- - SparseModelVersion: Sparse model version- - Dim: Dense vector dimension- - RetryTimes: Retry count- - RetryDelay: Base retry delay- """- # Merge default config and user config- self.full_config = config- super().__init__(self.full_config)-- self.ak = self.full_config.get("AK", os.environ.get("VOLC_AK"))- self.sk = self.full_config.get("SK", os.environ.get("VOLC_SK"))- self.host = self.full_config.get("Host", os.environ.get("VOLC_HOST"))- self.region = self.full_config.get("Region", os.environ.get("VOLC_REGION"))-- if not self.ak or not self.sk or not self.host or not self.region:- raise ValueError("AK, SK, Host, Region must set")-- # Initialize AK/SK signature client- self.api_client = ClientForDataApi(- ak=self.ak, sk=self.sk, host=self.host, region=self.region- )-- # Extract core configuration- self.api_path = self.full_config.get("APIPath", "/api/vikingdb/embedding")- self.retry_times = self.full_config.get("RetryTimes", 3)- self.retry_delay = self.full_config.get("RetryDelay", 1)- self.dim = self.full_config.get("Dim", 0)-- def vectorize_query(self, texts: List[str]) -> VectorizeResult:- """- Vectorize query text-- Args:- texts: Text list-- Returns:- VectorizeResult: Vectorization result- """- dense_model = {- "name": self.full_config.get("DenseModelName", ""),- "version": self.full_config.get("DenseModelVersion", "default"),- }- sparse_model = None- if self.full_config.get("SparseModelName"):- sparse_model = {- "name": self.full_config.get("SparseModelName", ""),- "version": self.full_config.get("SparseModelVersion", "default"),- }-- data = [{"text": t} for t in texts]- return self.vectorize_document(data, dense_model, sparse_model)-- def _build_request_body(- self, data: List[Any], dense_model: Dict[str, Any], sparse_model: Dict[str, Any] = None- ) -> Dict[str, Any]:- """Build embedding request body"""- if not isinstance(data, list) or not all(isinstance(t, dict) for t in data):- raise ValueError("data must be a list of dictionaries")-- req: Dict[str, Any] = {- "dense_model": {- "name": dense_model["name"],- "version": dense_model.get("version", "default"),- },- "data": data,- }- if "dim" in dense_model:- req["dense_model"]["dim"] = dense_model["dim"]-- if sparse_model:- req["sparse_model"] = {- "name": sparse_model["name"],- "version": sparse_model.get("version", "default"),- }- return req-- def _parse_response(- self,- response_dict: Dict[str, Any],- dense_model: Dict[str, Any],- sparse_model: Dict[str, Any] = None,- ) -> VectorizeResult:- """Parse API response"""- # Basic validation- if not isinstance(response_dict, dict):- raise ValueError("response must be dictionary type")-- # Check call status- if response_dict.get("code") != "Success":- raise RuntimeError(- f"API call failed: {response_dict.get('message', 'unknown error')}, "- f"request_id: {response_dict.get('request_id', 'none')}"- )-- # Wrap result- result = VectorizeResult()- result.request_id = response_dict.get("request_id", "")-- # Parse embedding data- embedding_data = response_dict["result"]["data"]- result.dense_vectors = [item["dense"] for item in embedding_data]- result.sparse_vectors = [item["sparse"] for item in embedding_data] if sparse_model else []-- # Parse token usage information- result.token_usage = response_dict["result"]["token_usage"]- return result-- def vectorize_document(- self,- data: List[Any],- dense_model: Dict[str, Any],- sparse_model: Optional[Dict[str, Any]] = None,- ) -> VectorizeResult:- """- Text vectorization core method-- Args:- data: Data list to vectorize-- Returns:- VectorizeResult: Vectorization result-- Raises:- ValueError: Input parameter error- RuntimeError: Request failed or response parsing failed- """- if not data:- raise ValueError("data list cannot be empty")-- # Build request body- req_body = self._build_request_body(data, dense_model, sparse_model)-- # Request logic with retry- retry_count = 0- last_exception = None-- while retry_count <= self.retry_times:- try:- # Send request using AK/SK signature client- resp = self.api_client.do_req(- req_method="POST",- req_path=self.api_path,- req_params=None,- req_body=req_body,- )-- # Check HTTP status code- resp.raise_for_status()-- # Parse JSON response- resp_dict = resp.json()-- # Parse and return result- return self._parse_response(resp_dict, dense_model, sparse_model)-- except (- requests.exceptions.RequestException,- json.JSONDecodeError,- RuntimeError,- ) as e:- last_exception = e- retry_count += 1-- # Raise exception if retry limit exceeded- if retry_count > self.retry_times:- raise RuntimeError(- f"request failed (after {self.retry_times} retries): {str(e)}"- ) from e-- # Exponential backoff delay- delay = self.retry_delay * (2 ** (retry_count - 1))- time.sleep(delay)-- # Fallback exception (should not reach here)- raise RuntimeError(f"request exception: {str(last_exception)}")-- def get_dense_vector_dim(- self, dense_model: Dict[str, Any], sparse_model: Optional[Dict[str, Any]] = None- ) -> int:- """Get dense vector dimension"""- if self.dim > 0:- return self.dim- test_data = [{"text": "volcengine vectorizer health check"}]- try:- result = self.vectorize_document(test_data, dense_model, sparse_model)- # Validate result validity- return len(result.dense_vectors[0]) if result.dense_vectors else 0- except Exception:- return 0-- def close(self):- """Close resources (interface compatibility)"""- pass-- def __del__(self):- """Destructor"""- self.close()