Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Local GGUF embedders powered by llama-cpp-python."""
from __future__ import annotations
import importlib
import os
from dataclasses import dataclass
from pathlib import Path
from threading import Lock
from typing import Any, Dict, List, Optional
import requests
from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult
from openviking.storage.errors import EmbeddingConfigurationError
from openviking_cli.utils import get_logger
logger = get_logger(__name__)
DEFAULT_LOCAL_MODEL_CACHE_DIR = "~/.cache/openviking/models"
DEFAULT_LOCAL_DENSE_MODEL = "bge-small-zh-v1.5-f16"
DEFAULT_BGE_ZH_QUERY_INSTRUCTION = "为这个句子生成表示以用于检索相关文章:"
@dataclass(frozen=True)
class LocalModelSpec:
model_name: str
dimension: int
filename: str
download_url: str
query_instruction: Optional[str] = None
LOCAL_DENSE_MODEL_SPECS: Dict[str, LocalModelSpec] = {
DEFAULT_LOCAL_DENSE_MODEL: LocalModelSpec(
model_name=DEFAULT_LOCAL_DENSE_MODEL,
dimension=512,
filename="bge-small-zh-v1.5-f16.gguf",
download_url=(
"https://huggingface.co/CompendiumLabs/bge-small-zh-v1.5-gguf/resolve/main/"
"bge-small-zh-v1.5-f16.gguf?download=true"
),
query_instruction=DEFAULT_BGE_ZH_QUERY_INSTRUCTION,
)
}
def get_local_model_spec(model_name: str) -> LocalModelSpec:
try:
return LOCAL_DENSE_MODEL_SPECS[model_name]
except KeyError as exc:
raise ValueError(
f"Unknown local embedding model '{model_name}'. "
f"Supported models: {list(LOCAL_DENSE_MODEL_SPECS.keys())}"
) from exc
def get_local_model_default_dimension(model_name: str) -> int:
return get_local_model_spec(model_name).dimension
def get_local_model_cache_path(model_name: str, cache_dir: Optional[str] = None) -> Path:
spec = get_local_model_spec(model_name)
cache_root = Path(cache_dir or DEFAULT_LOCAL_MODEL_CACHE_DIR).expanduser().resolve()
return cache_root / spec.filename
def get_local_model_identity(model_name: str, model_path: Optional[str] = None) -> str:
if model_path:
resolved = Path(model_path).expanduser().resolve()
return str(resolved)
return get_local_model_spec(model_name).filename
class LocalDenseEmbedder(DenseEmbedderBase):
"""Dense embedder backed by a local GGUF model via llama-cpp-python."""
def __init__(
self,
model_name: str = DEFAULT_LOCAL_DENSE_MODEL,
model_path: Optional[str] = None,
cache_dir: Optional[str] = None,
dimension: Optional[int] = None,
query_instruction: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
):
runtime_config = dict(config or {})
runtime_config.setdefault("provider", "local")
super().__init__(model_name, runtime_config)
self.model_spec = get_local_model_spec(model_name)
self.model_path = model_path
self.cache_dir = cache_dir or DEFAULT_LOCAL_MODEL_CACHE_DIR
self.query_instruction = (
query_instruction
if query_instruction is not None
else self.model_spec.query_instruction
)
self._dimension = dimension or self.model_spec.dimension
if self._dimension != self.model_spec.dimension:
raise ValueError(
f"Local model '{model_name}' has fixed dimension {self.model_spec.dimension}, "
f"but got dimension={self._dimension}"
)
self._resolved_model_path = self._resolve_model_path()
self._llama = self._load_model()
self._llama_lock = Lock()
def _import_llama(self):
try:
module = importlib.import_module("llama_cpp")
except ImportError as exc:
raise EmbeddingConfigurationError(
"Local embedding is enabled but 'llama-cpp-python' is not installed. "
'Install it with: pip install "openviking[local-embed]". '
"If you prefer a remote provider, set embedding.dense.provider explicitly in ov.conf."
) from exc
llama_cls = getattr(module, "Llama", None)
if llama_cls is None:
raise EmbeddingConfigurationError(
"llama_cpp.Llama is unavailable in the installed llama-cpp-python package."
)
return llama_cls
def _resolve_model_path(self) -> Path:
if self.model_path:
resolved = Path(self.model_path).expanduser().resolve()
if not resolved.exists():
raise EmbeddingConfigurationError(
f"Local embedding model file not found: {resolved}"
)
return resolved
cache_root = Path(self.cache_dir).expanduser().resolve()
cache_root.mkdir(parents=True, exist_ok=True)
target = get_local_model_cache_path(self.model_name, self.cache_dir)
if target.exists():
return target
self._download_model(self.model_spec.download_url, target)
return target
def _download_model(self, url: str, target: Path) -> None:
logger.info("Downloading local embedding model %s to %s", self.model_name, target)
tmp_target = target.with_suffix(target.suffix + ".part")
try:
with requests.get(url, stream=True, timeout=(10, 300)) as response:
response.raise_for_status()
with tmp_target.open("wb") as fh:
for chunk in response.iter_content(chunk_size=1024 * 1024):
if chunk:
fh.write(chunk)
os.replace(tmp_target, target)
except Exception as exc:
tmp_target.unlink(missing_ok=True)
raise EmbeddingConfigurationError(
f"Failed to download local embedding model '{self.model_name}' from {url} "
f"to {target}: {exc}"
) from exc
def _load_model(self):
llama_cls = self._import_llama()
try:
return llama_cls(
model_path=str(self._resolved_model_path),
embedding=True,
verbose=False,
)
except Exception as exc:
raise EmbeddingConfigurationError(
f"Failed to load GGUF embedding model from {self._resolved_model_path}: {exc}"
) from exc
def _format_text(self, text: str, *, is_query: bool) -> str:
if is_query and self.query_instruction:
return f"{self.query_instruction}{text}"
return text
@staticmethod
def _extract_embedding(payload: Any) -> List[float]:
if isinstance(payload, dict):
data = payload.get("data")
if isinstance(data, list) and data:
item = data[0]
if isinstance(item, dict) and "embedding" in item:
return list(item["embedding"])
if "embedding" in payload:
return list(payload["embedding"])
raise RuntimeError("Unexpected llama-cpp-python embedding response format")
def _embed_formatted_text(self, formatted: str) -> EmbedResult:
# llama.cpp mutates the shared context, including during native calls that release the GIL.
with self._llama_lock:
payload = self._llama.create_embedding(formatted)
return EmbedResult(dense_vector=self._extract_embedding(payload))
def embed(self, text: str, is_query: bool = False) -> EmbedResult:
formatted = self._format_text(text, is_query=is_query)
try:
result = self._run_once(
lambda: self._embed_formatted_text(formatted),
logger=logger,
operation_name="local embedding",
)
except Exception as exc:
raise RuntimeError(f"Local embedding failed: {exc}") from exc
estimated_tokens = self._estimate_tokens(formatted)
self.update_token_usage(
model_name=self.model_name,
provider="local",
prompt_tokens=estimated_tokens,
completion_tokens=0,
)
return result
def get_dimension(self) -> int:
return self._dimension
def close(self):
with self._llama_lock:
close_fn = getattr(self._llama, "close", None)
if callable(close_fn):
close_fn()