Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Native async embedders and a small in-memory cosine index.
The default Hugging Face backend is optional. Merely enabling long-term
memory never imports PyTorch or downloads a model on the event-loop thread:
model construction and encoding run through :func:`asyncio.to_thread`.
"""
from __future__ import annotations
import asyncio
import importlib.util
import inspect
import math
import sys
import threading
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional, Protocol
from uuid import uuid4
from core.encryption import fingerprint_credential
from core.logging import get_logger
if TYPE_CHECKING:
from services.gateway.routing import GatewayRoute
logger = get_logger(__name__)
EmbeddingProvider = Literal["huggingface", "openai", "ollama"]
DEFAULT_EMBEDDING_MODELS: Dict[str, str] = {
"huggingface": "BAAI/bge-small-en-v1.5",
"openai": "text-embedding-3-small",
"ollama": "nomic-embed-text",
}
@dataclass(frozen=True)
class MemoryVectorStoreKey:
"""Non-secret identity for one session's compatible vector space."""
session_id: str
provider: str
model: str
endpoint: str
credential_fingerprint: str
# Legacy string keys remain accepted so state-clear can clean stores created by
# a worker running the pre-migration code during a rolling deployment.
_memory_vector_stores: Dict[MemoryVectorStoreKey | str, Any] = {}
_memory_vector_stores_lock = threading.RLock()
@dataclass(frozen=True)
class MemoryDocument:
"""Search result compatible with the historical ``page_content`` API."""
page_content: str
metadata: Dict[str, Any] = field(default_factory=dict)
class EmbeddingError(RuntimeError):
"""Base class for user-safe embedding configuration/request failures."""
class EmbedderUnavailableError(EmbeddingError):
"""The requested direct SDK or optional embedding extra is unavailable."""
class EmbeddingProviderError(EmbeddingError):
"""A provider SDK failed while generating embeddings."""
class AsyncEmbedder(Protocol):
"""Provider-neutral async embedding contract."""
async def embed_documents(
self,
texts: List[str],
*,
batch_size: int = 32,
) -> List[List[float]]: ...
async def embed_query(self, text: str) -> List[float]: ...
async def aclose(self) -> None: ...
# Compatibility name for code that imported the old protocol.
Embedder = AsyncEmbedder
def default_embedding_model(provider: str) -> str:
normalized = str(provider or "huggingface").strip().lower()
try:
return DEFAULT_EMBEDDING_MODELS[normalized]
except KeyError as exc:
raise ValueError(f"Unsupported embedding provider: {provider}") from exc
def _normalize_provider(provider: str) -> EmbeddingProvider:
normalized = str(provider or "huggingface").strip().lower()
if normalized not in DEFAULT_EMBEDDING_MODELS:
raise ValueError(f"Unsupported embedding provider: {provider}")
return normalized # type: ignore[return-value]
def _normalize_endpoint(endpoint: Optional[str]) -> str:
return str(endpoint or "").strip().rstrip("/")
def _module_available(module_name: str) -> bool:
if module_name in sys.modules:
return sys.modules[module_name] is not None
try:
return importlib.util.find_spec(module_name) is not None
except (ImportError, ValueError):
return False
def _vector_lists(vectors: Any) -> List[List[float]]:
if hasattr(vectors, "tolist"):
vectors = vectors.tolist()
return [
[float(value) for value in vector]
for vector in (vectors or [])
]
async def _close_sdk_client(client: Any) -> None:
close = getattr(client, "close", None)
if close is None:
return
result = close()
if inspect.isawaitable(result):
await result
class SentenceTransformerEmbedder:
"""Lazy, non-blocking adapter for the optional sentence-transformers SDK."""
def __init__(self, model_name: str) -> None:
self.model_name = model_name
self._model: Any = None
self._model_lock = threading.Lock()
def _encode(
self,
texts: List[str],
batch_size: int,
) -> List[List[float]]:
# Loading a model may touch disk or download model assets. It executes
# only in the worker thread that also performs CPU-heavy encoding.
with self._model_lock:
if self._model is None:
try:
from sentence_transformers import SentenceTransformer
except ImportError as exc:
raise EmbedderUnavailableError(
"HuggingFace embeddings unavailable. Install the "
"optional 'local-embeddings' extra."
) from exc
self._model = SentenceTransformer(self.model_name)
vectors = self._model.encode(
texts,
batch_size=batch_size,
convert_to_numpy=True,
show_progress_bar=False,
)
return _vector_lists(vectors)
async def embed_documents(
self,
texts: List[str],
*,
batch_size: int = 32,
) -> List[List[float]]:
if not texts:
return []
try:
return await asyncio.to_thread(
self._encode,
list(texts),
batch_size,
)
except EmbedderUnavailableError:
raise
except Exception as exc:
raise EmbeddingProviderError(
"HuggingFace embedding generation failed "
f"({type(exc).__name__})"
) from exc
async def embed_query(self, text: str) -> List[float]:
vectors = await self.embed_documents([text], batch_size=1)
return vectors[0] if vectors else []
async def aclose(self) -> None:
return None
class OpenAIEmbedder:
"""Direct ``AsyncOpenAI.embeddings`` adapter.
With a ``route``, the OpenCompany Gateway serves the calls: its key and
URL, the LiteLLM prefix on the model name, and the attribution headers
of whichever run makes each call.
"""
def __init__(
self,
model_name: str,
*,
api_key: Optional[str],
endpoint: Optional[str] = None,
route: Optional[GatewayRoute] = None,
) -> None:
try:
from openai import AsyncOpenAI
except ImportError as exc:
raise EmbedderUnavailableError(
"OpenAI embeddings unavailable. Install the 'openai' package."
) from exc
if route is not None:
api_key, endpoint = route.api_key, route.base_url
if not api_key:
# AsyncOpenAI would fall back to OPENAI_API_KEY from the
# environment and send it wherever ``endpoint`` points.
raise EmbedderUnavailableError(
"OpenAI API key is required for embeddings."
)
kwargs: Dict[str, Any] = {"api_key": api_key}
if endpoint:
kwargs["base_url"] = endpoint
self.model_name = model_name
self._route = route
try:
self._client = AsyncOpenAI(**kwargs)
except Exception as exc:
raise EmbedderUnavailableError(
"OpenAI embedding client initialization failed "
f"({type(exc).__name__})"
) from exc
def _request(self, batch: List[str]) -> Dict[str, Any]:
if self._route is None:
return {"model": self.model_name, "input": batch}
from services.gateway.attribution import request_headers
return {
"model": f"{self._route.model_prefix}{self.model_name}",
"input": batch,
"extra_headers": request_headers(),
}
def _failure(self, exc: Exception) -> str:
if self._route is not None:
from services.gateway.errors import describe
message = describe(exc)
if message:
return message
return f"OpenAI embedding request failed ({type(exc).__name__})"
async def embed_documents(
self,
texts: List[str],
*,
batch_size: int = 32,
) -> List[List[float]]:
embeddings: List[List[float]] = []
for start in range(0, len(texts), batch_size):
batch = texts[start : start + batch_size]
try:
response = await self._client.embeddings.create(
**self._request(batch)
)
except Exception as exc:
raise EmbeddingProviderError(self._failure(exc)) from exc
data = sorted(response.data, key=lambda item: int(item.index))
vectors = [
[float(value) for value in item.embedding]
for item in data
]
if len(vectors) != len(batch):
raise ValueError(
"OpenAI returned a different number of embeddings "
"than requested"
)
embeddings.extend(vectors)
return embeddings
async def embed_query(self, text: str) -> List[float]:
vectors = await self.embed_documents([text], batch_size=1)
return vectors[0] if vectors else []
async def aclose(self) -> None:
await _close_sdk_client(self._client)
class OllamaEmbedder:
"""Direct ``ollama.AsyncClient.embed`` adapter."""
def __init__(
self,
model_name: str,
*,
endpoint: Optional[str] = None,
) -> None:
try:
from ollama import AsyncClient
except ImportError as exc:
raise EmbedderUnavailableError(
"Ollama embeddings unavailable. Install the 'ollama' package."
) from exc
kwargs: Dict[str, Any] = {}
if endpoint:
kwargs["host"] = endpoint
self.model_name = model_name
try:
self._client = AsyncClient(**kwargs)
except Exception as exc:
raise EmbedderUnavailableError(
"Ollama embedding client initialization failed "
f"({type(exc).__name__})"
) from exc
async def embed_documents(
self,
texts: List[str],
*,
batch_size: int = 32,
) -> List[List[float]]:
embeddings: List[List[float]] = []
for start in range(0, len(texts), batch_size):
batch = texts[start : start + batch_size]
try:
response = await self._client.embed(
model=self.model_name,
input=batch,
)
except Exception as exc:
raise EmbeddingProviderError(
"Ollama embedding request failed "
f"({type(exc).__name__})"
) from exc
vectors = (
response.get("embeddings")
if isinstance(response, dict)
else getattr(response, "embeddings", None)
)
if vectors is None:
raise ValueError("Ollama returned no embeddings")
batch_vectors = _vector_lists(vectors)
if len(batch_vectors) != len(batch):
raise ValueError(
"Ollama returned a different number of embeddings "
"than requested"
)
embeddings.extend(batch_vectors)
return embeddings
async def embed_query(self, text: str) -> List[float]:
vectors = await self.embed_documents([text], batch_size=1)
return vectors[0] if vectors else []
async def aclose(self) -> None:
await _close_sdk_client(self._client)
def create_embedder(
provider: str,
*,
model: Optional[str] = None,
api_key: Optional[str] = None,
endpoint: Optional[str] = None,
route: Optional[GatewayRoute] = None,
) -> AsyncEmbedder:
"""Build a direct-SDK embedder without making a network request.
``route`` (OpenAI only) sends the calls through the OpenCompany Gateway;
see :func:`openai_embedding_access`.
"""
normalized = _normalize_provider(provider)
model_name = str(model or "").strip() or default_embedding_model(normalized)
normalized_endpoint = _normalize_endpoint(endpoint)
if normalized == "huggingface":
if not _module_available("sentence_transformers"):
raise EmbedderUnavailableError(
"HuggingFace embeddings unavailable. Install the optional "
"'local-embeddings' extra."
)
return SentenceTransformerEmbedder(model_name)
if normalized == "openai":
if route is None and not str(api_key or "").strip():
raise EmbedderUnavailableError(
"OpenAI API key is required for embeddings."
)
return OpenAIEmbedder(
model_name,
api_key=api_key,
endpoint=normalized_endpoint or None,
route=route,
)
return OllamaEmbedder(
model_name,
endpoint=normalized_endpoint or None,
)
async def openai_embedding_access(
api_key: Optional[str],
endpoint: str,
auth_service: Any,
) -> tuple[Optional[str], Optional[GatewayRoute]]:
"""The key for OpenAI embeddings, and the gateway route when it is the
OpenCompany Gateway's.
Own keys first: the given key, else the install's stored OpenAI key.
Without either, and only on OpenAI's own endpoint (an ``endpoint``
override names a server of the user's), the gateway serves the calls
when it covers OpenAI.
"""
key = str(api_key or "").strip() or None
if key is None and auth_service is not None:
stored = await auth_service.resolve_api_key("openai", "default")
key = str(stored or "").strip() or None
if key is None and not endpoint:
from services.gateway.routing import gateway_route
route = await gateway_route("openai")
if route is not None:
return route.api_key, route
return key, None
class NativeMemoryVectorStore:
"""Concurrency-safe async cosine index with the historical tiny API."""
def __init__(
self,
embedder: AsyncEmbedder,
*,
batch_size: int = 32,
) -> None:
self._embedder = embedder
self._batch_size = batch_size
self._entries: List[
tuple[str, str, List[float], Dict[str, Any]]
] = []
self._dimension: Optional[int] = None
self._lock = threading.RLock()
async def add_texts(
self,
texts: Iterable[str],
metadatas: Optional[Iterable[Dict[str, Any]]] = None,
**_: Any,
) -> List[str]:
values = [str(text) for text in texts]
if not values:
return []
metadata_values = list(metadatas or ({} for _ in values))
if len(metadata_values) != len(values):
raise ValueError("metadatas must have the same length as texts")
vectors = await self._embedder.embed_documents(
values,
batch_size=self._batch_size,
)
if len(vectors) != len(values):
raise ValueError("embedder returned a different number of vectors")
dimensions = {len(vector) for vector in vectors}
if len(dimensions) != 1 or not dimensions or 0 in dimensions:
raise ValueError("embedder returned empty or inconsistent vectors")
dimension = dimensions.pop()
ids = [uuid4().hex for _ in values]
with self._lock:
if self._dimension is not None and self._dimension != dimension:
raise ValueError(
"embedding dimension changed within one vector store"
)
self._dimension = dimension
self._entries.extend(
(
item_id,
text,
[float(value) for value in vector],
dict(metadata),
)
for item_id, text, vector, metadata in zip(
ids,
values,
vectors,
metadata_values,
)
)
return ids
@staticmethod
def _cosine(left: List[float], right: List[float]) -> float:
if len(left) != len(right) or not left:
return float("-inf")
denominator = math.sqrt(sum(v * v for v in left)) * math.sqrt(
sum(v * v for v in right)
)
if denominator == 0:
return 0.0
return sum(a * b for a, b in zip(left, right)) / denominator
async def similarity_search(
self,
query: str,
k: int = 4,
**_: Any,
) -> List[MemoryDocument]:
if k <= 0:
return []
with self._lock:
entries = list(self._entries)
dimension = self._dimension
if not entries:
return []
query_vector = [
float(value)
for value in await self._embedder.embed_query(str(query))
]
if dimension is not None and len(query_vector) != dimension:
raise ValueError(
"query embedding dimension does not match stored vectors"
)
ranked = sorted(
entries,
key=lambda entry: self._cosine(query_vector, entry[2]),
reverse=True,
)
return [
MemoryDocument(page_content=text, metadata=dict(metadata))
for _, text, _, metadata in ranked[:k]
]
async def aclose(self) -> None:
await self._embedder.aclose()
def _credential_fingerprint(api_key: Optional[str]) -> str:
if not api_key:
return "none"
return fingerprint_credential(api_key)[:16]
async def get_memory_vector_store(
session_id: str,
*,
provider: str = "huggingface",
model: Optional[str] = None,
endpoint: Optional[str] = None,
api_key: Optional[str] = None,
auth_service: Any = None,
) -> Optional[NativeMemoryVectorStore]:
"""Get one compatible native semantic store for a memory session.
OpenAI credentials are accepted only in-memory and represented in the
cache key by a short SHA-256 fingerprint. The plaintext credential is
never persisted in the key or store metadata.
Missing optional Hugging Face support and missing OpenAI credentials are
deliberate no-op outcomes so long-term memory cannot break agent runs.
"""
normalized = _normalize_provider(provider)
model_name = str(model or "").strip() or default_embedding_model(normalized)
normalized_endpoint = _normalize_endpoint(endpoint)
resolved_key = str(api_key or "").strip() or None
route: Optional[GatewayRoute] = None
if normalized == "openai":
try:
resolved_key, route = await openai_embedding_access(
resolved_key,
normalized_endpoint,
auth_service,
)
except Exception as exc: # noqa: BLE001 - best-effort memory
logger.warning(
"[Memory] Could not resolve OpenAI embedding credential: %s",
exc,
)
return None
if not resolved_key:
logger.warning(
"[Memory] OpenAI long-term memory is disabled because no "
"OpenAI credential is configured"
)
return None
if route is not None:
normalized_endpoint = route.base_url
cache_key = MemoryVectorStoreKey(
session_id=str(session_id),
provider=normalized,
model=model_name,
endpoint=normalized_endpoint,
credential_fingerprint=_credential_fingerprint(resolved_key),
)
with _memory_vector_stores_lock:
cached = _memory_vector_stores.get(cache_key)
if cached is not None:
return cached
try:
embedder = create_embedder(
normalized,
model=model_name,
api_key=resolved_key,
endpoint=normalized_endpoint,
route=route,
)
except EmbedderUnavailableError as exc:
logger.warning("[Memory] Vector store not available: %s", exc)
return None
store = NativeMemoryVectorStore(embedder)
_memory_vector_stores[cache_key] = store
logger.debug(
"[Memory] Created vector store for session=%s provider=%s model=%s",
session_id,
normalized,
model_name,
)
return store
async def clear_memory_vector_stores(session_id: str) -> bool:
"""Remove and close every provider/model store for ``session_id``."""
with _memory_vector_stores_lock:
matching = [
key
for key in _memory_vector_stores
if key == session_id
or (
isinstance(key, MemoryVectorStoreKey)
and key.session_id == session_id
)
]
stores = [_memory_vector_stores.pop(key) for key in matching]
seen: set[int] = set()
for store in stores:
if id(store) in seen:
continue
seen.add(id(store))
close = getattr(store, "aclose", None)
if close is None:
continue
try:
result = close()
if inspect.isawaitable(result):
await result
except Exception as exc: # noqa: BLE001 - clearing remains best effort
logger.warning("[Memory] Failed to close vector store: %s", exc)
return bool(matching)