Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: remove legacy embeddings (#3846)
Sarah Woodersf4740b138812
diff --git a/letta/data_sources/connectors.py b/letta/data_sources/connectors.pyindex 3248e83ca..cfafe2a2f 100644--- a/letta/data_sources/connectors.py+++ b/letta/data_sources/connectors.py@@ -4,7 +4,6 @@ import typerfrom letta.constants import EMBEDDING_BATCH_SIZEfrom letta.data_sources.connectors_helper import assert_all_files_exist_locally, extract_metadata_from_files, get_filenames_in_dir-from letta.embeddings import embedding_modelfrom letta.schemas.file import FileMetadatafrom letta.schemas.passage import Passagefrom letta.schemas.source import Source@@ -40,61 +39,29 @@ class DataConnector:async def load_data(connector: DataConnector, source: Source, passage_manager: PassageManager, file_manager: FileManager, actor: "User"):from letta.llm_api.llm_client import LLMClient- from letta.schemas.embedding_config import EmbeddingConfig"""Load data from a connector (generates file and passages) into a specified source_id, associated with a user_id."""embedding_config = source.embedding_config# insert passages/file- texts = []embedding_to_document_name = {}passage_count = 0file_count = 0- async def generate_embeddings(texts: List[str], embedding_config: EmbeddingConfig) -> List[Passage]:- passages = []- if embedding_config.embedding_endpoint_type == "openai":- texts.append(passage_text)-- client = LLMClient.create(- provider_type=embedding_config.embedding_endpoint_type,- actor=actor,- )- embeddings = await client.request_embeddings(texts, embedding_config)-- else:- embed_model = embedding_model(embedding_config)- embeddings = [embed_model.get_text_embedding(text) for text in texts]-- # collate passage and embedding- for text, embedding in zip(texts, embeddings):- passage = Passage(- text=text,- file_id=file_metadata.id,- source_id=source.id,- metadata=passage_metadata,- organization_id=source.organization_id,- embedding_config=source.embedding_config,- embedding=embedding,- )- hashable_embedding = tuple(passage.embedding)- file_name = file_metadata.file_name- if hashable_embedding in embedding_to_document_name:- typer.secho(- f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",- fg=typer.colors.YELLOW,- )- continue-- passages.append(passage)- embedding_to_document_name[hashable_embedding] = file_name- return passages+ # Use the new LLMClient for all embedding requests+ client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )for file_metadata in connector.find_files(source):file_count += 1await file_manager.create_file(file_metadata, actor)- # generate passages+ # generate passages for this file+ texts = []+ metadatas = []+for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):# for some reason, llama index parsers sometimes return empty stringsif len(passage_text) == 0:@@ -104,24 +71,74 @@ async def load_data(connector: DataConnector, source: Source, passage_manager: P)continue- # get embeddingtexts.append(passage_text)+ metadatas.append(passage_metadata)+if len(texts) >= EMBEDDING_BATCH_SIZE:- passages = await generate_embeddings(texts, embedding_config)+ # Process the batch+ embeddings = await client.request_embeddings(texts, embedding_config)+ passages = []++ for text, embedding, passage_metadata in zip(texts, embeddings, metadatas):+ passage = Passage(+ text=text,+ file_id=file_metadata.id,+ source_id=source.id,+ metadata=passage_metadata,+ organization_id=source.organization_id,+ embedding_config=source.embedding_config,+ embedding=embedding,+ )+ hashable_embedding = tuple(passage.embedding)+ file_name = file_metadata.file_name+ if hashable_embedding in embedding_to_document_name:+ typer.secho(+ f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",+ fg=typer.colors.YELLOW,+ )+ continue++ passages.append(passage)+ embedding_to_document_name[hashable_embedding] = file_name++ # insert passages into passage store+ await passage_manager.create_many_passages_async(passages, actor)+ passage_count += len(passages)++ # Reset for next batchtexts = []- else:- continue+ metadatas = []++ # Process final remaining texts for this file+ if len(texts) > 0:+ embeddings = await client.request_embeddings(texts, embedding_config)+ passages = []++ for text, embedding, passage_metadata in zip(texts, embeddings, metadatas):+ passage = Passage(+ text=text,+ file_id=file_metadata.id,+ source_id=source.id,+ metadata=passage_metadata,+ organization_id=source.organization_id,+ embedding_config=source.embedding_config,+ embedding=embedding,+ )+ hashable_embedding = tuple(passage.embedding)+ file_name = file_metadata.file_name+ if hashable_embedding in embedding_to_document_name:+ typer.secho(+ f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",+ fg=typer.colors.YELLOW,+ )+ continue++ passages.append(passage)+ embedding_to_document_name[hashable_embedding] = file_name- # insert passages into passage storeawait passage_manager.create_many_passages_async(passages, actor)passage_count += len(passages)- # final remaining- if len(texts) > 0:- passages = await generate_embeddings(texts, embedding_config)- await passage_manager.create_many_passages_async(passages, actor)- passage_count += len(passages)-return passage_count, file_countdiff --git a/letta/embeddings.py b/letta/embeddings.pyindex f302c6fb3..a07e16c3c 100644--- a/letta/embeddings.py+++ b/letta/embeddings.py@@ -1,13 +1,9 @@-import uuid-from typing import Any, List, Optional+from typing import List-import numpy as npimport tiktoken-from openai import OpenAI-from letta.constants import EMBEDDING_TO_TOKENIZER_DEFAULT, EMBEDDING_TO_TOKENIZER_MAP, MAX_EMBEDDING_DIM-from letta.schemas.embedding_config import EmbeddingConfig-from letta.utils import is_valid_url, printd+from letta.constants import EMBEDDING_TO_TOKENIZER_DEFAULT, EMBEDDING_TO_TOKENIZER_MAP+from letta.utils import printddef parse_and_chunk_text(text: str, chunk_size: int) -> List[str]:@@ -55,235 +51,3 @@ def check_and_split_text(text: str, embedding_model: str) -> List[str]:text = truncate_text(formatted_text, max_length, encoding)return [text]---class EmbeddingEndpoint:- """Implementation for OpenAI compatible endpoint"""-- # """ Based off llama index https://github.com/run-llama/llama_index/blob/a98bdb8ecee513dc2e880f56674e7fd157d1dc3a/llama_index/embeddings/text_embeddings_inference.py """-- # _user: str = PrivateAttr()- # _timeout: float = PrivateAttr()- # _base_url: str = PrivateAttr()-- def __init__(- self,- model: str,- base_url: str,- user: str,- timeout: float = 60.0,- **kwargs: Any,- ):- if not is_valid_url(base_url):- raise ValueError(- f"Embeddings endpoint was provided an invalid URL (set to: '{base_url}'). Make sure embedding_endpoint is set correctly in your Letta config."- )- # TODO: find a neater solution - re-mapping for letta endpoint- if model == "letta-free":- model = "BAAI/bge-large-en-v1.5"- self.model_name = model- self._user = user- self._base_url = base_url- self._timeout = timeout-- def _call_api(self, text: str) -> List[float]:- if not is_valid_url(self._base_url):- raise ValueError(- f"Embeddings endpoint does not have a valid URL (set to: '{self._base_url}'). Make sure embedding_endpoint is set correctly in your Letta config."- )- import httpx-- headers = {"Content-Type": "application/json"}- json_data = {"input": text, "model": self.model_name, "user": self._user}-- with httpx.Client() as client:- response = client.post(- f"{self._base_url}/embeddings",- headers=headers,- json=json_data,- timeout=self._timeout,- )-- response_json = response.json()-- if isinstance(response_json, list):- # embedding directly in response- embedding = response_json- elif isinstance(response_json, dict):- # TEI embedding packaged inside openai-style response- try:- embedding = response_json["data"][0]["embedding"]- except (KeyError, IndexError):- raise TypeError(f"Got back an unexpected payload from text embedding function, response=\n{response_json}")- else:- # unknown response, can't parse- raise TypeError(f"Got back an unexpected payload from text embedding function, response=\n{response_json}")-- return embedding-- def get_text_embedding(self, text: str) -> List[float]:- return self._call_api(text)---class AzureOpenAIEmbedding:- def __init__(self, api_endpoint: str, api_key: str, api_version: str, model: str):- from openai import AzureOpenAI-- self.client = AzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=api_endpoint)- self.model = model-- def get_text_embedding(self, text: str):- embeddings = self.client.embeddings.create(input=[text], model=self.model).data[0].embedding- return embeddings---class OllamaEmbeddings:-- # Format:- # curl http://localhost:11434/api/embeddings -d '{- # "model": "mxbai-embed-large",- # "prompt": "Llamas are members of the camelid family"- # }'-- def __init__(self, model: str, base_url: str, ollama_additional_kwargs: dict):- self.model = model- self.base_url = base_url- self.ollama_additional_kwargs = ollama_additional_kwargs-- def get_text_embedding(self, text: str):- import httpx-- headers = {"Content-Type": "application/json"}- json_data = {"model": self.model, "prompt": text}- json_data.update(self.ollama_additional_kwargs)-- with httpx.Client() as client:- response = client.post(- f"{self.base_url}/api/embeddings",- headers=headers,- json=json_data,- )-- response_json = response.json()- return response_json["embedding"]---class GoogleEmbeddings:- def __init__(self, api_key: str, model: str, base_url: str):- self.api_key = api_key- self.model = model- self.base_url = base_url # Expected to be "https://generativelanguage.googleapis.com"-- def get_text_embedding(self, text: str):- import httpx-- headers = {"Content-Type": "application/json"}- # Build the URL based on the provided base_url, model, and API key.- url = f"{self.base_url}/v1beta/models/{self.model}:embedContent?key={self.api_key}"- payload = {"model": self.model, "content": {"parts": [{"text": text}]}}- with httpx.Client() as client:- response = client.post(url, headers=headers, json=payload)- # Raise an error for non-success HTTP status codes.- response.raise_for_status()- response_json = response.json()- return response_json["embedding"]["values"]---class GoogleVertexEmbeddings:- def __init__(self, model: str, project_id: str, region: str):- from google import genai-- self.client = genai.Client(vertexai=True, project=project_id, location=region, http_options={"api_version": "v1"})- self.model = model-- def get_text_embedding(self, text: str):- response = self.client.generate_embeddings(content=text, model=self.model)- return response.embeddings[0].embedding---class OpenAIEmbeddings:- def __init__(self, api_key: str, model: str, base_url: str):- if base_url:- self.client = OpenAI(api_key=api_key, base_url=base_url)- else:- self.client = OpenAI(api_key=api_key)- self.model = model-- def get_text_embedding(self, text: str):- response = self.client.embeddings.create(input=text, model=self.model)-- return response.data[0].embedding---def query_embedding(embedding_model, query_text: str):- """Generate padded embedding for querying database"""- query_vec = embedding_model.get_text_embedding(query_text)- query_vec = np.array(query_vec)- query_vec = np.pad(query_vec, (0, MAX_EMBEDDING_DIM - query_vec.shape[0]), mode="constant").tolist()- return query_vec---def embedding_model(config: EmbeddingConfig, user_id: Optional[uuid.UUID] = None):- """Return LlamaIndex embedding model to use for embeddings"""-- endpoint_type = config.embedding_endpoint_type-- # TODO: refactor to pass in settings from server- from letta.settings import model_settings-- if endpoint_type == "openai":- return OpenAIEmbeddings(- api_key=model_settings.openai_api_key,- model=config.embedding_model,- base_url=config.embedding_endpoint or model_settings.openai_api_base,- )-- elif endpoint_type == "azure":- assert all(- [- model_settings.azure_api_key is not None,- model_settings.azure_base_url is not None,- model_settings.azure_api_version is not None,- ]- )- return AzureOpenAIEmbedding(- api_endpoint=model_settings.azure_base_url,- api_key=model_settings.azure_api_key,- api_version=model_settings.azure_api_version,- model=config.embedding_model,- )-- elif endpoint_type == "hugging-face":- return EmbeddingEndpoint(- model=config.embedding_model,- base_url=config.embedding_endpoint,- user=user_id,- )- elif endpoint_type == "ollama":-- model = OllamaEmbeddings(- model=config.embedding_model,- base_url=config.embedding_endpoint,- ollama_additional_kwargs={},- )- return model-- elif endpoint_type == "google_ai":- assert all([model_settings.gemini_api_key is not None, model_settings.gemini_base_url is not None])- model = GoogleEmbeddings(- model=config.embedding_model,- api_key=model_settings.gemini_api_key,- base_url=model_settings.gemini_base_url,- )- return model-- elif endpoint_type == "google_vertex":- model = GoogleVertexEmbeddings(- model=config.embedding_model,- api_key=model_settings.gemini_api_key,- base_url=model_settings.gemini_base_url,- )- return model-- else:- raise ValueError(f"Unknown endpoint type {endpoint_type}")diff --git a/letta/functions/function_sets/base.py b/letta/functions/function_sets/base.pyindex ee8df51b1..d0e2e94c4 100644--- a/letta/functions/function_sets/base.py+++ b/letta/functions/function_sets/base.py@@ -63,7 +63,7 @@ def conversation_search(self: "Agent", query: str, page: Optional[int] = 0) -> Oreturn results_str-def archival_memory_insert(self: "Agent", content: str) -> Optional[str]:+async def archival_memory_insert(self: "Agent", content: str) -> Optional[str]:"""Add to archival memory. Make sure to phrase the memory contents such that it can be easily queried later.@@ -73,7 +73,7 @@ def archival_memory_insert(self: "Agent", content: str) -> Optional[str]:Returns:Optional[str]: None is always returned as this function does not produce a response."""- self.passage_manager.insert_passage(+ await self.passage_manager.insert_passage(agent_state=self.agent_state,text=content,actor=self.user,@@ -82,7 +82,7 @@ def archival_memory_insert(self: "Agent", content: str) -> Optional[str]:return None-def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0, start: Optional[int] = 0) -> Optional[str]:+async def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0, start: Optional[int] = 0) -> Optional[str]:"""Search archival memory using semantic (embedding-based) search.@@ -107,7 +107,7 @@ def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0, stry:# Get results using passage manager- all_results = self.agent_manager.list_passages(+ all_results = await self.agent_manager.list_passages_async(actor=self.user,agent_id=self.agent_state.id,query_text=query,diff --git a/letta/schemas/providers/ollama.py b/letta/schemas/providers/ollama.pyindex df7d83468..d34d86d7f 100644--- a/letta/schemas/providers/ollama.py+++ b/letta/schemas/providers/ollama.py@@ -74,7 +74,7 @@ class OllamaProvider(OpenAIProvider):configs = []for model in response_json["models"]:- embedding_dim = await self._get_model_embedding_dim_async(model["name"])+ embedding_dim = await self._get_model_embedding_dim(model["name"])if not embedding_dim:print(f"Ollama model {model['name']} has no embedding dimension, using default 1024")# continuediff --git a/letta/server/server.py b/letta/server/server.pyindex 468f1cf72..8b396594b 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -1099,33 +1099,6 @@ class SyncServer(Server):def get_recall_memory_summary(self, agent_id: str, actor: User) -> RecallMemorySummary:return RecallMemorySummary(size=self.message_manager.size(actor=actor, agent_id=agent_id))- def get_agent_archival(- self,- user_id: str,- agent_id: str,- after: Optional[str] = None,- before: Optional[str] = None,- limit: Optional[int] = 100,- order_by: Optional[str] = "created_at",- reverse: Optional[bool] = False,- query_text: Optional[str] = None,- ascending: Optional[bool] = True,- ) -> List[Passage]:- # TODO: Thread actor directly through this function, since the top level caller most likely already retrieved the user- actor = self.user_manager.get_user_or_default(user_id=user_id)-- # iterate over records- records = self.agent_manager.list_passages(- actor=actor,- agent_id=agent_id,- after=after,- query_text=query_text,- before=before,- ascending=ascending,- limit=limit,- )- return records-async def get_agent_archival_async(self,agent_id: str,@@ -1153,7 +1126,7 @@ class SyncServer(Server):agent_state = await self.agent_manager.get_agent_by_id_async(agent_id=agent_id, actor=actor)# Insert passages into the archive- passages = await self.passage_manager.insert_passage_async(agent_state=agent_state, text=memory_contents, actor=actor)+ passages = await self.passage_manager.insert_passage(agent_state=agent_state, text=memory_contents, actor=actor)return passages@@ -1471,10 +1444,6 @@ class SyncServer(Server):passage_count, document_count = await load_data(connector, source, self.passage_manager, self.file_manager, actor=actor)return passage_count, document_count- def list_data_source_passages(self, user_id: str, source_id: str) -> List[Passage]:- # TODO: move this query into PassageManager- return self.agent_manager.list_passages(actor=self.user_manager.get_user_or_default(user_id=user_id), source_id=source_id)-def list_all_sources(self, actor: User) -> List[Source]:# TODO: legacy: remove"""List all sources (w/ extra metadata) belonging to a user"""diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 5751ac407..65e03d378 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -2354,7 +2354,7 @@ class AgentManager:@enforce_types@trace_method- def list_passages(+ async def list_passages(self,actor: PydanticUser,agent_id: Optional[str] = None,@@ -2372,8 +2372,8 @@ class AgentManager:agent_only: bool = False,) -> List[PydanticPassage]:"""Lists all passages attached to an agent."""- with db_registry.session() as session:- main_query = build_passage_query(+ async with db_registry.async_session() as session:+ main_query = await build_passage_query(actor=actor,agent_id=agent_id,file_id=file_id,@@ -2394,7 +2394,7 @@ class AgentManager:main_query = main_query.limit(limit)# Execute query- result = session.execute(main_query)+ result = await session.execute(main_query)passages = []for row in result:@@ -2437,7 +2437,7 @@ class AgentManager:) -> List[PydanticPassage]:"""Lists all passages attached to an agent."""async with db_registry.async_session() as session:- main_query = build_passage_query(+ main_query = await build_passage_query(actor=actor,agent_id=agent_id,file_id=file_id,@@ -2500,7 +2500,7 @@ class AgentManager:) -> List[PydanticPassage]:"""Lists all passages attached to an agent."""async with db_registry.async_session() as session:- main_query = build_source_passage_query(+ main_query = await build_source_passage_query(actor=actor,agent_id=agent_id,file_id=file_id,@@ -2546,7 +2546,7 @@ class AgentManager:) -> List[PydanticPassage]:"""Lists all passages attached to an agent."""async with db_registry.async_session() as session:- main_query = build_agent_passage_query(+ main_query = await build_agent_passage_query(actor=actor,agent_id=agent_id,query_text=query_text,@@ -2574,7 +2574,7 @@ class AgentManager:@enforce_types@trace_method- def passage_size(+ async def passage_size(self,actor: PydanticUser,agent_id: Optional[str] = None,@@ -2591,8 +2591,8 @@ class AgentManager:agent_only: bool = False,) -> int:"""Returns the count of passages matching the given criteria."""- with db_registry.session() as session:- main_query = build_passage_query(+ async with db_registry.async_session() as session:+ main_query = await build_passage_query(actor=actor,agent_id=agent_id,file_id=file_id,@@ -2610,7 +2610,7 @@ class AgentManager:# Convert to count querycount_query = select(func.count()).select_from(main_query.subquery())- return session.scalar(count_query) or 0+ return (await session.scalar(count_query)) or 0@enforce_typesasync def passage_size_async(@@ -2630,7 +2630,7 @@ class AgentManager:agent_only: bool = False,) -> int:async with db_registry.async_session() as session:- main_query = build_passage_query(+ main_query = await build_passage_query(actor=actor,agent_id=agent_id,file_id=file_id,diff --git a/letta/services/helpers/agent_manager_helper.py b/letta/services/helpers/agent_manager_helper.pyindex a97509f4d..1319175b9 100644--- a/letta/services/helpers/agent_manager_helper.py+++ b/letta/services/helpers/agent_manager_helper.py@@ -20,9 +20,9 @@ from letta.constants import (MULTI_AGENT_TOOLS,STRUCTURED_OUTPUT_MODELS,)-from letta.embeddings import embedding_modelfrom letta.helpers import ToolRulesSolverfrom letta.helpers.datetime_helpers import format_datetime, get_local_time, get_local_time_fast+from letta.llm_api.llm_client import LLMClientfrom letta.orm.agent import Agent as AgentModelfrom letta.orm.agents_tags import AgentsTagsfrom letta.orm.archives_agents import ArchivesAgents@@ -939,7 +939,7 @@ def _apply_relationship_filters(query, include_relationships: Optional[List[str]return query-def build_passage_query(+async def build_passage_query(actor: User,agent_id: Optional[str] = None,file_id: Optional[str] = None,@@ -963,8 +963,14 @@ def build_passage_query(if embed_query:assert embedding_config is not None, "embedding_config must be specified for vector search"assert query_text is not None, "query_text must be specified for vector search"- embedded_text = embedding_model(embedding_config).get_text_embedding(query_text)- embedded_text = np.array(embedded_text)++ # Use the new LLMClient for embeddings+ embedding_client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )+ embeddings = await embedding_client.request_embeddings([query_text], embedding_config)+ embedded_text = np.array(embeddings[0])embedded_text = np.pad(embedded_text, (0, MAX_EMBEDDING_DIM - embedded_text.shape[0]), mode="constant").tolist()# Start with base query for source passages@@ -1150,7 +1156,7 @@ def build_passage_query(return main_query-def build_source_passage_query(+async def build_source_passage_query(actor: User,agent_id: Optional[str] = None,file_id: Optional[str] = None,@@ -1171,8 +1177,14 @@ def build_source_passage_query(if embed_query:assert embedding_config is not None, "embedding_config must be specified for vector search"assert query_text is not None, "query_text must be specified for vector search"- embedded_text = embedding_model(embedding_config).get_text_embedding(query_text)- embedded_text = np.array(embedded_text)++ # Use the new LLMClient for embeddings+ embedding_client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )+ embeddings = await embedding_client.request_embeddings([query_text], embedding_config)+ embedded_text = np.array(embeddings[0])embedded_text = np.pad(embedded_text, (0, MAX_EMBEDDING_DIM - embedded_text.shape[0]), mode="constant").tolist()# Base query for source passages@@ -1248,7 +1260,7 @@ def build_source_passage_query(return query-def build_agent_passage_query(+async def build_agent_passage_query(actor: User,agent_id: str, # Required for agent passagesquery_text: Optional[str] = None,@@ -1267,8 +1279,14 @@ def build_agent_passage_query(if embed_query:assert embedding_config is not None, "embedding_config must be specified for vector search"assert query_text is not None, "query_text must be specified for vector search"- embedded_text = embedding_model(embedding_config).get_text_embedding(query_text)- embedded_text = np.array(embedded_text)++ # Use the new LLMClient for embeddings+ embedding_client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )+ embeddings = await embedding_client.request_embeddings([query_text], embedding_config)+ embedded_text = np.array(embeddings[0])embedded_text = np.pad(embedded_text, (0, MAX_EMBEDDING_DIM - embedded_text.shape[0]), mode="constant").tolist()# Base query for agent passages - join through archives_agentsdiff --git a/letta/services/passage_manager.py b/letta/services/passage_manager.pyindex 895ecef3e..39cd07ce1 100644--- a/letta/services/passage_manager.py+++ b/letta/services/passage_manager.py@@ -1,4 +1,3 @@-import asynciofrom datetime import datetime, timezonefrom functools import lru_cachefrom typing import List, Optional@@ -7,8 +6,9 @@ from openai import AsyncOpenAI, OpenAIfrom sqlalchemy import selectfrom letta.constants import MAX_EMBEDDING_DIM-from letta.embeddings import embedding_model, parse_and_chunk_text+from letta.embeddings import parse_and_chunk_textfrom letta.helpers.decorators import async_redis_cache+from letta.llm_api.llm_client import LLMClientfrom letta.orm import ArchivesAgentsfrom letta.orm.errors import NoResultFoundfrom letta.orm.passage import ArchivalPassage, SourcePassage@@ -460,7 +460,7 @@ class PassageManager:@enforce_types@trace_method- def insert_passage(+ async def insert_passage(self,agent_state: AgentState,text: str,@@ -469,45 +469,32 @@ class PassageManager:"""Insert passage(s) into archival memory"""embedding_chunk_size = agent_state.embedding_config.embedding_chunk_size+ embedding_client = LLMClient.create(+ provider_type=agent_state.embedding_config.embedding_endpoint_type,+ actor=actor,+ )- # TODO eventually migrate off of llama-index for embeddings?- # Already causing pain for OpenAI proxy endpoints like LM Studio...- if agent_state.embedding_config.embedding_endpoint_type != "openai":- embed_model = embedding_model(agent_state.embedding_config)+ # Get or create the default archive for the agent+ archive = await self.archive_manager.get_or_create_default_archive_for_agent_async(+ agent_id=agent_state.id, agent_name=agent_state.name, actor=actor+ )- passages = []+ text_chunks = list(parse_and_chunk_text(text, embedding_chunk_size))- try:- # breakup string into passages- for text in parse_and_chunk_text(text, embedding_chunk_size):- if agent_state.embedding_config.embedding_endpoint_type != "openai":- embedding = embed_model.get_text_embedding(text)- else:- # TODO should have the settings passed in via the server call- embedding = get_openai_embedding(- text,- agent_state.embedding_config.embedding_model,- agent_state.embedding_config.embedding_endpoint,- )+ if not text_chunks:+ return []- if isinstance(embedding, dict):- try:- embedding = embedding["data"][0]["embedding"]- except (KeyError, IndexError):- # TODO as a fallback, see if we can find any lists in the payload- raise TypeError(- f"Got back an unexpected payload from text embedding function, type={type(embedding)}, value={embedding}"- )- # Get or create the default archive for the agent- archive = self.archive_manager.get_or_create_default_archive_for_agent(- agent_id=agent_state.id, agent_name=agent_state.name, actor=actor- )+ try:+ # Generate embeddings for all chunks using the new async API+ embeddings = await embedding_client.request_embeddings(text_chunks, agent_state.embedding_config)- passage = self.create_agent_passage(+ passages = []+ for chunk_text, embedding in zip(text_chunks, embeddings):+ passage = await self.create_agent_passage_async(PydanticPassage(organization_id=actor.organization_id,archive_id=archive.id,- text=text,+ text=chunk_text,embedding=embedding,embedding_config=agent_state.embedding_config,),@@ -520,84 +507,16 @@ class PassageManager:except Exception as e:raise e- @enforce_types- @trace_method- async def insert_passage_async(- self,- agent_state: AgentState,- text: str,- actor: PydanticUser,- image_ids: Optional[List[str]] = None,- ) -> List[PydanticPassage]:- """Insert passage(s) into archival memory"""- # Get or create default archive for the agent- archive = await self.archive_manager.get_or_create_default_archive_for_agent_async(- agent_id=agent_state.id,- agent_name=agent_state.name,+ async def _generate_embeddings_concurrent(self, text_chunks: List[str], embedding_config, actor: PydanticUser) -> List[List[float]]:+ """Generate embeddings for all text chunks concurrently using LLMClient"""++ embedding_client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,actor=actor,)- archive_id = archive.id-- embedding_chunk_size = agent_state.embedding_config.embedding_chunk_size- text_chunks = list(parse_and_chunk_text(text, embedding_chunk_size))-- if not text_chunks:- return []-- try:- embeddings = await self._generate_embeddings_concurrent(text_chunks, agent_state.embedding_config)-- passages = [- PydanticPassage(- organization_id=actor.organization_id,- archive_id=archive_id,- text=chunk_text,- embedding=embedding,- embedding_config=agent_state.embedding_config,- )- for chunk_text, embedding in zip(text_chunks, embeddings)- ]-- passages = await self.create_many_archival_passages_async(passages=passages, actor=actor)-- return passages-- except Exception as e:- raise e-- async def _generate_embeddings_concurrent(self, text_chunks: List[str], embedding_config) -> List[List[float]]:- """Generate embeddings for all text chunks concurrently"""-- if embedding_config.embedding_endpoint_type != "openai":- embed_model = embedding_model(embedding_config)- loop = asyncio.get_event_loop()-- tasks = [loop.run_in_executor(None, embed_model.get_text_embedding, text) for text in text_chunks]- embeddings = await asyncio.gather(*tasks)- else:- tasks = [- get_openai_embedding_async(- text,- embedding_config.embedding_model,- embedding_config.embedding_endpoint,- )- for text in text_chunks- ]- embeddings = await asyncio.gather(*tasks)-- processed_embeddings = []- for embedding in embeddings:- if isinstance(embedding, dict):- try:- processed_embeddings.append(embedding["data"][0]["embedding"])- except (KeyError, IndexError):- raise TypeError(- f"Got back an unexpected payload from text embedding function, type={type(embedding)}, value={embedding}"- )- else:- processed_embeddings.append(embedding)- return processed_embeddings+ embeddings = await embedding_client.request_embeddings(text_chunks, embedding_config)+ return embeddings@enforce_types@trace_methoddiff --git a/letta/services/tool_executor/core_tool_executor.py b/letta/services/tool_executor/core_tool_executor.pyindex 40398dcfd..35da4c28e 100644--- a/letta/services/tool_executor/core_tool_executor.py+++ b/letta/services/tool_executor/core_tool_executor.py@@ -174,7 +174,7 @@ class LettaCoreToolExecutor(ToolExecutor):Returns:Optional[str]: None is always returned as this function does not produce a response."""- await PassageManager().insert_passage_async(+ await PassageManager().insert_passage(agent_state=agent_state,text=content,actor=actor,diff --git a/tests/helpers/endpoints_helper.py b/tests/helpers/endpoints_helper.pyindex 2fa78a486..72b91b696 100644--- a/tests/helpers/endpoints_helper.py+++ b/tests/helpers/endpoints_helper.py@@ -13,8 +13,8 @@ logger = logging.getLogger(__name__)from letta.config import LettaConfigfrom letta.constants import DEFAULT_HUMAN, DEFAULT_PERSONA-from letta.embeddings import embedding_modelfrom letta.errors import InvalidInnerMonologueError, InvalidToolCallError, MissingInnerMonologueError, MissingToolCallError+from letta.llm_api.llm_client import LLMClientfrom letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.schemas.agent import AgentState, CreateAgentfrom letta.schemas.embedding_config import EmbeddingConfig@@ -91,14 +91,21 @@ def setup_agent(# ======================================================================================================================-def run_embedding_endpoint(filename):+async def run_embedding_endpoint(filename, actor=None):# load JSON fileconfig_data = json.load(open(filename, "r"))print(config_data)embedding_config = EmbeddingConfig(**config_data)- model = embedding_model(embedding_config)++ # Use the new LLMClient for embeddings+ client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )+query_text = "hello"- query_vec = model.get_text_embedding(query_text)+ query_vecs = await client.request_embeddings([query_text], embedding_config)+ query_vec = query_vecs[0]print("vector dim", len(query_vec))assert query_vec is not Nonediff --git a/tests/test_base_functions.py b/tests/test_base_functions.pyindex 7ba28ee90..5a88de870 100644--- a/tests/test_base_functions.py+++ b/tests/test_base_functions.py@@ -1,3 +1,4 @@+import asyncioimport osimport threading@@ -96,53 +97,63 @@ def query_in_search_results(search_results, query):return False-def test_archival(agent_obj):+async def test_archival(agent_obj):"""Test archival memory functions comprehensively."""# Test 1: Basic insertion and retrieval- base_functions.archival_memory_insert(agent_obj, "The cat sleeps on the mat")- base_functions.archival_memory_insert(agent_obj, "The dog plays in the park")- base_functions.archival_memory_insert(agent_obj, "Python is a programming language")+ await base_functions.archival_memory_insert(agent_obj, "The cat sleeps on the mat")+ await asyncio.sleep(0.1) # Small delay to ensure session cleanup+ await base_functions.archival_memory_insert(agent_obj, "The dog plays in the park")+ await asyncio.sleep(0.1)+ await base_functions.archival_memory_insert(agent_obj, "Python is a programming language")+ await asyncio.sleep(0.1)# Test exact text search- results, _ = base_functions.archival_memory_search(agent_obj, "cat")+ results, _ = await base_functions.archival_memory_search(agent_obj, "cat")assert query_in_search_results(results, "cat")+ await asyncio.sleep(0.1)# Test semantic search (should return animal-related content)- results, _ = base_functions.archival_memory_search(agent_obj, "animal pets")+ results, _ = await base_functions.archival_memory_search(agent_obj, "animal pets")assert query_in_search_results(results, "cat") or query_in_search_results(results, "dog")+ await asyncio.sleep(0.1)# Test unrelated search (should not return animal content)- results, _ = base_functions.archival_memory_search(agent_obj, "programming computers")+ results, _ = await base_functions.archival_memory_search(agent_obj, "programming computers")assert query_in_search_results(results, "python")+ await asyncio.sleep(0.1)# Test 2: Test pagination# Insert more items to test paginationfor i in range(10):- base_functions.archival_memory_insert(agent_obj, f"Test passage number {i}")+ await base_functions.archival_memory_insert(agent_obj, f"Test passage number {i}")+ await asyncio.sleep(0.05) # Shorter delay for bulk operations# Get first page- page0_results, next_page = base_functions.archival_memory_search(agent_obj, "Test passage", page=0)+ page0_results, next_page = await base_functions.archival_memory_search(agent_obj, "Test passage", page=0)+ await asyncio.sleep(0.1)# Get second page- page1_results, _ = base_functions.archival_memory_search(agent_obj, "Test passage", page=1, start=next_page)+ page1_results, _ = await base_functions.archival_memory_search(agent_obj, "Test passage", page=1, start=next_page)+ await asyncio.sleep(0.1)assert page0_results != page1_resultsassert query_in_search_results(page0_results, "Test passage")assert query_in_search_results(page1_results, "Test passage")# Test 3: Test complex text patterns- base_functions.archival_memory_insert(agent_obj, "Important meeting on 2024-01-15 with John")- base_functions.archival_memory_insert(agent_obj, "Follow-up meeting scheduled for next week")- base_functions.archival_memory_insert(agent_obj, "Project deadline is approaching")+ await base_functions.archival_memory_insert(agent_obj, "Important meeting on 2024-01-15 with John")+ await base_functions.archival_memory_insert(agent_obj, "Follow-up meeting scheduled for next week")+ await base_functions.archival_memory_insert(agent_obj, "Project deadline is approaching")# Search for meeting-related content- results, _ = base_functions.archival_memory_search(agent_obj, "meeting schedule")+ results, _ = await base_functions.archival_memory_search(agent_obj, "meeting schedule")assert query_in_search_results(results, "meeting")assert query_in_search_results(results, "2024-01-15") or query_in_search_results(results, "next week")# Test 4: Test error handling# Test invalid page numbertry:- base_functions.archival_memory_search(agent_obj, "test", page="invalid")+ await base_functions.archival_memory_search(agent_obj, "test", page="invalid")assert False, "Should have raised ValueError"except ValueError:passdiff --git a/tests/test_sdk_client.py b/tests/test_sdk_client.pyindex fddfafa7f..250d3c55f 100644--- a/tests/test_sdk_client.py+++ b/tests/test_sdk_client.py@@ -408,6 +408,28 @@ def test_send_system_message(client: LettaSDKClient, agent: AgentState):assert send_system_message_response, "Sending message failed"+def test_insert_archival_memory(client: LettaSDKClient, agent: AgentState):+ passage = client.agents.passages.create(+ agent_id=agent.id,+ text="This is a test passage",+ )+ assert passage, "Inserting archival memory failed"++ # List archival memory and verify content+ archival_memory_response = client.agents.passages.list(agent_id=agent.id, limit=1)+ archival_memories = [memory.text for memory in archival_memory_response]+ assert "This is a test passage" in archival_memories, f"Retrieving archival memory failed: {archival_memories}"++ # Delete the memory+ memory_id_to_delete = archival_memory_response[0].id+ client.agents.passages.delete(agent_id=agent.id, memory_id=memory_id_to_delete)++ # Verify memory is gone (implicitly checks that the list call works)+ final_passages = client.agents.passages.list(agent_id=agent.id)+ passage_texts = [p.text for p in final_passages]+ assert "This is a test passage" not in passage_texts, f"Memory was not deleted: {passage_texts}"++def test_function_return_limit(disable_e2b_api_key, client: LettaSDKClient, agent: AgentState):"""Test to see if the function return limit works"""diff --git a/tests/test_server.py b/tests/test_server.pyindex bc093c126..58cfc1c2a 100644--- a/tests/test_server.py+++ b/tests/test_server.py@@ -425,56 +425,6 @@ def test_get_recall_memory(server, org_id, user, agent_id):assert message_id in message_ids, f"{message_id} not in {message_ids}"-# @pytest.mark.order(6)-# def test_get_archival_memory(server, user, agent_id):-# # test archival memory cursor pagination-# actor = user-#-# # List latest 2 passages-# passages_1 = server.agent_manager.list_passages(-# actor=actor,-# agent_id=agent_id,-# ascending=False,-# limit=2,-# )-# assert len(passages_1) == 2, f"Returned {[p.text for p in passages_1]}, not equal to 2"-#-# # List next 3 passages (earliest 3)-# cursor1 = passages_1[-1].id-# passages_2 = server.agent_manager.list_passages(-# actor=actor,-# agent_id=agent_id,-# ascending=False,-# before=cursor1,-# )-#-# # List all 5-# cursor2 = passages_1[0].created_at-# passages_3 = server.agent_manager.list_passages(-# actor=actor,-# agent_id=agent_id,-# ascending=False,-# end_date=cursor2,-# limit=1000,-# )-# assert len(passages_2) in [3, 4] # NOTE: exact size seems non-deterministic, so loosen test-# assert len(passages_3) in [4, 5] # NOTE: exact size seems non-deterministic, so loosen test-#-# latest = passages_1[0]-# earliest = passages_2[-1]-#-# # test archival memory-# passage_1 = server.agent_manager.list_passages(actor=actor, agent_id=agent_id, limit=1, ascending=True)-# assert len(passage_1) == 1-# assert passage_1[0].text == "alpha"-# passage_2 = server.agent_manager.list_passages(actor=actor, agent_id=agent_id, after=earliest.id, limit=1000, ascending=True)-# assert len(passage_2) in [4, 5] # NOTE: exact size seems non-deterministic, so loosen test-# assert all("alpha" not in passage.text for passage in passage_2)-# # test safe empty return-# passage_none = server.agent_manager.list_passages(actor=actor, agent_id=agent_id, after=latest.id, limit=1000, ascending=True)-# assert len(passage_none) == 0--@pytest.mark.asyncioasync def test_get_context_window_overview(server: SyncServer, user, agent_id):"""Test that the context window overview fetch works"""diff --git a/tests/test_sources.py b/tests/test_sources.pyindex 8005ab35e..efe2e73ae 100644--- a/tests/test_sources.py+++ b/tests/test_sources.py@@ -525,7 +525,7 @@ def test_agent_uses_search_files_correctly(disable_pinecone, client: LettaSDKCli# Check it returned successfullytool_returns = [msg for msg in search_files_response.messages if msg.message_type == "tool_return_message"]assert len(tool_returns) > 0, "No tool returns found"- assert all(tr.status == "success" for tr in tool_returns), "Tool call failed"+ assert all(tr.status == "success" for tr in tool_returns), f"Tool call failed {tr}"def test_agent_uses_grep_correctly_basic(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):