Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: concurrently make embedding request and use async client for OpenAI (#2482)
Sarah Wooders3354f5fe502a
diff --git a/letta/constants.py b/letta/constants.pyindex ac7cc74c1..88bb7f535 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -37,6 +37,9 @@ TOOL_CALL_ID_MAX_LEN = 29# minimum context window sizeMIN_CONTEXT_WINDOW = 4096+# number of concurrent embedding requests to sent+EMBEDDING_BATCH_SIZE = 200+# Voice Sleeptime message buffer lengthsDEFAULT_MAX_MESSAGE_BUFFER_LENGTH = 30DEFAULT_MIN_MESSAGE_BUFFER_LENGTH = 15diff --git a/letta/data_sources/connectors.py b/letta/data_sources/connectors.pyindex 5a0329ec3..c9cd42ac0 100644--- a/letta/data_sources/connectors.py+++ b/letta/data_sources/connectors.py@@ -2,6 +2,7 @@ from typing import Dict, Iterator, List, Tupleimport typer+from letta.constants import EMBEDDING_BATCH_SIZEfrom letta.data_sources.connectors_helper import assert_all_files_exist_locally, extract_metadata_from_files, get_filenames_in_dirfrom letta.embeddings import embedding_modelfrom letta.schemas.file import FileMetadata@@ -40,43 +41,37 @@ class DataConnector:async def load_data(connector: DataConnector, source: Source, passage_manager: PassageManager, source_manager: SourceManager, actor: "User"):+ from letta.llm_api.llm_client import LLMClient+ from letta.schemas.embedding_config import EmbeddingConfig+"""Load data from a connector (generates file and passages) into a specified source_id, associated with a user_id."""embedding_config = source.embedding_config- # embedding model- embed_model = embedding_model(embedding_config)-# insert passages/file- passages = []+ texts = []embedding_to_document_name = {}passage_count = 0file_count = 0- for file_metadata in connector.find_files(source):- file_count += 1- await source_manager.create_file(file_metadata, actor)- # generate passages- for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):- # for some reason, llama index parsers sometimes return empty strings- if len(passage_text) == 0:- typer.secho(- f"Warning: Llama index parser returned empty string, skipping insert of passage with metadata '{passage_metadata}' into VectorDB. You can usually ignore this warning.",- fg=typer.colors.YELLOW,- )- continue+ async def generate_embeddings(texts: List[str], embedding_config: EmbeddingConfig) -> List[Passage]:+ passages = []+ if embedding_config.embedding_endpoint_type == "openai":+ texts.append(passage_text)- # get embedding- try:- embedding = embed_model.get_text_embedding(passage_text)- except Exception as e:- typer.secho(- f"Warning: Failed to get embedding for {passage_text} (error: {str(e)}), skipping insert into VectorDB.",- fg=typer.colors.YELLOW,- )- continue+ client = LLMClient.create(+ provider_type=embedding_config.embedding_endpoint_type,+ actor=actor,+ )+ embeddings = await client.request_embeddings(texts, embedding_config)+ else:+ embed_model = embedding_model(embedding_config)+ embeddings = [embed_model.get_text_embedding(text) for text in texts]++ # collate passage and embedding+ for text, embedding in zip(texts, embeddings):passage = Passage(- text=passage_text,+ text=text,file_id=file_metadata.id,source_id=source.id,metadata=passage_metadata,@@ -84,7 +79,6 @@ async def load_data(embedding_config=source.embedding_config,embedding=embedding,)-hashable_embedding = tuple(passage.embedding)file_name = file_metadata.file_nameif hashable_embedding in embedding_to_document_name:@@ -96,15 +90,37 @@ async def load_data(passages.append(passage)embedding_to_document_name[hashable_embedding] = file_name- if len(passages) >= 100:- # insert passages into passage store- await passage_manager.create_many_passages_async(passages, actor)+ return passages++ for file_metadata in connector.find_files(source):+ file_count += 1+ await source_manager.create_file(file_metadata, actor)++ # generate passages+ for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):+ # for some reason, llama index parsers sometimes return empty strings+ if len(passage_text) == 0:+ typer.secho(+ f"Warning: Llama index parser returned empty string, skipping insert of passage with metadata '{passage_metadata}' into VectorDB. You can usually ignore this warning.",+ fg=typer.colors.YELLOW,+ )+ continue++ # get embedding+ texts.append(passage_text)+ if len(texts) >= EMBEDDING_BATCH_SIZE:+ passages = await generate_embeddings(texts, embedding_config)+ texts = []+ else:+ continue- passage_count += len(passages)- passages = []+ # insert passages into passage store+ await passage_manager.create_many_passages_async(passages, actor)+ passage_count += len(passages)- if len(passages) > 0:- # insert passages into passage store+ # final remaining+ if len(texts) > 0:+ passages = await generate_embeddings(texts, embedding_config)await passage_manager.create_many_passages_async(passages, actor)passage_count += len(passages)diff --git a/letta/llm_api/llm_client_base.py b/letta/llm_api/llm_client_base.pyindex 6374a85c9..e88b60810 100644--- a/letta/llm_api/llm_client_base.py+++ b/letta/llm_api/llm_client_base.py@@ -6,6 +6,7 @@ from openai import AsyncStream, Streamfrom openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.errors import LLMError+from letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Messagefrom letta.schemas.openai.chat_completion_response import ChatCompletionResponse@@ -138,6 +139,20 @@ class LLMClientBase:"""raise NotImplementedError+ @abstractmethod+ async def request_embeddings(self, texts: List[str], embedding_config: EmbeddingConfig) -> List[List[float]]:+ """+ Generate embeddings for a batch of texts.++ Args:+ texts (List[str]): List of texts to generate embeddings for.+ embedding_config (EmbeddingConfig): Configuration for the embedding model.++ Returns:+ embeddings (List[List[float]]): List of embeddings for the input texts.+ """+ raise NotImplementedError+@abstractmethoddef convert_response_to_chat_completion(self,diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex 919120941..8be87ccc8 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -22,6 +22,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_stfrom letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRSTfrom letta.log import get_logger+from letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import ProviderCategory, ProviderTypefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessage@@ -125,6 +126,18 @@ class OpenAIClient(LLMClientBase):return kwargs+ def _prepare_client_kwargs_embedding(self, embedding_config: EmbeddingConfig) -> dict:+ api_key = None+ if embedding_config.embedding_endpoint_type == ProviderType.together:+ api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")++ if not api_key:+ api_key = model_settings.openai_api_key or os.environ.get("OPENAI_API_KEY")+ # supposedly the openai python client requires a dummy API key+ api_key = api_key or "DUMMY_API_KEY"+ kwargs = {"api_key": api_key, "base_url": embedding_config.embedding_endpoint}+ return kwargs+async def _prepare_client_kwargs_async(self, llm_config: LLMConfig) -> dict:api_key = Noneif llm_config.provider_category == ProviderCategory.byok:@@ -291,6 +304,16 @@ class OpenAIClient(LLMClientBase):)return response_stream+ @trace_method+ async def request_embeddings(self, inputs: List[str], embedding_config: EmbeddingConfig) -> List[dict]:+ """Request embeddings given texts and embedding config"""+ kwargs = self._prepare_client_kwargs_embedding(embedding_config)+ client = AsyncOpenAI(**kwargs)+ response = await client.embeddings.create(model=embedding_config.embedding_model, input=inputs)++ # TODO: add total usage+ return [r.embedding for r in response.data]+@trace_methoddef handle_llm_error(self, e: Exception) -> Exception:"""