Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: concurrently make embedding request and use async client for OpenAI (#2482)

Sarah Wooders3354f5fe502a
diff --git a/letta/constants.py b/letta/constants.py
index ac7cc74c1..88bb7f535 100644
--- a/letta/constants.py
+++ b/letta/constants.py
@@ -37,6 +37,9 @@ TOOL_CALL_ID_MAX_LEN = 29
# minimum context window size
MIN_CONTEXT_WINDOW = 4096
+# number of concurrent embedding requests to sent
+EMBEDDING_BATCH_SIZE = 200
+
# Voice Sleeptime message buffer lengths
DEFAULT_MAX_MESSAGE_BUFFER_LENGTH = 30
DEFAULT_MIN_MESSAGE_BUFFER_LENGTH = 15
diff --git a/letta/data_sources/connectors.py b/letta/data_sources/connectors.py
index 5a0329ec3..c9cd42ac0 100644
--- a/letta/data_sources/connectors.py
+++ b/letta/data_sources/connectors.py
@@ -2,6 +2,7 @@ from typing import Dict, Iterator, List, Tuple
import typer
+from letta.constants import EMBEDDING_BATCH_SIZE
from letta.data_sources.connectors_helper import assert_all_files_exist_locally, extract_metadata_from_files, get_filenames_in_dir
from letta.embeddings import embedding_model
from letta.schemas.file import FileMetadata
@@ -40,43 +41,37 @@ class DataConnector:
async def load_data(
connector: DataConnector, source: Source, passage_manager: PassageManager, source_manager: SourceManager, actor: "User"
):
+ from letta.llm_api.llm_client import LLMClient
+ from letta.schemas.embedding_config import EmbeddingConfig
+
"""Load data from a connector (generates file and passages) into a specified source_id, associated with a user_id."""
embedding_config = source.embedding_config
- # embedding model
- embed_model = embedding_model(embedding_config)
-
# insert passages/file
- passages = []
+ texts = []
embedding_to_document_name = {}
passage_count = 0
file_count = 0
- for file_metadata in connector.find_files(source):
- file_count += 1
- await source_manager.create_file(file_metadata, actor)
- # generate passages
- for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
- # for some reason, llama index parsers sometimes return empty strings
- if len(passage_text) == 0:
- typer.secho(
- f"Warning: Llama index parser returned empty string, skipping insert of passage with metadata '{passage_metadata}' into VectorDB. You can usually ignore this warning.",
- fg=typer.colors.YELLOW,
- )
- continue
+ async def generate_embeddings(texts: List[str], embedding_config: EmbeddingConfig) -> List[Passage]:
+ passages = []
+ if embedding_config.embedding_endpoint_type == "openai":
+ texts.append(passage_text)
- # get embedding
- try:
- embedding = embed_model.get_text_embedding(passage_text)
- except Exception as e:
- typer.secho(
- f"Warning: Failed to get embedding for {passage_text} (error: {str(e)}), skipping insert into VectorDB.",
- fg=typer.colors.YELLOW,
- )
- continue
+ client = LLMClient.create(
+ provider_type=embedding_config.embedding_endpoint_type,
+ actor=actor,
+ )
+ embeddings = await client.request_embeddings(texts, embedding_config)
+ else:
+ embed_model = embedding_model(embedding_config)
+ embeddings = [embed_model.get_text_embedding(text) for text in texts]
+
+ # collate passage and embedding
+ for text, embedding in zip(texts, embeddings):
passage = Passage(
- text=passage_text,
+ text=text,
file_id=file_metadata.id,
source_id=source.id,
metadata=passage_metadata,
@@ -84,7 +79,6 @@ async def load_data(
embedding_config=source.embedding_config,
embedding=embedding,
)
-
hashable_embedding = tuple(passage.embedding)
file_name = file_metadata.file_name
if hashable_embedding in embedding_to_document_name:
@@ -96,15 +90,37 @@ async def load_data(
passages.append(passage)
embedding_to_document_name[hashable_embedding] = file_name
- if len(passages) >= 100:
- # insert passages into passage store
- await passage_manager.create_many_passages_async(passages, actor)
+ return passages
+
+ for file_metadata in connector.find_files(source):
+ file_count += 1
+ await source_manager.create_file(file_metadata, actor)
+
+ # generate passages
+ for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
+ # for some reason, llama index parsers sometimes return empty strings
+ if len(passage_text) == 0:
+ typer.secho(
+ f"Warning: Llama index parser returned empty string, skipping insert of passage with metadata '{passage_metadata}' into VectorDB. You can usually ignore this warning.",
+ fg=typer.colors.YELLOW,
+ )
+ continue
+
+ # get embedding
+ texts.append(passage_text)
+ if len(texts) >= EMBEDDING_BATCH_SIZE:
+ passages = await generate_embeddings(texts, embedding_config)
+ texts = []
+ else:
+ continue
- passage_count += len(passages)
- passages = []
+ # insert passages into passage store
+ await passage_manager.create_many_passages_async(passages, actor)
+ passage_count += len(passages)
- if len(passages) > 0:
- # insert passages into passage store
+ # final remaining
+ if len(texts) > 0:
+ passages = await generate_embeddings(texts, embedding_config)
await passage_manager.create_many_passages_async(passages, actor)
passage_count += len(passages)
diff --git a/letta/llm_api/llm_client_base.py b/letta/llm_api/llm_client_base.py
index 6374a85c9..e88b60810 100644
--- a/letta/llm_api/llm_client_base.py
+++ b/letta/llm_api/llm_client_base.py
@@ -6,6 +6,7 @@ from openai import AsyncStream, Stream
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
from letta.errors import LLMError
+from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message
from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
@@ -138,6 +139,20 @@ class LLMClientBase:
"""
raise NotImplementedError
+ @abstractmethod
+ async def request_embeddings(self, texts: List[str], embedding_config: EmbeddingConfig) -> List[List[float]]:
+ """
+ Generate embeddings for a batch of texts.
+
+ Args:
+ texts (List[str]): List of texts to generate embeddings for.
+ embedding_config (EmbeddingConfig): Configuration for the embedding model.
+
+ Returns:
+ embeddings (List[List[float]]): List of embeddings for the input texts.
+ """
+ raise NotImplementedError
+
@abstractmethod
def convert_response_to_chat_completion(
self,
diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.py
index 919120941..8be87ccc8 100644
--- a/letta/llm_api/openai_client.py
+++ b/letta/llm_api/openai_client.py
@@ -22,6 +22,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_st
from letta.llm_api.llm_client_base import LLMClientBase
from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRST
from letta.log import get_logger
+from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import ProviderCategory, ProviderType
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message as PydanticMessage
@@ -125,6 +126,18 @@ class OpenAIClient(LLMClientBase):
return kwargs
+ def _prepare_client_kwargs_embedding(self, embedding_config: EmbeddingConfig) -> dict:
+ api_key = None
+ if embedding_config.embedding_endpoint_type == ProviderType.together:
+ api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")
+
+ if not api_key:
+ api_key = model_settings.openai_api_key or os.environ.get("OPENAI_API_KEY")
+ # supposedly the openai python client requires a dummy API key
+ api_key = api_key or "DUMMY_API_KEY"
+ kwargs = {"api_key": api_key, "base_url": embedding_config.embedding_endpoint}
+ return kwargs
+
async def _prepare_client_kwargs_async(self, llm_config: LLMConfig) -> dict:
api_key = None
if llm_config.provider_category == ProviderCategory.byok:
@@ -291,6 +304,16 @@ class OpenAIClient(LLMClientBase):
)
return response_stream
+ @trace_method
+ async def request_embeddings(self, inputs: List[str], embedding_config: EmbeddingConfig) -> List[dict]:
+ """Request embeddings given texts and embedding config"""
+ kwargs = self._prepare_client_kwargs_embedding(embedding_config)
+ client = AsyncOpenAI(**kwargs)
+ response = await client.embeddings.create(model=embedding_config.embedding_model, input=inputs)
+
+ # TODO: add total usage
+ return [r.embedding for r in response.data]
+
@trace_method
def handle_llm_error(self, e: Exception) -> Exception:
"""